From 07833379b49107cdae5ff4ab0d709e693b3b5b90 Mon Sep 17 00:00:00 2001 From: RustFS Date: Sun, 6 Sep 2026 10:12:06 +0800 Subject: [PATCH] test(e2e): add distributed cluster regression coverage (#7158) * test(e2e): add distributed 4x4 validation * test(e2e): prove operations overlap data movement --------- Co-authored-by: Zhengchao An --- .config/e2e-distributed-selection.txt | 2 + .config/nextest.toml | 33 + .github/scheduled-validations.json | 5 + .github/workflows/e2e-distributed.yml | 199 +++ .../scheduled-validation-watchdog.yml | 1 + crates/e2e_test/README.md | 5 + crates/e2e_test/src/common.rs | 63 + crates/e2e_test/src/distributed/chaos_test.rs | 222 +++ .../distributed/concurrency_stability_test.rs | 98 ++ .../concurrent_data_movement_test.rs | 74 + .../data_integrity_movement_test.rs | 156 +++ .../expand_decommission_rebalance_test.rs | 81 ++ crates/e2e_test/src/distributed/extra_test.rs | 149 ++ crates/e2e_test/src/distributed/harness.rs | 1235 +++++++++++++++++ crates/e2e_test/src/distributed/mod.rs | 35 + .../src/distributed/object_lock_test.rs | 219 +++ .../src/distributed/observability_test.rs | 236 ++++ .../src/distributed/replication_quota_test.rs | 191 +++ .../e2e_test/src/distributed/s3_basic_test.rs | 258 ++++ .../s3_during_data_movement_test.rs | 94 ++ .../src/distributed/site_replication_test.rs | 128 ++ .../e2e_test/src/distributed/upgrade_test.rs | 345 +++++ .../src/distributed/versioning_test.rs | 188 +++ crates/e2e_test/src/lib.rs | 5 + docs/testing/README.md | 5 +- docs/testing/ci-gates.md | 1 + docs/testing/distributed-e2e.md | 75 + 27 files changed, 4101 insertions(+), 2 deletions(-) create mode 100644 .config/e2e-distributed-selection.txt create mode 100644 .github/workflows/e2e-distributed.yml create mode 100644 crates/e2e_test/src/distributed/chaos_test.rs create mode 100644 crates/e2e_test/src/distributed/concurrency_stability_test.rs create mode 100644 crates/e2e_test/src/distributed/concurrent_data_movement_test.rs create mode 100644 crates/e2e_test/src/distributed/data_integrity_movement_test.rs create mode 100644 crates/e2e_test/src/distributed/expand_decommission_rebalance_test.rs create mode 100644 crates/e2e_test/src/distributed/extra_test.rs create mode 100644 crates/e2e_test/src/distributed/harness.rs create mode 100644 crates/e2e_test/src/distributed/mod.rs create mode 100644 crates/e2e_test/src/distributed/object_lock_test.rs create mode 100644 crates/e2e_test/src/distributed/observability_test.rs create mode 100644 crates/e2e_test/src/distributed/replication_quota_test.rs create mode 100644 crates/e2e_test/src/distributed/s3_basic_test.rs create mode 100644 crates/e2e_test/src/distributed/s3_during_data_movement_test.rs create mode 100644 crates/e2e_test/src/distributed/site_replication_test.rs create mode 100644 crates/e2e_test/src/distributed/upgrade_test.rs create mode 100644 crates/e2e_test/src/distributed/versioning_test.rs create mode 100644 docs/testing/distributed-e2e.md diff --git a/.config/e2e-distributed-selection.txt b/.config/e2e-distributed-selection.txt new file mode 100644 index 000000000..7291d5e03 --- /dev/null +++ b/.config/e2e-distributed-selection.txt @@ -0,0 +1,2 @@ +sha256-linux=9785867929047dfd8c6f768e0d2b1e0a8fdba85216f4a4139093b1619d03ff07 +sha256-darwin=9785867929047dfd8c6f768e0d2b1e0a8fdba85216f4a4139093b1619d03ff07 diff --git a/.config/nextest.toml b/.config/nextest.toml index bc57b8b8f..17c5dc7a5 100644 --- a/.config/nextest.toml +++ b/.config/nextest.toml @@ -183,6 +183,13 @@ test-group = 'e2e-reliability' filter = 'package(e2e_test) & test(/^inline_fast_path_cluster_test::/)' test-group = 'e2e-inline-boundaries' +# 4-node 4-drive distributed Actions suite: each case starts four rustfs +# processes and up to sixteen data directories. Serialize across nextest's +# process boundary so several 4x4 clusters never overlap. +[[profile.default.overrides]] +filter = 'package(e2e_test) & test(/^distributed::/)' +test-group = 'e2e-cluster-nightly' + # Vault KMS tests share the fixed dev-server port 8200. serial_test's #[serial] # does not cross nextest process boundaries, so keep every Vault-backed test in # one group. @@ -526,6 +533,27 @@ path = "junit.xml" filter = 'package(e2e_test)' test-group = 'e2e-cluster-nightly' +# --------------------------------------------------------------------------- +# e2e-distributed profile — 4-node 4-disk Actions suite +# --------------------------------------------------------------------------- +# Storage-sensitive PR / nightly / dispatch lane owned by +# .github/workflows/e2e-distributed.yml. +# Each case starts four rustfs processes (and for site replication, two +# clusters). Upgrade cases also require RUSTFS_UPGRADE_SOURCE_BINARY. +# Serialized via e2e-cluster-nightly with no retries. +[profile.e2e-distributed] +default-filter = 'package(e2e_test) & test(/^distributed::/)' +fail-fast = false +# Decommission / rebalance cases poll for up to 180s with little stdout. +slow-timeout = { period = "120s", terminate-after = 6 } + +[profile.e2e-distributed.junit] +path = "junit.xml" + +[[profile.e2e-distributed.overrides]] +filter = 'package(e2e_test)' +test-group = 'e2e-cluster-nightly' + # --------------------------------------------------------------------------- # e2e-odm-interop profile — on-demand migration provider interop lane (ODM-20) # --------------------------------------------------------------------------- @@ -586,6 +614,10 @@ path = "junit.xml" # cluster-fault lane. heal_erasure_disk_rebuild is intentionally not # excluded here because backlog#2213 promotes core heal rebuild coverage to # this merge/main lane while retaining nightly coverage. +# * distributed:: — 4-node 4-disk Actions suite (S3, lock, versioning, +# replication, quota, observability, expand/decommission/rebalance, site +# replication, chaos, upgrade history/IAM). Owns [profile.e2e-distributed] and +# .github/workflows/e2e-distributed.yml. # * on_demand_migration::interop_test — the ODM-20 provider interoperability # cases, which are meaningless without a source: they run in the dedicated # [profile.e2e-odm-interop] lane below, where the workflow points them at a @@ -607,6 +639,7 @@ default-filter = """ package(e2e_test) & !test(/^protocols::/) & !test(/^(admin_timeout_regression_test|cluster_concurrency_test|cluster_multidrive_pool_test|degraded_listing_availability_test|namespace_lock_quorum_test|object_lambda_test|stale_multipart_cleanup_cluster_test)::/) + & !test(/^distributed::/) & !test(/^replication_extension_test::/) & !test(/^replication_target_matrix_test::/) & !test(/^on_demand_migration::(concurrency_test|fault_test|interop_test|real_source_test)::/) diff --git a/.github/scheduled-validations.json b/.github/scheduled-validations.json index 9ac7f2614..e55ef3d56 100644 --- a/.github/scheduled-validations.json +++ b/.github/scheduled-validations.json @@ -4,6 +4,11 @@ { "workflow": ".github/workflows/ci.yml", "max_age_hours": 192 }, { "workflow": ".github/workflows/coverage.yml", "max_age_hours": 192 }, { "workflow": ".github/workflows/e2e-replication-nightly.yml", "max_age_hours": 36 }, + { + "workflow": ".github/workflows/e2e-distributed.yml", + "max_age_hours": 36, + "never_ran_grace_until": "2026-09-18T00:00:00Z" + }, { "workflow": ".github/workflows/e2e-s3tests.yml", "max_age_hours": 192 }, { "workflow": ".github/workflows/fuzz.yml", "max_age_hours": 36 }, { "workflow": ".github/workflows/mint.yml", "max_age_hours": 192 }, diff --git a/.github/workflows/e2e-distributed.yml b/.github/workflows/e2e-distributed.yml new file mode 100644 index 000000000..187defc32 --- /dev/null +++ b/.github/workflows/e2e-distributed.yml @@ -0,0 +1,199 @@ +# Copyright 2024 RustFS Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# 4-node 4-disk distributed e2e lane. +# +# Each selected test starts a real localhost cluster via +# `RustFSTestClusterEnvironment` (4 processes; 4 drives per node unless the +# case is a two-site 4-node 1-drive pair or a 4-node upgrade). Membership is +# `[profile.e2e-distributed]` in `.config/nextest.toml`. Storage-sensitive PRs, +# nightly runs, and manual dispatches all execute the same fail-closed suite. +# Upgrade cases download the same pinned previous release as e2e-upgrade.yml. + +name: e2e-distributed + +on: + pull_request: + paths: + - "Cargo.lock" + - "Cargo.toml" + - ".config/nextest.toml" + - ".github/workflows/e2e-distributed.yml" + - "crates/audit/**" + - "crates/common/**" + - "crates/config/**" + - "crates/e2e_test/**" + - "crates/ecstore/**" + - "crates/filemeta/**" + - "crates/heal/**" + - "crates/iam/**" + - "crates/lock/**" + - "crates/madmin/**" + - "crates/notify/**" + - "crates/replication/**" + - "crates/s3-client/**" + - "crates/s3-ops/**" + - "crates/s3-types/**" + - "crates/scanner/**" + - "crates/storage-api/**" + - "crates/utils/**" + - "rustfs/**" + workflow_dispatch: + inputs: + filter: + description: "Optional nextest -E filter (default: the whole e2e-distributed profile)" + required: false + default: "" + schedule: + # 05:53 UTC nightly — clear of e2e-nightly (04:29) and ODM interop (05:23). + - cron: "53 5 * * *" + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.event_name != 'schedule' }} + +jobs: + distributed: + name: Distributed 4-node 4-disk e2e + runs-on: sm-standard-4 + timeout-minutes: 180 + env: + FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true" + NO_PROXY: 127.0.0.1,localhost + HTTP_PROXY: "" + HTTPS_PROXY: "" + # Pinned previous release used by distributed::upgrade_test (same pin as e2e-upgrade.yml). + UPGRADE_SOURCE_VERSION: 1.0.0-rc.2 + UPGRADE_SOURCE_ASSET: rustfs-linux-x86_64-gnu-v1.0.0-rc.2.zip + UPGRADE_SOURCE_SHA256: 7c789386bf85278f865b8e0d359bf4edb84d5aa408cc3fa54a18c25ca74cd6e7 + steps: + - name: Checkout repository + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 + with: + persist-credentials: false + + - name: Setup Rust environment + uses: ./.github/actions/setup + with: + rust-version: stable + cache-shared-key: ci-e2e-distributed + cache-save-if: ${{ github.ref == 'refs/heads/main' }} + install-build-packaging-tools: 'false' + + - name: Prepare isolated filesystems for pool movement + run: | + set -euo pipefail + mount_base="${RUNNER_TEMP}/rustfs-e2e-pools" + mkdir -p "${mount_base}" + roots=() + for pool in 0 1 2 3; do + image="${mount_base}/pool-${pool}.img" + mountpoint="${mount_base}/pool-${pool}" + truncate -s 1G "${image}" + mkfs.ext4 -q -F "${image}" + mkdir -p "${mountpoint}" + sudo mount -o loop,nosuid,nodev "${image}" "${mountpoint}" + sudo chmod 1777 "${mountpoint}" + roots+=("${mountpoint}") + done + printf -v joined_roots '%s:' "${roots[@]}" + echo "RUSTFS_E2E_POOL_ROOTS=${joined_roots%:}" >> "${GITHUB_ENV}" + findmnt --noheadings --output TARGET,SOURCE,FSTYPE --target "${roots[0]}" + findmnt --noheadings --output TARGET,SOURCE,FSTYPE --target "${roots[1]}" + findmnt --noheadings --output TARGET,SOURCE,FSTYPE --target "${roots[2]}" + findmnt --noheadings --output TARGET,SOURCE,FSTYPE --target "${roots[3]}" + + - name: Download pinned previous release + env: + SOURCE_DIR: ${{ runner.temp }}/rustfs-upgrade-source + run: | + set -euo pipefail + mkdir -p "$SOURCE_DIR" + archive="$SOURCE_DIR/$UPGRADE_SOURCE_ASSET" + curl --fail --location --retry 3 --output "$archive" \ + "https://github.com/${GITHUB_REPOSITORY}/releases/download/${UPGRADE_SOURCE_VERSION}/${UPGRADE_SOURCE_ASSET}" + echo "$UPGRADE_SOURCE_SHA256 $archive" | sha256sum --check --strict + unzip -q "$archive" -d "$SOURCE_DIR" + chmod +x "$SOURCE_DIR/rustfs" + test -x "$SOURCE_DIR/rustfs" + echo "RUSTFS_UPGRADE_SOURCE_BINARY=$SOURCE_DIR/rustfs" >> "$GITHUB_ENV" + + - name: Build rustfs binary + run: | + cargo build -p rustfs --bins + : > target/debug/rustfs.features + + - name: Verify distributed e2e membership + env: + NEXTEST_LISTING: ${{ runner.temp }}/rustfs-e2e-distributed-list.json + run: | + cargo nextest list --profile e2e-distributed -p e2e_test --message-format json > "${NEXTEST_LISTING}" + python3 ./scripts/check_test_wiring.py --check-profile e2e-distributed "${NEXTEST_LISTING}" + + - name: Run distributed 4-node e2e suite + env: + RUSTFS_E2E_LOG_DIR: ${{ runner.temp }}/rustfs-e2e-distributed-logs + FILTER: ${{ inputs.filter }} + run: | + set -euo pipefail + if [ -n "${FILTER}" ]; then + cargo nextest run --profile e2e-distributed -p e2e_test -E "${FILTER}" + else + cargo nextest run --profile e2e-distributed -p e2e_test --no-tests=fail + fi + + - name: Upload distributed e2e diagnostics + if: always() + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 + with: + name: e2e-distributed-${{ github.run_number }} + path: | + target/nextest/e2e-distributed/junit.xml + ${{ runner.temp }}/rustfs-e2e-distributed-list.json + ${{ runner.temp }}/rustfs-e2e-distributed-logs/ + retention-days: 7 + if-no-files-found: warn + + - name: Unmount isolated pool filesystems + if: always() + run: | + set -euo pipefail + mount_base="${RUNNER_TEMP}/rustfs-e2e-pools" + for pool in 0 1 2 3; do + mountpoint="${mount_base}/pool-${pool}" + if mountpoint --quiet "${mountpoint}"; then + sudo umount "${mountpoint}" + fi + done + + alert-on-failure: + name: Alert on scheduled failure + needs: [distributed] + if: always() && github.event_name == 'schedule' && contains(needs.*.result, 'failure') + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: read + issues: write + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 + with: + persist-credentials: false + - name: Open or update failure-tracking issue + uses: ./.github/actions/schedule-failure-issue + with: + github-token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/scheduled-validation-watchdog.yml b/.github/workflows/scheduled-validation-watchdog.yml index e778ec640..154d541d3 100644 --- a/.github/workflows/scheduled-validation-watchdog.yml +++ b/.github/workflows/scheduled-validation-watchdog.yml @@ -22,6 +22,7 @@ on: - "Continuous Integration" - "coverage" - "e2e-nightly" + - "e2e-distributed" - "e2e-s3tests" - "Fuzz" - "mint" diff --git a/crates/e2e_test/README.md b/crates/e2e_test/README.md index ed5a65d97..9c6d90c0a 100644 --- a/crates/e2e_test/README.md +++ b/crates/e2e_test/README.md @@ -26,6 +26,7 @@ Registered in [`src/lib.rs`](src/lib.rs). Grouped by concern: | **protocols** | [`src/protocols/`](src/protocols) | FTPS, WebDAV, SFTP compliance. Fixed ports, own guide: [`src/protocols/README.md`](src/protocols/README.md) | | **reliant** | [`src/reliant/`](src/reliant) | Tests that reuse an **externally started** server (SQL/select, conditional writes, lifecycle, deleted-object reads, node-interact). Run via [`scripts/run_e2e_tests.sh`](../../scripts/run_e2e_tests.sh); see [`src/reliant/README.md`](src/reliant/README.md) | | **cluster** | `cluster_concurrency_test`, `stale_multipart_cleanup_cluster_test`, `namespace_lock_quorum_test`, `admin_timeout_regression_test`, `object_lambda_test`, `replication_extension_test`, `tier_stats_cluster_test` | Multi-node scenarios via `RustFSTestClusterEnvironment` | +| **distributed 4×4** | [`src/distributed/`](src/distributed) | Storage-sensitive PR and nightly `e2e-distributed` lane: S3, object lock/WORM, versioning, bucket/site replication, quota, expand/decommission/rebalance, concurrency, chaos, 4-node upgrade of historical data and IAM AK/SK. Map: [`docs/testing/distributed-e2e.md`](../../docs/testing/distributed-e2e.md) | | **chaos / reliability** | [`src/chaos.rs`](src/chaos.rs), `reliability_disk_fault_test`, `heal_erasure_disk_rebuild_test`, `server_startup_failfast_test` | Disk offline/replace/corrupt, EC rebuild, heal, fail-fast startup | | **upgrade compatibility** | `upgrade_compatibility_test` | Pinned previous-release writes followed by current-build reads on the same data directory | @@ -171,6 +172,7 @@ the same profile for membership and execution with one nightly worker. | KMS suite | `e2e-full` job, merge queue + main | **Active** | | Direct and mixed-version rolling upgrades from pinned previous release | `e2e-upgrade.yml`, storage-sensitive PRs + release tags + weekly | **Active** | | Cluster faults (`e2e-nightly` profile) | consolidated nightly workflow | **Active** (backlog#1149 ci-7) | +| Distributed 4-node 4-disk (`e2e-distributed` profile) | `.github/workflows/e2e-distributed.yml` | **Active** (storage-sensitive PR / nightly / dispatch) | | Protocols (FTPS/WebDAV/SFTP) | consolidated nightly workflow, serial | **Active** (backlog#1149 ci-7) | | Replication (fast subset) | `e2e-smoke` profile, `e2e-tests` job, every PR | **Active** (backlog#1147 repl-1) | | Replication (slow + multi-node) | `e2e-repl-nightly` profile, consolidated nightly workflow | **Active** (backlog#1147 repl-1) | @@ -191,6 +193,9 @@ cargo nextest run --profile e2e-smoke -p e2e_test cargo nextest run --profile e2e-full -p e2e_test # Cluster fault nightly lane cargo nextest run --profile e2e-nightly -p e2e_test +# 4-node 4-disk distributed lane (S3 / lock / versioning / replication / decommission / chaos / upgrade) +# Upgrade cases need RUSTFS_UPGRADE_SOURCE_BINARY; without it they fail closed. +cargo nextest run --profile e2e-distributed -p e2e_test # Replication nightly lane; awscurl is required for STS paths cargo nextest run --profile e2e-repl-nightly -p e2e_test # Fixed-port protocol nightly lane diff --git a/crates/e2e_test/src/common.rs b/crates/e2e_test/src/common.rs index b108cd074..daa744ab8 100644 --- a/crates/e2e_test/src/common.rs +++ b/crates/e2e_test/src/common.rs @@ -1700,6 +1700,69 @@ impl RustFSTestClusterEnvironment { Ok(()) } + /// Append a new single-node erasure pool to a stopped multi-pool cluster. + /// + /// Used to simulate pool expansion on localhost: every pool already owns + /// exactly one node with `drives_per_node >= 2` (the only multi-pool layout + /// the single-host `RUSTFS_VOLUMES` syntax can express). The new node is + /// allocated a fresh port and empty drive directories; callers must + /// [`Self::start`] afterwards so every process picks up the extended + /// volumes argument. Existing data directories are left untouched. + pub async fn append_single_node_pool(&mut self) -> Result> { + if self.nodes.iter().any(|node| node.process.is_some()) { + return Err("stop the cluster before appending a pool".into()); + } + if self.topology.drives_per_node < 2 { + return Err( + "append_single_node_pool requires drives_per_node >= 2 (the server parser rejects a single-drive ellipses pool)" + .into(), + ); + } + + let mut pools = self.topology.normalized_pools(); + for (pool_idx, nodes) in pools.iter().enumerate() { + if nodes.len() != 1 { + return Err(format!( + "pool {pool_idx} spans {} nodes; append_single_node_pool requires one node per pool", + nodes.len() + ) + .into()); + } + } + + let new_idx = self.nodes.len(); + let port = RustFSTestEnvironment::find_available_port().await?; + let address = format!("127.0.0.1:{port}"); + let data_dirs: Vec = (0..self.topology.drives_per_node) + .map(|drive| format!("{}/node{}/drive{}", self.temp_dir, new_idx, drive)) + .collect(); + for dir in &data_dirs { + fs::create_dir_all(dir).await?; + } + + self.nodes.push(ClusterNode { + url: format!("http://{address}"), + address, + data_dir: data_dirs[0].clone(), + data_dirs, + pool_idx: pools.len(), + process: None, + }); + pools.push(vec![new_idx]); + self.topology.node_count = self.nodes.len(); + self.topology.pools = pools; + self.node_extra_env.push(Vec::new()); + self.node_capture_log_paths.push(None); + self.volume_proxy_addresses.push(None); + + if !self.extra_env.iter().any(|(key, _)| key == "RUSTFS_UNSAFE_BYPASS_DISK_CHECK") { + self.extra_env + .push(("RUSTFS_UNSAFE_BYPASS_DISK_CHECK".to_string(), "true".to_string())); + } + + Ok(new_idx) + } + /// Gracefully stop one cluster node and wait for its process to exit. /// /// This is intentionally separate from [`Self::stop_node`]: the latter is diff --git a/crates/e2e_test/src/distributed/chaos_test.rs b/crates/e2e_test/src/distributed/chaos_test.rs new file mode 100644 index 000000000..df058fbda --- /dev/null +++ b/crates/e2e_test/src/distributed/chaos_test.rs @@ -0,0 +1,222 @@ +// Copyright 2026 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use super::harness::{ + DistCluster, DistLayout, TestResult, assert_object_bytes, payload_for, put_object, retrying_get_equals, unique_bucket, + wait_for_ready, wait_until, +}; +use crate::chaos::{census_object_version_on_disk, signed_admin_post}; +use crate::common::{build_test_s3_config, init_logging}; +use crate::fault_proxy::FaultMode; +use aws_sdk_s3::Client; +use std::path::PathBuf; +use std::sync::Arc; +use std::time::Duration; +use tokio::sync::{Barrier, mpsc}; +use tokio::time::timeout; + +#[tokio::test] +async fn kill_and_restart_node_preserves_objects() -> TestResult { + init_logging(); + let mut dist = DistCluster::start(DistLayout::FourByFour).await?; + let bucket = unique_bucket("killnode"); + dist.create_bucket(&bucket).await?; + let body = vec![0x11u8; 128 * 1024]; + put_object(&dist.client(0)?, &bucket, "keep.bin", body.clone()).await?; + + dist.cluster.stop_node(3)?; + retrying_get_equals(&dist.client(0)?, &bucket, "keep.bin", &body, Duration::from_secs(20)).await?; + + dist.cluster.start_node(3).await?; + wait_for_ready(&dist.cluster).await?; + assert_object_bytes(&dist.client(3)?, &bucket, "keep.bin", &body).await?; + Ok(()) +} + +#[tokio::test] +async fn full_cluster_restart_preserves_objects() -> TestResult { + init_logging(); + let mut dist = DistCluster::start(DistLayout::FourByFour).await?; + let bucket = unique_bucket("pwr"); + dist.create_bucket(&bucket).await?; + let body = vec![0x44u8; 64 * 1024]; + put_object(&dist.client(1)?, &bucket, "survive.bin", body.clone()).await?; + + dist.cluster.stop(); + dist.cluster.start().await?; + wait_for_ready(&dist.cluster).await?; + for node_idx in 0..dist.cluster.nodes.len() { + assert_object_bytes(&dist.client(node_idx)?, &bucket, "survive.bin", &body).await?; + } + Ok(()) +} + +#[tokio::test] +async fn fresh_drive_replacement_is_physically_healed_without_data_change() -> TestResult { + init_logging(); + let mut dist = DistCluster::start_with_env(DistLayout::FourByFour, &[("RUSTFS_HEAL_ENABLED", "true")]).await?; + let bucket = unique_bucket("baddrive"); + dist.create_bucket(&bucket).await?; + let body = payload_for("fresh-drive/durable.bin", 8 * 1024 * 1024); + put_object(&dist.client(1)?, &bucket, "durable.bin", body.clone()).await?; + + let replaced_drive = PathBuf::from(&dist.cluster.nodes[0].data_dirs[0]); + let baseline = census_object_version_on_disk(&replaced_drive, &bucket, "durable.bin", None)?; + assert!( + baseline.is_complete(), + "replacement target did not hold a complete baseline shard: {baseline:?}" + ); + assert!( + !baseline.expected_part_numbers.is_empty(), + "replacement witness must use physical part shards: {baseline:?}" + ); + + dist.cluster.stop_node(0)?; + let format_path = replaced_drive.join(".rustfs.sys/format.json"); + let format = std::fs::read(&format_path)?; + let retired_drive = PathBuf::from(format!("{}.retired", replaced_drive.display())); + std::fs::rename(&replaced_drive, &retired_drive)?; + std::fs::create_dir_all(format_path.parent().ok_or("replacement format path omitted parent")?)?; + std::fs::write(&format_path, format)?; + let empty = census_object_version_on_disk(&replaced_drive, &bucket, "durable.bin", None)?; + assert!(!empty.has_xl_meta, "fresh replacement unexpectedly retained object metadata: {empty:?}"); + + dist.cluster.start_node(0).await?; + wait_for_ready(&dist.cluster).await?; + let heal_body = + r#"{"recursive":true,"dryRun":false,"remove":false,"recreate":true,"scanMode":2,"updateParity":false,"nolock":false}"#; + let heal_url = format!("{}/rustfs/admin/v3/heal/{bucket}?forceStart=true", dist.cluster.nodes[1].url); + signed_admin_post(&heal_url, Some(heal_body), &dist.cluster.access_key, &dist.cluster.secret_key).await?; + wait_until( + Duration::from_secs(90), + || async { + let healed = census_object_version_on_disk(&replaced_drive, &bucket, "durable.bin", None)?; + Ok(healed.matches_manifest(&baseline)) + }, + "fresh replacement contains the original complete shard manifest", + ) + .await?; + + for node_idx in 0..dist.cluster.nodes.len() { + assert_object_bytes(&dist.client(node_idx)?, &bucket, "durable.bin", &body).await?; + } + Ok(()) +} + +#[tokio::test] +async fn concurrent_gets_survive_peer_node_kill() -> TestResult { + init_logging(); + let mut dist = DistCluster::start(DistLayout::FourByFour).await?; + let bucket = unique_bucket("getkill"); + dist.create_bucket(&bucket).await?; + let body = payload_for("inflight/steady.bin", 8 * 1024 * 1024); + put_object(&dist.client(0)?, &bucket, "steady.bin", body.clone()).await?; + + let live: Vec<_> = (0..3).map(|idx| dist.client(idx)).collect::, _>>()?; + let worker_count = 12; + let release = Arc::new(Barrier::new(worker_count + 1)); + let (started_tx, mut started_rx) = mpsc::unbounded_channel(); + let mut handles = Vec::new(); + for idx in 0..worker_count { + let client = live[idx % live.len()].clone(); + let bucket = bucket.clone(); + let body = body.clone(); + let release = release.clone(); + let started_tx = started_tx.clone(); + handles.push(tokio::spawn(async move { + let response = client.get_object().bucket(&bucket).key("steady.bin").send().await?; + if response.content_length() != Some(body.len() as i64) { + return Err::<(), Box>( + format!("worker {idx} received a wrong content length").into(), + ); + } + started_tx.send(idx)?; + release.wait().await; + let actual = response.body.collect().await?.into_bytes(); + if actual.as_ref() != body.as_slice() { + return Err(format!("worker {idx} received corrupted bytes after peer kill").into()); + } + Ok(()) + })); + } + drop(started_tx); + for _ in 0..worker_count { + timeout(Duration::from_secs(30), started_rx.recv()) + .await? + .ok_or("a streaming GET exited before reaching the kill barrier")?; + } + + dist.cluster.stop_node(3)?; + release.wait().await; + for handle in handles { + handle.await??; + } + + dist.cluster.start_node(3).await?; + wait_for_ready(&dist.cluster).await?; + assert_object_bytes(&dist.client(3)?, &bucket, "steady.bin", &body).await?; + Ok(()) +} + +#[tokio::test] +async fn blackholed_node_client_network_preserves_cluster_availability_and_recovers() -> TestResult { + init_logging(); + let dist = DistCluster::start(DistLayout::FourByFour).await?; + let proxy = crate::fault_proxy::FaultProxy::start(dist.cluster.nodes[3].address.parse()?).await?; + let proxied_url = format!("http://{}", proxy.local_addr()); + let proxied_client = Client::from_conf(build_test_s3_config( + &proxied_url, + &dist.cluster.access_key, + &dist.cluster.secret_key, + None, + "distributed-network-chaos", + )); + + let result: TestResult = async { + let bucket = unique_bucket("netfault"); + dist.create_bucket(&bucket).await?; + let baseline = payload_for("network/baseline.bin", 1024 * 1024); + put_object(&dist.client(0)?, &bucket, "baseline.bin", baseline.clone()).await?; + assert_object_bytes(&proxied_client, &bucket, "baseline.bin", &baseline).await?; + + proxy.set_mode(FaultMode::Blackhole); + assert_eq!(proxy.mode(), FaultMode::Blackhole); + if let Ok(Ok(_)) = timeout( + Duration::from_secs(5), + proxied_client.get_object().bucket(&bucket).key("baseline.bin").send(), + ) + .await + { + return Err("blackholed node endpoint unexpectedly completed a GET".into()); + } + + let during = payload_for("network/during.bin", 1024 * 1024); + timeout(Duration::from_secs(30), async { + put_object(&dist.client(1)?, &bucket, "during-blackhole.bin", during.clone()).await?; + assert_object_bytes(&dist.client(2)?, &bucket, "baseline.bin", &baseline).await?; + assert_object_bytes(&dist.client(0)?, &bucket, "during-blackhole.bin", &during).await?; + Ok::<_, Box>(()) + }) + .await??; + + proxy.set_mode(FaultMode::Pass); + retrying_get_equals(&proxied_client, &bucket, "during-blackhole.bin", &during, Duration::from_secs(30)).await?; + Ok(()) + } + .await; + + proxy.set_mode(FaultMode::Pass); + proxy.shutdown().await; + result +} diff --git a/crates/e2e_test/src/distributed/concurrency_stability_test.rs b/crates/e2e_test/src/distributed/concurrency_stability_test.rs new file mode 100644 index 000000000..06a24e9a4 --- /dev/null +++ b/crates/e2e_test/src/distributed/concurrency_stability_test.rs @@ -0,0 +1,98 @@ +// Copyright 2026 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use super::harness::{DistCluster, DistLayout, TestResult, assert_object_bytes, payload_for, put_object, unique_bucket}; +use crate::common::init_logging; +use std::collections::BTreeSet; +use std::sync::Arc; +use tokio::sync::Barrier; + +#[tokio::test] +async fn four_node_high_concurrency_mixed_workload_is_consistent_on_every_node() -> TestResult { + init_logging(); + let dist = DistCluster::start(DistLayout::FourByFour).await?; + let bucket = unique_bucket("conc"); + dist.create_bucket(&bucket).await?; + let clients = Arc::new(dist.clients()?); + let worker_count = 24; + let rounds = 4; + let barrier = Arc::new(Barrier::new(worker_count)); + + let mut handles = Vec::new(); + for idx in 0..worker_count { + let clients = clients.clone(); + let barrier = barrier.clone(); + let bucket = bucket.clone(); + handles.push(tokio::spawn(async move { + barrier.wait().await; + let writer = &clients[idx % clients.len()]; + let reader = &clients[(idx + 1) % clients.len()]; + let copier = &clients[(idx + 2) % clients.len()]; + let mut retained = Vec::with_capacity(rounds); + for round in 0..rounds { + let key = format!("source/worker-{idx:02}-round-{round}.bin"); + let copy_key = format!("retained/worker-{idx:02}-round-{round}.bin"); + let body = payload_for(&key, 64 * 1024); + put_object(writer, &bucket, &key, body.clone()).await?; + + let head = reader.head_object().bucket(&bucket).key(&key).send().await?; + if head.content_length() != Some(body.len() as i64) { + return Err(format!("HEAD returned the wrong size for {key}: {head:?}").into()); + } + assert_object_bytes(reader, &bucket, &key, &body).await?; + + copier + .copy_object() + .bucket(&bucket) + .key(©_key) + .copy_source(format!("{bucket}/{key}")) + .send() + .await?; + assert_object_bytes(writer, &bucket, ©_key, &body).await?; + + writer.delete_object().bucket(&bucket).key(&key).send().await?; + let missing = reader + .head_object() + .bucket(&bucket) + .key(&key) + .send() + .await + .expect_err("deleted source key must not remain visible"); + if missing.raw_response().map(|response| response.status().as_u16()) != Some(404) { + return Err(format!("deleted source {key} returned an unexpected result: {missing:?}").into()); + } + retained.push((copy_key, body)); + } + Ok::<_, Box>(retained) + })); + } + + let mut inventory = Vec::new(); + for handle in handles { + inventory.extend(handle.await??); + } + + let expected_keys: BTreeSet<_> = inventory.iter().map(|(key, _)| key.as_str()).collect(); + for (node_idx, client) in clients.iter().enumerate() { + let listed = client.list_objects_v2().bucket(&bucket).prefix("retained/").send().await?; + let listed_keys: BTreeSet<_> = listed.contents().iter().filter_map(|object| object.key()).collect(); + assert_eq!(listed_keys, expected_keys, "node {node_idx} returned a divergent retained-key listing"); + for (key, body) in &inventory { + assert_object_bytes(client, &bucket, key, body) + .await + .map_err(|error| format!("node {node_idx} failed to read {key}: {error}"))?; + } + } + Ok(()) +} diff --git a/crates/e2e_test/src/distributed/concurrent_data_movement_test.rs b/crates/e2e_test/src/distributed/concurrent_data_movement_test.rs new file mode 100644 index 000000000..66b30f5de --- /dev/null +++ b/crates/e2e_test/src/distributed/concurrent_data_movement_test.rs @@ -0,0 +1,74 @@ +// Copyright 2026 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use super::harness::{ + DECOMMISSION_POOL_ID, DistCluster, DistLayout, TestResult, assert_inventory, decommission_running_with_progress, + decommission_status_json, payload_for, put_inventory_retrying, retrying_get_equals, retrying_put, start_decommission, + unique_bucket, wait_for_decommission_complete, wait_for_decommission_running_with_progress, +}; +use crate::common::init_logging; +use std::sync::Arc; +use std::time::Duration; +use tokio::sync::Barrier; + +#[tokio::test] +async fn concurrent_puts_during_decommission_do_not_lose_baseline_or_new_objects() -> TestResult { + init_logging(); + let mut dist = DistCluster::start(DistLayout::SingleNodeFourDrive).await?; + let bucket = unique_bucket("concdecom"); + dist.create_bucket(&bucket).await?; + let baseline_client = dist.client(0)?; + let inventory = put_inventory_retrying(&baseline_client, &bucket, 96, 256 * 1024, Duration::from_secs(30)).await?; + dist.expand_to_four_pools().await?; + + start_decommission(&dist.cluster, DECOMMISSION_POOL_ID).await?; + + let clients = Arc::new(dist.clients()?); + let barrier = Arc::new(Barrier::new(17)); + let mut handles = Vec::new(); + for idx in 0..16 { + let clients = clients.clone(); + let barrier = barrier.clone(); + let bucket = bucket.clone(); + handles.push(tokio::spawn(async move { + barrier.wait().await; + let client = &clients[idx % clients.len()]; + let key = format!("live/{idx:02}.bin"); + let body = payload_for(&key, 8 * 1024); + retrying_put(client, &bucket, &key, body.clone(), Duration::from_secs(45)).await?; + Ok::<_, Box>((key, body)) + })); + } + + wait_for_decommission_running_with_progress(&dist.cluster, DECOMMISSION_POOL_ID, Duration::from_secs(30)).await?; + barrier.wait().await; + + let mut live_objects = Vec::new(); + for handle in handles { + live_objects.push(handle.await??); + } + let status = decommission_status_json(&dist.cluster).await?; + if !decommission_running_with_progress(&status, DECOMMISSION_POOL_ID)? { + return Err(format!("decommission did not remain active across concurrent PUTs: {status}").into()); + } + + wait_for_decommission_complete(&dist.cluster, DECOMMISSION_POOL_ID, Duration::from_secs(180)).await?; + + let checker = dist.client(2)?; + assert_inventory(&checker, &bucket, &inventory).await?; + for (key, body) in live_objects { + retrying_get_equals(&checker, &bucket, &key, &body, Duration::from_secs(30)).await?; + } + Ok(()) +} diff --git a/crates/e2e_test/src/distributed/data_integrity_movement_test.rs b/crates/e2e_test/src/distributed/data_integrity_movement_test.rs new file mode 100644 index 000000000..266f31f9d --- /dev/null +++ b/crates/e2e_test/src/distributed/data_integrity_movement_test.rs @@ -0,0 +1,156 @@ +// Copyright 2026 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use super::harness::{ + DECOMMISSION_POOL_ID, DistCluster, DistLayout, TestResult, assert_inventory, enable_versioning, put_inventory_retrying, + sha256_hex, start_decommission, unique_bucket, wait_for_decommission_active, wait_for_decommission_complete, +}; +use crate::common::init_logging; +use aws_sdk_s3::primitives::ByteStream; +use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart}; +use std::time::Duration; + +#[tokio::test] +async fn decommission_does_not_alter_object_sha256_across_pools() -> TestResult { + init_logging(); + let mut dist = DistCluster::start(DistLayout::SingleNodeFourDrive).await?; + let bucket = unique_bucket("integrity"); + dist.create_bucket(&bucket).await?; + let client = dist.client(0)?; + enable_versioning(&client, &bucket).await?; + let inventory = put_inventory_retrying(&client, &bucket, 96, 256 * 1024, Duration::from_secs(30)).await?; + let before: Vec<(String, String)> = inventory.iter().map(|(key, body)| (key.clone(), sha256_hex(body))).collect(); + + let versioned_key = "history/versioned.bin"; + let version_one = b"historical bytes before data movement".to_vec(); + let version_two = b"current bytes before data movement".to_vec(); + let version_one_id = client + .put_object() + .bucket(&bucket) + .key(versioned_key) + .body(ByteStream::from(version_one.clone())) + .send() + .await? + .version_id() + .ok_or("historical PUT omitted version ID")? + .to_string(); + let version_two_id = client + .put_object() + .bucket(&bucket) + .key(versioned_key) + .body(ByteStream::from(version_two.clone())) + .send() + .await? + .version_id() + .ok_or("current PUT omitted version ID")? + .to_string(); + + let multipart_key = "multipart/moved.bin"; + let first_part = vec![0x31; 5 * 1024 * 1024]; + let second_part = vec![0x72; 1024 * 1024]; + let upload = client + .create_multipart_upload() + .bucket(&bucket) + .key(multipart_key) + .send() + .await?; + let upload_id = upload.upload_id().ok_or("movement multipart upload omitted upload ID")?; + let uploaded_one = client + .upload_part() + .bucket(&bucket) + .key(multipart_key) + .upload_id(upload_id) + .part_number(1) + .body(ByteStream::from(first_part.clone())) + .send() + .await?; + let uploaded_two = client + .upload_part() + .bucket(&bucket) + .key(multipart_key) + .upload_id(upload_id) + .part_number(2) + .body(ByteStream::from(second_part.clone())) + .send() + .await?; + client + .complete_multipart_upload() + .bucket(&bucket) + .key(multipart_key) + .upload_id(upload_id) + .multipart_upload( + CompletedMultipartUpload::builder() + .parts( + CompletedPart::builder() + .part_number(1) + .e_tag(uploaded_one.e_tag().ok_or("movement part 1 omitted ETag")?) + .build(), + ) + .parts( + CompletedPart::builder() + .part_number(2) + .e_tag(uploaded_two.e_tag().ok_or("movement part 2 omitted ETag")?) + .build(), + ) + .build(), + ) + .send() + .await?; + + dist.expand_to_four_pools().await?; + + start_decommission(&dist.cluster, DECOMMISSION_POOL_ID).await?; + wait_for_decommission_active(&dist.cluster, DECOMMISSION_POOL_ID, Duration::from_secs(30)).await?; + wait_for_decommission_complete(&dist.cluster, DECOMMISSION_POOL_ID, Duration::from_secs(180)).await?; + + let after_client = dist.client(2)?; + assert_inventory(&after_client, &bucket, &inventory).await?; + for (key, expected_hash) in before { + let got = after_client.get_object().bucket(&bucket).key(&key).send().await?; + let body = got.body.collect().await?.into_bytes(); + assert_eq!(sha256_hex(body.as_ref()), expected_hash, "checksum changed for {key} after decommission"); + } + for (version_id, expected) in [(&version_one_id, &version_one), (&version_two_id, &version_two)] { + let got = after_client + .get_object() + .bucket(&bucket) + .key(versioned_key) + .version_id(version_id) + .send() + .await? + .body + .collect() + .await? + .into_bytes(); + assert_eq!(got.as_ref(), expected.as_slice(), "version {version_id} changed after decommission"); + } + let mut expected_multipart = first_part; + expected_multipart.extend_from_slice(&second_part); + let got_multipart = after_client + .get_object() + .bucket(&bucket) + .key(multipart_key) + .send() + .await? + .body + .collect() + .await? + .into_bytes(); + assert_eq!( + sha256_hex(got_multipart.as_ref()), + sha256_hex(&expected_multipart), + "multipart checksum changed after decommission" + ); + Ok(()) +} diff --git a/crates/e2e_test/src/distributed/expand_decommission_rebalance_test.rs b/crates/e2e_test/src/distributed/expand_decommission_rebalance_test.rs new file mode 100644 index 000000000..d76aa9e05 --- /dev/null +++ b/crates/e2e_test/src/distributed/expand_decommission_rebalance_test.rs @@ -0,0 +1,81 @@ +// Copyright 2026 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use super::harness::{ + DECOMMISSION_POOL_ID, DistCluster, DistLayout, TestResult, assert_inventory, list_pools_json, put_inventory, + put_inventory_retrying, start_decommission, start_rebalance, unique_bucket, wait_for_decommission_active, + wait_for_decommission_complete, wait_for_rebalance_active, wait_for_rebalance_complete, +}; +use crate::common::init_logging; +use std::time::Duration; + +#[tokio::test] +async fn four_node_pool_expand_preserves_objects_then_rebalance() -> TestResult { + init_logging(); + let mut dist = DistCluster::start(DistLayout::SingleNodeFourDrive).await?; + let bucket = unique_bucket("expand"); + dist.create_bucket(&bucket).await?; + let client = dist.client(0)?; + let inventory = put_inventory(&client, &bucket, 64, 256 * 1024).await?; + assert_inventory(&client, &bucket, &inventory).await?; + + for expected_nodes in 2..=4 { + let new_node = dist.append_pool_and_restart().await?; + assert_eq!(new_node + 1, expected_nodes); + assert_inventory(&dist.client(new_node)?, &bucket, &inventory).await?; + } + assert_eq!(dist.cluster.nodes.len(), 4); + + // Prove that the expanded pool map is durable, and clear any recovery + // latch raised while the newly-added pool replicas converged. + dist.restart_current_binary_gracefully().await?; + + let after_expand = dist.client(0)?; + assert_inventory(&after_expand, &bucket, &inventory).await?; + let peer = dist.client(3)?; + assert_inventory(&peer, &bucket, &inventory).await?; + + let rebalance_id = start_rebalance(&dist.cluster).await?; + wait_for_rebalance_active(&dist.cluster, &rebalance_id, Duration::from_secs(30)).await?; + wait_for_rebalance_complete(&dist.cluster, &rebalance_id, Duration::from_secs(180)).await?; + assert_inventory(&peer, &bucket, &inventory).await?; + Ok(()) +} + +#[tokio::test] +async fn four_pool_decommission_moves_objects_without_loss() -> TestResult { + init_logging(); + let mut dist = DistCluster::start(DistLayout::SingleNodeFourDrive).await?; + let bucket = unique_bucket("decom"); + dist.create_bucket(&bucket).await?; + let client = dist.client(0)?; + let inventory = put_inventory_retrying(&client, &bucket, 96, 128 * 1024, Duration::from_secs(30)).await?; + dist.expand_to_four_pools().await?; + + let pools_before = list_pools_json(&dist.cluster).await?; + let pool_count = pools_before + .as_array() + .map(Vec::len) + .or_else(|| pools_before.get("pools").and_then(serde_json::Value::as_array).map(Vec::len)) + .ok_or_else(|| format!("pool list omitted an array: {pools_before}"))?; + assert_eq!(pool_count, 4, "expected exactly four pools before decommission: {pools_before}"); + + start_decommission(&dist.cluster, DECOMMISSION_POOL_ID).await?; + wait_for_decommission_active(&dist.cluster, DECOMMISSION_POOL_ID, Duration::from_secs(30)).await?; + wait_for_decommission_complete(&dist.cluster, DECOMMISSION_POOL_ID, Duration::from_secs(180)).await?; + + let after = dist.client(2)?; + assert_inventory(&after, &bucket, &inventory).await?; + Ok(()) +} diff --git a/crates/e2e_test/src/distributed/extra_test.rs b/crates/e2e_test/src/distributed/extra_test.rs new file mode 100644 index 000000000..590dd13f4 --- /dev/null +++ b/crates/e2e_test/src/distributed/extra_test.rs @@ -0,0 +1,149 @@ +// Copyright 2026 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use super::harness::{ + DistCluster, DistLayout, TestResult, assert_object_bytes, get_object_bytes, put_object, unique_bucket, wait_until, +}; +use crate::common::init_logging; +use aws_sdk_s3::primitives::ByteStream; +use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart}; +use std::time::Duration; + +#[tokio::test] +async fn four_node_four_drive_multipart_and_cross_node_listing_agree() -> TestResult { + init_logging(); + let dist = DistCluster::start(DistLayout::FourByFour).await?; + let bucket = unique_bucket("extra"); + dist.create_bucket(&bucket).await?; + let client = dist.client(0)?; + + let key = "multipart.bin"; + let part1 = vec![0x41u8; 5 * 1024 * 1024]; + let part2 = vec![0x42u8; 5 * 1024 * 1024]; + let upload = client.create_multipart_upload().bucket(&bucket).key(key).send().await?; + let upload_id = upload.upload_id().ok_or("missing upload id")?.to_string(); + + let uploaded1 = client + .upload_part() + .bucket(&bucket) + .key(key) + .upload_id(&upload_id) + .part_number(1) + .body(ByteStream::from(part1.clone())) + .send() + .await?; + let uploaded2 = client + .upload_part() + .bucket(&bucket) + .key(key) + .upload_id(&upload_id) + .part_number(2) + .body(ByteStream::from(part2.clone())) + .send() + .await?; + + client + .complete_multipart_upload() + .bucket(&bucket) + .key(key) + .upload_id(&upload_id) + .multipart_upload( + CompletedMultipartUpload::builder() + .parts( + CompletedPart::builder() + .part_number(1) + .e_tag(uploaded1.e_tag().unwrap_or_default()) + .build(), + ) + .parts( + CompletedPart::builder() + .part_number(2) + .e_tag(uploaded2.e_tag().unwrap_or_default()) + .build(), + ) + .build(), + ) + .send() + .await?; + + let mut expected = part1; + expected.extend_from_slice(&part2); + for node_idx in 0..dist.cluster.nodes.len() { + assert_object_bytes(&dist.client(node_idx)?, &bucket, key, &expected).await?; + } + + put_object(&client, &bucket, "list/a", b"a".to_vec()).await?; + put_object(&dist.client(2)?, &bucket, "list/b", b"b".to_vec()).await?; + let mut seen = Vec::new(); + for node_idx in 0..dist.cluster.nodes.len() { + let listed = dist + .client(node_idx)? + .list_objects_v2() + .bucket(&bucket) + .prefix("list/") + .send() + .await?; + let keys: Vec = listed + .contents() + .iter() + .filter_map(|object| object.key().map(str::to_string)) + .collect(); + seen.push(keys); + } + for keys in &seen[1..] { + assert_eq!(&seen[0], keys, "list results diverged across nodes: {seen:?}"); + } + + let got = get_object_bytes(&dist.client(3)?, &bucket, "list/a").await?; + assert_eq!(got, b"a"); + Ok(()) +} + +#[tokio::test] +async fn four_node_list_buckets_agree_across_all_nodes() -> TestResult { + init_logging(); + let dist = DistCluster::start(DistLayout::FourByFour).await?; + let bucket = unique_bucket("listed"); + dist.create_bucket(&bucket).await?; + put_object(&dist.client(0)?, &bucket, "seed.bin", b"seed".to_vec()).await?; + + for node_idx in 0..dist.cluster.nodes.len() { + let client = dist.client(node_idx)?; + let name = bucket.clone(); + wait_until( + Duration::from_secs(20), + || { + let client = client.clone(); + let name = name.clone(); + async move { + let listed = client.list_buckets().send().await?; + Ok(listed.buckets().iter().any(|entry| entry.name() == Some(name.as_str()))) + } + }, + &format!("node {node_idx} lists {bucket}"), + ) + .await?; + wait_until( + Duration::from_secs(20), + || { + let client = dist.client(node_idx).expect("client"); + let name = bucket.clone(); + async move { Ok(get_object_bytes(&client, &name, "seed.bin").await.ok() == Some(b"seed".to_vec())) } + }, + &format!("node {node_idx} reads seed.bin"), + ) + .await?; + } + Ok(()) +} diff --git a/crates/e2e_test/src/distributed/harness.rs b/crates/e2e_test/src/distributed/harness.rs new file mode 100644 index 000000000..f610beaa4 --- /dev/null +++ b/crates/e2e_test/src/distributed/harness.rs @@ -0,0 +1,1235 @@ +// Copyright 2026 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Shared 4-node distributed e2e helpers. +//! +//! Two localhost-expressible layouts cover the suite: +//! +//! * **4×4 single pool** (`four_by_four`) — four processes, four drives each, +//! one `DistErasure` pool (16 explicit volume endpoints). This is the +//! default S3 / lock / versioning / chaos topology. +//! * **4×4 four pool** — start one durable single-node pool, then append three +//! single-node pools one at a time. Required for decommission/rebalance/expand, +//! which the server rejects on a single pool. Cold-starting multiple empty +//! pools races their bootstrap identities on localhost DistErasure. +//! +//! Genuine multi-node *striped* pools still need multi-host CI (backlog +//! #1313 / #1314). Site replication uses two 4-node 1-drive clusters so the +//! process count stays at eight rather than sixteen. + +use crate::common::{ + ClusterTopology, FAST_DATA_USAGE_SCANNER_ENV, RustFSTestClusterEnvironment, admin_request, build_test_s3_config, + local_http_client, replication_fast_env, signed_request, +}; +use crate::replication_extension_test::LOOPBACK_REPLICATION_TARGET_ENV; +use aws_sdk_s3::Client; +use aws_sdk_s3::primitives::ByteStream; +use aws_sdk_s3::types::{BucketVersioningStatus, VersioningConfiguration}; +use http::{Method, StatusCode}; +use sha2::{Digest, Sha256}; +use std::collections::BTreeMap; +use std::path::{Path, PathBuf}; +use std::time::Duration; +use tokio::time::{Instant, sleep}; +use uuid::Uuid; + +pub(crate) type TestResult = Result>; + +pub(crate) const NODE_COUNT: usize = 4; +pub(crate) const DRIVES_PER_NODE: usize = 4; +/// Retire the seed pool after test data is written there, proving that user +/// objects—not only internal metadata—move to the expansion pools. +pub(crate) const DECOMMISSION_POOL_ID: usize = 0; +const POOL_ROOTS_ENV: &str = "RUSTFS_E2E_POOL_ROOTS"; +const POOL_META_V3_ENV: [(&str, &str); 2] = [ + ("RUSTFS_POOL_META_V3_WRITE", "true"), + ("RUSTFS_POOL_META_V3_FLEET_CONFIRMED", "true"), +]; + +#[derive(Clone, Copy, Debug)] +pub(crate) enum DistLayout { + /// 4 nodes × 4 drives, one erasure pool spanning every endpoint. + FourByFour, + /// 4 nodes × 1 drive, one erasure pool (minimum 4-node 4-disk layout). + FourNodeFourDisk, + /// 1 node × 4 drives, used only as the durable seed for pool expansion. + SingleNodeFourDrive, +} + +pub(crate) struct DistCluster { + pub cluster: RustFSTestClusterEnvironment, + pool_storage_roots: Option>, + owned_pool_dirs: Vec, +} + +impl DistCluster { + pub async fn start(layout: DistLayout) -> TestResult { + Self::start_with_env(layout, &[]).await + } + + pub async fn start_with_env(layout: DistLayout, extra_env: &[(&str, &str)]) -> TestResult { + let mut dist = Self::new_stopped_with_env(layout, extra_env).await?; + dist.cluster.start().await?; + Ok(dist) + } + + /// Allocate ports and data dirs without spawning processes. + /// + /// Upgrade tests configure capture logs, then start a pinned previous + /// binary against the same directories. + pub async fn new_stopped(layout: DistLayout) -> TestResult { + Self::new_stopped_with_env(layout, &[]).await + } + + pub async fn new_stopped_with_env(layout: DistLayout, extra_env: &[(&str, &str)]) -> TestResult { + let topology = match layout { + DistLayout::FourByFour => ClusterTopology::single_pool_multidrive(NODE_COUNT, DRIVES_PER_NODE), + DistLayout::FourNodeFourDisk => ClusterTopology::single_pool(NODE_COUNT), + DistLayout::SingleNodeFourDrive => ClusterTopology::per_node_pools(DRIVES_PER_NODE, vec![vec![0]]), + }; + let mut cluster = RustFSTestClusterEnvironment::with_topology(topology).await?; + let pool_storage_roots = match layout { + DistLayout::SingleNodeFourDrive => Some(configured_pool_storage_roots()?), + DistLayout::FourByFour | DistLayout::FourNodeFourDisk => None, + }; + let mut owned_pool_dirs = Vec::new(); + if let Some(roots) = pool_storage_roots.as_deref() { + owned_pool_dirs.push(relocate_pool_storage(&mut cluster, 0, roots)?); + } + cluster.set_env("NO_PROXY", "127.0.0.1,localhost"); + cluster.set_env("HTTP_PROXY", ""); + cluster.set_env("HTTPS_PROXY", ""); + if matches!(layout, DistLayout::SingleNodeFourDrive) { + // This fresh, same-version fleet is safe to initialize at V3. The + // durable generation protocol is required for concurrent + // decommission progress updates and crash-recoverable movement. + for (key, value) in POOL_META_V3_ENV { + cluster.set_env(key, value); + } + } + for &(key, value) in extra_env { + cluster.set_env(key, value); + } + configure_node_logs(&mut cluster)?; + Ok(Self { + cluster, + pool_storage_roots, + owned_pool_dirs, + }) + } + + /// Start every node with a specific `rustfs` binary, keeping the allocated + /// data directories. Used to seed an old on-disk format before upgrading. + pub async fn start_from_binary(&mut self, binary: &Path) -> TestResult { + self.cluster.start_with_binary(binary).await?; + wait_for_ready(&self.cluster).await?; + Ok(()) + } + + /// Stop every node and bring the same data directories up on the workspace + /// binary (direct upgrade). + pub async fn restart_with_current_binary(&mut self) -> TestResult { + self.cluster.stop(); + self.cluster.start().await?; + wait_for_ready(&self.cluster).await?; + Ok(()) + } + + /// Planned topology changes use graceful shutdown. Crash semantics are + /// exercised separately by the chaos cases. + pub async fn restart_current_binary_gracefully(&mut self) -> TestResult { + self.stop_all_gracefully().await?; + self.cluster.start().await?; + wait_for_ready(&self.cluster).await?; + Ok(()) + } + + /// Replace one running node with the workspace binary (rolling upgrade). + pub async fn replace_node_with_current_binary(&mut self, node_idx: usize) -> TestResult { + self.cluster.stop_node(node_idx)?; + self.cluster.start_node(node_idx).await?; + wait_for_ready(&self.cluster).await?; + Ok(()) + } + + pub fn client_with_credentials(&self, node_idx: usize, access_key: &str, secret_key: &str) -> TestResult { + if node_idx >= self.cluster.nodes.len() { + return Err("node_idx is invalid".into()); + } + Ok(Client::from_conf(build_test_s3_config( + &self.cluster.nodes[node_idx].url, + access_key, + secret_key, + None, + "cluster-iam-test", + ))) + } + + pub async fn append_pool_and_restart(&mut self) -> TestResult { + self.stop_all_gracefully().await?; + let node_idx = self.cluster.append_single_node_pool().await?; + let roots = self + .pool_storage_roots + .as_deref() + .ok_or("pool expansion requires configured isolated pool filesystems")?; + self.owned_pool_dirs + .push(relocate_pool_storage(&mut self.cluster, node_idx, roots)?); + configure_node_logs(&mut self.cluster)?; + self.cluster.start().await?; + wait_for_ready(&self.cluster).await?; + Ok(node_idx) + } + + async fn stop_all_gracefully(&mut self) -> TestResult { + for node_idx in 0..self.cluster.nodes.len() { + self.cluster.stop_node_gracefully(node_idx).await?; + } + Ok(()) + } + + /// Expand the durable seed into four single-node pools through three + /// serialized additions. Callers can seed objects before this step so a + /// later pool-0 decommission proves that user data crosses pool boundaries. + pub async fn expand_to_four_pools(&mut self) -> TestResult { + for _ in 0..3 { + self.append_pool_and_restart().await?; + } + self.restart_current_binary_gracefully().await?; + Ok(()) + } + + pub async fn start_replication_pair() -> TestResult<(Self, Self)> { + let mut extra: Vec<(&str, &str)> = replication_fast_env(); + extra.extend_from_slice(LOOPBACK_REPLICATION_TARGET_ENV); + extra.extend_from_slice(FAST_DATA_USAGE_SCANNER_ENV); + let source = Self::start_with_env(DistLayout::FourNodeFourDisk, &extra).await?; + let target = Self::start_with_env(DistLayout::FourNodeFourDisk, &extra).await?; + Ok((source, target)) + } + + pub fn client(&self, node_idx: usize) -> TestResult { + self.cluster.create_s3_client(node_idx) + } + + pub fn clients(&self) -> TestResult> { + self.cluster.create_all_clients() + } + + pub async fn create_bucket(&self, bucket: &str) -> TestResult { + self.cluster.create_test_bucket(bucket).await + } +} + +impl Drop for DistCluster { + fn drop(&mut self) { + self.cluster.stop(); + for path in &self.owned_pool_dirs { + if let Err(error) = std::fs::remove_dir_all(path) { + eprintln!("failed to clean up isolated distributed E2E pool directory {}: {error}", path.display()); + } + } + } +} + +fn configured_pool_storage_roots() -> TestResult> { + let raw = std::env::var_os(POOL_ROOTS_ENV).ok_or_else(|| { + format!( + "{POOL_ROOTS_ENV} must name {NODE_COUNT} isolated filesystems for pool expansion, decommission, and rebalance tests" + ) + })?; + validate_pool_storage_roots(&raw) +} + +fn validate_pool_storage_roots(raw: &std::ffi::OsStr) -> TestResult> { + let roots: Vec = std::env::split_paths(raw).collect(); + if roots.len() != NODE_COUNT { + return Err(format!("{POOL_ROOTS_ENV} must contain exactly {NODE_COUNT} paths, got {}", roots.len()).into()); + } + + let mut canonical_roots = Vec::with_capacity(roots.len()); + for root in roots { + if !root.is_absolute() { + return Err(format!("{POOL_ROOTS_ENV} path must be absolute: {}", root.display()).into()); + } + let canonical = root + .canonicalize() + .map_err(|error| format!("{POOL_ROOTS_ENV} path {} is unavailable: {error}", root.display()))?; + if !canonical.is_dir() { + return Err(format!("{POOL_ROOTS_ENV} path is not a directory: {}", canonical.display()).into()); + } + if canonical_roots.contains(&canonical) { + return Err(format!("{POOL_ROOTS_ENV} contains a duplicate path: {}", canonical.display()).into()); + } + canonical_roots.push(canonical); + } + + #[cfg(unix)] + { + use std::os::unix::fs::MetadataExt; + + let devices: std::collections::BTreeSet = canonical_roots + .iter() + .map(|root| std::fs::metadata(root).map(|metadata| metadata.dev())) + .collect::>()?; + if devices.len() != canonical_roots.len() { + return Err(format!( + "{POOL_ROOTS_ENV} paths must be backed by distinct filesystems; found {} devices for {} paths", + devices.len(), + canonical_roots.len() + ) + .into()); + } + } + + Ok(canonical_roots) +} + +fn relocate_pool_storage(cluster: &mut RustFSTestClusterEnvironment, node_idx: usize, roots: &[PathBuf]) -> TestResult { + let root = roots + .get(node_idx) + .ok_or_else(|| format!("no isolated pool filesystem configured for node {node_idx}"))?; + let cluster_name = Path::new(&cluster.temp_dir) + .file_name() + .ok_or("cluster temp directory omitted a basename")?; + let pool_run_root = root.join(cluster_name); + let node_root = pool_run_root.join(format!("node{node_idx}")); + let data_dirs: Vec = (0..cluster.topology.drives_per_node) + .map(|drive| node_root.join(format!("drive{drive}")).to_string_lossy().into_owned()) + .collect(); + for data_dir in &data_dirs { + std::fs::create_dir_all(data_dir)?; + } + let node = cluster + .nodes + .get_mut(node_idx) + .ok_or_else(|| format!("cannot relocate missing cluster node {node_idx}"))?; + node.data_dir = data_dirs[0].clone(); + node.data_dirs = data_dirs; + Ok(pool_run_root) +} + +fn configure_node_logs(cluster: &mut RustFSTestClusterEnvironment) -> TestResult { + let Some(log_dir) = std::env::var_os("RUSTFS_E2E_LOG_DIR") else { + return Ok(()); + }; + std::fs::create_dir_all(&log_dir)?; + let cluster_id = Uuid::new_v4().simple().to_string(); + for node_idx in 0..cluster.nodes.len() { + let path = Path::new(&log_dir).join(format!("cluster-{cluster_id}-node-{node_idx}.log")); + cluster.set_node_capture_log_path(node_idx, path.to_string_lossy().into_owned())?; + } + Ok(()) +} + +pub(crate) fn unique_bucket(prefix: &str) -> String { + let id = Uuid::new_v4().simple().to_string(); + format!("{prefix}-{}", &id[..12]) +} + +pub(crate) fn sha256_hex(bytes: &[u8]) -> String { + let digest = Sha256::digest(bytes); + digest.iter().map(|byte| format!("{byte:02x}")).collect() +} + +pub(crate) fn payload_for(key: &str, size: usize) -> Vec { + let seed = key.as_bytes(); + (0..size) + .map(|idx| seed.get(idx % seed.len()).copied().unwrap_or(0) ^ (idx as u8)) + .collect() +} + +pub(crate) async fn put_object(client: &Client, bucket: &str, key: &str, body: Vec) -> TestResult { + client + .put_object() + .bucket(bucket) + .key(key) + .body(ByteStream::from(body)) + .send() + .await?; + Ok(()) +} + +pub(crate) async fn get_object_bytes(client: &Client, bucket: &str, key: &str) -> TestResult> { + let output = client.get_object().bucket(bucket).key(key).send().await?; + Ok(output.body.collect().await?.into_bytes().to_vec()) +} + +pub(crate) async fn assert_object_bytes(client: &Client, bucket: &str, key: &str, expected: &[u8]) -> TestResult { + let got = get_object_bytes(client, bucket, key).await?; + if got.as_slice() != expected { + return Err(format!( + "object {bucket}/{key} bytes mismatch: expected {} bytes sha256={} got {} bytes sha256={}", + expected.len(), + sha256_hex(expected), + got.len(), + sha256_hex(&got) + ) + .into()); + } + Ok(()) +} + +pub(crate) async fn put_inventory( + client: &Client, + bucket: &str, + count: usize, + size: usize, +) -> TestResult>> { + let mut inventory = BTreeMap::new(); + for idx in 0..count { + let key = format!("obj-{idx:04}"); + let body = payload_for(&key, size); + put_object(client, bucket, &key, body.clone()).await?; + inventory.insert(key, body); + } + Ok(inventory) +} + +/// Retry only transport-level service availability failures while a data +/// movement operation changes the pool map. Generic InternalError responses +/// remain fatal because accepting them would hide server defects. +pub(crate) async fn put_inventory_retrying( + client: &Client, + bucket: &str, + count: usize, + size: usize, + timeout: Duration, +) -> TestResult>> { + let mut inventory = BTreeMap::new(); + for idx in 0..count { + let key = format!("obj-{idx:04}"); + let body = payload_for(&key, size); + retrying_put(client, bucket, &key, body.clone(), timeout).await?; + inventory.insert(key, body); + } + Ok(inventory) +} + +pub(crate) async fn assert_inventory(client: &Client, bucket: &str, inventory: &BTreeMap>) -> TestResult { + for (key, expected) in inventory { + assert_object_bytes(client, bucket, key, expected).await?; + } + Ok(()) +} + +pub(crate) async fn enable_versioning(client: &Client, bucket: &str) -> TestResult { + client + .put_bucket_versioning() + .bucket(bucket) + .versioning_configuration( + VersioningConfiguration::builder() + .status(BucketVersioningStatus::Enabled) + .build(), + ) + .send() + .await?; + Ok(()) +} + +pub(crate) async fn wait_until(timeout: Duration, mut probe: F, label: &str) -> TestResult +where + F: FnMut() -> Fut, + Fut: std::future::Future>, +{ + let deadline = Instant::now() + timeout; + let mut delay = Duration::from_millis(50); + loop { + let last_error = match probe().await { + Ok(true) => return Ok(()), + Ok(false) => format!("{label} still false"), + Err(error) => error.to_string(), + }; + if Instant::now() >= deadline { + return Err(format!("{label} did not become true within {timeout:?}: {last_error}").into()); + } + sleep(delay).await; + delay = (delay * 2).min(Duration::from_secs(1)); + } +} + +pub(crate) async fn cluster_admin( + cluster: &RustFSTestClusterEnvironment, + method: Method, + path_and_query: &str, + body: Option, +) -> TestResult<(StatusCode, String)> { + admin_request( + &cluster.nodes[0].url, + method, + path_and_query, + body, + &cluster.access_key, + &cluster.secret_key, + ) + .await +} + +pub(crate) async fn cluster_admin_ok( + cluster: &RustFSTestClusterEnvironment, + method: Method, + path_and_query: &str, + body: Option, +) -> TestResult { + let (status, response) = cluster_admin(cluster, method.clone(), path_and_query, body).await?; + if !status.is_success() { + return Err(format!("{method} {path_and_query} failed: {status} {response}").into()); + } + Ok(response) +} + +pub(crate) async fn wait_for_ready(cluster: &RustFSTestClusterEnvironment) -> TestResult { + let client = local_http_client(); + for node in &cluster.nodes { + let url = format!("{}/health/ready", node.url); + wait_until( + Duration::from_secs(30), + || { + let client = client.clone(); + let url = url.clone(); + async move { + match client.get(&url).send().await { + Ok(response) if response.status().is_success() => Ok(true), + _ => Ok(false), + } + } + }, + &format!("node {} ready", node.address), + ) + .await?; + } + Ok(()) +} + +pub(crate) async fn set_remote_target( + source: &RustFSTestClusterEnvironment, + source_bucket: &str, + target: &RustFSTestClusterEnvironment, + target_bucket: &str, +) -> TestResult { + let body = serde_json::json!({ + "endpoint": target.nodes[0].address, + "credentials": { + "accessKey": target.access_key, + "secretKey": target.secret_key + }, + "targetbucket": target_bucket, + "secure": false, + "type": "replication" + }); + let url = format!( + "{}/rustfs/admin/v3/set-remote-target?bucket={}", + source.nodes[0].url, + urlencoding::encode(source_bucket) + ); + let response = signed_request( + Method::PUT, + &url, + &source.access_key, + &source.secret_key, + Some(body.to_string().into_bytes()), + Some("application/json"), + ) + .await?; + if response.status() != StatusCode::OK { + let status = response.status(); + let body = response.text().await.unwrap_or_default(); + return Err(format!("set remote target failed: {status} {body}").into()); + } + Ok(serde_json::from_slice(&response.bytes().await?)?) +} + +pub(crate) async fn put_bucket_replication(source: &RustFSTestClusterEnvironment, bucket: &str, target_arn: &str) -> TestResult { + let body = format!( + r#" + + + rule-1 + 1 + Enabled + + Enabled + + + Enabled + + + {target_arn} + + +"# + ); + let url = format!("{}/{bucket}?replication", source.nodes[0].url); + let response = signed_request( + Method::PUT, + &url, + &source.access_key, + &source.secret_key, + Some(body.into_bytes()), + Some("application/xml"), + ) + .await?; + if !response.status().is_success() { + let status = response.status(); + let body = response.text().await.unwrap_or_default(); + return Err(format!("put bucket replication failed: {status} {body}").into()); + } + Ok(()) +} + +pub(crate) async fn wait_for_replicated_bytes( + client: &Client, + bucket: &str, + key: &str, + expected: &[u8], + timeout: Duration, +) -> TestResult { + wait_until( + timeout, + || async { + match get_object_bytes(client, bucket, key).await { + Ok(got) if got.as_slice() == expected => Ok(true), + Ok(_) => Ok(false), + Err(error) => { + let message = error.to_string(); + if message.contains("NoSuchKey") || message.contains("NotFound") { + Ok(false) + } else { + Err(error) + } + } + } + }, + &format!("replicated object {bucket}/{key}"), + ) + .await +} + +pub(crate) async fn set_bucket_quota(cluster: &RustFSTestClusterEnvironment, bucket: &str, quota_bytes: u64) -> TestResult { + wait_until( + Duration::from_secs(30), + || async { + let (status, _) = + cluster_admin(cluster, Method::GET, &format!("/rustfs/admin/v3/quota-stats/{bucket}"), None).await?; + Ok(status.is_success() || status == StatusCode::NOT_FOUND) + }, + "quota stats ready", + ) + .await?; + let body = serde_json::json!({ "quota": quota_bytes, "quota_type": "HARD" }).to_string(); + wait_until( + Duration::from_secs(30), + || async { + let (status, response) = + cluster_admin(cluster, Method::PUT, &format!("/rustfs/admin/v3/quota/{bucket}"), Some(body.clone())).await?; + if status.is_success() { + return Ok(true); + } + if status == StatusCode::SERVICE_UNAVAILABLE { + return Ok(false); + } + Err(format!("failed to set quota for {bucket}: {status} {response}").into()) + }, + "set hard quota", + ) + .await +} + +/// Start decommission and fail closed unless the admin API acknowledges it. +pub(crate) async fn start_decommission(cluster: &RustFSTestClusterEnvironment, pool_id: usize) -> TestResult { + let path = format!("/rustfs/admin/v3/pools/decommission?pool={pool_id}&by-id=true"); + let deadline = Instant::now() + Duration::from_secs(45); + loop { + let (status, response) = cluster_admin(cluster, Method::POST, &path, None).await?; + if status.is_success() { + return Ok(()); + } + if status == StatusCode::INTERNAL_SERVER_ERROR + && response.contains("requires a live fleet capability proof") + && Instant::now() < deadline + { + sleep(Duration::from_secs(1)).await; + continue; + } + return Err(format!("POST {path} did not start decommission: {status} {response}").into()); + } +} + +pub(crate) async fn decommission_status_json(cluster: &RustFSTestClusterEnvironment) -> TestResult { + let body = cluster_admin_ok(cluster, Method::GET, "/rustfs/admin/v3/decommission/status", None).await?; + Ok(serde_json::from_str(&body)?) +} + +fn pool_entry(status: &serde_json::Value, pool_id: usize) -> Option<&serde_json::Value> { + if let Some(pools) = status.get("pools").and_then(serde_json::Value::as_array) { + return pools + .iter() + .find(|pool| pool.get("id").and_then(serde_json::Value::as_u64) == Some(pool_id as u64)); + } + if status.get("id").and_then(serde_json::Value::as_u64) == Some(pool_id as u64) { + Some(status) + } else { + None + } +} + +fn nonzero_u64(value: Option<&serde_json::Value>) -> bool { + value.and_then(serde_json::Value::as_u64).is_some_and(|count| count > 0) +} + +fn decommission_failure(pool: &serde_json::Value) -> Option { + let info = pool.get("decommissionInfo"); + let flagged = |key: &str| info.and_then(|value| value.get(key)).and_then(serde_json::Value::as_bool) == Some(true); + let terminal_flag = flagged("failed") + || flagged("canceled") + || pool + .get("status") + .and_then(serde_json::Value::as_str) + .is_some_and(|status| status.eq_ignore_ascii_case("failed") || status.eq_ignore_ascii_case("canceled")); + let object_failures = nonzero_u64(info.and_then(|value| value.get("objectsDecommissionedFailed"))); + let byte_failures = nonzero_u64(info.and_then(|value| value.get("bytesDecommissionedFailed"))); + let unresolved = info + .and_then(|value| value.get("unresolvedEntries")) + .and_then(serde_json::Value::as_array) + .is_some_and(|entries| !entries.is_empty()); + terminal_flag + .then(|| "decommission reported failed or canceled".to_string()) + .or_else(|| object_failures.then(|| "decommission reported failed objects".to_string())) + .or_else(|| byte_failures.then(|| "decommission reported failed bytes".to_string())) + .or_else(|| unresolved.then(|| "decommission reported unresolved entries".to_string())) +} + +pub(crate) fn decommission_active(status: &serde_json::Value, pool_id: usize) -> TestResult { + let pool = pool_entry(status, pool_id).ok_or_else(|| format!("pool {pool_id} missing from decommission status: {status}"))?; + if let Some(reason) = decommission_failure(pool) { + return Err(format!("{reason}: {pool}").into()); + } + let info = pool + .get("decommissionInfo") + .ok_or_else(|| format!("pool {pool_id} has no decommissionInfo: {pool}"))?; + let status_text = pool.get("status").and_then(serde_json::Value::as_str).unwrap_or(""); + let pool_status = pool.get("poolStatus").and_then(serde_json::Value::as_str).unwrap_or(""); + let queued = info.get("queued").and_then(serde_json::Value::as_bool) == Some(true); + Ok(queued || status_text.eq_ignore_ascii_case("running") || pool_status.eq_ignore_ascii_case("decommissioning")) +} + +pub(crate) fn decommission_running_with_progress(status: &serde_json::Value, pool_id: usize) -> TestResult { + let pool = pool_entry(status, pool_id).ok_or_else(|| format!("pool {pool_id} missing from decommission status: {status}"))?; + if let Some(reason) = decommission_failure(pool) { + return Err(format!("{reason}: {pool}").into()); + } + let info = pool + .get("decommissionInfo") + .ok_or_else(|| format!("pool {pool_id} has no decommissionInfo: {pool}"))?; + let status_text = pool.get("status").and_then(serde_json::Value::as_str).unwrap_or(""); + let pool_status = pool.get("poolStatus").and_then(serde_json::Value::as_str).unwrap_or(""); + let running = status_text.eq_ignore_ascii_case("running") || pool_status.eq_ignore_ascii_case("decommissioning"); + let progressed = nonzero_u64(info.get("objectsDecommissioned")) || nonzero_u64(info.get("bytesDecommissioned")); + Ok(running && progressed) +} + +pub(crate) fn decommission_complete(status: &serde_json::Value, pool_id: usize) -> TestResult { + let pool = pool_entry(status, pool_id).ok_or_else(|| format!("pool {pool_id} missing from decommission status: {status}"))?; + if let Some(reason) = decommission_failure(pool) { + return Err(format!("{reason}: {pool}").into()); + } + let info = pool + .get("decommissionInfo") + .ok_or_else(|| format!("pool {pool_id} has no decommissionInfo: {pool}"))?; + let complete = info.get("complete").and_then(serde_json::Value::as_bool) == Some(true); + let status_text = pool.get("status").and_then(serde_json::Value::as_str).unwrap_or(""); + let pool_status = pool.get("poolStatus").and_then(serde_json::Value::as_str).unwrap_or(""); + let terminal = status_text.eq_ignore_ascii_case("complete") && pool_status.eq_ignore_ascii_case("decommissioned"); + let moved_data = nonzero_u64(info.get("objectsDecommissioned")) && nonzero_u64(info.get("bytesDecommissioned")); + Ok(complete && terminal && moved_data) +} + +pub(crate) async fn wait_for_decommission_active( + cluster: &RustFSTestClusterEnvironment, + pool_id: usize, + timeout: Duration, +) -> TestResult { + wait_for_decommission_state(cluster, pool_id, timeout, "active", decommission_active).await +} + +pub(crate) async fn wait_for_decommission_running_with_progress( + cluster: &RustFSTestClusterEnvironment, + pool_id: usize, + timeout: Duration, +) -> TestResult { + let deadline = Instant::now() + timeout; + loop { + let status = decommission_status_json(cluster).await?; + if decommission_running_with_progress(&status, pool_id)? { + return Ok(()); + } + if Instant::now() >= deadline { + return Err(format!( + "decommission did not become active with non-zero progress within {timeout:?}; last status: {status}" + ) + .into()); + } + sleep(Duration::from_millis(100)).await; + } +} + +pub(crate) async fn wait_for_decommission_complete( + cluster: &RustFSTestClusterEnvironment, + pool_id: usize, + timeout: Duration, +) -> TestResult { + wait_for_decommission_state(cluster, pool_id, timeout, "complete with non-zero progress", decommission_complete).await +} + +async fn wait_for_decommission_state( + cluster: &RustFSTestClusterEnvironment, + pool_id: usize, + timeout: Duration, + expected: &str, + predicate: fn(&serde_json::Value, usize) -> TestResult, +) -> TestResult { + let deadline = Instant::now() + timeout; + loop { + let status = decommission_status_json(cluster).await?; + if predicate(&status, pool_id)? { + return Ok(()); + } + if Instant::now() >= deadline { + return Err(format!("decommission did not become {expected} within {timeout:?}; last status: {status}").into()); + } + sleep(Duration::from_secs(1)).await; + } +} + +pub(crate) async fn start_rebalance(cluster: &RustFSTestClusterEnvironment) -> TestResult { + let path = "/rustfs/admin/v3/rebalance/start"; + let deadline = Instant::now() + Duration::from_secs(45); + let response = loop { + let (status, response) = cluster_admin(cluster, Method::POST, path, None).await?; + if status.is_success() { + break response; + } + if status == StatusCode::INTERNAL_SERVER_ERROR + && response.contains("requires a live fleet capability proof") + && Instant::now() < deadline + { + sleep(Duration::from_secs(1)).await; + continue; + } + return Err(format!("POST {path} did not start rebalance: {status} {response}").into()); + }; + let parsed: serde_json::Value = serde_json::from_str(&response)?; + parsed + .get("id") + .and_then(serde_json::Value::as_str) + .filter(|id| !id.is_empty()) + .map(str::to_string) + .ok_or_else(|| format!("rebalance start response omitted id: {response}").into()) +} + +pub(crate) async fn rebalance_status_json(cluster: &RustFSTestClusterEnvironment) -> TestResult { + let body = cluster_admin_ok(cluster, Method::GET, "/rustfs/admin/v3/rebalance/status", None).await?; + Ok(serde_json::from_str(&body)?) +} + +fn validate_rebalance_status<'a>(status: &'a serde_json::Value, expected_id: &str) -> TestResult<&'a [serde_json::Value]> { + let id = status + .get("id") + .and_then(serde_json::Value::as_str) + .ok_or("rebalance status omitted id")?; + if id != expected_id { + return Err(format!("rebalance status id changed: expected {expected_id}, got {id}").into()); + } + let pools = status + .get("pools") + .and_then(serde_json::Value::as_array) + .filter(|pools| !pools.is_empty()) + .ok_or("rebalance status omitted non-empty pools")?; + for pool in pools { + if pool.get("status").and_then(serde_json::Value::as_str).is_some_and(|status| { + ["failed", "stopped", "canceled", "cancelled"] + .iter() + .any(|terminal| status.eq_ignore_ascii_case(terminal)) + }) { + return Err(format!("rebalance entered an unsuccessful terminal state: {status}").into()); + } + if pool.get("stopping").and_then(serde_json::Value::as_bool) == Some(true) { + return Err(format!("rebalance entered stopping state: {status}").into()); + } + if pool + .get("lastError") + .and_then(serde_json::Value::as_str) + .is_some_and(|error| !error.is_empty()) + { + return Err(format!("rebalance reported lastError: {status}").into()); + } + if nonzero_u64(pool.get("cleanupWarnings").and_then(|warnings| warnings.get("count"))) { + return Err(format!("rebalance reported cleanup warnings: {status}").into()); + } + } + let propagation_failed = status.get("stopPropagation").is_some_and(|propagation| { + propagation + .get("failedPeers") + .and_then(serde_json::Value::as_array) + .is_some_and(|peers| !peers.is_empty()) + || propagation + .get("terminalReloadFailedPeers") + .and_then(serde_json::Value::as_array) + .is_some_and(|peers| !peers.is_empty()) + || propagation.get("pendingTerminalReload").and_then(serde_json::Value::as_bool) == Some(true) + }); + if propagation_failed { + return Err(format!("rebalance stop propagation is incomplete: {status}").into()); + } + Ok(pools) +} + +pub(crate) fn rebalance_active(status: &serde_json::Value, expected_id: &str) -> TestResult { + Ok(validate_rebalance_status(status, expected_id)?.iter().any(|pool| { + pool.get("status") + .and_then(serde_json::Value::as_str) + .is_some_and(|value| value.eq_ignore_ascii_case("started")) + })) +} + +pub(crate) fn rebalance_running_with_progress(status: &serde_json::Value, expected_id: &str) -> TestResult { + Ok(validate_rebalance_status(status, expected_id)?.iter().any(|pool| { + let started = pool + .get("status") + .and_then(serde_json::Value::as_str) + .is_some_and(|value| value.eq_ignore_ascii_case("started")); + let progress = pool.get("progress"); + started + && (nonzero_u64(progress.and_then(|value| value.get("objects"))) + || nonzero_u64(progress.and_then(|value| value.get("versions"))) + || nonzero_u64(progress.and_then(|value| value.get("bytes")))) + })) +} + +pub(crate) fn rebalance_complete(status: &serde_json::Value, expected_id: &str) -> TestResult { + let pools = validate_rebalance_status(status, expected_id)?; + let completed: Vec<&serde_json::Value> = pools + .iter() + .filter(|pool| { + pool.get("status") + .and_then(serde_json::Value::as_str) + .is_some_and(|value| value.eq_ignore_ascii_case("completed")) + }) + .collect(); + let all_terminal = pools.iter().all(|pool| { + pool.get("status") + .and_then(serde_json::Value::as_str) + .is_some_and(|value| value.eq_ignore_ascii_case("completed") || value.eq_ignore_ascii_case("none")) + }); + let any_progress = completed.iter().any(|pool| { + let progress = pool.get("progress"); + nonzero_u64(progress.and_then(|value| value.get("objects"))) + || nonzero_u64(progress.and_then(|value| value.get("versions"))) + || nonzero_u64(progress.and_then(|value| value.get("bytes"))) + }); + Ok(all_terminal && !completed.is_empty() && any_progress) +} + +pub(crate) async fn wait_for_rebalance_active( + cluster: &RustFSTestClusterEnvironment, + expected_id: &str, + timeout: Duration, +) -> TestResult { + let deadline = Instant::now() + timeout; + loop { + let status = rebalance_status_json(cluster).await?; + if rebalance_active(&status, expected_id)? { + return Ok(()); + } + if Instant::now() >= deadline { + return Err(format!("rebalance did not become active within {timeout:?}; last status: {status}").into()); + } + sleep(Duration::from_secs(1)).await; + } +} + +pub(crate) async fn wait_for_rebalance_running_with_progress( + cluster: &RustFSTestClusterEnvironment, + expected_id: &str, + timeout: Duration, +) -> TestResult { + let deadline = Instant::now() + timeout; + loop { + let status = rebalance_status_json(cluster).await?; + if rebalance_running_with_progress(&status, expected_id)? { + return Ok(()); + } + if Instant::now() >= deadline { + return Err(format!( + "rebalance did not become active with non-zero progress within {timeout:?}; last status: {status}" + ) + .into()); + } + sleep(Duration::from_millis(100)).await; + } +} + +pub(crate) async fn wait_for_rebalance_complete( + cluster: &RustFSTestClusterEnvironment, + expected_id: &str, + timeout: Duration, +) -> TestResult { + let deadline = Instant::now() + timeout; + loop { + let status = rebalance_status_json(cluster).await?; + if rebalance_complete(&status, expected_id)? { + return Ok(()); + } + if Instant::now() >= deadline { + return Err( + format!("rebalance did not complete with non-zero progress within {timeout:?}; last status: {status}").into(), + ); + } + sleep(Duration::from_secs(1)).await; + } +} + +pub(crate) async fn list_pools_json(cluster: &RustFSTestClusterEnvironment) -> TestResult { + let body = cluster_admin_ok(cluster, Method::GET, "/rustfs/admin/v3/pools/list", None).await?; + Ok(serde_json::from_str(&body)?) +} + +pub(crate) async fn retrying_put(client: &Client, bucket: &str, key: &str, body: Vec, timeout: Duration) -> TestResult { + wait_until( + timeout, + || { + let client = client.clone(); + let bucket = bucket.to_string(); + let key = key.to_string(); + let body = body.clone(); + async move { + match put_object(&client, &bucket, &key, body).await { + Ok(()) => Ok(true), + Err(error) => { + let message = error.to_string(); + if message.contains("SlowDown") || message.contains("ServiceUnavailable") || message.contains("503") { + Ok(false) + } else { + Err(error) + } + } + } + } + }, + &format!("put {bucket}/{key} during data movement"), + ) + .await +} + +pub(crate) async fn retrying_get_equals( + client: &Client, + bucket: &str, + key: &str, + expected: &[u8], + timeout: Duration, +) -> TestResult { + wait_until( + timeout, + || async { + match get_object_bytes(client, bucket, key).await { + Ok(got) if got.as_slice() == expected => Ok(true), + Ok(_) => Ok(false), + Err(error) => { + let message = error.to_string(); + if message.contains("NoSuchKey") + || message.contains("SlowDown") + || message.contains("ServiceUnavailable") + || message.contains("503") + { + Ok(false) + } else { + Err(error) + } + } + } + }, + &format!("get {bucket}/{key} during data movement"), + ) + .await +} + +#[tokio::test] +async fn append_single_node_pool_extends_ellipses_volumes() { + let mut env = RustFSTestClusterEnvironment::with_topology(ClusterTopology::per_node_pools(2, vec![vec![0], vec![1]])) + .await + .expect("two-pool seed topology"); + assert_eq!(env.rustfs_volumes_arg().split(' ').count(), 2); + + let added = env.append_single_node_pool().await.expect("append third pool"); + assert_eq!(added, 2); + assert_eq!(env.nodes.len(), 3); + assert_eq!(env.nodes[2].pool_idx, 2); + assert_eq!(env.nodes[2].data_dirs.len(), 2); + let volumes = env.rustfs_volumes_arg(); + assert_eq!(volumes.split(' ').count(), 3, "expected three pool arguments, got: {volumes}"); + assert!(volumes.contains("/drive{0...1}"), "expanded layout must keep drive ellipses: {volumes}"); +} + +#[tokio::test] +async fn append_single_node_pool_rejects_striped_single_pool() { + let mut env = RustFSTestClusterEnvironment::new(4).await.expect("four-node single pool"); + let err = env + .append_single_node_pool() + .await + .expect_err("a striped single pool cannot gain a localhost pool"); + let message = err.to_string(); + assert!( + message.contains("drives_per_node") || message.contains("one node per pool"), + "unexpected error: {message}" + ); +} + +#[test] +fn pool_storage_roots_require_exactly_four_unique_paths() { + let temp_root = std::env::temp_dir(); + let too_few = std::env::join_paths([temp_root.as_path()]).expect("join one path"); + let error = validate_pool_storage_roots(&too_few).expect_err("one pool root must be rejected"); + assert!(error.to_string().contains("exactly 4 paths")); + + let duplicates = std::env::join_paths([ + temp_root.as_path(), + temp_root.as_path(), + temp_root.as_path(), + temp_root.as_path(), + ]) + .expect("join duplicate paths"); + let error = validate_pool_storage_roots(&duplicates).expect_err("duplicate pool roots must be rejected"); + assert!(error.to_string().contains("duplicate path")); +} + +#[cfg(unix)] +#[tokio::test] +async fn pool_storage_roots_reject_distinct_paths_on_the_same_device() { + let env = RustFSTestClusterEnvironment::new(NODE_COUNT) + .await + .expect("create same-device pool root fixture"); + let roots: Vec = env.nodes.iter().map(|node| PathBuf::from(&node.data_dir)).collect(); + let joined = std::env::join_paths(&roots).expect("join same-device paths"); + let error = validate_pool_storage_roots(&joined).expect_err("same-device pool roots must be rejected"); + assert!(error.to_string().contains("distinct filesystems"), "unexpected error: {error}"); +} + +#[test] +fn decommission_complete_requires_terminal_status_and_clean_counters() { + let status = serde_json::json!({ + "pools": [ + { + "id": 0, + "status": "complete", + "poolStatus": "decommissioned", + "decommissionInfo": { + "complete": true, + "failed": false, + "canceled": false, + "objectsDecommissioned": 1, + "bytesDecommissioned": 1024 + } + }, + { "id": 1, "status": "none", "poolStatus": "active" } + ] + }); + assert!(decommission_complete(&status, 0).expect("complete fixture should be accepted")); + assert!(decommission_complete(&status, 1).is_err()); + + for missing_counter in ["objectsDecommissioned", "bytesDecommissioned"] { + let mut one_sided = status.clone(); + one_sided["pools"][0]["decommissionInfo"][missing_counter] = serde_json::json!(0); + assert!( + !decommission_complete(&one_sided, 0).expect("one-sided progress fixture should be readable"), + "completion must require both movement counters; zeroed {missing_counter}" + ); + } +} + +#[test] +fn decommission_overlap_requires_running_state_and_progress() { + let mut status = serde_json::json!({ + "pools": [{ + "id": 0, + "status": "queued", + "poolStatus": "active", + "decommissionInfo": { + "queued": true, + "objectsDecommissioned": 1, + "bytesDecommissioned": 1024 + } + }] + }); + assert!( + !decommission_running_with_progress(&status, 0).expect("queued fixture should be readable"), + "queued work is not temporal overlap" + ); + status["pools"][0]["status"] = serde_json::json!("running"); + assert!(decommission_running_with_progress(&status, 0).expect("running fixture should be readable")); + status["pools"][0]["decommissionInfo"]["objectsDecommissioned"] = serde_json::json!(0); + status["pools"][0]["decommissionInfo"]["bytesDecommissioned"] = serde_json::json!(0); + assert!( + !decommission_running_with_progress(&status, 0).expect("zero-progress fixture should be readable"), + "running state alone does not prove movement started" + ); +} + +#[test] +fn rebalance_active_treats_started_as_in_progress() { + let started = serde_json::json!({ "id": "run-1", "pools": [{ "id": 0, "status": "Started", "stopping": false }] }); + let done = serde_json::json!({ "id": "run-1", "pools": [{ "id": 0, "status": "Completed", "stopping": false }] }); + assert!(rebalance_active(&started, "run-1").unwrap()); + assert!(!rebalance_active(&done, "run-1").unwrap()); +} + +#[test] +fn rebalance_overlap_requires_started_state_and_progress() { + let mut status = serde_json::json!({ + "id": "run-1", + "pools": [{ "id": 0, "status": "Started", "stopping": false, "progress": { "objects": 0, "bytes": 0 } }] + }); + assert!( + !rebalance_running_with_progress(&status, "run-1").expect("zero-progress fixture should be readable"), + "started state alone does not prove movement" + ); + status["pools"][0]["progress"]["objects"] = serde_json::json!(1); + assert!(rebalance_running_with_progress(&status, "run-1").expect("progress fixture should be readable")); + status["pools"][0]["status"] = serde_json::json!("Completed"); + assert!( + !rebalance_running_with_progress(&status, "run-1").expect("completed fixture should be readable"), + "completed movement is not temporal overlap" + ); +} + +#[test] +fn rebalance_complete_accepts_non_participating_pools_but_requires_progress() { + let completed = serde_json::json!({ + "id": "run-1", + "pools": [ + { "id": 0, "status": "Completed", "stopping": false, "progress": { "objects": 2, "bytes": 1024 } }, + { "id": 1, "status": "None", "stopping": false, "progress": null }, + { "id": 2, "status": "None", "stopping": false, "progress": null }, + { "id": 3, "status": "None", "stopping": false, "progress": null } + ] + }); + assert!(rebalance_complete(&completed, "run-1").unwrap()); + + let no_movement = serde_json::json!({ + "id": "run-1", + "pools": [ + { "id": 0, "status": "None", "stopping": false, "progress": null }, + { "id": 1, "status": "None", "stopping": false, "progress": null } + ] + }); + assert!(!rebalance_complete(&no_movement, "run-1").unwrap()); +} diff --git a/crates/e2e_test/src/distributed/mod.rs b/crates/e2e_test/src/distributed/mod.rs new file mode 100644 index 000000000..108721f4c --- /dev/null +++ b/crates/e2e_test/src/distributed/mod.rs @@ -0,0 +1,35 @@ +// Copyright 2026 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! 4-node 4-drive distributed e2e coverage. +//! +//! Selected by `[profile.e2e-distributed]` and run from +//! `.github/workflows/e2e-distributed.yml`. Excluded from `e2e-full` because +//! each case starts four real `rustfs` processes. + +mod chaos_test; +mod concurrency_stability_test; +mod concurrent_data_movement_test; +mod data_integrity_movement_test; +mod expand_decommission_rebalance_test; +mod extra_test; +mod harness; +mod object_lock_test; +mod observability_test; +mod replication_quota_test; +mod s3_basic_test; +mod s3_during_data_movement_test; +mod site_replication_test; +mod upgrade_test; +mod versioning_test; diff --git a/crates/e2e_test/src/distributed/object_lock_test.rs b/crates/e2e_test/src/distributed/object_lock_test.rs new file mode 100644 index 000000000..b6681e624 --- /dev/null +++ b/crates/e2e_test/src/distributed/object_lock_test.rs @@ -0,0 +1,219 @@ +// Copyright 2026 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use super::harness::{DistCluster, DistLayout, TestResult, unique_bucket}; +use crate::common::init_logging; +use crate::object_lock::common::{ + delete_object_with_bypass, put_object_lock_configuration, put_object_with_legal_hold, put_object_with_retention, +}; +use aws_sdk_s3::Client; +use aws_sdk_s3::error::ProvideErrorMetadata; +use aws_sdk_s3::error::SdkError; +use aws_sdk_s3::operation::delete_object::DeleteObjectError; +use aws_sdk_s3::primitives::ByteStream; +use aws_sdk_s3::types::{ + DefaultRetention, ObjectLockConfiguration, ObjectLockEnabled, ObjectLockLegalHoldStatus, ObjectLockRetentionMode, + ObjectLockRule, +}; +use chrono::{Duration as ChronoDuration, Utc}; + +fn delete_denied(error: &SdkError, context: &str) -> TestResult { + let code = error.as_service_error().and_then(ProvideErrorMetadata::code); + if code == Some("AccessDenied") { + Ok(()) + } else { + Err(format!("{context}: expected AccessDenied, got {error:?}").into()) + } +} + +async fn expect_versioned_delete_denied( + client: &Client, + bucket: &str, + key: &str, + version_id: &str, + bypass: bool, + context: &str, +) -> TestResult { + match delete_object_with_bypass(client, bucket, key, Some(version_id), bypass).await { + Ok(_) => Err(format!("{context}: DeleteObject of retained version must be denied").into()), + Err(error) => delete_denied(error.as_ref(), context), + } +} + +#[tokio::test] +async fn four_node_four_drive_object_lock_worm_blocks_delete() -> TestResult { + init_logging(); + let dist = DistCluster::start(DistLayout::FourByFour).await?; + let client = dist.client(0)?; + let peer = dist.client(2)?; + let bucket = unique_bucket("objlock"); + + client + .create_bucket() + .bucket(&bucket) + .object_lock_enabled_for_bucket(true) + .send() + .await?; + + let retain_until = Utc::now() + ChronoDuration::days(1); + + let compliance_key = "compliance.bin"; + let compliance_version = put_object_with_retention( + &client, + &bucket, + compliance_key, + b"locked-compliance", + ObjectLockRetentionMode::Compliance, + retain_until, + ) + .await?; + + // Unversioned DELETE is allowed: it only creates a delete marker. WORM + // applies to a specific version id. + let marker = peer.delete_object().bucket(&bucket).key(compliance_key).send().await?; + assert_eq!( + marker.delete_marker(), + Some(true), + "unversioned DELETE on a locked object must create a delete marker" + ); + + expect_versioned_delete_denied(&peer, &bucket, compliance_key, &compliance_version, false, "COMPLIANCE without bypass") + .await?; + expect_versioned_delete_denied(&peer, &bucket, compliance_key, &compliance_version, true, "COMPLIANCE with bypass").await?; + + let governance_key = "governance.bin"; + let governance_version = put_object_with_retention( + &client, + &bucket, + governance_key, + b"locked-governance", + ObjectLockRetentionMode::Governance, + retain_until, + ) + .await?; + + expect_versioned_delete_denied(&peer, &bucket, governance_key, &governance_version, false, "GOVERNANCE without bypass") + .await?; + delete_object_with_bypass(&peer, &bucket, governance_key, Some(&governance_version), true).await?; + let deleted_governance = peer + .head_object() + .bucket(&bucket) + .key(governance_key) + .version_id(&governance_version) + .send() + .await + .expect_err("GOVERNANCE bypass must remove the retained version"); + assert_eq!( + deleted_governance.raw_response().map(|response| response.status().as_u16()), + Some(404), + "deleted GOVERNANCE version returned an unexpected HEAD result: {deleted_governance:?}" + ); + + let hold_key = "legal-hold.bin"; + let hold_version = + put_object_with_legal_hold(&client, &bucket, hold_key, b"legal-hold", ObjectLockLegalHoldStatus::On).await?; + expect_versioned_delete_denied(&peer, &bucket, hold_key, &hold_version, false, "legal hold without bypass").await?; + expect_versioned_delete_denied(&peer, &bucket, hold_key, &hold_version, true, "legal hold with bypass").await?; + + Ok(()) +} + +#[tokio::test] +async fn four_node_default_retention_is_visible_and_non_lock_bucket_rejects_configuration() -> TestResult { + init_logging(); + let dist = DistCluster::start(DistLayout::FourByFour).await?; + let writer = dist.client(0)?; + let reader = dist.client(3)?; + let bucket = unique_bucket("default-lock"); + + writer + .create_bucket() + .bucket(&bucket) + .object_lock_enabled_for_bucket(true) + .send() + .await?; + put_object_lock_configuration(&writer, &bucket, ObjectLockRetentionMode::Governance, Some(1), None).await?; + + let key = "default-governance.bin"; + let put = writer + .put_object() + .bucket(&bucket) + .key(key) + .body(ByteStream::from_static(b"default retention payload")) + .send() + .await?; + let version_id = put.version_id().ok_or("default-retained PUT omitted version ID")?; + + let config = reader.get_object_lock_configuration().bucket(&bucket).send().await?; + let default_retention = config + .object_lock_configuration() + .and_then(|configuration| configuration.rule()) + .and_then(|rule| rule.default_retention()) + .ok_or("GetObjectLockConfiguration omitted default retention")?; + assert_eq!(default_retention.mode().map(|mode| mode.as_str()), Some("GOVERNANCE")); + assert_eq!(default_retention.days(), Some(1)); + + let retention = reader + .get_object_retention() + .bucket(&bucket) + .key(key) + .version_id(version_id) + .send() + .await?; + let retention = retention.retention().ok_or("GetObjectRetention omitted applied retention")?; + assert_eq!(retention.mode().map(|mode| mode.as_str()), Some("GOVERNANCE")); + let retain_until = retention + .retain_until_date() + .ok_or("default retention omitted retain-until date")?; + assert!(retain_until.secs() > Utc::now().timestamp(), "default retention is not in the future"); + + let versioning = reader.get_bucket_versioning().bucket(&bucket).send().await?; + assert_eq!(versioning.status().map(|status| status.as_str()), Some("Enabled")); + expect_versioned_delete_denied(&reader, &bucket, key, version_id, false, "default GOVERNANCE retention without bypass") + .await?; + + let plain_bucket = unique_bucket("no-lock"); + dist.create_bucket(&plain_bucket).await?; + let configuration = ObjectLockConfiguration::builder() + .object_lock_enabled(ObjectLockEnabled::Enabled) + .rule( + ObjectLockRule::builder() + .default_retention( + DefaultRetention::builder() + .mode(ObjectLockRetentionMode::Governance) + .days(1) + .build(), + ) + .build(), + ) + .build(); + let error = writer + .put_object_lock_configuration() + .bucket(&plain_bucket) + .object_lock_configuration(configuration) + .send() + .await + .expect_err("an unversioned bucket must reject Object Lock enablement"); + let service_error = error + .as_service_error() + .ok_or("non-lock bucket rejection was not an S3 service error")?; + assert_eq!(service_error.code(), Some("InvalidBucketState"), "unexpected error: {error:?}"); + assert_eq!( + service_error.message(), + Some("Object Lock configuration cannot be enabled on existing buckets"), + "unexpected error: {error:?}" + ); + + Ok(()) +} diff --git a/crates/e2e_test/src/distributed/observability_test.rs b/crates/e2e_test/src/distributed/observability_test.rs new file mode 100644 index 000000000..bb722b92e --- /dev/null +++ b/crates/e2e_test/src/distributed/observability_test.rs @@ -0,0 +1,236 @@ +// Copyright 2026 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use super::harness::{DistCluster, DistLayout, TestResult, cluster_admin_ok, unique_bucket, wait_for_ready}; +use crate::common::{admin_request, init_logging, local_http_client}; +use aws_sdk_s3::operation::RequestId; +use aws_sdk_s3::primitives::ByteStream; +use bytes::Bytes; +use http::Method; +use http_body_util::{BodyExt, Empty}; +use hyper::body::Incoming; +use hyper::service::service_fn; +use hyper::{Request, Response}; +use hyper_util::rt::TokioIo; +use local_ip_address::local_ip; +use rustfs_madmin::metrics::RealtimeMetrics; +use rustfs_utils::egress::ENV_OUTBOUND_ALLOW_ORIGINS; +use serde_json::Value; +use std::convert::Infallible; +use std::time::Duration; +use tokio::net::TcpListener; +use tokio::sync::mpsc; +use tokio::task::JoinHandle; +use tokio::time::{Instant, timeout}; + +async fn spawn_audit_collector() -> TestResult<(String, mpsc::UnboundedReceiver, JoinHandle<()>)> { + let listener = TcpListener::bind("0.0.0.0:0").await?; + let endpoint = format!("http://{}/audit", std::net::SocketAddr::new(local_ip()?, listener.local_addr()?.port())); + let (tx, rx) = mpsc::unbounded_channel(); + let handle = tokio::spawn(async move { + loop { + let Ok((stream, _)) = listener.accept().await else { + return; + }; + let tx = tx.clone(); + tokio::spawn(async move { + let service = service_fn(move |request: Request| { + let tx = tx.clone(); + async move { + let method = request.method().clone(); + if let Ok(body) = request.into_body().collect().await + && method == Method::POST + && let Ok(payload) = serde_json::from_slice::(&body.to_bytes()) + { + if let Some(records) = payload["Records"].as_array() { + for entry in records { + let _ = tx.send(entry.clone()); + } + } else { + let _ = tx.send(payload); + } + } + Ok::<_, Infallible>(Response::new(Empty::::new())) + } + }); + let _ = hyper::server::conn::http1::Builder::new() + .serve_connection(TokioIo::new(stream), service) + .await; + }); + } + }); + Ok((endpoint, rx, handle)) +} + +async fn wait_for_audit_entry( + rx: &mut mpsc::UnboundedReceiver, + bucket: &str, + key: &str, + request_id: &str, +) -> TestResult { + let deadline = Instant::now() + Duration::from_secs(30); + let mut seen = Vec::new(); + loop { + let remaining = deadline.saturating_duration_since(Instant::now()); + if remaining.is_zero() { + return Err(format!( + "audit webhook did not receive PutObject for {bucket}/{key}; received {} other records: {seen:?}", + seen.len() + ) + .into()); + } + let entry = match timeout(remaining, rx.recv()).await { + Ok(Some(entry)) => entry, + Ok(None) => return Err("audit collector stopped before the expected entry arrived".into()), + Err(_) => { + return Err(format!( + "audit webhook did not receive PutObject for {bucket}/{key}; received {} other records: {seen:?}", + seen.len() + ) + .into()); + } + }; + if entry["api"]["name"].as_str() == Some("s3:PutObject") + && entry["api"]["bucket"].as_str() == Some(bucket) + && entry["api"]["object"].as_str() == Some(key) + && entry["requestID"].as_str() == Some(request_id) + { + return Ok(entry); + } + if seen.len() < 8 { + seen.push(format!( + "api={:?} bucket={:?} object={:?} requestID={:?}", + entry["api"]["name"].as_str(), + entry["api"]["bucket"].as_str(), + entry["api"]["object"].as_str(), + entry["requestID"].as_str() + )); + } + } +} + +#[tokio::test] +async fn four_node_health_inventory_metrics_and_audit_delivery_are_consistent() -> TestResult { + init_logging(); + let (audit_endpoint, mut audit_entries, collector) = spawn_audit_collector().await?; + let audit_origin = reqwest::Url::parse(&audit_endpoint)?.origin().ascii_serialization(); + let audit_env = [ + ("RUSTFS_AUDIT_ENABLE", "true"), + ("RUSTFS_AUDIT_WEBHOOK_ENABLE_DISTRIBUTED", "on"), + ("RUSTFS_AUDIT_WEBHOOK_ENDPOINT_DISTRIBUTED", audit_endpoint.as_str()), + (ENV_OUTBOUND_ALLOW_ORIGINS, audit_origin.as_str()), + ]; + let mut dist = DistCluster::new_stopped_with_env(DistLayout::FourByFour, &audit_env).await?; + for node_idx in 0..dist.cluster.nodes.len() { + let queue_dir = format!("{}/audit-queue-node-{node_idx}", dist.cluster.temp_dir); + tokio::fs::create_dir_all(&queue_dir).await?; + dist.cluster + .set_node_env(node_idx, "RUSTFS_AUDIT_WEBHOOK_QUEUE_DIR_DISTRIBUTED", queue_dir)?; + } + dist.cluster.start().await?; + wait_for_ready(&dist.cluster).await?; + + let http = local_http_client(); + for node in &dist.cluster.nodes { + for probe in ["ready", "live"] { + let response = http.get(format!("{}/health/{probe}", node.url)).send().await?; + assert!( + response.status().is_success(), + "node {} {probe} probe failed: {}", + node.address, + response.status() + ); + } + } + + let info_body = cluster_admin_ok(&dist.cluster, Method::GET, "/rustfs/admin/v3/info", None).await?; + let info: Value = serde_json::from_str(&info_body)?; + let servers = info["info"]["servers"] + .as_array() + .ok_or_else(|| format!("admin info omitted servers: {info}"))?; + assert_eq!(servers.len(), 4, "admin info did not report all four nodes: {info}"); + + let storage_body = cluster_admin_ok(&dist.cluster, Method::GET, "/rustfs/admin/v3/storageinfo", None).await?; + let storage: Value = serde_json::from_str(&storage_body)?; + let disks = storage["info"]["disks"] + .as_array() + .ok_or_else(|| format!("storageinfo omitted disks: {storage}"))?; + assert_eq!(disks.len(), 16, "storageinfo did not report all sixteen drives: {storage}"); + assert!( + disks.iter().all(|disk| { + disk["state"].as_str().is_some_and(|state| state.eq_ignore_ascii_case("ok")) + && disk["runtimeState"] + .as_str() + .is_some_and(|state| state.eq_ignore_ascii_case("online")) + }), + "storageinfo reported a drive that was not healthy and online: {storage}" + ); + + for (node_idx, node) in dist.cluster.nodes.iter().enumerate() { + let (status, metrics_body) = admin_request( + &node.url, + Method::GET, + "/rustfs/admin/v3/metrics?n=1&by-host=true&by-disk=true", + None, + &dist.cluster.access_key, + &dist.cluster.secret_key, + ) + .await?; + assert!(status.is_success(), "node {node_idx} metrics failed: {status} {metrics_body}"); + let sample: RealtimeMetrics = serde_json::from_str( + metrics_body + .lines() + .next() + .ok_or_else(|| format!("node {node_idx} returned empty metrics"))?, + )?; + assert!(sample.finally, "node {node_idx} metrics sample was not terminal"); + assert!(sample.errors.is_empty(), "node {node_idx} metrics reported errors: {:?}", sample.errors); + assert!(!sample.hosts.is_empty(), "node {node_idx} metrics omitted hosts"); + } + + let targets_body = cluster_admin_ok(&dist.cluster, Method::GET, "/rustfs/admin/v3/audit/target/list", None).await?; + let targets: Value = serde_json::from_str(&targets_body)?; + let configured = targets["audit_endpoints"] + .as_array() + .ok_or_else(|| format!("audit target list omitted audit_endpoints: {targets}"))? + .iter() + .any(|target| target["account_id"].as_str() == Some("distributed") && target["service"].as_str() == Some("webhook")); + assert!(configured, "configured audit webhook was missing: {targets}"); + + let bucket = unique_bucket("audit"); + dist.create_bucket(&bucket).await?; + let key = "correlated/audit-object.bin"; + let put = dist + .client(2)? + .put_object() + .bucket(&bucket) + .key(key) + .body(ByteStream::from_static(b"distributed audit payload")) + .send() + .await?; + let request_id = put.request_id().ok_or("PutObject response omitted request ID")?; + let audit = wait_for_audit_entry(&mut audit_entries, &bucket, key, request_id).await?; + assert_eq!( + audit["api"]["status_code"].as_i64(), + Some(200), + "audit entry did not report success: {audit}" + ); + assert!( + !audit.to_string().contains(&dist.cluster.secret_key), + "audit entry leaked the root secret key" + ); + + collector.abort(); + Ok(()) +} diff --git a/crates/e2e_test/src/distributed/replication_quota_test.rs b/crates/e2e_test/src/distributed/replication_quota_test.rs new file mode 100644 index 000000000..ac7ed0d40 --- /dev/null +++ b/crates/e2e_test/src/distributed/replication_quota_test.rs @@ -0,0 +1,191 @@ +// Copyright 2026 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use super::harness::{ + DistCluster, DistLayout, TestResult, enable_versioning, put_bucket_replication, put_object, retrying_put, set_bucket_quota, + set_remote_target, unique_bucket, wait_for_ready, wait_for_replicated_bytes, wait_until, +}; +use crate::common::{FAST_DATA_USAGE_SCANNER_ENV, init_logging}; +use aws_sdk_s3::error::ProvideErrorMetadata; +use aws_sdk_s3::primitives::ByteStream; +use http::Method; +use std::time::Duration; + +async fn wait_for_replication_status( + client: &aws_sdk_s3::Client, + bucket: &str, + key: &str, + expected: &[&str], + timeout: Duration, +) -> TestResult { + wait_until( + timeout, + || async { + let head = client.head_object().bucket(bucket).key(key).send().await?; + Ok(head + .replication_status() + .is_some_and(|status| expected.contains(&status.as_str()))) + }, + &format!("replication status for {bucket}/{key} in {expected:?}"), + ) + .await +} + +#[tokio::test] +async fn four_node_bucket_replication_converges_to_peer_cluster() -> TestResult { + init_logging(); + let (source, mut target) = DistCluster::start_replication_pair().await?; + let source_bucket = unique_bucket("replsrc"); + let target_bucket = unique_bucket("repldst"); + source.create_bucket(&source_bucket).await?; + target.create_bucket(&target_bucket).await?; + + let source_client = source.client(0)?; + let target_client = target.client(0)?; + enable_versioning(&source_client, &source_bucket).await?; + enable_versioning(&target_client, &target_bucket).await?; + + let arn = set_remote_target(&source.cluster, &source_bucket, &target.cluster, &target_bucket).await?; + put_bucket_replication(&source.cluster, &source_bucket, &arn).await?; + + let key = "replicated/metadata-and-tags.bin"; + let body = b"distributed-bucket-replication".to_vec(); + source_client + .put_object() + .bucket(&source_bucket) + .key(key) + .metadata("origin", "four-node-source") + .tagging("suite=distributed&shape=metadata") + .body(ByteStream::from(body.clone())) + .send() + .await?; + wait_for_replicated_bytes(&target_client, &target_bucket, key, &body, Duration::from_secs(45)).await?; + wait_for_replication_status(&source_client, &source_bucket, key, &["COMPLETED"], Duration::from_secs(30)).await?; + + let peer_read = target.client(3)?; + wait_for_replicated_bytes(&peer_read, &target_bucket, key, &body, Duration::from_secs(15)).await?; + let replica_head = peer_read.head_object().bucket(&target_bucket).key(key).send().await?; + assert_eq!( + replica_head + .metadata() + .and_then(|metadata| metadata.get("origin")) + .map(String::as_str), + Some("four-node-source") + ); + assert_eq!(replica_head.replication_status().map(|status| status.as_str()), Some("REPLICA")); + let replica_tags = peer_read.get_object_tagging().bucket(&target_bucket).key(key).send().await?; + let tags: std::collections::BTreeMap<_, _> = replica_tags.tag_set().iter().map(|tag| (tag.key(), tag.value())).collect(); + assert_eq!(tags.get("suite"), Some(&"distributed")); + assert_eq!(tags.get("shape"), Some(&"metadata")); + + target.cluster.stop(); + let outage_key = "replicated/queued-during-target-outage.bin"; + let outage_body = b"retry-after-target-restart".to_vec(); + put_object(&source_client, &source_bucket, outage_key, outage_body.clone()).await?; + wait_for_replication_status( + &source_client, + &source_bucket, + outage_key, + &["PENDING", "FAILED"], + Duration::from_secs(30), + ) + .await?; + + target.cluster.start().await?; + wait_for_ready(&target.cluster).await?; + wait_for_replicated_bytes(&target.client(2)?, &target_bucket, outage_key, &outage_body, Duration::from_secs(90)).await?; + wait_for_replication_status(&source_client, &source_bucket, outage_key, &["COMPLETED"], Duration::from_secs(45)).await?; + Ok(()) +} + +#[tokio::test] +async fn four_node_four_drive_hard_quota_rejects_over_limit_put() -> TestResult { + init_logging(); + let dist = DistCluster::start_with_env(DistLayout::FourByFour, FAST_DATA_USAGE_SCANNER_ENV).await?; + let bucket = unique_bucket("quota"); + dist.create_bucket(&bucket).await?; + set_bucket_quota(&dist.cluster, &bucket, 8 * 1024).await?; + + let client = dist.client(1)?; + retrying_put(&client, &bucket, "small.bin", vec![0u8; 1024], Duration::from_secs(30)).await?; + wait_until( + Duration::from_secs(30), + || async { + let (status, body) = super::harness::cluster_admin( + &dist.cluster, + Method::GET, + &format!("/rustfs/admin/v3/quota-stats/{bucket}"), + None, + ) + .await?; + if !status.is_success() { + return Ok(false); + } + let stats: serde_json::Value = + serde_json::from_str(&body).map_err(|error| format!("quota stats returned invalid JSON: {error}: {body}"))?; + let usage = stats + .get("current_usage") + .and_then(serde_json::Value::as_u64) + .ok_or_else(|| format!("quota stats omitted current_usage: {stats}"))?; + Ok(usage >= 1024) + }, + "quota stats observe small object", + ) + .await?; + + let oversized_key = "too-big.bin"; + let error = client + .put_object() + .bucket(&bucket) + .key(oversized_key) + .body(vec![0u8; 16 * 1024].into()) + .send() + .await + .expect_err("hard quota must reject the oversized PUT"); + let service_error = error + .as_service_error() + .ok_or("quota rejection was not an S3 service error")?; + assert_eq!( + error.raw_response().map(|response| response.status().as_u16()), + Some(400), + "quota rejection must be HTTP 400: {error:?}" + ); + assert_eq!(service_error.code(), Some("InvalidRequest"), "unexpected quota error: {error:?}"); + assert!( + service_error + .message() + .is_some_and(|message| message.starts_with("Bucket quota exceeded")), + "PUT must fail specifically at quota admission: {error:?}" + ); + + let missing = client + .head_object() + .bucket(&bucket) + .key(oversized_key) + .send() + .await + .expect_err("an object rejected by quota must not become visible"); + assert_eq!( + missing.raw_response().map(|response| response.status().as_u16()), + Some(404), + "quota-rejected object returned an unexpected HEAD result: {missing:?}" + ); + + let listed = client.list_objects_v2().bucket(&bucket).send().await?; + assert!( + listed.contents().iter().all(|object| object.key() != Some(oversized_key)), + "quota-rejected key leaked into ListObjectsV2" + ); + Ok(()) +} diff --git a/crates/e2e_test/src/distributed/s3_basic_test.rs b/crates/e2e_test/src/distributed/s3_basic_test.rs new file mode 100644 index 000000000..cd0048f10 --- /dev/null +++ b/crates/e2e_test/src/distributed/s3_basic_test.rs @@ -0,0 +1,258 @@ +// Copyright 2026 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use super::harness::{DistCluster, DistLayout, TestResult, assert_object_bytes, get_object_bytes, put_object, unique_bucket}; +use crate::common::{init_logging, local_http_client}; +use aws_sdk_s3::error::ProvideErrorMetadata; +use aws_sdk_s3::presigning::PresigningConfig; +use aws_sdk_s3::primitives::ByteStream; +use aws_sdk_s3::types::{Delete, MetadataDirective, ObjectIdentifier}; +use std::time::Duration; + +#[tokio::test] +async fn four_node_four_drive_s3_put_get_head_list_copy_rename_delete_and_presign() -> TestResult { + init_logging(); + let dist = DistCluster::start(DistLayout::FourByFour).await?; + let bucket = unique_bucket("s3basic"); + dist.create_bucket(&bucket).await?; + + let writer = dist.client(0)?; + let reader = dist.client(3)?; + let key = "dir/object.bin"; + let body = vec![0xA5u8; 256 * 1024]; + put_object(&writer, &bucket, key, body.clone()).await?; + + let head = reader.head_object().bucket(&bucket).key(key).send().await?; + assert_eq!(head.content_length(), Some(body.len() as i64)); + assert_object_bytes(&reader, &bucket, key, &body).await?; + + let ranged = reader + .get_object() + .bucket(&bucket) + .key(key) + .range("bytes=0-15") + .send() + .await?; + let ranged_body = ranged.body.collect().await?.into_bytes(); + assert_eq!(ranged_body.as_ref(), &body[..16]); + + let listed = reader.list_objects_v2().bucket(&bucket).prefix("dir/").send().await?; + let keys: Vec<_> = listed.contents().iter().filter_map(|object| object.key()).collect(); + assert_eq!(keys, vec![key]); + + let copy_key = "dir/object-copy.bin"; + reader + .copy_object() + .bucket(&bucket) + .key(copy_key) + .copy_source(format!("{bucket}/{key}")) + .metadata_directive(MetadataDirective::Copy) + .send() + .await?; + assert_object_bytes(&writer, &bucket, copy_key, &body).await?; + + let moved_key = "dir/object-moved.bin"; + writer + .copy_object() + .bucket(&bucket) + .key(moved_key) + .copy_source(format!("{bucket}/{copy_key}")) + .send() + .await?; + writer.delete_object().bucket(&bucket).key(copy_key).send().await?; + match writer.head_object().bucket(&bucket).key(copy_key).send().await { + Ok(_) => return Err("copied source still present after rename delete".into()), + Err(error) if error.as_service_error().is_some_and(|err| err.is_not_found()) => {} + Err(error) => return Err(error.into()), + } + assert_object_bytes(&reader, &bucket, moved_key, &body).await?; + + let presigned = writer + .get_object() + .bucket(&bucket) + .key(key) + .presigned(PresigningConfig::expires_in(Duration::from_secs(120))?) + .await?; + let response = local_http_client().get(presigned.uri().to_string()).send().await?; + assert!(response.status().is_success(), "presigned GET failed: {}", response.status()); + let presigned_body = response.bytes().await?; + assert_eq!(presigned_body.as_ref(), body.as_slice()); + + let empty_key = "empty"; + put_object(&writer, &bucket, empty_key, Vec::new()).await?; + let empty = get_object_bytes(&reader, &bucket, empty_key).await?; + assert!(empty.is_empty()); + + let deleted = writer + .delete_objects() + .bucket(&bucket) + .delete( + Delete::builder() + .objects(ObjectIdentifier::builder().key(key).build()?) + .objects(ObjectIdentifier::builder().key(moved_key).build()?) + .objects(ObjectIdentifier::builder().key(empty_key).build()?) + .build()?, + ) + .send() + .await?; + assert!(deleted.errors().is_empty(), "DeleteObjects reported failures: {deleted:?}"); + assert_eq!(deleted.deleted().len(), 3, "DeleteObjects did not acknowledge every key"); + + let remaining = reader.list_objects_v2().bucket(&bucket).send().await?; + assert!(remaining.contents().is_empty(), "bucket still has objects after delete"); + Ok(()) +} + +#[tokio::test] +async fn four_node_s3_metadata_tags_special_keys_pagination_and_multipart_abort() -> TestResult { + init_logging(); + let dist = DistCluster::start(DistLayout::FourByFour).await?; + let bucket = unique_bucket("s3matrix"); + dist.create_bucket(&bucket).await?; + let writer = dist.client(0)?; + let reader = dist.client(3)?; + + let special_key = "unicode/测试 space+percent%25.txt"; + let special_body = b"metadata and tagging survive distributed routing".to_vec(); + let put = writer + .put_object() + .bucket(&bucket) + .key(special_key) + .metadata("test-meta", "distributed") + .tagging("purpose=compatibility&scope=four-by-four") + .body(ByteStream::from(special_body.clone())) + .send() + .await?; + let etag = put.e_tag().ok_or("PutObject omitted ETag")?.to_string(); + + let head = reader.head_object().bucket(&bucket).key(special_key).send().await?; + assert_eq!( + head.metadata() + .and_then(|metadata| metadata.get("test-meta")) + .map(String::as_str), + Some("distributed") + ); + assert_eq!(head.e_tag(), Some(etag.as_str())); + let tags = reader.get_object_tagging().bucket(&bucket).key(special_key).send().await?; + let actual_tags: std::collections::BTreeMap<_, _> = tags + .tag_set() + .iter() + .map(|tag| (tag.key().to_string(), tag.value().to_string())) + .collect(); + assert_eq!(actual_tags.get("purpose").map(String::as_str), Some("compatibility")); + assert_eq!(actual_tags.get("scope").map(String::as_str), Some("four-by-four")); + + let conditional = reader + .get_object() + .bucket(&bucket) + .key(special_key) + .if_match(&etag) + .send() + .await?; + assert_eq!(conditional.body.collect().await?.into_bytes().as_ref(), special_body.as_slice()); + let invalid_range = reader + .get_object() + .bucket(&bucket) + .key(special_key) + .range("bytes=999999-1000000") + .send() + .await + .expect_err("an unsatisfiable range must fail"); + assert_eq!( + invalid_range.as_service_error().and_then(ProvideErrorMetadata::code), + Some("InvalidRange"), + "unexpected invalid-range error: {invalid_range:?}" + ); + + let upload_key = "multipart/aborted.bin"; + let upload = writer + .create_multipart_upload() + .bucket(&bucket) + .key(upload_key) + .send() + .await?; + let upload_id = upload.upload_id().ok_or("CreateMultipartUpload omitted upload ID")?; + writer + .upload_part() + .bucket(&bucket) + .key(upload_key) + .upload_id(upload_id) + .part_number(1) + .body(ByteStream::from(vec![0x5Au8; 5 * 1024 * 1024])) + .send() + .await?; + let pending = reader + .list_multipart_uploads() + .bucket(&bucket) + .prefix("multipart/") + .send() + .await?; + assert!(pending.uploads().iter().any(|entry| entry.upload_id() == Some(upload_id))); + writer + .abort_multipart_upload() + .bucket(&bucket) + .key(upload_key) + .upload_id(upload_id) + .send() + .await?; + let after_abort = reader + .list_multipart_uploads() + .bucket(&bucket) + .prefix("multipart/") + .send() + .await?; + assert!(after_abort.uploads().iter().all(|entry| entry.upload_id() != Some(upload_id))); + let aborted_head = reader + .head_object() + .bucket(&bucket) + .key(upload_key) + .send() + .await + .expect_err("aborted multipart upload must not create an object"); + assert_eq!( + aborted_head.raw_response().map(|response| response.status().as_u16()), + Some(404), + "aborted multipart object returned an unexpected HEAD result: {aborted_head:?}" + ); + + for index in 0..113 { + let key = format!("page/{index:04}.txt"); + put_object(&writer, &bucket, &key, format!("page-{index}").into_bytes()).await?; + } + let mut token = None; + let mut paged_keys = Vec::new(); + loop { + let page = reader + .list_objects_v2() + .bucket(&bucket) + .prefix("page/") + .max_keys(37) + .set_continuation_token(token.take()) + .send() + .await?; + paged_keys.extend(page.contents().iter().filter_map(|object| object.key().map(str::to_string))); + if page.is_truncated() != Some(true) { + break; + } + token = Some( + page.next_continuation_token() + .ok_or("truncated ListObjectsV2 page omitted next continuation token")? + .to_string(), + ); + } + assert_eq!(paged_keys.len(), 113); + let expected: Vec<_> = (0..113).map(|index| format!("page/{index:04}.txt")).collect(); + assert_eq!(paged_keys, expected, "pagination lost, duplicated, or reordered keys"); + Ok(()) +} diff --git a/crates/e2e_test/src/distributed/s3_during_data_movement_test.rs b/crates/e2e_test/src/distributed/s3_during_data_movement_test.rs new file mode 100644 index 000000000..66c835bbd --- /dev/null +++ b/crates/e2e_test/src/distributed/s3_during_data_movement_test.rs @@ -0,0 +1,94 @@ +// Copyright 2026 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use super::harness::{ + DECOMMISSION_POOL_ID, DistCluster, DistLayout, TestResult, assert_inventory, decommission_running_with_progress, + decommission_status_json, put_inventory_retrying, rebalance_running_with_progress, rebalance_status_json, + retrying_get_equals, retrying_put, start_decommission, start_rebalance, unique_bucket, wait_for_decommission_complete, + wait_for_decommission_running_with_progress, wait_for_rebalance_complete, wait_for_rebalance_running_with_progress, +}; +use crate::common::init_logging; +use std::time::Duration; + +#[tokio::test] +async fn s3_put_get_list_succeed_during_decommission_and_rebalance() -> TestResult { + init_logging(); + let mut dist = DistCluster::start(DistLayout::SingleNodeFourDrive).await?; + let bucket = unique_bucket("s3move"); + dist.create_bucket(&bucket).await?; + let client = dist.client(0)?; + let inventory = put_inventory_retrying(&client, &bucket, 96, 256 * 1024, Duration::from_secs(30)).await?; + dist.expand_to_four_pools().await?; + + start_decommission(&dist.cluster, DECOMMISSION_POOL_ID).await?; + wait_for_decommission_running_with_progress(&dist.cluster, DECOMMISSION_POOL_ID, Duration::from_secs(30)).await?; + let live = dist.client(2)?; + retrying_put( + &live, + &bucket, + "during-decommission.bin", + b"written-while-decommissioning".to_vec(), + Duration::from_secs(30), + ) + .await?; + retrying_get_equals( + &live, + &bucket, + "during-decommission.bin", + b"written-while-decommissioning", + Duration::from_secs(30), + ) + .await?; + let listed = live.list_objects_v2().bucket(&bucket).send().await?; + assert!( + listed + .contents() + .iter() + .any(|object| object.key() == Some("during-decommission.bin")), + "list during decommission missed the newly written key" + ); + let status = decommission_status_json(&dist.cluster).await?; + if !decommission_running_with_progress(&status, DECOMMISSION_POOL_ID)? { + return Err(format!("decommission did not remain active across the S3 operations: {status}").into()); + } + + wait_for_decommission_complete(&dist.cluster, DECOMMISSION_POOL_ID, Duration::from_secs(180)).await?; + assert_inventory(&live, &bucket, &inventory).await?; + + let rebalance_id = start_rebalance(&dist.cluster).await?; + wait_for_rebalance_running_with_progress(&dist.cluster, &rebalance_id, Duration::from_secs(30)).await?; + retrying_put( + &live, + &bucket, + "during-rebalance.bin", + b"written-while-rebalancing".to_vec(), + Duration::from_secs(30), + ) + .await?; + retrying_get_equals( + &live, + &bucket, + "during-rebalance.bin", + b"written-while-rebalancing", + Duration::from_secs(30), + ) + .await?; + let status = rebalance_status_json(&dist.cluster).await?; + if !rebalance_running_with_progress(&status, &rebalance_id)? { + return Err(format!("rebalance did not remain active across the S3 operations: {status}").into()); + } + wait_for_rebalance_complete(&dist.cluster, &rebalance_id, Duration::from_secs(180)).await?; + assert_inventory(&dist.client(1)?, &bucket, &inventory).await?; + Ok(()) +} diff --git a/crates/e2e_test/src/distributed/site_replication_test.rs b/crates/e2e_test/src/distributed/site_replication_test.rs new file mode 100644 index 000000000..1a5b33e42 --- /dev/null +++ b/crates/e2e_test/src/distributed/site_replication_test.rs @@ -0,0 +1,128 @@ +// Copyright 2026 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use super::harness::{ + DistCluster, TestResult, cluster_admin_ok, enable_versioning, put_object, unique_bucket, wait_for_replicated_bytes, + wait_until, +}; +use crate::common::{init_logging, signed_request}; +use http::{Method, StatusCode}; +use rustfs_madmin::{PeerSite, ReplicateAddStatus, SiteReplicationInfo, SyncStatus}; +use std::time::Duration; + +async fn site_replication_add( + cluster: &crate::common::RustFSTestClusterEnvironment, + sites: &[PeerSite], +) -> TestResult { + let url = format!("{}/rustfs/admin/v3/site-replication/add?replicateILMExpiry=false", cluster.nodes[0].url); + let response = signed_request( + Method::PUT, + &url, + &cluster.access_key, + &cluster.secret_key, + Some(serde_json::to_vec(sites)?), + Some("application/json"), + ) + .await?; + if response.status() != StatusCode::OK { + let status = response.status(); + let body = response.text().await.unwrap_or_default(); + return Err(format!("site replication add failed: {status} {body}").into()); + } + Ok(serde_json::from_slice(&response.bytes().await?)?) +} + +async fn site_replication_info(cluster: &crate::common::RustFSTestClusterEnvironment) -> TestResult { + let body = cluster_admin_ok(cluster, Method::GET, "/rustfs/admin/v3/site-replication/info", None).await?; + Ok(serde_json::from_str(&body)?) +} + +async fn wait_for_site_replication_enabled(cluster: &crate::common::RustFSTestClusterEnvironment) -> TestResult { + wait_until( + Duration::from_secs(30), + || async { + let info = site_replication_info(cluster).await?; + Ok(info.enabled && info.sites.len() == 2 && info.sites.iter().all(|site| site.sync_state == SyncStatus::Enable)) + }, + "site replication enabled with two synchronized sites", + ) + .await +} + +#[tokio::test] +async fn four_node_site_replication_replicates_object_to_peer_site() -> TestResult { + init_logging(); + let (site_a, site_b) = DistCluster::start_replication_pair().await?; + let bucket = unique_bucket("siterepl"); + site_a.create_bucket(&bucket).await?; + site_b.create_bucket(&bucket).await?; + + let client_a = site_a.client(0)?; + let client_b = site_b.client(0)?; + enable_versioning(&client_a, &bucket).await?; + enable_versioning(&client_b, &bucket).await?; + + let sites = vec![ + PeerSite { + name: "site-a".to_string(), + endpoint: site_a.cluster.nodes[0].url.clone(), + access_key: site_a.cluster.access_key.clone(), + secret_key: site_a.cluster.secret_key.clone(), + ..Default::default() + }, + PeerSite { + name: "site-b".to_string(), + endpoint: site_b.cluster.nodes[0].url.clone(), + access_key: site_b.cluster.access_key.clone(), + secret_key: site_b.cluster.secret_key.clone(), + ..Default::default() + }, + ]; + let add_status = site_replication_add(&site_a.cluster, &sites).await?; + assert!( + add_status.success && add_status.err_detail.is_empty() && add_status.initial_sync_error_message.is_empty(), + "site replication add reported failure: {add_status:?}" + ); + wait_for_site_replication_enabled(&site_a.cluster).await?; + wait_for_site_replication_enabled(&site_b.cluster).await?; + + let info_a = site_replication_info(&site_a.cluster).await?; + let remote = info_a + .sites + .iter() + .find(|site| site.name == "site-b") + .ok_or_else(|| format!("site A info omitted the configured site-b peer: {info_a:?}"))?; + assert_eq!(remote.endpoint, site_b.cluster.nodes[0].url); + let deployment_ids: std::collections::BTreeSet<_> = info_a.sites.iter().map(|site| site.deployment_id.as_str()).collect(); + assert!( + deployment_ids.iter().all(|deployment_id| !deployment_id.is_empty()) && deployment_ids.len() == 2, + "site peers must have two distinct non-empty deployment IDs: {info_a:?}" + ); + assert!(info_a.retry_stats.is_none(), "site A has pending replication retries: {info_a:?}"); + assert!(info_a.pending_operation.is_none(), "site A has a pending operation: {info_a:?}"); + + let key = "site-object.bin"; + let body = b"four-node-site-replication".to_vec(); + put_object(&client_a, &bucket, key, body.clone()).await?; + wait_for_replicated_bytes(&client_b, &bucket, key, &body, Duration::from_secs(60)).await?; + + let peer_b = site_b.client(3)?; + wait_for_replicated_bytes(&peer_b, &bucket, key, &body, Duration::from_secs(20)).await?; + + let reverse_key = "reverse/site-object.bin"; + let reverse_body = b"site-b-to-site-a".to_vec(); + put_object(&site_b.client(2)?, &bucket, reverse_key, reverse_body.clone()).await?; + wait_for_replicated_bytes(&site_a.client(3)?, &bucket, reverse_key, &reverse_body, Duration::from_secs(60)).await?; + Ok(()) +} diff --git a/crates/e2e_test/src/distributed/upgrade_test.rs b/crates/e2e_test/src/distributed/upgrade_test.rs new file mode 100644 index 000000000..33d3c1c7a --- /dev/null +++ b/crates/e2e_test/src/distributed/upgrade_test.rs @@ -0,0 +1,345 @@ +// Copyright 2026 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! 4-node upgrade coverage for historical objects and IAM AK/SK. +//! +//! Complements `upgrade_compatibility_test` (single-node SSE/multipart and +//! mixed-version listing). This module pins the distributed contract the +//! hardware upgrade chain is meant to catch: after a 4-node upgrade, objects +//! written on the previous release still read back, and IAM user credentials +//! created before the upgrade still authenticate. +//! +//! Requires `RUSTFS_UPGRADE_SOURCE_BINARY` pointing at the pinned previous +//! release. The `e2e-distributed` workflow downloads that binary; a local run +//! without it fails closed rather than skipping. + +use super::harness::{ + DistCluster, DistLayout, TestResult, assert_object_bytes, cluster_admin_ok, enable_versioning, get_object_bytes, put_object, + unique_bucket, wait_until, +}; +use crate::common::{ + AdminTransport, admin_add_canned_policy_via, admin_attach_user_policy_via, admin_create_user_via, init_logging, +}; +use aws_sdk_s3::Client; +use aws_sdk_s3::error::ProvideErrorMetadata; +use std::path::{Path, PathBuf}; +use std::time::Duration; +use uuid::Uuid; + +const SOURCE_BINARY_ENV: &str = "RUSTFS_UPGRADE_SOURCE_BINARY"; +const IAM_SECRET: &str = "UpgradeTestSecretKey1"; +const WRONG_SECRET: &str = "WrongSecretKey000000"; +const CREDENTIAL_TIMEOUT: Duration = Duration::from_secs(30); + +struct UpgradeSeed { + history_bucket: String, + history_key: &'static str, + history_body: Vec, + versioned_bucket: String, + versioned_key: &'static str, + version1: String, + version1_body: Vec, + version2: String, + version2_body: Vec, + iam_bucket: String, + iam_key: &'static str, + iam_body: Vec, + iam_user: String, + iam_secret: &'static str, +} + +fn source_binary() -> TestResult { + let path = std::env::var_os(SOURCE_BINARY_ENV).map(PathBuf::from).ok_or_else(|| { + format!( + "{SOURCE_BINARY_ENV} must point to the pinned previous release binary (the e2e-distributed workflow downloads it)" + ) + })?; + if !path.is_file() { + return Err(format!("upgrade source binary does not exist: {}", path.display()).into()); + } + Ok(path) +} + +fn capture_upgrade_logs(cluster: &mut DistCluster, label: &str) -> TestResult { + let Some(log_dir) = std::env::var_os("RUSTFS_E2E_LOG_DIR") else { + return Ok(()); + }; + std::fs::create_dir_all(&log_dir)?; + for node_idx in 0..cluster.cluster.nodes.len() { + let path = Path::new(&log_dir).join(format!("{label}-node-{node_idx}.log")); + cluster + .cluster + .set_node_capture_log_path(node_idx, path.to_string_lossy().into_owned())?; + } + Ok(()) +} + +fn iam_rw_policy(bucket: &str) -> String { + serde_json::json!({ + "Version": "2012-10-17", + "Statement": [{ + "Effect": "Allow", + "Action": ["s3:*"], + "Resource": [ + format!("arn:aws:s3:::{bucket}"), + format!("arn:aws:s3:::{bucket}/*") + ] + }] + }) + .to_string() +} + +async fn create_iam_user(dist: &DistCluster, user: &str, secret: &str, policy_name: &str, bucket: &str) -> TestResult { + let url = &dist.cluster.nodes[0].url; + let access = &dist.cluster.access_key; + let admin_secret = &dist.cluster.secret_key; + admin_create_user_via(AdminTransport::Signed, url, access, admin_secret, user, secret).await?; + admin_add_canned_policy_via(AdminTransport::Signed, url, access, admin_secret, policy_name, &iam_rw_policy(bucket)).await?; + admin_attach_user_policy_via(AdminTransport::Signed, url, access, admin_secret, policy_name, user).await?; + Ok(()) +} + +async fn wait_for_put(client: &Client, bucket: &str, key: &str, body: Vec, label: &str) -> TestResult { + wait_until( + CREDENTIAL_TIMEOUT, + || { + let client = client.clone(); + let bucket = bucket.to_string(); + let key = key.to_string(); + let body = body.clone(); + async move { + put_object(&client, &bucket, &key, body).await?; + Ok(true) + } + }, + label, + ) + .await +} + +async fn wait_for_bytes(client: &Client, bucket: &str, key: &str, expected: &[u8], label: &str) -> TestResult { + wait_until( + CREDENTIAL_TIMEOUT, + || { + let client = client.clone(); + let bucket = bucket.to_string(); + let key = key.to_string(); + let expected = expected.to_vec(); + async move { + let got = get_object_bytes(&client, &bucket, &key).await?; + Ok(got == expected) + } + }, + label, + ) + .await +} + +async fn seed_history_and_iam(dist: &DistCluster) -> TestResult { + let history_bucket = unique_bucket("upg-hist"); + let versioned_bucket = unique_bucket("upg-ver"); + let iam_bucket = unique_bucket("upg-iam"); + dist.create_bucket(&history_bucket).await?; + dist.create_bucket(&versioned_bucket).await?; + dist.create_bucket(&iam_bucket).await?; + + let root = dist.client(0)?; + enable_versioning(&root, &versioned_bucket).await?; + + let history_key = "plain-history.bin"; + let history_body = b"written by the previous 4-node release".to_vec(); + put_object(&root, &history_bucket, history_key, history_body.clone()).await?; + + let versioned_key = "versioned-history.txt"; + let version1_body = b"version-one-before-upgrade".to_vec(); + let version1 = root + .put_object() + .bucket(&versioned_bucket) + .key(versioned_key) + .body(aws_sdk_s3::primitives::ByteStream::from(version1_body.clone())) + .send() + .await? + .version_id() + .ok_or("first versioned PUT omitted version ID")? + .to_string(); + let version2_body = b"version-two-before-upgrade".to_vec(); + let version2 = root + .put_object() + .bucket(&versioned_bucket) + .key(versioned_key) + .body(aws_sdk_s3::primitives::ByteStream::from(version2_body.clone())) + .send() + .await? + .version_id() + .ok_or("second versioned PUT omitted version ID")? + .to_string(); + + let iam_user = format!("upg{}", &Uuid::new_v4().simple().to_string()[..8]); + let policy_name = format!("upgpol{}", &Uuid::new_v4().simple().to_string()[..8]); + create_iam_user(dist, &iam_user, IAM_SECRET, &policy_name, &iam_bucket).await?; + + let iam_key = "iam-history.bin"; + let iam_body = b"written with pre-upgrade IAM AK/SK".to_vec(); + let iam_client = dist.client_with_credentials(1, &iam_user, IAM_SECRET)?; + wait_for_put(&iam_client, &iam_bucket, iam_key, iam_body.clone(), "IAM user PUT before upgrade").await?; + + Ok(UpgradeSeed { + history_bucket, + history_key, + history_body, + versioned_bucket, + versioned_key, + version1, + version1_body, + version2, + version2_body, + iam_bucket, + iam_key, + iam_body, + iam_user, + iam_secret: IAM_SECRET, + }) +} + +async fn assert_history_and_iam(dist: &DistCluster, seed: &UpgradeSeed, context: &str) -> TestResult { + let root_a = dist.client(0)?; + let root_b = dist.client(3)?; + wait_for_bytes( + &root_b, + &seed.history_bucket, + seed.history_key, + &seed.history_body, + &format!("{context}: root GET historical object"), + ) + .await?; + assert_object_bytes(&root_a, &seed.history_bucket, seed.history_key, &seed.history_body).await?; + + let v1 = root_b + .get_object() + .bucket(&seed.versioned_bucket) + .key(seed.versioned_key) + .version_id(&seed.version1) + .send() + .await?; + let v1_body = v1.body.collect().await?.into_bytes(); + if v1_body.as_ref() != seed.version1_body.as_slice() { + return Err(format!("{context}: version 1 bytes changed after upgrade").into()); + } + let v2 = root_a + .get_object() + .bucket(&seed.versioned_bucket) + .key(seed.versioned_key) + .version_id(&seed.version2) + .send() + .await?; + let v2_body = v2.body.collect().await?.into_bytes(); + if v2_body.as_ref() != seed.version2_body.as_slice() { + return Err(format!("{context}: version 2 bytes changed after upgrade").into()); + } + + let users = cluster_admin_ok(&dist.cluster, http::Method::GET, "/rustfs/admin/v3/list-users", None).await?; + if !users.contains(&seed.iam_user) { + return Err(format!("{context}: list-users lost IAM user {}: {users}", seed.iam_user).into()); + } + + let iam_on_upgraded = dist.client_with_credentials(0, &seed.iam_user, seed.iam_secret)?; + let iam_on_peer = dist.client_with_credentials(3, &seed.iam_user, seed.iam_secret)?; + wait_for_bytes( + &iam_on_upgraded, + &seed.iam_bucket, + seed.iam_key, + &seed.iam_body, + &format!("{context}: IAM GET historical object on node 0"), + ) + .await?; + wait_for_bytes( + &iam_on_peer, + &seed.iam_bucket, + seed.iam_key, + &seed.iam_body, + &format!("{context}: IAM GET historical object on node 3"), + ) + .await?; + + let post_key = format!("after-upgrade-{context}.txt"); + let post_body = format!("{context}: written with the same IAM AK/SK after upgrade").into_bytes(); + wait_for_put( + &iam_on_peer, + &seed.iam_bucket, + &post_key, + post_body.clone(), + &format!("{context}: IAM PUT after upgrade"), + ) + .await?; + assert_object_bytes(&iam_on_upgraded, &seed.iam_bucket, &post_key, &post_body).await?; + + let bad = dist.client_with_credentials(1, &seed.iam_user, WRONG_SECRET)?; + match bad.get_object().bucket(&seed.iam_bucket).key(seed.iam_key).send().await { + Ok(_) => return Err(format!("{context}: wrong secret must not read the IAM object").into()), + Err(error) => { + let code = error.as_service_error().and_then(ProvideErrorMetadata::code); + let rejected = code == Some("SignatureDoesNotMatch") + || code == Some("InvalidAccessKeyId") + || code == Some("AccessDenied") + || code == Some("InvalidArgument") + || error.raw_response().is_some_and(|response| response.status().as_u16() == 403); + if !rejected { + return Err(format!("{context}: wrong secret failed with unexpected error {error:?}").into()); + } + } + } + + let post_root_key = format!("root-after-{context}.bin"); + let post_root_body = format!("{context}: root write after upgrade").into_bytes(); + put_object(&root_a, &seed.history_bucket, &post_root_key, post_root_body.clone()).await?; + assert_object_bytes(&root_b, &seed.history_bucket, &post_root_key, &post_root_body).await?; + Ok(()) +} + +#[tokio::test] +async fn four_node_direct_upgrade_preserves_history_and_iam_credentials() -> TestResult { + init_logging(); + let previous = source_binary()?; + let mut dist = DistCluster::new_stopped(DistLayout::FourNodeFourDisk).await?; + capture_upgrade_logs(&mut dist, "direct-upgrade")?; + dist.start_from_binary(&previous).await?; + + let seed = seed_history_and_iam(&dist).await?; + dist.restart_with_current_binary().await?; + assert_history_and_iam(&dist, &seed, "direct").await?; + Ok(()) +} + +#[tokio::test] +async fn four_node_rolling_upgrade_preserves_history_and_iam_credentials() -> TestResult { + init_logging(); + let previous = source_binary()?; + let mut dist = DistCluster::new_stopped(DistLayout::FourNodeFourDisk).await?; + capture_upgrade_logs(&mut dist, "rolling-upgrade")?; + dist.start_from_binary(&previous).await?; + + let seed = seed_history_and_iam(&dist).await?; + + dist.replace_node_with_current_binary(0).await?; + assert_history_and_iam(&dist, &seed, "one-current-node").await?; + + for node_idx in [1, 2] { + dist.replace_node_with_current_binary(node_idx).await?; + } + assert_history_and_iam(&dist, &seed, "one-previous-node").await?; + + dist.replace_node_with_current_binary(3).await?; + assert_history_and_iam(&dist, &seed, "homogeneous-current").await?; + Ok(()) +} diff --git a/crates/e2e_test/src/distributed/versioning_test.rs b/crates/e2e_test/src/distributed/versioning_test.rs new file mode 100644 index 000000000..0a86fe145 --- /dev/null +++ b/crates/e2e_test/src/distributed/versioning_test.rs @@ -0,0 +1,188 @@ +// Copyright 2026 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use super::harness::{DistCluster, DistLayout, TestResult, enable_versioning, get_object_bytes, put_object, unique_bucket}; +use crate::common::init_logging; +use aws_sdk_s3::error::ProvideErrorMetadata; +use aws_sdk_s3::types::{BucketVersioningStatus, VersioningConfiguration}; + +#[tokio::test] +async fn four_node_four_drive_versioning_put_list_get_delete_marker() -> TestResult { + init_logging(); + let dist = DistCluster::start(DistLayout::FourByFour).await?; + let bucket = unique_bucket("version"); + dist.create_bucket(&bucket).await?; + let writer = dist.client(0)?; + let reader = dist.client(3)?; + enable_versioning(&writer, &bucket).await?; + + let key = "versioned.txt"; + let v1_id = writer + .put_object() + .bucket(&bucket) + .key(key) + .body(b"v1".to_vec().into()) + .send() + .await? + .version_id() + .ok_or("v1 PUT omitted version ID")? + .to_string(); + let v2_id = writer + .put_object() + .bucket(&bucket) + .key(key) + .body(b"v2".to_vec().into()) + .send() + .await? + .version_id() + .ok_or("v2 PUT omitted version ID")? + .to_string(); + + let versions = reader.list_object_versions().bucket(&bucket).prefix(key).send().await?; + let matching_versions: Vec<_> = versions + .versions() + .iter() + .filter(|version| version.key() == Some(key)) + .collect(); + assert_eq!(matching_versions.len(), 2, "fresh key must have exactly two versions: {versions:?}"); + assert!(versions.delete_markers().is_empty(), "fresh key unexpectedly has a delete marker"); + assert!( + matching_versions + .iter() + .any(|version| version.version_id() == Some(v1_id.as_str()) && version.is_latest() != Some(true)), + "v1 was not the historical version: {versions:?}" + ); + assert!( + matching_versions + .iter() + .any(|version| version.version_id() == Some(v2_id.as_str()) && version.is_latest() == Some(true)), + "v2 was not the latest version: {versions:?}" + ); + + let latest = get_object_bytes(&reader, &bucket, key).await?; + assert_eq!(latest, b"v2"); + + let older = reader.get_object().bucket(&bucket).key(key).version_id(&v1_id).send().await?; + let older_body = older.body.collect().await?.into_bytes(); + assert_eq!(older_body.as_ref(), b"v1"); + + let deleted = writer.delete_object().bucket(&bucket).key(key).send().await?; + assert_eq!(deleted.delete_marker(), Some(true)); + let marker_id = deleted.version_id().ok_or("DeleteObject omitted delete-marker version ID")?; + let after_delete = reader.list_object_versions().bucket(&bucket).prefix(key).send().await?; + let matching_markers: Vec<_> = after_delete + .delete_markers() + .iter() + .filter(|marker| marker.key() == Some(key)) + .collect(); + assert_eq!( + matching_markers.len(), + 1, + "delete marker missing or duplicated after current-version delete: {after_delete:?}" + ); + assert!( + matching_markers[0].version_id() == Some(marker_id) && matching_markers[0].is_latest() == Some(true), + "DeleteObject response and ListObjectVersions disagree about the marker: {after_delete:?}" + ); + + let latest_after_delete = reader.get_object().bucket(&bucket).key(key).send().await; + match latest_after_delete { + Ok(_) => return Err("current version should be a delete marker".into()), + Err(error) + if error + .as_service_error() + .and_then(ProvideErrorMetadata::code) + .is_some_and(|code| code == "NoSuchKey" || code == "NotFound") => {} + Err(error) => return Err(error.into()), + } + + let restored = reader.get_object().bucket(&bucket).key(key).version_id(&v1_id).send().await?; + let restored_body = restored.body.collect().await?.into_bytes(); + assert_eq!(restored_body.as_ref(), b"v1"); + + writer + .delete_object() + .bucket(&bucket) + .key(key) + .version_id(marker_id) + .send() + .await?; + assert_eq!(get_object_bytes(&reader, &bucket, key).await?, b"v2"); + Ok(()) +} + +#[tokio::test] +async fn four_node_versioning_suspension_keeps_one_null_version_and_history() -> TestResult { + init_logging(); + let dist = DistCluster::start(DistLayout::FourByFour).await?; + let bucket = unique_bucket("suspend"); + dist.create_bucket(&bucket).await?; + let writer = dist.client(0)?; + let reader = dist.client(3)?; + enable_versioning(&writer, &bucket).await?; + + let key = "suspended.txt"; + let original = writer + .put_object() + .bucket(&bucket) + .key(key) + .body(b"enabled-history".to_vec().into()) + .send() + .await? + .version_id() + .ok_or("enabled PUT omitted version ID")? + .to_string(); + writer + .put_bucket_versioning() + .bucket(&bucket) + .versioning_configuration( + VersioningConfiguration::builder() + .status(BucketVersioningStatus::Suspended) + .build(), + ) + .send() + .await?; + + put_object(&writer, &bucket, key, b"null-one".to_vec()).await?; + put_object(&writer, &bucket, key, b"null-two".to_vec()).await?; + assert_eq!(get_object_bytes(&reader, &bucket, key).await?, b"null-two"); + + let versions = reader.list_object_versions().bucket(&bucket).prefix(key).send().await?; + let matching: Vec<_> = versions + .versions() + .iter() + .filter(|version| version.key() == Some(key)) + .collect(); + assert!(matching.iter().any(|version| version.version_id() == Some(original.as_str()))); + let null_version_count = matching + .iter() + .filter(|version| { + matches!( + version.version_id(), + None | Some("") | Some("null") | Some("00000000-0000-0000-0000-000000000000") + ) + }) + .count(); + assert_eq!(null_version_count, 1, "suspended overwrites must keep one null version: {versions:?}"); + + let historical = reader + .get_object() + .bucket(&bucket) + .key(key) + .version_id(&original) + .send() + .await?; + assert_eq!(historical.body.collect().await?.into_bytes().as_ref(), b"enabled-history"); + Ok(()) +} diff --git a/crates/e2e_test/src/lib.rs b/crates/e2e_test/src/lib.rs index 9279c39c4..72f5121fb 100644 --- a/crates/e2e_test/src/lib.rs +++ b/crates/e2e_test/src/lib.rs @@ -378,6 +378,11 @@ mod bucket_stats_regression_test; #[cfg(test)] mod distributed_startup_regression_test; +// 4-node / 4-disk distributed Actions suite (S3, lock, versioning, replication, +// quota, observability, expand/decommission/rebalance, site replication, chaos). +#[cfg(test)] +mod distributed; + // P1 regression: tier/ILM transition (rustfs#5218, #5130, #5011, #4826, #5024) #[cfg(test)] mod tier_transition_regression_test; diff --git a/docs/testing/README.md b/docs/testing/README.md index 60343b9e2..55dc6a4e0 100644 --- a/docs/testing/README.md +++ b/docs/testing/README.md @@ -12,10 +12,10 @@ Pick the lowest layer that can prove the change; add a higher-layer test only wh |---|---|---|---| | Unit & crate integration | Per-crate logic and in-process integration tests | `cargo nextest run --all --exclude e2e_test` (or `-p `); `make test` wraps it | Every PR, required (`Test and Lint`, `ci` profile) | | ecstore black-box | Erasure-coded read/write/recovery validation; profiles `quick` / `full` / `destructive` / `fuzz` | `scripts/run_ecstore_validation_suite.sh --profile quick` | Local and release validation only; not wired into any workflow. Contract: [ecstore-validation-suite-design.md](ecstore-validation-suite-design.md) | -| e2e (`e2e_test` crate) | A real `rustfs` binary per test, driven over the S3, admin, and protocol APIs | `cargo nextest run --profile e2e-smoke -p e2e_test` | PR: `e2e-smoke` (report-only); merge queue / main push: `e2e-full`; nightly: `e2e-repl-nightly`, `e2e-nightly`, `e2e-protocols`. Guide: [`crates/e2e_test/README.md`](../../crates/e2e_test/README.md) | +| e2e (`e2e_test` crate) | A real `rustfs` binary per test, driven over the S3, admin, and protocol APIs | `cargo nextest run --profile e2e-smoke -p e2e_test` | PR: `e2e-smoke` (report-only); merge queue / main push: `e2e-full`; nightly: `e2e-repl-nightly`, `e2e-nightly`, `e2e-protocols`, `e2e-distributed`. Guide: [`crates/e2e_test/README.md`](../../crates/e2e_test/README.md); 4-node 4-disk map: [distributed-e2e.md](distributed-e2e.md) | | s3s-e2e conformance | External S3 conformance tool against a live server | `./scripts/e2e-run.sh ./target/debug/rustfs ` | PR, report-only (second half of the `End-to-End Tests` job) | | S3 compatibility | `ceph/s3-tests` (boto3; allow-list `scripts/s3-tests/implemented_tests.txt`) and MinIO `mint` | `scripts/s3-tests/run.sh`; mint via `.github/workflows/mint.yml` | s3-tests: PR report-only plus a weekly full sweep; mint: weekly, report-only | -| Chaos / fault-injection | Single-node disk fault injection (`crates/e2e_test/src/chaos.rs`, `crates/e2e_test/src/fault_proxy.rs`) used by the reliability and heal e2e modules | Part of the e2e crate (`e2e-reliability` test-group) | With the `e2e-full` and nightly e2e lanes. A multi-node power-loss harness is not in tree | +| Chaos / fault-injection | Single-node disk fault injection (`crates/e2e_test/src/chaos.rs`, `crates/e2e_test/src/fault_proxy.rs`) plus the 4-node kill/fresh-drive/blackhole cases in `crates/e2e_test/src/distributed/chaos_test.rs` | Part of the e2e crate (`e2e-reliability` and `e2e-distributed`) | Reliability cases with `e2e-full`; 4-node chaos on storage-sensitive PRs and nightly via `e2e-distributed` | | Fuzz | `cargo-fuzz` targets over untrusted parsing surfaces; isolated sub-workspace under `fuzz/` | `./scripts/fuzz/run.sh` (see [`fuzz/README.md`](../../fuzz/README.md)) | PR smoke on the paths listed in `.github/workflows/fuzz.yml`, plus nightly corpus | | Benchmarks | Criterion benches under each crate's `benches/` | `cargo bench -p ` | On demand; never a gate | @@ -63,6 +63,7 @@ All profiles are defined in `.config/nextest.toml`; its block comments hold the | `e2e-full` | Merge-queue / main-push single-node e2e lane | | `e2e-repl-nightly` | Nightly slow / cross-process replication lane | | `e2e-nightly` | Nightly serial multi-process cluster fault lane | +| `e2e-distributed` | Storage-sensitive PR and nightly 4-node 4-disk S3 / lock / versioning / replication / quota / expand / decommission / rebalance / site-replication / chaos / upgrade (history + IAM AK/SK) lane | | `e2e-protocols` | Nightly fixed-port FTPS/SFTP/WebDAV lane, run with `-j 1` | Membership of each e2e profile is pinned by a digest in `.config/e2e--selection.txt` and checked by `scripts/check_test_wiring.py --check-profile ` before the lane runs. To list what a profile selects on your platform (the result is platform-dependent because some modules are linux-only): diff --git a/docs/testing/ci-gates.md b/docs/testing/ci-gates.md index 81418c911..1779fe458 100644 --- a/docs/testing/ci-gates.md +++ b/docs/testing/ci-gates.md @@ -74,6 +74,7 @@ Scheduled lanes never block a PR. Their workflow-local gate fails the run, sched | `ci.yml` (weekly) | full matrix, including the schedule/dispatch-only rio-v2 jobs `build-rustfs-debug-binary-rio-v2` and `e2e-tests-rio-v2` | per-job | yes | dispatch `ci.yml` | | `build.yml` (weekly) | `build-rustfs` over the six-target platform matrix in `prepare-platform-matrix` (four Linux, macOS aarch64, Windows x86_64) | build/package integrity | yes | dispatch `build.yml` with an exact platform set | | `e2e-replication-nightly.yml` (nightly) | `repl-nightly`, `cluster-nightly`, `protocols-nightly` | three independent gates; JUnit, membership listing, server logs | yes | `cargo nextest run --profile e2e-repl-nightly -p e2e_test`; `--profile e2e-nightly`; `-j 1 --profile e2e-protocols` | +| `e2e-distributed.yml` (storage-sensitive PRs + nightly) | `distributed` | fail-closed 4-node 4-disk S3, durability, replication, movement, fault, and direct/rolling upgrade gate; JUnit, membership listing, per-node server logs | yes, with `never_ran_grace_until` | download the pinned previous release as in the workflow, export `RUSTFS_UPGRADE_SOURCE_BINARY`, then `cargo nextest run --profile e2e-distributed -p e2e_test` | | `e2e-s3tests.yml` (weekly) | `s3tests` (single and distributed, four shards each), `upstream-head-canary` | compatibility gate; report, JUnit, node IDs, server logs | yes | `scripts/s3-tests/run.sh` against an existing single or distributed target | | `fuzz.yml` (nightly) | `nightly-fuzz-corpus` per target | gate; corpus and crash artifacts | yes | `MAX_TOTAL_TIME= ./scripts/fuzz/run.sh` | | `minio-interop.yml` (nightly) | `minio-interop` | EC + SSE read-parity gate | yes, with `never_ran_grace_until` | pinned Docker fixture steps in the workflow | diff --git a/docs/testing/distributed-e2e.md b/docs/testing/distributed-e2e.md new file mode 100644 index 000000000..d8e3ec2bc --- /dev/null +++ b/docs/testing/distributed-e2e.md @@ -0,0 +1,75 @@ +# Distributed 4-node 4-disk e2e + +**Use this when:** adding or diagnosing GitHub Actions coverage for a 4-node cluster, or deciding whether a behaviour belongs in `e2e-distributed` versus the single-node `e2e-full` lane, the nightly cluster-fault lane, or the hardware functional chain. +**Source of truth:** `crates/e2e_test/src/distributed/`, `[profile.e2e-distributed]` in `.config/nextest.toml`, `.github/workflows/e2e-distributed.yml`. + +## Topology + +The in-tree harness runs every node on `127.0.0.1` with a distinct port. That matches `RustFSTestClusterEnvironment` in `crates/e2e_test/src/common.rs`: + +| Layout | Constructor | Use | +|---|---|---| +| 4 nodes × 4 drives, one pool | `ClusterTopology::single_pool_multidrive(4, 4)` | S3, object lock, versioning, quota, observability, concurrency, chaos | +| 4 nodes × 1 drive, one pool | `ClusterTopology::single_pool(4)` | Two-site replication (8 processes total); direct/rolling upgrade from the pinned previous release | +| 1 single-node pool × 4 drives, then `append_single_node_pool` three times | expansion seed | Pool expand, then decommission / rebalance / integrity | + +A multi-pool layout in which any pool spans several localhost ports is not expressible (`RUSTFS_VOLUMES` host ellipses would collide on disk paths). Multi-host striped expansion pools remain the hardware functional-chain / backlog #1313 / #1314 lane. + +Data-movement cases fail closed. A decommission or rebalance test must observe a successful start response, an active state, a clean terminal state, non-zero movement counters, and post-operation object integrity. An unsupported response, HTTP 5xx, missing status fields, cleanup warning, or zero-progress terminal response fails the case; pre/post S3 availability alone is not evidence that movement ran. + +The four expansion pools must report independent capacity. Four directories on one runner filesystem all return the same `statfs` totals, so RustFS correctly concludes that no pool is less free than the cluster average and performs no rebalance. The Actions job mounts four isolated ext4 loopback filesystems and exports their absolute paths through `RUSTFS_E2E_POOL_ROOTS`. The harness rejects missing, duplicate, relative, nonexistent, or same-device roots instead of allowing a vacuous movement pass. Planned pool additions stop every process with SIGTERM; hard process termination remains a chaos-only fault. After the fourth pool joins, the harness performs one full graceful persistent restart: this proves the expanded pool map survives restart and ensures movement begins only after every replica can load the converged metadata. + +The expansion fixture is an all-current-binary fleet, so it initializes pool metadata with the documented V3 write and fleet-confirmation gates. Decommission cases write their baseline objects, version history, and multipart data into pool 0 before adding pools 1–3, then retire pool 0. This makes a passing result evidence of user-data movement rather than merely an internal-metadata counter changing. + +## What this lane covers + +`cargo nextest run --profile e2e-distributed -p e2e_test` selects `distributed::*`: + +- S3 put / get / head / list / copy / rename / delete / presign, range and conditional reads, special keys, metadata, tags, pagination, empty objects, multipart complete and abort +- Object Lock COMPLIANCE, GOVERNANCE and bypass, legal hold, bucket default retention, and non-lock bucket rejection +- Versioning, exact historical reads, delete-marker removal, and suspended null-version overwrite semantics +- Bucket replication between two 4-node clusters, including metadata/tags and target-outage retry; hard quota admission and absence of rejected keys +- Ready/live probes on every node, exact 4-server/16-disk inventory, realtime metrics on every node, and correlated audit-webhook delivery +- Pool expand, decommission, rebalance, checksum integrity, versioned and multipart data, and S3 during active movement +- Bidirectional site-replication convergence plus enabled/synchronized peer state on both sites +- A 24-worker mixed PUT/HEAD/GET/COPY/DELETE workload; concurrent PUT during active decommission +- Node kill/restart, full process restart, node-facing TCP blackhole/recovery, in-flight streaming GETs across a peer kill, and fresh-drive replacement verified by physical `xl.meta`/part-shard census +- Multipart, cross-node listing, list-buckets agreement +- Direct and rolling upgrade from the pinned previous release: historical objects, versioned history, and IAM user AK/SK still work afterwards + +## Existing Actions gaps this lane does not replace + +Those suites stay in place; this lane fills the in-tree 4×4 hole they leave. + +| Existing lane | Gap | +|---|---| +| `rustfs-*-test.yml` functional chain | Clones private `rustfs/auto-testing`, runs on three shared VMs (`vm000`–`vm002`), `continue-on-error: true`, not a merge signal, not 4 nodes. Hardware `rustfs-upgrade-test.yml` stays there | +| `e2e-upgrade.yml` | Single-node SSE/multipart/delete-marker contracts plus mixed-version listing; does not pin IAM user AK/SK on a 4-node cluster | +| `e2e-smoke` / `e2e-full` | Most selected cases are single-node; distributed modules are intentionally owned by this serialized lane | +| `e2e-nightly` | 4-node cluster faults and heal, not S3/lock/versioning/quota/decommission matrix | +| `e2e-repl-nightly` | Site and bucket replication on 1–3 *single-node* processes | +| `e2e-s3tests.yml` `multi` | Weekly ceph/s3-tests against Docker 4-node; not lock/WORM, decommission, chaos, or checksum integrity | +| `crates/e2e_test/src/chaos.rs` | Single-node disk faults only | + +Hardware power-loss, physical NIC pull, authenticated inter-node partition, firmware/media errors, and replacement-server provisioning still belong on the hardware validation VMs. This lane provides deterministic process kill, fresh local-volume replacement, and node-facing TCP blackhole analogues; it does not claim physical fault certification. + +## Run + +```bash +cargo build -p rustfs --bins +# Expansion/decommission/rebalance cases require four paths on distinct filesystems. +export RUSTFS_E2E_POOL_ROOTS=/mnt/rustfs-pool-0:/mnt/rustfs-pool-1:/mnt/rustfs-pool-2:/mnt/rustfs-pool-3 +# Upgrade cases require the pinned previous binary (CI downloads it). +export RUSTFS_UPGRADE_SOURCE_BINARY=/path/to/rustfs-1.0.0-rc.2 +cargo nextest run --profile e2e-distributed -p e2e_test +``` + +Without `RUSTFS_UPGRADE_SOURCE_BINARY` the two `distributed::upgrade_test::*` cases fail closed. Without four distinct `RUSTFS_E2E_POOL_ROOTS`, the expansion and data-movement cases fail closed. Filter upgrades out for a local run that is not checking upgrade: + +```bash +cargo nextest run --profile e2e-distributed -p e2e_test -E 'not test(/^distributed::upgrade_test::/)' +``` + +The upgrade topology is `ClusterTopology::single_pool(4)` (4 nodes × 1 drive). That matches the proven mixed-version fixture in `upgrade_compatibility_test`; 4×4 localhost drives are rejected by the previous release's same-device disk check. + +Membership is pinned by `.config/e2e-distributed-selection.txt`. Update the Linux and Darwin entries with `python3 ./scripts/check_test_wiring.py --update-profile e2e-distributed ` after adding or renaming a case.