rustfs/crates/e2e_test/src/distributed/extra_test.rs

// Copyright 2026 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

use super::harness::{
    DistCluster, DistLayout, TestResult, assert_object_bytes, get_object_bytes, put_object, unique_bucket, wait_until,
};
use crate::common::init_logging;
use aws_sdk_s3::primitives::ByteStream;
use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart};
use std::time::Duration;

#[tokio::test]
async fn four_node_four_drive_multipart_and_cross_node_listing_agree() -> TestResult {
    init_logging();
    let dist = DistCluster::start(DistLayout::FourByFour).await?;
    let bucket = unique_bucket("extra");
    dist.create_bucket(&bucket).await?;
    let client = dist.client(0)?;

    let key = "multipart.bin";
    let part1 = vec![0x41u8; 5 * 1024 * 1024];
    let part2 = vec![0x42u8; 5 * 1024 * 1024];
    let upload = client.create_multipart_upload().bucket(&bucket).key(key).send().await?;
    let upload_id = upload.upload_id().ok_or("missing upload id")?.to_string();

    let uploaded1 = client
        .upload_part()
        .bucket(&bucket)
        .key(key)
        .upload_id(&upload_id)
        .part_number(1)
        .body(ByteStream::from(part1.clone()))
        .send()
        .await?;
    let uploaded2 = client
        .upload_part()
        .bucket(&bucket)
        .key(key)
        .upload_id(&upload_id)
        .part_number(2)
        .body(ByteStream::from(part2.clone()))
        .send()
        .await?;

    client
        .complete_multipart_upload()
        .bucket(&bucket)
        .key(key)
        .upload_id(&upload_id)
        .multipart_upload(
            CompletedMultipartUpload::builder()
                .parts(
                    CompletedPart::builder()
                        .part_number(1)
                        .e_tag(uploaded1.e_tag().unwrap_or_default())
                        .build(),
                )
                .parts(
                    CompletedPart::builder()
                        .part_number(2)
                        .e_tag(uploaded2.e_tag().unwrap_or_default())
                        .build(),
                )
                .build(),
        )
        .send()
        .await?;

    let mut expected = part1;
    expected.extend_from_slice(&part2);
    for node_idx in 0..dist.cluster.nodes.len() {
        assert_object_bytes(&dist.client(node_idx)?, &bucket, key, &expected).await?;
    }

    put_object(&client, &bucket, "list/a", b"a".to_vec()).await?;
    put_object(&dist.client(2)?, &bucket, "list/b", b"b".to_vec()).await?;
    let mut seen = Vec::new();
    for node_idx in 0..dist.cluster.nodes.len() {
        let listed = dist
            .client(node_idx)?
            .list_objects_v2()
            .bucket(&bucket)
            .prefix("list/")
            .send()
            .await?;
        let keys: Vec<String> = listed
            .contents()
            .iter()
            .filter_map(|object| object.key().map(str::to_string))
            .collect();
        seen.push(keys);
    }
    for keys in &seen[1..] {
        assert_eq!(&seen[0], keys, "list results diverged across nodes: {seen:?}");
    }

    let got = get_object_bytes(&dist.client(3)?, &bucket, "list/a").await?;
    assert_eq!(got, b"a");
    Ok(())
}

#[tokio::test]
async fn four_node_list_buckets_agree_across_all_nodes() -> TestResult {
    init_logging();
    let dist = DistCluster::start(DistLayout::FourByFour).await?;
    let bucket = unique_bucket("listed");
    dist.create_bucket(&bucket).await?;
    put_object(&dist.client(0)?, &bucket, "seed.bin", b"seed".to_vec()).await?;

    for node_idx in 0..dist.cluster.nodes.len() {
        let client = dist.client(node_idx)?;
        let name = bucket.clone();
        wait_until(
            Duration::from_secs(20),
            || {
                let client = client.clone();
                let name = name.clone();
                async move {
                    let listed = client.list_buckets().send().await?;
                    Ok(listed.buckets().iter().any(|entry| entry.name() == Some(name.as_str())))
                }
            },
            &format!("node {node_idx} lists {bucket}"),
        )
        .await?;
        wait_until(
            Duration::from_secs(20),
            || {
                let client = dist.client(node_idx).expect("client");
                let name = bucket.clone();
                async move { Ok(get_object_bytes(&client, &name, "seed.bin").await.ok() == Some(b"seed".to_vec())) }
            },
            &format!("node {node_idx} reads seed.bin"),
        )
        .await?;
    }
    Ok(())
}