rustfs/crates/ecstore/src/set_disk/transition_matrix_tests.rs

// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

use super::*;
use crate::bucket::lifecycle::lifecycle::{TRANSITION_COMPLETE, TRANSITION_PENDING, TransitionOptions, expected_expiry_time};
use crate::ecstore_validation_blackbox::make_local_set_disks;
use crate::services::tier::test_util::register_mock_tier;
use crate::storage_api_contracts::object::{ObjectIO as _, ObjectOperations as _};
use rustfs_filemeta::{RestoreStatusOps as _, parse_restore_obj_status};
use tokio::io::AsyncReadExt;

async fn prime_metadata_generation(set_disks: &SetDisks, bucket: &str, object: &str) -> GetObjectMetadataCacheKey {
    set_disks
        .get_object_fileinfo(bucket, object, &ObjectOptions::default(), true, false)
        .await
        .expect("object metadata should resolve");
    let generation = set_disks
        .get_object_metadata_cache_generation(bucket, object)
        .expect("metadata generation should be active");
    let key = GetObjectMetadataCacheKey::new(bucket, object, generation);
    assert!(
        set_disks.get_object_metadata_cache.get(&key).await.is_some(),
        "metadata read should publish the generation under test"
    );
    key
}

async fn assert_generation_reclaimed(set_disks: &SetDisks, key: &GetObjectMetadataCacheKey) {
    set_disks.get_object_metadata_cache.run_pending_tasks().await;
    assert!(
        set_disks.get_object_metadata_cache.get(key).await.is_none(),
        "metadata mutation must physically reclaim the prior generation"
    );
}

#[tokio::test]
#[serial_test::serial]
async fn transition_and_restore_reclaim_prior_metadata_generations() {
    let (_dirs, set_disks) = make_local_set_disks(4, 2).await;
    let bucket = "transition-restore-generation-bucket";
    let object = "object.bin";
    let payload = vec![0x5au8; 1024 * 1024];
    set_disks
        .make_bucket(bucket, &MakeBucketOptions::default())
        .await
        .expect("bucket should be created");
    let mut reader = PutObjReader::from_vec(payload.clone());
    let original = set_disks
        .put_object(bucket, object, &mut reader, &ObjectOptions::default())
        .await
        .expect("source object should be written");
    let source_generation = prime_metadata_generation(&set_disks, bucket, object).await;

    let tier_name = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase();
    let backend = register_mock_tier(&set_disks.instance_ctx().tier_config_mgr(), &tier_name).await;
    let transition_opts = ObjectOptions {
        transition: TransitionOptions {
            status: TRANSITION_PENDING.to_string(),
            tier: tier_name,
            etag: original.etag.clone().expect("source ETag should be present"),
            ..Default::default()
        },
        version_id: original.version_id.map(|version| version.to_string()),
        mod_time: original.mod_time,
        ..Default::default()
    };
    set_disks
        .transition_object(bucket, object, &transition_opts)
        .await
        .expect("transition should succeed");
    assert_generation_reclaimed(&set_disks, &source_generation).await;

    let transitioned_generation = prime_metadata_generation(&set_disks, bucket, object).await;
    let mut restore_opts = ObjectOptions::default();
    restore_opts.transition.restore_request.days = Some(1);
    let restore_started = OffsetDateTime::now_utc();
    let expiry_from_restore_start = temp_env::async_with_vars(
        [
            ("RUSTFS_ILM_DEBUG_DAY_SECS", Some("1")),
            ("RUSTFS_ILM_PROCESS_TIME", Some("1")),
        ],
        async {
            let expiry_from_restore_start = expected_expiry_time(restore_started, 1);
            let get_barrier = backend.arm_get_barrier().await;
            let restore_set = Arc::clone(&set_disks);
            let restore =
                tokio::spawn(async move { restore_set.restore_transitioned_object(bucket, object, &restore_opts).await });
            get_barrier.wait_until_paused().await;
            tokio::time::timeout(Duration::from_secs(5), async {
                loop {
                    if expected_expiry_time(OffsetDateTime::now_utc(), 1) > expiry_from_restore_start {
                        break;
                    }
                    tokio::time::sleep(Duration::from_millis(10)).await;
                }
            })
            .await
            .expect("test clock should cross the next accelerated lifecycle boundary");
            get_barrier.release();
            restore
                .await
                .expect("restore task should join")
                .expect("restore should succeed");
            expiry_from_restore_start
        },
    )
    .await;
    assert_generation_reclaimed(&set_disks, &transitioned_generation).await;
    assert_eq!(backend.get_count().await, 1, "restore should read the remote candidate exactly once");

    let restored_info = set_disks
        .get_object_info(bucket, object, &ObjectOptions::default())
        .await
        .expect("restored object metadata should be readable");
    let restore_status = parse_restore_obj_status(
        restored_info
            .user_defined
            .get(s3s::header::X_AMZ_RESTORE.as_str())
            .expect("completed restore header should be present"),
    )
    .expect("completed restore header should parse");
    assert!(
        restore_status.expiry().expect("completed restore should have an expiry") > expiry_from_restore_start,
        "restore expiry must be based on completion, not the time the remote copy started"
    );

    let mut restored = Vec::new();
    set_disks
        .get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
        .await
        .expect("restored object should be readable")
        .stream
        .read_to_end(&mut restored)
        .await
        .expect("restored body should drain");
    assert_eq!(restored, payload);
    assert_eq!(backend.get_count().await, 1, "restored GET should use the local copy");
}

#[tokio::test]
#[serial_test::serial]
async fn prepared_snapshot_transition_duplicate_and_late_get_use_committed_remote_generation() {
    let (_dirs, set_disks) = make_local_set_disks(4, 2).await;
    let bucket = "transition-prepared-combo-bucket";
    let object = "object.bin";
    let payload = b"prepared metadata must not revive the released transition source".repeat(32_768);
    set_disks
        .make_bucket(bucket, &MakeBucketOptions::default())
        .await
        .expect("bucket should be created");
    let mut reader = PutObjReader::from_vec(payload.clone());
    let original = set_disks
        .put_object(bucket, object, &mut reader, &ObjectOptions::default())
        .await
        .expect("source object should be written");

    let stale_generation = prime_metadata_generation(&set_disks, bucket, object).await;
    let stale_prepared = set_disks
        .prepare_get_object_metadata(bucket, object, &ObjectOptions::default())
        .await
        .expect("prepared metadata snapshot should resolve before transition");
    assert_ne!(
        stale_prepared.object_info().transitioned_object.status,
        TRANSITION_COMPLETE,
        "prepared snapshot must represent the pre-transition local source"
    );

    let tier_name = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase();
    let backend = register_mock_tier(&set_disks.instance_ctx().tier_config_mgr(), &tier_name).await;
    let transition_opts = ObjectOptions {
        transition: TransitionOptions {
            status: TRANSITION_PENDING.to_string(),
            tier: tier_name,
            etag: original.etag.clone().expect("source ETag should be present"),
            ..Default::default()
        },
        version_id: original.version_id.map(|version| version.to_string()),
        mod_time: original.mod_time,
        ..Default::default()
    };

    set_disks
        .transition_object(bucket, object, &transition_opts)
        .await
        .expect("first transition should commit");
    assert_generation_reclaimed(&set_disks, &stale_generation).await;
    assert_eq!(backend.put_count().await, 1, "the winning transition should upload one remote candidate");
    let put_versions = backend.put_versions().await;
    assert_eq!(put_versions.len(), 1, "the winner should expose exactly one remote object/version");
    let remote_object = put_versions[0].0.clone();
    assert_eq!(
        backend
            .bytes(&remote_object)
            .await
            .expect("winning remote object should remain stored"),
        payload,
        "the committed remote candidate must contain the exact source body"
    );

    set_disks
        .transition_object(bucket, object, &transition_opts)
        .await
        .expect("duplicate transition should observe the committed transition and return success");
    assert_eq!(
        backend.put_count().await,
        1,
        "duplicate transition must not stream the already-released local source into a second remote PUT"
    );
    assert_eq!(
        backend.remove_count().await,
        0,
        "committed transition data must not be cleaned up by the duplicate"
    );

    let info = set_disks
        .get_object_info(bucket, object, &ObjectOptions::default())
        .await
        .expect("transitioned metadata should be visible after duplicate transition");
    assert_eq!(info.transitioned_object.status, TRANSITION_COMPLETE);
    assert_eq!(info.transitioned_object.name, remote_object);

    let mut restored = Vec::new();
    set_disks
        .get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
        .await
        .expect("late GET should open the committed transitioned object")
        .stream
        .read_to_end(&mut restored)
        .await
        .expect("late GET should drain the remote body");
    assert_eq!(restored, payload, "late GET must read the committed remote body byte-for-byte");
    assert_eq!(
        backend.get_count().await,
        1,
        "late GET must route to the remote generation instead of reopening the released local source"
    );
}