diff --git a/.config/e2e-full-selection.txt b/.config/e2e-full-selection.txt index de20924ff..f1eefbd17 100644 --- a/.config/e2e-full-selection.txt +++ b/.config/e2e-full-selection.txt @@ -1,2 +1,2 @@ -sha256-darwin=88ee9684ece0e27294f2b3f0c9c8fe62890feff76aa47279d42dab0af3196fe2 -sha256-linux=01fc549f07ccd60ceb58abee8e437eacd028104eb0a107129c3f10b1cf21e519 +sha256-darwin=d6aa36cfaae2c4d8590482c7e47138c5965b335b34a75f50d11ffc3366e9021e +sha256-linux=373ba2e6c2f5dc8508c0119b03068d2e853068f1152ccc06c962c4823c2d660a diff --git a/.config/e2e-smoke-selection.txt b/.config/e2e-smoke-selection.txt index 0d5c5214c..d1112f9c5 100644 --- a/.config/e2e-smoke-selection.txt +++ b/.config/e2e-smoke-selection.txt @@ -1 +1 @@ -sha256=a183730b7b84232160ce5fdcfc06b5800e15bc79b3e03d43683ab3c7bdc0a23b +sha256=294350518743cac8d7c41880a2835216e4b697908d7b0b1bc92b62816d94c59d diff --git a/crates/e2e_test/src/reliant/tiering.rs b/crates/e2e_test/src/reliant/tiering.rs index cb478cbf9..401e3752c 100644 --- a/crates/e2e_test/src/reliant/tiering.rs +++ b/crates/e2e_test/src/reliant/tiering.rs @@ -27,8 +27,8 @@ //! loopback/SSRF restriction (that guard is replication-only), so `hot` can tier //! to `cold` over `http://127.0.0.1:`. //! -//! A single test drives the full transition main path and pins the chain -//! required by ilm-7: +//! The hermetic tests drive the transition and restore paths and pin the +//! chains required by ilm-7 and the restore follow-up: //! 1. `AddTier(RustFS)` on `hot` targeting `cold` โ€” the real connectivity / //! in-use probe runs (no `force`), so this also proves the tier is reachable. //! 2. A `Transition Days=0` rule installed before a multipart PUT transitions @@ -42,6 +42,9 @@ //! 6. The remote object is present in the cold-tier bucket after transition. //! 7. `DeleteObject` on `hot` drives free-version cleanup: the cold-tier copy //! eventually disappears and the hot object is gone (no local residue). +//! 8. `RestoreObject` copy-back failures clear the in-progress marker, a +//! retry serves the object locally until expiry, and expiry leaves the +//! remote object available for a second restore. use crate::common::{RustFSTestEnvironment, local_http_client}; use aws_sdk_s3::Client; @@ -49,7 +52,8 @@ use aws_sdk_s3::error::ProvideErrorMetadata; use aws_sdk_s3::primitives::ByteStream; use aws_sdk_s3::types::{ BucketLifecycleConfiguration, BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, ExpirationStatus, - LifecycleRule, LifecycleRuleFilter, NoncurrentVersionTransition, Transition, TransitionStorageClass, VersioningConfiguration, + LifecycleRule, LifecycleRuleFilter, NoncurrentVersionTransition, RestoreRequest, Transition, TransitionStorageClass, + VersioningConfiguration, }; use http::Method; use http::header::HOST; @@ -739,6 +743,77 @@ async fn wait_for_transition(client: &Client, bucket: &str, key: &str, deadline: } } +/// Poll `HEAD` until the asynchronous copy-back reports a completed restore. +async fn wait_for_restore_complete(client: &Client, bucket: &str, key: &str, deadline: StdDuration) -> TestResult { + let start = Instant::now(); + loop { + let head = client.head_object().bucket(bucket).key(key).send().await?; + if head + .restore() + .is_some_and(|restore| restore.contains("ongoing-request=\"false\"")) + { + return Ok(()); + } + if start.elapsed() >= deadline { + return Err(format!( + "restore for {bucket}/{key} did not complete within {}s; restore={:?}", + deadline.as_secs(), + head.restore() + ) + .into()); + } + tokio::time::sleep(StdDuration::from_millis(250)).await; + } +} + +/// Poll `HEAD` until the lifecycle restore-expiry action removes restore metadata. +async fn wait_for_restore_clear(client: &Client, bucket: &str, key: &str, deadline: StdDuration) -> TestResult { + let start = Instant::now(); + loop { + let head = client.head_object().bucket(bucket).key(key).send().await?; + if head.restore().is_none() { + return Ok(()); + } + if start.elapsed() >= deadline { + return Err(format!( + "restore metadata for {bucket}/{key} was not cleared within {}s; restore={:?}", + deadline.as_secs(), + head.restore() + ) + .into()); + } + tokio::time::sleep(StdDuration::from_millis(250)).await; + } +} + +/// Poll `HEAD` through the failed-copy transition, proving that the request +/// first published an in-progress marker and that the failure then removed it. +async fn wait_for_restore_failure(client: &Client, bucket: &str, key: &str, deadline: StdDuration) -> TestResult { + let start = Instant::now(); + let mut saw_ongoing = false; + loop { + let head = client.head_object().bucket(bucket).key(key).send().await?; + if head + .restore() + .is_some_and(|restore| restore.contains("ongoing-request=\"true\"")) + { + saw_ongoing = true; + } else if saw_ongoing && head.restore().is_none() { + return Ok(()); + } + if start.elapsed() >= deadline { + return Err(format!( + "failed restore for {bucket}/{key} did not publish and clear its in-progress marker within {}s; \ + saw_ongoing={saw_ongoing}, restore={:?}", + deadline.as_secs(), + head.restore() + ) + .into()); + } + tokio::time::sleep(StdDuration::from_millis(100)).await; + } +} + /// Poll until the cold-tier bucket is empty (remote free-version cleanup done), /// or fail after `deadline`. async fn wait_for_cold_tier_empty(cold_client: &Client, deadline: StdDuration) -> TestResult { @@ -894,6 +969,143 @@ async fn test_hermetic_transition_main_path() -> TestResult { Ok(()) } +/// Restore a transitioned object through a real RustFS remote tier. +/// +/// The test covers the externally visible copy-back contract that a mock tier +/// cannot prove: a failed remote read clears `x-amz-restore`, a retry creates a +/// local copy that remains readable while the cold tier is unavailable, expiry +/// removes only that local copy, and the same remote object can be restored a +/// second time. The accelerated lifecycle clock keeps the expiry assertion +/// bounded while the scanner remains enabled. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn test_hermetic_transition_restore_failure_expiry_and_retry() -> TestResult { + let mut cold = RustFSTestEnvironment::new().await?; + cold.access_key = "restorecoldtieradmin".to_string(); + cold.secret_key = "restorecoldtiersecret".to_string(); + cold.start_rustfs_server_without_cleanup(vec![]).await?; + let cold_client = cold.create_s3_client(); + cold_client.create_bucket().bucket(TIER_BUCKET).send().await?; + + let mut hot = RustFSTestEnvironment::new().await?; + hot.start_rustfs_server_with_env(vec![], &[("RUSTFS_SCANNER_CYCLE", "1"), ("RUSTFS_ILM_DEBUG_DAY_SECS", "5")]) + .await?; + let hot_client = hot.create_s3_client(); + add_rustfs_tier(&hot, &cold).await?; + + hot_client.create_bucket().bucket(SOURCE_BUCKET).send().await?; + hot_client + .put_bucket_lifecycle_configuration() + .bucket(SOURCE_BUCKET) + .lifecycle_configuration(BucketLifecycleConfiguration::builder().rules(transition_rule()?).build()?) + .send() + .await?; + + let data = payload(); + put_multipart_object(&hot_client, SOURCE_BUCKET, OBJECT_KEY, &data).await?; + wait_for_transition(&hot_client, SOURCE_BUCKET, OBJECT_KEY, StdDuration::from_secs(90)).await?; + let transitioned = hot_client.head_object().bucket(SOURCE_BUCKET).key(OBJECT_KEY).send().await?; + assert_eq!( + transitioned.storage_class().map(|storage_class| storage_class.as_str()), + Some(TIER_NAME), + "restore fixture must be transitioned before the copy-back request" + ); + assert!(transitioned.restore().is_none(), "transitioned object must not already be restored"); + assert_eq!( + cold_tier_object_count(&cold_client).await?, + 1, + "cold tier should contain one restore candidate" + ); + + // A failed remote read is accepted asynchronously, but it must not leave + // an object permanently advertising an in-progress restore. + cold.stop_server(); + hot_client + .restore_object() + .bucket(SOURCE_BUCKET) + .key(OBJECT_KEY) + .restore_request(RestoreRequest::builder().days(1).build()) + .send() + .await?; + wait_for_restore_failure(&hot_client, SOURCE_BUCKET, OBJECT_KEY, StdDuration::from_secs(30)).await?; + + // Once the tier is available again, the same object can be restored. + cold.restart_server_preserving_data(vec![], &[]).await?; + hot_client + .restore_object() + .bucket(SOURCE_BUCKET) + .key(OBJECT_KEY) + .restore_request(RestoreRequest::builder().days(1).build()) + .send() + .await?; + wait_for_restore_complete(&hot_client, SOURCE_BUCKET, OBJECT_KEY, StdDuration::from_secs(30)).await?; + + let restored = hot_client.head_object().bucket(SOURCE_BUCKET).key(OBJECT_KEY).send().await?; + assert_eq!( + restored.storage_class().map(|storage_class| storage_class.as_str()), + Some(TIER_NAME), + "restore must retain the transitioned storage class" + ); + assert!( + restored + .restore() + .is_some_and(|restore| restore.contains("ongoing-request=\"false\"")), + "completed restore must advertise a finished temporary copy" + ); + + // A completed restore must be served locally even if the remote tier is + // temporarily unavailable. + cold.stop_server(); + let local_get = hot_client.get_object().bucket(SOURCE_BUCKET).key(OBJECT_KEY).send().await?; + let local_body = local_get.body.collect().await?.into_bytes(); + assert_eq!(local_body.as_ref(), data.as_slice(), "restored local copy must be byte-identical"); + + cold.restart_server_preserving_data(vec![], &[]).await?; + + // The test clock makes the one-day restore expire in roughly five seconds. + wait_for_restore_clear(&hot_client, SOURCE_BUCKET, OBJECT_KEY, StdDuration::from_secs(30)).await?; + let expired = hot_client.head_object().bucket(SOURCE_BUCKET).key(OBJECT_KEY).send().await?; + assert_eq!( + expired.storage_class().map(|storage_class| storage_class.as_str()), + Some(TIER_NAME), + "restore expiry must not clear the transitioned storage class" + ); + assert_eq!( + cold_tier_object_count(&cold_client).await?, + 1, + "restore expiry must retain the remote object" + ); + + // After expiry the local copy is gone, so an unavailable tier must make the + // read fail; with the tier back, the original bytes remain readable. + cold.stop_server(); + let expired_get = hot_client.get_object().bucket(SOURCE_BUCKET).key(OBJECT_KEY).send().await; + assert!(expired_get.is_err(), "expired restore must not leave a local copy behind"); + cold.restart_server_preserving_data(vec![], &[]).await?; + let remote_get = hot_client.get_object().bucket(SOURCE_BUCKET).key(OBJECT_KEY).send().await?; + let remote_body = remote_get.body.collect().await?.into_bytes(); + assert_eq!(remote_body.as_ref(), data.as_slice(), "post-expiry GET must read the remote copy"); + + // The remote candidate survives expiry and can be restored again. + hot_client + .restore_object() + .bucket(SOURCE_BUCKET) + .key(OBJECT_KEY) + .restore_request(RestoreRequest::builder().days(1).build()) + .send() + .await?; + wait_for_restore_complete(&hot_client, SOURCE_BUCKET, OBJECT_KEY, StdDuration::from_secs(30)).await?; + + hot_client + .delete_object() + .bucket(SOURCE_BUCKET) + .key(OBJECT_KEY) + .send() + .await?; + wait_for_cold_tier_empty(&cold_client, StdDuration::from_secs(90)).await?; + + Ok(()) +} + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn test_manual_transition_run_black_box_semantics() -> TestResult { let mut cold = RustFSTestEnvironment::new().await?; diff --git a/docs/testing/e2e-suite-inventory.md b/docs/testing/e2e-suite-inventory.md index 502b07e60..c8412b236 100644 --- a/docs/testing/e2e-suite-inventory.md +++ b/docs/testing/e2e-suite-inventory.md @@ -18,6 +18,7 @@ |---|---|---| | admin_auth_test | 4 | โœ… | | admin_iam_crud_test | 3 | โœ… | +| admin_mfa_test | 4 | | | admin_pools_test | 1 | โœ… | | admin_timeout_regression_test | 1 | ๐ŸŒ™ | | anonymous_access_test | 4 | โœ… | @@ -30,7 +31,7 @@ | checksum_upload_test | 7 | | | cluster_concurrency_test | 3 | ๐ŸŒ™ | | cluster_multidrive_pool_test | 2 | ๐ŸŒ™ | -| common | 16 | | +| common | 17 | | | compression_test | 6 | โœ… | | connection_cap_test | 2 | | | console_smoke_test | 1 | โœ… | @@ -59,7 +60,7 @@ | heal_erasure_disk_rebuild_test | 4 | ๐ŸŒ™ | | inline_fast_path_cluster_test | 16 | | | internode_rpc_signature_e2e_test | 5 | | -| kms | 48 | | +| kms | 50 | | | leading_slash_key_test | 2 | โœ… | | lifecycle_regression_test | 4 | | | list_buckets_auth_test | 1 | โœ… | @@ -75,7 +76,7 @@ | multipart_auth_test | 75 | | | multipart_storage_class_test | 3 | โœ… | | namespace_lock_quorum_test | 2 | ๐ŸŒ™ | -| negative_sigv4_test | 6 | โœ… | +| negative_sigv4_test | 7 | โœ… | | notification_startup_regression_test | 2 | | | notification_webhook_test | 3 | โœ… | | object_lambda_test | 16 | ๐ŸŒ™ | @@ -86,7 +87,7 @@ | protocols | 16 | ๐ŸŒ™ | | quota_test | 14 | | | reliability_disk_fault_test | 4 | | -| reliant | 31 | 19 โœ… | +| reliant | 43 | 20 โœ… | | replication_extension_test | 76 | 20 โœ… +56 ๐ŸŒ™ | | replication_lww_receiver_test | 1 | | | security_boundary_test | 4 | | @@ -102,4 +103,4 @@ | tls_hot_reload_test | 1 | โœ… | | version_id_regression_test | 10 | โœ… | -**Total listed: 591 tests across 83 modules ยท PR smoke: 163 tests / 36 modules ยท merge/main full: 467 tests / 74 modules ยท nightly replication: 56 tests ยท nightly cluster faults: 29 tests / 7 modules ยท nightly protocols: 16 tests** ยท updated 2026-08-23. +**Total listed: 619 tests across 86 modules ยท PR smoke: 165 tests / 36 modules ยท merge/main full: 495 tests / 77 modules ยท nightly replication: 56 tests ยท nightly cluster faults: 29 tests / 7 modules ยท nightly protocols: 16 tests** ยท updated 2026-08-26.