diff --git a/.config/nextest.toml b/.config/nextest.toml index 7981a29d2..c0bce1796 100644 --- a/.config/nextest.toml +++ b/.config/nextest.toml @@ -542,6 +542,8 @@ test-group = 'e2e-cluster-nightly' [profile.e2e-distributed] default-filter = 'package(e2e_test) & test(/^distributed::/)' fail-fast = false +# Decommission / rebalance cases poll for up to 180s with little stdout. +slow-timeout = { period = "120s", terminate-after = 6 } [profile.e2e-distributed.junit] path = "junit.xml" diff --git a/.github/workflows/e2e-distributed.yml b/.github/workflows/e2e-distributed.yml index e52ccbc31..55fb6a9df 100644 --- a/.github/workflows/e2e-distributed.yml +++ b/.github/workflows/e2e-distributed.yml @@ -48,6 +48,9 @@ jobs: timeout-minutes: 180 env: FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true" + NO_PROXY: 127.0.0.1,localhost + HTTP_PROXY: "" + HTTPS_PROXY: "" steps: - name: Checkout repository uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 diff --git a/crates/e2e_test/src/distributed/harness.rs b/crates/e2e_test/src/distributed/harness.rs index 88bb5f8cd..ccdebb1af 100644 --- a/crates/e2e_test/src/distributed/harness.rs +++ b/crates/e2e_test/src/distributed/harness.rs @@ -79,6 +79,9 @@ impl DistCluster { DistLayout::TwoPoolFourDrive => ClusterTopology::per_node_pools(DRIVES_PER_NODE, vec![vec![0], vec![1]]), }; let mut cluster = RustFSTestClusterEnvironment::with_topology(topology).await?; + cluster.set_env("NO_PROXY", "127.0.0.1,localhost"); + cluster.set_env("HTTP_PROXY", ""); + cluster.set_env("HTTPS_PROXY", ""); for &(key, value) in extra_env { cluster.set_env(key, value); } diff --git a/crates/e2e_test/src/distributed/object_lock_test.rs b/crates/e2e_test/src/distributed/object_lock_test.rs index fc419934f..f2e47807e 100644 --- a/crates/e2e_test/src/distributed/object_lock_test.rs +++ b/crates/e2e_test/src/distributed/object_lock_test.rs @@ -4,7 +4,7 @@ // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // -// http://www.apache.org/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, @@ -12,15 +12,39 @@ // See the License for the specific language governing permissions and // limitations under the License. -use super::harness::{DistCluster, DistLayout, TestResult, put_object, unique_bucket}; +use super::harness::{DistCluster, DistLayout, TestResult, unique_bucket}; use crate::common::init_logging; +use crate::object_lock::common::{delete_object_with_bypass, put_object_with_legal_hold, put_object_with_retention}; +use aws_sdk_s3::Client; use aws_sdk_s3::error::ProvideErrorMetadata; -use aws_sdk_s3::types::{ - DefaultRetention, ObjectLockConfiguration, ObjectLockEnabled, ObjectLockLegalHold, ObjectLockLegalHoldStatus, - ObjectLockRetention, ObjectLockRetentionMode, ObjectLockRule, -}; +use aws_sdk_s3::error::SdkError; +use aws_sdk_s3::operation::delete_object::DeleteObjectError; +use aws_sdk_s3::types::{ObjectLockLegalHoldStatus, ObjectLockRetentionMode}; use chrono::{Duration as ChronoDuration, Utc}; +fn delete_denied(error: &SdkError, context: &str) -> TestResult { + let code = error.as_service_error().and_then(ProvideErrorMetadata::code); + if code == Some("AccessDenied") { + Ok(()) + } else { + Err(format!("{context}: expected AccessDenied, got {error:?}").into()) + } +} + +async fn expect_versioned_delete_denied( + client: &Client, + bucket: &str, + key: &str, + version_id: &str, + bypass: bool, + context: &str, +) -> TestResult { + match delete_object_with_bypass(client, bucket, key, Some(version_id), bypass).await { + Ok(_) => Err(format!("{context}: DeleteObject of retained version must be denied").into()), + Err(error) => delete_denied(error.as_ref(), context), + } +} + #[tokio::test] async fn four_node_four_drive_object_lock_worm_blocks_delete() -> TestResult { init_logging(); @@ -37,101 +61,51 @@ async fn four_node_four_drive_object_lock_worm_blocks_delete() -> TestResult { .await?; let retain_until = Utc::now() + ChronoDuration::days(1); - let retain_until_s3 = aws_sdk_s3::primitives::DateTime::from_secs(retain_until.timestamp()); - - client - .put_object_lock_configuration() - .bucket(&bucket) - .object_lock_configuration( - ObjectLockConfiguration::builder() - .object_lock_enabled(ObjectLockEnabled::Enabled) - .rule( - ObjectLockRule::builder() - .default_retention( - DefaultRetention::builder() - .mode(ObjectLockRetentionMode::Governance) - .days(1) - .build(), - ) - .build(), - ) - .build(), - ) - .send() - .await?; let compliance_key = "compliance.bin"; - put_object(&client, &bucket, compliance_key, b"locked-compliance".to_vec()).await?; - client - .put_object_retention() - .bucket(&bucket) - .key(compliance_key) - .retention( - ObjectLockRetention::builder() - .mode(ObjectLockRetentionMode::Compliance) - .retain_until_date(retain_until_s3) - .build(), - ) - .send() - .await?; + let compliance_version = put_object_with_retention( + &client, + &bucket, + compliance_key, + b"locked-compliance", + ObjectLockRetentionMode::Compliance, + retain_until, + ) + .await?; - let compliance_delete = peer.delete_object().bucket(&bucket).key(compliance_key).send().await; - match compliance_delete { - Ok(_) => return Err("COMPLIANCE retention must block DeleteObject".into()), - Err(error) => { - let code = error.as_service_error().and_then(ProvideErrorMetadata::code); - assert_eq!(code, Some("AccessDenied"), "unexpected COMPLIANCE delete error: {error:?}"); - } - } + // Unversioned DELETE is allowed: it only creates a delete marker. WORM + // applies to a specific version id. + let marker = peer.delete_object().bucket(&bucket).key(compliance_key).send().await?; + assert_eq!( + marker.delete_marker(), + Some(true), + "unversioned DELETE on a locked object must create a delete marker" + ); + + expect_versioned_delete_denied(&peer, &bucket, compliance_key, &compliance_version, false, "COMPLIANCE without bypass") + .await?; + expect_versioned_delete_denied(&peer, &bucket, compliance_key, &compliance_version, true, "COMPLIANCE with bypass").await?; let governance_key = "governance.bin"; - put_object(&client, &bucket, governance_key, b"locked-governance".to_vec()).await?; - client - .put_object_retention() - .bucket(&bucket) - .key(governance_key) - .retention( - ObjectLockRetention::builder() - .mode(ObjectLockRetentionMode::Governance) - .retain_until_date(retain_until_s3) - .build(), - ) - .send() - .await?; + let governance_version = put_object_with_retention( + &client, + &bucket, + governance_key, + b"locked-governance", + ObjectLockRetentionMode::Governance, + retain_until, + ) + .await?; - let governance_blocked = peer.delete_object().bucket(&bucket).key(governance_key).send().await; - match governance_blocked { - Ok(_) => return Err("GOVERNANCE retention must block DeleteObject without bypass".into()), - Err(error) => { - let code = error.as_service_error().and_then(ProvideErrorMetadata::code); - assert_eq!(code, Some("AccessDenied"), "unexpected GOVERNANCE delete error: {error:?}"); - } - } - - peer.delete_object() - .bucket(&bucket) - .key(governance_key) - .bypass_governance_retention(true) - .send() + expect_versioned_delete_denied(&peer, &bucket, governance_key, &governance_version, false, "GOVERNANCE without bypass") .await?; + delete_object_with_bypass(&peer, &bucket, governance_key, Some(&governance_version), true).await?; let hold_key = "legal-hold.bin"; - put_object(&client, &bucket, hold_key, b"legal-hold".to_vec()).await?; - client - .put_object_legal_hold() - .bucket(&bucket) - .key(hold_key) - .legal_hold(ObjectLockLegalHold::builder().status(ObjectLockLegalHoldStatus::On).build()) - .send() - .await?; - let hold_delete = peer.delete_object().bucket(&bucket).key(hold_key).send().await; - match hold_delete { - Ok(_) => return Err("legal hold must block DeleteObject".into()), - Err(error) => { - let code = error.as_service_error().and_then(ProvideErrorMetadata::code); - assert_eq!(code, Some("AccessDenied"), "unexpected legal-hold delete error: {error:?}"); - } - } + let hold_version = + put_object_with_legal_hold(&client, &bucket, hold_key, b"legal-hold", ObjectLockLegalHoldStatus::On).await?; + expect_versioned_delete_denied(&peer, &bucket, hold_key, &hold_version, false, "legal hold without bypass").await?; + expect_versioned_delete_denied(&peer, &bucket, hold_key, &hold_version, true, "legal hold with bypass").await?; Ok(()) }