diff --git a/.github/workflows/e2e-replication-nightly.yml b/.github/workflows/e2e-replication-nightly.yml index ef312d180..145ad317e 100644 --- a/.github/workflows/e2e-replication-nightly.yml +++ b/.github/workflows/e2e-replication-nightly.yml @@ -196,6 +196,9 @@ jobs: cache-save-if: 'false' install-build-packaging-tools: 'false' + - name: Verify protocol socket oracle + run: ss -tn state CLOSE-WAIT >/dev/null + # The suite owns fixed protocol ports and serializes its internal cases. - name: Verify protocol e2e membership env: diff --git a/crates/e2e_test/src/object_lambda_test.rs b/crates/e2e_test/src/object_lambda_test.rs index aa6f7d4a1..d2d5f0036 100644 --- a/crates/e2e_test/src/object_lambda_test.rs +++ b/crates/e2e_test/src/object_lambda_test.rs @@ -16,6 +16,7 @@ use crate::common::{RustFSTestClusterEnvironment, RustFSTestEnvironment, init_lo use aws_sdk_s3::primitives::ByteStream; use http::header::{CONTENT_TYPE, HOST}; use reqwest::StatusCode; +use rustfs_config::{ENV_DRIVE_ACTIVE_CHECK_INTERVAL_SECS, ENV_NOTIFY_ENABLE}; use rustfs_signer::pre_sign_v4; use rustfs_utils::egress::ENV_OUTBOUND_ALLOW_ORIGINS; use s3s::Body; @@ -976,7 +977,8 @@ async fn test_get_object_lambda_rejects_disabled_target() -> Result<(), Box Result<(), Box Resul init_logging(); let mut env = RustFSTestEnvironment::new().await?; - env.start_rustfs_server(vec![]).await?; + env.start_rustfs_server_with_env(vec![], &[(ENV_NOTIFY_ENABLE, "true")]) + .await?; let bucket = "object-lambda-e2e-invalid-endpoint"; @@ -1064,7 +1074,8 @@ async fn test_configure_object_lambda_notify_webhook_rejects_response_header_tim init_logging(); let mut env = RustFSTestEnvironment::new().await?; - env.start_rustfs_server(vec![]).await?; + env.start_rustfs_server_with_env(vec![], &[(ENV_NOTIFY_ENABLE, "true")]) + .await?; let response = send_configure_webhook_target_request( &env, @@ -1173,6 +1184,8 @@ async fn test_listen_notification_fans_in_remote_node_events() -> Result<(), Box init_logging(); let mut cluster = RustFSTestClusterEnvironment::new(2).await?; + cluster.set_env(ENV_NOTIFY_ENABLE, "true"); + cluster.set_env(ENV_DRIVE_ACTIVE_CHECK_INTERVAL_SECS, "1"); cluster.start().await?; let bucket = "listen-notification-cluster"; diff --git a/crates/e2e_test/src/stale_multipart_cleanup_cluster_test.rs b/crates/e2e_test/src/stale_multipart_cleanup_cluster_test.rs index c5fc45e17..82d0924bc 100644 --- a/crates/e2e_test/src/stale_multipart_cleanup_cluster_test.rs +++ b/crates/e2e_test/src/stale_multipart_cleanup_cluster_test.rs @@ -15,7 +15,6 @@ use crate::common::{RustFSTestClusterEnvironment, init_logging}; use aws_sdk_s3::error::SdkError; use aws_sdk_s3::primitives::ByteStream; -use aws_sdk_s3::types::CompletedMultipartUpload; use tokio::time::{Duration, sleep}; use tracing::info; use uuid::Uuid; @@ -43,32 +42,18 @@ async fn list_parts_reports_missing_upload( } } -async fn complete_reports_missing_upload( +async fn multipart_listing_reports_missing_upload( client: &aws_sdk_s3::Client, bucket: &str, key: &str, upload_id: &str, ) -> Result> { - let result = client - .complete_multipart_upload() - .bucket(bucket) - .key(key) - .upload_id(upload_id) - .multipart_upload(CompletedMultipartUpload::builder().build()) - .send() - .await; - match result { - Ok(_) => Ok(false), - Err(SdkError::ServiceError(err)) => { - let code = err.err().meta().code().unwrap_or(""); - if code == "NoSuchUpload" { - Ok(true) - } else { - Err(format!("unexpected complete_multipart_upload service error: code={code}, err={err:?}").into()) - } - } - Err(err) => Err(format!("unexpected complete_multipart_upload error: {err:?}").into()), - } + let result = client.list_multipart_uploads().bucket(bucket).prefix(key).send().await?; + + Ok(!result + .uploads() + .iter() + .any(|upload| upload.key() == Some(key) && upload.upload_id() == Some(upload_id))) } async fn wait_for_cleanup_on_all_nodes( @@ -81,8 +66,8 @@ async fn wait_for_cleanup_on_all_nodes( let mut all_cleaned = true; for (idx, client) in clients.iter().enumerate() { let list_parts_missing = list_parts_reports_missing_upload(client, bucket, key, upload_id).await?; - let complete_missing = complete_reports_missing_upload(client, bucket, key, upload_id).await?; - if !(list_parts_missing && complete_missing) { + let listing_missing = multipart_listing_reports_missing_upload(client, bucket, key, upload_id).await?; + if !(list_parts_missing && listing_missing) { info!("stale multipart still visible on node {} at attempt {}", idx, attempt + 1); all_cleaned = false; break; @@ -146,6 +131,10 @@ async fn test_stale_multipart_cleanup_removes_incomplete_upload_across_cluster() 1, "multipart upload should be visible before background cleanup" ); + assert!( + !multipart_listing_reports_missing_upload(&clients[2], CLEANUP_BUCKET, &key, &upload_id).await?, + "multipart upload listing should contain the upload before background cleanup" + ); wait_for_cleanup_on_all_nodes(&clients, CLEANUP_BUCKET, &key, &upload_id).await?;