test(e2e): wait for heal peers after node rejoin

.github/actions/schedule-failure-issue/action.yml

@@ -14,10 +14,9 @@
 
 name: "Schedule Failure Issue"
 description: >-
-  Open (or update) a tracking issue when a scheduled workflow run fails or
-  does not complete normally.
+  Open (or update) a tracking issue when a scheduled workflow run fails.
   Dedupes by workflow name: if an open issue titled
-  "[scheduled-failure] <workflow name>" already exists, the result is
+  "[scheduled-failure] <workflow name>" already exists, the failure is
   appended as a comment; otherwise a new issue is created. This is the
   single alerting mechanism for all scheduled pipelines (backlog#1149 ci-8).
 
@@ -39,26 +38,6 @@ inputs:
       Set to an empty string to skip labeling.
     required: false
     default: "infrastructure"
-  source-run-id:
-    description: "Run ID to report. Defaults to the current workflow run."
-    required: false
-    default: ${{ github.run_id }}
-  source-run-attempt:
-    description: "Run attempt to report. Defaults to the current attempt."
-    required: false
-    default: ${{ github.run_attempt }}
-  source-event:
-    description: "Trigger event of the run being reported."
-    required: false
-    default: ${{ github.event_name }}
-  source-ref-name:
-    description: "Ref name of the run being reported."
-    required: false
-    default: ${{ github.ref_name }}
-  source-sha:
-    description: "Commit SHA of the run being reported."
-    required: false
-    default: ${{ github.sha }}
 
 runs:
   using: "composite"
@@ -69,21 +48,17 @@ runs:
         GH_TOKEN: ${{ inputs.github-token }}
         WORKFLOW_NAME: ${{ inputs.workflow-name }}
         ISSUE_LABEL: ${{ inputs.label }}
-        SOURCE_RUN_ID: ${{ inputs.source-run-id }}
-        SOURCE_RUN_ATTEMPT: ${{ inputs.source-run-attempt }}
-        SOURCE_EVENT: ${{ inputs.source-event }}
-        SOURCE_REF_NAME: ${{ inputs.source-ref-name }}
-        SOURCE_SHA: ${{ inputs.source-sha }}
       run: |
         set -euo pipefail
 
         title="[scheduled-failure] ${WORKFLOW_NAME}"
-        run_url="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${SOURCE_RUN_ID}"
+        run_url="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
 
-        # Inspect the reported run attempt. It can be the current in-workflow
-        # failure or a completed run observed by the external watchdog.
+        # Failed job names for this run attempt. The alert job runs while the
+        # run as a whole is still in progress, so inspect the jobs that have
+        # already completed with a non-success conclusion.
         failed_jobs="$(gh api \
-          "repos/${GITHUB_REPOSITORY}/actions/runs/${SOURCE_RUN_ID}/attempts/${SOURCE_RUN_ATTEMPT}/jobs" \
+          "repos/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/attempts/${GITHUB_RUN_ATTEMPT}/jobs" \
           --paginate \
           --jq '.jobs[]
                 | select(.conclusion == "failure" or .conclusion == "timed_out" or .conclusion == "cancelled")
@@ -93,13 +68,13 @@ runs:
         fi
 
         body="$(cat <<EOF
-        Scheduled run of **${WORKFLOW_NAME}** did not complete successfully.
+        Scheduled run of **${WORKFLOW_NAME}** failed.
 
-        - Run: ${run_url} (attempt ${SOURCE_RUN_ATTEMPT})
-        - Event: \`${SOURCE_EVENT}\`
-        - Ref: \`${SOURCE_REF_NAME}\` @ \`${SOURCE_SHA}\`
+        - Run: ${run_url} (attempt ${GITHUB_RUN_ATTEMPT})
+        - Event: \`${GITHUB_EVENT_NAME}\`
+        - Ref: \`${GITHUB_REF_NAME}\` @ \`${GITHUB_SHA}\`
 
-        Non-success jobs:
+        Failed jobs:
         ${failed_jobs}
         EOF
         )"

.github/workflows/build.yml

@@ -1032,23 +1032,3 @@ jobs:
 
           echo "🎉 Released $TAG successfully!"
           echo "📄 Release URL: ${{ needs.create-release.outputs.release_url }}"
-
-  alert-on-failure:
-    name: Alert on scheduled failure
-    needs: [build-check, prepare-platform-matrix, build-rustfs, build-summary]
-    if: >-
-      always() && github.event_name == 'schedule' &&
-      (contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled'))
-    runs-on: ubuntu-latest
-    timeout-minutes: 10
-    permissions:
-      contents: read
-      issues: write
-    steps:
-      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
-        with:
-          persist-credentials: false
-      - name: Open or update failure-tracking issue
-        uses: ./.github/actions/schedule-failure-issue
-        with:
-          github-token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/ci.yml

@@ -1032,37 +1032,3 @@ jobs:
           path: artifacts/s3tests-single/**
           if-no-files-found: ignore
           retention-days: 3
-
-  alert-on-failure:
-    name: Alert on scheduled failure
-    needs:
-      - typos
-      - quick-checks
-      - test-and-lint
-      - test-ilm-integration-serial
-      - test-and-lint-rio-v2
-      - test-and-lint-protocols
-      - build-rustfs-debug-binary
-      - build-rustfs-debug-binary-rio-v2
-      - uring-integration
-      - e2e-tests
-      - e2e-full
-      - e2e-tests-rio-v2
-      - s3-implemented-tests
-      - s3-lifecycle-behavior-tests
-    if: >-
-      always() && github.event_name == 'schedule' &&
-      (contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled'))
-    runs-on: ubuntu-latest
-    timeout-minutes: 10
-    permissions:
-      contents: read
-      issues: write
-    steps:
-      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
-        with:
-          persist-credentials: false
-      - name: Open or update failure-tracking issue
-        uses: ./.github/actions/schedule-failure-issue
-        with:
-          github-token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/minio-interop.yml

@@ -121,21 +121,3 @@ jobs:
           cargo nextest run --run-ignored ignored-only --no-tests=fail \
             -p "$INTEROP_PACKAGE" --features "$INTEROP_FEATURES" \
             -E "$INTEROP_FILTER"
-
-  alert-on-failure:
-    name: Alert on scheduled failure
-    needs: [minio-interop]
-    if: always() && github.event_name == 'schedule' && contains(needs.*.result, 'failure')
-    runs-on: ubuntu-latest
-    timeout-minutes: 10
-    permissions:
-      contents: read
-      issues: write
-    steps:
-      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
-        with:
-          persist-credentials: false
-      - name: Open or update failure-tracking issue
-        uses: ./.github/actions/schedule-failure-issue
-        with:
-          github-token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/nightly-gnu.yml

@@ -194,23 +194,3 @@ jobs:
 
       - name: Run HA leader failover live checks (three-node Raft cluster in Docker)
         run: bash scripts/test/vault_ha_kms_live.sh
-
-  alert-on-failure:
-    name: Alert on scheduled failure
-    needs: [build, kms-vault-lane, kms-vault-ha-failover]
-    if: >-
-      always() && github.event_name == 'schedule' &&
-      (contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled'))
-    runs-on: ubuntu-latest
-    timeout-minutes: 10
-    permissions:
-      contents: read
-      issues: write
-    steps:
-      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
-        with:
-          persist-credentials: false
-      - name: Open or update failure-tracking issue
-        uses: ./.github/actions/schedule-failure-issue
-        with:
-          github-token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/scheduled-validation-watchdog.yml

@@ -1,63 +0,0 @@
-# Copyright 2024 RustFS Team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-name: Scheduled Validation Watchdog
-
-on:
-  workflow_run:
-    workflows:
-      - "Security Audit"
-      - "Build and Release"
-      - "Continuous Integration"
-      - "coverage"
-      - "e2e-nightly"
-      - "e2e-s3tests"
-      - "Fuzz"
-      - "mint"
-      - "minio-interop"
-      - "Nightly GNU Build"
-      - "Performance A/B"
-      - "Runner Hygiene"
-    types: [completed]
-
-permissions:
-  contents: read
-
-jobs:
-  alert-on-incomplete-run:
-    name: Alert on incomplete scheduled run
-    if: >-
-      github.event.workflow_run.event == 'schedule' &&
-      github.event.workflow_run.conclusion != 'success' &&
-      github.event.workflow_run.conclusion != 'failure'
-    runs-on: ubuntu-latest
-    timeout-minutes: 10
-    permissions:
-      actions: read
-      contents: read
-      issues: write
-    steps:
-      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
-        with:
-          persist-credentials: false
-      - name: Open or update incomplete-run issue
-        uses: ./.github/actions/schedule-failure-issue
-        with:
-          github-token: ${{ secrets.GITHUB_TOKEN }}
-          workflow-name: ${{ github.event.workflow_run.name }}
-          source-run-id: ${{ github.event.workflow_run.id }}
-          source-run-attempt: ${{ github.event.workflow_run.run_attempt }}
-          source-event: ${{ github.event.workflow_run.event }}
-          source-ref-name: ${{ github.event.workflow_run.head_branch }}
-          source-sha: ${{ github.event.workflow_run.head_sha }}

crates/e2e_test/src/common.rs

@@ -30,7 +30,6 @@ use reqwest::StatusCode;
 use rustfs_signer::constants::UNSIGNED_PAYLOAD;
 use rustfs_signer::sign_v4;
 use s3s::Body;
-use serde_json;
 use std::ffi::OsStr;
 use std::fs as stdfs;
 use std::io::ErrorKind;
@@ -1584,156 +1583,6 @@ impl Drop for RustFSTestClusterEnvironment {
     }
 }
 
-/// Send a SigV4-signed HTTP request and return the raw `reqwest::Response`.
-///
-/// Unlike [`signed_s3_request`], this variant accepts `body: Option<Vec<u8>>`
-/// (binary-safe) and reorders parameters so that `access_key`/`secret_key`
-/// appear before the body — matching the convention used by the replication
-/// extension and object-lambda e2e suites.
-pub(crate) async fn signed_request(
-    method: http::Method,
-    url: &str,
-    access_key: &str,
-    secret_key: &str,
-    body: Option<Vec<u8>>,
-    content_type: Option<&str>,
-) -> Result<reqwest::Response, Box<dyn std::error::Error + Send + Sync>> {
-    let uri = url.parse::<http::Uri>()?;
-    let authority = uri.authority().ok_or("request URL missing authority")?.to_string();
-    let mut request = http::Request::builder().method(method.clone()).uri(uri);
-    request = request.header(HOST, authority);
-    request = request.header("x-amz-content-sha256", UNSIGNED_PAYLOAD);
-    if let Some(content_type) = content_type {
-        request = request.header(CONTENT_TYPE, content_type);
-    }
-
-    let content_len = body.as_ref().map(|body| body.len() as i64).unwrap_or_default();
-    let signed = sign_v4(request.body(Body::empty())?, content_len, access_key, secret_key, "", "us-east-1");
-
-    let reqwest_method = reqwest::Method::from_bytes(method.as_str().as_bytes())?;
-    let client = local_http_client();
-    let mut request_builder = client.request(reqwest_method, url);
-    for (name, value) in signed.headers() {
-        request_builder = request_builder.header(name, value);
-    }
-    if let Some(body) = body {
-        request_builder = request_builder.body(body);
-    }
-
-    Ok(request_builder.send().await?)
-}
-
-/// Like [`signed_request`], but uses a caller-supplied `reqwest::Client`
-/// instead of the shared [`local_http_client`].
-pub(crate) async fn signed_request_with_client(
-    client: &reqwest::Client,
-    method: http::Method,
-    url: &str,
-    access_key: &str,
-    secret_key: &str,
-    body: Option<Vec<u8>>,
-    content_type: Option<&str>,
-) -> Result<reqwest::Response, Box<dyn std::error::Error + Send + Sync>> {
-    let uri = url.parse::<http::Uri>()?;
-    let authority = uri.authority().ok_or("request URL missing authority")?.to_string();
-    let mut request = http::Request::builder().method(method.clone()).uri(uri);
-    request = request.header(HOST, authority);
-    request = request.header("x-amz-content-sha256", UNSIGNED_PAYLOAD);
-    if let Some(content_type) = content_type {
-        request = request.header(CONTENT_TYPE, content_type);
-    }
-
-    let content_len = body.as_ref().map(|body| body.len() as i64).unwrap_or_default();
-    let signed = sign_v4(request.body(Body::empty())?, content_len, access_key, secret_key, "", "us-east-1");
-
-    let reqwest_method = reqwest::Method::from_bytes(method.as_str().as_bytes())?;
-    let mut request_builder = client.request(reqwest_method, url);
-    for (name, value) in signed.headers() {
-        request_builder = request_builder.header(name, value);
-    }
-    if let Some(body) = body {
-        request_builder = request_builder.body(body);
-    }
-
-    Ok(request_builder.send().await?)
-}
-
-/// Like [`signed_request`], but includes a `session_token` in the
-/// `x-amz-security-token` header and passes it to the SigV4 signer.
-pub(crate) async fn signed_request_with_session_token(
-    method: http::Method,
-    url: &str,
-    access_key: &str,
-    secret_key: &str,
-    session_token: &str,
-    body: Option<Vec<u8>>,
-    content_type: Option<&str>,
-) -> Result<reqwest::Response, Box<dyn std::error::Error + Send + Sync>> {
-    let uri = url.parse::<http::Uri>()?;
-    let authority = uri.authority().ok_or("request URL missing authority")?.to_string();
-    let mut request = http::Request::builder().method(method.clone()).uri(uri);
-    request = request.header(HOST, authority);
-    request = request.header("x-amz-content-sha256", UNSIGNED_PAYLOAD);
-    if !session_token.is_empty() {
-        request = request.header("x-amz-security-token", session_token);
-    }
-    if let Some(content_type) = content_type {
-        request = request.header(CONTENT_TYPE, content_type);
-    }
-
-    let content_len = body.as_ref().map(|body| body.len() as i64).unwrap_or_default();
-    let signed = sign_v4(
-        request.body(Body::empty())?,
-        content_len,
-        access_key,
-        secret_key,
-        session_token,
-        "us-east-1",
-    );
-
-    let reqwest_method = reqwest::Method::from_bytes(method.as_str().as_bytes())?;
-    let client = local_http_client();
-    let mut request_builder = client.request(reqwest_method, url);
-    for (name, value) in signed.headers() {
-        request_builder = request_builder.header(name, value);
-    }
-    if let Some(body) = body {
-        request_builder = request_builder.body(body);
-    }
-
-    Ok(request_builder.send().await?)
-}
-
-/// Create a new user via the admin API.
-pub(crate) async fn admin_create_user(
-    env: &RustFSTestEnvironment,
-    username: &str,
-    secret_key: &str,
-) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
-    let url = format!("{}/rustfs/admin/v3/add-user?accessKey={}", env.url, username);
-    let body = serde_json::json!({
-        "secretKey": secret_key,
-        "status": "enabled"
-    });
-    let response = signed_request(
-        http::Method::PUT,
-        &url,
-        &env.access_key,
-        &env.secret_key,
-        Some(body.to_string().into_bytes()),
-        Some("application/json"),
-    )
-    .await?;
-
-    if response.status() != reqwest::StatusCode::OK {
-        let status = response.status();
-        let body = response.text().await.unwrap_or_default();
-        return Err(format!("create user failed: {status} {body}").into());
-    }
-
-    Ok(())
-}
-
 #[cfg(test)]
 mod tests {
     use super::*;

crates/e2e_test/src/heal_erasure_disk_rebuild_test.rs

@@ -380,10 +380,24 @@ mod tests {
         cluster.start_node(1).await?;
 
         let status_url = format!("{}/rustfs/admin/v3/background-heal/status", cluster.nodes[0].url);
-        let status_body = signed_admin_post(&status_url, None, &cluster.access_key, &cluster.secret_key).await?;
-        assert!(
-            !status_body.contains("MissingContentLength"),
-            "background heal status should not fail without an explicit Content-Length: {status_body}"
+        let mut recovered = serde_json::Value::Null;
+        for _ in 0..60 {
+            let status_body = signed_admin_post(&status_url, None, &cluster.access_key, &cluster.secret_key).await?;
+            assert!(
+                !status_body.contains("MissingContentLength"),
+                "background heal status should not fail without an explicit Content-Length: {status_body}"
+            );
+            recovered = serde_json::from_str(&status_body)
+                .map_err(|err| format!("background heal status is not JSON ({err}): {status_body}"))?;
+            if recovered["clusterStatusComplete"] == serde_json::Value::Bool(true) {
+                break;
+            }
+            sleep(Duration::from_secs(1)).await;
+        }
+        assert_eq!(
+            recovered["clusterStatusComplete"],
+            serde_json::Value::Bool(true),
+            "cluster heal status should recover before root heal starts: {recovered}"
         );
 
         let heal_body = r#"{"recursive":true,"dryRun":false,"remove":false,"recreate":true,"scanMode":2,"updateParity":false,"nolock":false}"#;

crates/e2e_test/src/kms/common.rs

@@ -40,7 +40,7 @@ use std::time::Duration;
 use tokio::fs;
 use tokio::net::TcpStream;
 use tokio::time::sleep;
-use tracing::{debug, error, info, warn};
+use tracing::{debug, error, info};
 
 // KMS-specific constants
 pub const TEST_BUCKET: &str = "kms-test-bucket";
@@ -177,49 +177,6 @@ pub async fn get_kms_status(
     Ok(status)
 }
 
-/// Poll the KMS status endpoint until the backend reports ready or the timeout
-/// expires.  Replaces hard-coded `sleep(Duration::from_secs(3))` startup waits
-/// with an active readiness probe so tests start as soon as KMS is usable
-/// (typically < 1 s) instead of always waiting the full 3 s.
-///
-/// Uses exponential back-off starting at 200 ms (doubling each attempt, capped
-/// at 1 s) up to a total wall-clock budget of 5 s.
-pub async fn wait_for_kms_ready(
-    base_url: &str,
-    access_key: &str,
-    secret_key: &str,
-) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
-    let total_deadline = Duration::from_secs(5);
-    let start = tokio::time::Instant::now();
-    let mut backoff = Duration::from_millis(200);
-    let max_backoff = Duration::from_secs(1);
-    let mut first_attempt = true;
-
-    loop {
-        if !first_attempt {
-            if start.elapsed() >= total_deadline {
-                return Err("KMS failed to become ready within 5 seconds".into());
-            }
-            sleep(backoff).await;
-            backoff = (backoff * 2).min(max_backoff);
-        }
-        first_attempt = false;
-
-        match get_kms_status(base_url, access_key, secret_key).await {
-            Ok(status) => {
-                info!("KMS is ready (status: {})", status);
-                return Ok(());
-            }
-            Err(e) => {
-                if start.elapsed() >= total_deadline {
-                    return Err(format!("KMS did not become ready within 5 s: last error: {e}").into());
-                }
-                warn!(error = %e, elapsed_ms = start.elapsed().as_millis() as u64, "KMS not ready yet, retrying…");
-            }
-        }
-    }
-}
-
 /// Create a default KMS key for testing and return the created key ID
 pub async fn create_default_key(
     base_url: &str,
@@ -904,13 +861,6 @@ impl LocalKMSTestEnvironment {
         Ok(default_key_id.to_string())
     }
 
-    /// Poll the KMS status endpoint until the backend reports ready.
-    ///
-    /// Prefer this over a fixed `sleep` after calling `start_rustfs_for_local_kms`.
-    pub async fn wait_for_kms_ready(&self) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
-        wait_for_kms_ready(&self.base_env.url, &self.base_env.access_key, &self.base_env.secret_key).await
-    }
-
     /// Configure Local KMS backend with a predefined default key
     pub async fn configure_local_kms(&self) -> Result<String, Box<dyn std::error::Error + Send + Sync>> {
         // Use a fixed, predictable default key ID

crates/e2e_test/src/object_lambda_test.rs

@@ -12,11 +12,12 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use crate::common::{RustFSTestClusterEnvironment, RustFSTestEnvironment, init_logging, local_http_client, signed_request};
+use crate::common::{RustFSTestClusterEnvironment, RustFSTestEnvironment, init_logging, local_http_client};
 use aws_sdk_s3::primitives::ByteStream;
 use http::header::{CONTENT_TYPE, HOST};
 use reqwest::StatusCode;
-use rustfs_signer::pre_sign_v4;
+use rustfs_signer::constants::UNSIGNED_PAYLOAD;
+use rustfs_signer::{pre_sign_v4, sign_v4};
 use rustfs_utils::egress::ENV_OUTBOUND_ALLOW_ORIGINS;
 use s3s::Body;
 use std::collections::HashMap;
@@ -226,6 +227,39 @@ async fn presigned_get_request(
     Ok(local_http_client().get(signed.uri().to_string()).send().await?)
 }
 
+async fn signed_request(
+    method: http::Method,
+    url: &str,
+    access_key: &str,
+    secret_key: &str,
+    body: Option<Vec<u8>>,
+    content_type: Option<&str>,
+) -> Result<reqwest::Response, Box<dyn Error + Send + Sync>> {
+    let uri = url.parse::<http::Uri>()?;
+    let authority = uri.authority().ok_or("request URL missing authority")?.to_string();
+    let mut request = http::Request::builder().method(method.clone()).uri(uri);
+    request = request.header(HOST, authority);
+    request = request.header("x-amz-content-sha256", UNSIGNED_PAYLOAD);
+    if let Some(content_type) = content_type {
+        request = request.header(CONTENT_TYPE, content_type);
+    }
+
+    let content_len = body.as_ref().map(|body| body.len() as i64).unwrap_or_default();
+    let signed = sign_v4(request.body(Body::empty())?, content_len, access_key, secret_key, "", "us-east-1");
+
+    let reqwest_method = reqwest::Method::from_bytes(method.as_str().as_bytes())?;
+    let client = local_http_client();
+    let mut request_builder = client.request(reqwest_method, url);
+    for (name, value) in signed.headers() {
+        request_builder = request_builder.header(name, value);
+    }
+    if let Some(body) = body {
+        request_builder = request_builder.body(body);
+    }
+
+    Ok(request_builder.send().await?)
+}
+
 async fn configure_webhook_target(
     env: &RustFSTestEnvironment,
     target_name: &str,

crates/e2e_test/src/replication_extension_test.rs

@@ -13,9 +13,8 @@
 // limitations under the License.
 
 use crate::common::{
-    RustFSTestEnvironment, admin_create_user, awscurl_available, awscurl_post_sts_form_urlencoded, init_logging,
-    local_http_client, replication_fast_env, rustfs_binary_path, signed_request, signed_request_with_client,
-    signed_request_with_session_token,
+    RustFSTestEnvironment, awscurl_available, awscurl_post_sts_form_urlencoded, init_logging, local_http_client,
+    replication_fast_env, rustfs_binary_path,
 };
 use crate::fake_s3_target::{
     FAKE_ACCESS_KEY, FAKE_SECRET_KEY, FakeS3Target, FaultAction as FakeTargetFault, Operation as FakeTargetOperation,
@@ -36,7 +35,7 @@ use base64::{Engine, engine::general_purpose::STANDARD as BASE64_STANDARD};
 use bytes::Bytes;
 use flate2::read::GzDecoder;
 use futures::{Stream, StreamExt};
-use http::header::CONTENT_ENCODING;
+use http::header::{CONTENT_ENCODING, CONTENT_TYPE, HOST};
 use http_body_util::{BodyExt, Full};
 use hyper::body::Incoming;
 use hyper::server::conn::http1;
@@ -57,6 +56,9 @@ use rustfs_madmin::{
     AddServiceAccountReq, ListServiceAccountsResp, PeerInfo, PeerSite, ReplicateAddStatus, ReplicateEditStatus,
     ReplicateRemoveStatus, SRRemoveReq, SRResyncOpStatus, SRStatusInfo, SiteReplicationInfo, SyncStatus,
 };
+use rustfs_signer::constants::UNSIGNED_PAYLOAD;
+use rustfs_signer::sign_v4;
+use s3s::Body;
 use s3s::header::X_AMZ_REPLICATION_STATUS;
 use sha2::{Digest, Sha256};
 use std::collections::BTreeMap;
@@ -385,6 +387,116 @@ struct ReplicationResetStatusTarget {
     object: String,
 }
 
+async fn signed_request(
+    method: http::Method,
+    url: &str,
+    access_key: &str,
+    secret_key: &str,
+    body: Option<Vec<u8>>,
+    content_type: Option<&str>,
+) -> Result<reqwest::Response, Box<dyn Error + Send + Sync>> {
+    let uri = url.parse::<http::Uri>()?;
+    let authority = uri.authority().ok_or("request URL missing authority")?.to_string();
+    let mut request = http::Request::builder().method(method.clone()).uri(uri);
+    request = request.header(HOST, authority);
+    request = request.header("x-amz-content-sha256", UNSIGNED_PAYLOAD);
+    if let Some(content_type) = content_type {
+        request = request.header(CONTENT_TYPE, content_type);
+    }
+
+    let content_len = body.as_ref().map(|body| body.len() as i64).unwrap_or_default();
+    let signed = sign_v4(request.body(Body::empty())?, content_len, access_key, secret_key, "", "us-east-1");
+
+    let reqwest_method = reqwest::Method::from_bytes(method.as_str().as_bytes())?;
+    let client = local_http_client();
+    let mut request_builder = client.request(reqwest_method, url);
+    for (name, value) in signed.headers() {
+        request_builder = request_builder.header(name, value);
+    }
+    if let Some(body) = body {
+        request_builder = request_builder.body(body);
+    }
+
+    Ok(request_builder.send().await?)
+}
+
+async fn signed_request_with_client(
+    client: &reqwest::Client,
+    method: http::Method,
+    url: &str,
+    access_key: &str,
+    secret_key: &str,
+    body: Option<Vec<u8>>,
+    content_type: Option<&str>,
+) -> Result<reqwest::Response, Box<dyn Error + Send + Sync>> {
+    let uri = url.parse::<http::Uri>()?;
+    let authority = uri.authority().ok_or("request URL missing authority")?.to_string();
+    let mut request = http::Request::builder().method(method.clone()).uri(uri);
+    request = request.header(HOST, authority);
+    request = request.header("x-amz-content-sha256", UNSIGNED_PAYLOAD);
+    if let Some(content_type) = content_type {
+        request = request.header(CONTENT_TYPE, content_type);
+    }
+
+    let content_len = body.as_ref().map(|body| body.len() as i64).unwrap_or_default();
+    let signed = sign_v4(request.body(Body::empty())?, content_len, access_key, secret_key, "", "us-east-1");
+
+    let reqwest_method = reqwest::Method::from_bytes(method.as_str().as_bytes())?;
+    let mut request_builder = client.request(reqwest_method, url);
+    for (name, value) in signed.headers() {
+        request_builder = request_builder.header(name, value);
+    }
+    if let Some(body) = body {
+        request_builder = request_builder.body(body);
+    }
+
+    Ok(request_builder.send().await?)
+}
+
+async fn signed_request_with_session_token(
+    method: http::Method,
+    url: &str,
+    access_key: &str,
+    secret_key: &str,
+    session_token: &str,
+    body: Option<Vec<u8>>,
+    content_type: Option<&str>,
+) -> Result<reqwest::Response, Box<dyn Error + Send + Sync>> {
+    let uri = url.parse::<http::Uri>()?;
+    let authority = uri.authority().ok_or("request URL missing authority")?.to_string();
+    let mut request = http::Request::builder().method(method.clone()).uri(uri);
+    request = request.header(HOST, authority);
+    request = request.header("x-amz-content-sha256", UNSIGNED_PAYLOAD);
+    if !session_token.is_empty() {
+        request = request.header("x-amz-security-token", session_token);
+    }
+    if let Some(content_type) = content_type {
+        request = request.header(CONTENT_TYPE, content_type);
+    }
+
+    let content_len = body.as_ref().map(|body| body.len() as i64).unwrap_or_default();
+    let signed = sign_v4(
+        request.body(Body::empty())?,
+        content_len,
+        access_key,
+        secret_key,
+        session_token,
+        "us-east-1",
+    );
+
+    let reqwest_method = reqwest::Method::from_bytes(method.as_str().as_bytes())?;
+    let client = local_http_client();
+    let mut request_builder = client.request(reqwest_method, url);
+    for (name, value) in signed.headers() {
+        request_builder = request_builder.header(name, value);
+    }
+    if let Some(body) = body {
+        request_builder = request_builder.body(body);
+    }
+
+    Ok(request_builder.send().await?)
+}
+
 fn extract_xml_tag(xml: &str, tag: &str) -> Option<String> {
     let open = format!("<{tag}>");
     let close = format!("</{tag}>");
@@ -904,6 +1016,35 @@ fn create_user_s3_client(env: &RustFSTestEnvironment, access_key: &str, secret_k
     Client::from_conf(config)
 }
 
+async fn admin_create_user(
+    env: &RustFSTestEnvironment,
+    username: &str,
+    secret_key: &str,
+) -> Result<(), Box<dyn Error + Send + Sync>> {
+    let url = format!("{}/rustfs/admin/v3/add-user?accessKey={}", env.url, username);
+    let body = serde_json::json!({
+        "secretKey": secret_key,
+        "status": "enabled"
+    });
+    let response = signed_request(
+        http::Method::PUT,
+        &url,
+        &env.access_key,
+        &env.secret_key,
+        Some(body.to_string().into_bytes()),
+        Some("application/json"),
+    )
+    .await?;
+
+    if response.status() != StatusCode::OK {
+        let status = response.status();
+        let body = response.text().await.unwrap_or_default();
+        return Err(format!("create user failed: {status} {body}").into());
+    }
+
+    Ok(())
+}
+
 async fn admin_add_canned_policy(
     env: &RustFSTestEnvironment,
     policy_name: &str,

crates/madmin/src/trace.rs

@@ -12,16 +12,18 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use std::{collections::HashMap, time::Duration};
+
+use jiff::Timestamp;
 use serde::{Deserialize, Serialize};
 
-/// Bitflag helper for service trace categories.
-///
-/// Each variant occupies a single bit so that a `TraceType` value can represent
-/// an arbitrary combination of categories via bitwise OR.
+use crate::heal_commands::HealResultItem;
+
 #[derive(Debug, Clone, Copy, Serialize, Deserialize, Default)]
 pub struct TraceType(u64);
 
 impl TraceType {
+    // Define some constants
     pub const OS: TraceType = TraceType(1 << 0);
     pub const STORAGE: TraceType = TraceType(1 << 1);
     pub const S3: TraceType = TraceType(1 << 2);
@@ -38,13 +40,15 @@ impl TraceType {
     pub const FTP: TraceType = TraceType(1 << 13);
     pub const ILM: TraceType = TraceType(1 << 14);
 
-    /// All trace categories combined. Must be updated when adding new variants.
+    // MetricsAll must be last.
     pub const ALL: TraceType = TraceType((1 << 15) - 1);
 
     pub fn new(t: u64) -> Self {
         Self(t)
     }
+}
 
+impl TraceType {
     pub fn contains(&self, x: &TraceType) -> bool {
         (self.0 & x.0) == x.0
     }
@@ -72,38 +76,140 @@ impl TraceType {
     }
 }
 
+#[derive(Debug, Default, Clone, Serialize, Deserialize)]
+pub struct TraceInfo {
+    #[serde(rename = "type")]
+    trace_type: u64,
+    #[serde(rename = "nodename")]
+    node_name: String,
+    #[serde(rename = "funcname")]
+    func_name: String,
+    #[serde(rename = "time")]
+    time: Timestamp,
+    #[serde(rename = "path")]
+    path: String,
+    #[serde(rename = "dur")]
+    duration: Duration,
+    #[serde(rename = "bytes", skip_serializing_if = "Option::is_none")]
+    bytes: Option<i64>,
+    #[serde(rename = "msg", skip_serializing_if = "Option::is_none")]
+    message: Option<String>,
+    #[serde(rename = "error", skip_serializing_if = "Option::is_none")]
+    error: Option<String>,
+    #[serde(rename = "custom", skip_serializing_if = "Option::is_none")]
+    custom: Option<HashMap<String, String>>,
+    #[serde(rename = "http", skip_serializing_if = "Option::is_none")]
+    http: Option<TraceHTTPStats>,
+    #[serde(rename = "healResult", skip_serializing_if = "Option::is_none")]
+    heal_result: Option<HealResultItem>,
+}
+
+impl TraceInfo {
+    pub fn mask(&self) -> u64 {
+        TraceType::new(self.trace_type).mask()
+    }
+}
+
+#[derive(Debug, Default, Clone, Serialize, Deserialize)]
+pub struct TraceInfoLegacy {
+    trace_info: TraceInfo,
+    #[serde(rename = "request")]
+    req_info: Option<TraceRequestInfo>,
+    #[serde(rename = "response")]
+    resp_info: Option<TraceResponseInfo>,
+    #[serde(rename = "stats")]
+    call_stats: Option<TraceCallStats>,
+    #[serde(rename = "storageStats")]
+    storage_stats: Option<StorageStats>,
+    #[serde(rename = "osStats")]
+    os_stats: Option<OSStats>,
+}
+
+#[derive(Debug, Default, Clone, Serialize, Deserialize)]
+pub struct StorageStats {
+    path: String,
+    duration: Duration,
+}
+
+#[derive(Debug, Default, Clone, Serialize, Deserialize)]
+pub struct OSStats {
+    path: String,
+    duration: Duration,
+}
+
+#[derive(Debug, Default, Clone, Serialize, Deserialize)]
+pub struct TraceHTTPStats {
+    req_info: TraceRequestInfo,
+    resp_info: TraceResponseInfo,
+    call_stats: TraceCallStats,
+}
+
+#[derive(Debug, Default, Clone, Serialize, Deserialize)]
+pub struct TraceCallStats {
+    input_bytes: i32,
+    output_bytes: i32,
+    latency: Duration,
+    time_to_first_byte: Duration,
+}
+
+#[derive(Debug, Default, Clone, Serialize, Deserialize)]
+pub struct TraceRequestInfo {
+    time: Timestamp,
+    proto: String,
+    method: String,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    path: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    raw_query: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    headers: Option<HashMap<String, String>>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    body: Option<Vec<u8>>,
+    client: String,
+}
+
+#[derive(Debug, Default, Clone, Serialize, Deserialize)]
+pub struct TraceResponseInfo {
+    time: Timestamp,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    headers: Option<HashMap<String, String>>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    body: Option<Vec<u8>>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    status_code: Option<i32>,
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
 
     #[test]
-    fn trace_type_contains_and_overlaps() {
-        let mut combined = TraceType::default();
-        combined.merge(&TraceType::S3);
-        combined.merge(&TraceType::HEALING);
+    fn trace_timestamps_serialize_as_rfc3339_utc() {
+        let timestamp = Timestamp::constant(1_700_000_000, 123_456_000);
+        let trace = TraceInfo {
+            time: timestamp,
+            http: Some(TraceHTTPStats {
+                req_info: TraceRequestInfo {
+                    time: timestamp,
+                    ..Default::default()
+                },
+                resp_info: TraceResponseInfo {
+                    time: timestamp,
+                    ..Default::default()
+                },
+                ..Default::default()
+            }),
+            ..Default::default()
+        };
 
-        assert!(combined.contains(&TraceType::S3));
-        assert!(combined.contains(&TraceType::HEALING));
-        assert!(!combined.contains(&TraceType::SCANNER));
-        assert!(combined.overlaps(&TraceType::S3));
-        assert!(combined.overlaps(&TraceType::HEALING));
-        assert!(!combined.overlaps(&TraceType::SCANNER));
-    }
-
-    #[test]
-    fn trace_type_set_if() {
-        let mut tt = TraceType::default();
-        tt.set_if(true, &TraceType::OS);
-        tt.set_if(false, &TraceType::S3);
-        assert!(tt.contains(&TraceType::OS));
-        assert!(!tt.contains(&TraceType::S3));
-    }
-
-    #[test]
-    fn trace_type_single_type() {
-        assert!(TraceType::S3.single_type());
-        let mut combined = TraceType::S3;
-        combined.merge(&TraceType::HEALING);
-        assert!(!combined.single_type());
+        let value = serde_json::to_value(trace).expect("trace should serialize");
+        assert_eq!(value["time"], "2023-11-14T22:13:20.123456Z");
+        assert_eq!(value["http"]["req_info"]["time"], "2023-11-14T22:13:20.123456Z");
+        assert_eq!(value["http"]["resp_info"]["time"], "2023-11-14T22:13:20.123456Z");
+        let trace: TraceInfo = serde_json::from_value(value).expect("trace should deserialize");
+        assert_eq!(trace.time, timestamp);
+        let http = trace.http.expect("http trace should deserialize");
+        assert_eq!(http.req_info.time, timestamp);
+        assert_eq!(http.resp_info.time, timestamp);
     }
 }

scripts/check_test_wiring.py

@@ -15,20 +15,6 @@ from pathlib import Path
 
 
 ROOT = Path(__file__).resolve().parents[1]
-SCHEDULED_ALERT_WORKFLOWS = (
-    ".github/workflows/audit.yml",
-    ".github/workflows/build.yml",
-    ".github/workflows/ci.yml",
-    ".github/workflows/coverage.yml",
-    ".github/workflows/e2e-replication-nightly.yml",
-    ".github/workflows/e2e-s3tests.yml",
-    ".github/workflows/fuzz.yml",
-    ".github/workflows/mint.yml",
-    ".github/workflows/minio-interop.yml",
-    ".github/workflows/nightly-gnu.yml",
-    ".github/workflows/performance-ab.yml",
-    ".github/workflows/runner-hygiene.yml",
-)
 
 
 def words(value: str) -> set[str]:
@@ -220,13 +206,6 @@ def check_runner_selection(root: Path) -> list[str]:
     return errors
 
 
-def check_s3_tests_runner(root: Path) -> list[str]:
-    runner = (root / "scripts/s3-tests/run.sh").read_text()
-    if "--showlocals" in runner:
-        return ["scripts/s3-tests/run.sh: pytest failure diagnostics must not dump local values"]
-    return []
-
-
 def profile_selection(root: Path, profile: str) -> str:
     if not re.fullmatch(r"e2e-[a-z0-9-]+", profile):
         raise ValueError(f"invalid e2e profile name: {profile}")
@@ -266,79 +245,6 @@ def check_profile_definitions(root: Path) -> list[str]:
     return errors
 
 
-def check_scheduled_alerts(root: Path) -> list[str]:
-    errors: list[str] = []
-    for relative in SCHEDULED_ALERT_WORKFLOWS:
-        path = root / relative
-        try:
-            lines = path.read_text().splitlines()
-        except FileNotFoundError:
-            errors.append(f"{relative}: missing scheduled validation workflow")
-            continue
-
-        try:
-            start = lines.index("  alert-on-failure:") + 1
-        except ValueError:
-            errors.append(f"{relative}: missing alert-on-failure job")
-            continue
-        end = next(
-            (index for index in range(start, len(lines)) if re.fullmatch(r"  [A-Za-z0-9_-]+:", lines[index])),
-            len(lines),
-        )
-        job = "\n".join(line.split("#", 1)[0] for line in lines[start:end])
-        required = (
-            "always()",
-            "github.event_name == 'schedule'",
-            "contains(needs.*.result, 'failure')",
-            "issues: write",
-            "uses: ./.github/actions/schedule-failure-issue",
-            "github-token: ${{ secrets.GITHUB_TOKEN }}",
-        )
-        missing = [token for token in required if token not in job]
-        if missing:
-            errors.append(f"{relative}: alert-on-failure missing {', '.join(missing)}")
-
-    watchdog_path = root / ".github/workflows/scheduled-validation-watchdog.yml"
-    try:
-        watchdog = "\n".join(
-            line.split("#", 1)[0] for line in watchdog_path.read_text().splitlines()
-        )
-    except FileNotFoundError:
-        errors.append(".github/workflows/scheduled-validation-watchdog.yml: missing completion watchdog")
-        return errors
-    for relative in SCHEDULED_ALERT_WORKFLOWS:
-        path = root / relative
-        if not path.is_file():
-            continue
-        source = path.read_text()
-        match = re.search(r"^name:\s*[\"']?([^\"'\n]+)", source, re.MULTILINE)
-        if not match:
-            errors.append(f"{relative}: missing workflow name")
-        elif f'- "{match.group(1).strip()}"' not in watchdog:
-            errors.append(f"{relative}: missing from scheduled completion watchdog")
-    required = (
-        "github.event.workflow_run.event == 'schedule'",
-        "github.event.workflow_run.conclusion != 'success'",
-        "github.event.workflow_run.conclusion != 'failure'",
-        "actions: read",
-        "issues: write",
-        "uses: ./.github/actions/schedule-failure-issue",
-        "github-token: ${{ secrets.GITHUB_TOKEN }}",
-        "workflow-name: ${{ github.event.workflow_run.name }}",
-        "source-event: ${{ github.event.workflow_run.event }}",
-        "source-ref-name: ${{ github.event.workflow_run.head_branch }}",
-        "source-sha: ${{ github.event.workflow_run.head_sha }}",
-        "source-run-id: ${{ github.event.workflow_run.id }}",
-        "source-run-attempt: ${{ github.event.workflow_run.run_attempt }}",
-    )
-    missing = [token for token in required if token not in watchdog]
-    if missing:
-        errors.append(
-            ".github/workflows/scheduled-validation-watchdog.yml: missing " + ", ".join(missing)
-        )
-    return errors
-
-
 def check_profile_listing(root: Path, profile: str, listing: Path) -> list[str]:
     try:
         expected_digest = profile_selection(root, profile)
@@ -366,9 +272,7 @@ def validate(root: Path) -> list[str]:
     errors.extend(check_e2e_modules(root))
     errors.extend(check_fuzz_targets(root))
     errors.extend(check_runner_selection(root))
-    errors.extend(check_s3_tests_runner(root))
     errors.extend(check_profile_definitions(root))
-    errors.extend(check_scheduled_alerts(root))
     return errors
 
 
@@ -437,24 +341,6 @@ class SelfTests(unittest.TestCase):
             )
             self.assertEqual(len(check_fuzz_targets(root)), 1)
 
-    def test_s3_runner_rejects_unbounded_failure_locals(self) -> None:
-        with tempfile.TemporaryDirectory() as tmp:
-            root = Path(tmp)
-            runner = root / "scripts/s3-tests/run.sh"
-            runner.parent.mkdir(parents=True)
-            runner.write_text("tox -- -vv -ra --tb=long\n")
-            self.assertEqual(check_s3_tests_runner(root), [])
-            runner.write_text("tox -- -vv -ra --showlocals --tb=long\n")
-            self.assertEqual(len(check_s3_tests_runner(root)), 1)
-            with (
-                mock.patch(__name__ + ".check_e2e_modules", return_value=[]),
-                mock.patch(__name__ + ".check_fuzz_targets", return_value=[]),
-                mock.patch(__name__ + ".check_runner_selection", return_value=[]),
-                mock.patch(__name__ + ".check_profile_definitions", return_value=[]),
-                mock.patch(__name__ + ".check_scheduled_alerts", return_value=[]),
-            ):
-                self.assertEqual(len(validate(root)), 1)
-
     def test_profile_listing_enforces_selection(self) -> None:
         with tempfile.TemporaryDirectory() as tmp:
             root = Path(tmp)
@@ -502,75 +388,6 @@ class SelfTests(unittest.TestCase):
             with mock.patch.object(sys, "platform", "linux"):
                 self.assertEqual(len(check_profile_listing(root, "e2e-full", listing)), 1)
 
-    def test_scheduled_alerts_require_completion_watchdog(self) -> None:
-        with tempfile.TemporaryDirectory() as tmp:
-            root = Path(tmp)
-            alert = (
-                "  alert-on-failure:\n"
-                "    if: always() && github.event_name == 'schedule' && "
-                "contains(needs.*.result, 'failure')\n"
-                "    permissions:\n"
-                "      issues: write\n"
-                "    steps:\n"
-                "      - uses: ./.github/actions/schedule-failure-issue\n"
-                "        with:\n"
-                "          github-token: ${{ secrets.GITHUB_TOKEN }}\n"
-            )
-            names: list[str] = []
-            for relative in SCHEDULED_ALERT_WORKFLOWS:
-                path = root / relative
-                path.parent.mkdir(parents=True, exist_ok=True)
-                names.append(path.stem)
-                path.write_text(f'name: "{path.stem}"\n{alert}')
-            watchdog = root / ".github/workflows/scheduled-validation-watchdog.yml"
-            watchdog_source = (
-                "\n".join(f'- "{name}"' for name in names)
-                + "\ngithub.event.workflow_run.event == 'schedule'\n"
-                + "github.event.workflow_run.conclusion != 'success'\n"
-                + "github.event.workflow_run.conclusion != 'failure'\n"
-                + "actions: read\n"
-                + "issues: write\n"
-                + "uses: ./.github/actions/schedule-failure-issue\n"
-                + "github-token: ${{ secrets.GITHUB_TOKEN }}\n"
-                + "workflow-name: ${{ github.event.workflow_run.name }}\n"
-                + "source-event: ${{ github.event.workflow_run.event }}\n"
-                + "source-ref-name: ${{ github.event.workflow_run.head_branch }}\n"
-                + "source-sha: ${{ github.event.workflow_run.head_sha }}\n"
-                + "source-run-id: ${{ github.event.workflow_run.id }}\n"
-                + "source-run-attempt: ${{ github.event.workflow_run.run_attempt }}\n"
-            )
-            watchdog.write_text(watchdog_source)
-            self.assertEqual(check_scheduled_alerts(root), [])
-
-            first = root / SCHEDULED_ALERT_WORKFLOWS[0]
-            mutations = (
-                ("contains(needs.*.result, 'failure')", "false"),
-                ("issues: write", "issues: read"),
-                ("uses: ./.github/actions/schedule-failure-issue", "uses: actions/checkout@v7"),
-                ("github-token: ${{ secrets.GITHUB_TOKEN }}", "github-token: missing"),
-            )
-            for required, replacement in mutations:
-                original = first.read_text()
-                first.write_text(original.replace(required, replacement))
-                self.assertEqual(len(check_scheduled_alerts(root)), 1)
-                first.write_text(original)
-
-            watchdog.write_text(watchdog_source.replace(f'- "{names[0]}"\n', ""))
-            self.assertEqual(len(check_scheduled_alerts(root)), 1)
-
-            watchdog_mutations = (
-                ("actions: read", "actions: none"),
-                ("issues: write", "issues: read"),
-                ("uses: ./.github/actions/schedule-failure-issue", "uses: actions/checkout@v7"),
-                ("github-token: ${{ secrets.GITHUB_TOKEN }}", "github-token: missing"),
-                ("source-event: ${{ github.event.workflow_run.event }}", "source-event: wrong"),
-                ("source-ref-name: ${{ github.event.workflow_run.head_branch }}", "source-ref-name: wrong"),
-                ("source-sha: ${{ github.event.workflow_run.head_sha }}", "source-sha: wrong"),
-            )
-            for required, replacement in watchdog_mutations:
-                watchdog.write_text(watchdog_source.replace(required, replacement))
-                self.assertEqual(len(check_scheduled_alerts(root)), 1)
-
 def main() -> int:
     if sys.argv[1:] == ["--self-test"]:
         suite = unittest.defaultTestLoader.loadTestsFromTestCase(SelfTests)
@@ -594,7 +411,7 @@ def main() -> int:
         for error in errors:
             print(f"ERROR: {error}", file=sys.stderr)
         return 1
-    print("OK: e2e modules, runner selection, fuzz matrices, profiles, scheduled alerts, and bounded diagnostics are wired")
+    print("OK: e2e modules, runner selection, fuzz matrices, and profile guards are wired")
     return 0
 
 

scripts/s3-tests/run.sh

@@ -1028,11 +1028,10 @@ else
 fi
 
 # Run tests from s3tests/functional
-# Failure locals can contain multi-MiB request bodies; keep tracebacks without expanding local values.
 set +e
 S3TEST_CONF="${CONF_OUTPUT_PATH}" \
     tox -- \
-    -vv -ra --tb=long \
+    -vv -ra --showlocals --tb=long \
     --maxfail="${MAXFAIL}" \
     --timeout="${TEST_TIMEOUT}" \
     --junitxml="${ARTIFACTS_DIR}/junit.xml" \