test(ci): guard watchdog alert wiring

.github/actions/schedule-failure-issue/action.yml

@@ -14,9 +14,10 @@
 
 name: "Schedule Failure Issue"
 description: >-
-  Open (or update) a tracking issue when a scheduled workflow run fails.
+  Open (or update) a tracking issue when a scheduled workflow run fails or
+  does not complete normally.
   Dedupes by workflow name: if an open issue titled
-  "[scheduled-failure] <workflow name>" already exists, the failure is
+  "[scheduled-failure] <workflow name>" already exists, the result is
   appended as a comment; otherwise a new issue is created. This is the
   single alerting mechanism for all scheduled pipelines (backlog#1149 ci-8).
 
@@ -38,6 +39,26 @@ inputs:
       Set to an empty string to skip labeling.
     required: false
     default: "infrastructure"
+  source-run-id:
+    description: "Run ID to report. Defaults to the current workflow run."
+    required: false
+    default: ${{ github.run_id }}
+  source-run-attempt:
+    description: "Run attempt to report. Defaults to the current attempt."
+    required: false
+    default: ${{ github.run_attempt }}
+  source-event:
+    description: "Trigger event of the run being reported."
+    required: false
+    default: ${{ github.event_name }}
+  source-ref-name:
+    description: "Ref name of the run being reported."
+    required: false
+    default: ${{ github.ref_name }}
+  source-sha:
+    description: "Commit SHA of the run being reported."
+    required: false
+    default: ${{ github.sha }}
 
 runs:
   using: "composite"
@@ -48,17 +69,21 @@ runs:
         GH_TOKEN: ${{ inputs.github-token }}
         WORKFLOW_NAME: ${{ inputs.workflow-name }}
         ISSUE_LABEL: ${{ inputs.label }}
+        SOURCE_RUN_ID: ${{ inputs.source-run-id }}
+        SOURCE_RUN_ATTEMPT: ${{ inputs.source-run-attempt }}
+        SOURCE_EVENT: ${{ inputs.source-event }}
+        SOURCE_REF_NAME: ${{ inputs.source-ref-name }}
+        SOURCE_SHA: ${{ inputs.source-sha }}
       run: |
         set -euo pipefail
 
         title="[scheduled-failure] ${WORKFLOW_NAME}"
-        run_url="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
+        run_url="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${SOURCE_RUN_ID}"
 
-        # Failed job names for this run attempt. The alert job runs while the
-        # run as a whole is still in progress, so inspect the jobs that have
-        # already completed with a non-success conclusion.
+        # Inspect the reported run attempt. It can be the current in-workflow
+        # failure or a completed run observed by the external watchdog.
         failed_jobs="$(gh api \
-          "repos/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/attempts/${GITHUB_RUN_ATTEMPT}/jobs" \
+          "repos/${GITHUB_REPOSITORY}/actions/runs/${SOURCE_RUN_ID}/attempts/${SOURCE_RUN_ATTEMPT}/jobs" \
           --paginate \
           --jq '.jobs[]
                 | select(.conclusion == "failure" or .conclusion == "timed_out" or .conclusion == "cancelled")
@@ -68,13 +93,13 @@ runs:
         fi
 
         body="$(cat <<EOF
-        Scheduled run of **${WORKFLOW_NAME}** failed.
+        Scheduled run of **${WORKFLOW_NAME}** did not complete successfully.
 
-        - Run: ${run_url} (attempt ${GITHUB_RUN_ATTEMPT})
-        - Event: \`${GITHUB_EVENT_NAME}\`
-        - Ref: \`${GITHUB_REF_NAME}\` @ \`${GITHUB_SHA}\`
+        - Run: ${run_url} (attempt ${SOURCE_RUN_ATTEMPT})
+        - Event: \`${SOURCE_EVENT}\`
+        - Ref: \`${SOURCE_REF_NAME}\` @ \`${SOURCE_SHA}\`
 
-        Failed jobs:
+        Non-success jobs:
         ${failed_jobs}
         EOF
         )"

.github/workflows/build.yml

@@ -1032,3 +1032,23 @@ jobs:
 
           echo "🎉 Released $TAG successfully!"
           echo "📄 Release URL: ${{ needs.create-release.outputs.release_url }}"
+
+  alert-on-failure:
+    name: Alert on scheduled failure
+    needs: [build-check, prepare-platform-matrix, build-rustfs, build-summary]
+    if: >-
+      always() && github.event_name == 'schedule' &&
+      (contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled'))
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    permissions:
+      contents: read
+      issues: write
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
+      - name: Open or update failure-tracking issue
+        uses: ./.github/actions/schedule-failure-issue
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/ci.yml

@@ -1032,3 +1032,37 @@ jobs:
           path: artifacts/s3tests-single/**
           if-no-files-found: ignore
           retention-days: 3
+
+  alert-on-failure:
+    name: Alert on scheduled failure
+    needs:
+      - typos
+      - quick-checks
+      - test-and-lint
+      - test-ilm-integration-serial
+      - test-and-lint-rio-v2
+      - test-and-lint-protocols
+      - build-rustfs-debug-binary
+      - build-rustfs-debug-binary-rio-v2
+      - uring-integration
+      - e2e-tests
+      - e2e-full
+      - e2e-tests-rio-v2
+      - s3-implemented-tests
+      - s3-lifecycle-behavior-tests
+    if: >-
+      always() && github.event_name == 'schedule' &&
+      (contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled'))
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    permissions:
+      contents: read
+      issues: write
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
+      - name: Open or update failure-tracking issue
+        uses: ./.github/actions/schedule-failure-issue
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/minio-interop.yml

@@ -121,3 +121,21 @@ jobs:
           cargo nextest run --run-ignored ignored-only --no-tests=fail \
             -p "$INTEROP_PACKAGE" --features "$INTEROP_FEATURES" \
             -E "$INTEROP_FILTER"
+
+  alert-on-failure:
+    name: Alert on scheduled failure
+    needs: [minio-interop]
+    if: always() && github.event_name == 'schedule' && contains(needs.*.result, 'failure')
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    permissions:
+      contents: read
+      issues: write
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
+      - name: Open or update failure-tracking issue
+        uses: ./.github/actions/schedule-failure-issue
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/nightly-gnu.yml

@@ -194,3 +194,23 @@ jobs:
 
       - name: Run HA leader failover live checks (three-node Raft cluster in Docker)
         run: bash scripts/test/vault_ha_kms_live.sh
+
+  alert-on-failure:
+    name: Alert on scheduled failure
+    needs: [build, kms-vault-lane, kms-vault-ha-failover]
+    if: >-
+      always() && github.event_name == 'schedule' &&
+      (contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled'))
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    permissions:
+      contents: read
+      issues: write
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
+      - name: Open or update failure-tracking issue
+        uses: ./.github/actions/schedule-failure-issue
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/scheduled-validation-watchdog.yml

@@ -0,0 +1,63 @@
+# Copyright 2024 RustFS Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: Scheduled Validation Watchdog
+
+on:
+  workflow_run:
+    workflows:
+      - "Security Audit"
+      - "Build and Release"
+      - "Continuous Integration"
+      - "coverage"
+      - "e2e-nightly"
+      - "e2e-s3tests"
+      - "Fuzz"
+      - "mint"
+      - "minio-interop"
+      - "Nightly GNU Build"
+      - "Performance A/B"
+      - "Runner Hygiene"
+    types: [completed]
+
+permissions:
+  contents: read
+
+jobs:
+  alert-on-incomplete-run:
+    name: Alert on incomplete scheduled run
+    if: >-
+      github.event.workflow_run.event == 'schedule' &&
+      github.event.workflow_run.conclusion != 'success' &&
+      github.event.workflow_run.conclusion != 'failure'
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    permissions:
+      actions: read
+      contents: read
+      issues: write
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
+      - name: Open or update incomplete-run issue
+        uses: ./.github/actions/schedule-failure-issue
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          workflow-name: ${{ github.event.workflow_run.name }}
+          source-run-id: ${{ github.event.workflow_run.id }}
+          source-run-attempt: ${{ github.event.workflow_run.run_attempt }}
+          source-event: ${{ github.event.workflow_run.event }}
+          source-ref-name: ${{ github.event.workflow_run.head_branch }}
+          source-sha: ${{ github.event.workflow_run.head_sha }}

crates/config/src/constants/runtime.rs

@@ -57,13 +57,6 @@ pub const DEFAULT_MAX_IO_EVENTS_PER_TICK: usize = 1024;
 pub const DEFAULT_EVENT_INTERVAL: u32 = 61;
 pub const DEFAULT_RNG_SEED: Option<u64> = None; // None means random
 
-/// Dedicated blocking thread pool for fsync/fdatasync operations.
-/// When > 1, fsync operations are isolated from the main blocking pool to
-/// prevent device-bound fsync from starving read operations (pread/stat/open).
-/// Default 0 means auto (no isolation, use main runtime).
-pub const ENV_FSYNC_BLOCKING_THREADS: &str = "RUSTFS_RUNTIME_FSYNC_BLOCKING_THREADS";
-pub const DEFAULT_FSYNC_BLOCKING_THREADS: usize = 0;
-
 // Dial9 Tokio Telemetry Default values
 pub const DEFAULT_RUNTIME_DIAL9_ENABLED: bool = false; // Disabled by default
 pub const DEFAULT_RUNTIME_DIAL9_OUTPUT_DIR: &str = "/var/log/rustfs/telemetry";

crates/ecstore/src/disk/os.rs

@@ -315,7 +315,7 @@ pub async fn fsync_dir(dir: impl AsRef<Path>) -> io::Result<()> {
     #[cfg(unix)]
     {
         let dir = dir.as_ref().to_path_buf();
-        fsync_spawn_blocking(move || fsync_dir_std(dir)).await?
+        tokio::task::spawn_blocking(move || fsync_dir_std(dir)).await?
     }
 
     #[cfg(not(unix))]
@@ -683,7 +683,7 @@ async fn fsync_open_dst_dir_group(group: &DstDirFsyncGroup) -> io::Result<()> {
     #[cfg(test)]
     let dir = group.dir.clone();
     let dir_file = group.dir_file.clone();
-    fsync_spawn_blocking(move || {
+    tokio::task::spawn_blocking(move || {
         #[cfg(test)]
         {
             if let Some(kind) = fsync_dir_recorder::take_grouped_failure(&dir) {
@@ -1080,44 +1080,6 @@ const TEST_GLOBAL_FILE_SYNCS: usize = 64;
 
 static FILE_SYNC_PERMITS: LazyLock<Semaphore> = LazyLock::new(|| Semaphore::new(global_file_sync_limit()));
 static DISK_FILE_SYNC_LIMITERS: LazyLock<Mutex<HashMap<PathBuf, Weak<Semaphore>>>> = LazyLock::new(|| Mutex::new(HashMap::new()));
-
-/// Dedicated tokio runtime for fsync/fdatasync blocking operations. When
-/// configured with >1 threads, isolates device-bound fsync from the main
-/// blocking pool so reads (pread/stat/open) are not starved. `None` means
-/// fall back to the main runtime (zero behavior change).
-static FSYNC_RUNTIME: LazyLock<Option<tokio::runtime::Runtime>> = LazyLock::new(|| {
-    let threads =
-        rustfs_utils::get_env_usize(rustfs_config::ENV_FSYNC_BLOCKING_THREADS, rustfs_config::DEFAULT_FSYNC_BLOCKING_THREADS);
-    if threads <= 1 {
-        return None;
-    }
-    let mut builder = tokio::runtime::Builder::new_multi_thread();
-    builder
-        .worker_threads(num_cpus::get().min(8))
-        .max_blocking_threads(threads)
-        .thread_name("rustfs-fsync")
-        .thread_stack_size(512 * 1024)
-        .enable_all();
-    match builder.build() {
-        Ok(rt) => {
-            tracing::info!(threads, "fsync dedicated blocking pool enabled");
-            Some(rt)
-        }
-        Err(err) => {
-            tracing::warn!(%err, "failed to build fsync runtime, falling back to main pool");
-            None
-        }
-    }
-});
-
-/// Spawn a blocking task on the fsync-dedicated runtime if configured,
-/// otherwise fall back to the main tokio blocking pool.
-fn fsync_spawn_blocking<T: Send + 'static>(f: impl FnOnce() -> T + Send + 'static) -> tokio::task::JoinHandle<T> {
-    match FSYNC_RUNTIME.as_ref() {
-        Some(rt) => rt.spawn_blocking(f),
-        None => tokio::task::spawn_blocking(f),
-    }
-}
 static DISK_VOLUME_MUTATION_LOCKS: LazyLock<Mutex<HashMap<PathBuf, Weak<RwLock<()>>>>> =
     LazyLock::new(|| Mutex::new(HashMap::new()));
 type NamespaceMutationLock = AsyncMutex<()>;
@@ -1255,7 +1217,7 @@ where
     F: FnOnce() -> io::Result<T> + Send + 'static,
 {
     let (disk_permit, global_permit) = acquire_file_sync_permits(disk_permits).await?;
-    let result = fsync_spawn_blocking(move || {
+    let result = tokio::task::spawn_blocking(move || {
         let _disk_permit = disk_permit;
         work()
     })
@@ -2184,7 +2146,7 @@ async fn run_blocking_namespace_file_sync_operation_with_global<T: Send + 'stati
         wait_started,
     );
     let disk_permit = admission.disk_permit.clone();
-    let result = fsync_spawn_blocking(move || {
+    let result = tokio::task::spawn_blocking(move || {
         let _lease = lease;
         let _disk_permit = disk_permit;
         operation()

crates/ecstore/src/store/heal.rs

@@ -297,16 +297,10 @@ impl ECStore {
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::bucket::metadata_sys;
     use crate::core::pools::{PoolDecommissionInfo, PoolStatus};
-    use crate::disk::{DeleteOptions, DiskOption, format::FormatV3, new_disk};
-    use crate::layout::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints};
-    use crate::runtime::instance::InstanceContext;
-    use crate::storage_api_contracts::bucket::{BucketOperations, MakeBucketOptions};
-    use crate::storage_api_contracts::object::{ObjectIO as _, ObjectOperations};
+    use crate::disk::{DiskOption, format::FormatV3, new_disk};
+    use crate::layout::endpoints::{Endpoints, PoolEndpoints};
     use crate::store::init_format::{load_format_erasure, save_format_file};
-    use crate::store::init_local_disks_with_instance_ctx;
-    use tokio_util::sync::CancellationToken;
 
     async fn minimal_heal_pool(pool_idx: usize) -> Arc<Sets> {
         let format = FormatV3::new(1, 1);
@@ -353,51 +347,6 @@ mod tests {
         }
     }
 
-    async fn multi_pool_heal_store() -> (tempfile::TempDir, Arc<ECStore>, CancellationToken) {
-        let temp_dir = tempfile::tempdir().expect("multi-pool heal test directory should be created");
-        let mut pool_endpoints = Vec::new();
-        for pool_index in 0..2 {
-            let mut endpoints = Vec::new();
-            for disk_index in 0..4 {
-                let disk_path = temp_dir.path().join(format!("pool{pool_index}-disk{disk_index}"));
-                tokio::fs::create_dir_all(&disk_path)
-                    .await
-                    .expect("multi-pool heal test disk should be created");
-                let mut endpoint = Endpoint::try_from(disk_path.to_str().expect("disk path should be utf8"))
-                    .expect("test endpoint should parse");
-                endpoint.set_pool_index(pool_index);
-                endpoint.set_set_index(0);
-                endpoint.set_disk_index(disk_index);
-                endpoints.push(endpoint);
-            }
-            pool_endpoints.push(PoolEndpoints {
-                legacy: false,
-                set_count: 1,
-                drives_per_set: 4,
-                endpoints: Endpoints::from(endpoints),
-                cmd_line: format!("heal-owner-pool-{pool_index}"),
-                platform: "test".to_string(),
-            });
-        }
-
-        let endpoint_pools = EndpointServerPools::from(pool_endpoints);
-        let instance_ctx = Arc::new(InstanceContext::new());
-        init_local_disks_with_instance_ctx(&instance_ctx, endpoint_pools.clone())
-            .await
-            .expect("multi-pool local disks should initialize");
-        let shutdown = CancellationToken::new();
-        let store = ECStore::new_with_instance_ctx(
-            "127.0.0.1:0".parse().expect("test address should parse"),
-            endpoint_pools,
-            shutdown.clone(),
-            instance_ctx,
-        )
-        .await
-        .expect("multi-pool test store should initialize");
-        metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
-        (temp_dir, store, shutdown)
-    }
-
     #[tokio::test]
     async fn heal_object_pool_scope_selects_only_requested_pool() {
         let store = minimal_heal_store().await;
@@ -557,204 +506,6 @@ mod tests {
         }
     }
 
-    #[tokio::test]
-    #[serial_test::serial]
-    async fn unscoped_heal_object_suspended_owner_semantics() {
-        let (_temp_dir, store, shutdown) = multi_pool_heal_store().await;
-        let bucket = format!("heal-owner-{}", Uuid::new_v4().simple());
-        let active_object = "active-owner";
-        let suspended_only_object = "suspended-only";
-        let duplicate_object = "duplicate-owner";
-        let marker_object = "marker-owner";
-        let quorum_object = "quorum-owner";
-        store
-            .make_bucket(&bucket, &MakeBucketOptions::default())
-            .await
-            .expect("bucket should be created in all pools");
-
-        let mut active_reader = PutObjReader::from_vec(b"active owner".to_vec());
-        store.pools[0]
-            .put_object(&bucket, active_object, &mut active_reader, &ObjectOptions::default())
-            .await
-            .expect("active owner object should be written");
-        let active_disks = store.pools[0].disk_set[0].disks.read().await.clone();
-        let missing_active_disk = active_disks[0].clone().expect("active disk should be online");
-        missing_active_disk
-            .delete(
-                &bucket,
-                active_object,
-                DeleteOptions {
-                    recursive: true,
-                    immediate: true,
-                    ..Default::default()
-                },
-            )
-            .await
-            .expect("active owner shard should be removed for repair");
-        assert!(
-            missing_active_disk.read_xl(&bucket, active_object, false).await.is_err(),
-            "the active owner fixture must start with one missing metadata copy"
-        );
-
-        let mut suspended_reader = PutObjReader::from_vec(b"suspended owner".to_vec());
-        store.pools[1]
-            .put_object(&bucket, suspended_only_object, &mut suspended_reader, &ObjectOptions::default())
-            .await
-            .expect("suspended owner object should be written");
-        for (pool_index, mod_time) in [1_i64, 2_i64].into_iter().enumerate() {
-            let mut duplicate_reader = PutObjReader::from_vec(format!("duplicate-pool-{pool_index}").into_bytes());
-            store.pools[pool_index]
-                .put_object(
-                    &bucket,
-                    duplicate_object,
-                    &mut duplicate_reader,
-                    &ObjectOptions {
-                        mod_time: Some(OffsetDateTime::UNIX_EPOCH + time::Duration::seconds(mod_time)),
-                        ..Default::default()
-                    },
-                )
-                .await
-                .expect("duplicate owner object should be written");
-        }
-        let history_version = Uuid::new_v4();
-        let mut history_reader = PutObjReader::from_vec(b"marker history".to_vec());
-        store.pools[0]
-            .put_object(
-                &bucket,
-                marker_object,
-                &mut history_reader,
-                &ObjectOptions {
-                    versioned: true,
-                    version_id: Some(history_version.to_string()),
-                    mod_time: Some(OffsetDateTime::UNIX_EPOCH + time::Duration::seconds(1)),
-                    ..Default::default()
-                },
-            )
-            .await
-            .expect("versioned marker history should be written");
-        store.pools[0]
-            .delete_object(
-                &bucket,
-                marker_object,
-                ObjectOptions {
-                    versioned: true,
-                    mod_time: Some(OffsetDateTime::UNIX_EPOCH + time::Duration::seconds(2)),
-                    ..Default::default()
-                },
-            )
-            .await
-            .expect("delete marker should be written");
-        let mut quorum_reader = PutObjReader::from_vec(b"quorum boundary".to_vec());
-        store.pools[0]
-            .put_object(&bucket, quorum_object, &mut quorum_reader, &ObjectOptions::default())
-            .await
-            .expect("quorum boundary object should be written");
-        {
-            let mut pool_meta = store.pool_meta.write().await;
-            let mut next = PoolMeta::new(&store.pools, &pool_meta);
-            next.pools[1].decommission = Some(PoolDecommissionInfo {
-                start_time: Some(OffsetDateTime::UNIX_EPOCH),
-                ..Default::default()
-            });
-            *pool_meta = next;
-        }
-
-        let (_, duplicate_owner) = store
-            .get_latest_object_info_with_idx(&bucket, duplicate_object, &ObjectOptions::default())
-            .await
-            .expect("duplicate owner should resolve");
-        assert_eq!(duplicate_owner, 1, "latest duplicate must win when all pools are eligible");
-        let (_, active_duplicate_owner) = store
-            .get_latest_object_info_with_idx(
-                &bucket,
-                duplicate_object,
-                &ObjectOptions {
-                    skip_decommissioned: true,
-                    ..Default::default()
-                },
-            )
-            .await
-            .expect("active duplicate owner should resolve");
-        assert_eq!(
-            active_duplicate_owner, 0,
-            "suspended duplicate must be excluded from active owner selection"
-        );
-        let (marker_info, marker_owner) = store
-            .get_latest_object_info_with_idx(
-                &bucket,
-                marker_object,
-                &ObjectOptions {
-                    skip_decommissioned: true,
-                    versioned: true,
-                    ..Default::default()
-                },
-            )
-            .await
-            .expect("latest delete marker should resolve");
-        assert_eq!(marker_owner, 0);
-        assert!(marker_info.delete_marker, "latest version must preserve delete-marker semantics");
-
-        let (active_result, active_err) = store
-            .handle_heal_object(&bucket, active_object, "", &HealOpts::default())
-            .await
-            .expect("unscoped active-owner heal should complete");
-        assert_eq!(active_result.object, active_object);
-        assert!(active_err.is_none(), "active owner must be selected even with a suspended pool");
-        assert!(
-            missing_active_disk.read_xl(&bucket, active_object, false).await.is_ok(),
-            "active owner heal must write the missing disk metadata: result={active_result:?}, err={active_err:?}"
-        );
-        assert!(
-            store.pools[1]
-                .get_object_info(&bucket, active_object, &ObjectOptions::default())
-                .await
-                .is_err(),
-            "the suspended pool must not be written for an active-owner object"
-        );
-
-        let (suspended_result, suspended_err) = store
-            .handle_heal_object(&bucket, suspended_only_object, "", &HealOpts::default())
-            .await
-            .expect("unscoped suspended-only heal should return a terminal result");
-        assert!(suspended_result.object.is_empty());
-        assert!(matches!(suspended_err, Some(Error::FileNotFound)));
-        assert!(
-            store.pools[1]
-                .get_object_info(&bucket, suspended_only_object, &ObjectOptions::default())
-                .await
-                .is_ok(),
-            "suspended-only data must remain untouched when unscoped heal reports absent"
-        );
-
-        let (_, explicit_err) = store
-            .handle_heal_object(
-                &bucket,
-                suspended_only_object,
-                "",
-                &HealOpts {
-                    pool: Some(1),
-                    ..Default::default()
-                },
-            )
-            .await
-            .expect("explicit suspended-owner heal should return a mapped error");
-        assert!(matches!(explicit_err, Some(Error::SlowDown)));
-
-        let original_quorum_disks = store.pools[0].disk_set[0].disks.read().await.clone();
-        let surviving_quorum_disk = original_quorum_disks[3].clone();
-        *store.pools[0].disk_set[0].disks.write().await = vec![None, None, None, surviving_quorum_disk];
-        let (_, quorum_err) = store
-            .handle_heal_object(&bucket, quorum_object, "", &HealOpts::default())
-            .await
-            .expect("quorum boundary heal should return a mapped result");
-        *store.pools[0].disk_set[0].disks.write().await = original_quorum_disks;
-        assert!(
-            matches!(quorum_err, Some(Error::ErasureReadQuorum)),
-            "quorum-boundary heal must preserve quorum error, got {quorum_err:?}"
-        );
-        shutdown.cancel();
-    }
-
     #[tokio::test]
     async fn handle_heal_format_continues_after_a_pool_error() {
         let canonical_format = FormatV3::new(1, 3);

scripts/check_test_wiring.py

@@ -15,6 +15,20 @@ from pathlib import Path
 
 
 ROOT = Path(__file__).resolve().parents[1]
+SCHEDULED_ALERT_WORKFLOWS = (
+    ".github/workflows/audit.yml",
+    ".github/workflows/build.yml",
+    ".github/workflows/ci.yml",
+    ".github/workflows/coverage.yml",
+    ".github/workflows/e2e-replication-nightly.yml",
+    ".github/workflows/e2e-s3tests.yml",
+    ".github/workflows/fuzz.yml",
+    ".github/workflows/mint.yml",
+    ".github/workflows/minio-interop.yml",
+    ".github/workflows/nightly-gnu.yml",
+    ".github/workflows/performance-ab.yml",
+    ".github/workflows/runner-hygiene.yml",
+)
 
 
 def words(value: str) -> set[str]:
@@ -252,6 +266,79 @@ def check_profile_definitions(root: Path) -> list[str]:
     return errors
 
 
+def check_scheduled_alerts(root: Path) -> list[str]:
+    errors: list[str] = []
+    for relative in SCHEDULED_ALERT_WORKFLOWS:
+        path = root / relative
+        try:
+            lines = path.read_text().splitlines()
+        except FileNotFoundError:
+            errors.append(f"{relative}: missing scheduled validation workflow")
+            continue
+
+        try:
+            start = lines.index("  alert-on-failure:") + 1
+        except ValueError:
+            errors.append(f"{relative}: missing alert-on-failure job")
+            continue
+        end = next(
+            (index for index in range(start, len(lines)) if re.fullmatch(r"  [A-Za-z0-9_-]+:", lines[index])),
+            len(lines),
+        )
+        job = "\n".join(line.split("#", 1)[0] for line in lines[start:end])
+        required = (
+            "always()",
+            "github.event_name == 'schedule'",
+            "contains(needs.*.result, 'failure')",
+            "issues: write",
+            "uses: ./.github/actions/schedule-failure-issue",
+            "github-token: ${{ secrets.GITHUB_TOKEN }}",
+        )
+        missing = [token for token in required if token not in job]
+        if missing:
+            errors.append(f"{relative}: alert-on-failure missing {', '.join(missing)}")
+
+    watchdog_path = root / ".github/workflows/scheduled-validation-watchdog.yml"
+    try:
+        watchdog = "\n".join(
+            line.split("#", 1)[0] for line in watchdog_path.read_text().splitlines()
+        )
+    except FileNotFoundError:
+        errors.append(".github/workflows/scheduled-validation-watchdog.yml: missing completion watchdog")
+        return errors
+    for relative in SCHEDULED_ALERT_WORKFLOWS:
+        path = root / relative
+        if not path.is_file():
+            continue
+        source = path.read_text()
+        match = re.search(r"^name:\s*[\"']?([^\"'\n]+)", source, re.MULTILINE)
+        if not match:
+            errors.append(f"{relative}: missing workflow name")
+        elif f'- "{match.group(1).strip()}"' not in watchdog:
+            errors.append(f"{relative}: missing from scheduled completion watchdog")
+    required = (
+        "github.event.workflow_run.event == 'schedule'",
+        "github.event.workflow_run.conclusion != 'success'",
+        "github.event.workflow_run.conclusion != 'failure'",
+        "actions: read",
+        "issues: write",
+        "uses: ./.github/actions/schedule-failure-issue",
+        "github-token: ${{ secrets.GITHUB_TOKEN }}",
+        "workflow-name: ${{ github.event.workflow_run.name }}",
+        "source-event: ${{ github.event.workflow_run.event }}",
+        "source-ref-name: ${{ github.event.workflow_run.head_branch }}",
+        "source-sha: ${{ github.event.workflow_run.head_sha }}",
+        "source-run-id: ${{ github.event.workflow_run.id }}",
+        "source-run-attempt: ${{ github.event.workflow_run.run_attempt }}",
+    )
+    missing = [token for token in required if token not in watchdog]
+    if missing:
+        errors.append(
+            ".github/workflows/scheduled-validation-watchdog.yml: missing " + ", ".join(missing)
+        )
+    return errors
+
+
 def check_profile_listing(root: Path, profile: str, listing: Path) -> list[str]:
     try:
         expected_digest = profile_selection(root, profile)
@@ -281,6 +368,7 @@ def validate(root: Path) -> list[str]:
     errors.extend(check_runner_selection(root))
     errors.extend(check_s3_tests_runner(root))
     errors.extend(check_profile_definitions(root))
+    errors.extend(check_scheduled_alerts(root))
     return errors
 
 
@@ -363,6 +451,7 @@ class SelfTests(unittest.TestCase):
                 mock.patch(__name__ + ".check_fuzz_targets", return_value=[]),
                 mock.patch(__name__ + ".check_runner_selection", return_value=[]),
                 mock.patch(__name__ + ".check_profile_definitions", return_value=[]),
+                mock.patch(__name__ + ".check_scheduled_alerts", return_value=[]),
             ):
                 self.assertEqual(len(validate(root)), 1)
 
@@ -413,6 +502,75 @@ class SelfTests(unittest.TestCase):
             with mock.patch.object(sys, "platform", "linux"):
                 self.assertEqual(len(check_profile_listing(root, "e2e-full", listing)), 1)
 
+    def test_scheduled_alerts_require_completion_watchdog(self) -> None:
+        with tempfile.TemporaryDirectory() as tmp:
+            root = Path(tmp)
+            alert = (
+                "  alert-on-failure:\n"
+                "    if: always() && github.event_name == 'schedule' && "
+                "contains(needs.*.result, 'failure')\n"
+                "    permissions:\n"
+                "      issues: write\n"
+                "    steps:\n"
+                "      - uses: ./.github/actions/schedule-failure-issue\n"
+                "        with:\n"
+                "          github-token: ${{ secrets.GITHUB_TOKEN }}\n"
+            )
+            names: list[str] = []
+            for relative in SCHEDULED_ALERT_WORKFLOWS:
+                path = root / relative
+                path.parent.mkdir(parents=True, exist_ok=True)
+                names.append(path.stem)
+                path.write_text(f'name: "{path.stem}"\n{alert}')
+            watchdog = root / ".github/workflows/scheduled-validation-watchdog.yml"
+            watchdog_source = (
+                "\n".join(f'- "{name}"' for name in names)
+                + "\ngithub.event.workflow_run.event == 'schedule'\n"
+                + "github.event.workflow_run.conclusion != 'success'\n"
+                + "github.event.workflow_run.conclusion != 'failure'\n"
+                + "actions: read\n"
+                + "issues: write\n"
+                + "uses: ./.github/actions/schedule-failure-issue\n"
+                + "github-token: ${{ secrets.GITHUB_TOKEN }}\n"
+                + "workflow-name: ${{ github.event.workflow_run.name }}\n"
+                + "source-event: ${{ github.event.workflow_run.event }}\n"
+                + "source-ref-name: ${{ github.event.workflow_run.head_branch }}\n"
+                + "source-sha: ${{ github.event.workflow_run.head_sha }}\n"
+                + "source-run-id: ${{ github.event.workflow_run.id }}\n"
+                + "source-run-attempt: ${{ github.event.workflow_run.run_attempt }}\n"
+            )
+            watchdog.write_text(watchdog_source)
+            self.assertEqual(check_scheduled_alerts(root), [])
+
+            first = root / SCHEDULED_ALERT_WORKFLOWS[0]
+            mutations = (
+                ("contains(needs.*.result, 'failure')", "false"),
+                ("issues: write", "issues: read"),
+                ("uses: ./.github/actions/schedule-failure-issue", "uses: actions/checkout@v7"),
+                ("github-token: ${{ secrets.GITHUB_TOKEN }}", "github-token: missing"),
+            )
+            for required, replacement in mutations:
+                original = first.read_text()
+                first.write_text(original.replace(required, replacement))
+                self.assertEqual(len(check_scheduled_alerts(root)), 1)
+                first.write_text(original)
+
+            watchdog.write_text(watchdog_source.replace(f'- "{names[0]}"\n', ""))
+            self.assertEqual(len(check_scheduled_alerts(root)), 1)
+
+            watchdog_mutations = (
+                ("actions: read", "actions: none"),
+                ("issues: write", "issues: read"),
+                ("uses: ./.github/actions/schedule-failure-issue", "uses: actions/checkout@v7"),
+                ("github-token: ${{ secrets.GITHUB_TOKEN }}", "github-token: missing"),
+                ("source-event: ${{ github.event.workflow_run.event }}", "source-event: wrong"),
+                ("source-ref-name: ${{ github.event.workflow_run.head_branch }}", "source-ref-name: wrong"),
+                ("source-sha: ${{ github.event.workflow_run.head_sha }}", "source-sha: wrong"),
+            )
+            for required, replacement in watchdog_mutations:
+                watchdog.write_text(watchdog_source.replace(required, replacement))
+                self.assertEqual(len(check_scheduled_alerts(root)), 1)
+
 def main() -> int:
     if sys.argv[1:] == ["--self-test"]:
         suite = unittest.defaultTestLoader.loadTestsFromTestCase(SelfTests)
@@ -436,7 +594,7 @@ def main() -> int:
         for error in errors:
             print(f"ERROR: {error}", file=sys.stderr)
         return 1
-    print("OK: e2e modules, runner selection, fuzz matrices, profiles, and bounded diagnostics are wired")
+    print("OK: e2e modules, runner selection, fuzz matrices, profiles, scheduled alerts, and bounded diagnostics are wired")
     return 0