feat(ecstore): pin bitrot algorithms with a startup self-test (HS-11) (#6165)

feat(ecstore): pin bitrot algorithms with a startup self-test A drifted HighwayHash implementation fails silently: every shard reads back corrupt, heal rewrites healthy data, and cross-platform clusters disagree about which copy is good. Mirror MinIO's bitrotSelfTest by verifying, once at process start: - known-answer digests for HighwayHash256S / HighwayHash256SLegacy over a deterministic 4096-byte xorshift64* payload, plus the externally verifiable FIPS SHA-256 "abc" vector guarding the HashAlgorithm plumbing itself; - an end-to-end roundtrip per streaming variant (encode -> size formula -> bitrot_verify -> BitrotReader read-back), over full blocks and a partial tail; - tamper detection: one flipped byte in the final data block and one in the leading hash must both be rejected as a hash mismatch, not by an incidental read error. The check costs microseconds and runs inline in init_background_service_runtime before any shard can be written or verified. Outcome surfaces as one structured bitrot_selftest log event, the rustfs_bitrot_selftest_status gauge (1=passed / 0=failed / 2=skipped), a bitrotSelftest field on the admin server-info response, and RUSTFS_BITROT_SELFTEST_STRICT=on turns a failure into a startup error (MinIO Fatal parity; the default only degrades the status so a bad build cannot brick an existing fleet on upgrade). Closes rustfs/backlog#1873 (HS-11). Co-authored-by: heihutu <heihutu@gmail.com>
2026-08-19 02:56:18 +00:00 · 2026-08-17 15:04:23 +08:00
parent d091554ffe
commit 89e2513205
9 changed files with 535 additions and 16 deletions
@@ -417,6 +417,13 @@ struct SystemAdminDiscovery {
 struct ServerInfoResponse {
    info: InfoMessage,
    admin_discovery: SystemAdminDiscovery,
+    /// Startup bitrot algorithm self-test outcome (rustfs/backlog#1873):
+    /// `passed` (algorithms verified at boot), `failed` (a drifted hash
+    /// implementation — the process is serving with degraded integrity
+    /// checking unless `RUSTFS_BITROT_SELFTEST_STRICT` aborted it), or
+    /// `unknown` (not yet run or disabled).
+    #[serde(rename = "bitrotSelftest")]
+    bitrot_selftest: &'static str,
 }

 #[derive(Serialize)]
@@ -433,6 +440,14 @@ fn system_admin_discovery(usecase: &DefaultAdminUsecase) -> SystemAdminDiscovery
    }
 }

+fn bitrot_selftest_status_str() -> &'static str {
+    match crate::bitrot_selftest::bitrot_selftest_passed() {
+        Some(true) => "passed",
+        Some(false) => "failed",
+        None => "unknown",
+    }
+}
+
 #[async_trait::async_trait]
 impl Operation for ServerInfoHandler {
    async fn call(&self, req: S3Request<Body>, _params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
@@ -464,6 +479,7 @@ impl Operation for ServerInfoHandler {
        let response = ServerInfoResponse {
            info,
            admin_discovery: system_admin_discovery(&usecase),
+            bitrot_selftest: bitrot_selftest_status_str(),
        };

        let data = serde_json::to_vec(&response).map_err(|e| {
@@ -1535,6 +1551,18 @@ mod tests {
        );
    }

+    /// The startup bitrot self-test outcome must surface in server info as one
+    /// of three closed-set strings, never an internal enum or a null
+    /// (rustfs/backlog#1873). This test pins the string mapping; whether the
+    /// process-global cell holds Some(true)/Some(false)/None is owned by
+    /// `crate::bitrot_selftest`'s own tests.
+    #[test]
+    fn bitrot_selftest_status_str_is_a_closed_set_of_operators_strings() {
+        let rendered = super::bitrot_selftest_status_str();
+        assert!(matches!(rendered, "passed" | "failed" | "unknown"));
+        assert_eq!(super::bitrot_selftest_status_str(), rendered);
+    }
+
    #[test]
    fn server_info_response_exposes_admin_discovery_paths() {
        let usecase = DefaultAdminUsecase::without_context();
@@ -1556,6 +1584,7 @@ mod tests {
                pools: None,
            },
            admin_discovery: system_admin_discovery(&usecase),
+            bitrot_selftest: super::bitrot_selftest_status_str(),
        };

        let value = serde_json::to_value(response).expect("server info response should serialize");
@@ -0,0 +1,181 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Startup bitrot algorithm self-test (rustfs/backlog#1873).
+//!
+//! A drifted hash implementation fails silently in production: every shard
+//! reads back "corrupt", heal rewrites healthy data, and cross-platform
+//! clusters disagree about which copy is good. [`run_startup_bitrot_self_test`]
+//! pins the algorithms once at process start — the check itself runs in well
+//! under a millisecond on 4 KiB, so it executes inline before background
+//! services come up and the result is published before the server accepts
+//! traffic.
+//!
+//! Outcome surface:
+//! - one structured `bitrot_selftest` log event (`passed`/`failed`/`skipped`),
+//! - the `rustfs_bitrot_selftest_status` gauge (1=passed, 0=failed, 2=skipped),
+//! - [`bitrot_selftest_passed`] for admin/health surfaces,
+//! - `RUSTFS_BITROT_SELFTEST_STRICT=on` turns a failure into a startup error
+//!   (MinIO `bitrotSelfTest` Fatal parity); the default only degrades the
+//!   status so a bad build cannot brick an existing fleet on upgrade.
+
+use crate::storage_api::startup::background::{BitrotSelfTestError, bitrot_self_test};
+use metrics::gauge;
+use std::future::Future;
+use std::io;
+use std::sync::atomic::{AtomicU8, Ordering};
+use std::time::Instant;
+use tracing::{debug, error, info};
+
+const LOG_COMPONENT_MAIN: &str = "main";
+const LOG_SUBSYSTEM_STARTUP: &str = "startup";
+const EVENT_BITROT_SELFTEST: &str = "bitrot_selftest";
+const METRIC_BITROT_SELFTEST_STATUS: &str = "rustfs_bitrot_selftest_status";
+
+/// Gauge values for [`METRIC_BITROT_SELFTEST_STATUS`].
+const STATUS_PASSED: f64 = 1.0;
+const STATUS_FAILED: f64 = 0.0;
+const STATUS_SKIPPED: f64 = 2.0;
+
+/// Internal cell values for [`BITROT_SELF_TEST_STATUS`].
+const STATUS_CELL_UNSET: u8 = 0;
+const STATUS_CELL_PASSED: u8 = 1;
+const STATUS_CELL_FAILED: u8 = 2;
+
+static BITROT_SELF_TEST_STATUS: AtomicU8 = AtomicU8::new(STATUS_CELL_UNSET);
+
+/// Last recorded self-test outcome: `None` before the first run, then
+/// `Some(true)` on a passing check and `Some(false)` on a failed one (a
+/// skipped check never publishes, so it cannot read as a pass). The cell is
+/// last-writer-wins rather than set-once: production runs the self-test once,
+/// and last-writer-wins keeps tests that exercise both outcomes
+/// order-independent.
+pub fn bitrot_selftest_passed() -> Option<bool> {
+    match BITROT_SELF_TEST_STATUS.load(Ordering::Acquire) {
+        STATUS_CELL_UNSET => None,
+        STATUS_CELL_PASSED => Some(true),
+        STATUS_CELL_FAILED => Some(false),
+        _ => None,
+    }
+}
+
+/// Run the bitrot self-test and publish the outcome. In strict mode a failure
+/// is returned as an error so the caller aborts startup.
+pub(crate) async fn run_startup_bitrot_self_test(enabled: bool, strict: bool) -> io::Result<()> {
+    run_startup_bitrot_self_test_with(enabled, strict, bitrot_self_test).await
+}
+
+async fn run_startup_bitrot_self_test_with<F, Fut>(enabled: bool, strict: bool, run_check: F) -> io::Result<()>
+where
+    F: FnOnce() -> Fut,
+    Fut: Future<Output = Result<(), BitrotSelfTestError>>,
+{
+    if !enabled {
+        gauge!(METRIC_BITROT_SELFTEST_STATUS).set(STATUS_SKIPPED);
+        debug!(
+            target: "rustfs::main::run",
+            event = EVENT_BITROT_SELFTEST,
+            component = LOG_COMPONENT_MAIN,
+            subsystem = LOG_SUBSYSTEM_STARTUP,
+            state = "skipped",
+            reason = "disabled",
+            "Bitrot self-test skipped"
+        );
+        return Ok(());
+    }
+
+    let started = Instant::now();
+    match run_check().await {
+        Ok(()) => {
+            BITROT_SELF_TEST_STATUS.store(STATUS_CELL_PASSED, Ordering::Release);
+            gauge!(METRIC_BITROT_SELFTEST_STATUS).set(STATUS_PASSED);
+            info!(
+                target: "rustfs::main::run",
+                event = EVENT_BITROT_SELFTEST,
+                component = LOG_COMPONENT_MAIN,
+                subsystem = LOG_SUBSYSTEM_STARTUP,
+                state = "passed",
+                duration_us = started.elapsed().as_micros() as u64,
+                "Bitrot self-test passed"
+            );
+        }
+        Err(err) => {
+            BITROT_SELF_TEST_STATUS.store(STATUS_CELL_FAILED, Ordering::Release);
+            gauge!(METRIC_BITROT_SELFTEST_STATUS).set(STATUS_FAILED);
+            error!(
+                target: "rustfs::main::run",
+                event = EVENT_BITROT_SELFTEST,
+                component = LOG_COMPONENT_MAIN,
+                subsystem = LOG_SUBSYSTEM_STARTUP,
+                state = "failed",
+                duration_us = started.elapsed().as_micros() as u64,
+                error = %err,
+                "Bitrot self-test failed"
+            );
+            if strict {
+                return Err(io::Error::other(format!("bitrot self-test failed: {err}")));
+            }
+        }
+    }
+    Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::{BITROT_SELF_TEST_STATUS, STATUS_CELL_UNSET, bitrot_selftest_passed, run_startup_bitrot_self_test_with};
+    use crate::storage_api::startup::background::BitrotSelfTestError;
+    use std::future::ready;
+    use std::sync::atomic::Ordering;
+
+    fn failing_check() -> impl Future<Output = Result<(), BitrotSelfTestError>> {
+        ready(Err(BitrotSelfTestError::RoundtripReadback {
+            algorithm: "HighwayHash256S",
+        }))
+    }
+
+    /// All scenarios run sequentially inside one test: the status cell is
+    /// process-global, so parallel per-scenario tests would race the reset and
+    /// read each other's outcomes (the exact order-dependent flake class this
+    /// module exists to avoid).
+    #[tokio::test]
+    async fn startup_self_test_publishes_outcome_and_strict_gates_abort() {
+        BITROT_SELF_TEST_STATUS.store(STATUS_CELL_UNSET, Ordering::Release);
+
+        // Skipped: publishes nothing, never fails, never aborts.
+        run_startup_bitrot_self_test_with(false, true, || async { Ok(()) })
+            .await
+            .expect("a disabled self-test must not fail even in strict mode");
+        assert_eq!(bitrot_selftest_passed(), None, "a skipped run must leave the status unset");
+
+        // Passing: publishes Some(true), never fails.
+        run_startup_bitrot_self_test_with(true, false, || async { Ok(()) })
+            .await
+            .expect("a passing check must never fail startup");
+        assert_eq!(bitrot_selftest_passed(), Some(true), "a passing run must publish Some(true)");
+
+        // Failing, non-strict: publishes Some(false) but startup continues.
+        run_startup_bitrot_self_test_with(true, false, failing_check)
+            .await
+            .expect("a failed check must not abort startup in non-strict mode");
+        assert_eq!(bitrot_selftest_passed(), Some(false), "a failing run must publish Some(false)");
+
+        // Failing, strict: startup error carries the failure and the published
+        // outcome stays a failure.
+        let err = run_startup_bitrot_self_test_with(true, true, failing_check)
+            .await
+            .expect_err("strict mode must turn a failed check into a startup error");
+        assert!(err.to_string().contains("bitrot self-test failed"));
+        assert_eq!(bitrot_selftest_passed(), Some(false));
+    }
+}
@@ -76,6 +76,7 @@ pub mod allocator_reclaim;
 pub mod app;
 pub mod auth;
 pub mod auth_keystone;
+pub(crate) mod bitrot_selftest;
 pub mod capacity;
 pub mod cluster_snapshot;
 pub mod config;
@@ -33,6 +33,8 @@ pub(crate) const ENV_SCANNER_ENABLED: &str = "RUSTFS_SCANNER_ENABLED";
 pub(crate) const ENV_SCANNER_ENABLED_DEPRECATED: &str = "RUSTFS_ENABLE_SCANNER";
 pub(crate) const ENV_HEAL_ENABLED: &str = "RUSTFS_HEAL_ENABLED";
 pub(crate) const ENV_HEAL_ENABLED_DEPRECATED: &str = "RUSTFS_ENABLE_HEAL";
+pub(crate) const ENV_BITROT_SELFTEST_ENABLE: &str = "RUSTFS_BITROT_SELFTEST_ENABLE";
+pub(crate) const ENV_BITROT_SELFTEST_STRICT: &str = "RUSTFS_BITROT_SELFTEST_STRICT";

 static AUDIT_MODULE_ENABLED: AtomicBool = AtomicBool::new(rustfs_config::DEFAULT_AUDIT_ENABLE);
 static NOTIFY_MODULE_ENABLED: AtomicBool = AtomicBool::new(rustfs_config::DEFAULT_NOTIFY_ENABLE);
@@ -47,6 +49,18 @@ pub(crate) fn heal_enabled_from_env() -> bool {
    get_env_bool_with_aliases(ENV_HEAL_ENABLED, &[ENV_HEAL_ENABLED_DEPRECATED], true)
 }

+/// Whether the startup bitrot algorithm self-test runs, defaulting to on
+/// (rustfs/backlog#1873).
+pub(crate) fn bitrot_selftest_enabled_from_env() -> bool {
+    rustfs_utils::get_env_bool(ENV_BITROT_SELFTEST_ENABLE, true)
+}
+
+/// Whether a failed bitrot self-test aborts startup instead of only logging
+/// and exposing a failed status, defaulting to off.
+pub(crate) fn bitrot_selftest_strict_from_env() -> bool {
+    rustfs_utils::get_env_bool(ENV_BITROT_SELFTEST_STRICT, false)
+}
+
 /// Last published audit-module state.
 pub fn is_audit_module_enabled() -> bool {
    AUDIT_MODULE_ENABLED.load(Ordering::Relaxed)
@@ -12,7 +12,10 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.

-use crate::module_switches::{heal_enabled_from_env, scanner_enabled_from_env};
+use crate::bitrot_selftest::run_startup_bitrot_self_test;
+use crate::module_switches::{
+    bitrot_selftest_enabled_from_env, bitrot_selftest_strict_from_env, heal_enabled_from_env, scanner_enabled_from_env,
+};
 use crate::storage_api::startup::background::{ECStore, set_workload_admission_snapshot_provider};
 use crate::workload_admission::RustFsWorkloadAdmissionSnapshotProvider;
 use rustfs_concurrency::WorkloadAdmissionSnapshotProvider;
@@ -27,6 +30,12 @@ const LOG_SUBSYSTEM_STARTUP: &str = "startup";
 const EVENT_BACKGROUND_SERVICES_CONFIGURED: &str = "background_services_configured";

 pub(crate) async fn init_background_service_runtime(store: Arc<ECStore>) -> Result<bool> {
+    // Pin the bitrot algorithms before anything can write or verify a shard:
+    // the check costs well under a millisecond, and in strict mode a drifted
+    // build must abort here rather than after it has touched data
+    // (rustfs/backlog#1873).
+    run_startup_bitrot_self_test(bitrot_selftest_enabled_from_env(), bitrot_selftest_strict_from_env()).await?;
+
    let _ = create_ahm_services_cancel_token();

    let enable_scanner = scanner_enabled_from_env();
@@ -569,6 +569,10 @@ pub(crate) mod ecstore_erasure {
    pub(crate) use rustfs_ecstore::api::erasure::{BitrotReader, Erasure};
 }

+/// Startup bitrot algorithm self-test (rustfs/backlog#1873), re-exported for
+/// the root facade's background-startup section.
+pub(crate) use rustfs_ecstore::api::erasure::{BitrotSelfTestError, bitrot_self_test};
+
 pub(crate) mod ecstore_storage {
    #[cfg(test)]
    pub(crate) use rustfs_ecstore::api::storage::init_local_disks;
@@ -214,7 +214,9 @@ pub(crate) mod startup {
    }

    pub(crate) mod background {
-        pub(crate) use crate::storage::storage_api::{ECStore, set_workload_admission_snapshot_provider};
+        pub(crate) use crate::storage::storage_api::{
+            BitrotSelfTestError, ECStore, bitrot_self_test, set_workload_admission_snapshot_provider,
+        };
    }

    pub(crate) mod bucket_metadata {