feat(rio): rio_v2 is compatible with minio for storing data. (#3115)

* Set up a compatibility layer for replacing old Rio components with new ones. * fix(rio). compress range * feat(rio). Add the experimental feature rio_v2 to support minio data at the binary level. * feat(rio_v2): add sse-c test * test compression component * simple fix * fix minlz encode * fix metadata * fix kms key cache error * Update launch.json * ci: set nix crate download user agent * fix: gate obs pyroscope backend * ignore minio test * fix encrypt check * fix * fix * fix * Update object_usecase.rs * Update ci.yml * fix * ci add rio-v2 test * fix * ci fix * fix * Reconstructed into a more reasonable compatibility mode * fix * fix --------- Signed-off-by: houseme <housemecn@gmail.com> Signed-off-by: 唐小鸭 <tangtang1251@qq.com> Co-authored-by: houseme <housemecn@gmail.com> Co-authored-by: cxymds <Cxymds@qq.com> Co-authored-by: 安正超 <anzhengchao@gmail.com>
2026-08-08 06:13:14 +00:00 · 2026-06-08 19:59:14 +08:00
parent 9504dff595
commit f7724d223b
47 changed files with 8742 additions and 682 deletions
@@ -28,6 +28,21 @@

 use opentelemetry_sdk::{logs::SdkLoggerProvider, metrics::SdkMeterProvider, trace::SdkTracerProvider};

+#[cfg(all(
+    feature = "pyroscope",
+    any(target_os = "macos", all(target_os = "linux", target_env = "gnu", target_arch = "x86_64"))
+))]
+pub(crate) type ProfilingAgent = pyroscope::PyroscopeAgent<pyroscope::pyroscope::PyroscopeAgentRunning>;
+#[cfg(not(all(
+    feature = "pyroscope",
+    any(target_os = "macos", all(target_os = "linux", target_env = "gnu", target_arch = "x86_64"))
+)))]
+pub(crate) type ProfilingAgent = ();
+#[cfg(all(feature = "pyroscope", target_os = "linux", target_env = "gnu", target_arch = "x86_64"))]
+pub(crate) type MemoryProfilingAgent = pyroscope::PyroscopeAgent<pyroscope::pyroscope::PyroscopeAgentRunning>;
+#[cfg(not(all(feature = "pyroscope", target_os = "linux", target_env = "gnu", target_arch = "x86_64")))]
+pub(crate) type MemoryProfilingAgent = ();
+
 /// RAII guard that owns all active OpenTelemetry providers and the
 /// `tracing_appender` worker guard.
 ///
@@ -41,10 +56,8 @@ pub struct OtelGuard {
    pub(crate) meter_provider: Option<SdkMeterProvider>,
    /// Optional logger provider for OTLP log export.
    pub(crate) logger_provider: Option<SdkLoggerProvider>,
-    #[cfg(any(target_os = "macos", all(target_os = "linux", target_env = "gnu", target_arch = "x86_64")))]
-    pub(crate) profiling_agent: Option<pyroscope::PyroscopeAgent<pyroscope::pyroscope::PyroscopeAgentRunning>>,
-    #[cfg(all(target_os = "linux", target_env = "gnu", target_arch = "x86_64"))]
-    pub(crate) memory_profiling_agent: Option<pyroscope::PyroscopeAgent<pyroscope::pyroscope::PyroscopeAgentRunning>>,
+    pub(crate) profiling_agent: Option<ProfilingAgent>,
+    pub(crate) memory_profiling_agent: Option<MemoryProfilingAgent>,
    /// Handle to the background log-cleanup task; aborted on drop.
    pub(crate) cleanup_handle: Option<tokio::task::JoinHandle<()>>,
    /// Worker guard that keeps the non-blocking `tracing_appender` thread
@@ -59,12 +72,10 @@ impl std::fmt::Debug for OtelGuard {
        let mut s = f.debug_struct("OtelGuard");
        s.field("tracer_provider", &self.tracer_provider.is_some())
            .field("meter_provider", &self.meter_provider.is_some())
-            .field("logger_provider", &self.logger_provider.is_some());
-        #[cfg(any(target_os = "macos", all(target_os = "linux", target_env = "gnu", target_arch = "x86_64")))]
-        s.field("profiling_agent", &self.profiling_agent.is_some());
-        #[cfg(all(target_os = "linux", target_env = "gnu", target_arch = "x86_64"))]
-        s.field("memory_profiling_agent", &self.memory_profiling_agent.is_some());
-        s.field("cleanup_handle", &self.cleanup_handle.is_some())
+            .field("logger_provider", &self.logger_provider.is_some())
+            .field("profiling_agent", &self.profiling_agent.is_some())
+            .field("memory_profiling_agent", &self.memory_profiling_agent.is_some())
+            .field("cleanup_handle", &self.cleanup_handle.is_some())
            .field("tracing_guard", &self.tracing_guard.is_some())
            .field("stdout_guard", &self.stdout_guard.is_some())
            .finish()
@@ -95,7 +106,10 @@ impl Drop for OtelGuard {
            eprintln!("Logger shutdown error: {err:?}");
        }

-        #[cfg(any(target_os = "macos", all(target_os = "linux", target_env = "gnu", target_arch = "x86_64")))]
+        #[cfg(all(
+            feature = "pyroscope",
+            any(target_os = "macos", all(target_os = "linux", target_env = "gnu", target_arch = "x86_64"))
+        ))]
        if let Some(agent) = self.profiling_agent.take() {
            match agent.stop() {
                Err(err) => eprintln!("Profiling agent stop error: {err:?}"),
@@ -105,7 +119,7 @@ impl Drop for OtelGuard {
            }
        }

-        #[cfg(all(target_os = "linux", target_env = "gnu", target_arch = "x86_64"))]
+        #[cfg(all(feature = "pyroscope", target_os = "linux", target_env = "gnu", target_arch = "x86_64"))]
        if let Some(agent) = self.memory_profiling_agent.take() {
            match agent.stop() {
                Err(err) => eprintln!("Memory profiling agent stop error: {err:?}"),
@@ -155,9 +155,7 @@ fn init_stdout_only(_config: &OtelConfig, logger_level: &str, is_production: boo
        tracer_provider: None,
        meter_provider: None,
        logger_provider: None,
-        #[cfg(any(target_os = "macos", all(target_os = "linux", target_env = "gnu", target_arch = "x86_64")))]
        profiling_agent: None,
-        #[cfg(all(target_os = "linux", target_env = "gnu", target_arch = "x86_64"))]
        memory_profiling_agent: None,
        tracing_guard: Some(guard),
        stdout_guard: None,
@@ -291,9 +289,7 @@ fn init_file_logging_internal(
        tracer_provider: None,
        meter_provider: None,
        logger_provider: None,
-        #[cfg(any(target_os = "macos", all(target_os = "linux", target_env = "gnu", target_arch = "x86_64")))]
        profiling_agent: None,
-        #[cfg(all(target_os = "linux", target_env = "gnu", target_arch = "x86_64"))]
        memory_profiling_agent: None,
        tracing_guard: Some(guard),
        stdout_guard,
@@ -40,7 +40,7 @@ use crate::cleaner::types::FileMatchMode;
 use crate::config::OtelConfig;
 use crate::global::set_observability_metric_enabled;
 use crate::telemetry::filter::build_env_filter;
-use crate::telemetry::guard::OtelGuard;
+use crate::telemetry::guard::{MemoryProfilingAgent, OtelGuard, ProfilingAgent};
 use crate::telemetry::local::spawn_cleanup_task;
 use crate::telemetry::recorder::Recorder;
 use crate::telemetry::resource::build_resource;
@@ -164,10 +164,7 @@ pub(super) fn init_observability_http(
    // ── Meter provider (HTTP) ─────────────────────────────────────────────────
    let meter_provider = build_meter_provider(&metric_ep, config, res.clone(), &service_name, use_stdout)?;

-    #[cfg(any(target_os = "macos", all(target_os = "linux", target_env = "gnu", target_arch = "x86_64")))]
    let profiling_agent = init_profiler(config);
-
-    #[cfg(all(target_os = "linux", target_env = "gnu", target_arch = "x86_64"))]
    let memory_profiling_agent = init_memory_profiler(config);

    // ── Logger Logic ──────────────────────────────────────────────────────────
@@ -317,9 +314,7 @@ pub(super) fn init_observability_http(
        tracer_provider,
        meter_provider,
        logger_provider,
-        #[cfg(any(target_os = "macos", all(target_os = "linux", target_env = "gnu", target_arch = "x86_64")))]
        profiling_agent,
-        #[cfg(all(target_os = "linux", target_env = "gnu", target_arch = "x86_64"))]
        memory_profiling_agent,
        tracing_guard,
        stdout_guard,
@@ -496,8 +491,11 @@ fn build_logger_provider(
 /// Returns `None` when profiling export is disabled, when no usable
 /// profiling endpoint is configured, or when building or starting the agent
 /// fails.
-#[cfg(any(target_os = "macos", all(target_os = "linux", target_env = "gnu", target_arch = "x86_64")))]
-fn init_profiler(config: &OtelConfig) -> Option<pyroscope::PyroscopeAgent<pyroscope::pyroscope::PyroscopeAgentRunning>> {
+#[cfg(all(
+    feature = "pyroscope",
+    any(target_os = "macos", all(target_os = "linux", target_env = "gnu", target_arch = "x86_64"))
+))]
+fn init_profiler(config: &OtelConfig) -> Option<ProfilingAgent> {
    use pyroscope::backend::{BackendConfig, PprofConfig, pprof_backend};
    use pyroscope::pyroscope::PyroscopeAgentBuilder;
    use rustfs_config::VERSION;
@@ -534,6 +532,14 @@ fn init_profiler(config: &OtelConfig) -> Option<pyroscope::PyroscopeAgent<pyrosc
    }
 }

+#[cfg(not(all(
+    feature = "pyroscope",
+    any(target_os = "macos", all(target_os = "linux", target_env = "gnu", target_arch = "x86_64"))
+)))]
+fn init_profiler(_config: &OtelConfig) -> Option<ProfilingAgent> {
+    None
+}
+
 /// Initialise a Pyroscope agent for continuous **memory** profiling via jemalloc.
 ///
 /// This is only available on `linux + gnu + x86_64` where tikv-jemallocator
@@ -541,8 +547,8 @@ fn init_profiler(config: &OtelConfig) -> Option<pyroscope::PyroscopeAgent<pyrosc
 ///
 /// Returns `None` when profiling export is disabled, the endpoint is missing,
 /// jemalloc profiling is not activated, or the agent fails to build/start.
-#[cfg(all(target_os = "linux", target_env = "gnu", target_arch = "x86_64"))]
-fn init_memory_profiler(config: &OtelConfig) -> Option<pyroscope::PyroscopeAgent<pyroscope::pyroscope::PyroscopeAgentRunning>> {
+#[cfg(all(feature = "pyroscope", target_os = "linux", target_env = "gnu", target_arch = "x86_64"))]
+fn init_memory_profiler(config: &OtelConfig) -> Option<MemoryProfilingAgent> {
    use pyroscope::backend::jemalloc_backend;
    use pyroscope::pyroscope::PyroscopeAgentBuilder;
    use rustfs_config::VERSION;
@@ -588,6 +594,11 @@ fn init_memory_profiler(config: &OtelConfig) -> Option<pyroscope::PyroscopeAgent
    }
 }

+#[cfg(not(all(feature = "pyroscope", target_os = "linux", target_env = "gnu", target_arch = "x86_64")))]
+fn init_memory_profiler(_config: &OtelConfig) -> Option<MemoryProfilingAgent> {
+    None
+}
+
 /// Create a stdout periodic metrics reader for the given interval.
 ///
 /// This helper is primarily used for local development and diagnostics when