refactor(ecstore): Optimize memory usage for object integrity verification

Change the object integrity verification from reading all data to streaming processing to avoid memory overflow caused by large objects.

Modify the TLS key log check to use environment variables directly instead of configuration constants.

Add memory limits for object data reading in the AHM module.

Signed-off-by: junxiang Mu <1948535941@qq.com>

crates/ecstore/src/set_disk.rs

@@ -5354,9 +5354,10 @@ impl StorageAPI for SetDisks {
 
     #[tracing::instrument(skip(self))]
     async fn verify_object_integrity(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<()> {
-        let mut get_object_reader =
-            <Self as ObjectIO>::get_object_reader(self, bucket, object, None, HeaderMap::new(), opts).await?;
-        let _ = get_object_reader.read_all().await?;
+        let get_object_reader = <Self as ObjectIO>::get_object_reader(self, bucket, object, None, HeaderMap::new(), opts).await?;
+        // Stream to sink to avoid loading entire object into memory during verification
+        let mut reader = get_object_reader.stream;
+        tokio::io::copy(&mut reader, &mut tokio::io::sink()).await?;
         Ok(())
     }
 }

crates/ecstore/src/sets.rs

@@ -882,11 +882,15 @@ impl StorageAPI for Sets {
         unimplemented!()
     }
 
-    #[tracing::instrument(skip(self))]
+    #[tracing::instrument(level = "debug", skip(self))]
     async fn verify_object_integrity(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<()> {
-        self.get_disks_by_key(object)
-            .verify_object_integrity(bucket, object, opts)
-            .await
+        let gor = self.get_object_reader(bucket, object, None, HeaderMap::new(), opts).await?;
+        let mut reader = gor.stream;
+
+        // Stream data to sink instead of reading all into memory to prevent OOM
+        tokio::io::copy(&mut reader, &mut tokio::io::sink()).await?;
+
+        Ok(())
     }
 }
 

crates/ecstore/src/store.rs

@@ -2238,9 +2238,10 @@ impl StorageAPI for ECStore {
     }
 
     async fn verify_object_integrity(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<()> {
-        let mut get_object_reader =
-            <Self as ObjectIO>::get_object_reader(self, bucket, object, None, HeaderMap::new(), opts).await?;
-        let _ = get_object_reader.read_all().await?;
+        let get_object_reader = <Self as ObjectIO>::get_object_reader(self, bucket, object, None, HeaderMap::new(), opts).await?;
+        // Stream to sink to avoid loading entire object into memory during verification
+        let mut reader = get_object_reader.stream;
+        tokio::io::copy(&mut reader, &mut tokio::io::sink()).await?;
         Ok(())
     }
 }