feat:decom,rebalance

2026-08-06 05:17:42 +00:00 · 2025-04-15 10:03:01 +08:00
parent 38b7bbbc49
commit f624ec5ba6
14 changed files with 1971 additions and 368 deletions
@@ -50,6 +50,7 @@ use tracing::{error, info, warn};
 pub mod group;
 pub mod policys;
 pub mod pools;
+pub mod rebalance;
 pub mod service_account;
 pub mod sts;
 pub mod trace;
@@ -735,40 +736,6 @@ impl Operation for BackgroundHealStatusHandler {
    }
 }

-pub struct RebalanceStart {}
-
-#[async_trait::async_trait]
-impl Operation for RebalanceStart {
-    async fn call(&self, _req: S3Request<Body>, _params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
-        warn!("handle RebalanceStart");
-
-        return Err(s3_error!(NotImplemented));
-    }
-}
-
-// RebalanceStatus
-pub struct RebalanceStatus {}
-
-#[async_trait::async_trait]
-impl Operation for RebalanceStatus {
-    async fn call(&self, _req: S3Request<Body>, _params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
-        warn!("handle RebalanceStatus");
-
-        return Err(s3_error!(NotImplemented));
-    }
-}
-// RebalanceStop
-pub struct RebalanceStop {}
-
-#[async_trait::async_trait]
-impl Operation for RebalanceStop {
-    async fn call(&self, _req: S3Request<Body>, _params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
-        warn!("handle RebalanceStop");
-
-        return Err(s3_error!(NotImplemented));
-    }
-}
-
 #[cfg(test)]
 mod test {
    use ecstore::heal::heal_commands::HealOpts;
@@ -0,0 +1,238 @@
+use ecstore::{
+    new_object_layer_fn,
+    notification_sys::get_global_notification_sys,
+    rebalance::{DiskStat, RebalSaveOpt},
+    store_api::BucketOptions,
+    StorageAPI,
+};
+use http::{HeaderMap, StatusCode};
+use matchit::Params;
+use s3s::{header::CONTENT_TYPE, s3_error, Body, S3Request, S3Response, S3Result};
+use serde::{Deserialize, Serialize};
+use std::time::{Duration, SystemTime};
+use tracing::warn;
+
+use crate::admin::router::Operation;
+use ecstore::rebalance::RebalanceMeta;
+
+#[derive(Debug, Clone, Deserialize, Serialize)]
+pub struct RebalanceResp {
+    pub id: String,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize, Default)]
+pub struct RebalPoolProgress {
+    #[serde(rename = "objects")]
+    pub num_objects: u64,
+    #[serde(rename = "versions")]
+    pub num_versions: u64,
+    #[serde(rename = "bytes")]
+    pub bytes: u64,
+    #[serde(rename = "bucket")]
+    pub bucket: String,
+    #[serde(rename = "object")]
+    pub object: String,
+    #[serde(rename = "elapsed")]
+    pub elapsed: Duration,
+    #[serde(rename = "eta")]
+    pub eta: Duration,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize, Default)]
+pub struct RebalancePoolStatus {
+    #[serde(rename = "id")]
+    pub id: usize, // Pool index (zero-based)
+    #[serde(rename = "status")]
+    pub status: String, // Active if rebalance is running, empty otherwise
+    #[serde(rename = "used")]
+    pub used: f64, // Percentage used space
+    #[serde(rename = "progress")]
+    pub progress: Option<RebalPoolProgress>, // None when rebalance is not running
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize, Default)]
+pub struct RebalanceAdminStatus {
+    pub id: String, // Identifies the ongoing rebalance operation by a UUID
+    #[serde(rename = "pools")]
+    pub pools: Vec<RebalancePoolStatus>, // Contains all pools, including inactive
+    #[serde(rename = "stoppedAt")]
+    pub stopped_at: Option<SystemTime>, // Optional timestamp when rebalance was stopped
+}
+
+pub struct RebalanceStart {}
+
+#[async_trait::async_trait]
+impl Operation for RebalanceStart {
+    async fn call(&self, _req: S3Request<Body>, _params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
+        warn!("handle RebalanceStart");
+
+        let Some(store) = new_object_layer_fn() else {
+            return Err(s3_error!(InternalError, "Not init"));
+        };
+
+        if store.pools.len() == 1 {
+            return Err(s3_error!(NotImplemented));
+        }
+
+        if store.is_decommission_running().await {
+            return Err(s3_error!(
+                InternalError,
+                "Rebalance cannot be started, decommission is already in progress"
+            ));
+        }
+
+        if store.is_rebalance_started().await {
+            return Err(s3_error!(InternalError, "Rebalance already in progress"));
+        }
+
+        let bucket_infos = store
+            .list_bucket(&BucketOptions::default())
+            .await
+            .map_err(|e| s3_error!(InternalError, "Failed to list buckets: {}", e))?;
+
+        let buckets: Vec<String> = bucket_infos.into_iter().map(|bucket| bucket.name).collect();
+
+        let id = match store.init_rebalance_meta(buckets).await {
+            Ok(id) => id,
+            Err(e) => {
+                return Err(s3_error!(InternalError, "Failed to init rebalance meta: {}", e));
+            }
+        };
+
+        if let Some(notification_sys) = get_global_notification_sys() {
+            notification_sys.load_rebalance_meta(true).await;
+        }
+
+        let resp = RebalanceResp { id };
+        let data = serde_json::to_string(&resp).map_err(|e| s3_error!(InternalError, "Failed to serialize response: {}", e))?;
+
+        let mut header = HeaderMap::new();
+        header.insert(CONTENT_TYPE, "application/json".parse().unwrap());
+
+        Ok(S3Response::with_headers((StatusCode::OK, Body::from(data)), header))
+    }
+}
+
+// RebalanceStatus
+pub struct RebalanceStatus {}
+
+#[async_trait::async_trait]
+impl Operation for RebalanceStatus {
+    async fn call(&self, _req: S3Request<Body>, _params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
+        warn!("handle RebalanceStatus");
+
+        let Some(store) = new_object_layer_fn() else {
+            return Err(s3_error!(InternalError, "Not init"));
+        };
+
+        let mut meta = RebalanceMeta::new();
+        meta.load(store.pools[0].clone())
+            .await
+            .map_err(|e| s3_error!(InternalError, "Failed to load rebalance meta: {}", e))?;
+
+        // Compute disk usage percentage
+        let si = store.storage_info().await;
+        let mut disk_stats = vec![DiskStat::default(); store.pools.len()];
+
+        for disk in si.disks.iter() {
+            if disk.pool_index < 0 || disk_stats.len() <= disk.pool_index as usize {
+                continue;
+            }
+            disk_stats[disk.pool_index as usize].available_space += disk.available_space;
+            disk_stats[disk.pool_index as usize].total_space += disk.total_space;
+        }
+
+        let stop_time = meta.stopped_at;
+        let mut admin_status = RebalanceAdminStatus {
+            id: meta.id.clone(),
+            stopped_at: meta.stopped_at,
+            pools: vec![RebalancePoolStatus::default(); meta.pool_stats.len()],
+        };
+
+        for (i, ps) in meta.pool_stats.iter().enumerate() {
+            admin_status.pools[i] = RebalancePoolStatus {
+                id: i,
+                status: ps.info.status.to_string(),
+                used: (disk_stats[i].total_space - disk_stats[i].available_space) as f64 / disk_stats[i].total_space as f64,
+                progress: None,
+            };
+
+            if !ps.participating {
+                continue;
+            }
+
+            // Calculate total bytes to be rebalanced
+            let total_bytes_to_rebal = ps.init_capacity as f64 * meta.percent_free_goal - ps.init_free_space as f64;
+
+            let elapsed = if let Some(start_time) = ps.info.start_time {
+                SystemTime::now()
+                    .duration_since(start_time)
+                    .map_err(|e| s3_error!(InternalError, "Failed to calculate elapsed time: {}", e))?
+            } else {
+                return Err(s3_error!(InternalError, "Start time is not available"));
+            };
+
+            let eta = if ps.bytes > 0 {
+                Duration::from_secs_f64(total_bytes_to_rebal * elapsed.as_secs_f64() / ps.bytes as f64)
+            } else {
+                Duration::ZERO
+            };
+
+            let stop_time = ps.info.end_time.unwrap_or(stop_time.unwrap_or(SystemTime::now()));
+
+            let elapsed = if ps.info.end_time.is_some() || meta.stopped_at.is_some() {
+                stop_time
+                    .duration_since(ps.info.start_time.unwrap_or(stop_time))
+                    .unwrap_or_default()
+            } else {
+                elapsed
+            };
+
+            admin_status.pools[i].progress = Some(RebalPoolProgress {
+                num_objects: ps.num_objects,
+                num_versions: ps.num_versions,
+                bytes: ps.bytes,
+                bucket: ps.bucket.clone(),
+                object: ps.object.clone(),
+                elapsed,
+                eta,
+            });
+        }
+
+        let data =
+            serde_json::to_string(&admin_status).map_err(|e| s3_error!(InternalError, "Failed to serialize response: {}", e))?;
+        let mut header = HeaderMap::new();
+        header.insert(CONTENT_TYPE, "application/json".parse().unwrap());
+
+        Ok(S3Response::with_headers((StatusCode::OK, Body::from(data)), header))
+    }
+}
+
+// RebalanceStop
+pub struct RebalanceStop {}
+
+#[async_trait::async_trait]
+impl Operation for RebalanceStop {
+    async fn call(&self, _req: S3Request<Body>, _params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
+        warn!("handle RebalanceStop");
+
+        let Some(store) = new_object_layer_fn() else {
+            return Err(s3_error!(InternalError, "Not init"));
+        };
+
+        if let Some(notification_sys) = get_global_notification_sys() {
+            notification_sys.stop_rebalance().await;
+        }
+
+        store
+            .save_rebalance_stats(0, RebalSaveOpt::StoppedAt)
+            .await
+            .map_err(|e| s3_error!(InternalError, "Failed to stop rebalance: {}", e))?;
+
+        if let Some(notification_sys) = get_global_notification_sys() {
+            notification_sys.load_rebalance_meta(true).await;
+        }
+
+        return Err(s3_error!(NotImplemented));
+    }
+}
@@ -6,7 +6,7 @@ pub mod utils;
 use common::error::Result;
 // use ecstore::global::{is_dist_erasure, is_erasure};
 use handlers::{
-    group, policys, pools,
+    group, policys, pools, rebalance,
    service_account::{AddServiceAccount, DeleteServiceAccount, InfoServiceAccount, ListServiceAccount, UpdateServiceAccount},
    sts, user,
 };
@@ -94,17 +94,17 @@ pub fn make_admin_route() -> Result<impl S3Route> {
    r.insert(
        Method::POST,
        format!("{}{}", ADMIN_PREFIX, "/v3/rebalance/start").as_str(),
-        AdminOperation(&handlers::RebalanceStart {}),
+        AdminOperation(&rebalance::RebalanceStart {}),
    )?;
    r.insert(
        Method::GET,
        format!("{}{}", ADMIN_PREFIX, "/v3/rebalance/status").as_str(),
-        AdminOperation(&handlers::RebalanceStatus {}),
+        AdminOperation(&rebalance::RebalanceStatus {}),
    )?;
    r.insert(
        Method::POST,
        format!("{}{}", ADMIN_PREFIX, "/v3/rebalance/stop").as_str(),
-        AdminOperation(&handlers::RebalanceStop {}),
+        AdminOperation(&rebalance::RebalanceStop {}),
    )?;

    // Some APIs are only available in EC mode
@@ -2360,23 +2360,46 @@ impl Node for NodeService {
    }

    async fn stop_rebalance(&self, _request: Request<StopRebalanceRequest>) -> Result<Response<StopRebalanceResponse>, Status> {
-        let Some(_store) = new_object_layer_fn() else {
+        let Some(store) = new_object_layer_fn() else {
            return Ok(tonic::Response::new(StopRebalanceResponse {
                success: false,
                error_info: Some("errServerNotInitialized".to_string()),
            }));
        };

-        // todo
-        // store.stop_rebalance().await;
-        todo!()
+        let _ = store.stop_rebalance().await;
+        Ok(tonic::Response::new(StopRebalanceResponse {
+            success: true,
+            error_info: None,
+        }))
    }

    async fn load_rebalance_meta(
        &self,
-        _request: Request<LoadRebalanceMetaRequest>,
+        request: Request<LoadRebalanceMetaRequest>,
    ) -> Result<Response<LoadRebalanceMetaResponse>, Status> {
-        todo!()
+        let Some(store) = new_object_layer_fn() else {
+            return Ok(tonic::Response::new(LoadRebalanceMetaResponse {
+                success: false,
+                error_info: Some("errServerNotInitialized".to_string()),
+            }));
+        };
+
+        let LoadRebalanceMetaRequest { start_rebalance } = request.into_inner();
+
+        store.load_rebalance_meta().await.map_err(|err| {
+            error!("load_rebalance_meta err {:?}", err);
+            Status::internal(err.to_string())
+        })?;
+
+        if start_rebalance {
+            let store = store.clone();
+            tokio::spawn(async move {
+                store.start_rebalance().await;
+            });
+        }
+
+        unimplemented!()
    }

    async fn load_transition_tier_config(
@@ -21,8 +21,13 @@ use common::{
    globals::set_global_addr,
 };
 use config::{DEFAULT_ACCESS_KEY, DEFAULT_SECRET_KEY, RUSTFS_TLS_CERT, RUSTFS_TLS_KEY};
+use ecstore::bucket::metadata_sys::init_bucket_metadata_sys;
+use ecstore::config as ecconfig;
+use ecstore::config::GLOBAL_ConfigSys;
 use ecstore::heal::background_heal_ops::init_auto_heal;
+use ecstore::store_api::BucketOptions;
 use ecstore::utils::net::{self, get_available_port};
+use ecstore::StorageAPI;
 use ecstore::{
    endpoints::EndpointServerPools,
    heal::data_scanner::init_data_scanner,
@@ -379,11 +384,20 @@ async fn run(opt: config::Opt) -> Result<()> {
            Error::from_string(err.to_string())
        })?;

-    ECStore::init(store.clone()).await.map_err(|err| {
-        error!("ECStore init failed {:?}", &err);
-        Error::from_string(err.to_string())
-    })?;
-    debug!("init store success!");
+    ecconfig::init();
+    GLOBAL_ConfigSys.init(store.clone()).await?;
+
+    let buckets_list = store
+        .list_bucket(&BucketOptions {
+            no_metadata: true,
+            ..Default::default()
+        })
+        .await
+        .map_err(|err| Error::from_string(err.to_string()))?;
+
+    let buckets = buckets_list.into_iter().map(|v| v.name).collect();
+
+    init_bucket_metadata_sys(store.clone(), buckets).await;

    init_iam_sys(store.clone()).await?;

@@ -51,6 +51,7 @@ pub fn to_s3_error(err: Error) -> S3Error {
                object,
                version_id
            ),
+
            // extended
            StorageError::ObjectExistsAsDirectory(bucket, object) => {
                s3_error!(InvalidArgument, "Object exists on :{} as directory {}", bucket, object)
@@ -62,6 +63,7 @@ pub fn to_s3_error(err: Error) -> S3Error {
                s3_error!(SlowDown, "Storage resources are insufficient for the write operation")
            }
            StorageError::DecommissionNotStarted => s3_error!(InvalidArgument, "Decommission Not Started"),
+            StorageError::DecommissionAlreadyRunning => s3_error!(InternalError, "Decommission already running"),

            StorageError::VolumeNotFound(bucket) => {
                s3_error!(NoSuchBucket, "bucket not found {}", bucket)