From 124a9eb5218635ea6e5f5ff7d591cae4b34ea905 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Fri, 6 Mar 2026 10:08:12 +0100 Subject: [PATCH] admin api: export node statistics as structured json --- doc/api/garage-admin-v2.json | 135 ++++++++++++++++++++++++++++++++--- src/api/admin/api.rs | 44 ++++++++++++ src/api/admin/node.rs | 128 +++++++++++++++++++-------------- 3 files changed, 244 insertions(+), 63 deletions(-) diff --git a/doc/api/garage-admin-v2.json b/doc/api/garage-admin-v2.json index 82796e3a..a5ae37f8 100644 --- a/doc/api/garage-admin-v2.json +++ b/doc/api/garage-admin-v2.json @@ -3072,7 +3072,8 @@ ], "properties": { "dbEngine": { - "type": "string" + "type": "string", + "description": "database engine used for metadata" }, "garageFeatures": { "type": [ @@ -3081,16 +3082,23 @@ ], "items": { "type": "string" - } + }, + "description": "build-time features enabled for this garage release" }, "garageVersion": { - "type": "string" + "type": "string", + "description": "garage version running on this node" + }, + "hostname": { + "type": "string", + "description": "hostname of this node" }, "nodeId": { "type": "string" }, "rustVersion": { - "type": "string" + "type": "string", + "description": "rustc version with which this garage release was compiled" } } }, @@ -3100,8 +3108,20 @@ "freeform" ], "properties": { + "blockManagerStats": { + "$ref": "#/components/schemas/NodeBlockManagerStats", + "description": "block manager statistics" + }, "freeform": { - "type": "string" + "type": "string", + "description": "node statistics as a free-form string, kept for compatibility with nodes\nrunning older v2.x versions of garage" + }, + "tableStats": { + "type": "array", + "items": { + "$ref": "#/components/schemas/NodeTableStats" + }, + "description": "metadata table statistics" } } }, @@ -3402,7 +3422,8 @@ ], "properties": { "dbEngine": { - "type": "string" + "type": "string", + "description": "database engine used for metadata" }, "garageFeatures": { "type": [ @@ -3411,16 +3432,23 @@ ], "items": { "type": "string" - } + }, + "description": "build-time features enabled for this garage release" }, "garageVersion": { - "type": "string" + "type": "string", + "description": "garage version running on this node" + }, + "hostname": { + "type": "string", + "description": "hostname of this node" }, "nodeId": { "type": "string" }, "rustVersion": { - "type": "string" + "type": "string", + "description": "rustc version with which this garage release was compiled" } } }, @@ -3456,8 +3484,20 @@ "freeform" ], "properties": { + "blockManagerStats": { + "$ref": "#/components/schemas/NodeBlockManagerStats", + "description": "block manager statistics" + }, "freeform": { - "type": "string" + "type": "string", + "description": "node statistics as a free-form string, kept for compatibility with nodes\nrunning older v2.x versions of garage" + }, + "tableStats": { + "type": "array", + "items": { + "$ref": "#/components/schemas/NodeTableStats" + }, + "description": "metadata table statistics" } } }, @@ -3796,6 +3836,34 @@ } } }, + "NodeBlockManagerStats": { + "type": "object", + "required": [ + "rcEntries", + "resyncQueueLen", + "resyncErrors" + ], + "properties": { + "rcEntries": { + "type": "integer", + "format": "int64", + "description": "number of reference counter entries", + "minimum": 0 + }, + "resyncErrors": { + "type": "integer", + "format": "int64", + "description": "number of blocks with resync errors", + "minimum": 0 + }, + "resyncQueueLen": { + "type": "integer", + "format": "int64", + "description": "number of blocks in the resync queue", + "minimum": 0 + } + } + }, "NodeResp": { "type": "object", "required": [ @@ -3959,6 +4027,53 @@ } ] }, + "NodeTableStats": { + "type": "object", + "required": [ + "tableName", + "items", + "merkleItems", + "merkleQueueLen", + "insertQueueLen", + "gcQueueLen" + ], + "properties": { + "gcQueueLen": { + "type": "integer", + "format": "int64", + "description": "number of items in the garbage collection queue", + "minimum": 0 + }, + "insertQueueLen": { + "type": "integer", + "format": "int64", + "description": "number of items in the remote insert queue", + "minimum": 0 + }, + "items": { + "type": "integer", + "format": "int64", + "description": "number of items stored in metadata table", + "minimum": 0 + }, + "merkleItems": { + "type": "integer", + "format": "int64", + "description": "size of the merkle tree representing all items in the table", + "minimum": 0 + }, + "merkleQueueLen": { + "type": "integer", + "format": "int64", + "description": "number of items in the merkle tree update queue", + "minimum": 0 + }, + "tableName": { + "type": "string", + "description": "name of metadata table" + } + } + }, "NodeUpdateTrackers": { "type": "object", "required": [ diff --git a/src/api/admin/api.rs b/src/api/admin/api.rs index fb91354c..b375ecb9 100644 --- a/src/api/admin/api.rs +++ b/src/api/admin/api.rs @@ -1123,9 +1123,16 @@ pub struct LocalGetNodeInfoRequest; #[serde(rename_all = "camelCase")] pub struct LocalGetNodeInfoResponse { pub node_id: String, + /// hostname of this node + #[serde(default)] + pub hostname: String, + /// garage version running on this node pub garage_version: String, + /// build-time features enabled for this garage release pub garage_features: Option>, + /// rustc version with which this garage release was compiled pub rust_version: String, + /// database engine used for metadata pub db_engine: String, } @@ -1135,8 +1142,45 @@ pub struct LocalGetNodeInfoResponse { pub struct LocalGetNodeStatisticsRequest; #[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "camelCase")] pub struct LocalGetNodeStatisticsResponse { + /// node statistics as a free-form string, kept for compatibility with nodes + /// running older v2.x versions of garage pub freeform: String, + /// metadata table statistics + #[serde(default)] + pub table_stats: Vec, + /// block manager statistics + #[serde(default)] + pub block_manager_stats: NodeBlockManagerStats, +} + +#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "camelCase")] +pub struct NodeTableStats { + /// name of metadata table + pub table_name: String, + /// number of items stored in metadata table + pub items: u64, + /// size of the merkle tree representing all items in the table + pub merkle_items: u64, + /// number of items in the merkle tree update queue + pub merkle_queue_len: u64, + /// number of items in the remote insert queue + pub insert_queue_len: u64, + /// number of items in the garbage collection queue + pub gc_queue_len: u64, +} + +#[derive(Debug, Clone, Serialize, Deserialize, ToSchema, Default)] +#[serde(rename_all = "camelCase")] +pub struct NodeBlockManagerStats { + /// number of reference counter entries + pub rc_entries: u64, + /// number of blocks in the resync queue + pub resync_queue_len: u64, + /// number of blocks with resync errors + pub resync_errors: u64, } // ---- CreateMetadataSnapshot ---- diff --git a/src/api/admin/node.rs b/src/api/admin/node.rs index 12963aab..6e46a46d 100644 --- a/src/api/admin/node.rs +++ b/src/api/admin/node.rs @@ -22,8 +22,12 @@ impl RequestHandler for LocalGetNodeInfoRequest { garage: &Arc, _admin: &Admin, ) -> Result { + let sys_status = garage.system.local_status(); + let hostname = sys_status.hostname.unwrap_or_default().to_string(); + Ok(LocalGetNodeInfoResponse { node_id: hex::encode(garage.system.id), + hostname, garage_version: garage_util::version::garage_version().to_string(), garage_features: garage_util::version::garage_features() .map(|features| features.iter().map(ToString::to_string).collect()), @@ -57,46 +61,58 @@ impl RequestHandler for LocalGetNodeStatisticsRequest { ) -> Result { let sys_status = garage.system.local_status(); + let hostname = sys_status.hostname.unwrap_or_default().to_string(); + let garage_version = garage_util::version::garage_version().to_string(); + let garage_features = garage_util::version::garage_features() + .unwrap() + .iter() + .map(ToString::to_string) + .collect::>(); + let rustc_version = garage_util::version::rust_version().to_string(); + let db_engine_descr = garage.db.engine(); + let mut ret = format_table_to_string(vec![ format!("Node ID:\t{:?}", garage.system.id), - format!("Hostname:\t{}", sys_status.hostname.unwrap_or_default(),), - format!( - "Garage version:\t{}", - garage_util::version::garage_version(), - ), - format!( - "Garage features:\t{}", - garage_util::version::garage_features() - .map(|list| list.join(", ")) - .unwrap_or_else(|| "(unknown)".into()), - ), - format!( - "Rust compiler version:\t{}", - garage_util::version::rust_version(), - ), - format!("Database engine:\t{}", garage.db.engine()), + format!("Hostname:\t{}", hostname), + format!("Garage version:\t{}", garage_version), + format!("Garage features:\t{}", garage_features.join(", ")), + format!("Rust compiler version:\t{}", rustc_version), + format!("Database engine:\t{}", db_engine_descr), ]); - // Gather table statistics - let mut table = vec![" Table\tItems\tMklItems\tMklTodo\tInsQueue\tGcTodo".into()]; - table.push(gather_table_stats(&garage.admin_token_table)?); - table.push(gather_table_stats(&garage.bucket_table)?); - table.push(gather_table_stats(&garage.bucket_alias_table)?); - table.push(gather_table_stats(&garage.key_table)?); - - table.push(gather_table_stats(&garage.object_table)?); - table.push(gather_table_stats(&garage.object_counter_table.table)?); - table.push(gather_table_stats(&garage.mpu_table)?); - table.push(gather_table_stats(&garage.mpu_counter_table.table)?); - table.push(gather_table_stats(&garage.version_table)?); - table.push(gather_table_stats(&garage.block_ref_table)?); + let mut table_stats = vec![ + gather_table_stats(&garage.admin_token_table)?, + gather_table_stats(&garage.bucket_table)?, + gather_table_stats(&garage.bucket_alias_table)?, + gather_table_stats(&garage.key_table)?, + gather_table_stats(&garage.object_table)?, + gather_table_stats(&garage.object_counter_table.table)?, + gather_table_stats(&garage.mpu_table)?, + gather_table_stats(&garage.mpu_counter_table.table)?, + gather_table_stats(&garage.version_table)?, + gather_table_stats(&garage.block_ref_table)?, + ]; #[cfg(feature = "k2v")] { - table.push(gather_table_stats(&garage.k2v.item_table)?); - table.push(gather_table_stats(&garage.k2v.counter_table.table)?); + table_stats.push(gather_table_stats(&garage.k2v.item_table)?); + table_stats.push(gather_table_stats(&garage.k2v.counter_table.table)?); } + // Gather table statistics + let mut table = vec![" Table\tItems\tMklItems\tMklTodo\tInsQueue\tGcTodo".into()]; + table.extend(table_stats.iter().map(|ts| { + format!( + " {}\t{}\t{}\t{}\t{}\t{}", + ts.table_name, + ts.items, + ts.merkle_items, + ts.merkle_queue_len, + ts.insert_queue_len, + ts.gc_queue_len, + ) + })); + write!( &mut ret, "\nTable stats:\n{}", @@ -104,46 +120,52 @@ impl RequestHandler for LocalGetNodeStatisticsRequest { ) .unwrap(); + let block_manager_stats = NodeBlockManagerStats { + rc_entries: garage.block_manager.rc_approximate_len()? as u64, + resync_queue_len: garage.block_manager.resync.queue_approximate_len()? as u64, + resync_errors: garage.block_manager.resync.errors_approximate_len()? as u64, + }; + // Gather block manager statistics writeln!(&mut ret, "\nBlock manager stats:").unwrap(); - let rc_len = garage.block_manager.rc_approximate_len()?.to_string(); ret += &format_table_to_string(vec![ - format!(" number of RC entries:\t{} (~= number of blocks)", rc_len), + format!( + " number of RC entries:\t{} (~= number of blocks)", + block_manager_stats.rc_entries + ), format!( " resync queue length:\t{}", - garage.block_manager.resync.queue_approximate_len()? + block_manager_stats.resync_queue_len, ), format!( " blocks with resync errors:\t{}", - garage.block_manager.resync.errors_approximate_len()? + block_manager_stats.resync_errors ), ]); - Ok(LocalGetNodeStatisticsResponse { freeform: ret }) + Ok(LocalGetNodeStatisticsResponse { + freeform: ret, + table_stats, + block_manager_stats, + }) } } -fn gather_table_stats(t: &Arc>) -> Result +fn gather_table_stats(t: &Arc>) -> Result where F: TableSchema + 'static, R: TableReplication + 'static, { - let data_len = t - .data - .store - .approximate_len() - .map_err(GarageError::from)? - .to_string(); - let mkl_len = t.merkle_updater.merkle_tree_approximate_len()?.to_string(); + let data_len = t.data.store.approximate_len().map_err(GarageError::from)?; + let mkl_len = t.merkle_updater.merkle_tree_approximate_len()?; - Ok(format!( - " {}\t{}\t{}\t{}\t{}\t{}", - F::TABLE_NAME, - data_len, - mkl_len, - t.merkle_updater.todo_approximate_len()?, - t.data.insert_queue_approximate_len()?, - t.data.gc_todo_approximate_len()? - )) + Ok(NodeTableStats { + table_name: F::TABLE_NAME.to_string(), + items: data_len as u64, + merkle_items: mkl_len as u64, + merkle_queue_len: t.merkle_updater.todo_approximate_len()? as u64, + insert_queue_len: t.data.insert_queue_approximate_len()? as u64, + gc_queue_len: t.data.gc_todo_approximate_len()? as u64, + }) }