admin api: export node statistics as structured json

doc/api/garage-admin-v2.json

@@ -3072,7 +3072,8 @@
         ],
         "properties": {
           "dbEngine": {
-            "type": "string"
+            "type": "string",
+            "description": "database engine used for metadata"
           },
           "garageFeatures": {
             "type": [
@@ -3081,16 +3082,23 @@
             ],
             "items": {
               "type": "string"
-            }
+            },
+            "description": "build-time features enabled for this garage release"
           },
           "garageVersion": {
-            "type": "string"
+            "type": "string",
+            "description": "garage version running on this node"
+          },
+          "hostname": {
+            "type": "string",
+            "description": "hostname of this node"
           },
           "nodeId": {
             "type": "string"
           },
           "rustVersion": {
-            "type": "string"
+            "type": "string",
+            "description": "rustc version with which this garage release was compiled"
           }
         }
       },
@@ -3100,8 +3108,20 @@
           "freeform"
         ],
         "properties": {
+          "blockManagerStats": {
+            "$ref": "#/components/schemas/NodeBlockManagerStats",
+            "description": "block manager statistics"
+          },
           "freeform": {
-            "type": "string"
+            "type": "string",
+            "description": "node statistics as a free-form string, kept for compatibility with nodes\nrunning older v2.x versions of garage"
+          },
+          "tableStats": {
+            "type": "array",
+            "items": {
+              "$ref": "#/components/schemas/NodeTableStats"
+            },
+            "description": "metadata table statistics"
           }
         }
       },
@@ -3402,7 +3422,8 @@
               ],
               "properties": {
                 "dbEngine": {
-                  "type": "string"
+                  "type": "string",
+                  "description": "database engine used for metadata"
                 },
                 "garageFeatures": {
                   "type": [
@@ -3411,16 +3432,23 @@
                   ],
                   "items": {
                     "type": "string"
-                  }
+                  },
+                  "description": "build-time features enabled for this garage release"
                 },
                 "garageVersion": {
-                  "type": "string"
+                  "type": "string",
+                  "description": "garage version running on this node"
+                },
+                "hostname": {
+                  "type": "string",
+                  "description": "hostname of this node"
                 },
                 "nodeId": {
                   "type": "string"
                 },
                 "rustVersion": {
-                  "type": "string"
+                  "type": "string",
+                  "description": "rustc version with which this garage release was compiled"
                 }
               }
             },
@@ -3456,8 +3484,20 @@
                 "freeform"
               ],
               "properties": {
+                "blockManagerStats": {
+                  "$ref": "#/components/schemas/NodeBlockManagerStats",
+                  "description": "block manager statistics"
+                },
                 "freeform": {
-                  "type": "string"
+                  "type": "string",
+                  "description": "node statistics as a free-form string, kept for compatibility with nodes\nrunning older v2.x versions of garage"
+                },
+                "tableStats": {
+                  "type": "array",
+                  "items": {
+                    "$ref": "#/components/schemas/NodeTableStats"
+                  },
+                  "description": "metadata table statistics"
                 }
               }
             },
@@ -3796,6 +3836,34 @@
           }
         }
       },
+      "NodeBlockManagerStats": {
+        "type": "object",
+        "required": [
+          "rcEntries",
+          "resyncQueueLen",
+          "resyncErrors"
+        ],
+        "properties": {
+          "rcEntries": {
+            "type": "integer",
+            "format": "int64",
+            "description": "number of reference counter entries",
+            "minimum": 0
+          },
+          "resyncErrors": {
+            "type": "integer",
+            "format": "int64",
+            "description": "number of blocks with resync errors",
+            "minimum": 0
+          },
+          "resyncQueueLen": {
+            "type": "integer",
+            "format": "int64",
+            "description": "number of blocks in the resync queue",
+            "minimum": 0
+          }
+        }
+      },
       "NodeResp": {
         "type": "object",
         "required": [
@@ -3959,6 +4027,53 @@
           }
         ]
       },
+      "NodeTableStats": {
+        "type": "object",
+        "required": [
+          "tableName",
+          "items",
+          "merkleItems",
+          "merkleQueueLen",
+          "insertQueueLen",
+          "gcQueueLen"
+        ],
+        "properties": {
+          "gcQueueLen": {
+            "type": "integer",
+            "format": "int64",
+            "description": "number of items in the garbage collection queue",
+            "minimum": 0
+          },
+          "insertQueueLen": {
+            "type": "integer",
+            "format": "int64",
+            "description": "number of items in the remote insert queue",
+            "minimum": 0
+          },
+          "items": {
+            "type": "integer",
+            "format": "int64",
+            "description": "number of items stored in metadata table",
+            "minimum": 0
+          },
+          "merkleItems": {
+            "type": "integer",
+            "format": "int64",
+            "description": "size of the merkle tree representing all items in the table",
+            "minimum": 0
+          },
+          "merkleQueueLen": {
+            "type": "integer",
+            "format": "int64",
+            "description": "number of items in the merkle tree update queue",
+            "minimum": 0
+          },
+          "tableName": {
+            "type": "string",
+            "description": "name of metadata table"
+          }
+        }
+      },
       "NodeUpdateTrackers": {
         "type": "object",
         "required": [

src/api/admin/api.rs

@@ -1123,9 +1123,16 @@ pub struct LocalGetNodeInfoRequest;
 #[serde(rename_all = "camelCase")]
 pub struct LocalGetNodeInfoResponse {
 	pub node_id: String,
+	/// hostname of this node
+	#[serde(default)]
+	pub hostname: String,
+	/// garage version running on this node
 	pub garage_version: String,
+	/// build-time features enabled for this garage release
 	pub garage_features: Option<Vec<String>>,
+	/// rustc version with which this garage release was compiled
 	pub rust_version: String,
+	/// database engine used for metadata
 	pub db_engine: String,
 }
 
@@ -1135,8 +1142,45 @@ pub struct LocalGetNodeInfoResponse {
 pub struct LocalGetNodeStatisticsRequest;
 
 #[derive(Debug, Clone, Serialize, Deserialize, ToSchema)]
+#[serde(rename_all = "camelCase")]
 pub struct LocalGetNodeStatisticsResponse {
+	/// node statistics as a free-form string, kept for compatibility with nodes
+	/// running older v2.x versions of garage
 	pub freeform: String,
+	/// metadata table statistics
+	#[serde(default)]
+	pub table_stats: Vec<NodeTableStats>,
+	/// block manager statistics
+	#[serde(default)]
+	pub block_manager_stats: NodeBlockManagerStats,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)]
+#[serde(rename_all = "camelCase")]
+pub struct NodeTableStats {
+	/// name of metadata table
+	pub table_name: String,
+	/// number of items stored in metadata table
+	pub items: u64,
+	/// size of the merkle tree representing all items in the table
+	pub merkle_items: u64,
+	/// number of items in the merkle tree update queue
+	pub merkle_queue_len: u64,
+	/// number of items in the remote insert queue
+	pub insert_queue_len: u64,
+	/// number of items in the garbage collection queue
+	pub gc_queue_len: u64,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize, ToSchema, Default)]
+#[serde(rename_all = "camelCase")]
+pub struct NodeBlockManagerStats {
+	/// number of reference counter entries
+	pub rc_entries: u64,
+	/// number of blocks in the resync queue
+	pub resync_queue_len: u64,
+	/// number of blocks with resync errors
+	pub resync_errors: u64,
 }
 
 // ---- CreateMetadataSnapshot ----

src/api/admin/node.rs

@@ -22,8 +22,12 @@ impl RequestHandler for LocalGetNodeInfoRequest {
 		garage: &Arc<Garage>,
 		_admin: &Admin,
 	) -> Result<LocalGetNodeInfoResponse, Error> {
+		let sys_status = garage.system.local_status();
+		let hostname = sys_status.hostname.unwrap_or_default().to_string();
+
 		Ok(LocalGetNodeInfoResponse {
 			node_id: hex::encode(garage.system.id),
+			hostname,
 			garage_version: garage_util::version::garage_version().to_string(),
 			garage_features: garage_util::version::garage_features()
 				.map(|features| features.iter().map(ToString::to_string).collect()),
@@ -57,46 +61,58 @@ impl RequestHandler for LocalGetNodeStatisticsRequest {
 	) -> Result<LocalGetNodeStatisticsResponse, Error> {
 		let sys_status = garage.system.local_status();
 
+		let hostname = sys_status.hostname.unwrap_or_default().to_string();
+		let garage_version = garage_util::version::garage_version().to_string();
+		let garage_features = garage_util::version::garage_features()
+			.unwrap()
+			.iter()
+			.map(ToString::to_string)
+			.collect::<Vec<String>>();
+		let rustc_version = garage_util::version::rust_version().to_string();
+		let db_engine_descr = garage.db.engine();
+
 		let mut ret = format_table_to_string(vec![
 			format!("Node ID:\t{:?}", garage.system.id),
-			format!("Hostname:\t{}", sys_status.hostname.unwrap_or_default(),),
+			format!("Hostname:\t{}", hostname),
-			format!(
+			format!("Garage version:\t{}", garage_version),
-				"Garage version:\t{}",
+			format!("Garage features:\t{}", garage_features.join(", ")),
-				garage_util::version::garage_version(),
+			format!("Rust compiler version:\t{}", rustc_version),
-			),
+			format!("Database engine:\t{}", db_engine_descr),
-			format!(
-				"Garage features:\t{}",
-				garage_util::version::garage_features()
-					.map(|list| list.join(", "))
-					.unwrap_or_else(|| "(unknown)".into()),
-			),
-			format!(
-				"Rust compiler version:\t{}",
-				garage_util::version::rust_version(),
-			),
-			format!("Database engine:\t{}", garage.db.engine()),
 		]);
 
-		// Gather table statistics
+		let mut table_stats = vec![
-		let mut table = vec!["  Table\tItems\tMklItems\tMklTodo\tInsQueue\tGcTodo".into()];
+			gather_table_stats(&garage.admin_token_table)?,
-		table.push(gather_table_stats(&garage.admin_token_table)?);
+			gather_table_stats(&garage.bucket_table)?,
-		table.push(gather_table_stats(&garage.bucket_table)?);
+			gather_table_stats(&garage.bucket_alias_table)?,
-		table.push(gather_table_stats(&garage.bucket_alias_table)?);
+			gather_table_stats(&garage.key_table)?,
-		table.push(gather_table_stats(&garage.key_table)?);
+			gather_table_stats(&garage.object_table)?,
-
+			gather_table_stats(&garage.object_counter_table.table)?,
-		table.push(gather_table_stats(&garage.object_table)?);
+			gather_table_stats(&garage.mpu_table)?,
-		table.push(gather_table_stats(&garage.object_counter_table.table)?);
+			gather_table_stats(&garage.mpu_counter_table.table)?,
-		table.push(gather_table_stats(&garage.mpu_table)?);
+			gather_table_stats(&garage.version_table)?,
-		table.push(gather_table_stats(&garage.mpu_counter_table.table)?);
+			gather_table_stats(&garage.block_ref_table)?,
-		table.push(gather_table_stats(&garage.version_table)?);
+		];
-		table.push(gather_table_stats(&garage.block_ref_table)?);
 
 		#[cfg(feature = "k2v")]
 		{
-			table.push(gather_table_stats(&garage.k2v.item_table)?);
+			table_stats.push(gather_table_stats(&garage.k2v.item_table)?);
-			table.push(gather_table_stats(&garage.k2v.counter_table.table)?);
+			table_stats.push(gather_table_stats(&garage.k2v.counter_table.table)?);
 		}
 
+		// Gather table statistics
+		let mut table = vec!["  Table\tItems\tMklItems\tMklTodo\tInsQueue\tGcTodo".into()];
+		table.extend(table_stats.iter().map(|ts| {
+			format!(
+				"  {}\t{}\t{}\t{}\t{}\t{}",
+				ts.table_name,
+				ts.items,
+				ts.merkle_items,
+				ts.merkle_queue_len,
+				ts.insert_queue_len,
+				ts.gc_queue_len,
+			)
+		}));
+
 		write!(
 			&mut ret,
 			"\nTable stats:\n{}",
@@ -104,46 +120,52 @@ impl RequestHandler for LocalGetNodeStatisticsRequest {
 		)
 		.unwrap();
 
+		let block_manager_stats = NodeBlockManagerStats {
+			rc_entries: garage.block_manager.rc_approximate_len()? as u64,
+			resync_queue_len: garage.block_manager.resync.queue_approximate_len()? as u64,
+			resync_errors: garage.block_manager.resync.errors_approximate_len()? as u64,
+		};
+
 		// Gather block manager statistics
 		writeln!(&mut ret, "\nBlock manager stats:").unwrap();
-		let rc_len = garage.block_manager.rc_approximate_len()?.to_string();
 
 		ret += &format_table_to_string(vec![
-			format!("  number of RC entries:\t{} (~= number of blocks)", rc_len),
+			format!(
+				"  number of RC entries:\t{} (~= number of blocks)",
+				block_manager_stats.rc_entries
+			),
 			format!(
 				"  resync queue length:\t{}",
-				garage.block_manager.resync.queue_approximate_len()?
+				block_manager_stats.resync_queue_len,
 			),
 			format!(
 				"  blocks with resync errors:\t{}",
-				garage.block_manager.resync.errors_approximate_len()?
+				block_manager_stats.resync_errors
 			),
 		]);
 
-		Ok(LocalGetNodeStatisticsResponse { freeform: ret })
+		Ok(LocalGetNodeStatisticsResponse {
+			freeform: ret,
+			table_stats,
+			block_manager_stats,
+		})
 	}
 }
 
-fn gather_table_stats<F, R>(t: &Arc<Table<F, R>>) -> Result<String, Error>
+fn gather_table_stats<F, R>(t: &Arc<Table<F, R>>) -> Result<NodeTableStats, Error>
 where
 	F: TableSchema + 'static,
 	R: TableReplication + 'static,
 {
-	let data_len = t
+	let data_len = t.data.store.approximate_len().map_err(GarageError::from)?;
-		.data
+	let mkl_len = t.merkle_updater.merkle_tree_approximate_len()?;
-		.store
-		.approximate_len()
-		.map_err(GarageError::from)?
-		.to_string();
-	let mkl_len = t.merkle_updater.merkle_tree_approximate_len()?.to_string();
 
-	Ok(format!(
+	Ok(NodeTableStats {
-		"  {}\t{}\t{}\t{}\t{}\t{}",
+		table_name: F::TABLE_NAME.to_string(),
-		F::TABLE_NAME,
+		items: data_len as u64,
-		data_len,
+		merkle_items: mkl_len as u64,
-		mkl_len,
+		merkle_queue_len: t.merkle_updater.todo_approximate_len()? as u64,
-		t.merkle_updater.todo_approximate_len()?,
+		insert_queue_len: t.data.insert_queue_approximate_len()? as u64,
-		t.data.insert_queue_approximate_len()?,
+		gc_queue_len: t.data.gc_todo_approximate_len()? as u64,
-		t.data.gc_todo_approximate_len()?
+	})
-	))
 }