mirror of
https://github.com/deuxfleurs-org/garage.git
synced 2026-07-26 07:58:14 +00:00
admin api: export node statistics as structured json
This commit is contained in:
Alex Auvolat
+125
-10
@@ -3072,7 +3072,8 @@
|
||||
],
|
||||
"properties": {
|
||||
"dbEngine": {
|
||||
"type": "string"
|
||||
"type": "string",
|
||||
"description": "database engine used for metadata"
|
||||
},
|
||||
"garageFeatures": {
|
||||
"type": [
|
||||
@@ -3081,16 +3082,23 @@
|
||||
],
|
||||
"items": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"description": "build-time features enabled for this garage release"
|
||||
},
|
||||
"garageVersion": {
|
||||
"type": "string"
|
||||
"type": "string",
|
||||
"description": "garage version running on this node"
|
||||
},
|
||||
"hostname": {
|
||||
"type": "string",
|
||||
"description": "hostname of this node"
|
||||
},
|
||||
"nodeId": {
|
||||
"type": "string"
|
||||
},
|
||||
"rustVersion": {
|
||||
"type": "string"
|
||||
"type": "string",
|
||||
"description": "rustc version with which this garage release was compiled"
|
||||
}
|
||||
}
|
||||
},
|
||||
@@ -3100,8 +3108,20 @@
|
||||
"freeform"
|
||||
],
|
||||
"properties": {
|
||||
"blockManagerStats": {
|
||||
"$ref": "#/components/schemas/NodeBlockManagerStats",
|
||||
"description": "block manager statistics"
|
||||
},
|
||||
"freeform": {
|
||||
"type": "string"
|
||||
"type": "string",
|
||||
"description": "node statistics as a free-form string, kept for compatibility with nodes\nrunning older v2.x versions of garage"
|
||||
},
|
||||
"tableStats": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/components/schemas/NodeTableStats"
|
||||
},
|
||||
"description": "metadata table statistics"
|
||||
}
|
||||
}
|
||||
},
|
||||
@@ -3402,7 +3422,8 @@
|
||||
],
|
||||
"properties": {
|
||||
"dbEngine": {
|
||||
"type": "string"
|
||||
"type": "string",
|
||||
"description": "database engine used for metadata"
|
||||
},
|
||||
"garageFeatures": {
|
||||
"type": [
|
||||
@@ -3411,16 +3432,23 @@
|
||||
],
|
||||
"items": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"description": "build-time features enabled for this garage release"
|
||||
},
|
||||
"garageVersion": {
|
||||
"type": "string"
|
||||
"type": "string",
|
||||
"description": "garage version running on this node"
|
||||
},
|
||||
"hostname": {
|
||||
"type": "string",
|
||||
"description": "hostname of this node"
|
||||
},
|
||||
"nodeId": {
|
||||
"type": "string"
|
||||
},
|
||||
"rustVersion": {
|
||||
"type": "string"
|
||||
"type": "string",
|
||||
"description": "rustc version with which this garage release was compiled"
|
||||
}
|
||||
}
|
||||
},
|
||||
@@ -3456,8 +3484,20 @@
|
||||
"freeform"
|
||||
],
|
||||
"properties": {
|
||||
"blockManagerStats": {
|
||||
"$ref": "#/components/schemas/NodeBlockManagerStats",
|
||||
"description": "block manager statistics"
|
||||
},
|
||||
"freeform": {
|
||||
"type": "string"
|
||||
"type": "string",
|
||||
"description": "node statistics as a free-form string, kept for compatibility with nodes\nrunning older v2.x versions of garage"
|
||||
},
|
||||
"tableStats": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/components/schemas/NodeTableStats"
|
||||
},
|
||||
"description": "metadata table statistics"
|
||||
}
|
||||
}
|
||||
},
|
||||
@@ -3796,6 +3836,34 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"NodeBlockManagerStats": {
|
||||
"type": "object",
|
||||
"required": [
|
||||
"rcEntries",
|
||||
"resyncQueueLen",
|
||||
"resyncErrors"
|
||||
],
|
||||
"properties": {
|
||||
"rcEntries": {
|
||||
"type": "integer",
|
||||
"format": "int64",
|
||||
"description": "number of reference counter entries",
|
||||
"minimum": 0
|
||||
},
|
||||
"resyncErrors": {
|
||||
"type": "integer",
|
||||
"format": "int64",
|
||||
"description": "number of blocks with resync errors",
|
||||
"minimum": 0
|
||||
},
|
||||
"resyncQueueLen": {
|
||||
"type": "integer",
|
||||
"format": "int64",
|
||||
"description": "number of blocks in the resync queue",
|
||||
"minimum": 0
|
||||
}
|
||||
}
|
||||
},
|
||||
"NodeResp": {
|
||||
"type": "object",
|
||||
"required": [
|
||||
@@ -3959,6 +4027,53 @@
|
||||
}
|
||||
]
|
||||
},
|
||||
"NodeTableStats": {
|
||||
"type": "object",
|
||||
"required": [
|
||||
"tableName",
|
||||
"items",
|
||||
"merkleItems",
|
||||
"merkleQueueLen",
|
||||
"insertQueueLen",
|
||||
"gcQueueLen"
|
||||
],
|
||||
"properties": {
|
||||
"gcQueueLen": {
|
||||
"type": "integer",
|
||||
"format": "int64",
|
||||
"description": "number of items in the garbage collection queue",
|
||||
"minimum": 0
|
||||
},
|
||||
"insertQueueLen": {
|
||||
"type": "integer",
|
||||
"format": "int64",
|
||||
"description": "number of items in the remote insert queue",
|
||||
"minimum": 0
|
||||
},
|
||||
"items": {
|
||||
"type": "integer",
|
||||
"format": "int64",
|
||||
"description": "number of items stored in metadata table",
|
||||
"minimum": 0
|
||||
},
|
||||
"merkleItems": {
|
||||
"type": "integer",
|
||||
"format": "int64",
|
||||
"description": "size of the merkle tree representing all items in the table",
|
||||
"minimum": 0
|
||||
},
|
||||
"merkleQueueLen": {
|
||||
"type": "integer",
|
||||
"format": "int64",
|
||||
"description": "number of items in the merkle tree update queue",
|
||||
"minimum": 0
|
||||
},
|
||||
"tableName": {
|
||||
"type": "string",
|
||||
"description": "name of metadata table"
|
||||
}
|
||||
}
|
||||
},
|
||||
"NodeUpdateTrackers": {
|
||||
"type": "object",
|
||||
"required": [
|
||||
|
||||
@@ -1123,9 +1123,16 @@ pub struct LocalGetNodeInfoRequest;
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct LocalGetNodeInfoResponse {
|
||||
pub node_id: String,
|
||||
/// hostname of this node
|
||||
#[serde(default)]
|
||||
pub hostname: String,
|
||||
/// garage version running on this node
|
||||
pub garage_version: String,
|
||||
/// build-time features enabled for this garage release
|
||||
pub garage_features: Option<Vec<String>>,
|
||||
/// rustc version with which this garage release was compiled
|
||||
pub rust_version: String,
|
||||
/// database engine used for metadata
|
||||
pub db_engine: String,
|
||||
}
|
||||
|
||||
@@ -1135,8 +1142,45 @@ pub struct LocalGetNodeInfoResponse {
|
||||
pub struct LocalGetNodeStatisticsRequest;
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct LocalGetNodeStatisticsResponse {
|
||||
/// node statistics as a free-form string, kept for compatibility with nodes
|
||||
/// running older v2.x versions of garage
|
||||
pub freeform: String,
|
||||
/// metadata table statistics
|
||||
#[serde(default)]
|
||||
pub table_stats: Vec<NodeTableStats>,
|
||||
/// block manager statistics
|
||||
#[serde(default)]
|
||||
pub block_manager_stats: NodeBlockManagerStats,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct NodeTableStats {
|
||||
/// name of metadata table
|
||||
pub table_name: String,
|
||||
/// number of items stored in metadata table
|
||||
pub items: u64,
|
||||
/// size of the merkle tree representing all items in the table
|
||||
pub merkle_items: u64,
|
||||
/// number of items in the merkle tree update queue
|
||||
pub merkle_queue_len: u64,
|
||||
/// number of items in the remote insert queue
|
||||
pub insert_queue_len: u64,
|
||||
/// number of items in the garbage collection queue
|
||||
pub gc_queue_len: u64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, ToSchema, Default)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct NodeBlockManagerStats {
|
||||
/// number of reference counter entries
|
||||
pub rc_entries: u64,
|
||||
/// number of blocks in the resync queue
|
||||
pub resync_queue_len: u64,
|
||||
/// number of blocks with resync errors
|
||||
pub resync_errors: u64,
|
||||
}
|
||||
|
||||
// ---- CreateMetadataSnapshot ----
|
||||
|
||||
+75
-53
@@ -22,8 +22,12 @@ impl RequestHandler for LocalGetNodeInfoRequest {
|
||||
garage: &Arc<Garage>,
|
||||
_admin: &Admin,
|
||||
) -> Result<LocalGetNodeInfoResponse, Error> {
|
||||
let sys_status = garage.system.local_status();
|
||||
let hostname = sys_status.hostname.unwrap_or_default().to_string();
|
||||
|
||||
Ok(LocalGetNodeInfoResponse {
|
||||
node_id: hex::encode(garage.system.id),
|
||||
hostname,
|
||||
garage_version: garage_util::version::garage_version().to_string(),
|
||||
garage_features: garage_util::version::garage_features()
|
||||
.map(|features| features.iter().map(ToString::to_string).collect()),
|
||||
@@ -57,46 +61,58 @@ impl RequestHandler for LocalGetNodeStatisticsRequest {
|
||||
) -> Result<LocalGetNodeStatisticsResponse, Error> {
|
||||
let sys_status = garage.system.local_status();
|
||||
|
||||
let hostname = sys_status.hostname.unwrap_or_default().to_string();
|
||||
let garage_version = garage_util::version::garage_version().to_string();
|
||||
let garage_features = garage_util::version::garage_features()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.map(ToString::to_string)
|
||||
.collect::<Vec<String>>();
|
||||
let rustc_version = garage_util::version::rust_version().to_string();
|
||||
let db_engine_descr = garage.db.engine();
|
||||
|
||||
let mut ret = format_table_to_string(vec![
|
||||
format!("Node ID:\t{:?}", garage.system.id),
|
||||
format!("Hostname:\t{}", sys_status.hostname.unwrap_or_default(),),
|
||||
format!(
|
||||
"Garage version:\t{}",
|
||||
garage_util::version::garage_version(),
|
||||
),
|
||||
format!(
|
||||
"Garage features:\t{}",
|
||||
garage_util::version::garage_features()
|
||||
.map(|list| list.join(", "))
|
||||
.unwrap_or_else(|| "(unknown)".into()),
|
||||
),
|
||||
format!(
|
||||
"Rust compiler version:\t{}",
|
||||
garage_util::version::rust_version(),
|
||||
),
|
||||
format!("Database engine:\t{}", garage.db.engine()),
|
||||
format!("Hostname:\t{}", hostname),
|
||||
format!("Garage version:\t{}", garage_version),
|
||||
format!("Garage features:\t{}", garage_features.join(", ")),
|
||||
format!("Rust compiler version:\t{}", rustc_version),
|
||||
format!("Database engine:\t{}", db_engine_descr),
|
||||
]);
|
||||
|
||||
// Gather table statistics
|
||||
let mut table = vec![" Table\tItems\tMklItems\tMklTodo\tInsQueue\tGcTodo".into()];
|
||||
table.push(gather_table_stats(&garage.admin_token_table)?);
|
||||
table.push(gather_table_stats(&garage.bucket_table)?);
|
||||
table.push(gather_table_stats(&garage.bucket_alias_table)?);
|
||||
table.push(gather_table_stats(&garage.key_table)?);
|
||||
|
||||
table.push(gather_table_stats(&garage.object_table)?);
|
||||
table.push(gather_table_stats(&garage.object_counter_table.table)?);
|
||||
table.push(gather_table_stats(&garage.mpu_table)?);
|
||||
table.push(gather_table_stats(&garage.mpu_counter_table.table)?);
|
||||
table.push(gather_table_stats(&garage.version_table)?);
|
||||
table.push(gather_table_stats(&garage.block_ref_table)?);
|
||||
let mut table_stats = vec![
|
||||
gather_table_stats(&garage.admin_token_table)?,
|
||||
gather_table_stats(&garage.bucket_table)?,
|
||||
gather_table_stats(&garage.bucket_alias_table)?,
|
||||
gather_table_stats(&garage.key_table)?,
|
||||
gather_table_stats(&garage.object_table)?,
|
||||
gather_table_stats(&garage.object_counter_table.table)?,
|
||||
gather_table_stats(&garage.mpu_table)?,
|
||||
gather_table_stats(&garage.mpu_counter_table.table)?,
|
||||
gather_table_stats(&garage.version_table)?,
|
||||
gather_table_stats(&garage.block_ref_table)?,
|
||||
];
|
||||
|
||||
#[cfg(feature = "k2v")]
|
||||
{
|
||||
table.push(gather_table_stats(&garage.k2v.item_table)?);
|
||||
table.push(gather_table_stats(&garage.k2v.counter_table.table)?);
|
||||
table_stats.push(gather_table_stats(&garage.k2v.item_table)?);
|
||||
table_stats.push(gather_table_stats(&garage.k2v.counter_table.table)?);
|
||||
}
|
||||
|
||||
// Gather table statistics
|
||||
let mut table = vec![" Table\tItems\tMklItems\tMklTodo\tInsQueue\tGcTodo".into()];
|
||||
table.extend(table_stats.iter().map(|ts| {
|
||||
format!(
|
||||
" {}\t{}\t{}\t{}\t{}\t{}",
|
||||
ts.table_name,
|
||||
ts.items,
|
||||
ts.merkle_items,
|
||||
ts.merkle_queue_len,
|
||||
ts.insert_queue_len,
|
||||
ts.gc_queue_len,
|
||||
)
|
||||
}));
|
||||
|
||||
write!(
|
||||
&mut ret,
|
||||
"\nTable stats:\n{}",
|
||||
@@ -104,46 +120,52 @@ impl RequestHandler for LocalGetNodeStatisticsRequest {
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let block_manager_stats = NodeBlockManagerStats {
|
||||
rc_entries: garage.block_manager.rc_approximate_len()? as u64,
|
||||
resync_queue_len: garage.block_manager.resync.queue_approximate_len()? as u64,
|
||||
resync_errors: garage.block_manager.resync.errors_approximate_len()? as u64,
|
||||
};
|
||||
|
||||
// Gather block manager statistics
|
||||
writeln!(&mut ret, "\nBlock manager stats:").unwrap();
|
||||
let rc_len = garage.block_manager.rc_approximate_len()?.to_string();
|
||||
|
||||
ret += &format_table_to_string(vec![
|
||||
format!(" number of RC entries:\t{} (~= number of blocks)", rc_len),
|
||||
format!(
|
||||
" number of RC entries:\t{} (~= number of blocks)",
|
||||
block_manager_stats.rc_entries
|
||||
),
|
||||
format!(
|
||||
" resync queue length:\t{}",
|
||||
garage.block_manager.resync.queue_approximate_len()?
|
||||
block_manager_stats.resync_queue_len,
|
||||
),
|
||||
format!(
|
||||
" blocks with resync errors:\t{}",
|
||||
garage.block_manager.resync.errors_approximate_len()?
|
||||
block_manager_stats.resync_errors
|
||||
),
|
||||
]);
|
||||
|
||||
Ok(LocalGetNodeStatisticsResponse { freeform: ret })
|
||||
Ok(LocalGetNodeStatisticsResponse {
|
||||
freeform: ret,
|
||||
table_stats,
|
||||
block_manager_stats,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
fn gather_table_stats<F, R>(t: &Arc<Table<F, R>>) -> Result<String, Error>
|
||||
fn gather_table_stats<F, R>(t: &Arc<Table<F, R>>) -> Result<NodeTableStats, Error>
|
||||
where
|
||||
F: TableSchema + 'static,
|
||||
R: TableReplication + 'static,
|
||||
{
|
||||
let data_len = t
|
||||
.data
|
||||
.store
|
||||
.approximate_len()
|
||||
.map_err(GarageError::from)?
|
||||
.to_string();
|
||||
let mkl_len = t.merkle_updater.merkle_tree_approximate_len()?.to_string();
|
||||
let data_len = t.data.store.approximate_len().map_err(GarageError::from)?;
|
||||
let mkl_len = t.merkle_updater.merkle_tree_approximate_len()?;
|
||||
|
||||
Ok(format!(
|
||||
" {}\t{}\t{}\t{}\t{}\t{}",
|
||||
F::TABLE_NAME,
|
||||
data_len,
|
||||
mkl_len,
|
||||
t.merkle_updater.todo_approximate_len()?,
|
||||
t.data.insert_queue_approximate_len()?,
|
||||
t.data.gc_todo_approximate_len()?
|
||||
))
|
||||
Ok(NodeTableStats {
|
||||
table_name: F::TABLE_NAME.to_string(),
|
||||
items: data_len as u64,
|
||||
merkle_items: mkl_len as u64,
|
||||
merkle_queue_len: t.merkle_updater.todo_approximate_len()? as u64,
|
||||
insert_queue_len: t.data.insert_queue_approximate_len()? as u64,
|
||||
gc_queue_len: t.data.gc_todo_approximate_len()? as u64,
|
||||
})
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user