From 8d85301808762fba6a9197fefc56750a1bacb0d9 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Sat, 25 Jul 2026 15:12:37 +0200 Subject: [PATCH] Admin API: add parameters `details`, `offset` and `limit` to ListKeys and ListBuckets --- doc/api/garage-admin-v2.json | 206 ++++++++++++++++++++++++++++++-- src/api/admin/api.rs | 42 ++++++- src/api/admin/bucket.rs | 86 ++++++++----- src/api/admin/key.rs | 67 +++++++---- src/api/admin/openapi.rs | 4 +- src/api/admin/router_v2.rs | 15 ++- src/garage/cli/remote/bucket.rs | 13 +- src/garage/cli/remote/key.rs | 14 ++- 8 files changed, 365 insertions(+), 82 deletions(-) diff --git a/doc/api/garage-admin-v2.json b/doc/api/garage-admin-v2.json index 15975c6a..1207a5bd 100644 --- a/doc/api/garage-admin-v2.json +++ b/doc/api/garage-admin-v2.json @@ -12,7 +12,7 @@ "name": "AGPL-3.0", "identifier": "AGPL-3.0" }, - "version": "v2.3.0" + "version": "v2.4.0" }, "servers": [ { @@ -1243,6 +1243,36 @@ ], "description": "List all the buckets on the cluster with their UUID and their global and local aliases.", "operationId": "ListBuckets", + "parameters": [ + { + "name": "details", + "in": "query", + "description": "Returned detailed informations in the same format as GetBucketInfo for each bucket", + "required": false, + "schema": { + "type": "boolean" + } + }, + { + "name": "offset", + "in": "query", + "description": "Bucket ID of the first bucket to return", + "required": false, + "schema": { + "type": "string" + } + }, + { + "name": "limit", + "in": "query", + "description": "Maximum number of buckets to return in a single call", + "required": false, + "schema": { + "type": "integer", + "minimum": 0 + } + } + ], "responses": { "200": { "description": "Returns the UUID of all the buckets and all their aliases", @@ -1267,6 +1297,36 @@ ], "description": "Returns all API access keys in the cluster.", "operationId": "ListKeys", + "parameters": [ + { + "name": "details", + "in": "query", + "description": "Returned detailed informations in the same format as GetKeyInfo for each bucket", + "required": false, + "schema": { + "type": "boolean" + } + }, + { + "name": "offset", + "in": "query", + "description": "Key ID of the first key to return", + "required": false, + "schema": { + "type": "string" + } + }, + { + "name": "limit", + "in": "query", + "description": "Maximum number of keys to return in a single call", + "required": false, + "schema": { + "type": "integer", + "minimum": 0 + } + } + ], "responses": { "200": { "description": "Returns the key identifier (aka `AWS_ACCESS_KEY_ID`) and its associated, human friendly, name if any (otherwise return an empty string)", @@ -3200,10 +3260,20 @@ } }, "ListBucketsResponse": { - "type": "array", - "items": { - "$ref": "#/components/schemas/ListBucketsResponseItem" - } + "oneOf": [ + { + "type": "array", + "items": { + "$ref": "#/components/schemas/ListBucketsResponseItem" + } + }, + { + "type": "array", + "items": { + "$ref": "#/components/schemas/GetBucketInfoResponse" + } + } + ] }, "ListBucketsResponseItem": { "type": "object", @@ -3236,10 +3306,20 @@ } }, "ListKeysResponse": { - "type": "array", - "items": { - "$ref": "#/components/schemas/ListKeysResponseItem" - } + "oneOf": [ + { + "type": "array", + "items": { + "$ref": "#/components/schemas/ListKeysResponseItem" + } + }, + { + "type": "array", + "items": { + "$ref": "#/components/schemas/GetKeyInfoResponse" + } + } + ] }, "ListKeysResponseItem": { "type": "object", @@ -3321,10 +3401,35 @@ "dbEngine" ], "properties": { + "addr": { + "type": [ + "string", + "null" + ], + "description": "Socket address used by other nodes to connect to this node for RPC" + }, + "dataPartition": { + "oneOf": [ + { + "type": "null" + }, + { + "$ref": "#/components/schemas/FreeSpaceResp", + "description": "Total and available space on the disk partition(s) containing the data\ndirectory(ies)" + } + ] + }, "dbEngine": { "type": "string", "description": "database engine used for metadata" }, + "draining": { + "type": [ + "boolean", + "null" + ], + "description": "Whether this node is part of an older layout version and is draining data." + }, "garageFeatures": { "type": [ "array", @@ -3346,9 +3451,38 @@ ], "description": "hostname of this node" }, + "isUp": { + "type": [ + "boolean", + "null" + ], + "description": "Whether this node is connected in the cluster" + }, + "metadataPartition": { + "oneOf": [ + { + "type": "null" + }, + { + "$ref": "#/components/schemas/FreeSpaceResp", + "description": "Total and available space on the disk partition containing the\nmetadata directory" + } + ] + }, "nodeId": { "type": "string" }, + "role": { + "oneOf": [ + { + "type": "null" + }, + { + "$ref": "#/components/schemas/NodeAssignedRole", + "description": "Role assigned to this node in the current cluster layout" + } + ] + }, "rustVersion": { "type": "string", "description": "rustc version with which this garage release was compiled" @@ -3684,10 +3818,35 @@ "dbEngine" ], "properties": { + "addr": { + "type": [ + "string", + "null" + ], + "description": "Socket address used by other nodes to connect to this node for RPC" + }, + "dataPartition": { + "oneOf": [ + { + "type": "null" + }, + { + "$ref": "#/components/schemas/FreeSpaceResp", + "description": "Total and available space on the disk partition(s) containing the data\ndirectory(ies)" + } + ] + }, "dbEngine": { "type": "string", "description": "database engine used for metadata" }, + "draining": { + "type": [ + "boolean", + "null" + ], + "description": "Whether this node is part of an older layout version and is draining data." + }, "garageFeatures": { "type": [ "array", @@ -3709,9 +3868,38 @@ ], "description": "hostname of this node" }, + "isUp": { + "type": [ + "boolean", + "null" + ], + "description": "Whether this node is connected in the cluster" + }, + "metadataPartition": { + "oneOf": [ + { + "type": "null" + }, + { + "$ref": "#/components/schemas/FreeSpaceResp", + "description": "Total and available space on the disk partition containing the\nmetadata directory" + } + ] + }, "nodeId": { "type": "string" }, + "role": { + "oneOf": [ + { + "type": "null" + }, + { + "$ref": "#/components/schemas/NodeAssignedRole", + "description": "Role assigned to this node in the current cluster layout" + } + ] + }, "rustVersion": { "type": "string", "description": "rustc version with which this garage release was compiled" diff --git a/src/api/admin/api.rs b/src/api/admin/api.rs index 124702ce..acde68f7 100644 --- a/src/api/admin/api.rs +++ b/src/api/admin/api.rs @@ -688,11 +688,26 @@ pub struct ClusterLayoutSkipDeadNodesResponse { // ---- ListKeys ---- -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct ListKeysRequest; +#[derive(Debug, Clone, Serialize, Deserialize, Default, IntoParams)] +#[into_params(parameter_in = Query)] +pub struct ListKeysRequest { + /// Returned detailed informations in the same format as GetKeyInfo for each bucket + #[serde(default)] + pub details: bool, + /// Key ID of the first key to return + #[serde(default)] + pub offset: Option, + /// Maximum number of keys to return in a single call + #[serde(default)] + pub limit: Option, +} #[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] -pub struct ListKeysResponse(pub Vec); +#[serde(untagged)] +pub enum ListKeysResponse { + WithoutDetails(Vec), + WithDetails(Vec), +} #[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] #[serde(rename_all = "camelCase")] @@ -830,11 +845,26 @@ pub struct DeleteKeyResponse; // ---- ListBuckets ---- -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct ListBucketsRequest; +#[derive(Debug, Clone, Serialize, Deserialize, Default, IntoParams)] +#[into_params(parameter_in = Query)] +pub struct ListBucketsRequest { + /// Returned detailed informations in the same format as GetBucketInfo for each bucket + #[serde(default)] + pub details: bool, + /// Bucket ID of the first bucket to return + #[serde(default)] + pub offset: Option, + /// Maximum number of buckets to return in a single call + #[serde(default)] + pub limit: Option, +} #[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] -pub struct ListBucketsResponse(pub Vec); +#[serde(untagged)] +pub enum ListBucketsResponse { + WithoutDetails(Vec), + WithDetails(Vec), +} #[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] #[serde(rename_all = "camelCase")] diff --git a/src/api/admin/bucket.rs b/src/api/admin/bucket.rs index 3b30465a..8b86126c 100644 --- a/src/api/admin/bucket.rs +++ b/src/api/admin/bucket.rs @@ -3,6 +3,7 @@ use std::sync::Arc; use std::time::Duration; use chrono::DateTime; +use futures::StreamExt; use garage_util::crdt::*; use garage_util::data::*; @@ -32,47 +33,70 @@ impl RequestHandler for ListBucketsRequest { garage: &Arc, _admin: &Admin, ) -> Result { + let limit = self + .limit + .unwrap_or_else(|| if self.details { 1000 } else { 10_000 }); + + let offset = match self.offset { + Some(id) => Some(parse_bucket_id(&id)?), + None => None, + }; + let buckets = garage .bucket_table .get_range( &EmptyKey, - None, + offset, Some(DeletedFilter::NotDeleted), - 1_000_000, + limit, EnumerationOrder::Forward, ) .await?; - let res = buckets - .into_iter() - .map(|b| { - let state = b.state.as_option().unwrap(); - ListBucketsResponseItem { - id: hex::encode(b.id), - created: DateTime::from_timestamp_millis(state.creation_date as i64) - .expect("invalid timestamp stored in db"), - global_aliases: state - .aliases - .items() - .iter() - .filter(|(_, _, a)| *a) - .map(|(n, _, _)| n.to_string()) - .collect::>(), - local_aliases: state - .local_aliases - .items() - .iter() - .filter(|(_, _, a)| *a) - .map(|((k, n), _, _)| BucketLocalAlias { - access_key_id: k.to_string(), - alias: n.to_string(), - }) - .collect::>(), - } - }) - .collect::>(); + if self.details { + let mut stream = buckets + .into_iter() + .map(|b| bucket_info_results(garage, b.id)) + .collect::>(); - Ok(ListBucketsResponse(res)) + let mut res = vec![]; + while let Some(next) = stream.next().await { + res.push(next?); + } + + Ok(ListBucketsResponse::WithDetails(res)) + } else { + let res = buckets + .into_iter() + .map(|b| { + let state = b.state.as_option().unwrap(); + ListBucketsResponseItem { + id: hex::encode(b.id), + created: DateTime::from_timestamp_millis(state.creation_date as i64) + .expect("invalid timestamp stored in db"), + global_aliases: state + .aliases + .items() + .iter() + .filter(|(_, _, a)| *a) + .map(|(n, _, _)| n.to_string()) + .collect::>(), + local_aliases: state + .local_aliases + .items() + .iter() + .filter(|(_, _, a)| *a) + .map(|((k, n), _, _)| BucketLocalAlias { + access_key_id: k.to_string(), + alias: n.to_string(), + }) + .collect::>(), + } + }) + .collect::>(); + + Ok(ListBucketsResponse::WithoutDetails(res)) + } } } diff --git a/src/api/admin/key.rs b/src/api/admin/key.rs index 65b9c537..d8c50f9a 100644 --- a/src/api/admin/key.rs +++ b/src/api/admin/key.rs @@ -2,6 +2,7 @@ use std::collections::HashMap; use std::sync::Arc; use chrono::DateTime; +use futures::StreamExt; use garage_table::*; use garage_util::time::now_msec; @@ -20,37 +21,57 @@ impl RequestHandler for ListKeysRequest { async fn handle(self, garage: &Arc, _admin: &Admin) -> Result { let now = now_msec(); - let res = garage + let limit = self + .limit + .unwrap_or_else(|| if self.details { 1000 } else { 10_000 }); + + let keys = garage .key_table .get_range( &EmptyKey, - None, + self.offset, Some(KeyFilter::Deleted(DeletedFilter::NotDeleted)), - 10000, + limit, EnumerationOrder::Forward, ) - .await? - .iter() - .map(|k| { - let p = k.params().unwrap(); + .await?; - ListKeysResponseItem { - id: k.key_id.to_string(), - name: p.name.get().clone(), - created: p.created.map(|x| { - DateTime::from_timestamp_millis(x as i64) - .expect("invalid timestamp stored in db") - }), - expiration: p.expiration.get().inner().map(|x| { - DateTime::from_timestamp_millis(x.0 as i64) - .expect("invalid timestamp stored in db") - }), - expired: p.is_expired(now), - } - }) - .collect::>(); + if self.details { + let mut stream = keys + .into_iter() + .map(|k| key_info_results(garage, k, false)) + .collect::>(); - Ok(ListKeysResponse(res)) + let mut res = vec![]; + while let Some(next) = stream.next().await { + res.push(next?); + } + + Ok(ListKeysResponse::WithDetails(res)) + } else { + let res = keys + .iter() + .map(|k| { + let p = k.params().unwrap(); + + ListKeysResponseItem { + id: k.key_id.to_string(), + name: p.name.get().clone(), + created: p.created.map(|x| { + DateTime::from_timestamp_millis(x as i64) + .expect("invalid timestamp stored in db") + }), + expiration: p.expiration.get().inner().map(|x| { + DateTime::from_timestamp_millis(x.0 as i64) + .expect("invalid timestamp stored in db") + }), + expired: p.is_expired(now), + } + }) + .collect::>(); + + Ok(ListKeysResponse::WithoutDetails(res)) + } } } diff --git a/src/api/admin/openapi.rs b/src/api/admin/openapi.rs index 2bb5b2a4..148ec2ea 100644 --- a/src/api/admin/openapi.rs +++ b/src/api/admin/openapi.rs @@ -364,6 +364,7 @@ fn ClusterLayoutSkipDeadNodes() {} path = "/v2/ListKeys", tag = "Access key", description = "Returns all API access keys in the cluster.", + params(ListKeysRequest), responses( (status = 200, description = "Returns the key identifier (aka `AWS_ACCESS_KEY_ID`) and its associated, human friendly, name if any (otherwise return an empty string)", body = ListKeysResponse), (status = 500, description = "Internal server error") @@ -453,6 +454,7 @@ fn DeleteKey() {} path = "/v2/ListBuckets", tag = "Bucket", description = "List all the buckets on the cluster with their UUID and their global and local aliases.", + params(ListBucketsRequest), responses( (status = 200, description = "Returns the UUID of all the buckets and all their aliases", body = ListBucketsResponse), (status = 500, description = "Internal server error") @@ -876,7 +878,7 @@ impl Modify for SecurityAddon { #[derive(OpenApi)] #[openapi( info( - version = "v2.3.0", + version = "v2.4.0", title = "Garage administration API", description = "Administrate your Garage cluster programmatically, including status, layout, keys, buckets, and maintenance tasks. diff --git a/src/api/admin/router_v2.rs b/src/api/admin/router_v2.rs index 3a9a567d..0096053e 100644 --- a/src/api/admin/router_v2.rs +++ b/src/api/admin/router_v2.rs @@ -55,10 +55,10 @@ impl AdminApiRequest { POST CreateKey (body), POST ImportKey (body), POST DeleteKey (query::id), - GET ListKeys (), + GET ListKeys (parse_default(false)::details, query_opt::offset, opt_parse::limit), // Bucket endpoints GET GetBucketInfo (query_opt::id, query_opt::global_alias, query_opt::search), - GET ListBuckets (), + GET ListBuckets (parse_default(false)::details, query_opt::offset, opt_parse::limit), POST CreateBucket (body), POST DeleteBucket (query::id), POST UpdateBucket (body_field, query::id), @@ -129,7 +129,7 @@ impl AdminApiRequest { )), // Keys - Endpoint::ListKeys => Ok(AdminApiRequest::ListKeys(ListKeysRequest)), + Endpoint::ListKeys => Ok(AdminApiRequest::ListKeys(ListKeysRequest::default())), Endpoint::GetKeyInfo { id, search, @@ -161,7 +161,9 @@ impl AdminApiRequest { // Endpoint::DeleteKey { id } => Ok(AdminApiRequest::DeleteKey(DeleteKeyRequest { id })), // Buckets - Endpoint::ListBuckets => Ok(AdminApiRequest::ListBuckets(ListBucketsRequest)), + Endpoint::ListBuckets => { + Ok(AdminApiRequest::ListBuckets(ListBucketsRequest::default())) + } Endpoint::GetBucketInfo { id, global_alias } => { Ok(AdminApiRequest::GetBucketInfo(GetBucketInfoRequest { id, @@ -271,6 +273,9 @@ generateQueryParameters! { "accessKeyId" => access_key_id, "showSecretKey" => show_secret_key, "bucketId" => bucket_id, - "key" => key + "key" => key, + "details" => details, + "offset" => offset, + "limit" => limit ] } diff --git a/src/garage/cli/remote/bucket.rs b/src/garage/cli/remote/bucket.rs index 314ea041..5937ea5f 100644 --- a/src/garage/cli/remote/bucket.rs +++ b/src/garage/cli/remote/bucket.rs @@ -31,12 +31,19 @@ impl Cli { } pub async fn cmd_list_buckets(&self) -> Result<(), Error> { - let mut buckets = self.api_request(ListBucketsRequest).await?; + let mut buckets = match self.api_request(ListBucketsRequest::default()).await? { + ListBucketsResponse::WithoutDetails(list) => list, + _ => { + return Err(Error::Message( + "Unexpected ListBuckets response format".into(), + )) + } + }; - buckets.0.sort_by_key(|x| x.created); + buckets.sort_by_key(|x| x.created); let mut table = vec!["ID\tCreated\tGlobal aliases\tLocal aliases".to_string()]; - for bucket in buckets.0.iter() { + for bucket in buckets.iter() { table.push(format!( "{:.16}\t{}\t{}\t{}", bucket.id, diff --git a/src/garage/cli/remote/key.rs b/src/garage/cli/remote/key.rs index 54d44b78..063bf4ad 100644 --- a/src/garage/cli/remote/key.rs +++ b/src/garage/cli/remote/key.rs @@ -28,12 +28,15 @@ impl Cli { } pub async fn cmd_list_keys(&self) -> Result<(), Error> { - let mut keys = self.api_request(ListKeysRequest).await?; + let mut keys = match self.api_request(ListKeysRequest::default()).await? { + ListKeysResponse::WithoutDetails(list) => list, + _ => return Err(Error::Message("Unexpected ListKeys response format".into())), + }; - keys.0.sort_by_key(|x| x.created); + keys.sort_by_key(|x| x.created); let mut table = vec!["ID\tCreated\tName\tExpiration".to_string()]; - for key in keys.0.iter() { + for key in keys.iter() { let exp = if key.expired { Cow::from("expired") } else { @@ -243,7 +246,10 @@ impl Cli { } pub async fn cmd_delete_expired_keys(&self, yes: bool) -> Result<(), Error> { - let mut list = self.api_request(ListKeysRequest).await?.0; + let mut list = match self.api_request(ListKeysRequest::default()).await? { + ListKeysResponse::WithoutDetails(list) => list, + _ => return Err(Error::Message("Unexpected ListKeys response format".into())), + }; list.retain(|key| key.expired);