Compare commits

..

2 Commits

Author SHA1 Message Date
Alex Auvolat be17e25bee CLI: garage json-api: allow payload to be omitted more often 2026-07-25 15:24:31 +02:00
Alex Auvolat 8d85301808 Admin API: add parameters details, offset and limit to ListKeys and ListBuckets 2026-07-25 15:12:37 +02:00
12 changed files with 395 additions and 105 deletions
+197 -9
View File
@@ -12,7 +12,7 @@
"name": "AGPL-3.0",
"identifier": "AGPL-3.0"
},
"version": "v2.3.0"
"version": "v2.4.0"
},
"servers": [
{
@@ -1243,6 +1243,36 @@
],
"description": "List all the buckets on the cluster with their UUID and their global and local aliases.",
"operationId": "ListBuckets",
"parameters": [
{
"name": "details",
"in": "query",
"description": "Returned detailed informations in the same format as GetBucketInfo for each bucket",
"required": false,
"schema": {
"type": "boolean"
}
},
{
"name": "offset",
"in": "query",
"description": "Bucket ID of the first bucket to return",
"required": false,
"schema": {
"type": "string"
}
},
{
"name": "limit",
"in": "query",
"description": "Maximum number of buckets to return in a single call",
"required": false,
"schema": {
"type": "integer",
"minimum": 0
}
}
],
"responses": {
"200": {
"description": "Returns the UUID of all the buckets and all their aliases",
@@ -1267,6 +1297,36 @@
],
"description": "Returns all API access keys in the cluster.",
"operationId": "ListKeys",
"parameters": [
{
"name": "details",
"in": "query",
"description": "Returned detailed informations in the same format as GetKeyInfo for each bucket",
"required": false,
"schema": {
"type": "boolean"
}
},
{
"name": "offset",
"in": "query",
"description": "Key ID of the first key to return",
"required": false,
"schema": {
"type": "string"
}
},
{
"name": "limit",
"in": "query",
"description": "Maximum number of keys to return in a single call",
"required": false,
"schema": {
"type": "integer",
"minimum": 0
}
}
],
"responses": {
"200": {
"description": "Returns the key identifier (aka `AWS_ACCESS_KEY_ID`) and its associated, human friendly, name if any (otherwise return an empty string)",
@@ -3200,10 +3260,20 @@
}
},
"ListBucketsResponse": {
"type": "array",
"items": {
"$ref": "#/components/schemas/ListBucketsResponseItem"
}
"oneOf": [
{
"type": "array",
"items": {
"$ref": "#/components/schemas/ListBucketsResponseItem"
}
},
{
"type": "array",
"items": {
"$ref": "#/components/schemas/GetBucketInfoResponse"
}
}
]
},
"ListBucketsResponseItem": {
"type": "object",
@@ -3236,10 +3306,20 @@
}
},
"ListKeysResponse": {
"type": "array",
"items": {
"$ref": "#/components/schemas/ListKeysResponseItem"
}
"oneOf": [
{
"type": "array",
"items": {
"$ref": "#/components/schemas/ListKeysResponseItem"
}
},
{
"type": "array",
"items": {
"$ref": "#/components/schemas/GetKeyInfoResponse"
}
}
]
},
"ListKeysResponseItem": {
"type": "object",
@@ -3321,10 +3401,35 @@
"dbEngine"
],
"properties": {
"addr": {
"type": [
"string",
"null"
],
"description": "Socket address used by other nodes to connect to this node for RPC"
},
"dataPartition": {
"oneOf": [
{
"type": "null"
},
{
"$ref": "#/components/schemas/FreeSpaceResp",
"description": "Total and available space on the disk partition(s) containing the data\ndirectory(ies)"
}
]
},
"dbEngine": {
"type": "string",
"description": "database engine used for metadata"
},
"draining": {
"type": [
"boolean",
"null"
],
"description": "Whether this node is part of an older layout version and is draining data."
},
"garageFeatures": {
"type": [
"array",
@@ -3346,9 +3451,38 @@
],
"description": "hostname of this node"
},
"isUp": {
"type": [
"boolean",
"null"
],
"description": "Whether this node is connected in the cluster"
},
"metadataPartition": {
"oneOf": [
{
"type": "null"
},
{
"$ref": "#/components/schemas/FreeSpaceResp",
"description": "Total and available space on the disk partition containing the\nmetadata directory"
}
]
},
"nodeId": {
"type": "string"
},
"role": {
"oneOf": [
{
"type": "null"
},
{
"$ref": "#/components/schemas/NodeAssignedRole",
"description": "Role assigned to this node in the current cluster layout"
}
]
},
"rustVersion": {
"type": "string",
"description": "rustc version with which this garage release was compiled"
@@ -3684,10 +3818,35 @@
"dbEngine"
],
"properties": {
"addr": {
"type": [
"string",
"null"
],
"description": "Socket address used by other nodes to connect to this node for RPC"
},
"dataPartition": {
"oneOf": [
{
"type": "null"
},
{
"$ref": "#/components/schemas/FreeSpaceResp",
"description": "Total and available space on the disk partition(s) containing the data\ndirectory(ies)"
}
]
},
"dbEngine": {
"type": "string",
"description": "database engine used for metadata"
},
"draining": {
"type": [
"boolean",
"null"
],
"description": "Whether this node is part of an older layout version and is draining data."
},
"garageFeatures": {
"type": [
"array",
@@ -3709,9 +3868,38 @@
],
"description": "hostname of this node"
},
"isUp": {
"type": [
"boolean",
"null"
],
"description": "Whether this node is connected in the cluster"
},
"metadataPartition": {
"oneOf": [
{
"type": "null"
},
{
"$ref": "#/components/schemas/FreeSpaceResp",
"description": "Total and available space on the disk partition containing the\nmetadata directory"
}
]
},
"nodeId": {
"type": "string"
},
"role": {
"oneOf": [
{
"type": "null"
},
{
"$ref": "#/components/schemas/NodeAssignedRole",
"description": "Role assigned to this node in the current cluster layout"
}
]
},
"rustVersion": {
"type": "string",
"description": "rustc version with which this garage release was compiled"
+6 -11
View File
@@ -133,17 +133,12 @@ Use the following command to launch the Garage server:
garage server --single-node --default-bucket
```
- the `--single-node` flag instructs Garage to automatically configure a
single-node cluster without data replication;
- the `--default-bucket` flag instructs Garage to create a default access key
and a default bucket using the environment variables we defined above (it
implies `--default-access-key`).
The `--single-node` flag instructs Garage to automatically configure a single-node cluster without data replication.
The `--default-bucket` flag instructs Garage to create a default access key and a default bucket using the environment variables we defined above.
Both flags are optional and can be omitted, in which case you will have to follow manual configuration steps described below.
> You can refer to the [manual configuration
> steps](#manual-configuration) if:
>
> - you decide to no use these optional flags;
> - you are running an **older version of Garage (before v2.3.0)**.
**For older versions of Garage (before v2.3.0):** automatic configuration using `--single-node` and `--default-bucket` is not available,
you must follow the manual configuration steps.
Alternatively, if you cannot or do not wish to run the Garage binary directly,
you may use Docker to run Garage in a container using the following command:
@@ -297,7 +292,7 @@ An exhaustive list is maintained in the ["Integrations" > "Browsing tools" secti
## Manual configuration {#manual-configuration}
## Manual configuration
This section provides instructions that are equivalent to using the
`--single-node` and `--default-bucket` flags for automatic configuration. If
+1 -1
View File
@@ -18,7 +18,7 @@ fi
$GARAGE_BIN -c /tmp/config.1.toml bucket create eprouvette
if [ "$GARAGE_OLDVER" = "v08" ]; then
KEY_INFO=$($GARAGE_BIN -c /tmp/config.1.toml key new --name opérateur)
KEY_INFO=$($GARAGE_BIN -c /tmp/config.1.toml key create opérateur)
ACCESS_KEY=`echo $KEY_INFO|grep -Po 'GK[a-f0-9]+'`
SECRET_KEY=`echo $KEY_INFO|grep -Po 'Secret key: [a-f0-9]+'|grep -Po '[a-f0-9]+$'`
elif [ "$GARAGE_OLDVER" = "v1" ]; then
+36 -6
View File
@@ -688,11 +688,26 @@ pub struct ClusterLayoutSkipDeadNodesResponse {
// ---- ListKeys ----
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ListKeysRequest;
#[derive(Debug, Clone, Serialize, Deserialize, Default, IntoParams)]
#[into_params(parameter_in = Query)]
pub struct ListKeysRequest {
/// Returned detailed informations in the same format as GetKeyInfo for each bucket
#[serde(default)]
pub details: bool,
/// Key ID of the first key to return
#[serde(default)]
pub offset: Option<String>,
/// Maximum number of keys to return in a single call
#[serde(default)]
pub limit: Option<usize>,
}
#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)]
pub struct ListKeysResponse(pub Vec<ListKeysResponseItem>);
#[serde(untagged)]
pub enum ListKeysResponse {
WithoutDetails(Vec<ListKeysResponseItem>),
WithDetails(Vec<GetKeyInfoResponse>),
}
#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)]
#[serde(rename_all = "camelCase")]
@@ -830,11 +845,26 @@ pub struct DeleteKeyResponse;
// ---- ListBuckets ----
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ListBucketsRequest;
#[derive(Debug, Clone, Serialize, Deserialize, Default, IntoParams)]
#[into_params(parameter_in = Query)]
pub struct ListBucketsRequest {
/// Returned detailed informations in the same format as GetBucketInfo for each bucket
#[serde(default)]
pub details: bool,
/// Bucket ID of the first bucket to return
#[serde(default)]
pub offset: Option<String>,
/// Maximum number of buckets to return in a single call
#[serde(default)]
pub limit: Option<usize>,
}
#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)]
pub struct ListBucketsResponse(pub Vec<ListBucketsResponseItem>);
#[serde(untagged)]
pub enum ListBucketsResponse {
WithoutDetails(Vec<ListBucketsResponseItem>),
WithDetails(Vec<GetBucketInfoResponse>),
}
#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)]
#[serde(rename_all = "camelCase")]
+55 -31
View File
@@ -3,6 +3,7 @@ use std::sync::Arc;
use std::time::Duration;
use chrono::DateTime;
use futures::StreamExt;
use garage_util::crdt::*;
use garage_util::data::*;
@@ -32,47 +33,70 @@ impl RequestHandler for ListBucketsRequest {
garage: &Arc<Garage>,
_admin: &Admin,
) -> Result<ListBucketsResponse, Error> {
let limit = self
.limit
.unwrap_or_else(|| if self.details { 1000 } else { 10_000 });
let offset = match self.offset {
Some(id) => Some(parse_bucket_id(&id)?),
None => None,
};
let buckets = garage
.bucket_table
.get_range(
&EmptyKey,
None,
offset,
Some(DeletedFilter::NotDeleted),
1_000_000,
limit,
EnumerationOrder::Forward,
)
.await?;
let res = buckets
.into_iter()
.map(|b| {
let state = b.state.as_option().unwrap();
ListBucketsResponseItem {
id: hex::encode(b.id),
created: DateTime::from_timestamp_millis(state.creation_date as i64)
.expect("invalid timestamp stored in db"),
global_aliases: state
.aliases
.items()
.iter()
.filter(|(_, _, a)| *a)
.map(|(n, _, _)| n.to_string())
.collect::<Vec<_>>(),
local_aliases: state
.local_aliases
.items()
.iter()
.filter(|(_, _, a)| *a)
.map(|((k, n), _, _)| BucketLocalAlias {
access_key_id: k.to_string(),
alias: n.to_string(),
})
.collect::<Vec<_>>(),
}
})
.collect::<Vec<_>>();
if self.details {
let mut stream = buckets
.into_iter()
.map(|b| bucket_info_results(garage, b.id))
.collect::<futures::stream::FuturesOrdered<_>>();
Ok(ListBucketsResponse(res))
let mut res = vec![];
while let Some(next) = stream.next().await {
res.push(next?);
}
Ok(ListBucketsResponse::WithDetails(res))
} else {
let res = buckets
.into_iter()
.map(|b| {
let state = b.state.as_option().unwrap();
ListBucketsResponseItem {
id: hex::encode(b.id),
created: DateTime::from_timestamp_millis(state.creation_date as i64)
.expect("invalid timestamp stored in db"),
global_aliases: state
.aliases
.items()
.iter()
.filter(|(_, _, a)| *a)
.map(|(n, _, _)| n.to_string())
.collect::<Vec<_>>(),
local_aliases: state
.local_aliases
.items()
.iter()
.filter(|(_, _, a)| *a)
.map(|((k, n), _, _)| BucketLocalAlias {
access_key_id: k.to_string(),
alias: n.to_string(),
})
.collect::<Vec<_>>(),
}
})
.collect::<Vec<_>>();
Ok(ListBucketsResponse::WithoutDetails(res))
}
}
}
+44 -23
View File
@@ -2,6 +2,7 @@ use std::collections::HashMap;
use std::sync::Arc;
use chrono::DateTime;
use futures::StreamExt;
use garage_table::*;
use garage_util::time::now_msec;
@@ -20,37 +21,57 @@ impl RequestHandler for ListKeysRequest {
async fn handle(self, garage: &Arc<Garage>, _admin: &Admin) -> Result<ListKeysResponse, Error> {
let now = now_msec();
let res = garage
let limit = self
.limit
.unwrap_or_else(|| if self.details { 1000 } else { 10_000 });
let keys = garage
.key_table
.get_range(
&EmptyKey,
None,
self.offset,
Some(KeyFilter::Deleted(DeletedFilter::NotDeleted)),
10000,
limit,
EnumerationOrder::Forward,
)
.await?
.iter()
.map(|k| {
let p = k.params().unwrap();
.await?;
ListKeysResponseItem {
id: k.key_id.to_string(),
name: p.name.get().clone(),
created: p.created.map(|x| {
DateTime::from_timestamp_millis(x as i64)
.expect("invalid timestamp stored in db")
}),
expiration: p.expiration.get().inner().map(|x| {
DateTime::from_timestamp_millis(x.0 as i64)
.expect("invalid timestamp stored in db")
}),
expired: p.is_expired(now),
}
})
.collect::<Vec<_>>();
if self.details {
let mut stream = keys
.into_iter()
.map(|k| key_info_results(garage, k, false))
.collect::<futures::stream::FuturesOrdered<_>>();
Ok(ListKeysResponse(res))
let mut res = vec![];
while let Some(next) = stream.next().await {
res.push(next?);
}
Ok(ListKeysResponse::WithDetails(res))
} else {
let res = keys
.iter()
.map(|k| {
let p = k.params().unwrap();
ListKeysResponseItem {
id: k.key_id.to_string(),
name: p.name.get().clone(),
created: p.created.map(|x| {
DateTime::from_timestamp_millis(x as i64)
.expect("invalid timestamp stored in db")
}),
expiration: p.expiration.get().inner().map(|x| {
DateTime::from_timestamp_millis(x.0 as i64)
.expect("invalid timestamp stored in db")
}),
expired: p.is_expired(now),
}
})
.collect::<Vec<_>>();
Ok(ListKeysResponse::WithoutDetails(res))
}
}
}
+3 -1
View File
@@ -364,6 +364,7 @@ fn ClusterLayoutSkipDeadNodes() {}
path = "/v2/ListKeys",
tag = "Access key",
description = "Returns all API access keys in the cluster.",
params(ListKeysRequest),
responses(
(status = 200, description = "Returns the key identifier (aka `AWS_ACCESS_KEY_ID`) and its associated, human friendly, name if any (otherwise return an empty string)", body = ListKeysResponse),
(status = 500, description = "Internal server error")
@@ -453,6 +454,7 @@ fn DeleteKey() {}
path = "/v2/ListBuckets",
tag = "Bucket",
description = "List all the buckets on the cluster with their UUID and their global and local aliases.",
params(ListBucketsRequest),
responses(
(status = 200, description = "Returns the UUID of all the buckets and all their aliases", body = ListBucketsResponse),
(status = 500, description = "Internal server error")
@@ -876,7 +878,7 @@ impl Modify for SecurityAddon {
#[derive(OpenApi)]
#[openapi(
info(
version = "v2.3.0",
version = "v2.4.0",
title = "Garage administration API",
description = "Administrate your Garage cluster programmatically, including status, layout, keys, buckets, and maintenance tasks.
+10 -5
View File
@@ -55,10 +55,10 @@ impl AdminApiRequest {
POST CreateKey (body),
POST ImportKey (body),
POST DeleteKey (query::id),
GET ListKeys (),
GET ListKeys (parse_default(false)::details, query_opt::offset, opt_parse::limit),
// Bucket endpoints
GET GetBucketInfo (query_opt::id, query_opt::global_alias, query_opt::search),
GET ListBuckets (),
GET ListBuckets (parse_default(false)::details, query_opt::offset, opt_parse::limit),
POST CreateBucket (body),
POST DeleteBucket (query::id),
POST UpdateBucket (body_field, query::id),
@@ -129,7 +129,7 @@ impl AdminApiRequest {
)),
// Keys
Endpoint::ListKeys => Ok(AdminApiRequest::ListKeys(ListKeysRequest)),
Endpoint::ListKeys => Ok(AdminApiRequest::ListKeys(ListKeysRequest::default())),
Endpoint::GetKeyInfo {
id,
search,
@@ -161,7 +161,9 @@ impl AdminApiRequest {
// Endpoint::DeleteKey { id } => Ok(AdminApiRequest::DeleteKey(DeleteKeyRequest { id })),
// Buckets
Endpoint::ListBuckets => Ok(AdminApiRequest::ListBuckets(ListBucketsRequest)),
Endpoint::ListBuckets => {
Ok(AdminApiRequest::ListBuckets(ListBucketsRequest::default()))
}
Endpoint::GetBucketInfo { id, global_alias } => {
Ok(AdminApiRequest::GetBucketInfo(GetBucketInfoRequest {
id,
@@ -271,6 +273,9 @@ generateQueryParameters! {
"accessKeyId" => access_key_id,
"showSecretKey" => show_secret_key,
"bucketId" => bucket_id,
"key" => key
"key" => key,
"details" => details,
"offset" => offset,
"limit" => limit
]
}
+10 -3
View File
@@ -31,12 +31,19 @@ impl Cli {
}
pub async fn cmd_list_buckets(&self) -> Result<(), Error> {
let mut buckets = self.api_request(ListBucketsRequest).await?;
let mut buckets = match self.api_request(ListBucketsRequest::default()).await? {
ListBucketsResponse::WithoutDetails(list) => list,
_ => {
return Err(Error::Message(
"Unexpected ListBuckets response format".into(),
))
}
};
buckets.0.sort_by_key(|x| x.created);
buckets.sort_by_key(|x| x.created);
let mut table = vec!["ID\tCreated\tGlobal aliases\tLocal aliases".to_string()];
for bucket in buckets.0.iter() {
for bucket in buckets.iter() {
table.push(format!(
"{:.16}\t{}\t{}\t{}",
bucket.id,
+10 -4
View File
@@ -28,12 +28,15 @@ impl Cli {
}
pub async fn cmd_list_keys(&self) -> Result<(), Error> {
let mut keys = self.api_request(ListKeysRequest).await?;
let mut keys = match self.api_request(ListKeysRequest::default()).await? {
ListKeysResponse::WithoutDetails(list) => list,
_ => return Err(Error::Message("Unexpected ListKeys response format".into())),
};
keys.0.sort_by_key(|x| x.created);
keys.sort_by_key(|x| x.created);
let mut table = vec!["ID\tCreated\tName\tExpiration".to_string()];
for key in keys.0.iter() {
for key in keys.iter() {
let exp = if key.expired {
Cow::from("expired")
} else {
@@ -243,7 +246,10 @@ impl Cli {
}
pub async fn cmd_delete_expired_keys(&self, yes: bool) -> Result<(), Error> {
let mut list = self.api_request(ListKeysRequest).await?.0;
let mut list = match self.api_request(ListKeysRequest::default()).await? {
ListKeysResponse::WithoutDetails(list) => list,
_ => return Err(Error::Message("Unexpected ListKeys response format".into())),
};
list.retain(|key| key.expired);
+22 -9
View File
@@ -110,16 +110,29 @@ impl Cli {
Ok(resp.success.into_iter().next().unwrap().1)
}
pub async fn cmd_json_api(&self, endpoint: String, payload: String) -> Result<(), Error> {
let payload: serde_json::Value = if payload == "-" {
serde_json::from_reader(&std::io::stdin())?
} else {
serde_json::from_str(&payload)?
};
pub async fn cmd_json_api(
&self,
endpoint: String,
payload: Option<String>,
) -> Result<(), Error> {
let request: AdminApiRequest = if let Some(payload) = payload {
let payload: serde_json::Value = if payload == "-" {
serde_json::from_reader(&std::io::stdin())?
} else {
serde_json::from_str(&payload)?
};
let request: AdminApiRequest = serde_json::from_value(serde_json::json!({
endpoint.clone(): payload,
}))?;
serde_json::from_value(serde_json::json!({
endpoint.clone(): payload,
}))?
} else {
serde_json::from_value(serde_json::json!({
endpoint.clone(): null,
}))
.or(serde_json::from_value(serde_json::json!({
endpoint.clone(): {},
})))?
};
let resp = match self
.proxy_rpc_endpoint
+1 -2
View File
@@ -78,8 +78,7 @@ pub enum Command {
/// The admin API endpoint to invoke, e.g. `GetClusterStatus`
endpoint: String,
/// The JSON payload, or `-` to read from `stdin`
#[structopt(default_value = "null")]
payload: String,
payload: Option<String>,
},
/// Generate completions for a shell