CLI: garage json-api: allow payload to be omitted more often

.woodpecker/helm.yaml

@@ -1,38 +0,0 @@
-steps:
-  helm:
-    when:
-      - event: pull_request
-        branch: main-v2
-      - event: push
-        branch: main-v2
-    image: alpine/helm:4.1.1
-    environment:
-      chart_dir: script/helm/garage
-      chart_name: garage
-      registry_host: codeberg.org
-      registry_user: ${CI_REPO_OWNER}
-      registry_path: ${CI_REPO}
-
-      # To create registry_password:
-      # 1. Go to https://codeberg.org/user/settings/applications.
-      # 2. Create a token with these settings:
-      #   - Token name: https://git.deuxfleurs.fr/3uzbcqje/garage/src/branch/main-v2/.woodpecker/helm.yaml
-      #   - Set package permission to "Read and write".
-      # 3. Click "Generate token".
-      # 4. Copy the token.
-      # 5. Log into https://ci.codeberg.org.
-      # 6. Find your repo, and add a secret with these settings:
-      #   - Name: registry_password
-      #   - Value: <the token you copied earlier>
-      # 7. Click "Add secret".
-
-      registry_password:
-        from_secret: registry_password
-
-    commands:
-      - |
-        if [[ "${CI_PIPELINE_EVENT}" == "push" && "${CI_COMMIT_BRANCH}" == "${CI_REPO_DEFAULT_BRANCH}" ]]; then
-          ./pipeline.sh push
-        else
-          ./pipeline.sh
-        fi

doc/api/garage-admin-v2.json

@@ -12,7 +12,7 @@
       "name": "AGPL-3.0",
       "identifier": "AGPL-3.0"
     },
-    "version": "v2.3.0"
+    "version": "v2.4.0"
   },
   "servers": [
     {
@@ -1243,6 +1243,36 @@
         ],
         "description": "List all the buckets on the cluster with their UUID and their global and local aliases.",
         "operationId": "ListBuckets",
+        "parameters": [
+          {
+            "name": "details",
+            "in": "query",
+            "description": "Returned detailed informations in the same format as GetBucketInfo for each bucket",
+            "required": false,
+            "schema": {
+              "type": "boolean"
+            }
+          },
+          {
+            "name": "offset",
+            "in": "query",
+            "description": "Bucket ID of the first bucket to return",
+            "required": false,
+            "schema": {
+              "type": "string"
+            }
+          },
+          {
+            "name": "limit",
+            "in": "query",
+            "description": "Maximum number of buckets to return in a single call",
+            "required": false,
+            "schema": {
+              "type": "integer",
+              "minimum": 0
+            }
+          }
+        ],
         "responses": {
           "200": {
             "description": "Returns the UUID of all the buckets and all their aliases",
@@ -1267,6 +1297,36 @@
         ],
         "description": "Returns all API access keys in the cluster.",
         "operationId": "ListKeys",
+        "parameters": [
+          {
+            "name": "details",
+            "in": "query",
+            "description": "Returned detailed informations in the same format as GetKeyInfo for each bucket",
+            "required": false,
+            "schema": {
+              "type": "boolean"
+            }
+          },
+          {
+            "name": "offset",
+            "in": "query",
+            "description": "Key ID of the first key to return",
+            "required": false,
+            "schema": {
+              "type": "string"
+            }
+          },
+          {
+            "name": "limit",
+            "in": "query",
+            "description": "Maximum number of keys to return in a single call",
+            "required": false,
+            "schema": {
+              "type": "integer",
+              "minimum": 0
+            }
+          }
+        ],
         "responses": {
           "200": {
             "description": "Returns the key identifier (aka `AWS_ACCESS_KEY_ID`) and its associated, human friendly, name if any (otherwise return an empty string)",
@@ -3200,10 +3260,20 @@
         }
       },
       "ListBucketsResponse": {
-        "type": "array",
+        "oneOf": [
-        "items": {
+          {
-          "$ref": "#/components/schemas/ListBucketsResponseItem"
+            "type": "array",
-        }
+            "items": {
+              "$ref": "#/components/schemas/ListBucketsResponseItem"
+            }
+          },
+          {
+            "type": "array",
+            "items": {
+              "$ref": "#/components/schemas/GetBucketInfoResponse"
+            }
+          }
+        ]
       },
       "ListBucketsResponseItem": {
         "type": "object",
@@ -3236,10 +3306,20 @@
         }
       },
       "ListKeysResponse": {
-        "type": "array",
+        "oneOf": [
-        "items": {
+          {
-          "$ref": "#/components/schemas/ListKeysResponseItem"
+            "type": "array",
-        }
+            "items": {
+              "$ref": "#/components/schemas/ListKeysResponseItem"
+            }
+          },
+          {
+            "type": "array",
+            "items": {
+              "$ref": "#/components/schemas/GetKeyInfoResponse"
+            }
+          }
+        ]
       },
       "ListKeysResponseItem": {
         "type": "object",
@@ -3321,10 +3401,35 @@
           "dbEngine"
         ],
         "properties": {
+          "addr": {
+            "type": [
+              "string",
+              "null"
+            ],
+            "description": "Socket address used by other nodes to connect to this node for RPC"
+          },
+          "dataPartition": {
+            "oneOf": [
+              {
+                "type": "null"
+              },
+              {
+                "$ref": "#/components/schemas/FreeSpaceResp",
+                "description": "Total and available space on the disk partition(s) containing the data\ndirectory(ies)"
+              }
+            ]
+          },
           "dbEngine": {
             "type": "string",
             "description": "database engine used for metadata"
           },
+          "draining": {
+            "type": [
+              "boolean",
+              "null"
+            ],
+            "description": "Whether this node is part of an older layout version and is draining data."
+          },
           "garageFeatures": {
             "type": [
               "array",
@@ -3346,9 +3451,38 @@
             ],
             "description": "hostname of this node"
           },
+          "isUp": {
+            "type": [
+              "boolean",
+              "null"
+            ],
+            "description": "Whether this node is connected in the cluster"
+          },
+          "metadataPartition": {
+            "oneOf": [
+              {
+                "type": "null"
+              },
+              {
+                "$ref": "#/components/schemas/FreeSpaceResp",
+                "description": "Total and available space on the disk partition containing the\nmetadata directory"
+              }
+            ]
+          },
           "nodeId": {
             "type": "string"
           },
+          "role": {
+            "oneOf": [
+              {
+                "type": "null"
+              },
+              {
+                "$ref": "#/components/schemas/NodeAssignedRole",
+                "description": "Role assigned to this node in the current cluster layout"
+              }
+            ]
+          },
           "rustVersion": {
             "type": "string",
             "description": "rustc version with which this garage release was compiled"
@@ -3684,10 +3818,35 @@
                 "dbEngine"
               ],
               "properties": {
+                "addr": {
+                  "type": [
+                    "string",
+                    "null"
+                  ],
+                  "description": "Socket address used by other nodes to connect to this node for RPC"
+                },
+                "dataPartition": {
+                  "oneOf": [
+                    {
+                      "type": "null"
+                    },
+                    {
+                      "$ref": "#/components/schemas/FreeSpaceResp",
+                      "description": "Total and available space on the disk partition(s) containing the data\ndirectory(ies)"
+                    }
+                  ]
+                },
                 "dbEngine": {
                   "type": "string",
                   "description": "database engine used for metadata"
                 },
+                "draining": {
+                  "type": [
+                    "boolean",
+                    "null"
+                  ],
+                  "description": "Whether this node is part of an older layout version and is draining data."
+                },
                 "garageFeatures": {
                   "type": [
                     "array",
@@ -3709,9 +3868,38 @@
                   ],
                   "description": "hostname of this node"
                 },
+                "isUp": {
+                  "type": [
+                    "boolean",
+                    "null"
+                  ],
+                  "description": "Whether this node is connected in the cluster"
+                },
+                "metadataPartition": {
+                  "oneOf": [
+                    {
+                      "type": "null"
+                    },
+                    {
+                      "$ref": "#/components/schemas/FreeSpaceResp",
+                      "description": "Total and available space on the disk partition containing the\nmetadata directory"
+                    }
+                  ]
+                },
                 "nodeId": {
                   "type": "string"
                 },
+                "role": {
+                  "oneOf": [
+                    {
+                      "type": "null"
+                    },
+                    {
+                      "$ref": "#/components/schemas/NodeAssignedRole",
+                      "description": "Role assigned to this node in the current cluster layout"
+                    }
+                  ]
+                },
                 "rustVersion": {
                   "type": "string",
                   "description": "rustc version with which this garage release was compiled"

doc/book/quick-start/_index.md

@@ -133,17 +133,12 @@ Use the following command to launch the Garage server:
 garage server --single-node --default-bucket
 ```
 
-- the `--single-node` flag instructs Garage to automatically configure a
+The `--single-node` flag instructs Garage to automatically configure a single-node cluster without data replication.
-  single-node cluster without data replication;
+The `--default-bucket` flag instructs Garage to create a default access key and a default bucket using the environment variables we defined above.
-- the `--default-bucket` flag instructs Garage to create a default access key
+Both flags are optional and can be omitted, in which case you will have to follow manual configuration steps described below.
-  and a default bucket using the environment variables we defined above (it
-  implies `--default-access-key`).
 
-> You can refer to the [manual configuration
+**For older versions of Garage (before v2.3.0):** automatic configuration using `--single-node` and `--default-bucket` is not available,
-> steps](#manual-configuration) if:
+you must follow the manual configuration steps.
->
-> - you decide to no use these optional flags;
-> - you are running an **older version of Garage (before v2.3.0)**.
 
 Alternatively, if you cannot or do not wish to run the Garage binary directly,
 you may use Docker to run Garage in a container using the following command:
@@ -297,7 +292,7 @@ An exhaustive list is maintained in the ["Integrations" > "Browsing tools" secti
 
 
 
-## Manual configuration {#manual-configuration}
+## Manual configuration
 
 This section provides instructions that are equivalent to using the
 `--single-node` and `--default-bucket` flags for automatic configuration.  If

doc/book/reference-manual/configuration.md

@@ -175,9 +175,6 @@ they do not exist in the configuration file:
   Garage daemon send its logs to `journald` (using the native protocol of `systemd-journald`)
   instead of printing to stderr.
 
-- `NO_COLOR` (since `v2.4.0`): set this to `0` or `false` to disable
-  ANSI color codes in Garage's logs.
-
 The following environment variables can be used to override the corresponding
 values in the configuration file:
 

doc/book/reference-manual/k2v.md

@@ -8,11 +8,12 @@ which is an alternative storage API designed to help efficiently store
 many small values in buckets (in opposition to S3 which is more designed
 to store large blobs).
 
-K2V is included in release builds since version 0.8.0. Precompiled builds
+K2V is currently disabled at compile time in all builds, as the
-of earlier versions including `k2v` can be found in our download page under
+specification is still subject to changes. To build a Garage version with
-"Extra builds": they can be easily identified as their tag name ends with
+K2V, the Cargo feature flag `k2v` must be activated.  Special builds with
-`-k2v` (example: `v0.7.2-k2v`). Otherwise, when compiling Garage, the Cargo
+the `k2v` feature flag enabled can be obtained from our download page under
-feature flag `k2v` must be activated.
+"Extra builds": such builds can be identified easily as their tag name ends
+with `-k2v` (example: `v0.7.2-k2v`).
 
 The specification of the K2V API can be found
 [here](https://git.deuxfleurs.fr/Deuxfleurs/garage/src/commit/f8be15c37db857e177d543de7be863692628d567/doc/drafts/k2v-spec.md).

pipeline.sh

@@ -1,22 +0,0 @@
-#!/usr/bin/env bash
-
-# https://vaneyckt.io/posts/safer_bash_scripts_with_set_euxo_pipefail/
-# Not using "-x" because we aren't debugging.
-set -Eeuo pipefail
-
-# We get unbound var err if we don't set arg
-arg="${1:-}"
-
-echo "${registry_password}" | helm registry login -u "${registry_user}" --password-stdin "${registry_host}"
-helm lint "${chart_dir}"
-chart_version=$(grep '^version: ' "${chart_dir}/Chart.yaml" | cut -d' ' -f 2)
-helm package "${chart_dir}"
-
-# If chart already exists in the chart repository, don't push.
-if helm pull "oci://${registry_host}/${registry_path}/${chart_name}" --version "${chart_version}" &> /dev/null; then
-  echo -e "\e[31mWARNING: Chart ${chart_name} version ${chart_version} already exists in the repository.\nThis means that the chart's code has not changed, or you forgot to update the version in Chart.yaml.\e[0m"
-else
-  if [[ $arg == 'push' ]]; then
-    helm push "${chart_name}-${chart_version}.tgz" "oci://${registry_host}/${registry_path}"
-  fi
-fi

script/dev-bucket.sh

@@ -18,7 +18,7 @@ fi
 
 $GARAGE_BIN -c /tmp/config.1.toml bucket create eprouvette
 if [ "$GARAGE_OLDVER" = "v08" ]; then
-	KEY_INFO=$($GARAGE_BIN -c /tmp/config.1.toml key new --name opérateur)
+	KEY_INFO=$($GARAGE_BIN -c /tmp/config.1.toml key create opérateur)
 	ACCESS_KEY=`echo $KEY_INFO|grep -Po 'GK[a-f0-9]+'`
 	SECRET_KEY=`echo $KEY_INFO|grep -Po 'Secret key: [a-f0-9]+'|grep -Po '[a-f0-9]+$'`
 elif [ "$GARAGE_OLDVER" = "v1" ]; then

src/api/admin/api.rs

@@ -688,11 +688,26 @@ pub struct ClusterLayoutSkipDeadNodesResponse {
 
 // ---- ListKeys ----
 
-#[derive(Debug, Clone, Serialize, Deserialize)]
+#[derive(Debug, Clone, Serialize, Deserialize, Default, IntoParams)]
-pub struct ListKeysRequest;
+#[into_params(parameter_in = Query)]
+pub struct ListKeysRequest {
+	/// Returned detailed informations in the same format as GetKeyInfo for each bucket
+	#[serde(default)]
+	pub details: bool,
+	/// Key ID of the first key to return
+	#[serde(default)]
+	pub offset: Option<String>,
+	/// Maximum number of keys to return in a single call
+	#[serde(default)]
+	pub limit: Option<usize>,
+}
 
 #[derive(Debug, Clone, Serialize, Deserialize, ToSchema)]
-pub struct ListKeysResponse(pub Vec<ListKeysResponseItem>);
+#[serde(untagged)]
+pub enum ListKeysResponse {
+	WithoutDetails(Vec<ListKeysResponseItem>),
+	WithDetails(Vec<GetKeyInfoResponse>),
+}
 
 #[derive(Debug, Clone, Serialize, Deserialize, ToSchema)]
 #[serde(rename_all = "camelCase")]
@@ -830,11 +845,26 @@ pub struct DeleteKeyResponse;
 
 // ---- ListBuckets ----
 
-#[derive(Debug, Clone, Serialize, Deserialize)]
+#[derive(Debug, Clone, Serialize, Deserialize, Default, IntoParams)]
-pub struct ListBucketsRequest;
+#[into_params(parameter_in = Query)]
+pub struct ListBucketsRequest {
+	/// Returned detailed informations in the same format as GetBucketInfo for each bucket
+	#[serde(default)]
+	pub details: bool,
+	/// Bucket ID of the first bucket to return
+	#[serde(default)]
+	pub offset: Option<String>,
+	/// Maximum number of buckets to return in a single call
+	#[serde(default)]
+	pub limit: Option<usize>,
+}
 
 #[derive(Debug, Clone, Serialize, Deserialize, ToSchema)]
-pub struct ListBucketsResponse(pub Vec<ListBucketsResponseItem>);
+#[serde(untagged)]
+pub enum ListBucketsResponse {
+	WithoutDetails(Vec<ListBucketsResponseItem>),
+	WithDetails(Vec<GetBucketInfoResponse>),
+}
 
 #[derive(Debug, Clone, Serialize, Deserialize, ToSchema)]
 #[serde(rename_all = "camelCase")]

src/api/admin/bucket.rs

@@ -3,6 +3,7 @@ use std::sync::Arc;
 use std::time::Duration;
 
 use chrono::DateTime;
+use futures::StreamExt;
 
 use garage_util::crdt::*;
 use garage_util::data::*;
@@ -32,47 +33,70 @@ impl RequestHandler for ListBucketsRequest {
 		garage: &Arc<Garage>,
 		_admin: &Admin,
 	) -> Result<ListBucketsResponse, Error> {
+		let limit = self
+			.limit
+			.unwrap_or_else(|| if self.details { 1000 } else { 10_000 });
+
+		let offset = match self.offset {
+			Some(id) => Some(parse_bucket_id(&id)?),
+			None => None,
+		};
+
 		let buckets = garage
 			.bucket_table
 			.get_range(
 				&EmptyKey,
-				None,
+				offset,
 				Some(DeletedFilter::NotDeleted),
-				1_000_000,
+				limit,
 				EnumerationOrder::Forward,
 			)
 			.await?;
 
-		let res = buckets
+		if self.details {
-			.into_iter()
+			let mut stream = buckets
-			.map(|b| {
+				.into_iter()
-				let state = b.state.as_option().unwrap();
+				.map(|b| bucket_info_results(garage, b.id))
-				ListBucketsResponseItem {
+				.collect::<futures::stream::FuturesOrdered<_>>();
-					id: hex::encode(b.id),
-					created: DateTime::from_timestamp_millis(state.creation_date as i64)
-						.expect("invalid timestamp stored in db"),
-					global_aliases: state
-						.aliases
-						.items()
-						.iter()
-						.filter(|(_, _, a)| *a)
-						.map(|(n, _, _)| n.to_string())
-						.collect::<Vec<_>>(),
-					local_aliases: state
-						.local_aliases
-						.items()
-						.iter()
-						.filter(|(_, _, a)| *a)
-						.map(|((k, n), _, _)| BucketLocalAlias {
-							access_key_id: k.to_string(),
-							alias: n.to_string(),
-						})
-						.collect::<Vec<_>>(),
-				}
-			})
-			.collect::<Vec<_>>();
 
-		Ok(ListBucketsResponse(res))
+			let mut res = vec![];
+			while let Some(next) = stream.next().await {
+				res.push(next?);
+			}
+
+			Ok(ListBucketsResponse::WithDetails(res))
+		} else {
+			let res = buckets
+				.into_iter()
+				.map(|b| {
+					let state = b.state.as_option().unwrap();
+					ListBucketsResponseItem {
+						id: hex::encode(b.id),
+						created: DateTime::from_timestamp_millis(state.creation_date as i64)
+							.expect("invalid timestamp stored in db"),
+						global_aliases: state
+							.aliases
+							.items()
+							.iter()
+							.filter(|(_, _, a)| *a)
+							.map(|(n, _, _)| n.to_string())
+							.collect::<Vec<_>>(),
+						local_aliases: state
+							.local_aliases
+							.items()
+							.iter()
+							.filter(|(_, _, a)| *a)
+							.map(|((k, n), _, _)| BucketLocalAlias {
+								access_key_id: k.to_string(),
+								alias: n.to_string(),
+							})
+							.collect::<Vec<_>>(),
+					}
+				})
+				.collect::<Vec<_>>();
+
+			Ok(ListBucketsResponse::WithoutDetails(res))
+		}
 	}
 }
 

src/api/admin/key.rs

@@ -2,6 +2,7 @@ use std::collections::HashMap;
 use std::sync::Arc;
 
 use chrono::DateTime;
+use futures::StreamExt;
 
 use garage_table::*;
 use garage_util::time::now_msec;
@@ -20,37 +21,57 @@ impl RequestHandler for ListKeysRequest {
 	async fn handle(self, garage: &Arc<Garage>, _admin: &Admin) -> Result<ListKeysResponse, Error> {
 		let now = now_msec();
 
-		let res = garage
+		let limit = self
+			.limit
+			.unwrap_or_else(|| if self.details { 1000 } else { 10_000 });
+
+		let keys = garage
 			.key_table
 			.get_range(
 				&EmptyKey,
-				None,
+				self.offset,
 				Some(KeyFilter::Deleted(DeletedFilter::NotDeleted)),
-				10000,
+				limit,
 				EnumerationOrder::Forward,
 			)
-			.await?
+			.await?;
-			.iter()
-			.map(|k| {
-				let p = k.params().unwrap();
 
-				ListKeysResponseItem {
+		if self.details {
-					id: k.key_id.to_string(),
+			let mut stream = keys
-					name: p.name.get().clone(),
+				.into_iter()
-					created: p.created.map(|x| {
+				.map(|k| key_info_results(garage, k, false))
-						DateTime::from_timestamp_millis(x as i64)
+				.collect::<futures::stream::FuturesOrdered<_>>();
-							.expect("invalid timestamp stored in db")
-					}),
-					expiration: p.expiration.get().inner().map(|x| {
-						DateTime::from_timestamp_millis(x.0 as i64)
-							.expect("invalid timestamp stored in db")
-					}),
-					expired: p.is_expired(now),
-				}
-			})
-			.collect::<Vec<_>>();
 
-		Ok(ListKeysResponse(res))
+			let mut res = vec![];
+			while let Some(next) = stream.next().await {
+				res.push(next?);
+			}
+
+			Ok(ListKeysResponse::WithDetails(res))
+		} else {
+			let res = keys
+				.iter()
+				.map(|k| {
+					let p = k.params().unwrap();
+
+					ListKeysResponseItem {
+						id: k.key_id.to_string(),
+						name: p.name.get().clone(),
+						created: p.created.map(|x| {
+							DateTime::from_timestamp_millis(x as i64)
+								.expect("invalid timestamp stored in db")
+						}),
+						expiration: p.expiration.get().inner().map(|x| {
+							DateTime::from_timestamp_millis(x.0 as i64)
+								.expect("invalid timestamp stored in db")
+						}),
+						expired: p.is_expired(now),
+					}
+				})
+				.collect::<Vec<_>>();
+
+			Ok(ListKeysResponse::WithoutDetails(res))
+		}
 	}
 }
 

src/api/admin/openapi.rs

@@ -364,6 +364,7 @@ fn ClusterLayoutSkipDeadNodes() {}
     path = "/v2/ListKeys",
     tag = "Access key",
     description = "Returns all API access keys in the cluster.",
+    params(ListKeysRequest),
 	responses(
             (status = 200, description = "Returns the key identifier (aka `AWS_ACCESS_KEY_ID`) and its associated, human friendly, name if any (otherwise return an empty string)", body = ListKeysResponse),
             (status = 500, description = "Internal server error")
@@ -453,6 +454,7 @@ fn DeleteKey() {}
     path = "/v2/ListBuckets",
     tag = "Bucket",
     description = "List all the buckets on the cluster with their UUID and their global and local aliases.",
+    params(ListBucketsRequest),
 	responses(
             (status = 200, description = "Returns the UUID of all the buckets and all their aliases", body = ListBucketsResponse),
             (status = 500, description = "Internal server error")
@@ -876,7 +878,7 @@ impl Modify for SecurityAddon {
 #[derive(OpenApi)]
 #[openapi(
     info(
-        version = "v2.3.0",
+        version = "v2.4.0",
         title = "Garage administration API",
         description = "Administrate your Garage cluster programmatically, including status, layout, keys, buckets, and maintenance tasks.
 

src/api/admin/router_v2.rs

@@ -55,10 +55,10 @@ impl AdminApiRequest {
 			POST CreateKey (body),
 			POST ImportKey (body),
 			POST DeleteKey (query::id),
-			GET ListKeys (),
+			GET ListKeys (parse_default(false)::details, query_opt::offset, opt_parse::limit),
 			// Bucket endpoints
 			GET GetBucketInfo (query_opt::id, query_opt::global_alias, query_opt::search),
-			GET ListBuckets (),
+			GET ListBuckets (parse_default(false)::details, query_opt::offset, opt_parse::limit),
 			POST CreateBucket (body),
 			POST DeleteBucket (query::id),
 			POST UpdateBucket (body_field, query::id),
@@ -129,7 +129,7 @@ impl AdminApiRequest {
 			)),
 
 			// Keys
-			Endpoint::ListKeys => Ok(AdminApiRequest::ListKeys(ListKeysRequest)),
+			Endpoint::ListKeys => Ok(AdminApiRequest::ListKeys(ListKeysRequest::default())),
 			Endpoint::GetKeyInfo {
 				id,
 				search,
@@ -161,7 +161,9 @@ impl AdminApiRequest {
 			// Endpoint::DeleteKey { id } => Ok(AdminApiRequest::DeleteKey(DeleteKeyRequest { id })),
 
 			// Buckets
-			Endpoint::ListBuckets => Ok(AdminApiRequest::ListBuckets(ListBucketsRequest)),
+			Endpoint::ListBuckets => {
+				Ok(AdminApiRequest::ListBuckets(ListBucketsRequest::default()))
+			}
 			Endpoint::GetBucketInfo { id, global_alias } => {
 				Ok(AdminApiRequest::GetBucketInfo(GetBucketInfoRequest {
 					id,
@@ -271,6 +273,9 @@ generateQueryParameters! {
 		"accessKeyId" => access_key_id,
 		"showSecretKey" => show_secret_key,
 		"bucketId" => bucket_id,
-		"key" => key
+		"key" => key,
+		"details" => details,
+		"offset" => offset,
+		"limit" => limit
 	]
 }

src/garage/cli/remote/bucket.rs

@@ -31,12 +31,19 @@ impl Cli {
 	}
 
 	pub async fn cmd_list_buckets(&self) -> Result<(), Error> {
-		let mut buckets = self.api_request(ListBucketsRequest).await?;
+		let mut buckets = match self.api_request(ListBucketsRequest::default()).await? {
+			ListBucketsResponse::WithoutDetails(list) => list,
+			_ => {
+				return Err(Error::Message(
+					"Unexpected ListBuckets response format".into(),
+				))
+			}
+		};
 
-		buckets.0.sort_by_key(|x| x.created);
+		buckets.sort_by_key(|x| x.created);
 
 		let mut table = vec!["ID\tCreated\tGlobal aliases\tLocal aliases".to_string()];
-		for bucket in buckets.0.iter() {
+		for bucket in buckets.iter() {
 			table.push(format!(
 				"{:.16}\t{}\t{}\t{}",
 				bucket.id,

src/garage/cli/remote/key.rs

@@ -28,12 +28,15 @@ impl Cli {
 	}
 
 	pub async fn cmd_list_keys(&self) -> Result<(), Error> {
-		let mut keys = self.api_request(ListKeysRequest).await?;
+		let mut keys = match self.api_request(ListKeysRequest::default()).await? {
+			ListKeysResponse::WithoutDetails(list) => list,
+			_ => return Err(Error::Message("Unexpected ListKeys response format".into())),
+		};
 
-		keys.0.sort_by_key(|x| x.created);
+		keys.sort_by_key(|x| x.created);
 
 		let mut table = vec!["ID\tCreated\tName\tExpiration".to_string()];
-		for key in keys.0.iter() {
+		for key in keys.iter() {
 			let exp = if key.expired {
 				Cow::from("expired")
 			} else {
@@ -243,7 +246,10 @@ impl Cli {
 	}
 
 	pub async fn cmd_delete_expired_keys(&self, yes: bool) -> Result<(), Error> {
-		let mut list = self.api_request(ListKeysRequest).await?.0;
+		let mut list = match self.api_request(ListKeysRequest::default()).await? {
+			ListKeysResponse::WithoutDetails(list) => list,
+			_ => return Err(Error::Message("Unexpected ListKeys response format".into())),
+		};
 
 		list.retain(|key| key.expired);
 

src/garage/cli/remote/mod.rs

@@ -110,16 +110,29 @@ impl Cli {
 		Ok(resp.success.into_iter().next().unwrap().1)
 	}
 
-	pub async fn cmd_json_api(&self, endpoint: String, payload: String) -> Result<(), Error> {
+	pub async fn cmd_json_api(
-		let payload: serde_json::Value = if payload == "-" {
+		&self,
-			serde_json::from_reader(&std::io::stdin())?
+		endpoint: String,
-		} else {
+		payload: Option<String>,
-			serde_json::from_str(&payload)?
+	) -> Result<(), Error> {
-		};
+		let request: AdminApiRequest = if let Some(payload) = payload {
+			let payload: serde_json::Value = if payload == "-" {
+				serde_json::from_reader(&std::io::stdin())?
+			} else {
+				serde_json::from_str(&payload)?
+			};
 
-		let request: AdminApiRequest = serde_json::from_value(serde_json::json!({
+			serde_json::from_value(serde_json::json!({
-			endpoint.clone(): payload,
+				endpoint.clone(): payload,
-		}))?;
+			}))?
+		} else {
+			serde_json::from_value(serde_json::json!({
+				endpoint.clone(): null,
+			}))
+			.or(serde_json::from_value(serde_json::json!({
+				endpoint.clone(): {},
+			})))?
+		};
 
 		let resp = match self
 			.proxy_rpc_endpoint

src/garage/cli/structs.rs

@@ -78,8 +78,7 @@ pub enum Command {
 		/// The admin API endpoint to invoke, e.g. `GetClusterStatus`
 		endpoint: String,
 		/// The JSON payload, or `-` to read from `stdin`
-		#[structopt(default_value = "null")]
+		payload: Option<String>,
-		payload: String,
 	},
 
 	/// Generate completions for a shell

src/garage/main.rs

@@ -276,11 +276,6 @@ fn init_logging(opt: &Opt) {
 	tracing_subscriber::fmt()
 		.with_writer(std::io::stderr)
 		.with_env_filter(env_filter)
-		.with_ansi(
-			std::env::var("NO_COLOR")
-				.map(|x| x != "0" && !x.eq_ignore_ascii_case("false"))
-				.unwrap_or(true),
-		)
 		.init();
 }
 

src/garage/server.rs

@@ -152,14 +152,6 @@ pub async fn run_server(
 		}
 	}
 
-	// Deregister from Consul (if enabled) in the background, in parallel with the
-	// rest of the shutdown sequence, so that it doesn't add to shutdown latency.
-	#[cfg(feature = "consul-discovery")]
-	let deregister_consul_task = tokio::spawn({
-		let system = garage.system.clone();
-		async move { system.deregister_from_discovery().await }
-	});
-
 	// Remove RPC handlers for system to break reference cycles
 	info!("Deregistering RPC handlers for shutdown...");
 	garage.system.netapp.drop_all_handlers();
@@ -176,12 +168,6 @@ pub async fn run_server(
 	// Await for all background tasks to end
 	await_background_done.await?;
 
-	// Await for Consul deregistration to end, if it hasn't already
-	#[cfg(feature = "consul-discovery")]
-	if let Err(e) = deregister_consul_task.await {
-		error!("Error while joining Consul deregistration task: {}", e);
-	}
-
 	info!("Cleaning up...");
 
 	Ok(())

src/rpc/consul.rs

@@ -183,36 +183,6 @@ impl ConsulDiscovery {
 	}
 	// ---- PUBLISHING TO CONSUL CATALOG ----
 
-	#[cfg(feature = "consul-discovery")]
-	pub async fn deregister_consul_service(&self, node_id: NodeID) -> Result<(), ConsulError> {
-		let node = format!("garage:{}", hex::encode(&node_id[..8]));
-		let url = format!(
-			"{}/v1/{}",
-			self.config.consul_http_addr,
-			(match &self.config.api {
-				ConsulDiscoveryAPI::Catalog => format!("catalog/deregister"),
-				ConsulDiscoveryAPI::Agent => format!("agent/service/deregister/{}", node),
-			})
-		);
-
-		let req = self.client.put(&url);
-
-		let http = if matches!(&self.config.api, ConsulDiscoveryAPI::Catalog) {
-			let deregister_request = serde_json::json!({
-				"Node": node,
-				"ServiceID": node,
-			});
-			let req = req.json(&deregister_request);
-			req.send().await?
-		} else {
-			req.send().await?
-		};
-		http.error_for_status()?;
-
-		debug!("Deregistered service {} from Consul", node);
-		Ok(())
-	}
-
 	pub async fn publish_consul_service(
 		&self,
 		node_id: NodeID,

src/rpc/system.rs

@@ -358,7 +358,7 @@ impl System {
 		);
 	}
 
-	pub fn cleanup(self: &Arc<Self>) {
+	pub fn cleanup(&self) {
 		// Break reference cycle
 		self.metrics.store(None);
 	}
@@ -650,15 +650,6 @@ impl System {
 		}
 	}
 
-	#[cfg(feature = "consul-discovery")]
-	pub async fn deregister_from_discovery(self: &Arc<Self>) {
-		if let Some(c) = &self.consul_discovery {
-			if let Err(e) = c.deregister_consul_service(self.netapp.id).await {
-				error!("Error while deregistering from Consul: {}", e);
-			}
-		}
-	}
-
 	async fn discovery_loop(self: &Arc<Self>, mut stop_signal: watch::Receiver<bool>) {
 		while !*stop_signal.borrow() {
 			let peers_up = self