add boto3 test for STREAMING-UNSIGNED-PAYLOAD-TRAILER

Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/820

.woodpecker/release.yaml

@@ -35,7 +35,15 @@ steps:
       - matrix:
           ARCH: i386
 
-  - name: upgrade tests
+  - name: upgrade tests from v1.0.0
+    image: nixpkgs/nix:nixos-22.05
+    commands:
+      - nix-shell --attr ci --run "./script/test-upgrade.sh v1.0.0 x86_64-unknown-linux-musl" || (cat /tmp/garage.log; false)
+    when:
+      - matrix:
+          ARCH: amd64
+
+  - name: upgrade tests from v0.8.4
     image: nixpkgs/nix:nixos-22.05
     commands:
       - nix-shell --attr ci --run "./script/test-upgrade.sh v0.8.4 x86_64-unknown-linux-musl" || (cat /tmp/garage.log; false)

Cargo.toml

@@ -24,18 +24,18 @@ default-members = ["src/garage"]
 
 # Internal Garage crates
 format_table = { version = "0.1.1", path = "src/format-table" }
-garage_api_common = { version = "1.0.1", path = "src/api/common" }
-garage_api_admin = { version = "1.0.1", path = "src/api/admin" }
-garage_api_s3 = { version = "1.0.1", path = "src/api/s3" }
-garage_api_k2v = { version = "1.0.1", path = "src/api/k2v" }
-garage_block = { version = "1.0.1", path = "src/block" }
-garage_db = { version = "1.0.1", path = "src/db", default-features = false }
-garage_model = { version = "1.0.1", path = "src/model", default-features = false }
-garage_net = { version = "1.0.1", path = "src/net" }
-garage_rpc = { version = "1.0.1", path = "src/rpc" }
-garage_table = { version = "1.0.1", path = "src/table" }
-garage_util = { version = "1.0.1", path = "src/util" }
-garage_web = { version = "1.0.1", path = "src/web" }
+garage_api_common = { version = "2.0.0", path = "src/api/common" }
+garage_api_admin = { version = "2.0.0", path = "src/api/admin" }
+garage_api_s3 = { version = "2.0.0", path = "src/api/s3" }
+garage_api_k2v = { version = "2.0.0", path = "src/api/k2v" }
+garage_block = { version = "2.0.0", path = "src/block" }
+garage_db = { version = "2.0.0", path = "src/db", default-features = false }
+garage_model = { version = "2.0.0", path = "src/model", default-features = false }
+garage_net = { version = "2.0.0", path = "src/net" }
+garage_rpc = { version = "2.0.0", path = "src/rpc" }
+garage_table = { version = "2.0.0", path = "src/table" }
+garage_util = { version = "2.0.0", path = "src/util" }
+garage_web = { version = "2.0.0", path = "src/web" }
 k2v-client = { version = "0.0.4", path = "src/k2v-client" }
 
 # External crates from crates.io
@@ -48,9 +48,10 @@ blake2 = "0.10"
 bytes = "1.0"
 bytesize = "1.1"
 cfg-if = "1.0"
-chrono = "0.4"
+chrono = { version = "0.4", features = ["serde"] }
 crc32fast = "1.4"
 crc32c = "0.6"
+crc64fast-nvme = "1.2"
 crypto-common = "0.1"
 err-derive = "0.3"
 gethostname = "0.4"
@@ -67,6 +68,7 @@ mktemp = "0.5"
 nix = { version = "0.29", default-features = false, features = ["fs"] }
 nom = "7.1"
 parse_duration = "2.1"
+paste = "1.0"
 pin-project = "1.0.12"
 pnet_datalink = "0.34"
 rand = "0.8"
@@ -100,6 +102,7 @@ serde = { version = "1.0", default-features = false, features = ["derive", "rc"]
 serde_bytes = "0.11"
 serde_json = "1.0"
 toml = { version = "0.8", default-features = false, features = ["parse"] }
+utoipa = { version = "5.3.1", features = ["chrono"] }
 
 # newer version requires rust edition 2021
 k8s-openapi = { version = "0.21", features = ["v1_24"] }
@@ -132,8 +135,8 @@ opentelemetry-contrib = "0.9"
 prometheus = "0.13"
 
 # used by the k2v-client crate only
-aws-sigv4 = { version = "1.1" }
-hyper-rustls = { version = "0.26", features = ["http2"] }
+aws-sigv4 = { version = "1.1", default-features = false }
+hyper-rustls = { version = "0.26", default-features = false, features = ["http1", "http2", "ring", "rustls-native-certs"] }
 log = "0.4"
 thiserror = "1.0"
 
@@ -141,8 +144,9 @@ thiserror = "1.0"
 assert-json-diff = "2.0"
 rustc_version = "0.4.0"
 static_init = "1.0"
-aws-sdk-config = "1.13"
-aws-sdk-s3 = "1.14"
+aws-smithy-runtime = { version = "1.8", default-features = false, features = ["tls-rustls"] }
+aws-sdk-config = { version = "1.62", default-features = false }
+aws-sdk-s3 = { version = "1.79", default-features = false, features = ["rt-tokio"] }
 
 [profile.dev]
 #lto = "thin"	# disabled for now, adds 2-4 min to each CI build

doc/api/garage-admin-v0.html

@@ -1,7 +1,7 @@
 <!DOCTYPE html>
 <html>
   <head>
-    <title>Garage Adminstration API v0</title>
+    <title>Garage adminstration API v0</title>
     <!-- needed for adaptive design -->
     <meta charset="utf-8"/>
     <meta name="viewport" content="width=device-width, initial-scale=1">

doc/api/garage-admin-v1.html

@@ -1,7 +1,7 @@
 <!DOCTYPE html>
 <html>
   <head>
-    <title>Garage Adminstration API v0</title>
+    <title>Garage adminstration API v1</title>
     <!-- needed for adaptive design -->
     <meta charset="utf-8"/>
     <meta name="viewport" content="width=device-width, initial-scale=1">

doc/api/garage-admin-v1.yml

@@ -687,7 +687,7 @@ paths:
       operationId: "GetBucketInfo"
       summary: "Get a bucket"
       description: |
-        Given a bucket identifier (`id`) or a global alias (`alias`), get its information.
+        Given a bucket identifier (`id`) or a global alias (`globalAlias`), get its information.
         It includes its aliases, its web configuration, keys that have some permissions
         on it, some statistics (number of objects, size), number of dangling multipart uploads,
         and its quotas (if any).
@@ -701,7 +701,7 @@ paths:
           example: "b4018dc61b27ccb5c64ec1b24f53454bbbd180697c758c4d47a22a8921864a87"
           schema:
             type: string
-        - name: alias
+        - name: globalAlias
           in: query
           description: |
             The exact global alias of one of the existing buckets.

doc/api/garage-admin-v2.html

@@ -0,0 +1,24 @@
+<!DOCTYPE html>
+<html>
+  <head>
+    <title>Garage adminstration API v2</title>
+    <!-- needed for adaptive design -->
+    <meta charset="utf-8"/>
+    <meta name="viewport" content="width=device-width, initial-scale=1">
+    <link href="./css/redoc.css" rel="stylesheet">
+
+    <!--
+    Redoc doesn't change outer page styles
+    -->
+    <style>
+      body {
+        margin: 0;
+        padding: 0;
+      }
+    </style>
+  </head>
+  <body>
+    <redoc spec-url='./garage-admin-v2.json'></redoc>
+    <script src="./redoc.standalone.js"> </script>
+  </body>
+</html>

doc/book/cookbook/kubernetes.md

@@ -86,3 +86,62 @@ helm delete --namespace garage garage
 ```
 
 Note that this will leave behind custom CRD `garagenodes.deuxfleurs.fr`, which must be removed manually if desired.
+
+## Increase PVC size on running Garage instances
+
+Since the Garage Helm chart creates the data and meta PVC based on `StatefulSet` templates, increasing the PVC size can be a bit tricky.
+
+### Confirm the `StorageClass` used for Garage supports volume expansion
+
+Confirm the storage class used for garage.
+
+```bash
+kubectl -n garage get pvc 
+NAME            STATUS   VOLUME                                     CAPACITY   ACCESS MODES   STORAGECLASS     VOLUMEATTRIBUTESCLASS   AGE
+data-garage-0   Bound    pvc-080360c9-8ce3-4acf-8579-1701e57b7f3f   30Gi       RWO            longhorn-local   <unset>                 77d
+data-garage-1   Bound    pvc-ab8ba697-6030-4fc7-ab3c-0d6df9e3dbc0   30Gi       RWO            longhorn-local   <unset>                 5d8h
+data-garage-2   Bound    pvc-3ab37551-0231-4604-986d-136d0fd950ec   30Gi       RWO            longhorn-local   <unset>                 5d5h
+meta-garage-0   Bound    pvc-3b457302-3023-4169-846e-c928c5f2ea65   3Gi        RWO            longhorn-local   <unset>                 77d
+meta-garage-1   Bound    pvc-49ace2b9-5c85-42df-9247-51c4cf64b460   3Gi        RWO            longhorn-local   <unset>                 5d8h
+meta-garage-2   Bound    pvc-99e2e50f-42b4-4128-ae2f-b52629259723   3Gi        RWO            longhorn-local   <unset>                 5d5h
+```
+
+In this case, the storage class is `longhorn-local`. Now, check if `ALLOWVOLUMEEXPANSION` is true for the used `StorageClass`.
+
+```bash
+kubectl get storageclasses.storage.k8s.io longhorn-local
+NAME             PROVISIONER          RECLAIMPOLICY   VOLUMEBINDINGMODE   ALLOWVOLUMEEXPANSION   AGE
+longhorn-local   driver.longhorn.io   Delete          Immediate           true                   103d
+```
+
+If your `StorageClass` does not support volume expansion, double check if you can enable it. Otherwise, your only real option is to spin up a new Garage cluster with increased size and migrate all data over.
+
+If your `StorageClass` supports expansion, you are free to continue.
+
+### Increase the size of the PVCs
+
+Increase the size of all PVCs to your desired size.
+
+```bash
+kubectl -n garage edit pvc data-garage-0
+kubectl -n garage edit pvc data-garage-1
+kubectl -n garage edit pvc data-garage-2
+kubectl -n garage edit pvc meta-garage-0
+kubectl -n garage edit pvc meta-garage-1
+kubectl -n garage edit pvc meta-garage-2
+```
+
+### Increase the size of the `StatefulSet` PVC template
+
+This is an optional step, but if not done, future instances of Garage will be created with the original size from the template.
+
+```bash
+kubectl -n garage delete sts --cascade=orphan garage 
+statefulset.apps "garage" deleted
+```
+
+This will remove the Garage `StatefulSet` but leave the pods running. It may seem destructive but needs to be done this way since edits to the size of PVC templates are prohibited.
+
+### Redeploy the `StatefulSet`
+
+Now the size of future PVCs can be increased, and the Garage Helm chart can be upgraded. The new `StatefulSet` should take ownership of the orphaned pods again.

doc/book/cookbook/real-world.md

@@ -96,14 +96,14 @@ to store 2 TB of data in total.
 ## Get a Docker image
 
 Our docker image is currently named `dxflrs/garage` and is stored on the [Docker Hub](https://hub.docker.com/r/dxflrs/garage/tags?page=1&ordering=last_updated).
-We encourage you to use a fixed tag (eg. `v1.0.1`) and not the `latest` tag.
-For this example, we will use the latest published version at the time of the writing which is `v1.0.1` but it's up to you
+We encourage you to use a fixed tag (eg. `v2.0.0`) and not the `latest` tag.
+For this example, we will use the latest published version at the time of the writing which is `v2.0.0` but it's up to you
 to check [the most recent versions on the Docker Hub](https://hub.docker.com/r/dxflrs/garage/tags?page=1&ordering=last_updated).
 
 For example:
 
 ```
-sudo docker pull dxflrs/garage:v1.0.1
+sudo docker pull dxflrs/garage:v2.0.0
 ```
 
 ## Deploying and configuring Garage
@@ -171,7 +171,7 @@ docker run \
   -v /etc/garage.toml:/etc/garage.toml \
   -v /var/lib/garage/meta:/var/lib/garage/meta \
   -v /var/lib/garage/data:/var/lib/garage/data \
-  dxflrs/garage:v1.0.1
+  dxflrs/garage:v2.0.0
 ```
 
 With this command line, Garage should be started automatically at each boot.
@@ -185,7 +185,7 @@ If you want to use `docker-compose`, you may use the following `docker-compose.y
 version: "3"
 services:
   garage:
-    image: dxflrs/garage:v1.0.1
+    image: dxflrs/garage:v2.0.0
     network_mode: "host"
     restart: unless-stopped
     volumes:

doc/book/operations/upgrading.md

@@ -71,7 +71,7 @@ The entire procedure would look something like this:
 
 2. Take each node offline individually to back up its metadata folder, bring them back online once the backup is done.
   You can do all of the nodes in a single zone at once as that won't impact global cluster availability.
-  Do not try to make a backup of the metadata folder of a running node.
+  Do not try to manually copy the metadata folder of a running node.
 
   **Since Garage v0.9.4,** you can use the `garage meta snapshot --all` command
   to take a simultaneous snapshot of the metadata database files of all your

doc/book/quick-start/_index.md

@@ -132,7 +132,7 @@ docker run \
   -v /etc/garage.toml:/path/to/garage.toml \
   -v /var/lib/garage/meta:/path/to/garage/meta \
   -v /var/lib/garage/data:/path/to/garage/data \
-  dxflrs/garage:v0.9.4
+  dxflrs/garage:v2.0.0
 ```
 
 Under Linux, you can substitute `--network host` for `-p 3900:3900 -p 3901:3901 -p 3902:3902 -p 3903:3903`

doc/book/reference-manual/configuration.md

@@ -80,6 +80,7 @@ add_host_to_metrics = true
 [admin]
 api_bind_addr = "0.0.0.0:3903"
 metrics_token = "BCAdFjoa9G0KJR0WXnHHm7fs1ZAbfpI8iIZ+Z/a2NgI="
+metrics_require_token = true
 admin_token = "UkLeGWEvHnXBqnueR3ISEMWpOnm40jH2tM2HnnL/0F4="
 trace_sink = "http://localhost:4317"
 ```
@@ -145,6 +146,7 @@ The `[s3_web]` section:
 
 The `[admin]` section:
 [`api_bind_addr`](#admin_api_bind_addr),
+[`metrics_require_token`](#admin_metrics_require_token),
 [`metrics_token`/`metrics_token_file`](#admin_metrics_token),
 [`admin_token`/`admin_token_file`](#admin_token),
 [`trace_sink`](#admin_trace_sink),
@@ -154,7 +156,7 @@ The `[admin]` section:
 The following configuration parameter must be specified as an environment
 variable, it does not exist in the configuration file:
 
-- `GARAGE_LOG_TO_SYSLOG` (since v0.9.4): set this to `1` or `true` to make the
+- `GARAGE_LOG_TO_SYSLOG` (since `v0.9.4`): set this to `1` or `true` to make the
   Garage daemon send its logs to `syslog` (using the libc `syslog` function)
   instead of printing to stderr.
 
@@ -300,7 +302,7 @@ data_dir = [
 See [the dedicated documentation page](@/documentation/operations/multi-hdd.md)
 on how to operate Garage in such a setup.
 
-#### `metadata_snapshots_dir` (since Garage `v1.0.2`) {#metadata_snapshots_dir}
+#### `metadata_snapshots_dir` (since `v1.1.0`) {#metadata_snapshots_dir}
 
 The directory in which Garage will store metadata snapshots when it
 performs a snapshot of the metadata database, either when instructed to do
@@ -416,7 +418,7 @@ at the cost of a moderate drop in write performance.
 Similarly to `metatada_fsync`, this is likely not necessary
 if geographical replication is used.
 
-#### `metadata_auto_snapshot_interval` (since Garage v0.9.4) {#metadata_auto_snapshot_interval}
+#### `metadata_auto_snapshot_interval` (since `v0.9.4`) {#metadata_auto_snapshot_interval}
 
 If this value is set, Garage will automatically take a snapshot of the metadata
 DB file at a regular interval and save it in the metadata directory.
@@ -453,7 +455,7 @@ you should delete it from the data directory and then call `garage repair
 blocks` on the node to ensure that it re-obtains a copy from another node on
 the network.
 
-#### `use_local_tz` {#use_local_tz}
+#### `use_local_tz` (since `v1.1.0`) {#use_local_tz}
 
 By default, Garage runs the lifecycle worker every day at midnight in UTC. Set the
 `use_local_tz` configuration value to `true` if you want Garage to run the
@@ -475,7 +477,7 @@ files will remain available. This however means that chunks from existing files
 will not be deduplicated with chunks from newly uploaded files, meaning you
 might use more storage space that is optimally possible.
 
-#### `block_ram_buffer_max` (since v0.9.4) {#block_ram_buffer_max}
+#### `block_ram_buffer_max` (since `v0.9.4`) {#block_ram_buffer_max}
 
 A limit on the total size of data blocks kept in RAM by S3 API nodes awaiting
 to be sent to storage nodes asynchronously.
@@ -562,7 +564,7 @@ the node, even in the case of a NAT: the NAT should be configured to forward the
 port number to the same internal port nubmer. This means that if you have several nodes running
 behind a NAT, they should each use a different RPC port number.
 
-#### `rpc_bind_outgoing`(since v0.9.2) {#rpc_bind_outgoing}
+#### `rpc_bind_outgoing` (since `v0.9.2`) {#rpc_bind_outgoing}
 
 If enabled, pre-bind all sockets for outgoing connections to the same IP address
 used for listening (the IP address specified in `rpc_bind_addr`) before
@@ -767,10 +769,34 @@ See [administration API reference](@/documentation/reference-manual/admin-api.md
 Alternatively, since `v0.8.5`, a path can be used to create a unix socket. Note that for security reasons,
 the socket will have 0220 mode. Make sure to set user and group permissions accordingly.
 
+#### `admin_token`, `admin_token_file` or `GARAGE_ADMIN_TOKEN`, `GARAGE_ADMIN_TOKEN_FILE` (env) {#admin_token}
+
+The token for accessing all administration functions on the admin endpoint,
+with the exception of the metrics endpoint (see `metrics_token`).
+
+You can use any random string for this value. We recommend generating a random
+token with `openssl rand -base64 32`.
+
+For Garage version earlier than `v2.0`, if this token is not set,
+access to these endpoints is disabled entirely.
+
+Since Garage `v2.0`, additional admin API tokens can be defined dynamically
+in your Garage cluster using administration commands. This new admin token system
+is more flexible since it allows admin tokens to have an expiration date,
+and to have a scope restricted to certain admin API functions. If `admin_token`
+is set, it behaves as an admin token without expiration and with full scope.
+Otherwise, only admin API tokens defined dynamically can be used.
+
+`admin_token` was introduced in Garage `v0.7.2`.
+`admin_token_file` and the `GARAGE_ADMIN_TOKEN` environment variable are supported since Garage `v0.8.2`.
+
+`GARAGE_ADMIN_TOKEN_FILE` is supported since `v0.8.5` / `v0.9.1`.
+
 #### `metrics_token`, `metrics_token_file` or `GARAGE_METRICS_TOKEN`, `GARAGE_METRICS_TOKEN_FILE` (env) {#admin_metrics_token}
 
-The token for accessing the Metrics endpoint. If this token is not set, the
-Metrics endpoint can be accessed without access control.
+The token for accessing the Prometheus metrics endpoint (`/metrics`).
+If this token is not set, and unless `metrics_require_token` is set to `true`,
+the metrics endpoint can be accessed without access control.
 
 You can use any random string for this value. We recommend generating a random token with `openssl rand -base64 32`.
 
@@ -779,17 +805,12 @@ You can use any random string for this value. We recommend generating a random t
 
 `GARAGE_METRICS_TOKEN_FILE` is supported since `v0.8.5` / `v0.9.1`.
 
-#### `admin_token`, `admin_token_file` or `GARAGE_ADMIN_TOKEN`, `GARAGE_ADMIN_TOKEN_FILE` (env) {#admin_token}
+#### `metrics_require_token` (since `v2.0.0`) {#admin_metrics_require_token}
 
-The token for accessing all of the other administration endpoints.  If this
-token is not set, access to these endpoints is disabled entirely.
-
-You can use any random string for this value. We recommend generating a random token with `openssl rand -base64 32`.
-
-`admin_token` was introduced in Garage `v0.7.2`.
-`admin_token_file` and the `GARAGE_ADMIN_TOKEN` environment variable are supported since Garage `v0.8.2`.
-
-`GARAGE_ADMIN_TOKEN_FILE` is supported since `v0.8.5` / `v0.9.1`.
+If this is set to `true`, accessing the metrics endpoint will always require
+an access token. Valid tokens include the `metrics_token` if it is set,
+and admin API token defined dynamicaly in Garage which have
+the `Metrics` endpoint in their scope.
 
 #### `trace_sink` {#admin_trace_sink}
 

doc/drafts/admin-api.md

@@ -13,8 +13,9 @@ We will bump the version numbers prefixed to each API endpoint each time the syn
 or semantics change, meaning that code that relies on these endpoints will break
 when changes are introduced.
 
-The Garage administration API was introduced in version 0.7.2, this document
-does not apply to older versions of Garage.
+The Garage administration API was introduced in version 0.7.2, and was
+changed several times.
+This document applies only to the Garage v2 API (starting with Garage v2.0.0).
 
 
 ## Access control
@@ -52,34 +53,28 @@ Returns an HTTP status 200 if the node is ready to answer user's requests,
 and an HTTP status 503 (Service Unavailable) if there are some partitions
 for which a quorum of nodes is not available.
 A simple textual message is also returned in a body with content-type `text/plain`.
-See `/v1/health` for an API that also returns JSON output.
+See `/v2/GetClusterHealth` for an API that also returns JSON output.
+
+### Other special endpoints
+
+#### CheckDomain `GET /check?domain=<domain>`
+
+Checks whether this Garage cluster serves a website for domain `<domain>`.
+Returns HTTP 200 Ok if yes, or HTTP 4xx if no website is available for this domain.
 
 ### Cluster operations
 
-#### GetClusterStatus `GET /v1/status`
+#### GetClusterStatus `GET /v2/GetClusterStatus`
 
 Returns the cluster's current status in JSON, including:
 
-- ID of the node being queried and its version of the Garage daemon
 - Live nodes
 - Currently configured cluster layout
-- Staged changes to the cluster layout
 
 Example response body:
 
 ```json
 {
-  "node": "b10c110e4e854e5aa3f4637681befac755154b20059ec163254ddbfae86b09df",
-  "garageVersion": "v1.0.1",
-  "garageFeatures": [
-    "k2v",
-    "lmdb",
-    "sqlite",
-    "metrics",
-    "bundled-libs"
-  ],
-  "rustVersion": "1.68.0",
-  "dbEngine": "LMDB (using Heed crate)",
   "layoutVersion": 5,
   "nodes": [
     {
@@ -169,7 +164,7 @@ Example response body:
 }
 ```
 
-#### GetClusterHealth `GET /v1/health`
+#### GetClusterHealth `GET /v2/GetClusterHealth`
 
 Returns the cluster's current health in JSON format, with the following variables:
 
@@ -202,7 +197,7 @@ Example response body:
 }
 ```
 
-#### ConnectClusterNodes `POST /v1/connect`
+#### ConnectClusterNodes `POST /v2/ConnectClusterNodes`
 
 Instructs this Garage node to connect to other Garage nodes at specified addresses.
 
@@ -232,7 +227,7 @@ Example response:
 ]
 ```
 
-#### GetClusterLayout `GET /v1/layout`
+#### GetClusterLayout `GET /v2/GetClusterLayout`
 
 Returns the cluster's current layout in JSON, including:
 
@@ -293,7 +288,7 @@ Example response body:
 }
 ```
 
-#### UpdateClusterLayout `POST /v1/layout`
+#### UpdateClusterLayout `POST /v2/UpdateClusterLayout`
 
 Send modifications to the cluster layout. These modifications will
 be included in the staged role changes, visible in subsequent calls
@@ -330,7 +325,7 @@ This returns the new cluster layout with the proposed staged changes,
 as returned by GetClusterLayout.
 
 
-#### ApplyClusterLayout `POST /v1/layout/apply`
+#### ApplyClusterLayout `POST /v2/ApplyClusterLayout`
 
 Applies to the cluster the layout changes currently registered as
 staged layout changes.
@@ -350,23 +345,11 @@ existing layout in the cluster.
 This returns the message describing all the calculations done to compute the new
 layout, as well as the description of the layout as returned by GetClusterLayout.
 
-#### RevertClusterLayout `POST /v1/layout/revert`
+#### RevertClusterLayout `POST /v2/RevertClusterLayout`
 
 Clears all of the staged layout changes.
 
-Request body format:
-
-```json
-{
-  "version": 13
-}
-```
-
-Reverting the staged changes is done by incrementing the version number
-and clearing the contents of the staged change list.
-Similarly to the CLI, the body must include the incremented
-version number, which MUST be 1 + the value of the currently
-existing layout in the cluster.
+This requests contains an empty body.
 
 This returns the new cluster layout with all changes reverted,
 as returned by GetClusterLayout.
@@ -374,7 +357,7 @@ as returned by GetClusterLayout.
 
 ### Access key operations
 
-#### ListKeys `GET /v1/key`
+#### ListKeys `GET /v2/ListKeys`
 
 Returns all API access keys in the cluster.
 
@@ -393,8 +376,8 @@ Example response:
 ]
 ```
 
-#### GetKeyInfo `GET /v1/key?id=<acces key id>`
-#### GetKeyInfo `GET /v1/key?search=<pattern>`
+#### GetKeyInfo `GET /v2/GetKeyInfo?id=<acces key id>`
+#### GetKeyInfo `GET /v2/GetKeyInfo?search=<pattern>`
 
 Returns information about the requested API access key.
 
@@ -468,7 +451,7 @@ Example response:
 }
 ```
 
-#### CreateKey `POST /v1/key`
+#### CreateKey `POST /v2/CreateKey`
 
 Creates a new API access key.
 
@@ -483,7 +466,7 @@ Request body format:
 This returns the key info, including the created secret key,
 in the same format as the result of GetKeyInfo.
 
-#### ImportKey `POST /v1/key/import`
+#### ImportKey `POST /v2/ImportKey`
 
 Imports an existing API key.
 This will check that the imported key is in the valid format, i.e.
@@ -501,7 +484,7 @@ Request body format:
 
 This returns the key info in the same format as the result of GetKeyInfo.
 
-#### UpdateKey `POST /v1/key?id=<acces key id>`
+#### UpdateKey `POST /v2/UpdateKey?id=<acces key id>`
 
 Updates information about the specified API access key.
 
@@ -523,14 +506,14 @@ The possible flags in `allow` and `deny` are: `createBucket`.
 
 This returns the key info in the same format as the result of GetKeyInfo.
 
-#### DeleteKey `DELETE /v1/key?id=<acces key id>`
+#### DeleteKey `POST /v2/DeleteKey?id=<acces key id>`
 
 Deletes an API access key.
 
 
 ### Bucket operations
 
-#### ListBuckets `GET /v1/bucket`
+#### ListBuckets `GET /v2/ListBuckets`
 
 Returns all storage buckets in the cluster.
 
@@ -572,8 +555,8 @@ Example response:
 ]
 ```
 
-#### GetBucketInfo `GET /v1/bucket?id=<bucket id>`
-#### GetBucketInfo `GET /v1/bucket?globalAlias=<alias>`
+#### GetBucketInfo `GET /v2/GetBucketInfo?id=<bucket id>`
+#### GetBucketInfo `GET /v2/GetBucketInfo?globalAlias=<alias>`
 
 Returns information about the requested storage bucket.
 
@@ -616,7 +599,7 @@ Example response:
 }
 ```
 
-#### CreateBucket `POST /v1/bucket`
+#### CreateBucket `POST /v2/CreateBucket`
 
 Creates a new storage bucket.
 
@@ -656,7 +639,7 @@ or no alias at all.
 Technically, you can also specify both `globalAlias` and `localAlias` and that would create
 two aliases, but I don't see why you would want to do that.
 
-#### UpdateBucket `PUT /v1/bucket?id=<bucket id>`
+#### UpdateBucket `POST /v2/UpdateBucket?id=<bucket id>`
 
 Updates configuration of the given bucket.
 
@@ -688,16 +671,38 @@ In `quotas`: new values of `maxSize` and `maxObjects` must both be specified, or
 to remove the quotas. An absent value will be considered the same as a `null`. It is not possible
 to change only one of the two quotas.
 
-#### DeleteBucket `DELETE /v1/bucket?id=<bucket id>`
+#### DeleteBucket `POST /v2/DeleteBucket?id=<bucket id>`
 
 Deletes a storage bucket. A bucket cannot be deleted if it is not empty.
 
 Warning: this will delete all aliases associated with the bucket!
 
+#### CleanupIncompleteUploads `POST /v2/CleanupIncompleteUploads`
+
+Cleanup all incomplete uploads in a bucket that are older than a specified number
+of seconds.
+
+Request body format:
+
+```json
+{
+    "bucketId": "e6a14cd6a27f48684579ec6b381c078ab11697e6bc8513b72b2f5307e25fff9b",
+    "olderThanSecs": 3600
+}
+```
+
+Response format
+
+```json
+{
+    "uploadsDeleted": 12
+}
+```
+
 
 ### Operations on permissions for keys on buckets
 
-#### BucketAllowKey `POST /v1/bucket/allow`
+#### AllowBucketKey `POST /v2/AllowBucketKey`
 
 Allows a key to do read/write/owner operations on a bucket.
 
@@ -718,7 +723,7 @@ Request body format:
 Flags in `permissions` which have the value `true` will be activated.
 Other flags will remain unchanged.
 
-#### BucketDenyKey `POST /v1/bucket/deny`
+#### DenyBucketKey `POST /v2/DenyBucketKey`
 
 Denies a key from doing read/write/owner operations on a bucket.
 
@@ -742,19 +747,35 @@ Other flags will remain unchanged.
 
 ### Operations on bucket aliases
 
-#### GlobalAliasBucket `PUT /v1/bucket/alias/global?id=<bucket id>&alias=<global alias>`
+#### AddBucketAlias `POST /v2/AddBucketAlias`
 
-Empty body. Creates a global alias for a bucket.
+Creates an alias for a bucket in the namespace of a specific access key.
+To create a global alias, specify the `globalAlias` field.
+To create a local alias, specify the `localAlias` and `accessKeyId` fields.
 
-#### GlobalUnaliasBucket `DELETE /v1/bucket/alias/global?id=<bucket id>&alias=<global alias>`
+Request body format:
 
-Removes a global alias for a bucket.
+```json
+{
+    "bucketId": "e6a14cd6a27f48684579ec6b381c078ab11697e6bc8513b72b2f5307e25fff9b",
+    "globalAlias": "my-bucket"
+}
+```
 
-#### LocalAliasBucket `PUT /v1/bucket/alias/local?id=<bucket id>&accessKeyId=<access key ID>&alias=<local alias>`
+or:
 
-Empty body. Creates a local alias for a bucket in the namespace of a specific access key.
+```json
+{
+    "bucketId": "e6a14cd6a27f48684579ec6b381c078ab11697e6bc8513b72b2f5307e25fff9b",
+    "accessKeyId": "GK31c2f218a2e44f485b94239e",
+    "localAlias": "my-bucket"
+}
+```
 
-#### LocalUnaliasBucket `DELETE /v1/bucket/alias/local?id=<bucket id>&accessKeyId<access key ID>&alias=<local alias>`
+#### RemoveBucketAlias `POST /v2/RemoveBucketAlias`
 
-Removes a local alias for a bucket in the namespace of a specific access key.
+Removes an alias for a bucket in the namespace of a specific access key.
+To remove a global alias, specify the `globalAlias` field.
+To remove a local alias, specify the `localAlias` and `accessKeyId` fields.
 
+Request body format: same as AddBucketAlias.

flake.lock

@@ -50,17 +50,17 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1736692550,
-        "narHash": "sha256-7tk8xH+g0sJkKLTJFOxphJxxOjMDFMWv24nXslaU2ro=",
+        "lastModified": 1747825515,
+        "narHash": "sha256-BWpMQymVI73QoKZdcVCxUCCK3GNvr/xa2Dc4DM1o2BE=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "7c4869c47090dd7f9f1bdfb49a22aea026996815",
+        "rev": "cd2812de55cf87df88a9e09bf3be1ce63d50c1a6",
         "type": "github"
       },
       "original": {
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "7c4869c47090dd7f9f1bdfb49a22aea026996815",
+        "rev": "cd2812de55cf87df88a9e09bf3be1ce63d50c1a6",
         "type": "github"
       }
     },

flake.nix

@@ -2,9 +2,9 @@
   description =
     "Garage, an S3-compatible distributed object store for self-hosted deployments";
 
-  # Nixpkgs 24.11 as of 2025-01-12
+  # Nixpkgs 25.05 as of 2025-05-22
   inputs.nixpkgs.url =
-    "github:NixOS/nixpkgs/7c4869c47090dd7f9f1bdfb49a22aea026996815";
+    "github:NixOS/nixpkgs/cd2812de55cf87df88a9e09bf3be1ce63d50c1a6";
 
   # Rust overlay as of 2025-02-03
   inputs.rust-overlay.url =

script/dev-bucket.sh

@@ -17,13 +17,19 @@ else
 fi
 
 $GARAGE_BIN -c /tmp/config.1.toml bucket create eprouvette
-if [ "$GARAGE_08" = "1" ]; then
+if [ "$GARAGE_OLDVER" = "v08" ]; then
 	KEY_INFO=$($GARAGE_BIN -c /tmp/config.1.toml key new --name opérateur)
-else
+	ACCESS_KEY=`echo $KEY_INFO|grep -Po 'GK[a-f0-9]+'`
+	SECRET_KEY=`echo $KEY_INFO|grep -Po 'Secret key: [a-f0-9]+'|grep -Po '[a-f0-9]+$'`
+elif [ "$GARAGE_OLDVER" = "v1" ]; then
 	KEY_INFO=$($GARAGE_BIN -c /tmp/config.1.toml key create opérateur)
+	ACCESS_KEY=`echo $KEY_INFO|grep -Po 'GK[a-f0-9]+'`
+	SECRET_KEY=`echo $KEY_INFO|grep -Po 'Secret key: [a-f0-9]+'|grep -Po '[a-f0-9]+$'`
+else
+	KEY_INFO=$($GARAGE_BIN -c /tmp/config.1.toml json-api CreateKey '{"name":"opérateur"}')
+	ACCESS_KEY=`echo $KEY_INFO|jq -r .accessKeyId`
+	SECRET_KEY=`echo $KEY_INFO|jq -r .secretAccessKey`
 fi
-ACCESS_KEY=`echo $KEY_INFO|grep -Po 'GK[a-f0-9]+'`
-SECRET_KEY=`echo $KEY_INFO|grep -Po 'Secret key: [a-f0-9]+'|grep -Po '[a-f0-9]+$'`
 $GARAGE_BIN -c /tmp/config.1.toml bucket allow eprouvette --read --write --owner --key $ACCESS_KEY
 echo "$ACCESS_KEY $SECRET_KEY" > /tmp/garage.s3
 

script/dev-cluster.sh

@@ -11,7 +11,7 @@ PATH="${GARAGE_DEBUG}:${GARAGE_RELEASE}:${NIX_RELEASE}:$PATH"
 FANCYCOLORS=("41m" "42m" "44m" "45m" "100m" "104m")
 
 export RUST_BACKTRACE=1 
-export RUST_LOG=garage=info,garage_api=debug
+export RUST_LOG=garage=info,garage_api_common=debug,garage_api_s3=debug
 MAIN_LABEL="\e[${FANCYCOLORS[0]}[main]\e[49m"
 
 if [ -z "$GARAGE_BIN" ]; then

script/dev-configure.sh

@@ -29,7 +29,7 @@ until $GARAGE_BIN -c /tmp/config.1.toml status 2>&1|grep -q HEALTHY ; do
 	sleep 1
 done
 
-if [ "$GARAGE_08" = "1" ]; then
+if [ "$GARAGE_OLDVER" = "v08" ]; then
 	$GARAGE_BIN -c /tmp/config.1.toml status \
 		| grep 'NO ROLE' \
 		| grep -Po '^[0-9a-f]+' \

script/helm/README.md

@@ -1,3 +1,3 @@
 # Garage helm3 chart
 
-Documentation is located [here](/doc/book/cookbook/kubernetes.md).
+Documentation is located [here](https://garagehq.deuxfleurs.fr/documentation/cookbook/kubernetes/).

script/helm/garage/Chart.yaml

@@ -15,10 +15,10 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.6.0
+version: 0.8.0
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to
 # follow Semantic Versioning. They should reflect the version the application is using.
 # It is recommended to use it with quotes.
-appVersion: "v1.0.1"
+appVersion: "v2.0.0"

script/helm/garage/templates/service-headless.yaml

@@ -0,0 +1,22 @@
+{{- if eq .Values.deployment.kind "StatefulSet" -}}
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "garage.fullname" . }}-headless
+  labels:
+    {{- include "garage.labels" . | nindent 4 }}
+spec:
+  type: ClusterIP
+  clusterIP: None
+  ports:
+    - port: {{ .Values.service.s3.api.port }}
+      targetPort: 3900
+      protocol: TCP
+      name: s3-api
+    - port: {{ .Values.service.s3.web.port }}
+      targetPort: 3902
+      protocol: TCP
+      name: s3-web
+  selector:
+    {{- include "garage.selectorLabels" . | nindent 4 }}
+{{- end }}

script/helm/garage/templates/workload.yaml

@@ -10,12 +10,11 @@ spec:
       {{- include "garage.selectorLabels" . | nindent 6 }}
   {{- if eq .Values.deployment.kind "StatefulSet" }}
   replicas: {{ .Values.deployment.replicaCount }}
-  serviceName: {{ include "garage.fullname" . }}
+  serviceName: {{ include "garage.fullname" . }}-headless
   podManagementPolicy: {{ .Values.deployment.podManagementPolicy }}
   {{- end }}
   template:
     metadata:
-
       annotations:
         checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }}
       {{- with .Values.podAnnotations }}

script/test-smoke.sh

@@ -112,6 +112,19 @@ if [ -z "$SKIP_S3CMD" ]; then
   done
 fi
 
+# BOTO3
+if [ -z "$SKIP_BOTO3" ]; then
+  echo "🛠️ Testing with boto3 for STREAMING-UNSIGNED-PAYLOAD-TRAILER"
+  source ${SCRIPT_FOLDER}/dev-env-aws.sh
+  AWS_ENDPOINT_URL=https://localhost:4443 python <<EOF
+import boto3
+client = boto3.client('s3', verify=False)
+client.put_object(Body=b'hello world', Bucket='eprouvette', Key='test.s3.txt')
+client.delete_object(Bucket='eprouvette', Key='test.s3.txt')
+print("OK!")
+EOF
+fi
+
 # Minio Client
 if [ -z "$SKIP_MC" ]; then
   echo "🛠️ Testing with mc (minio client)"

script/test-upgrade.sh

@@ -24,9 +24,17 @@ echo "============= insert data into old version cluster ================="
 export GARAGE_BIN=/tmp/old_garage
 if echo $OLD_VERSION | grep 'v0\.8\.'; then
 	echo "Detected Garage v0.8.x"
-	export GARAGE_08=1
+	export GARAGE_OLDVER=v08
+elif (echo $OLD_VERSION | grep 'v0\.9\.') || (echo $OLD_VERSION | grep 'v1\.'); then
+	echo "Detected Garage v0.9.x / v1.x"
+	export GARAGE_OLDVER=v1
 fi
 
+if echo $OLD_VERSION | grep 'v1\.'; then
+	DO_SSEC_TEST=1
+fi
+SSEC_KEY="u8zCfnEyt5Imo/krN+sxA1DQXxLWtPJavU6T6gOVj1Y="
+
 echo "⏳ Setup cluster using old version"
 $GARAGE_BIN --version
 ${SCRIPT_FOLDER}/dev-clean.sh
@@ -37,7 +45,23 @@ ${SCRIPT_FOLDER}/dev-bucket.sh
 
 echo "🛠️ Inserting data in old cluster"
 source ${SCRIPT_FOLDER}/dev-env-rclone.sh
-rclone copy "${SCRIPT_FOLDER}/../.git/" garage:eprouvette/test_dotgit --stats=1s --stats-log-level=NOTICE --stats-one-line
+rclone copy "${SCRIPT_FOLDER}/../.git/" garage:eprouvette/test_dotgit \
+	--stats=1s --stats-log-level=NOTICE --stats-one-line
+
+if [ "$DO_SSEC_TEST" = "1" ]; then
+	# upload small file (should be single part)
+	rclone copy "${SCRIPT_FOLDER}/test-upgrade.sh" garage:eprouvette/test-ssec \
+		--s3-sse-customer-algorithm AES256 \
+		--s3-sse-customer-key-base64 "$SSEC_KEY" \
+		--stats=1s --stats-log-level=NOTICE --stats-one-line
+	# do a multipart upload
+	dd if=/dev/urandom of=/tmp/randfile-for-upgrade bs=5M count=5
+	rclone copy "/tmp/randfile-for-upgrade" garage:eprouvette/test-ssec \
+		--s3-chunk-size 5M \
+		--s3-sse-customer-algorithm AES256 \
+		--s3-sse-customer-key-base64 "$SSEC_KEY" \
+		--stats=1s --stats-log-level=NOTICE --stats-one-line
+fi
 
 echo "🏁 Stopping old cluster"
 killall -INT old_garage
@@ -47,7 +71,7 @@ killall -9 old_garage || true
 echo "🏁 Removing old garage version"
 rm -rv $GARAGE_BIN
 export -n GARAGE_BIN
-export -n GARAGE_08
+export -n GARAGE_OLDVER
 
 echo "================ read data from new cluster ==================="
 
@@ -60,7 +84,8 @@ ${SCRIPT_FOLDER}/dev-cluster.sh >> /tmp/garage.log 2>&1 &
 sleep 3
 
 echo "🛠️ Retrieving data from old cluster"
-rclone copy garage:eprouvette/test_dotgit /tmp/test_dotgit --stats=1s --stats-log-level=NOTICE --stats-one-line --fast-list
+rclone copy garage:eprouvette/test_dotgit /tmp/test_dotgit \
+	--stats=1s --stats-log-level=NOTICE --stats-one-line --fast-list
 
 if ! diff <(find "${SCRIPT_FOLDER}/../.git" -type f | xargs md5sum | cut -d ' ' -f 1 | sort) <(find /tmp/test_dotgit -type f | xargs md5sum | cut -d ' ' -f 1 | sort); then
 	echo "TEST FAILURE: directories are different"
@@ -68,6 +93,23 @@ if ! diff <(find "${SCRIPT_FOLDER}/../.git" -type f | xargs md5sum | cut -d ' '
 fi
 rm -r /tmp/test_dotgit
 
+if [ "$DO_SSEC_TEST" = "1" ]; then
+	rclone copy garage:eprouvette/test-ssec /tmp/test_ssec_out \
+		--s3-sse-customer-algorithm AES256 \
+		--s3-sse-customer-key-base64 "$SSEC_KEY" \
+		--stats=1s --stats-log-level=NOTICE --stats-one-line
+	if ! diff "/tmp/test_ssec_out/test-upgrade.sh" "${SCRIPT_FOLDER}/test-upgrade.sh"; then
+		echo "SSEC-FAILURE (small file)"
+		exit 1
+	fi
+	if ! diff "/tmp/test_ssec_out/randfile-for-upgrade" "/tmp/randfile-for-upgrade"; then
+		echo "SSEC-FAILURE (big file)"
+		exit 1
+	fi
+	rm -r /tmp/test_ssec_out
+	rm /tmp/randfile-for-upgrade
+fi
+
 echo "🏁 Teardown"
 rm -rf /tmp/garage-{data,meta}-*
 rm -rf /tmp/config.*.toml

shell.nix

@@ -26,6 +26,8 @@ in
       s3cmd
       minio-client
       rclone
+      (python312.withPackages (ps: [ ps.boto3 ]))
+
       socat
       psmisc
       which

src/api/admin/Cargo.toml

@@ -1,12 +1,12 @@
 [package]
 name = "garage_api_admin"
-version = "1.0.1"
+version = "2.0.0"
 authors = ["Alex Auvolat <alex@adnab.me>"]
 edition = "2018"
 license = "AGPL-3.0"
 description = "Admin API server crate for the Garage object store"
 repository = "https://git.deuxfleurs.fr/Deuxfleurs/garage"
-readme = "../../README.md"
+readme = "../../../README.md"
 
 [lib]
 path = "lib.rs"
@@ -14,7 +14,9 @@ path = "lib.rs"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
+format_table.workspace = true
 garage_model.workspace = true
+garage_block.workspace = true
 garage_table.workspace = true
 garage_util.workspace = true
 garage_rpc.workspace = true
@@ -22,8 +24,11 @@ garage_api_common.workspace = true
 
 argon2.workspace = true
 async-trait.workspace = true
+bytesize.workspace = true
+chrono.workspace = true
 err-derive.workspace = true
 hex.workspace = true
+paste.workspace = true
 tracing.workspace = true
 
 futures.workspace = true
@@ -34,6 +39,7 @@ url.workspace = true
 
 serde.workspace = true
 serde_json.workspace = true
+utoipa.workspace = true
 
 opentelemetry.workspace = true
 opentelemetry-prometheus = { workspace = true, optional = true }
@@ -41,3 +47,4 @@ prometheus = { workspace = true, optional = true }
 
 [features]
 metrics = [ "opentelemetry-prometheus", "prometheus" ]
+k2v = [ "garage_model/k2v" ]

src/api/admin/admin_token.rs

@@ -0,0 +1,235 @@
+use std::sync::Arc;
+
+use chrono::{DateTime, Utc};
+
+use garage_table::*;
+use garage_util::time::now_msec;
+
+use garage_model::admin_token_table::*;
+use garage_model::garage::Garage;
+
+use crate::api::*;
+use crate::error::*;
+use crate::{Admin, RequestHandler};
+
+impl RequestHandler for ListAdminTokensRequest {
+	type Response = ListAdminTokensResponse;
+
+	async fn handle(
+		self,
+		garage: &Arc<Garage>,
+		_admin: &Admin,
+	) -> Result<ListAdminTokensResponse, Error> {
+		let now = now_msec();
+
+		let mut res = garage
+			.admin_token_table
+			.get_range(
+				&EmptyKey,
+				None,
+				Some(KeyFilter::Deleted(DeletedFilter::NotDeleted)),
+				10000,
+				EnumerationOrder::Forward,
+			)
+			.await?
+			.iter()
+			.map(|t| admin_token_info_results(t, now))
+			.collect::<Vec<_>>();
+
+		if garage.config.admin.metrics_token.is_some() {
+			res.insert(
+				0,
+				GetAdminTokenInfoResponse {
+					id: None,
+					created: None,
+					name: "metrics_token (from daemon configuration)".into(),
+					expiration: None,
+					expired: false,
+					scope: vec!["Metrics".into()],
+				},
+			);
+		}
+
+		if garage.config.admin.admin_token.is_some() {
+			res.insert(
+				0,
+				GetAdminTokenInfoResponse {
+					id: None,
+					created: None,
+					name: "admin_token (from daemon configuration)".into(),
+					expiration: None,
+					expired: false,
+					scope: vec!["*".into()],
+				},
+			);
+		}
+
+		Ok(ListAdminTokensResponse(res))
+	}
+}
+
+impl RequestHandler for GetAdminTokenInfoRequest {
+	type Response = GetAdminTokenInfoResponse;
+
+	async fn handle(
+		self,
+		garage: &Arc<Garage>,
+		_admin: &Admin,
+	) -> Result<GetAdminTokenInfoResponse, Error> {
+		let token = match (self.id, self.search) {
+			(Some(id), None) => get_existing_admin_token(garage, &id).await?,
+			(None, Some(search)) => {
+				let candidates = garage
+					.admin_token_table
+					.get_range(
+						&EmptyKey,
+						None,
+						Some(KeyFilter::MatchesAndNotDeleted(search.to_string())),
+						10,
+						EnumerationOrder::Forward,
+					)
+					.await?
+					.into_iter()
+					.collect::<Vec<_>>();
+				if candidates.len() != 1 {
+					return Err(Error::bad_request(format!(
+						"{} matching admin tokens",
+						candidates.len()
+					)));
+				}
+				candidates.into_iter().next().unwrap()
+			}
+			_ => {
+				return Err(Error::bad_request(
+					"Either id or search must be provided (but not both)",
+				));
+			}
+		};
+
+		Ok(admin_token_info_results(&token, now_msec()))
+	}
+}
+
+impl RequestHandler for CreateAdminTokenRequest {
+	type Response = CreateAdminTokenResponse;
+
+	async fn handle(
+		self,
+		garage: &Arc<Garage>,
+		_admin: &Admin,
+	) -> Result<CreateAdminTokenResponse, Error> {
+		let (mut token, secret) = if self.0.name.is_some() {
+			AdminApiToken::new("")
+		} else {
+			AdminApiToken::new(&format!("token_{}", Utc::now().format("%Y%m%d_%H%M")))
+		};
+
+		apply_token_updates(&mut token, self.0)?;
+
+		garage.admin_token_table.insert(&token).await?;
+
+		Ok(CreateAdminTokenResponse {
+			secret_token: secret,
+			info: admin_token_info_results(&token, now_msec()),
+		})
+	}
+}
+
+impl RequestHandler for UpdateAdminTokenRequest {
+	type Response = UpdateAdminTokenResponse;
+
+	async fn handle(
+		self,
+		garage: &Arc<Garage>,
+		_admin: &Admin,
+	) -> Result<UpdateAdminTokenResponse, Error> {
+		let mut token = get_existing_admin_token(&garage, &self.id).await?;
+
+		apply_token_updates(&mut token, self.body)?;
+
+		garage.admin_token_table.insert(&token).await?;
+
+		Ok(UpdateAdminTokenResponse(admin_token_info_results(
+			&token,
+			now_msec(),
+		)))
+	}
+}
+
+impl RequestHandler for DeleteAdminTokenRequest {
+	type Response = DeleteAdminTokenResponse;
+
+	async fn handle(
+		self,
+		garage: &Arc<Garage>,
+		_admin: &Admin,
+	) -> Result<DeleteAdminTokenResponse, Error> {
+		let token = get_existing_admin_token(&garage, &self.id).await?;
+
+		garage
+			.admin_token_table
+			.insert(&AdminApiToken::delete(token.prefix))
+			.await?;
+
+		Ok(DeleteAdminTokenResponse)
+	}
+}
+
+// ---- helpers ----
+
+fn admin_token_info_results(token: &AdminApiToken, now: u64) -> GetAdminTokenInfoResponse {
+	let params = token.params().unwrap();
+
+	GetAdminTokenInfoResponse {
+		id: Some(token.prefix.clone()),
+		created: Some(
+			DateTime::from_timestamp_millis(params.created as i64)
+				.expect("invalid timestamp stored in db"),
+		),
+		name: params.name.get().to_string(),
+		expiration: params.expiration.get().map(|x| {
+			DateTime::from_timestamp_millis(x as i64).expect("invalid timestamp stored in db")
+		}),
+		expired: params.is_expired(now),
+		scope: params.scope.get().0.clone(),
+	}
+}
+
+async fn get_existing_admin_token(garage: &Garage, id: &String) -> Result<AdminApiToken, Error> {
+	garage
+		.admin_token_table
+		.get(&EmptyKey, id)
+		.await?
+		.filter(|k| !k.state.is_deleted())
+		.ok_or_else(|| Error::NoSuchAdminToken(id.to_string()))
+}
+
+fn apply_token_updates(
+	token: &mut AdminApiToken,
+	updates: UpdateAdminTokenRequestBody,
+) -> Result<(), Error> {
+	if updates.never_expires && updates.expiration.is_some() {
+		return Err(Error::bad_request(
+			"cannot specify `expiration` and `never_expires`",
+		));
+	}
+
+	let params = token.params_mut().unwrap();
+
+	if let Some(name) = updates.name {
+		params.name.update(name);
+	}
+	if let Some(expiration) = updates.expiration {
+		params
+			.expiration
+			.update(Some(expiration.timestamp_millis() as u64));
+	}
+	if updates.never_expires {
+		params.expiration.update(None);
+	}
+	if let Some(scope) = updates.scope {
+		params.scope.update(AdminApiTokenScope(scope));
+	}
+
+	Ok(())
+}

src/api/admin/api_server.rs

@@ -1,333 +1,230 @@
-use std::collections::HashMap;
+use std::borrow::Cow;
 use std::sync::Arc;
 
-use argon2::password_hash::PasswordHash;
-
-use http::header::{ACCESS_CONTROL_ALLOW_METHODS, ACCESS_CONTROL_ALLOW_ORIGIN, ALLOW};
-use hyper::{body::Incoming as IncomingBody, Request, Response, StatusCode};
+use http::header::{HeaderValue, ACCESS_CONTROL_ALLOW_ORIGIN, AUTHORIZATION};
+use hyper::{body::Incoming as IncomingBody, Request, Response};
+use serde::{Deserialize, Serialize};
 use tokio::sync::watch;
 
 use opentelemetry::trace::SpanRef;
 
 #[cfg(feature = "metrics")]
 use opentelemetry_prometheus::PrometheusExporter;
-#[cfg(feature = "metrics")]
-use prometheus::{Encoder, TextEncoder};
 
 use garage_model::garage::Garage;
-use garage_rpc::system::ClusterHealthStatus;
+use garage_rpc::{Endpoint as RpcEndpoint, *};
+use garage_table::EmptyKey;
+use garage_util::background::BackgroundRunner;
+use garage_util::data::Uuid;
 use garage_util::error::Error as GarageError;
 use garage_util::socket_address::UnixOrTCPSocketAddress;
+use garage_util::time::now_msec;
 
 use garage_api_common::generic_server::*;
 use garage_api_common::helpers::*;
 
-use crate::bucket::*;
-use crate::cluster::*;
+use crate::api::*;
 use crate::error::*;
-use crate::key::*;
 use crate::router_v0;
-use crate::router_v1::{Authorization, Endpoint};
+use crate::router_v1;
+use crate::Authorization;
+use crate::RequestHandler;
+
+// ---- FOR RPC ----
+
+pub const ADMIN_RPC_PATH: &str = "garage_api/admin/rpc.rs/Rpc";
+
+#[derive(Debug, Serialize, Deserialize)]
+pub enum AdminRpc {
+	Proxy(AdminApiRequest),
+	Internal(LocalAdminApiRequest),
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+pub enum AdminRpcResponse {
+	ProxyApiOkResponse(TaggedAdminApiResponse),
+	InternalApiOkResponse(LocalAdminApiResponse),
+	ApiErrorResponse {
+		http_code: u16,
+		error_code: String,
+		message: String,
+	},
+}
+
+impl Rpc for AdminRpc {
+	type Response = Result<AdminRpcResponse, GarageError>;
+}
+
+impl EndpointHandler<AdminRpc> for AdminApiServer {
+	async fn handle(
+		self: &Arc<Self>,
+		message: &AdminRpc,
+		_from: NodeID,
+	) -> Result<AdminRpcResponse, GarageError> {
+		match message {
+			AdminRpc::Proxy(req) => {
+				info!("Proxied admin API request: {}", req.name());
+				let res = req.clone().handle(&self.garage, &self).await;
+				match res {
+					Ok(res) => Ok(AdminRpcResponse::ProxyApiOkResponse(res.tagged())),
+					Err(e) => Ok(AdminRpcResponse::ApiErrorResponse {
+						http_code: e.http_status_code().as_u16(),
+						error_code: e.code().to_string(),
+						message: e.to_string(),
+					}),
+				}
+			}
+			AdminRpc::Internal(req) => {
+				info!("Internal admin API request: {}", req.name());
+				let res = req.clone().handle(&self.garage, &self).await;
+				match res {
+					Ok(res) => Ok(AdminRpcResponse::InternalApiOkResponse(res)),
+					Err(e) => Ok(AdminRpcResponse::ApiErrorResponse {
+						http_code: e.http_status_code().as_u16(),
+						error_code: e.code().to_string(),
+						message: e.to_string(),
+					}),
+				}
+			}
+		}
+	}
+}
+
+// ---- FOR HTTP ----
 
 pub type ResBody = BoxBody<Error>;
 
 pub struct AdminApiServer {
 	garage: Arc<Garage>,
 	#[cfg(feature = "metrics")]
-	exporter: PrometheusExporter,
+	pub(crate) exporter: PrometheusExporter,
 	metrics_token: Option<String>,
+	metrics_require_token: bool,
 	admin_token: Option<String>,
+	pub(crate) background: Arc<BackgroundRunner>,
+	pub(crate) endpoint: Arc<RpcEndpoint<AdminRpc, Self>>,
+}
+
+pub enum HttpEndpoint {
+	Old(router_v1::Endpoint),
+	New(String),
 }
 
 impl AdminApiServer {
 	pub fn new(
 		garage: Arc<Garage>,
+		background: Arc<BackgroundRunner>,
 		#[cfg(feature = "metrics")] exporter: PrometheusExporter,
-	) -> Self {
+	) -> Arc<Self> {
 		let cfg = &garage.config.admin;
 		let metrics_token = cfg.metrics_token.as_deref().map(hash_bearer_token);
 		let admin_token = cfg.admin_token.as_deref().map(hash_bearer_token);
-		Self {
+		let metrics_require_token = cfg.metrics_require_token;
+
+		let endpoint = garage.system.netapp.endpoint(ADMIN_RPC_PATH.into());
+		let admin = Arc::new(Self {
 			garage,
 			#[cfg(feature = "metrics")]
 			exporter,
 			metrics_token,
+			metrics_require_token,
 			admin_token,
-		}
+			background,
+			endpoint,
+		});
+		admin.endpoint.set_handler(admin.clone());
+		admin
 	}
 
 	pub async fn run(
-		self,
+		self: Arc<Self>,
 		bind_addr: UnixOrTCPSocketAddress,
 		must_exit: watch::Receiver<bool>,
 	) -> Result<(), GarageError> {
 		let region = self.garage.config.s3_api.s3_region.clone();
-		ApiServer::new(region, self)
+		ApiServer::new(region, ArcAdminApiServer(self))
 			.run_server(bind_addr, Some(0o220), must_exit)
 			.await
 	}
 
-	fn handle_options(&self, _req: &Request<IncomingBody>) -> Result<Response<ResBody>, Error> {
-		Ok(Response::builder()
-			.status(StatusCode::NO_CONTENT)
-			.header(ALLOW, "OPTIONS, GET, POST")
-			.header(ACCESS_CONTROL_ALLOW_METHODS, "OPTIONS, GET, POST")
-			.header(ACCESS_CONTROL_ALLOW_ORIGIN, "*")
-			.body(empty_body())?)
-	}
-
-	async fn handle_check_domain(
+	async fn handle_http_api(
 		&self,
 		req: Request<IncomingBody>,
+		endpoint: HttpEndpoint,
 	) -> Result<Response<ResBody>, Error> {
-		let query_params: HashMap<String, String> = req
-			.uri()
-			.query()
-			.map(|v| {
-				url::form_urlencoded::parse(v.as_bytes())
-					.into_owned()
-					.collect()
-			})
-			.unwrap_or_else(HashMap::new);
+		let auth_header = req.headers().get(AUTHORIZATION).cloned();
 
-		let has_domain_key = query_params.contains_key("domain");
-
-		if !has_domain_key {
-			return Err(Error::bad_request("No domain query string found"));
-		}
-
-		let domain = query_params
-			.get("domain")
-			.ok_or_internal_error("Could not parse domain query string")?;
-
-		if self.check_domain(domain).await? {
-			Ok(Response::builder()
-				.status(StatusCode::OK)
-				.body(string_body(format!(
-					"Domain '{domain}' is managed by Garage"
-				)))?)
-		} else {
-			Err(Error::bad_request(format!(
-				"Domain '{domain}' is not managed by Garage"
-			)))
-		}
-	}
-
-	async fn check_domain(&self, domain: &str) -> Result<bool, Error> {
-		// Resolve bucket from domain name, inferring if the website must be activated for the
-		// domain to be valid.
-		let (bucket_name, must_check_website) = if let Some(bname) = self
-			.garage
-			.config
-			.s3_api
-			.root_domain
-			.as_ref()
-			.and_then(|rd| host_to_bucket(domain, rd))
-		{
-			(bname.to_string(), false)
-		} else if let Some(bname) = self
-			.garage
-			.config
-			.s3_web
-			.as_ref()
-			.and_then(|sw| host_to_bucket(domain, sw.root_domain.as_str()))
-		{
-			(bname.to_string(), true)
-		} else {
-			(domain.to_string(), true)
+		let request = match endpoint {
+			HttpEndpoint::Old(endpoint_v1) => AdminApiRequest::from_v1(endpoint_v1, req).await?,
+			HttpEndpoint::New(_) => AdminApiRequest::from_request(req).await?,
 		};
 
-		let bucket_id = match self
-			.garage
-			.bucket_helper()
-			.resolve_global_bucket_name(&bucket_name)
-			.await?
-		{
-			Some(bucket_id) => bucket_id,
-			None => return Ok(false),
-		};
-
-		if !must_check_website {
-			return Ok(true);
-		}
-
-		let bucket = self
-			.garage
-			.bucket_helper()
-			.get_existing_bucket(bucket_id)
-			.await?;
-
-		let bucket_state = bucket.state.as_option().unwrap();
-		let bucket_website_config = bucket_state.website_config.get();
-
-		match bucket_website_config {
-			Some(_v) => Ok(true),
-			None => Ok(false),
-		}
-	}
-
-	fn handle_health(&self) -> Result<Response<ResBody>, Error> {
-		let health = self.garage.system.health();
-
-		let (status, status_str) = match health.status {
-			ClusterHealthStatus::Healthy => (StatusCode::OK, "Garage is fully operational"),
-			ClusterHealthStatus::Degraded => (
-				StatusCode::OK,
-				"Garage is operational but some storage nodes are unavailable",
-			),
-			ClusterHealthStatus::Unavailable => (
-				StatusCode::SERVICE_UNAVAILABLE,
-				"Quorum is not available for some/all partitions, reads and writes will fail",
+		let (global_token_hash, token_required) = match request.authorization_type() {
+			Authorization::None => (None, false),
+			Authorization::MetricsToken => (
+				self.metrics_token.as_deref(),
+				self.metrics_token.is_some() || self.metrics_require_token,
 			),
+			Authorization::AdminToken => (self.admin_token.as_deref(), true),
 		};
-		let status_str = format!(
-			"{}\nConsult the full health check API endpoint at /v1/health for more details\n",
-			status_str
-		);
 
-		Ok(Response::builder()
-			.status(status)
-			.header(http::header::CONTENT_TYPE, "text/plain")
-			.body(string_body(status_str))?)
-	}
-
-	fn handle_metrics(&self) -> Result<Response<ResBody>, Error> {
-		#[cfg(feature = "metrics")]
-		{
-			use opentelemetry::trace::Tracer;
-
-			let mut buffer = vec![];
-			let encoder = TextEncoder::new();
-
-			let tracer = opentelemetry::global::tracer("garage");
-			let metric_families = tracer.in_span("admin/gather_metrics", |_| {
-				self.exporter.registry().gather()
-			});
-
-			encoder
-				.encode(&metric_families, &mut buffer)
-				.ok_or_internal_error("Could not serialize metrics")?;
-
-			Ok(Response::builder()
-				.status(StatusCode::OK)
-				.header(http::header::CONTENT_TYPE, encoder.format_type())
-				.body(bytes_body(buffer.into()))?)
+		if token_required {
+			verify_authorization(&self.garage, global_token_hash, auth_header, request.name())?;
+		}
+
+		match request {
+			AdminApiRequest::Options(req) => req.handle(&self.garage, &self).await,
+			AdminApiRequest::CheckDomain(req) => req.handle(&self.garage, &self).await,
+			AdminApiRequest::Health(req) => req.handle(&self.garage, &self).await,
+			AdminApiRequest::Metrics(req) => req.handle(&self.garage, &self).await,
+			req => {
+				let res = req.handle(&self.garage, &self).await?;
+				let mut res = json_ok_response(&res)?;
+				res.headers_mut()
+					.insert(ACCESS_CONTROL_ALLOW_ORIGIN, HeaderValue::from_static("*"));
+				Ok(res)
+			}
 		}
-		#[cfg(not(feature = "metrics"))]
-		Err(Error::bad_request(
-			"Garage was built without the metrics feature".to_string(),
-		))
 	}
 }
 
-impl ApiHandler for AdminApiServer {
+struct ArcAdminApiServer(Arc<AdminApiServer>);
+
+impl ApiHandler for ArcAdminApiServer {
 	const API_NAME: &'static str = "admin";
 	const API_NAME_DISPLAY: &'static str = "Admin";
 
-	type Endpoint = Endpoint;
+	type Endpoint = HttpEndpoint;
 	type Error = Error;
 
-	fn parse_endpoint(&self, req: &Request<IncomingBody>) -> Result<Endpoint, Error> {
+	fn parse_endpoint(&self, req: &Request<IncomingBody>) -> Result<HttpEndpoint, Error> {
 		if req.uri().path().starts_with("/v0/") {
 			let endpoint_v0 = router_v0::Endpoint::from_request(req)?;
-			Endpoint::from_v0(endpoint_v0)
+			let endpoint_v1 = router_v1::Endpoint::from_v0(endpoint_v0)?;
+			Ok(HttpEndpoint::Old(endpoint_v1))
+		} else if req.uri().path().starts_with("/v1/") {
+			let endpoint_v1 = router_v1::Endpoint::from_request(req)?;
+			Ok(HttpEndpoint::Old(endpoint_v1))
 		} else {
-			Endpoint::from_request(req)
+			Ok(HttpEndpoint::New(req.uri().path().to_string()))
 		}
 	}
 
 	async fn handle(
 		&self,
 		req: Request<IncomingBody>,
-		endpoint: Endpoint,
+		endpoint: HttpEndpoint,
 	) -> Result<Response<ResBody>, Error> {
-		let required_auth_hash =
-			match endpoint.authorization_type() {
-				Authorization::None => None,
-				Authorization::MetricsToken => self.metrics_token.as_deref(),
-				Authorization::AdminToken => match self.admin_token.as_deref() {
-					None => return Err(Error::forbidden(
-						"Admin token isn't configured, admin API access is disabled for security.",
-					)),
-					Some(t) => Some(t),
-				},
-			};
-
-		if let Some(password_hash) = required_auth_hash {
-			match req.headers().get("Authorization") {
-				None => return Err(Error::forbidden("Authorization token must be provided")),
-				Some(authorization) => {
-					verify_bearer_token(&authorization, password_hash)?;
-				}
-			}
-		}
-
-		match endpoint {
-			Endpoint::Options => self.handle_options(&req),
-			Endpoint::CheckDomain => self.handle_check_domain(req).await,
-			Endpoint::Health => self.handle_health(),
-			Endpoint::Metrics => self.handle_metrics(),
-			Endpoint::GetClusterStatus => handle_get_cluster_status(&self.garage).await,
-			Endpoint::GetClusterHealth => handle_get_cluster_health(&self.garage).await,
-			Endpoint::ConnectClusterNodes => handle_connect_cluster_nodes(&self.garage, req).await,
-			// Layout
-			Endpoint::GetClusterLayout => handle_get_cluster_layout(&self.garage).await,
-			Endpoint::UpdateClusterLayout => handle_update_cluster_layout(&self.garage, req).await,
-			Endpoint::ApplyClusterLayout => handle_apply_cluster_layout(&self.garage, req).await,
-			Endpoint::RevertClusterLayout => handle_revert_cluster_layout(&self.garage).await,
-			// Keys
-			Endpoint::ListKeys => handle_list_keys(&self.garage).await,
-			Endpoint::GetKeyInfo {
-				id,
-				search,
-				show_secret_key,
-			} => {
-				let show_secret_key = show_secret_key.map(|x| x == "true").unwrap_or(false);
-				handle_get_key_info(&self.garage, id, search, show_secret_key).await
-			}
-			Endpoint::CreateKey => handle_create_key(&self.garage, req).await,
-			Endpoint::ImportKey => handle_import_key(&self.garage, req).await,
-			Endpoint::UpdateKey { id } => handle_update_key(&self.garage, id, req).await,
-			Endpoint::DeleteKey { id } => handle_delete_key(&self.garage, id).await,
-			// Buckets
-			Endpoint::ListBuckets => handle_list_buckets(&self.garage).await,
-			Endpoint::GetBucketInfo { id, global_alias } => {
-				handle_get_bucket_info(&self.garage, id, global_alias).await
-			}
-			Endpoint::CreateBucket => handle_create_bucket(&self.garage, req).await,
-			Endpoint::DeleteBucket { id } => handle_delete_bucket(&self.garage, id).await,
-			Endpoint::UpdateBucket { id } => handle_update_bucket(&self.garage, id, req).await,
-			// Bucket-key permissions
-			Endpoint::BucketAllowKey => {
-				handle_bucket_change_key_perm(&self.garage, req, true).await
-			}
-			Endpoint::BucketDenyKey => {
-				handle_bucket_change_key_perm(&self.garage, req, false).await
-			}
-			// Bucket aliasing
-			Endpoint::GlobalAliasBucket { id, alias } => {
-				handle_global_alias_bucket(&self.garage, id, alias).await
-			}
-			Endpoint::GlobalUnaliasBucket { id, alias } => {
-				handle_global_unalias_bucket(&self.garage, id, alias).await
-			}
-			Endpoint::LocalAliasBucket {
-				id,
-				access_key_id,
-				alias,
-			} => handle_local_alias_bucket(&self.garage, id, access_key_id, alias).await,
-			Endpoint::LocalUnaliasBucket {
-				id,
-				access_key_id,
-				alias,
-			} => handle_local_unalias_bucket(&self.garage, id, access_key_id, alias).await,
-		}
+		self.0.handle_http_api(req, endpoint).await
 	}
 }
 
-impl ApiEndpoint for Endpoint {
-	fn name(&self) -> &'static str {
-		Endpoint::name(self)
+impl ApiEndpoint for HttpEndpoint {
+	fn name(&self) -> Cow<'static, str> {
+		match self {
+			Self::Old(endpoint_v1) => Cow::Borrowed(endpoint_v1.name()),
+			Self::New(path) => Cow::Owned(path.clone()),
+		}
 	}
 
 	fn add_span_attributes(&self, _span: SpanRef<'_>) {}
@@ -347,20 +244,87 @@ fn hash_bearer_token(token: &str) -> String {
 		.to_string()
 }
 
-fn verify_bearer_token(token: &hyper::http::HeaderValue, password_hash: &str) -> Result<(), Error> {
-	use argon2::{password_hash::PasswordVerifier, Argon2};
+fn verify_authorization(
+	garage: &Garage,
+	global_token_hash: Option<&str>,
+	auth_header: Option<hyper::http::HeaderValue>,
+	endpoint_name: &str,
+) -> Result<(), Error> {
+	use argon2::{password_hash::PasswordHash, password_hash::PasswordVerifier, Argon2};
 
-	let parsed_hash = PasswordHash::new(&password_hash).unwrap();
+	let invalid_msg = "Invalid bearer token";
 
-	token
-		.to_str()?
-		.strip_prefix("Bearer ")
-		.and_then(|token| {
-			Argon2::default()
-				.verify_password(token.trim().as_bytes(), &parsed_hash)
-				.ok()
-		})
-		.ok_or_else(|| Error::forbidden("Invalid authorization token"))?;
+	let token = match &auth_header {
+		None => {
+			return Err(Error::forbidden(
+				"Bearer token must be provided in Authorization header",
+			))
+		}
+		Some(authorization) => authorization
+			.to_str()?
+			.strip_prefix("Bearer ")
+			.ok_or_else(|| Error::forbidden("Invalid Authorization header"))?
+			.trim(),
+	};
+
+	let token_hash_string = if let Some((prefix, _)) = token.split_once('.') {
+		garage
+			.admin_token_table
+			.get_local(&EmptyKey, &prefix.to_string())?
+			.and_then(|k| k.state.into_option())
+			.filter(|p| !p.is_expired(now_msec()))
+			.filter(|p| p.has_scope(endpoint_name))
+			.ok_or_else(|| Error::forbidden(invalid_msg))?
+			.token_hash
+	} else {
+		global_token_hash
+			.ok_or_else(|| Error::forbidden(invalid_msg))?
+			.to_string()
+	};
+
+	let token_hash =
+		PasswordHash::new(&token_hash_string).ok_or_internal_error("Could not parse token hash")?;
+
+	Argon2::default()
+		.verify_password(token.as_bytes(), &token_hash)
+		.map_err(|_| Error::forbidden(invalid_msg))?;
 
 	Ok(())
 }
+
+pub(crate) fn find_matching_nodes(garage: &Garage, spec: &str) -> Result<Vec<Uuid>, Error> {
+	let mut res = vec![];
+	if spec == "*" {
+		res = garage.system.cluster_layout().all_nodes().to_vec();
+		for node in garage.system.get_known_nodes() {
+			if node.is_up && !res.contains(&node.id) {
+				res.push(node.id);
+			}
+		}
+	} else if spec == "self" {
+		res.push(garage.system.id);
+	} else {
+		let layout = garage.system.cluster_layout();
+		let known_nodes = garage.system.get_known_nodes();
+		let all_nodes = layout
+			.all_nodes()
+			.iter()
+			.copied()
+			.chain(known_nodes.iter().filter(|x| x.is_up).map(|x| x.id));
+		for node in all_nodes {
+			if !res.contains(&node) && hex::encode(node).starts_with(spec) {
+				res.push(node);
+			}
+		}
+		if res.is_empty() {
+			return Err(Error::bad_request(format!("No nodes matching {}", spec)));
+		}
+		if res.len() > 1 {
+			return Err(Error::bad_request(format!(
+				"Multiple nodes matching {}: {:?}",
+				spec, res
+			)));
+		}
+	}
+	Ok(res)
+}

src/api/admin/block.rs

@@ -0,0 +1,276 @@
+use std::sync::Arc;
+
+use garage_util::data::*;
+use garage_util::error::Error as GarageError;
+use garage_util::time::now_msec;
+
+use garage_table::EmptyKey;
+
+use garage_model::garage::Garage;
+use garage_model::s3::object_table::*;
+use garage_model::s3::version_table::*;
+
+use garage_api_common::common_error::CommonErrorDerivative;
+
+use crate::api::*;
+use crate::error::*;
+use crate::{Admin, RequestHandler};
+
+impl RequestHandler for LocalListBlockErrorsRequest {
+	type Response = LocalListBlockErrorsResponse;
+
+	async fn handle(
+		self,
+		garage: &Arc<Garage>,
+		_admin: &Admin,
+	) -> Result<LocalListBlockErrorsResponse, Error> {
+		let errors = garage.block_manager.list_resync_errors()?;
+		let now = now_msec();
+		let errors = errors
+			.into_iter()
+			.map(|e| BlockError {
+				block_hash: hex::encode(&e.hash),
+				refcount: e.refcount,
+				error_count: e.error_count,
+				last_try_secs_ago: now.saturating_sub(e.last_try) / 1000,
+				next_try_in_secs: e.next_try.saturating_sub(now) / 1000,
+			})
+			.collect();
+		Ok(LocalListBlockErrorsResponse(errors))
+	}
+}
+
+impl RequestHandler for LocalGetBlockInfoRequest {
+	type Response = LocalGetBlockInfoResponse;
+
+	async fn handle(
+		self,
+		garage: &Arc<Garage>,
+		_admin: &Admin,
+	) -> Result<LocalGetBlockInfoResponse, Error> {
+		let hash = find_block_hash_by_prefix(garage, &self.block_hash)?;
+		let refcount = garage.block_manager.get_block_rc(&hash)?;
+		let block_refs = garage
+			.block_ref_table
+			.get_range(&hash, None, None, 10000, Default::default())
+			.await?;
+		let mut versions = vec![];
+		for br in block_refs {
+			if let Some(v) = garage.version_table.get(&br.version, &EmptyKey).await? {
+				let bl = match &v.backlink {
+					VersionBacklink::MultipartUpload { upload_id } => {
+						if let Some(u) = garage.mpu_table.get(upload_id, &EmptyKey).await? {
+							BlockVersionBacklink::Upload {
+								upload_id: hex::encode(&upload_id),
+								upload_deleted: u.deleted.get(),
+								upload_garbage_collected: false,
+								bucket_id: Some(hex::encode(&u.bucket_id)),
+								key: Some(u.key.to_string()),
+							}
+						} else {
+							BlockVersionBacklink::Upload {
+								upload_id: hex::encode(&upload_id),
+								upload_deleted: true,
+								upload_garbage_collected: true,
+								bucket_id: None,
+								key: None,
+							}
+						}
+					}
+					VersionBacklink::Object { bucket_id, key } => BlockVersionBacklink::Object {
+						bucket_id: hex::encode(&bucket_id),
+						key: key.to_string(),
+					},
+				};
+				versions.push(BlockVersion {
+					version_id: hex::encode(&br.version),
+					ref_deleted: br.deleted.get(),
+					version_deleted: v.deleted.get(),
+					garbage_collected: false,
+					backlink: Some(bl),
+				});
+			} else {
+				versions.push(BlockVersion {
+					version_id: hex::encode(&br.version),
+					ref_deleted: br.deleted.get(),
+					version_deleted: true,
+					garbage_collected: true,
+					backlink: None,
+				});
+			}
+		}
+		Ok(LocalGetBlockInfoResponse {
+			block_hash: hex::encode(&hash),
+			refcount,
+			versions,
+		})
+	}
+}
+
+impl RequestHandler for LocalRetryBlockResyncRequest {
+	type Response = LocalRetryBlockResyncResponse;
+
+	async fn handle(
+		self,
+		garage: &Arc<Garage>,
+		_admin: &Admin,
+	) -> Result<LocalRetryBlockResyncResponse, Error> {
+		match self {
+			Self::All { all: true } => {
+				let blocks = garage.block_manager.list_resync_errors()?;
+				for b in blocks.iter() {
+					garage.block_manager.resync.clear_backoff(&b.hash)?;
+				}
+				Ok(LocalRetryBlockResyncResponse {
+					count: blocks.len() as u64,
+				})
+			}
+			Self::All { all: false } => Err(Error::bad_request("nonsense")),
+			Self::Blocks { block_hashes } => {
+				for hash in block_hashes.iter() {
+					let hash = hex::decode(hash).ok_or_bad_request("invalid hash")?;
+					let hash = Hash::try_from(&hash).ok_or_bad_request("invalid hash")?;
+					garage.block_manager.resync.clear_backoff(&hash)?;
+				}
+				Ok(LocalRetryBlockResyncResponse {
+					count: block_hashes.len() as u64,
+				})
+			}
+		}
+	}
+}
+
+impl RequestHandler for LocalPurgeBlocksRequest {
+	type Response = LocalPurgeBlocksResponse;
+
+	async fn handle(
+		self,
+		garage: &Arc<Garage>,
+		_admin: &Admin,
+	) -> Result<LocalPurgeBlocksResponse, Error> {
+		let mut obj_dels = 0;
+		let mut mpu_dels = 0;
+		let mut ver_dels = 0;
+
+		for hash in self.0.iter() {
+			let hash = hex::decode(hash).ok_or_bad_request("invalid hash")?;
+			let hash = Hash::try_from(&hash).ok_or_bad_request("invalid hash")?;
+			let block_refs = garage
+				.block_ref_table
+				.get_range(&hash, None, None, 10000, Default::default())
+				.await?;
+
+			for br in block_refs {
+				if let Some(version) = garage.version_table.get(&br.version, &EmptyKey).await? {
+					handle_block_purge_version_backlink(
+						garage,
+						&version,
+						&mut obj_dels,
+						&mut mpu_dels,
+					)
+					.await?;
+
+					if !version.deleted.get() {
+						let deleted_version = Version::new(version.uuid, version.backlink, true);
+						garage.version_table.insert(&deleted_version).await?;
+						ver_dels += 1;
+					}
+				}
+			}
+		}
+
+		Ok(LocalPurgeBlocksResponse {
+			blocks_purged: self.0.len() as u64,
+			versions_deleted: ver_dels,
+			objects_deleted: obj_dels,
+			uploads_deleted: mpu_dels,
+		})
+	}
+}
+
+fn find_block_hash_by_prefix(garage: &Arc<Garage>, prefix: &str) -> Result<Hash, Error> {
+	if prefix.len() < 4 {
+		return Err(Error::bad_request(
+			"Please specify at least 4 characters of the block hash",
+		));
+	}
+
+	let prefix_bin = hex::decode(&prefix[..prefix.len() & !1]).ok_or_bad_request("invalid hash")?;
+
+	let iter = garage
+		.block_ref_table
+		.data
+		.store
+		.range(&prefix_bin[..]..)
+		.map_err(GarageError::from)?;
+	let mut found = None;
+	for item in iter {
+		let (k, _v) = item.map_err(GarageError::from)?;
+		let hash = Hash::try_from(&k[..32]).unwrap();
+		if &hash.as_slice()[..prefix_bin.len()] != prefix_bin {
+			break;
+		}
+		if hex::encode(hash.as_slice()).starts_with(prefix) {
+			match &found {
+				Some(x) if *x == hash => (),
+				Some(_) => {
+					return Err(Error::bad_request(format!(
+						"Several blocks match prefix `{}`",
+						prefix
+					)));
+				}
+				None => {
+					found = Some(hash);
+				}
+			}
+		}
+	}
+
+	found.ok_or_else(|| Error::NoSuchBlock(prefix.to_string()))
+}
+
+async fn handle_block_purge_version_backlink(
+	garage: &Arc<Garage>,
+	version: &Version,
+	obj_dels: &mut u64,
+	mpu_dels: &mut u64,
+) -> Result<(), Error> {
+	let (bucket_id, key, ov_id) = match &version.backlink {
+		VersionBacklink::Object { bucket_id, key } => (*bucket_id, key.clone(), version.uuid),
+		VersionBacklink::MultipartUpload { upload_id } => {
+			if let Some(mut mpu) = garage.mpu_table.get(upload_id, &EmptyKey).await? {
+				if !mpu.deleted.get() {
+					mpu.parts.clear();
+					mpu.deleted.set();
+					garage.mpu_table.insert(&mpu).await?;
+					*mpu_dels += 1;
+				}
+				(mpu.bucket_id, mpu.key.clone(), *upload_id)
+			} else {
+				return Ok(());
+			}
+		}
+	};
+
+	if let Some(object) = garage.object_table.get(&bucket_id, &key).await? {
+		let ov = object.versions().iter().rev().find(|v| v.is_complete());
+		if let Some(ov) = ov {
+			if ov.uuid == ov_id {
+				let del_uuid = gen_uuid();
+				let deleted_object = Object::new(
+					bucket_id,
+					key,
+					vec![ObjectVersion {
+						uuid: del_uuid,
+						timestamp: ov.timestamp + 1,
+						state: ObjectVersionState::Complete(ObjectVersionData::DeleteMarker),
+					}],
+				);
+				garage.object_table.insert(&deleted_object).await?;
+				*obj_dels += 1;
+			}
+		}
+	}
+
+	Ok(())
+}

src/api/admin/cluster.rs

@@ -1,411 +1,276 @@
 use std::collections::HashMap;
-use std::net::SocketAddr;
+use std::fmt::Write;
 use std::sync::Arc;
 
-use hyper::{body::Incoming as IncomingBody, Request, Response};
-use serde::{Deserialize, Serialize};
+use format_table::format_table_to_string;
 
-use garage_util::crdt::*;
 use garage_util::data::*;
 
 use garage_rpc::layout;
+use garage_rpc::layout::PARTITION_BITS;
 
 use garage_model::garage::Garage;
 
-use garage_api_common::helpers::{json_ok_response, parse_json_body};
-
-use crate::api_server::ResBody;
+use crate::api::*;
 use crate::error::*;
+use crate::{Admin, RequestHandler};
 
-pub async fn handle_get_cluster_status(garage: &Arc<Garage>) -> Result<Response<ResBody>, Error> {
-	let layout = garage.system.cluster_layout();
-	let mut nodes = garage
-		.system
-		.get_known_nodes()
-		.into_iter()
-		.map(|i| {
-			(
-				i.id,
-				NodeResp {
-					id: hex::encode(i.id),
-					addr: i.addr,
-					hostname: i.status.hostname,
-					is_up: i.is_up,
-					last_seen_secs_ago: i.last_seen_secs_ago,
-					data_partition: i
-						.status
-						.data_disk_avail
-						.map(|(avail, total)| FreeSpaceResp {
-							available: avail,
-							total,
+impl RequestHandler for GetClusterStatusRequest {
+	type Response = GetClusterStatusResponse;
+
+	async fn handle(
+		self,
+		garage: &Arc<Garage>,
+		_admin: &Admin,
+	) -> Result<GetClusterStatusResponse, Error> {
+		let layout = garage.system.cluster_layout();
+		let mut nodes = garage
+			.system
+			.get_known_nodes()
+			.into_iter()
+			.map(|i| {
+				(
+					i.id,
+					NodeResp {
+						id: hex::encode(i.id),
+						garage_version: i.status.garage_version,
+						addr: i.addr,
+						hostname: i.status.hostname,
+						is_up: i.is_up,
+						last_seen_secs_ago: i.last_seen_secs_ago,
+						data_partition: i.status.data_disk_avail.map(|(avail, total)| {
+							FreeSpaceResp {
+								available: avail,
+								total,
+							}
 						}),
-					metadata_partition: i.status.meta_disk_avail.map(|(avail, total)| {
-						FreeSpaceResp {
-							available: avail,
-							total,
-						}
-					}),
-					..Default::default()
-				},
-			)
-		})
-		.collect::<HashMap<_, _>>();
+						metadata_partition: i.status.meta_disk_avail.map(|(avail, total)| {
+							FreeSpaceResp {
+								available: avail,
+								total,
+							}
+						}),
+						..Default::default()
+					},
+				)
+			})
+			.collect::<HashMap<_, _>>();
 
-	for (id, _, role) in layout.current().roles.items().iter() {
-		if let layout::NodeRoleV(Some(r)) = role {
-			let role = NodeRoleResp {
-				id: hex::encode(id),
-				zone: r.zone.to_string(),
-				capacity: r.capacity,
-				tags: r.tags.clone(),
-			};
-			match nodes.get_mut(id) {
-				None => {
-					nodes.insert(
-						*id,
-						NodeResp {
-							id: hex::encode(id),
-							role: Some(role),
-							..Default::default()
-						},
-					);
-				}
-				Some(n) => {
-					n.role = Some(role);
-				}
-			}
-		}
-	}
-
-	for ver in layout.versions().iter().rev().skip(1) {
-		for (id, _, role) in ver.roles.items().iter() {
+		for (id, _, role) in layout.current().roles.items().iter() {
 			if let layout::NodeRoleV(Some(r)) = role {
-				if r.capacity.is_some() {
-					if let Some(n) = nodes.get_mut(id) {
-						if n.role.is_none() {
-							n.draining = true;
-						}
-					} else {
+				let role = NodeAssignedRole {
+					zone: r.zone.to_string(),
+					capacity: r.capacity,
+					tags: r.tags.clone(),
+				};
+				match nodes.get_mut(id) {
+					None => {
 						nodes.insert(
 							*id,
 							NodeResp {
 								id: hex::encode(id),
-								draining: true,
+								role: Some(role),
 								..Default::default()
 							},
 						);
 					}
+					Some(n) => {
+						n.role = Some(role);
+					}
 				}
 			}
 		}
-	}
 
-	let mut nodes = nodes.into_values().collect::<Vec<_>>();
-	nodes.sort_by(|x, y| x.id.cmp(&y.id));
+		for ver in layout.versions().iter().rev().skip(1) {
+			for (id, _, role) in ver.roles.items().iter() {
+				if let layout::NodeRoleV(Some(r)) = role {
+					if r.capacity.is_some() {
+						if let Some(n) = nodes.get_mut(id) {
+							if n.role.is_none() {
+								n.draining = true;
+							}
+						} else {
+							nodes.insert(
+								*id,
+								NodeResp {
+									id: hex::encode(id),
+									draining: true,
+									..Default::default()
+								},
+							);
+						}
+					}
+				}
+			}
+		}
 
-	let res = GetClusterStatusResponse {
-		node: hex::encode(garage.system.id),
-		garage_version: garage_util::version::garage_version(),
-		garage_features: garage_util::version::garage_features(),
-		rust_version: garage_util::version::rust_version(),
-		db_engine: garage.db.engine(),
-		layout_version: layout.current().version,
-		nodes,
-	};
+		let mut nodes = nodes.into_values().collect::<Vec<_>>();
+		nodes.sort_by(|x, y| x.id.cmp(&y.id));
 
-	Ok(json_ok_response(&res)?)
-}
-
-pub async fn handle_get_cluster_health(garage: &Arc<Garage>) -> Result<Response<ResBody>, Error> {
-	use garage_rpc::system::ClusterHealthStatus;
-	let health = garage.system.health();
-	let health = ClusterHealth {
-		status: match health.status {
-			ClusterHealthStatus::Healthy => "healthy",
-			ClusterHealthStatus::Degraded => "degraded",
-			ClusterHealthStatus::Unavailable => "unavailable",
-		},
-		known_nodes: health.known_nodes,
-		connected_nodes: health.connected_nodes,
-		storage_nodes: health.storage_nodes,
-		storage_nodes_ok: health.storage_nodes_ok,
-		partitions: health.partitions,
-		partitions_quorum: health.partitions_quorum,
-		partitions_all_ok: health.partitions_all_ok,
-	};
-	Ok(json_ok_response(&health)?)
-}
-
-pub async fn handle_connect_cluster_nodes(
-	garage: &Arc<Garage>,
-	req: Request<IncomingBody>,
-) -> Result<Response<ResBody>, Error> {
-	let req = parse_json_body::<Vec<String>, _, Error>(req).await?;
-
-	let res = futures::future::join_all(req.iter().map(|node| garage.system.connect(node)))
-		.await
-		.into_iter()
-		.map(|r| match r {
-			Ok(()) => ConnectClusterNodesResponse {
-				success: true,
-				error: None,
-			},
-			Err(e) => ConnectClusterNodesResponse {
-				success: false,
-				error: Some(format!("{}", e)),
-			},
+		Ok(GetClusterStatusResponse {
+			layout_version: layout.current().version,
+			nodes,
 		})
-		.collect::<Vec<_>>();
-
-	Ok(json_ok_response(&res)?)
-}
-
-pub async fn handle_get_cluster_layout(garage: &Arc<Garage>) -> Result<Response<ResBody>, Error> {
-	let res = format_cluster_layout(garage.system.cluster_layout().inner());
-
-	Ok(json_ok_response(&res)?)
-}
-
-fn format_cluster_layout(layout: &layout::LayoutHistory) -> GetClusterLayoutResponse {
-	let roles = layout
-		.current()
-		.roles
-		.items()
-		.iter()
-		.filter_map(|(k, _, v)| v.0.clone().map(|x| (k, x)))
-		.map(|(k, v)| NodeRoleResp {
-			id: hex::encode(k),
-			zone: v.zone.clone(),
-			capacity: v.capacity,
-			tags: v.tags.clone(),
-		})
-		.collect::<Vec<_>>();
-
-	let staged_role_changes = layout
-		.staging
-		.get()
-		.roles
-		.items()
-		.iter()
-		.filter(|(k, _, v)| layout.current().roles.get(k) != Some(v))
-		.map(|(k, _, v)| match &v.0 {
-			None => NodeRoleChange {
-				id: hex::encode(k),
-				action: NodeRoleChangeEnum::Remove { remove: true },
-			},
-			Some(r) => NodeRoleChange {
-				id: hex::encode(k),
-				action: NodeRoleChangeEnum::Update {
-					zone: r.zone.clone(),
-					capacity: r.capacity,
-					tags: r.tags.clone(),
-				},
-			},
-		})
-		.collect::<Vec<_>>();
-
-	GetClusterLayoutResponse {
-		version: layout.current().version,
-		roles,
-		staged_role_changes,
 	}
 }
 
-// ----
+impl RequestHandler for GetClusterHealthRequest {
+	type Response = GetClusterHealthResponse;
 
-#[derive(Debug, Clone, Copy, Serialize)]
-#[serde(rename_all = "camelCase")]
-pub struct ClusterHealth {
-	status: &'static str,
-	known_nodes: usize,
-	connected_nodes: usize,
-	storage_nodes: usize,
-	storage_nodes_ok: usize,
-	partitions: usize,
-	partitions_quorum: usize,
-	partitions_all_ok: usize,
-}
-
-#[derive(Serialize)]
-#[serde(rename_all = "camelCase")]
-struct GetClusterStatusResponse {
-	node: String,
-	garage_version: &'static str,
-	garage_features: Option<&'static [&'static str]>,
-	rust_version: &'static str,
-	db_engine: String,
-	layout_version: u64,
-	nodes: Vec<NodeResp>,
-}
-
-#[derive(Serialize)]
-#[serde(rename_all = "camelCase")]
-struct ApplyClusterLayoutResponse {
-	message: Vec<String>,
-	layout: GetClusterLayoutResponse,
-}
-
-#[derive(Serialize)]
-#[serde(rename_all = "camelCase")]
-struct ConnectClusterNodesResponse {
-	success: bool,
-	error: Option<String>,
-}
-
-#[derive(Serialize)]
-#[serde(rename_all = "camelCase")]
-struct GetClusterLayoutResponse {
-	version: u64,
-	roles: Vec<NodeRoleResp>,
-	staged_role_changes: Vec<NodeRoleChange>,
-}
-
-#[derive(Serialize)]
-#[serde(rename_all = "camelCase")]
-struct NodeRoleResp {
-	id: String,
-	zone: String,
-	capacity: Option<u64>,
-	tags: Vec<String>,
-}
-
-#[derive(Serialize, Default)]
-#[serde(rename_all = "camelCase")]
-struct FreeSpaceResp {
-	available: u64,
-	total: u64,
-}
-
-#[derive(Serialize, Default)]
-#[serde(rename_all = "camelCase")]
-struct NodeResp {
-	id: String,
-	role: Option<NodeRoleResp>,
-	addr: Option<SocketAddr>,
-	hostname: Option<String>,
-	is_up: bool,
-	last_seen_secs_ago: Option<u64>,
-	draining: bool,
-	#[serde(skip_serializing_if = "Option::is_none")]
-	data_partition: Option<FreeSpaceResp>,
-	#[serde(skip_serializing_if = "Option::is_none")]
-	metadata_partition: Option<FreeSpaceResp>,
-}
-
-// ---- update functions ----
-
-pub async fn handle_update_cluster_layout(
-	garage: &Arc<Garage>,
-	req: Request<IncomingBody>,
-) -> Result<Response<ResBody>, Error> {
-	let updates = parse_json_body::<UpdateClusterLayoutRequest, _, Error>(req).await?;
-
-	let mut layout = garage.system.cluster_layout().inner().clone();
-
-	let mut roles = layout.current().roles.clone();
-	roles.merge(&layout.staging.get().roles);
-
-	for change in updates {
-		let node = hex::decode(&change.id).ok_or_bad_request("Invalid node identifier")?;
-		let node = Uuid::try_from(&node).ok_or_bad_request("Invalid node identifier")?;
-
-		let new_role = match change.action {
-			NodeRoleChangeEnum::Remove { remove: true } => None,
-			NodeRoleChangeEnum::Update {
-				zone,
-				capacity,
-				tags,
-			} => Some(layout::NodeRole {
-				zone,
-				capacity,
-				tags,
-			}),
-			_ => return Err(Error::bad_request("Invalid layout change")),
+	async fn handle(
+		self,
+		garage: &Arc<Garage>,
+		_admin: &Admin,
+	) -> Result<GetClusterHealthResponse, Error> {
+		use garage_rpc::system::ClusterHealthStatus;
+		let health = garage.system.health();
+		let health = GetClusterHealthResponse {
+			status: match health.status {
+				ClusterHealthStatus::Healthy => "healthy",
+				ClusterHealthStatus::Degraded => "degraded",
+				ClusterHealthStatus::Unavailable => "unavailable",
+			}
+			.to_string(),
+			known_nodes: health.known_nodes,
+			connected_nodes: health.connected_nodes,
+			storage_nodes: health.storage_nodes,
+			storage_nodes_ok: health.storage_nodes_ok,
+			partitions: health.partitions,
+			partitions_quorum: health.partitions_quorum,
+			partitions_all_ok: health.partitions_all_ok,
 		};
-
-		layout
-			.staging
-			.get_mut()
-			.roles
-			.merge(&roles.update_mutator(node, layout::NodeRoleV(new_role)));
+		Ok(health)
 	}
-
-	garage
-		.system
-		.layout_manager
-		.update_cluster_layout(&layout)
-		.await?;
-
-	let res = format_cluster_layout(&layout);
-	Ok(json_ok_response(&res)?)
 }
 
-pub async fn handle_apply_cluster_layout(
-	garage: &Arc<Garage>,
-	req: Request<IncomingBody>,
-) -> Result<Response<ResBody>, Error> {
-	let param = parse_json_body::<ApplyLayoutRequest, _, Error>(req).await?;
+impl RequestHandler for GetClusterStatisticsRequest {
+	type Response = GetClusterStatisticsResponse;
 
-	let layout = garage.system.cluster_layout().inner().clone();
-	let (layout, msg) = layout.apply_staged_changes(Some(param.version))?;
+	// FIXME: return this as a JSON struct instead of text
+	async fn handle(
+		self,
+		garage: &Arc<Garage>,
+		_admin: &Admin,
+	) -> Result<GetClusterStatisticsResponse, Error> {
+		let mut ret = String::new();
 
-	garage
-		.system
-		.layout_manager
-		.update_cluster_layout(&layout)
-		.await?;
+		// Gather storage node and free space statistics for current nodes
+		let layout = &garage.system.cluster_layout();
+		let mut node_partition_count = HashMap::<Uuid, u64>::new();
+		for short_id in layout.current().ring_assignment_data.iter() {
+			let id = layout.current().node_id_vec[*short_id as usize];
+			*node_partition_count.entry(id).or_default() += 1;
+		}
+		let node_info = garage
+			.system
+			.get_known_nodes()
+			.into_iter()
+			.map(|n| (n.id, n))
+			.collect::<HashMap<_, _>>();
 
-	let res = ApplyClusterLayoutResponse {
-		message: msg,
-		layout: format_cluster_layout(&layout),
-	};
-	Ok(json_ok_response(&res)?)
+		let mut table = vec!["  ID\tHostname\tZone\tCapacity\tPart.\tDataAvail\tMetaAvail".into()];
+		for (id, parts) in node_partition_count.iter() {
+			let info = node_info.get(id);
+			let status = info.map(|x| &x.status);
+			let role = layout.current().roles.get(id).and_then(|x| x.0.as_ref());
+			let hostname = status.and_then(|x| x.hostname.as_deref()).unwrap_or("?");
+			let zone = role.map(|x| x.zone.as_str()).unwrap_or("?");
+			let capacity = role
+				.map(|x| x.capacity_string())
+				.unwrap_or_else(|| "?".into());
+			let avail_str = |x| match x {
+				Some((avail, total)) => {
+					let pct = (avail as f64) / (total as f64) * 100.;
+					let avail = bytesize::ByteSize::b(avail);
+					let total = bytesize::ByteSize::b(total);
+					format!("{}/{} ({:.1}%)", avail, total, pct)
+				}
+				None => "?".into(),
+			};
+			let data_avail = avail_str(status.and_then(|x| x.data_disk_avail));
+			let meta_avail = avail_str(status.and_then(|x| x.meta_disk_avail));
+			table.push(format!(
+				"  {:?}\t{}\t{}\t{}\t{}\t{}\t{}",
+				id, hostname, zone, capacity, parts, data_avail, meta_avail
+			));
+		}
+		write!(
+			&mut ret,
+			"Storage nodes:\n{}",
+			format_table_to_string(table)
+		)
+		.unwrap();
+
+		let meta_part_avail = node_partition_count
+			.iter()
+			.filter_map(|(id, parts)| {
+				node_info
+					.get(id)
+					.and_then(|x| x.status.meta_disk_avail)
+					.map(|c| c.0 / *parts)
+			})
+			.collect::<Vec<_>>();
+		let data_part_avail = node_partition_count
+			.iter()
+			.filter_map(|(id, parts)| {
+				node_info
+					.get(id)
+					.and_then(|x| x.status.data_disk_avail)
+					.map(|c| c.0 / *parts)
+			})
+			.collect::<Vec<_>>();
+		if !meta_part_avail.is_empty() && !data_part_avail.is_empty() {
+			let meta_avail =
+				bytesize::ByteSize(meta_part_avail.iter().min().unwrap() * (1 << PARTITION_BITS));
+			let data_avail =
+				bytesize::ByteSize(data_part_avail.iter().min().unwrap() * (1 << PARTITION_BITS));
+			writeln!(
+				&mut ret,
+				"\nEstimated available storage space cluster-wide (might be lower in practice):"
+			)
+			.unwrap();
+			if meta_part_avail.len() < node_partition_count.len()
+				|| data_part_avail.len() < node_partition_count.len()
+			{
+				ret += &format_table_to_string(vec![
+					format!("  data: < {}", data_avail),
+					format!("  metadata: < {}", meta_avail),
+				]);
+				writeln!(&mut ret, "A precise estimate could not be given as information is missing for some storage nodes.").unwrap();
+			} else {
+				ret += &format_table_to_string(vec![
+					format!("  data: {}", data_avail),
+					format!("  metadata: {}", meta_avail),
+				]);
+			}
+		}
+
+		Ok(GetClusterStatisticsResponse { freeform: ret })
+	}
 }
 
-pub async fn handle_revert_cluster_layout(
-	garage: &Arc<Garage>,
-) -> Result<Response<ResBody>, Error> {
-	let layout = garage.system.cluster_layout().inner().clone();
-	let layout = layout.revert_staged_changes()?;
-	garage
-		.system
-		.layout_manager
-		.update_cluster_layout(&layout)
-		.await?;
+impl RequestHandler for ConnectClusterNodesRequest {
+	type Response = ConnectClusterNodesResponse;
 
-	let res = format_cluster_layout(&layout);
-	Ok(json_ok_response(&res)?)
-}
-
-// ----
-
-type UpdateClusterLayoutRequest = Vec<NodeRoleChange>;
-
-#[derive(Deserialize)]
-#[serde(rename_all = "camelCase")]
-struct ApplyLayoutRequest {
-	version: u64,
-}
-
-// ----
-
-#[derive(Serialize, Deserialize)]
-#[serde(rename_all = "camelCase")]
-struct NodeRoleChange {
-	id: String,
-	#[serde(flatten)]
-	action: NodeRoleChangeEnum,
-}
-
-#[derive(Serialize, Deserialize)]
-#[serde(untagged)]
-enum NodeRoleChangeEnum {
-	#[serde(rename_all = "camelCase")]
-	Remove { remove: bool },
-	#[serde(rename_all = "camelCase")]
-	Update {
-		zone: String,
-		capacity: Option<u64>,
-		tags: Vec<String>,
-	},
+	async fn handle(
+		self,
+		garage: &Arc<Garage>,
+		_admin: &Admin,
+	) -> Result<ConnectClusterNodesResponse, Error> {
+		let res = futures::future::join_all(self.0.iter().map(|node| garage.system.connect(node)))
+			.await
+			.into_iter()
+			.map(|r| match r {
+				Ok(()) => ConnectNodeResponse {
+					success: true,
+					error: None,
+				},
+				Err(e) => ConnectNodeResponse {
+					success: false,
+					error: Some(format!("{}", e)),
+				},
+			})
+			.collect::<Vec<_>>();
+		Ok(ConnectClusterNodesResponse(res))
+	}
 }

src/api/admin/error.rs

@@ -21,10 +21,26 @@ pub enum Error {
 	Common(#[error(source)] CommonError),
 
 	// Category: cannot process
+	/// The admin API token does not exist
+	#[error(display = "Admin token not found: {}", _0)]
+	NoSuchAdminToken(String),
+
 	/// The API access key does not exist
 	#[error(display = "Access key not found: {}", _0)]
 	NoSuchAccessKey(String),
 
+	/// The requested block does not exist
+	#[error(display = "Block not found: {}", _0)]
+	NoSuchBlock(String),
+
+	/// The requested worker does not exist
+	#[error(display = "Worker not found: {}", _0)]
+	NoSuchWorker(u64),
+
+	/// The object requested don't exists
+	#[error(display = "Key not found")]
+	NoSuchKey,
+
 	/// In Import key, the key already exists
 	#[error(
 		display = "Key {} already exists in data store. Even if it is deleted, we can't let you create a new key with the same ID. Sorry.",
@@ -49,11 +65,15 @@ impl From<HelperError> for Error {
 }
 
 impl Error {
-	fn code(&self) -> &'static str {
+	pub fn code(&self) -> &'static str {
 		match self {
 			Error::Common(c) => c.aws_code(),
+			Error::NoSuchAdminToken(_) => "NoSuchAdminToken",
 			Error::NoSuchAccessKey(_) => "NoSuchAccessKey",
+			Error::NoSuchWorker(_) => "NoSuchWorker",
+			Error::NoSuchBlock(_) => "NoSuchBlock",
 			Error::KeyAlreadyExists(_) => "KeyAlreadyExists",
+			Error::NoSuchKey => "NoSuchKey",
 		}
 	}
 }
@@ -63,7 +83,11 @@ impl ApiError for Error {
 	fn http_status_code(&self) -> StatusCode {
 		match self {
 			Error::Common(c) => c.http_status_code(),
-			Error::NoSuchAccessKey(_) => StatusCode::NOT_FOUND,
+			Error::NoSuchAdminToken(_)
+			| Error::NoSuchAccessKey(_)
+			| Error::NoSuchWorker(_)
+			| Error::NoSuchBlock(_)
+			| Error::NoSuchKey => StatusCode::NOT_FOUND,
 			Error::KeyAlreadyExists(_) => StatusCode::CONFLICT,
 		}
 	}

src/api/admin/key.rs

@@ -1,173 +1,190 @@
 use std::collections::HashMap;
 use std::sync::Arc;
 
-use hyper::{body::Incoming as IncomingBody, Request, Response, StatusCode};
-use serde::{Deserialize, Serialize};
+use chrono::DateTime;
 
 use garage_table::*;
+use garage_util::time::now_msec;
 
 use garage_model::garage::Garage;
 use garage_model::key_table::*;
 
-use garage_api_common::helpers::*;
-
-use crate::api_server::ResBody;
+use crate::api::*;
 use crate::error::*;
+use crate::{Admin, RequestHandler};
 
-pub async fn handle_list_keys(garage: &Arc<Garage>) -> Result<Response<ResBody>, Error> {
-	let res = garage
-		.key_table
-		.get_range(
-			&EmptyKey,
-			None,
-			Some(KeyFilter::Deleted(DeletedFilter::NotDeleted)),
-			10000,
-			EnumerationOrder::Forward,
-		)
-		.await?
-		.iter()
-		.map(|k| ListKeyResultItem {
-			id: k.key_id.to_string(),
-			name: k.params().unwrap().name.get().clone(),
-		})
-		.collect::<Vec<_>>();
+impl RequestHandler for ListKeysRequest {
+	type Response = ListKeysResponse;
 
-	Ok(json_ok_response(&res)?)
-}
+	async fn handle(self, garage: &Arc<Garage>, _admin: &Admin) -> Result<ListKeysResponse, Error> {
+		let now = now_msec();
 
-#[derive(Serialize)]
-#[serde(rename_all = "camelCase")]
-struct ListKeyResultItem {
-	id: String,
-	name: String,
-}
-
-pub async fn handle_get_key_info(
-	garage: &Arc<Garage>,
-	id: Option<String>,
-	search: Option<String>,
-	show_secret_key: bool,
-) -> Result<Response<ResBody>, Error> {
-	let key = if let Some(id) = id {
-		garage.key_helper().get_existing_key(&id).await?
-	} else if let Some(search) = search {
-		garage
-			.key_helper()
-			.get_existing_matching_key(&search)
+		let res = garage
+			.key_table
+			.get_range(
+				&EmptyKey,
+				None,
+				Some(KeyFilter::Deleted(DeletedFilter::NotDeleted)),
+				10000,
+				EnumerationOrder::Forward,
+			)
 			.await?
-	} else {
-		unreachable!();
-	};
+			.iter()
+			.map(|k| {
+				let p = k.params().unwrap();
 
-	key_info_results(garage, key, show_secret_key).await
-}
+				ListKeysResponseItem {
+					id: k.key_id.to_string(),
+					name: p.name.get().clone(),
+					created: p.created.map(|x| {
+						DateTime::from_timestamp_millis(x as i64)
+							.expect("invalid timestamp stored in db")
+					}),
+					expiration: p.expiration.get().map(|x| {
+						DateTime::from_timestamp_millis(x as i64)
+							.expect("invalid timestamp stored in db")
+					}),
+					expired: p.is_expired(now),
+				}
+			})
+			.collect::<Vec<_>>();
 
-pub async fn handle_create_key(
-	garage: &Arc<Garage>,
-	req: Request<IncomingBody>,
-) -> Result<Response<ResBody>, Error> {
-	let req = parse_json_body::<CreateKeyRequest, _, Error>(req).await?;
-
-	let key = Key::new(req.name.as_deref().unwrap_or("Unnamed key"));
-	garage.key_table.insert(&key).await?;
-
-	key_info_results(garage, key, true).await
-}
-
-#[derive(Deserialize)]
-#[serde(rename_all = "camelCase")]
-struct CreateKeyRequest {
-	name: Option<String>,
-}
-
-pub async fn handle_import_key(
-	garage: &Arc<Garage>,
-	req: Request<IncomingBody>,
-) -> Result<Response<ResBody>, Error> {
-	let req = parse_json_body::<ImportKeyRequest, _, Error>(req).await?;
-
-	let prev_key = garage.key_table.get(&EmptyKey, &req.access_key_id).await?;
-	if prev_key.is_some() {
-		return Err(Error::KeyAlreadyExists(req.access_key_id.to_string()));
+		Ok(ListKeysResponse(res))
 	}
-
-	let imported_key = Key::import(
-		&req.access_key_id,
-		&req.secret_access_key,
-		req.name.as_deref().unwrap_or("Imported key"),
-	)
-	.ok_or_bad_request("Invalid key format")?;
-	garage.key_table.insert(&imported_key).await?;
-
-	key_info_results(garage, imported_key, false).await
 }
 
-#[derive(Deserialize)]
-#[serde(rename_all = "camelCase")]
-struct ImportKeyRequest {
-	access_key_id: String,
-	secret_access_key: String,
-	name: Option<String>,
-}
+impl RequestHandler for GetKeyInfoRequest {
+	type Response = GetKeyInfoResponse;
 
-pub async fn handle_update_key(
-	garage: &Arc<Garage>,
-	id: String,
-	req: Request<IncomingBody>,
-) -> Result<Response<ResBody>, Error> {
-	let req = parse_json_body::<UpdateKeyRequest, _, Error>(req).await?;
+	async fn handle(
+		self,
+		garage: &Arc<Garage>,
+		_admin: &Admin,
+	) -> Result<GetKeyInfoResponse, Error> {
+		let key = match (self.id, self.search) {
+			(Some(id), None) => garage.key_helper().get_existing_key(&id).await?,
+			(None, Some(search)) => {
+				let candidates = garage
+					.key_table
+					.get_range(
+						&EmptyKey,
+						None,
+						Some(KeyFilter::MatchesAndNotDeleted(search.to_string())),
+						10,
+						EnumerationOrder::Forward,
+					)
+					.await?
+					.into_iter()
+					.collect::<Vec<_>>();
+				if candidates.len() != 1 {
+					return Err(Error::bad_request(format!(
+						"{} matching keys",
+						candidates.len()
+					)));
+				}
+				candidates.into_iter().next().unwrap()
+			}
+			_ => {
+				return Err(Error::bad_request(
+					"Either id or search must be provided (but not both)",
+				));
+			}
+		};
 
-	let mut key = garage.key_helper().get_existing_key(&id).await?;
-
-	let key_state = key.state.as_option_mut().unwrap();
-
-	if let Some(new_name) = req.name {
-		key_state.name.update(new_name);
+		Ok(key_info_results(garage, key, self.show_secret_key).await?)
 	}
-	if let Some(allow) = req.allow {
-		if allow.create_bucket {
-			key_state.allow_create_bucket.update(true);
+}
+
+impl RequestHandler for CreateKeyRequest {
+	type Response = CreateKeyResponse;
+
+	async fn handle(
+		self,
+		garage: &Arc<Garage>,
+		_admin: &Admin,
+	) -> Result<CreateKeyResponse, Error> {
+		let mut key = Key::new("Unnamed key");
+
+		apply_key_updates(&mut key, self.0)?;
+
+		garage.key_table.insert(&key).await?;
+
+		Ok(CreateKeyResponse(
+			key_info_results(garage, key, true).await?,
+		))
+	}
+}
+
+impl RequestHandler for ImportKeyRequest {
+	type Response = ImportKeyResponse;
+
+	async fn handle(
+		self,
+		garage: &Arc<Garage>,
+		_admin: &Admin,
+	) -> Result<ImportKeyResponse, Error> {
+		let prev_key = garage.key_table.get(&EmptyKey, &self.access_key_id).await?;
+		if prev_key.is_some() {
+			return Err(Error::KeyAlreadyExists(self.access_key_id.to_string()));
 		}
-	}
-	if let Some(deny) = req.deny {
-		if deny.create_bucket {
-			key_state.allow_create_bucket.update(false);
-		}
-	}
 
-	garage.key_table.insert(&key).await?;
+		let imported_key = Key::import(
+			&self.access_key_id,
+			&self.secret_access_key,
+			self.name.as_deref().unwrap_or("Imported key"),
+		)
+		.ok_or_bad_request("Invalid key format")?;
+		garage.key_table.insert(&imported_key).await?;
 
-	key_info_results(garage, key, false).await
+		Ok(ImportKeyResponse(
+			key_info_results(garage, imported_key, false).await?,
+		))
+	}
 }
 
-#[derive(Deserialize)]
-#[serde(rename_all = "camelCase")]
-struct UpdateKeyRequest {
-	name: Option<String>,
-	allow: Option<KeyPerm>,
-	deny: Option<KeyPerm>,
+impl RequestHandler for UpdateKeyRequest {
+	type Response = UpdateKeyResponse;
+
+	async fn handle(
+		self,
+		garage: &Arc<Garage>,
+		_admin: &Admin,
+	) -> Result<UpdateKeyResponse, Error> {
+		let mut key = garage.key_helper().get_existing_key(&self.id).await?;
+
+		apply_key_updates(&mut key, self.body)?;
+
+		garage.key_table.insert(&key).await?;
+
+		Ok(UpdateKeyResponse(
+			key_info_results(garage, key, false).await?,
+		))
+	}
 }
 
-pub async fn handle_delete_key(
-	garage: &Arc<Garage>,
-	id: String,
-) -> Result<Response<ResBody>, Error> {
-	let helper = garage.locked_helper().await;
+impl RequestHandler for DeleteKeyRequest {
+	type Response = DeleteKeyResponse;
 
-	let mut key = helper.key().get_existing_key(&id).await?;
+	async fn handle(
+		self,
+		garage: &Arc<Garage>,
+		_admin: &Admin,
+	) -> Result<DeleteKeyResponse, Error> {
+		let helper = garage.locked_helper().await;
 
-	helper.delete_key(&mut key).await?;
+		let mut key = helper.key().get_existing_key(&self.id).await?;
 
-	Ok(Response::builder()
-		.status(StatusCode::NO_CONTENT)
-		.body(empty_body())?)
+		helper.delete_key(&mut key).await?;
+
+		Ok(DeleteKeyResponse)
+	}
 }
 
 async fn key_info_results(
 	garage: &Arc<Garage>,
 	key: Key,
 	show_secret: bool,
-) -> Result<Response<ResBody>, Error> {
+) -> Result<GetKeyInfoResponse, Error> {
 	let mut relevant_buckets = HashMap::new();
 
 	let key_state = key.state.as_option().unwrap();
@@ -193,8 +210,15 @@ async fn key_info_results(
 		}
 	}
 
-	let res = GetKeyInfoResult {
+	let res = GetKeyInfoResponse {
 		name: key_state.name.get().clone(),
+		created: key_state.created.map(|x| {
+			DateTime::from_timestamp_millis(x as i64).expect("invalid timestamp stored in db")
+		}),
+		expiration: key_state.expiration.get().map(|x| {
+			DateTime::from_timestamp_millis(x as i64).expect("invalid timestamp stored in db")
+		}),
+		expired: key_state.is_expired(now_msec()),
 		access_key_id: key.key_id.clone(),
 		secret_access_key: if show_secret {
 			Some(key_state.secret_key.clone())
@@ -208,7 +232,7 @@ async fn key_info_results(
 			.into_values()
 			.map(|bucket| {
 				let state = bucket.state.as_option().unwrap();
-				KeyInfoBucketResult {
+				KeyInfoBucketResponse {
 					id: hex::encode(bucket.id),
 					global_aliases: state
 						.aliases
@@ -238,43 +262,39 @@ async fn key_info_results(
 			.collect::<Vec<_>>(),
 	};
 
-	Ok(json_ok_response(&res)?)
+	Ok(res)
 }
 
-#[derive(Serialize)]
-#[serde(rename_all = "camelCase")]
-struct GetKeyInfoResult {
-	name: String,
-	access_key_id: String,
-	#[serde(skip_serializing_if = "is_default")]
-	secret_access_key: Option<String>,
-	permissions: KeyPerm,
-	buckets: Vec<KeyInfoBucketResult>,
-}
+fn apply_key_updates(key: &mut Key, updates: UpdateKeyRequestBody) -> Result<(), Error> {
+	if updates.never_expires && updates.expiration.is_some() {
+		return Err(Error::bad_request(
+			"cannot specify `expiration` and `never_expires`",
+		));
+	}
 
-#[derive(Serialize, Deserialize)]
-#[serde(rename_all = "camelCase")]
-struct KeyPerm {
-	#[serde(default)]
-	create_bucket: bool,
-}
+	let key_state = key.state.as_option_mut().unwrap();
 
-#[derive(Serialize)]
-#[serde(rename_all = "camelCase")]
-struct KeyInfoBucketResult {
-	id: String,
-	global_aliases: Vec<String>,
-	local_aliases: Vec<String>,
-	permissions: ApiBucketKeyPerm,
-}
+	if let Some(new_name) = updates.name {
+		key_state.name.update(new_name);
+	}
+	if let Some(expiration) = updates.expiration {
+		key_state
+			.expiration
+			.update(Some(expiration.timestamp_millis() as u64));
+	}
+	if updates.never_expires {
+		key_state.expiration.update(None);
+	}
+	if let Some(allow) = updates.allow {
+		if allow.create_bucket {
+			key_state.allow_create_bucket.update(true);
+		}
+	}
+	if let Some(deny) = updates.deny {
+		if deny.create_bucket {
+			key_state.allow_create_bucket.update(false);
+		}
+	}
 
-#[derive(Serialize, Deserialize, Default)]
-#[serde(rename_all = "camelCase")]
-pub(crate) struct ApiBucketKeyPerm {
-	#[serde(default)]
-	pub(crate) read: bool,
-	#[serde(default)]
-	pub(crate) write: bool,
-	#[serde(default)]
-	pub(crate) owner: bool,
+	Ok(())
 }

src/api/admin/layout.rs

@@ -0,0 +1,406 @@
+use std::sync::Arc;
+
+use garage_util::crdt::*;
+use garage_util::data::*;
+use garage_util::error::Error as GarageError;
+
+use garage_rpc::layout;
+
+use garage_model::garage::Garage;
+
+use crate::api::*;
+use crate::error::*;
+use crate::{Admin, RequestHandler};
+
+impl RequestHandler for GetClusterLayoutRequest {
+	type Response = GetClusterLayoutResponse;
+
+	async fn handle(
+		self,
+		garage: &Arc<Garage>,
+		_admin: &Admin,
+	) -> Result<GetClusterLayoutResponse, Error> {
+		Ok(format_cluster_layout(
+			garage.system.cluster_layout().inner(),
+		))
+	}
+}
+
+fn format_cluster_layout(layout: &layout::LayoutHistory) -> GetClusterLayoutResponse {
+	let current = layout.current();
+
+	let roles = current
+		.roles
+		.items()
+		.iter()
+		.filter_map(|(k, _, v)| v.0.clone().map(|x| (k, x)))
+		.map(|(k, v)| {
+			let stored_partitions = current.get_node_usage(k).ok().map(|x| x as u64);
+			LayoutNodeRole {
+				id: hex::encode(k),
+				zone: v.zone.clone(),
+				capacity: v.capacity,
+				stored_partitions,
+				usable_capacity: stored_partitions.map(|x| x * current.partition_size),
+				tags: v.tags.clone(),
+			}
+		})
+		.collect::<Vec<_>>();
+
+	let staged_role_changes = layout
+		.staging
+		.get()
+		.roles
+		.items()
+		.iter()
+		.filter(|(k, _, v)| current.roles.get(k) != Some(v))
+		.map(|(k, _, v)| match &v.0 {
+			None => NodeRoleChange {
+				id: hex::encode(k),
+				action: NodeRoleChangeEnum::Remove { remove: true },
+			},
+			Some(r) => NodeRoleChange {
+				id: hex::encode(k),
+				action: NodeRoleChangeEnum::Update(NodeAssignedRole {
+					zone: r.zone.clone(),
+					capacity: r.capacity,
+					tags: r.tags.clone(),
+				}),
+			},
+		})
+		.collect::<Vec<_>>();
+
+	let staged_parameters = if *layout.staging.get().parameters.get() != current.parameters {
+		Some((*layout.staging.get().parameters.get()).into())
+	} else {
+		None
+	};
+
+	GetClusterLayoutResponse {
+		version: current.version,
+		roles,
+		partition_size: current.partition_size,
+		parameters: current.parameters.into(),
+		staged_role_changes,
+		staged_parameters,
+	}
+}
+
+impl RequestHandler for GetClusterLayoutHistoryRequest {
+	type Response = GetClusterLayoutHistoryResponse;
+
+	async fn handle(
+		self,
+		garage: &Arc<Garage>,
+		_admin: &Admin,
+	) -> Result<GetClusterLayoutHistoryResponse, Error> {
+		let layout_helper = garage.system.cluster_layout();
+		let layout = layout_helper.inner();
+		let min_stored = layout.min_stored();
+
+		let versions = layout
+			.versions
+			.iter()
+			.rev()
+			.chain(layout.old_versions.iter().rev())
+			.map(|ver| {
+				let status = if ver.version == layout.current().version {
+					ClusterLayoutVersionStatus::Current
+				} else if ver.version >= min_stored {
+					ClusterLayoutVersionStatus::Draining
+				} else {
+					ClusterLayoutVersionStatus::Historical
+				};
+				ClusterLayoutVersion {
+					version: ver.version,
+					status,
+					storage_nodes: ver
+						.roles
+						.items()
+						.iter()
+						.filter(
+							|(_, _, x)| matches!(x, layout::NodeRoleV(Some(c)) if c.capacity.is_some()),
+						)
+						.count() as u64,
+					gateway_nodes: ver
+						.roles
+						.items()
+						.iter()
+						.filter(
+							|(_, _, x)| matches!(x, layout::NodeRoleV(Some(c)) if c.capacity.is_none()),
+						)
+						.count() as u64,
+				}
+			})
+			.collect::<Vec<_>>();
+
+		let all_nodes = layout.get_all_nodes();
+		let min_ack = layout_helper.ack_map_min();
+
+		let update_trackers = if layout.versions.len() > 1 {
+			Some(
+				all_nodes
+					.iter()
+					.map(|node| {
+						(
+							hex::encode(&node),
+							NodeUpdateTrackers {
+								ack: layout.update_trackers.ack_map.get(node, min_stored),
+								sync: layout.update_trackers.sync_map.get(node, min_stored),
+								sync_ack: layout.update_trackers.sync_ack_map.get(node, min_stored),
+							},
+						)
+					})
+					.collect(),
+			)
+		} else {
+			None
+		};
+
+		Ok(GetClusterLayoutHistoryResponse {
+			current_version: layout.current().version,
+			min_ack,
+			versions,
+			update_trackers,
+		})
+	}
+}
+
+// ----
+
+// ---- update functions ----
+
+impl RequestHandler for UpdateClusterLayoutRequest {
+	type Response = UpdateClusterLayoutResponse;
+
+	async fn handle(
+		self,
+		garage: &Arc<Garage>,
+		_admin: &Admin,
+	) -> Result<UpdateClusterLayoutResponse, Error> {
+		let mut layout = garage.system.cluster_layout().inner().clone();
+
+		let mut roles = layout.current().roles.clone();
+		roles.merge(&layout.staging.get().roles);
+
+		for change in self.roles {
+			let node = hex::decode(&change.id).ok_or_bad_request("Invalid node identifier")?;
+			let node = Uuid::try_from(&node).ok_or_bad_request("Invalid node identifier")?;
+
+			let new_role = match change.action {
+				NodeRoleChangeEnum::Remove { remove: true } => None,
+				NodeRoleChangeEnum::Update(NodeAssignedRole {
+					zone,
+					capacity,
+					tags,
+				}) => {
+					if matches!(capacity, Some(cap) if cap < 1024) {
+						return Err(Error::bad_request("Capacity should be at least 1K (1024)"));
+					}
+					Some(layout::NodeRole {
+						zone,
+						capacity,
+						tags,
+					})
+				}
+				_ => return Err(Error::bad_request("Invalid layout change")),
+			};
+
+			layout
+				.staging
+				.get_mut()
+				.roles
+				.merge(&roles.update_mutator(node, layout::NodeRoleV(new_role)));
+		}
+
+		if let Some(param) = self.parameters {
+			if let ZoneRedundancy::AtLeast(r_int) = param.zone_redundancy {
+				if r_int > layout.current().replication_factor {
+					return Err(Error::bad_request(format!(
+						"The zone redundancy must be smaller or equal to the replication factor ({}).",
+						layout.current().replication_factor
+					)));
+				} else if r_int < 1 {
+					return Err(Error::bad_request(
+						"The zone redundancy must be at least 1.",
+					));
+				}
+			}
+			layout.staging.get_mut().parameters.update(param.into());
+		}
+
+		garage
+			.system
+			.layout_manager
+			.update_cluster_layout(&layout)
+			.await?;
+
+		let res = format_cluster_layout(&layout);
+		Ok(UpdateClusterLayoutResponse(res))
+	}
+}
+
+impl RequestHandler for PreviewClusterLayoutChangesRequest {
+	type Response = PreviewClusterLayoutChangesResponse;
+
+	async fn handle(
+		self,
+		garage: &Arc<Garage>,
+		_admin: &Admin,
+	) -> Result<PreviewClusterLayoutChangesResponse, Error> {
+		let layout = garage.system.cluster_layout().inner().clone();
+		let new_ver = layout.current().version + 1;
+		match layout.apply_staged_changes(new_ver) {
+			Err(GarageError::Message(error)) => {
+				Ok(PreviewClusterLayoutChangesResponse::Error { error })
+			}
+			Err(e) => Err(e.into()),
+			Ok((new_layout, msg)) => Ok(PreviewClusterLayoutChangesResponse::Success {
+				message: msg,
+				new_layout: format_cluster_layout(&new_layout),
+			}),
+		}
+	}
+}
+
+impl RequestHandler for ApplyClusterLayoutRequest {
+	type Response = ApplyClusterLayoutResponse;
+
+	async fn handle(
+		self,
+		garage: &Arc<Garage>,
+		_admin: &Admin,
+	) -> Result<ApplyClusterLayoutResponse, Error> {
+		let layout = garage.system.cluster_layout().inner().clone();
+		let (layout, msg) = layout.apply_staged_changes(self.version)?;
+
+		garage
+			.system
+			.layout_manager
+			.update_cluster_layout(&layout)
+			.await?;
+
+		Ok(ApplyClusterLayoutResponse {
+			message: msg,
+			layout: format_cluster_layout(&layout),
+		})
+	}
+}
+
+impl RequestHandler for RevertClusterLayoutRequest {
+	type Response = RevertClusterLayoutResponse;
+
+	async fn handle(
+		self,
+		garage: &Arc<Garage>,
+		_admin: &Admin,
+	) -> Result<RevertClusterLayoutResponse, Error> {
+		let layout = garage.system.cluster_layout().inner().clone();
+		let layout = layout.revert_staged_changes()?;
+		garage
+			.system
+			.layout_manager
+			.update_cluster_layout(&layout)
+			.await?;
+
+		let res = format_cluster_layout(&layout);
+		Ok(RevertClusterLayoutResponse(res))
+	}
+}
+
+impl RequestHandler for ClusterLayoutSkipDeadNodesRequest {
+	type Response = ClusterLayoutSkipDeadNodesResponse;
+
+	async fn handle(
+		self,
+		garage: &Arc<Garage>,
+		_admin: &Admin,
+	) -> Result<ClusterLayoutSkipDeadNodesResponse, Error> {
+		let status = garage.system.get_known_nodes();
+
+		let mut layout = garage.system.cluster_layout().inner().clone();
+		let mut ack_updated = vec![];
+		let mut sync_updated = vec![];
+
+		if layout.versions.len() == 1 {
+			return Err(Error::bad_request(
+				"This command cannot be called when there is only one live cluster layout version",
+			));
+		}
+
+		let min_v = layout.min_stored();
+		if self.version <= min_v || self.version > layout.current().version {
+			return Err(Error::bad_request(format!(
+				"Invalid version, you may use the following version numbers: {}",
+				(min_v + 1..=layout.current().version)
+					.map(|x| x.to_string())
+					.collect::<Vec<_>>()
+					.join(" ")
+			)));
+		}
+
+		let all_nodes = layout.get_all_nodes();
+		for node in all_nodes.iter() {
+			// Update ACK tracker for dead nodes or for all nodes if --allow-missing-data
+			if self.allow_missing_data || !status.iter().any(|x| x.id == *node && x.is_up) {
+				if layout.update_trackers.ack_map.set_max(*node, self.version) {
+					ack_updated.push(hex::encode(node));
+				}
+			}
+
+			// If --allow-missing-data, update SYNC tracker for all nodes.
+			if self.allow_missing_data {
+				if layout.update_trackers.sync_map.set_max(*node, self.version) {
+					sync_updated.push(hex::encode(node));
+				}
+			}
+		}
+
+		garage
+			.system
+			.layout_manager
+			.update_cluster_layout(&layout)
+			.await?;
+
+		Ok(ClusterLayoutSkipDeadNodesResponse {
+			ack_updated,
+			sync_updated,
+		})
+	}
+}
+
+// ----
+
+impl From<layout::ZoneRedundancy> for ZoneRedundancy {
+	fn from(x: layout::ZoneRedundancy) -> Self {
+		match x {
+			layout::ZoneRedundancy::Maximum => ZoneRedundancy::Maximum,
+			layout::ZoneRedundancy::AtLeast(x) => ZoneRedundancy::AtLeast(x),
+		}
+	}
+}
+
+impl Into<layout::ZoneRedundancy> for ZoneRedundancy {
+	fn into(self) -> layout::ZoneRedundancy {
+		match self {
+			ZoneRedundancy::Maximum => layout::ZoneRedundancy::Maximum,
+			ZoneRedundancy::AtLeast(x) => layout::ZoneRedundancy::AtLeast(x),
+		}
+	}
+}
+
+impl From<layout::LayoutParameters> for LayoutParameters {
+	fn from(x: layout::LayoutParameters) -> Self {
+		LayoutParameters {
+			zone_redundancy: x.zone_redundancy.into(),
+		}
+	}
+}
+
+impl Into<layout::LayoutParameters> for LayoutParameters {
+	fn into(self) -> layout::LayoutParameters {
+		layout::LayoutParameters {
+			zone_redundancy: self.zone_redundancy.into(),
+		}
+	}
+}

src/api/admin/lib.rs

@@ -3,9 +3,44 @@ extern crate tracing;
 
 pub mod api_server;
 mod error;
+mod macros;
+
+pub mod api;
+pub mod openapi;
 mod router_v0;
 mod router_v1;
+mod router_v2;
 
+mod admin_token;
 mod bucket;
 mod cluster;
 mod key;
+mod layout;
+mod special;
+
+mod block;
+mod node;
+mod repair;
+mod worker;
+
+use std::sync::Arc;
+
+use garage_model::garage::Garage;
+
+pub use api_server::AdminApiServer as Admin;
+
+pub enum Authorization {
+	None,
+	MetricsToken,
+	AdminToken,
+}
+
+pub trait RequestHandler {
+	type Response;
+
+	fn handle(
+		self,
+		garage: &Arc<Garage>,
+		admin: &Admin,
+	) -> impl std::future::Future<Output = Result<Self::Response, error::Error>> + Send;
+}

src/api/admin/macros.rs

@@ -0,0 +1,206 @@
+macro_rules! admin_endpoints {
+    [
+        $(@special $special_endpoint:ident,)*
+        $($endpoint:ident,)*
+    ] => {
+        paste! {
+            #[derive(Debug, Clone, Serialize, Deserialize)]
+            pub enum AdminApiRequest {
+                $(
+                    $special_endpoint( [<$special_endpoint Request>] ),
+                )*
+                $(
+                    $endpoint( [<$endpoint Request>] ),
+                )*
+            }
+
+            #[derive(Debug, Clone, Serialize)]
+            #[serde(untagged)]
+            pub enum AdminApiResponse {
+                $(
+                    $endpoint( [<$endpoint Response>] ),
+                )*
+            }
+
+            #[derive(Debug, Clone, Serialize, Deserialize)]
+            pub enum TaggedAdminApiResponse {
+                $(
+                    $endpoint( [<$endpoint Response>] ),
+                )*
+            }
+
+            impl AdminApiRequest {
+                pub fn name(&self) -> &'static str {
+                    match self {
+                        $(
+                            Self::$special_endpoint(_) => stringify!($special_endpoint),
+                        )*
+                        $(
+                            Self::$endpoint(_) => stringify!($endpoint),
+                        )*
+                    }
+                }
+            }
+
+            impl AdminApiResponse {
+                pub fn tagged(self) -> TaggedAdminApiResponse {
+                    match self {
+                        $(
+                            Self::$endpoint(res) => TaggedAdminApiResponse::$endpoint(res),
+                        )*
+                    }
+                }
+            }
+
+            $(
+                impl From< [< $endpoint Request >] > for AdminApiRequest {
+                    fn from(req: [< $endpoint Request >]) -> AdminApiRequest {
+                        AdminApiRequest::$endpoint(req)
+                    }
+                }
+
+                impl TryFrom<TaggedAdminApiResponse> for [< $endpoint Response >] {
+                    type Error = TaggedAdminApiResponse;
+                    fn try_from(resp: TaggedAdminApiResponse) -> Result< [< $endpoint Response >], TaggedAdminApiResponse> {
+                        match resp {
+                            TaggedAdminApiResponse::$endpoint(v) => Ok(v),
+                            x => Err(x),
+                        }
+                    }
+                }
+            )*
+
+            impl RequestHandler for AdminApiRequest {
+                type Response = AdminApiResponse;
+
+                async fn handle(self, garage: &Arc<Garage>, admin: &Admin) -> Result<AdminApiResponse, Error> {
+                    Ok(match self {
+                        $(
+                            AdminApiRequest::$special_endpoint(_) => panic!(
+                                concat!(stringify!($special_endpoint), " needs to go through a special handler")
+                            ),
+                        )*
+                        $(
+                            AdminApiRequest::$endpoint(req) => AdminApiResponse::$endpoint(req.handle(garage, admin).await?),
+                        )*
+                    })
+                }
+            }
+        }
+    };
+}
+
+macro_rules! local_admin_endpoints {
+    [
+        $($endpoint:ident,)*
+    ] => {
+        paste! {
+            #[derive(Debug, Clone, Serialize, Deserialize)]
+            pub enum LocalAdminApiRequest {
+                $(
+                    $endpoint( [<Local $endpoint Request>] ),
+                )*
+            }
+
+            #[derive(Debug, Clone, Serialize, Deserialize)]
+            pub enum LocalAdminApiResponse {
+                $(
+                    $endpoint( [<Local $endpoint Response>] ),
+                )*
+            }
+
+            $(
+                pub type [< $endpoint Request >] = MultiRequest< [< Local $endpoint Request >] >;
+
+                pub type [< $endpoint RequestBody >] = [< Local $endpoint Request >];
+
+                pub type [< $endpoint Response >] = MultiResponse< [< Local $endpoint Response >] >;
+
+                impl From< [< Local $endpoint Request >] > for LocalAdminApiRequest {
+                    fn from(req: [< Local $endpoint Request >]) -> LocalAdminApiRequest {
+                        LocalAdminApiRequest::$endpoint(req)
+                    }
+                }
+
+                impl TryFrom<LocalAdminApiResponse> for [< Local $endpoint Response >] {
+                    type Error = LocalAdminApiResponse;
+                    fn try_from(resp: LocalAdminApiResponse) -> Result< [< Local $endpoint Response >], LocalAdminApiResponse> {
+                        match resp {
+                            LocalAdminApiResponse::$endpoint(v) => Ok(v),
+                            x => Err(x),
+                        }
+                    }
+                }
+
+                impl RequestHandler for [< $endpoint Request >] {
+                    type Response = [< $endpoint Response >];
+
+	                async fn handle(self, garage: &Arc<Garage>, admin: &Admin) -> Result<Self::Response, Error> {
+                        let to = find_matching_nodes(garage, self.node.as_str())?;
+
+                        let resps = garage.system.rpc_helper().call_many(&admin.endpoint,
+                            &to,
+                            AdminRpc::Internal(self.body.into()),
+                            RequestStrategy::with_priority(PRIO_NORMAL),
+                        ).await?;
+
+                        let mut ret = [< $endpoint Response >] {
+                            success: HashMap::new(),
+                            error: HashMap::new(),
+                        };
+                        for (node, resp) in resps {
+                            match resp {
+                                Ok(AdminRpcResponse::InternalApiOkResponse(r)) => {
+                                    match [< Local $endpoint Response >]::try_from(r) {
+                                        Ok(r) => {
+                                            ret.success.insert(hex::encode(node), r);
+                                        }
+                                        Err(_) => {
+                                            ret.error.insert(hex::encode(node), "returned invalid value".to_string());
+                                        }
+                                    }
+                                }
+                                Ok(AdminRpcResponse::ApiErrorResponse{error_code, http_code, message}) => {
+                                    ret.error.insert(hex::encode(node), format!("{} ({}): {}", error_code, http_code, message));
+                                }
+                                Ok(_) => {
+                                    ret.error.insert(hex::encode(node), "returned invalid value".to_string());
+                                }
+                                Err(e) => {
+                                    ret.error.insert(hex::encode(node), e.to_string());
+                                }
+                            }
+                        }
+
+                        Ok(ret)
+                    }
+                }
+            )*
+
+            impl LocalAdminApiRequest {
+                pub fn name(&self) -> &'static str {
+                    match self {
+                        $(
+                            Self::$endpoint(_) => stringify!($endpoint),
+                        )*
+                    }
+                }
+            }
+
+            impl RequestHandler for LocalAdminApiRequest {
+                type Response = LocalAdminApiResponse;
+
+                async fn handle(self, garage: &Arc<Garage>, admin: &Admin) -> Result<LocalAdminApiResponse, Error> {
+                    Ok(match self {
+                        $(
+                            LocalAdminApiRequest::$endpoint(req) => LocalAdminApiResponse::$endpoint(req.handle(garage, admin).await?),
+                        )*
+                    })
+                }
+            }
+        }
+    };
+}
+
+pub(crate) use admin_endpoints;
+pub(crate) use local_admin_endpoints;

src/api/admin/node.rs

@@ -0,0 +1,144 @@
+use std::fmt::Write;
+use std::sync::Arc;
+
+use format_table::format_table_to_string;
+
+use garage_util::error::Error as GarageError;
+
+use garage_table::replication::*;
+use garage_table::*;
+
+use garage_model::garage::Garage;
+
+use crate::api::*;
+use crate::error::Error;
+use crate::{Admin, RequestHandler};
+
+impl RequestHandler for LocalGetNodeInfoRequest {
+	type Response = LocalGetNodeInfoResponse;
+
+	async fn handle(
+		self,
+		garage: &Arc<Garage>,
+		_admin: &Admin,
+	) -> Result<LocalGetNodeInfoResponse, Error> {
+		Ok(LocalGetNodeInfoResponse {
+			node_id: hex::encode(garage.system.id),
+			garage_version: garage_util::version::garage_version().to_string(),
+			garage_features: garage_util::version::garage_features()
+				.map(|features| features.iter().map(ToString::to_string).collect()),
+			rust_version: garage_util::version::rust_version().to_string(),
+			db_engine: garage.db.engine(),
+		})
+	}
+}
+
+impl RequestHandler for LocalCreateMetadataSnapshotRequest {
+	type Response = LocalCreateMetadataSnapshotResponse;
+
+	async fn handle(
+		self,
+		garage: &Arc<Garage>,
+		_admin: &Admin,
+	) -> Result<LocalCreateMetadataSnapshotResponse, Error> {
+		garage_model::snapshot::async_snapshot_metadata(garage).await?;
+		Ok(LocalCreateMetadataSnapshotResponse)
+	}
+}
+
+impl RequestHandler for LocalGetNodeStatisticsRequest {
+	type Response = LocalGetNodeStatisticsResponse;
+
+	// FIXME: return this as a JSON struct instead of text
+	async fn handle(
+		self,
+		garage: &Arc<Garage>,
+		_admin: &Admin,
+	) -> Result<LocalGetNodeStatisticsResponse, Error> {
+		let sys_status = garage.system.local_status();
+
+		let mut ret = format_table_to_string(vec![
+			format!("Node ID:\t{:?}", garage.system.id),
+			format!("Hostname:\t{}", sys_status.hostname.unwrap_or_default(),),
+			format!(
+				"Garage version:\t{}",
+				garage_util::version::garage_version(),
+			),
+			format!(
+				"Garage features:\t{}",
+				garage_util::version::garage_features()
+					.map(|list| list.join(", "))
+					.unwrap_or_else(|| "(unknown)".into()),
+			),
+			format!(
+				"Rust compiler version:\t{}",
+				garage_util::version::rust_version(),
+			),
+			format!("Database engine:\t{}", garage.db.engine()),
+		]);
+
+		// Gather table statistics
+		let mut table = vec!["  Table\tItems\tMklItems\tMklTodo\tInsQueue\tGcTodo".into()];
+		table.push(gather_table_stats(&garage.admin_token_table)?);
+		table.push(gather_table_stats(&garage.bucket_table)?);
+		table.push(gather_table_stats(&garage.bucket_alias_table)?);
+		table.push(gather_table_stats(&garage.key_table)?);
+
+		table.push(gather_table_stats(&garage.object_table)?);
+		table.push(gather_table_stats(&garage.object_counter_table.table)?);
+		table.push(gather_table_stats(&garage.mpu_table)?);
+		table.push(gather_table_stats(&garage.mpu_counter_table.table)?);
+		table.push(gather_table_stats(&garage.version_table)?);
+		table.push(gather_table_stats(&garage.block_ref_table)?);
+
+		#[cfg(feature = "k2v")]
+		{
+			table.push(gather_table_stats(&garage.k2v.item_table)?);
+			table.push(gather_table_stats(&garage.k2v.counter_table.table)?);
+		}
+
+		write!(
+			&mut ret,
+			"\nTable stats:\n{}",
+			format_table_to_string(table)
+		)
+		.unwrap();
+
+		// Gather block manager statistics
+		writeln!(&mut ret, "\nBlock manager stats:").unwrap();
+		let rc_len = garage.block_manager.rc_len()?.to_string();
+
+		ret += &format_table_to_string(vec![
+			format!("  number of RC entries:\t{} (~= number of blocks)", rc_len),
+			format!(
+				"  resync queue length:\t{}",
+				garage.block_manager.resync.queue_len()?
+			),
+			format!(
+				"  blocks with resync errors:\t{}",
+				garage.block_manager.resync.errors_len()?
+			),
+		]);
+
+		Ok(LocalGetNodeStatisticsResponse { freeform: ret })
+	}
+}
+
+fn gather_table_stats<F, R>(t: &Arc<Table<F, R>>) -> Result<String, Error>
+where
+	F: TableSchema + 'static,
+	R: TableReplication + 'static,
+{
+	let data_len = t.data.store.len().map_err(GarageError::from)?.to_string();
+	let mkl_len = t.merkle_updater.merkle_tree_len()?.to_string();
+
+	Ok(format!(
+		"  {}\t{}\t{}\t{}\t{}\t{}",
+		F::TABLE_NAME,
+		data_len,
+		mkl_len,
+		t.merkle_updater.todo_len()?,
+		t.data.insert_queue_len()?,
+		t.data.gc_todo_len()?
+	))
+}

src/api/admin/openapi.rs

@@ -0,0 +1,924 @@
+#![allow(dead_code)]
+#![allow(non_snake_case)]
+
+use utoipa::{Modify, OpenApi};
+
+use crate::api::*;
+
+// **********************************************
+//      Special endpoints
+// **********************************************
+
+#[utoipa::path(get,
+    path = "/metrics",
+    tag = "Special endpoints",
+    description = "Prometheus metrics endpoint",
+    security((), ("bearerAuth" = [])),
+	responses(
+            (status = 200, description = "Garage daemon metrics exported in Prometheus format"),
+        ),
+)]
+fn Metrics() -> () {}
+
+#[utoipa::path(get,
+    path = "/health",
+    tag = "Special endpoints",
+    description = "
+Check cluster health. The status code returned by this function indicates
+whether this Garage daemon can answer API requests.
+Garage will return `200 OK` even if some storage nodes are disconnected,
+as long as it is able to have a quorum of nodes for read and write operations.
+    ",
+    security(()),
+	responses(
+            (status = 200, description = "Garage is able to answer requests"),
+            (status = 503, description = "This Garage daemon is not able to handle requests")
+        ),
+)]
+fn Health() -> () {}
+
+#[utoipa::path(get,
+    path = "/check",
+    tag = "Special endpoints",
+    description = "
+Static website domain name check. Checks whether a bucket is configured to serve
+a static website for the requested domain. This is used by reverse proxies such
+as Caddy or Tricot, to avoid requesting TLS certificates for domain names that
+do not correspond to an actual website.
+    ",
+    params(
+        ("domain", description = "The domain name to check for"),
+    ),
+    security(()),
+	responses(
+            (status = 200, description = "The domain name redirects to a static website bucket"),
+            (status = 400, description = "No static website bucket exists for this domain")
+        ),
+)]
+fn CheckDomain() -> () {}
+
+// **********************************************
+//      Cluster operations
+// **********************************************
+
+#[utoipa::path(get,
+    path = "/v2/GetClusterStatus",
+    tag = "Cluster",
+    description = "
+Returns the cluster's current status, including:
+
+- ID of the node being queried and its version of the Garage daemon
+- Live nodes
+- Currently configured cluster layout
+- Staged changes to the cluster layout
+
+*Capacity is given in bytes*
+    ",
+	responses(
+            (status = 200, description = "Cluster status report", body = GetClusterStatusResponse),
+            (status = 500, description = "Internal server error")
+        ),
+)]
+fn GetClusterStatus() -> () {}
+
+#[utoipa::path(get,
+    path = "/v2/GetClusterHealth",
+    tag = "Cluster",
+    description = "Returns the global status of the cluster, the number of connected nodes (over the number of known ones), the number of healthy storage nodes (over the declared ones), and the number of healthy partitions (over the total).",
+	responses(
+            (status = 200, description = "Cluster health report", body = GetClusterHealthResponse),
+        ),
+)]
+fn GetClusterHealth() -> () {}
+
+#[utoipa::path(get,
+    path = "/v2/GetClusterStatistics",
+    tag = "Cluster",
+    description = "
+Fetch global cluster statistics.
+
+*Note: do not try to parse the `freeform` field of the response, it is given as a string specifically because its format is not stable.*
+    ",
+	responses(
+            (status = 200, description = "Global cluster statistics", body = GetClusterStatisticsResponse),
+            (status = 500, description = "Internal server error")
+        ),
+)]
+fn GetClusterStatistics() -> () {}
+
+#[utoipa::path(post,
+    path = "/v2/ConnectClusterNodes",
+    tag = "Cluster",
+    description = "Instructs this Garage node to connect to other Garage nodes at specified `<node_id>@<net_address>`. `node_id` is generated automatically on node start.",
+    request_body=ConnectClusterNodesRequest,
+	responses(
+            (status = 200, description = "The request has been handled correctly but it does not mean that all connection requests succeeded; some might have fail, you need to check the body!", body = ConnectClusterNodesResponse),
+            (status = 500, description = "Internal server error")
+        ),
+)]
+fn ConnectClusterNodes() -> () {}
+
+// **********************************************
+//      Admin API token operations
+// **********************************************
+
+#[utoipa::path(get,
+    path = "/v2/ListAdminTokens",
+    tag = "Admin API token",
+    description = "Returns all admin API tokens in the cluster.",
+	responses(
+            (status = 200, description = "Returns info about all admin API tokens", body = ListAdminTokensResponse),
+            (status = 500, description = "Internal server error")
+        ),
+)]
+fn ListAdminTokens() -> () {}
+
+#[utoipa::path(get,
+    path = "/v2/GetAdminTokenInfo",
+    tag = "Admin API token",
+    description = "
+Return information about a specific admin API token.
+You can search by specifying the exact token identifier (`id`) or by specifying a pattern (`search`).
+    ",
+    params(GetAdminTokenInfoRequest),
+	responses(
+            (status = 200, description = "Information about the admin token", body = GetAdminTokenInfoResponse),
+            (status = 500, description = "Internal server error")
+        ),
+)]
+fn GetAdminTokenInfo() -> () {}
+
+#[utoipa::path(post,
+    path = "/v2/CreateAdminToken",
+    tag = "Admin API token",
+    description = "Creates a new admin API token",
+    request_body = UpdateAdminTokenRequestBody,
+	responses(
+            (status = 200, description = "Admin token has been created", body = CreateAdminTokenResponse),
+            (status = 500, description = "Internal server error")
+        ),
+)]
+fn CreateAdminToken() -> () {}
+
+#[utoipa::path(post,
+    path = "/v2/UpdateAdminToken",
+    tag = "Admin API token",
+    description = "
+Updates information about the specified admin API token.
+    ",
+    request_body = UpdateAdminTokenRequestBody,
+    params(
+        ("id", description = "Admin API token ID"),
+    ),
+	responses(
+            (status = 200, description = "Admin token has been updated", body = UpdateAdminTokenResponse),
+            (status = 500, description = "Internal server error")
+        ),
+)]
+fn UpdateAdminToken() -> () {}
+
+#[utoipa::path(post,
+    path = "/v2/DeleteAdminToken",
+    tag = "Admin API token",
+    description = "Delete an admin API token from the cluster, revoking all its permissions.",
+    params(
+        ("id", description = "Admin API token ID"),
+    ),
+	responses(
+            (status = 200, description = "Admin token has been deleted"),
+            (status = 500, description = "Internal server error")
+        ),
+)]
+fn DeleteAdminToken() -> () {}
+
+// **********************************************
+//      Layout operations
+// **********************************************
+
+#[utoipa::path(get,
+    path = "/v2/GetClusterLayout",
+    tag = "Cluster layout",
+    description = "
+Returns the cluster's current layout, including:
+
+- Currently configured cluster layout
+- Staged changes to the cluster layout
+
+*Capacity is given in bytes*
+    ",
+	responses(
+            (status = 200, description = "Current cluster layout", body = GetClusterLayoutResponse),
+            (status = 500, description = "Internal server error")
+        ),
+)]
+fn GetClusterLayout() -> () {}
+
+#[utoipa::path(get,
+    path = "/v2/GetClusterLayoutHistory",
+    tag = "Cluster layout",
+    description = "
+Returns the history of layouts in the cluster
+    ",
+	responses(
+            (status = 200, description = "Cluster layout history", body = GetClusterLayoutHistoryResponse),
+            (status = 500, description = "Internal server error")
+        ),
+)]
+fn GetClusterLayoutHistory() -> () {}
+
+#[utoipa::path(post,
+    path = "/v2/UpdateClusterLayout",
+    tag = "Cluster layout",
+    description = "
+Send modifications to the cluster layout. These modifications will be included in the staged role changes, visible in subsequent calls of `GET /GetClusterHealth`. Once the set of staged changes is satisfactory, the user may call `POST /ApplyClusterLayout` to apply the changed changes, or `POST /RevertClusterLayout` to clear all of the staged changes in the layout.
+
+Setting the capacity to `null` will configure the node as a gateway.
+Otherwise, capacity must be now set in bytes (before Garage 0.9 it was arbitrary weights).
+For example to declare 100GB, you must set `capacity: 100000000000`.
+
+Garage uses internally the International System of Units (SI), it assumes that 1kB = 1000 bytes, and displays storage as kB, MB, GB (and not KiB, MiB, GiB that assume 1KiB = 1024 bytes).
+    ",
+    request_body(
+        content=UpdateClusterLayoutRequest,
+        description="
+To add a new node to the layout or to change the configuration of an existing node, simply set the values you want (`zone`, `capacity`, and `tags`).
+To remove a node, simply pass the `remove: true` field.
+This logic is represented in OpenAPI with a 'One Of' object.
+
+Contrary to the CLI that may update only a subset of the fields capacity, zone and tags, when calling this API all of these values must be specified.
+        "
+    ),
+	responses(
+            (status = 200, description = "Proposed changes have been added to the list of pending changes", body = UpdateClusterLayoutResponse),
+            (status = 500, description = "Internal server error")
+        ),
+)]
+fn UpdateClusterLayout() -> () {}
+
+#[utoipa::path(post,
+    path = "/v2/PreviewClusterLayoutChanges",
+    tag = "Cluster layout",
+    description = "
+Computes a new layout taking into account the staged parameters, and returns it with detailed statistics. The new layout is not applied in the cluster.
+
+*Note: do not try to parse the `message` field of the response, it is given as an array of string specifically because its format is not stable.*
+    ",
+	responses(
+            (status = 200, description = "Information about the new layout", body = PreviewClusterLayoutChangesResponse),
+            (status = 500, description = "Internal server error")
+        ),
+)]
+fn PreviewClusterLayoutChanges() -> () {}
+
+#[utoipa::path(post,
+    path = "/v2/ApplyClusterLayout",
+    tag = "Cluster layout",
+    description = "
+Applies to the cluster the layout changes currently registered as staged layout changes.
+
+*Note: do not try to parse the `message` field of the response, it is given as an array of string specifically because its format is not stable.*
+    ",
+    request_body=ApplyClusterLayoutRequest,
+	responses(
+            (status = 200, description = "The updated cluster layout has been applied in the cluster", body = ApplyClusterLayoutResponse),
+            (status = 500, description = "Internal server error")
+        ),
+)]
+fn ApplyClusterLayout() -> () {}
+
+#[utoipa::path(post,
+    path = "/v2/RevertClusterLayout",
+    tag = "Cluster layout",
+    description = "Clear staged layout changes",
+	responses(
+            (status = 200, description = "All pending changes to the cluster layout have been erased", body = RevertClusterLayoutResponse),
+            (status = 500, description = "Internal server error")
+        ),
+)]
+fn RevertClusterLayout() -> () {}
+
+#[utoipa::path(post,
+    path = "/v2/ClusterLayoutSkipDeadNodes",
+    tag = "Cluster layout",
+    description = "Force progress in layout update trackers",
+    request_body = ClusterLayoutSkipDeadNodesRequest,
+	responses(
+            (status = 200, description = "Request has been taken into account", body = ClusterLayoutSkipDeadNodesResponse),
+            (status = 500, description = "Internal server error")
+        ),
+)]
+fn ClusterLayoutSkipDeadNodes() -> () {}
+
+// **********************************************
+//      Access key operations
+// **********************************************
+
+#[utoipa::path(get,
+    path = "/v2/ListKeys",
+    tag = "Access key",
+    description = "Returns all API access keys in the cluster.",
+	responses(
+            (status = 200, description = "Returns the key identifier (aka `AWS_ACCESS_KEY_ID`) and its associated, human friendly, name if any (otherwise return an empty string)", body = ListKeysResponse),
+            (status = 500, description = "Internal server error")
+        ),
+)]
+fn ListKeys() -> () {}
+
+#[utoipa::path(get,
+    path = "/v2/GetKeyInfo",
+    tag = "Access key",
+    description = "
+Return information about a specific key like its identifiers, its permissions and buckets on which it has permissions.
+You can search by specifying the exact key identifier (`id`) or by specifying a pattern (`search`).
+
+For confidentiality reasons, the secret key is not returned by default: you must pass the `showSecretKey` query parameter to get it.
+    ",
+    params(GetKeyInfoRequest),
+	responses(
+            (status = 200, description = "Information about the access key", body = GetKeyInfoResponse),
+            (status = 500, description = "Internal server error")
+        ),
+)]
+fn GetKeyInfo() -> () {}
+
+#[utoipa::path(post,
+    path = "/v2/CreateKey",
+    tag = "Access key",
+    description = "Creates a new API access key.",
+    request_body = CreateKeyRequest,
+	responses(
+            (status = 200, description = "Access key has been created", body = CreateKeyResponse),
+            (status = 500, description = "Internal server error")
+        ),
+)]
+fn CreateKey() -> () {}
+
+#[utoipa::path(post,
+    path = "/v2/ImportKey",
+    tag = "Access key",
+    description = "
+Imports an existing API key. This feature must only be used for migrations and backup restore.
+
+**Do not use it to generate custom key identifiers or you will break your Garage cluster.**
+    ",
+    request_body = ImportKeyRequest,
+	responses(
+            (status = 200, description = "Access key has been imported", body = ImportKeyResponse),
+            (status = 500, description = "Internal server error")
+        ),
+)]
+fn ImportKey() -> () {}
+
+#[utoipa::path(post,
+    path = "/v2/UpdateKey",
+    tag = "Access key",
+    description = "
+Updates information about the specified API access key.
+
+*Note: the secret key is not returned in the response, `null` is sent instead.*
+    ",
+    request_body = UpdateKeyRequestBody,
+    params(
+        ("id", description = "Access key ID"),
+    ),
+	responses(
+            (status = 200, description = "Access key has been updated", body = UpdateKeyResponse),
+            (status = 500, description = "Internal server error")
+        ),
+)]
+fn UpdateKey() -> () {}
+
+#[utoipa::path(post,
+    path = "/v2/DeleteKey",
+    tag = "Access key",
+    description = "Delete a key from the cluster. Its access will be removed from all the buckets. Buckets are not automatically deleted and can be dangling. You should manually delete them before. ",
+    params(
+        ("id", description = "Access key ID"),
+    ),
+	responses(
+            (status = 200, description = "Access key has been deleted"),
+            (status = 500, description = "Internal server error")
+        ),
+)]
+fn DeleteKey() -> () {}
+
+// **********************************************
+//      Bucket operations
+// **********************************************
+
+#[utoipa::path(get,
+    path = "/v2/ListBuckets",
+    tag = "Bucket",
+    description = "List all the buckets on the cluster with their UUID and their global and local aliases.",
+	responses(
+            (status = 200, description = "Returns the UUID of all the buckets and all their aliases", body = ListBucketsResponse),
+            (status = 500, description = "Internal server error")
+        ),
+)]
+fn ListBuckets() -> () {}
+
+#[utoipa::path(get,
+    path = "/v2/GetBucketInfo",
+    tag = "Bucket",
+    description = "
+Given a bucket identifier (`id`) or a global alias (`alias`), get its information.
+It includes its aliases, its web configuration, keys that have some permissions
+on it, some statistics (number of objects, size), number of dangling multipart uploads,
+and its quotas (if any).
+    ",
+    params(GetBucketInfoRequest),
+	responses(
+            (status = 200, description = "Returns exhaustive information about the bucket", body = GetBucketInfoResponse),
+            (status = 500, description = "Internal server error")
+        ),
+)]
+fn GetBucketInfo() -> () {}
+
+#[utoipa::path(post,
+    path = "/v2/CreateBucket",
+    tag = "Bucket",
+    description = "
+Creates a new bucket, either with a global alias, a local one, or no alias at all.
+Technically, you can also specify both `globalAlias` and `localAlias` and that would create two aliases.
+    ",
+    request_body = CreateBucketRequest,
+	responses(
+            (status = 200, description = "Returns exhaustive information about the bucket", body = CreateBucketResponse),
+            (status = 500, description = "Internal server error")
+        ),
+)]
+fn CreateBucket() -> () {}
+
+#[utoipa::path(post,
+    path = "/v2/UpdateBucket",
+    tag = "Bucket",
+    description = "
+All fields (`websiteAccess` and `quotas`) are optional.
+If they are present, the corresponding modifications are applied to the bucket, otherwise nothing is changed.
+
+In `websiteAccess`: if `enabled` is `true`, `indexDocument` must be specified.
+The field `errorDocument` is optional, if no error document is set a generic
+error message is displayed when errors happen. Conversely, if `enabled` is
+`false`, neither `indexDocument` nor `errorDocument` must be specified.
+
+In `quotas`: new values of `maxSize` and `maxObjects` must both be specified, or set to `null`
+to remove the quotas. An absent value will be considered the same as a `null`. It is not possible
+to change only one of the two quotas.
+    ",
+    params(
+        ("id", description = "ID of the bucket to update"),
+    ),
+    request_body = UpdateBucketRequestBody,
+	responses(
+            (status = 200, description = "Bucket has been updated", body = UpdateBucketResponse),
+            (status = 404, description = "Bucket not found"),
+            (status = 500, description = "Internal server error")
+        ),
+)]
+fn UpdateBucket() -> () {}
+
+#[utoipa::path(post,
+    path = "/v2/DeleteBucket",
+    tag = "Bucket",
+    description = "
+Deletes a storage bucket. A bucket cannot be deleted if it is not empty.
+
+**Warning:** this will delete all aliases associated with the bucket!
+    ",
+    params(
+        ("id", description = "ID of the bucket to delete"),
+    ),
+	responses(
+            (status = 200, description = "Bucket has been deleted"),
+            (status = 400, description = "Bucket is not empty"),
+            (status = 404, description = "Bucket not found"),
+            (status = 500, description = "Internal server error")
+        ),
+)]
+fn DeleteBucket() -> () {}
+
+#[utoipa::path(post,
+    path = "/v2/CleanupIncompleteUploads",
+    tag = "Bucket",
+    description = "Removes all incomplete multipart uploads that are older than the specified number of seconds.",
+    request_body = CleanupIncompleteUploadsRequest,
+	responses(
+            (status = 200, description = "The bucket was cleaned up successfully", body = CleanupIncompleteUploadsResponse),
+            (status = 500, description = "Internal server error")
+        ),
+)]
+fn CleanupIncompleteUploads() -> () {}
+
+#[utoipa::path(get,
+    path = "/v2/InspectObject",
+    tag = "Bucket",
+    description = "
+Returns detailed information about an object in a bucket, including its internal state in Garage.
+
+This API call can be used to list the data blocks referenced by an object,
+as well as to view metadata associated to the object.
+
+This call may return a list of more than one version for the object, for instance in the
+case where there is a currently stored version of the object, and a newer version whose
+upload is in progress and not yet finished.
+    ",
+    params(InspectObjectRequest),
+	responses(
+            (status = 200, description = "Returns exhaustive information about the object", body = InspectObjectResponse),
+            (status = 404, description = "Object not found"),
+            (status = 500, description = "Internal server error")
+        ),
+)]
+fn InspectObject() -> () {}
+
+// **********************************************
+//      Operations on permissions for keys on buckets
+// **********************************************
+
+#[utoipa::path(post,
+    path = "/v2/AllowBucketKey",
+    tag = "Permission",
+    description = "
+⚠️ **DISCLAIMER**: Garage's developers are aware that this endpoint has an unconventional semantic. Be extra careful when implementing it, its behavior is not obvious.
+
+Allows a key to do read/write/owner operations on a bucket.
+
+Flags in permissions which have the value true will be activated. Other flags will remain unchanged (ie. they will keep their internal value).
+
+For example, if you set read to true, the key will be allowed to read the bucket.
+If you set it to false, the key will keeps its previous read permission.
+If you want to disallow read for the key, check the DenyBucketKey operation.
+    ",
+    request_body = AllowBucketKeyRequest,
+	responses(
+            (status = 200, description = "Returns exhaustive information about the bucket", body = AllowBucketKeyResponse),
+            (status = 500, description = "Internal server error")
+        ),
+)]
+fn AllowBucketKey() -> () {}
+
+#[utoipa::path(post,
+    path = "/v2/DenyBucketKey",
+    tag = "Permission",
+    description = "
+⚠️ **DISCLAIMER**: Garage's developers are aware that this endpoint has an unconventional semantic. Be extra careful when implementing it, its behavior is not obvious.
+
+Denies a key from doing read/write/owner operations on a bucket.
+
+Flags in permissions which have the value true will be deactivated. Other flags will remain unchanged.
+
+For example, if you set read to true, the key will be denied from reading.
+If you set read to false,  the key will keep its previous permissions.
+If you want the key to have the reading permission, check the AllowBucketKey operation.
+    ",
+    request_body = DenyBucketKeyRequest,
+	responses(
+            (status = 200, description = "Returns exhaustive information about the bucket", body = DenyBucketKeyResponse),
+            (status = 500, description = "Internal server error")
+        ),
+)]
+fn DenyBucketKey() -> () {}
+
+// **********************************************
+//      Operations on bucket aliases
+// **********************************************
+
+#[utoipa::path(post,
+    path = "/v2/AddBucketAlias",
+    tag = "Bucket alias",
+    description = "Add an alias for the target bucket.  This can be either a global or a local alias, depending on which fields are specified.",
+    request_body = AddBucketAliasRequest,
+	responses(
+            (status = 200, description = "Returns exhaustive information about the bucket", body = AddBucketAliasResponse),
+            (status = 500, description = "Internal server error")
+        ),
+)]
+fn AddBucketAlias() -> () {}
+
+#[utoipa::path(post,
+    path = "/v2/RemoveBucketAlias",
+    tag = "Bucket alias",
+    description = "Remove an alias for the target bucket.  This can be either a global or a local alias, depending on which fields are specified.",
+    request_body = RemoveBucketAliasRequest,
+	responses(
+            (status = 200, description = "Returns exhaustive information about the bucket", body = RemoveBucketAliasResponse),
+            (status = 500, description = "Internal server error")
+        ),
+)]
+fn RemoveBucketAlias() -> () {}
+
+// **********************************************
+//      Node operations
+// **********************************************
+
+#[utoipa::path(get,
+    path = "/v2/GetNodeInfo",
+    tag = "Node",
+    description = "
+Return information about the Garage daemon running on one or several nodes.
+    ",
+    params(
+        ("node", description = "Node ID to query, or `*` for all nodes, or `self` for the node responding to the request"),
+    ),
+	responses(
+            (status = 200, description = "Responses from individual cluster nodes", body = MultiResponse<LocalGetNodeInfoResponse>),
+            (status = 500, description = "Internal server error")
+        ),
+)]
+fn GetNodeInfo() -> () {}
+
+#[utoipa::path(get,
+    path = "/v2/GetNodeStatistics",
+    tag = "Node",
+    description = "
+Fetch statistics for one or several Garage nodes.
+
+*Note: do not try to parse the `freeform` field of the response, it is given as a string specifically because its format is not stable.*
+    ",
+    params(
+        ("node", description = "Node ID to query, or `*` for all nodes, or `self` for the node responding to the request"),
+    ),
+	responses(
+            (status = 200, description = "Responses from individual cluster nodes", body = MultiResponse<LocalGetNodeStatisticsResponse>),
+            (status = 500, description = "Internal server error")
+        ),
+)]
+fn GetNodeStatistics() -> () {}
+
+#[utoipa::path(post,
+    path = "/v2/CreateMetadataSnapshot",
+    tag = "Node",
+    description = "
+Instruct one or several nodes to take a snapshot of their metadata databases.
+    ",
+    params(
+        ("node", description = "Node ID to query, or `*` for all nodes, or `self` for the node responding to the request"),
+    ),
+	responses(
+            (status = 200, description = "Responses from individual cluster nodes", body = MultiResponse<LocalCreateMetadataSnapshotResponse>),
+            (status = 500, description = "Internal server error")
+        ),
+)]
+fn CreateMetadataSnapshot() -> () {}
+
+#[utoipa::path(post,
+    path = "/v2/LaunchRepairOperation",
+    tag = "Node",
+    description = "
+Launch a repair operation on one or several cluster nodes.
+    ",
+    params(
+        ("node", description = "Node ID to query, or `*` for all nodes, or `self` for the node responding to the request"),
+    ),
+    request_body = LocalLaunchRepairOperationRequest,
+	responses(
+            (status = 200, description = "Responses from individual cluster nodes", body = MultiResponse<LocalLaunchRepairOperationResponse>),
+            (status = 500, description = "Internal server error")
+        ),
+)]
+fn LaunchRepairOperation() -> () {}
+
+// **********************************************
+//      Worker operations
+// **********************************************
+
+#[utoipa::path(post,
+    path = "/v2/ListWorkers",
+    tag = "Worker",
+    description = "
+List background workers currently running on one or several cluster nodes.
+    ",
+    params(
+        ("node", description = "Node ID to query, or `*` for all nodes, or `self` for the node responding to the request"),
+    ),
+    request_body = LocalListWorkersRequest,
+	responses(
+            (status = 200, description = "Responses from individual cluster nodes", body = MultiResponse<LocalListWorkersResponse>),
+            (status = 500, description = "Internal server error")
+        ),
+)]
+fn ListWorkers() -> () {}
+
+#[utoipa::path(post,
+    path = "/v2/GetWorkerInfo",
+    tag = "Worker",
+    description = "
+Get information about the specified background worker on one or several cluster nodes.
+    ",
+    params(
+        ("node", description = "Node ID to query, or `*` for all nodes, or `self` for the node responding to the request"),
+    ),
+    request_body = LocalGetWorkerInfoRequest,
+	responses(
+            (status = 200, description = "Responses from individual cluster nodes", body = MultiResponse<LocalGetWorkerInfoResponse>),
+            (status = 500, description = "Internal server error")
+        ),
+)]
+fn GetWorkerInfo() -> () {}
+
+#[utoipa::path(post,
+    path = "/v2/GetWorkerVariable",
+    tag = "Worker",
+    description = "
+Fetch values of one or several worker variables, from one or several cluster nodes.
+    ",
+    params(
+        ("node", description = "Node ID to query, or `*` for all nodes, or `self` for the node responding to the request"),
+    ),
+    request_body = LocalGetWorkerVariableRequest,
+	responses(
+            (status = 200, description = "Responses from individual cluster nodes", body = MultiResponse<LocalGetWorkerVariableResponse>),
+            (status = 500, description = "Internal server error")
+        ),
+)]
+fn GetWorkerVariable() -> () {}
+
+#[utoipa::path(post,
+    path = "/v2/SetWorkerVariable",
+    tag = "Worker",
+    description = "
+Set the value for a worker variable, on one or several cluster nodes.
+    ",
+    params(
+        ("node", description = "Node ID to query, or `*` for all nodes, or `self` for the node responding to the request"),
+    ),
+    request_body = LocalSetWorkerVariableRequest,
+	responses(
+            (status = 200, description = "Responses from individual cluster nodes", body = MultiResponse<LocalSetWorkerVariableResponse>),
+            (status = 500, description = "Internal server error")
+        ),
+)]
+fn SetWorkerVariable() -> () {}
+
+// **********************************************
+//      Block operations
+// **********************************************
+
+#[utoipa::path(get,
+    path = "/v2/ListBlockErrors",
+    tag = "Block",
+    description = "
+List data blocks that are currently in an errored state on one or several Garage nodes.
+    ",
+    params(
+        ("node", description = "Node ID to query, or `*` for all nodes, or `self` for the node responding to the request"),
+    ),
+	responses(
+            (status = 200, description = "Responses from individual cluster nodes", body = MultiResponse<LocalListBlockErrorsResponse>),
+            (status = 500, description = "Internal server error")
+        ),
+)]
+fn ListBlockErrors() -> () {}
+
+#[utoipa::path(post,
+    path = "/v2/GetBlockInfo",
+    tag = "Block",
+    description = "
+Get detailed information about a data block stored on a Garage node, including all object versions and in-progress multipart uploads that contain a reference to this block.
+    ",
+    params(
+        ("node", description = "Node ID to query, or `*` for all nodes, or `self` for the node responding to the request"),
+    ),
+    request_body = LocalGetBlockInfoRequest,
+	responses(
+            (status = 200, description = "Detailed block information", body = MultiResponse<LocalGetBlockInfoResponse>),
+            (status = 500, description = "Internal server error")
+        ),
+)]
+fn GetBlockInfo() -> () {}
+
+#[utoipa::path(post,
+    path = "/v2/RetryBlockResync",
+    tag = "Block",
+    description = "
+Instruct Garage node(s) to retry the resynchronization of one or several missing data block(s).
+    ",
+    params(
+        ("node", description = "Node ID to query, or `*` for all nodes, or `self` for the node responding to the request"),
+    ),
+    request_body = LocalRetryBlockResyncRequest,
+	responses(
+            (status = 200, description = "Responses from individual cluster nodes", body = MultiResponse<LocalRetryBlockResyncResponse>),
+            (status = 500, description = "Internal server error")
+        ),
+)]
+fn RetryBlockResync() -> () {}
+
+#[utoipa::path(post,
+    path = "/v2/PurgeBlocks",
+    tag = "Block",
+    description = "
+Purge references to one or several missing data blocks.
+
+This will remove all objects and in-progress multipart uploads that contain the specified data block(s). The objects will be permanently deleted from the buckets in which they appear. Use with caution.
+    ",
+    params(
+        ("node", description = "Node ID to query, or `*` for all nodes, or `self` for the node responding to the request"),
+    ),
+    request_body = LocalPurgeBlocksRequest,
+	responses(
+            (status = 200, description = "Responses from individual cluster nodes", body = MultiResponse<LocalPurgeBlocksResponse>),
+            (status = 500, description = "Internal server error")
+        ),
+)]
+fn PurgeBlocks() -> () {}
+
+// **********************************************
+// **********************************************
+// **********************************************
+
+struct SecurityAddon;
+
+impl Modify for SecurityAddon {
+	fn modify(&self, openapi: &mut utoipa::openapi::OpenApi) {
+		use utoipa::openapi::security::*;
+		let components = openapi.components.as_mut().unwrap(); // we can unwrap safely since there already is components registered.
+		components.add_security_scheme(
+			"bearerAuth",
+			SecurityScheme::Http(Http::builder().scheme(HttpAuthScheme::Bearer).build()),
+		)
+	}
+}
+
+#[derive(OpenApi)]
+#[openapi(
+    info(
+        version = "v2.0.0",
+        title = "Garage administration API",
+        description = "Administrate your Garage cluster programatically, including status, layout, keys, buckets, and maintainance tasks.
+
+*Disclaimer: This API may change in future Garage versions. Read the changelog and upgrade your scripts before upgrading. Additionnaly, this specification is early stage and can contain bugs, so be careful and please report any issues on our issue tracker.*",
+        contact(
+            name = "The Garage team",
+            email = "garagehq@deuxfleurs.fr",
+            url = "https://garagehq.deuxfleurs.fr/",
+        ),
+    ),
+    modifiers(&SecurityAddon),
+    security(("bearerAuth" = [])),
+    paths(
+        // Special ops
+        Metrics,
+        Health,
+        CheckDomain,
+        // Cluster operations
+        GetClusterHealth,
+        GetClusterStatus,
+        GetClusterStatistics,
+        ConnectClusterNodes,
+        // Admin token operations
+        ListAdminTokens,
+        GetAdminTokenInfo,
+        CreateAdminToken,
+        UpdateAdminToken,
+        DeleteAdminToken,
+        // Layout operations
+        GetClusterLayout,
+        GetClusterLayoutHistory,
+        UpdateClusterLayout,
+        PreviewClusterLayoutChanges,
+        ApplyClusterLayout,
+        RevertClusterLayout,
+        ClusterLayoutSkipDeadNodes,
+        // Key operations
+        ListKeys,
+        GetKeyInfo,
+        CreateKey,
+        ImportKey,
+        UpdateKey,
+        DeleteKey,
+        // Bucket operations
+        ListBuckets,
+        GetBucketInfo,
+        CreateBucket,
+        UpdateBucket,
+        DeleteBucket,
+        CleanupIncompleteUploads,
+        InspectObject,
+        // Operations on permissions
+        AllowBucketKey,
+        DenyBucketKey,
+        // Operations on aliases
+        AddBucketAlias,
+        RemoveBucketAlias,
+        // Node operations
+        GetNodeInfo,
+        GetNodeStatistics,
+        CreateMetadataSnapshot,
+        LaunchRepairOperation,
+        // Worker operations
+        ListWorkers,
+        GetWorkerInfo,
+        GetWorkerVariable,
+        SetWorkerVariable,
+        // Block operations
+        ListBlockErrors,
+        GetBlockInfo,
+        RetryBlockResync,
+        PurgeBlocks,
+    ),
+    servers(
+        (url = "http://localhost:3903/", description = "A local server")
+    ),
+)]
+pub struct ApiDoc;

src/api/admin/repair.rs

@@ -5,6 +5,14 @@ use std::time::Duration;
 use async_trait::async_trait;
 use tokio::sync::watch;
 
+use garage_util::background::*;
+use garage_util::data::*;
+use garage_util::error::{Error as GarageError, OkOrMessage};
+use garage_util::migrate::Migrate;
+
+use garage_table::replication::*;
+use garage_table::*;
+
 use garage_block::manager::BlockManager;
 use garage_block::repair::ScrubWorkerCommand;
 
@@ -14,82 +22,76 @@ use garage_model::s3::mpu_table::*;
 use garage_model::s3::object_table::*;
 use garage_model::s3::version_table::*;
 
-use garage_table::replication::*;
-use garage_table::*;
-
-use garage_util::background::*;
-use garage_util::data::*;
-use garage_util::error::Error;
-use garage_util::migrate::Migrate;
-
-use crate::*;
+use crate::api::*;
+use crate::error::Error;
+use crate::{Admin, RequestHandler};
 
 const RC_REPAIR_ITER_COUNT: usize = 64;
 
-pub async fn launch_online_repair(
-	garage: &Arc<Garage>,
-	bg: &BackgroundRunner,
-	opt: RepairOpt,
-) -> Result<(), Error> {
-	match opt.what {
-		RepairWhat::Tables => {
-			info!("Launching a full sync of tables");
-			garage.bucket_table.syncer.add_full_sync()?;
-			garage.object_table.syncer.add_full_sync()?;
-			garage.version_table.syncer.add_full_sync()?;
-			garage.block_ref_table.syncer.add_full_sync()?;
-			garage.key_table.syncer.add_full_sync()?;
-		}
-		RepairWhat::Versions => {
-			info!("Repairing the versions table");
-			bg.spawn_worker(TableRepairWorker::new(garage.clone(), RepairVersions));
-		}
-		RepairWhat::MultipartUploads => {
-			info!("Repairing the multipart uploads table");
-			bg.spawn_worker(TableRepairWorker::new(garage.clone(), RepairMpu));
-		}
-		RepairWhat::BlockRefs => {
-			info!("Repairing the block refs table");
-			bg.spawn_worker(TableRepairWorker::new(garage.clone(), RepairBlockRefs));
-		}
-		RepairWhat::BlockRc => {
-			info!("Repairing the block reference counters");
-			bg.spawn_worker(BlockRcRepair::new(
-				garage.block_manager.clone(),
-				garage.block_ref_table.clone(),
-			));
-		}
-		RepairWhat::Blocks => {
-			info!("Repairing the stored blocks");
-			bg.spawn_worker(garage_block::repair::RepairWorker::new(
-				garage.block_manager.clone(),
-			));
-		}
-		RepairWhat::Scrub { cmd } => {
-			let cmd = match cmd {
-				ScrubCmd::Start => ScrubWorkerCommand::Start,
-				ScrubCmd::Pause => ScrubWorkerCommand::Pause(Duration::from_secs(3600 * 24)),
-				ScrubCmd::Resume => ScrubWorkerCommand::Resume,
-				ScrubCmd::Cancel => ScrubWorkerCommand::Cancel,
-				ScrubCmd::SetTranquility { tranquility } => {
-					garage
-						.block_manager
-						.scrub_persister
-						.set_with(|x| x.tranquility = tranquility)?;
-					return Ok(());
-				}
-			};
-			info!("Sending command to scrub worker: {:?}", cmd);
-			garage.block_manager.send_scrub_command(cmd).await?;
-		}
-		RepairWhat::Rebalance => {
-			info!("Rebalancing the stored blocks among storage locations");
-			bg.spawn_worker(garage_block::repair::RebalanceWorker::new(
-				garage.block_manager.clone(),
-			));
+impl RequestHandler for LocalLaunchRepairOperationRequest {
+	type Response = LocalLaunchRepairOperationResponse;
+
+	async fn handle(
+		self,
+		garage: &Arc<Garage>,
+		admin: &Admin,
+	) -> Result<LocalLaunchRepairOperationResponse, Error> {
+		let bg = &admin.background;
+		match self.repair_type {
+			RepairType::Tables => {
+				info!("Launching a full sync of tables");
+				garage.bucket_table.syncer.add_full_sync()?;
+				garage.object_table.syncer.add_full_sync()?;
+				garage.version_table.syncer.add_full_sync()?;
+				garage.block_ref_table.syncer.add_full_sync()?;
+				garage.key_table.syncer.add_full_sync()?;
+			}
+			RepairType::Versions => {
+				info!("Repairing the versions table");
+				bg.spawn_worker(TableRepairWorker::new(garage.clone(), RepairVersions));
+			}
+			RepairType::MultipartUploads => {
+				info!("Repairing the multipart uploads table");
+				bg.spawn_worker(TableRepairWorker::new(garage.clone(), RepairMpu));
+			}
+			RepairType::BlockRefs => {
+				info!("Repairing the block refs table");
+				bg.spawn_worker(TableRepairWorker::new(garage.clone(), RepairBlockRefs));
+			}
+			RepairType::BlockRc => {
+				info!("Repairing the block reference counters");
+				bg.spawn_worker(BlockRcRepair::new(
+					garage.block_manager.clone(),
+					garage.block_ref_table.clone(),
+				));
+			}
+			RepairType::Blocks => {
+				info!("Repairing the stored blocks");
+				bg.spawn_worker(garage_block::repair::RepairWorker::new(
+					garage.block_manager.clone(),
+				));
+			}
+			RepairType::Scrub(cmd) => {
+				let cmd = match cmd {
+					ScrubCommand::Start => ScrubWorkerCommand::Start,
+					ScrubCommand::Pause => {
+						ScrubWorkerCommand::Pause(Duration::from_secs(3600 * 24))
+					}
+					ScrubCommand::Resume => ScrubWorkerCommand::Resume,
+					ScrubCommand::Cancel => ScrubWorkerCommand::Cancel,
+				};
+				info!("Sending command to scrub worker: {:?}", cmd);
+				garage.block_manager.send_scrub_command(cmd).await?;
+			}
+			RepairType::Rebalance => {
+				info!("Rebalancing the stored blocks among storage locations");
+				bg.spawn_worker(garage_block::repair::RebalanceWorker::new(
+					garage.block_manager.clone(),
+				));
+			}
 		}
+		Ok(LocalLaunchRepairOperationResponse)
 	}
-	Ok(())
 }
 
 // ----
@@ -103,7 +105,7 @@ trait TableRepair: Send + Sync + 'static {
 		&mut self,
 		garage: &Garage,
 		entry: <<Self as TableRepair>::T as TableSchema>::E,
-	) -> impl Future<Output = Result<bool, Error>> + Send;
+	) -> impl Future<Output = Result<bool, GarageError>> + Send;
 }
 
 struct TableRepairWorker<T: TableRepair> {
@@ -139,7 +141,10 @@ impl<R: TableRepair> Worker for TableRepairWorker<R> {
 		}
 	}
 
-	async fn work(&mut self, _must_exit: &mut watch::Receiver<bool>) -> Result<WorkerState, Error> {
+	async fn work(
+		&mut self,
+		_must_exit: &mut watch::Receiver<bool>,
+	) -> Result<WorkerState, GarageError> {
 		let (item_bytes, next_pos) = match R::table(&self.garage).data.store.get_gt(&self.pos)? {
 			Some((k, v)) => (v, k),
 			None => {
@@ -181,7 +186,7 @@ impl TableRepair for RepairVersions {
 		&garage.version_table
 	}
 
-	async fn process(&mut self, garage: &Garage, version: Version) -> Result<bool, Error> {
+	async fn process(&mut self, garage: &Garage, version: Version) -> Result<bool, GarageError> {
 		if !version.deleted.get() {
 			let ref_exists = match &version.backlink {
 				VersionBacklink::Object { bucket_id, key } => garage
@@ -227,7 +232,11 @@ impl TableRepair for RepairBlockRefs {
 		&garage.block_ref_table
 	}
 
-	async fn process(&mut self, garage: &Garage, mut block_ref: BlockRef) -> Result<bool, Error> {
+	async fn process(
+		&mut self,
+		garage: &Garage,
+		mut block_ref: BlockRef,
+	) -> Result<bool, GarageError> {
 		if !block_ref.deleted.get() {
 			let ref_exists = garage
 				.version_table
@@ -262,7 +271,11 @@ impl TableRepair for RepairMpu {
 		&garage.mpu_table
 	}
 
-	async fn process(&mut self, garage: &Garage, mut mpu: MultipartUpload) -> Result<bool, Error> {
+	async fn process(
+		&mut self,
+		garage: &Garage,
+		mut mpu: MultipartUpload,
+	) -> Result<bool, GarageError> {
 		if !mpu.deleted.get() {
 			let ref_exists = garage
 				.object_table
@@ -329,7 +342,10 @@ impl Worker for BlockRcRepair {
 		}
 	}
 
-	async fn work(&mut self, _must_exit: &mut watch::Receiver<bool>) -> Result<WorkerState, Error> {
+	async fn work(
+		&mut self,
+		_must_exit: &mut watch::Receiver<bool>,
+	) -> Result<WorkerState, GarageError> {
 		for _i in 0..RC_REPAIR_ITER_COUNT {
 			let next1 = self
 				.block_manager

src/api/admin/router_v1.rs

@@ -7,12 +7,6 @@ use garage_api_common::router_macros::*;
 use crate::error::*;
 use crate::router_v0;
 
-pub enum Authorization {
-	None,
-	MetricsToken,
-	AdminToken,
-}
-
 router_match! {@func
 
 /// List of all Admin API endpoints.
@@ -211,15 +205,6 @@ impl Endpoint {
 			))),
 		}
 	}
-	/// Get the kind of authorization which is required to perform the operation.
-	pub fn authorization_type(&self) -> Authorization {
-		match self {
-			Self::Health => Authorization::None,
-			Self::CheckDomain => Authorization::None,
-			Self::Metrics => Authorization::MetricsToken,
-			_ => Authorization::AdminToken,
-		}
-	}
 }
 
 generateQueryParameters! {

src/api/admin/router_v2.rs

@@ -0,0 +1,275 @@
+use std::borrow::Cow;
+
+use hyper::body::Incoming as IncomingBody;
+use hyper::{Method, Request};
+use paste::paste;
+
+use garage_api_common::helpers::*;
+use garage_api_common::router_macros::*;
+
+use crate::api::*;
+use crate::error::*;
+use crate::router_v1;
+use crate::Authorization;
+
+impl AdminApiRequest {
+	/// Determine which S3 endpoint a request is for using the request, and a bucket which was
+	/// possibly extracted from the Host header.
+	/// Returns Self plus bucket name, if endpoint is not Endpoint::ListBuckets
+	pub async fn from_request(req: Request<IncomingBody>) -> Result<Self, Error> {
+		let uri = req.uri().clone();
+		let path = uri.path();
+		let query = uri.query();
+
+		let method = req.method().clone();
+
+		let mut query = QueryParameters::from_query(query.unwrap_or_default())?;
+
+		let res = router_match!(@gen_path_parser_v2 (&method, path, "/v2/", query, req) [
+			@special OPTIONS _ => Options (),
+			@special GET "/check" => CheckDomain (query::domain),
+			@special GET "/health" => Health (),
+			@special GET "/metrics" => Metrics (),
+			// Cluster endpoints
+			GET GetClusterStatus (),
+			GET GetClusterHealth (),
+			POST ConnectClusterNodes (body),
+			// Admin token endpoints
+			GET ListAdminTokens (),
+			GET GetAdminTokenInfo (query_opt::id, query_opt::search),
+			POST CreateAdminToken (body),
+			POST UpdateAdminToken (body_field, query::id),
+			POST DeleteAdminToken (query::id),
+			// Layout endpoints
+			GET GetClusterLayout (),
+			GET GetClusterLayoutHistory (),
+			POST UpdateClusterLayout (body),
+			POST PreviewClusterLayoutChanges (),
+			POST ApplyClusterLayout (body),
+			POST RevertClusterLayout (),
+			POST ClusterLayoutSkipDeadNodes (body),
+			// API key endpoints
+			GET GetKeyInfo (query_opt::id, query_opt::search, parse_default(false)::show_secret_key),
+			POST UpdateKey (body_field, query::id),
+			POST CreateKey (body),
+			POST ImportKey (body),
+			POST DeleteKey (query::id),
+			GET ListKeys (),
+			// Bucket endpoints
+			GET GetBucketInfo (query_opt::id, query_opt::global_alias, query_opt::search),
+			GET ListBuckets (),
+			POST CreateBucket (body),
+			POST DeleteBucket (query::id),
+			POST UpdateBucket (body_field, query::id),
+			POST CleanupIncompleteUploads (body),
+			GET InspectObject (query::bucket_id, query::key),
+			// Bucket-key permissions
+			POST AllowBucketKey (body),
+			POST DenyBucketKey (body),
+			// Bucket aliases
+			POST AddBucketAlias (body),
+			POST RemoveBucketAlias (body),
+			// Node APIs
+			GET GetNodeInfo (default::body, query::node),
+			POST CreateMetadataSnapshot (default::body, query::node),
+			GET GetNodeStatistics (default::body, query::node),
+			GET GetClusterStatistics (),
+			POST LaunchRepairOperation (body_field, query::node),
+			// Worker APIs
+			POST ListWorkers (body_field, query::node),
+			POST GetWorkerInfo (body_field, query::node),
+			POST GetWorkerVariable (body_field, query::node),
+			POST SetWorkerVariable (body_field, query::node),
+			// Block APIs
+			GET ListBlockErrors (default::body, query::node),
+			POST GetBlockInfo (body_field, query::node),
+			POST RetryBlockResync (body_field, query::node),
+			POST PurgeBlocks (body_field, query::node),
+		]);
+
+		if let Some(message) = query.nonempty_message() {
+			debug!("Unused query parameter: {}", message)
+		}
+
+		Ok(res)
+	}
+
+	/// Some endpoints work exactly the same in their v2/ version as they did in their v1/ version.
+	/// For these endpoints, we can convert a v1/ call to its equivalent as if it was made using
+	/// its v2/ URL.
+	pub async fn from_v1(
+		v1_endpoint: router_v1::Endpoint,
+		req: Request<IncomingBody>,
+	) -> Result<Self, Error> {
+		use router_v1::Endpoint;
+
+		match v1_endpoint {
+			// GetClusterStatus semantics changed:
+			// info about local node is no longer returned
+			Endpoint::GetClusterHealth => {
+				Ok(AdminApiRequest::GetClusterHealth(GetClusterHealthRequest))
+			}
+			Endpoint::ConnectClusterNodes => {
+				let req = parse_json_body::<ConnectClusterNodesRequest, _, Error>(req).await?;
+				Ok(AdminApiRequest::ConnectClusterNodes(req))
+			}
+
+			// Layout
+			Endpoint::GetClusterLayout => {
+				Ok(AdminApiRequest::GetClusterLayout(GetClusterLayoutRequest))
+			}
+			// UpdateClusterLayout semantics changed
+			Endpoint::ApplyClusterLayout => {
+				let param = parse_json_body::<ApplyClusterLayoutRequest, _, Error>(req).await?;
+				Ok(AdminApiRequest::ApplyClusterLayout(param))
+			}
+			Endpoint::RevertClusterLayout => Ok(AdminApiRequest::RevertClusterLayout(
+				RevertClusterLayoutRequest,
+			)),
+
+			// Keys
+			Endpoint::ListKeys => Ok(AdminApiRequest::ListKeys(ListKeysRequest)),
+			Endpoint::GetKeyInfo {
+				id,
+				search,
+				show_secret_key,
+			} => {
+				let show_secret_key = show_secret_key.map(|x| x == "true").unwrap_or(false);
+				Ok(AdminApiRequest::GetKeyInfo(GetKeyInfoRequest {
+					id,
+					search,
+					show_secret_key,
+				}))
+			}
+			Endpoint::CreateKey => {
+				let req = parse_json_body::<CreateKeyRequest, _, Error>(req).await?;
+				Ok(AdminApiRequest::CreateKey(req))
+			}
+			Endpoint::ImportKey => {
+				let req = parse_json_body::<ImportKeyRequest, _, Error>(req).await?;
+				Ok(AdminApiRequest::ImportKey(req))
+			}
+			Endpoint::UpdateKey { id } => {
+				let body = parse_json_body::<UpdateKeyRequestBody, _, Error>(req).await?;
+				Ok(AdminApiRequest::UpdateKey(UpdateKeyRequest { id, body }))
+			}
+
+			// DeleteKey semantics changed:
+			// - in v1/ : HTTP DELETE => HTTP 204 No Content
+			// - in v2/ : HTTP POST => HTTP 200 Ok
+			// Endpoint::DeleteKey { id } => Ok(AdminApiRequest::DeleteKey(DeleteKeyRequest { id })),
+
+			// Buckets
+			Endpoint::ListBuckets => Ok(AdminApiRequest::ListBuckets(ListBucketsRequest)),
+			Endpoint::GetBucketInfo { id, global_alias } => {
+				Ok(AdminApiRequest::GetBucketInfo(GetBucketInfoRequest {
+					id,
+					global_alias,
+					search: None,
+				}))
+			}
+			Endpoint::CreateBucket => {
+				let req = parse_json_body::<CreateBucketRequest, _, Error>(req).await?;
+				Ok(AdminApiRequest::CreateBucket(req))
+			}
+
+			// DeleteBucket semantics changed::
+			// - in v1/ : HTTP DELETE => HTTP 204 No Content
+			// - in v2/ : HTTP POST => HTTP 200 Ok
+			// Endpoint::DeleteBucket { id } => {
+			// 	Ok(AdminApiRequest::DeleteBucket(DeleteBucketRequest { id }))
+			// }
+			Endpoint::UpdateBucket { id } => {
+				let body = parse_json_body::<UpdateBucketRequestBody, _, Error>(req).await?;
+				Ok(AdminApiRequest::UpdateBucket(UpdateBucketRequest {
+					id,
+					body,
+				}))
+			}
+
+			// Bucket-key permissions
+			Endpoint::BucketAllowKey => {
+				let req = parse_json_body::<BucketKeyPermChangeRequest, _, Error>(req).await?;
+				Ok(AdminApiRequest::AllowBucketKey(AllowBucketKeyRequest(req)))
+			}
+			Endpoint::BucketDenyKey => {
+				let req = parse_json_body::<BucketKeyPermChangeRequest, _, Error>(req).await?;
+				Ok(AdminApiRequest::DenyBucketKey(DenyBucketKeyRequest(req)))
+			}
+			// Bucket aliasing
+			Endpoint::GlobalAliasBucket { id, alias } => {
+				Ok(AdminApiRequest::AddBucketAlias(AddBucketAliasRequest {
+					bucket_id: id,
+					alias: BucketAliasEnum::Global {
+						global_alias: alias,
+					},
+				}))
+			}
+			Endpoint::GlobalUnaliasBucket { id, alias } => Ok(AdminApiRequest::RemoveBucketAlias(
+				RemoveBucketAliasRequest {
+					bucket_id: id,
+					alias: BucketAliasEnum::Global {
+						global_alias: alias,
+					},
+				},
+			)),
+			Endpoint::LocalAliasBucket {
+				id,
+				access_key_id,
+				alias,
+			} => Ok(AdminApiRequest::AddBucketAlias(AddBucketAliasRequest {
+				bucket_id: id,
+				alias: BucketAliasEnum::Local {
+					local_alias: alias,
+					access_key_id,
+				},
+			})),
+			Endpoint::LocalUnaliasBucket {
+				id,
+				access_key_id,
+				alias,
+			} => Ok(AdminApiRequest::RemoveBucketAlias(
+				RemoveBucketAliasRequest {
+					bucket_id: id,
+					alias: BucketAliasEnum::Local {
+						local_alias: alias,
+						access_key_id,
+					},
+				},
+			)),
+
+			// For endpoints that have different body content syntax, issue
+			// deprecation warning
+			_ => Err(Error::bad_request(format!(
+				"v1/ endpoint is no longer supported: {}",
+				v1_endpoint.name()
+			))),
+		}
+	}
+
+	/// Get the kind of authorization which is required to perform the operation.
+	pub fn authorization_type(&self) -> Authorization {
+		match self {
+			Self::Options(_) | Self::Health(_) | Self::CheckDomain(_) => Authorization::None,
+			Self::Metrics(_) => Authorization::MetricsToken,
+			_ => Authorization::AdminToken,
+		}
+	}
+}
+
+generateQueryParameters! {
+	keywords: [],
+	fields: [
+		"node" => node,
+		"domain" => domain,
+		"format" => format,
+		"id" => id,
+		"search" => search,
+		"globalAlias" => global_alias,
+		"alias" => alias,
+		"accessKeyId" => access_key_id,
+		"showSecretKey" => show_secret_key,
+		"bucketId" => bucket_id,
+		"key" => key
+	]
+}

src/api/admin/special.rs

@@ -0,0 +1,173 @@
+use std::sync::Arc;
+
+use http::header::{
+	ACCESS_CONTROL_ALLOW_HEADERS, ACCESS_CONTROL_ALLOW_METHODS, ACCESS_CONTROL_ALLOW_ORIGIN, ALLOW,
+};
+use hyper::{Response, StatusCode};
+
+#[cfg(feature = "metrics")]
+use prometheus::{Encoder, TextEncoder};
+
+use garage_model::garage::Garage;
+use garage_rpc::system::ClusterHealthStatus;
+
+use garage_api_common::helpers::*;
+
+use crate::api::{CheckDomainRequest, HealthRequest, MetricsRequest, OptionsRequest};
+use crate::api_server::ResBody;
+use crate::error::*;
+use crate::{Admin, RequestHandler};
+
+impl RequestHandler for OptionsRequest {
+	type Response = Response<ResBody>;
+
+	async fn handle(
+		self,
+		_garage: &Arc<Garage>,
+		_admin: &Admin,
+	) -> Result<Response<ResBody>, Error> {
+		Ok(Response::builder()
+			.status(StatusCode::OK)
+			.header(ALLOW, "OPTIONS,GET,POST")
+			.header(ACCESS_CONTROL_ALLOW_METHODS, "OPTIONS,GET,POST")
+			.header(ACCESS_CONTROL_ALLOW_HEADERS, "authorization,content-type")
+			.header(ACCESS_CONTROL_ALLOW_ORIGIN, "*")
+			.body(empty_body())?)
+	}
+}
+
+impl RequestHandler for MetricsRequest {
+	type Response = Response<ResBody>;
+
+	async fn handle(
+		self,
+		_garage: &Arc<Garage>,
+		admin: &Admin,
+	) -> Result<Response<ResBody>, Error> {
+		#[cfg(feature = "metrics")]
+		{
+			use opentelemetry::trace::Tracer;
+
+			let mut buffer = vec![];
+			let encoder = TextEncoder::new();
+
+			let tracer = opentelemetry::global::tracer("garage");
+			let metric_families = tracer.in_span("admin/gather_metrics", |_| {
+				admin.exporter.registry().gather()
+			});
+
+			encoder
+				.encode(&metric_families, &mut buffer)
+				.ok_or_internal_error("Could not serialize metrics")?;
+
+			Ok(Response::builder()
+				.status(StatusCode::OK)
+				.header(http::header::CONTENT_TYPE, encoder.format_type())
+				.body(bytes_body(buffer.into()))?)
+		}
+		#[cfg(not(feature = "metrics"))]
+		Err(Error::bad_request(
+			"Garage was built without the metrics feature".to_string(),
+		))
+	}
+}
+
+impl RequestHandler for HealthRequest {
+	type Response = Response<ResBody>;
+
+	async fn handle(
+		self,
+		garage: &Arc<Garage>,
+		_admin: &Admin,
+	) -> Result<Response<ResBody>, Error> {
+		let health = garage.system.health();
+
+		let (status, status_str) = match health.status {
+			ClusterHealthStatus::Healthy => (StatusCode::OK, "Garage is fully operational"),
+			ClusterHealthStatus::Degraded => (
+				StatusCode::OK,
+				"Garage is operational but some storage nodes are unavailable",
+			),
+			ClusterHealthStatus::Unavailable => (
+				StatusCode::SERVICE_UNAVAILABLE,
+				"Quorum is not available for some/all partitions, reads and writes will fail",
+			),
+		};
+		let status_str = format!(
+			"{}\nConsult the full health check API endpoint at /v2/GetClusterHealth for more details\n",
+			status_str
+		);
+
+		Ok(Response::builder()
+			.status(status)
+			.header(http::header::CONTENT_TYPE, "text/plain")
+			.body(string_body(status_str))?)
+	}
+}
+
+impl RequestHandler for CheckDomainRequest {
+	type Response = Response<ResBody>;
+
+	async fn handle(
+		self,
+		garage: &Arc<Garage>,
+		_admin: &Admin,
+	) -> Result<Response<ResBody>, Error> {
+		if check_domain(garage, &self.domain).await? {
+			Ok(Response::builder()
+				.status(StatusCode::OK)
+				.body(string_body(format!(
+					"Domain '{}' is managed by Garage",
+					self.domain
+				)))?)
+		} else {
+			Err(Error::bad_request(format!(
+				"Domain '{}' is not managed by Garage",
+				self.domain
+			)))
+		}
+	}
+}
+
+async fn check_domain(garage: &Arc<Garage>, domain: &str) -> Result<bool, Error> {
+	// Resolve bucket from domain name, inferring if the website must be activated for the
+	// domain to be valid.
+	let (bucket_name, must_check_website) = if let Some(bname) = garage
+		.config
+		.s3_api
+		.root_domain
+		.as_ref()
+		.and_then(|rd| host_to_bucket(domain, rd))
+	{
+		(bname.to_string(), false)
+	} else if let Some(bname) = garage
+		.config
+		.s3_web
+		.as_ref()
+		.and_then(|sw| host_to_bucket(domain, sw.root_domain.as_str()))
+	{
+		(bname.to_string(), true)
+	} else {
+		(domain.to_string(), true)
+	};
+
+	let bucket = match garage
+		.bucket_helper()
+		.resolve_global_bucket_fast(&bucket_name)?
+	{
+		Some(b) => b,
+		None => return Ok(false),
+	};
+
+	if !must_check_website {
+		return Ok(true);
+	}
+
+	let bucket_state = bucket.state.as_option().unwrap();
+	let bucket_website_config = bucket_state.website_config.get();
+
+	match bucket_website_config {
+		Some(_v) => Ok(true),
+		None => Ok(false),
+	}
+}

src/api/admin/worker.rs

@@ -0,0 +1,118 @@
+use std::collections::HashMap;
+use std::sync::Arc;
+
+use garage_util::background::*;
+use garage_util::time::now_msec;
+
+use garage_model::garage::Garage;
+
+use crate::api::*;
+use crate::error::Error;
+use crate::{Admin, RequestHandler};
+
+impl RequestHandler for LocalListWorkersRequest {
+	type Response = LocalListWorkersResponse;
+
+	async fn handle(
+		self,
+		_garage: &Arc<Garage>,
+		admin: &Admin,
+	) -> Result<LocalListWorkersResponse, Error> {
+		let workers = admin.background.get_worker_info();
+		let info = workers
+			.into_iter()
+			.filter(|(_, w)| {
+				(!self.busy_only
+					|| matches!(w.state, WorkerState::Busy | WorkerState::Throttled(_)))
+					&& (!self.error_only || w.errors > 0)
+			})
+			.map(|(id, w)| worker_info_to_api(id as u64, w))
+			.collect::<Vec<_>>();
+		Ok(LocalListWorkersResponse(info))
+	}
+}
+
+impl RequestHandler for LocalGetWorkerInfoRequest {
+	type Response = LocalGetWorkerInfoResponse;
+
+	async fn handle(
+		self,
+		_garage: &Arc<Garage>,
+		admin: &Admin,
+	) -> Result<LocalGetWorkerInfoResponse, Error> {
+		let info = admin
+			.background
+			.get_worker_info()
+			.get(&(self.id as usize))
+			.ok_or(Error::NoSuchWorker(self.id))?
+			.clone();
+		Ok(LocalGetWorkerInfoResponse(worker_info_to_api(
+			self.id, info,
+		)))
+	}
+}
+
+impl RequestHandler for LocalGetWorkerVariableRequest {
+	type Response = LocalGetWorkerVariableResponse;
+
+	async fn handle(
+		self,
+		garage: &Arc<Garage>,
+		_admin: &Admin,
+	) -> Result<LocalGetWorkerVariableResponse, Error> {
+		let mut res = HashMap::new();
+		if let Some(k) = self.variable {
+			res.insert(k.clone(), garage.bg_vars.get(&k)?);
+		} else {
+			let vars = garage.bg_vars.get_all();
+			for (k, v) in vars.iter() {
+				res.insert(k.to_string(), v.to_string());
+			}
+		}
+		Ok(LocalGetWorkerVariableResponse(res))
+	}
+}
+
+impl RequestHandler for LocalSetWorkerVariableRequest {
+	type Response = LocalSetWorkerVariableResponse;
+
+	async fn handle(
+		self,
+		garage: &Arc<Garage>,
+		_admin: &Admin,
+	) -> Result<LocalSetWorkerVariableResponse, Error> {
+		garage.bg_vars.set(&self.variable, &self.value)?;
+
+		Ok(LocalSetWorkerVariableResponse {
+			variable: self.variable,
+			value: self.value,
+		})
+	}
+}
+
+// ---- helper functions ----
+
+fn worker_info_to_api(id: u64, info: WorkerInfo) -> WorkerInfoResp {
+	WorkerInfoResp {
+		id,
+		name: info.name,
+		state: match info.state {
+			WorkerState::Busy => WorkerStateResp::Busy,
+			WorkerState::Throttled(t) => WorkerStateResp::Throttled { duration_secs: t },
+			WorkerState::Idle => WorkerStateResp::Idle,
+			WorkerState::Done => WorkerStateResp::Done,
+		},
+		errors: info.errors as u64,
+		consecutive_errors: info.consecutive_errors as u64,
+		last_error: info.last_error.map(|(message, t)| WorkerLastError {
+			message,
+			secs_ago: now_msec().saturating_sub(t) / 1000,
+		}),
+
+		tranquility: info.status.tranquility,
+		progress: info.status.progress,
+		queue_length: info.status.queue_length,
+		persistent_errors: info.status.persistent_errors,
+		freeform: info.status.freeform,
+	}
+}

src/api/common/Cargo.toml

@@ -1,12 +1,12 @@
 [package]
 name = "garage_api_common"
-version = "1.0.1"
+version = "2.0.0"
 authors = ["Alex Auvolat <alex@adnab.me>"]
 edition = "2018"
 license = "AGPL-3.0"
 description = "Common functions for the API server crates for the Garage object store"
 repository = "https://git.deuxfleurs.fr/Deuxfleurs/garage"
-readme = "../../README.md"
+readme = "../../../README.md"
 
 [lib]
 path = "lib.rs"
@@ -18,16 +18,22 @@ garage_model.workspace = true
 garage_table.workspace = true
 garage_util.workspace = true
 
+base64.workspace = true
 bytes.workspace = true
 chrono.workspace = true
+crc32fast.workspace = true
+crc32c.workspace = true
+crc64fast-nvme.workspace = true
 crypto-common.workspace = true
 err-derive.workspace = true
 hex.workspace = true
 hmac.workspace = true
+md-5.workspace = true
 idna.workspace = true
 tracing.workspace = true
 nom.workspace = true
 pin-project.workspace = true
+sha1.workspace = true
 sha2.workspace = true
 
 futures.workspace = true

src/api/common/cors.rs

@@ -9,14 +9,12 @@ use hyper::{body::Body, body::Incoming as IncomingBody, Request, Response, Statu
 use garage_model::bucket_table::{BucketParams, CorsRule as GarageCorsRule};
 use garage_model::garage::Garage;
 
-use crate::common_error::{
-	helper_error_as_internal, CommonError, OkOrBadRequest, OkOrInternalError,
-};
+use crate::common_error::{CommonError, OkOrBadRequest, OkOrInternalError};
 use crate::helpers::*;
 
-pub fn find_matching_cors_rule<'a>(
+pub fn find_matching_cors_rule<'a, B>(
 	bucket_params: &'a BucketParams,
-	req: &Request<impl Body>,
+	req: &Request<B>,
 ) -> Result<Option<&'a GarageCorsRule>, CommonError> {
 	if let Some(cors_config) = bucket_params.cors_config.get() {
 		if let Some(origin) = req.headers().get("Origin") {
@@ -76,7 +74,7 @@ pub fn add_cors_headers(
 	Ok(())
 }
 
-pub async fn handle_options_api(
+pub fn handle_options_api(
 	garage: Arc<Garage>,
 	req: &Request<IncomingBody>,
 	bucket_name: Option<String>,
@@ -93,16 +91,8 @@ pub async fn handle_options_api(
 	// OPTIONS calls are not auhtenticated).
 	if let Some(bn) = bucket_name {
 		let helper = garage.bucket_helper();
-		let bucket_id = helper
-			.resolve_global_bucket_name(&bn)
-			.await
-			.map_err(helper_error_as_internal)?;
-		if let Some(id) = bucket_id {
-			let bucket = garage
-				.bucket_helper()
-				.get_existing_bucket(id)
-				.await
-				.map_err(helper_error_as_internal)?;
+		let bucket_opt = helper.resolve_global_bucket_fast(&bn)?;
+		if let Some(bucket) = bucket_opt {
 			let bucket_params = bucket.state.into_option().unwrap();
 			handle_options_for_bucket(req, &bucket_params)
 		} else {
@@ -132,8 +122,8 @@ pub async fn handle_options_api(
 	}
 }
 
-pub fn handle_options_for_bucket(
-	req: &Request<IncomingBody>,
+pub fn handle_options_for_bucket<B>(
+	req: &Request<B>,
 	bucket_params: &BucketParams,
 ) -> Result<Response<EmptyBody>, CommonError> {
 	let origin = req

src/api/common/generic_server.rs

@@ -1,3 +1,4 @@
+use std::borrow::Cow;
 use std::convert::Infallible;
 use std::fs::{self, Permissions};
 use std::os::unix::fs::PermissionsExt;
@@ -35,7 +36,7 @@ use garage_util::socket_address::UnixOrTCPSocketAddress;
 use crate::helpers::{BoxBody, ErrorBody};
 
 pub trait ApiEndpoint: Send + Sync + 'static {
-	fn name(&self) -> &'static str;
+	fn name(&self) -> Cow<'static, str>;
 	fn add_span_attributes(&self, span: SpanRef<'_>);
 }
 

src/api/common/router_macros.rs

@@ -45,6 +45,68 @@ macro_rules! router_match {
             }
         }
     }};
+    (@gen_path_parser_v2 ($method:expr, $reqpath:expr, $pathprefix:literal, $query:expr, $req:expr)
+     [
+     $(@special $spec_meth:ident $spec_path:pat =>  $spec_api:ident $spec_params:tt,)*
+     $($meth:ident $api:ident $params:tt,)*
+     ]) => {{
+        {
+            #[allow(unused_parens)]
+            match ($method, $reqpath) {
+                $(
+                    (&Method::$spec_meth, $spec_path) => AdminApiRequest::$spec_api (
+                        router_match!(@@gen_parse_request $spec_api, $spec_params, $query, $req)
+                    ),
+                )*
+                $(
+                    (&Method::$meth, concat!($pathprefix, stringify!($api)))
+                    => AdminApiRequest::$api (
+                        router_match!(@@gen_parse_request $api, $params, $query, $req)
+                        ),
+                )*
+                (m, p) => {
+                    return Err(Error::bad_request(format!(
+                        "Unknown API endpoint: {} {}",
+                        m, p
+                    )))
+                }
+            }
+        }
+    }};
+    (@@gen_parse_request $api:ident, (), $query: expr, $req:expr) => {{
+        paste!(
+            [< $api Request >]
+        )
+    }};
+    (@@gen_parse_request $api:ident, (body), $query: expr, $req:expr) => {{
+        paste!({
+            parse_json_body::< [<$api Request>], _, Error>($req).await?
+        })
+    }};
+    (@@gen_parse_request $api:ident, (body_field, $($conv:ident $(($conv_arg:expr))? :: $param:ident),*), $query: expr, $req:expr)
+        =>
+    {{
+        paste!({
+            let body = parse_json_body::< [<$api RequestBody>], _, Error>($req).await?;
+            [< $api Request >] {
+                body,
+                $(
+                    $param: router_match!(@@parse_param $query, $conv $(($conv_arg))?, $param),
+                )+
+            }
+        })
+    }};
+    (@@gen_parse_request $api:ident, ($($conv:ident $(($conv_arg:expr))? :: $param:ident),*), $query: expr, $req:expr)
+        =>
+    {{
+        paste!({
+            [< $api Request >] {
+                $(
+                    $param: router_match!(@@parse_param $query, $conv $(($conv_arg))?, $param),
+                )+
+            }
+        })
+    }};
     (@gen_parser ($keyword:expr, $key:ident, $query:expr, $header:expr),
         key: [$($kw_k:ident $(if $required_k:ident)? $(header $header_k:expr)? => $api_k:ident $(($($conv_k:ident :: $param_k:ident),*))?,)*],
         no_key: [$($kw_nk:ident $(if $required_nk:ident)? $(if_header $header_nk:expr)? => $api_nk:ident $(($($conv_nk:ident :: $param_nk:ident),*))?,)*]) => {{
@@ -79,13 +141,19 @@ macro_rules! router_match {
         }
     }};
 
+    (@@parse_param $query:expr, default, $param:ident) => {{
+        Default::default()
+    }};
     (@@parse_param $query:expr, query_opt, $param:ident) => {{
         // extract optional query parameter
         $query.$param.take().map(|param| param.into_owned())
     }};
     (@@parse_param $query:expr, query, $param:ident) => {{
         // extract mendatory query parameter
-        $query.$param.take().ok_or_bad_request("Missing argument for endpoint")?.into_owned()
+        $query.$param.take()
+            .ok_or_bad_request(
+                format!("Missing argument `{}` for endpoint", stringify!($param))
+            )?.into_owned()
     }};
     (@@parse_param $query:expr, opt_parse, $param:ident) => {{
         // extract and parse optional query parameter
@@ -99,10 +167,22 @@ macro_rules! router_match {
     (@@parse_param $query:expr, parse, $param:ident) => {{
         // extract and parse mandatory query parameter
         // both missing and un-parseable parameters are reported as errors
-        $query.$param.take().ok_or_bad_request("Missing argument for endpoint")?
+        $query.$param.take()
+            .ok_or_bad_request(
+                format!("Missing argument `{}` for endpoint", stringify!($param))
+            )?
             .parse()
             .map_err(|_| Error::bad_request("Failed to parse query parameter"))?
     }};
+    (@@parse_param $query:expr, parse_default($default:expr), $param:ident) => {{
+        // extract and parse optional query parameter
+        // using provided value as default if paramter is missing
+        $query.$param.take().map(|x| x
+            .parse()
+            .map_err(|_| Error::bad_request("Failed to parse query parameter")))
+            .transpose()?
+            .unwrap_or($default)
+    }};
     (@func
     $(#[$doc:meta])*
      pub enum Endpoint {
@@ -187,6 +267,7 @@ macro_rules! generateQueryParameters {
                             },
                         )*
                         $(
+                            // FIXME: remove if !v.is_empty() ?
                             $f_param => if !v.is_empty() {
                                 if res.$f_name.replace(v).is_some() {
                                     return Err(Error::bad_request(format!(

src/api/common/signature/body.rs

@@ -0,0 +1,135 @@
+use std::sync::Mutex;
+
+use futures::prelude::*;
+use futures::stream::BoxStream;
+use http_body_util::{BodyExt, StreamBody};
+use hyper::body::{Bytes, Frame};
+use serde::Deserialize;
+use tokio::sync::mpsc;
+use tokio::task;
+
+use super::*;
+
+use crate::signature::checksum::*;
+
+pub struct ReqBody {
+	// why need mutex to be sync??
+	pub(crate) stream: Mutex<BoxStream<'static, Result<Frame<Bytes>, Error>>>,
+	pub(crate) checksummer: Checksummer,
+	pub(crate) expected_checksums: ExpectedChecksums,
+	pub(crate) trailer_algorithm: Option<ChecksumAlgorithm>,
+}
+
+pub type StreamingChecksumReceiver = task::JoinHandle<Result<Checksums, Error>>;
+
+impl ReqBody {
+	pub fn add_expected_checksums(&mut self, more: ExpectedChecksums) {
+		if more.md5.is_some() {
+			self.expected_checksums.md5 = more.md5;
+		}
+		if more.sha256.is_some() {
+			self.expected_checksums.sha256 = more.sha256;
+		}
+		if more.extra.is_some() {
+			self.expected_checksums.extra = more.extra;
+		}
+		self.checksummer.add_expected(&self.expected_checksums);
+	}
+
+	pub fn add_md5(&mut self) {
+		self.checksummer.add_md5();
+	}
+
+	// ============ non-streaming =============
+
+	pub async fn json<T: for<'a> Deserialize<'a>>(self) -> Result<T, Error> {
+		let body = self.collect().await?;
+		let resp: T = serde_json::from_slice(&body).ok_or_bad_request("Invalid JSON")?;
+		Ok(resp)
+	}
+
+	pub async fn collect(self) -> Result<Bytes, Error> {
+		self.collect_with_checksums().await.map(|(b, _)| b)
+	}
+
+	pub async fn collect_with_checksums(mut self) -> Result<(Bytes, Checksums), Error> {
+		let stream: BoxStream<_> = self.stream.into_inner().unwrap();
+		let bytes = BodyExt::collect(StreamBody::new(stream)).await?.to_bytes();
+
+		self.checksummer.update(&bytes);
+		let checksums = self.checksummer.finalize();
+		checksums.verify(&self.expected_checksums)?;
+
+		Ok((bytes, checksums))
+	}
+
+	// ============ streaming =============
+
+	pub fn streaming_with_checksums(
+		self,
+	) -> (
+		BoxStream<'static, Result<Bytes, Error>>,
+		StreamingChecksumReceiver,
+	) {
+		let Self {
+			stream,
+			mut checksummer,
+			mut expected_checksums,
+			trailer_algorithm,
+		} = self;
+
+		let (frame_tx, mut frame_rx) = mpsc::channel::<Frame<Bytes>>(5);
+
+		let join_checksums = tokio::spawn(async move {
+			while let Some(frame) = frame_rx.recv().await {
+				match frame.into_data() {
+					Ok(data) => {
+						checksummer = tokio::task::spawn_blocking(move || {
+							checksummer.update(&data);
+							checksummer
+						})
+						.await
+						.unwrap()
+					}
+					Err(frame) => {
+						let trailers = frame.into_trailers().unwrap();
+						let algo = trailer_algorithm.unwrap();
+						expected_checksums.extra = Some(extract_checksum_value(&trailers, algo)?);
+						break;
+					}
+				}
+			}
+
+			if trailer_algorithm.is_some() && expected_checksums.extra.is_none() {
+				return Err(Error::bad_request("trailing checksum was not sent"));
+			}
+
+			let checksums = checksummer.finalize();
+			checksums.verify(&expected_checksums)?;
+
+			Ok(checksums)
+		});
+
+		let stream: BoxStream<_> = stream.into_inner().unwrap();
+		let stream = stream.filter_map(move |x| {
+			let frame_tx = frame_tx.clone();
+			async move {
+				match x {
+					Err(e) => Some(Err(e)),
+					Ok(frame) => {
+						if frame.is_data() {
+							let data = frame.data_ref().unwrap().clone();
+							let _ = frame_tx.send(frame).await;
+							Some(Ok(data))
+						} else {
+							let _ = frame_tx.send(frame).await;
+							None
+						}
+					}
+				}
+			}
+		});
+
+		(stream.boxed(), join_checksums)
+	}
+}

src/api/common/signature/checksum.rs

@@ -4,6 +4,7 @@ use std::hash::Hasher;
 use base64::prelude::*;
 use crc32c::Crc32cHasher as Crc32c;
 use crc32fast::Hasher as Crc32;
+use crc64fast_nvme::Digest as Crc64Nvme;
 use md5::{Digest, Md5};
 use sha1::Sha1;
 use sha2::Sha256;
@@ -11,28 +12,32 @@ use sha2::Sha256;
 use http::{HeaderMap, HeaderName, HeaderValue};
 
 use garage_util::data::*;
-use garage_util::error::OkOrMessage;
 
-use garage_model::s3::object_table::*;
+use super::*;
 
-use crate::error::*;
+pub use garage_model::s3::object_table::{ChecksumAlgorithm, ChecksumValue};
+
+pub const CONTENT_MD5: HeaderName = HeaderName::from_static("content-md5");
 
 pub const X_AMZ_CHECKSUM_ALGORITHM: HeaderName =
 	HeaderName::from_static("x-amz-checksum-algorithm");
 pub const X_AMZ_CHECKSUM_MODE: HeaderName = HeaderName::from_static("x-amz-checksum-mode");
 pub const X_AMZ_CHECKSUM_CRC32: HeaderName = HeaderName::from_static("x-amz-checksum-crc32");
 pub const X_AMZ_CHECKSUM_CRC32C: HeaderName = HeaderName::from_static("x-amz-checksum-crc32c");
+pub const X_AMZ_CHECKSUM_CRC64NVME: HeaderName =
+	HeaderName::from_static("x-amz-checksum-crc64nvme");
 pub const X_AMZ_CHECKSUM_SHA1: HeaderName = HeaderName::from_static("x-amz-checksum-sha1");
 pub const X_AMZ_CHECKSUM_SHA256: HeaderName = HeaderName::from_static("x-amz-checksum-sha256");
 
 pub type Crc32Checksum = [u8; 4];
 pub type Crc32cChecksum = [u8; 4];
+pub type Crc64NvmeChecksum = [u8; 8];
 pub type Md5Checksum = [u8; 16];
 pub type Sha1Checksum = [u8; 20];
 pub type Sha256Checksum = [u8; 32];
 
-#[derive(Debug, Default)]
-pub(crate) struct ExpectedChecksums {
+#[derive(Debug, Default, Clone)]
+pub struct ExpectedChecksums {
 	// base64-encoded md5 (content-md5 header)
 	pub md5: Option<String>,
 	// content_sha256 (as a Hash / FixedBytes32)
@@ -41,52 +46,72 @@ pub(crate) struct ExpectedChecksums {
 	pub extra: Option<ChecksumValue>,
 }
 
-pub(crate) struct Checksummer {
+pub struct Checksummer {
 	pub crc32: Option<Crc32>,
 	pub crc32c: Option<Crc32c>,
+	pub crc64nvme: Option<Crc64Nvme>,
 	pub md5: Option<Md5>,
 	pub sha1: Option<Sha1>,
 	pub sha256: Option<Sha256>,
 }
 
 #[derive(Default)]
-pub(crate) struct Checksums {
+pub struct Checksums {
 	pub crc32: Option<Crc32Checksum>,
 	pub crc32c: Option<Crc32cChecksum>,
+	pub crc64nvme: Option<Crc64NvmeChecksum>,
 	pub md5: Option<Md5Checksum>,
 	pub sha1: Option<Sha1Checksum>,
 	pub sha256: Option<Sha256Checksum>,
 }
 
 impl Checksummer {
-	pub(crate) fn init(expected: &ExpectedChecksums, require_md5: bool) -> Self {
-		let mut ret = Self {
+	pub fn new() -> Self {
+		Self {
 			crc32: None,
 			crc32c: None,
+			crc64nvme: None,
 			md5: None,
 			sha1: None,
 			sha256: None,
-		};
+		}
+	}
 
-		if expected.md5.is_some() || require_md5 {
-			ret.md5 = Some(Md5::new());
-		}
-		if expected.sha256.is_some() || matches!(&expected.extra, Some(ChecksumValue::Sha256(_))) {
-			ret.sha256 = Some(Sha256::new());
-		}
-		if matches!(&expected.extra, Some(ChecksumValue::Crc32(_))) {
-			ret.crc32 = Some(Crc32::new());
-		}
-		if matches!(&expected.extra, Some(ChecksumValue::Crc32c(_))) {
-			ret.crc32c = Some(Crc32c::default());
-		}
-		if matches!(&expected.extra, Some(ChecksumValue::Sha1(_))) {
-			ret.sha1 = Some(Sha1::new());
+	pub fn init(expected: &ExpectedChecksums, add_md5: bool) -> Self {
+		let mut ret = Self::new();
+		ret.add_expected(expected);
+		if add_md5 {
+			ret.add_md5();
 		}
 		ret
 	}
 
-	pub(crate) fn add(mut self, algo: Option<ChecksumAlgorithm>) -> Self {
+	pub fn add_md5(&mut self) {
+		self.md5 = Some(Md5::new());
+	}
+
+	pub fn add_expected(&mut self, expected: &ExpectedChecksums) {
+		if expected.md5.is_some() {
+			self.md5 = Some(Md5::new());
+		}
+		if expected.sha256.is_some() || matches!(&expected.extra, Some(ChecksumValue::Sha256(_))) {
+			self.sha256 = Some(Sha256::new());
+		}
+		if matches!(&expected.extra, Some(ChecksumValue::Crc32(_))) {
+			self.crc32 = Some(Crc32::new());
+		}
+		if matches!(&expected.extra, Some(ChecksumValue::Crc32c(_))) {
+			self.crc32c = Some(Crc32c::default());
+		}
+		if matches!(&expected.extra, Some(ChecksumValue::Crc64Nvme(_))) {
+			self.crc64nvme = Some(Crc64Nvme::default());
+		}
+		if matches!(&expected.extra, Some(ChecksumValue::Sha1(_))) {
+			self.sha1 = Some(Sha1::new());
+		}
+	}
+
+	pub fn add(mut self, algo: Option<ChecksumAlgorithm>) -> Self {
 		match algo {
 			Some(ChecksumAlgorithm::Crc32) => {
 				self.crc32 = Some(Crc32::new());
@@ -94,6 +119,9 @@ impl Checksummer {
 			Some(ChecksumAlgorithm::Crc32c) => {
 				self.crc32c = Some(Crc32c::default());
 			}
+			Some(ChecksumAlgorithm::Crc64Nvme) => {
+				self.crc64nvme = Some(Crc64Nvme::default());
+			}
 			Some(ChecksumAlgorithm::Sha1) => {
 				self.sha1 = Some(Sha1::new());
 			}
@@ -105,13 +133,16 @@ impl Checksummer {
 		self
 	}
 
-	pub(crate) fn update(&mut self, bytes: &[u8]) {
+	pub fn update(&mut self, bytes: &[u8]) {
 		if let Some(crc32) = &mut self.crc32 {
 			crc32.update(bytes);
 		}
 		if let Some(crc32c) = &mut self.crc32c {
 			crc32c.write(bytes);
 		}
+		if let Some(crc64nvme) = &mut self.crc64nvme {
+			crc64nvme.write(bytes);
+		}
 		if let Some(md5) = &mut self.md5 {
 			md5.update(bytes);
 		}
@@ -123,12 +154,13 @@ impl Checksummer {
 		}
 	}
 
-	pub(crate) fn finalize(self) -> Checksums {
+	pub fn finalize(self) -> Checksums {
 		Checksums {
 			crc32: self.crc32.map(|x| u32::to_be_bytes(x.finalize())),
 			crc32c: self
 				.crc32c
 				.map(|x| u32::to_be_bytes(u32::try_from(x.finish()).unwrap())),
+			crc64nvme: self.crc64nvme.map(|x| u64::to_be_bytes(x.sum64())),
 			md5: self.md5.map(|x| x.finalize()[..].try_into().unwrap()),
 			sha1: self.sha1.map(|x| x.finalize()[..].try_into().unwrap()),
 			sha256: self.sha256.map(|x| x.finalize()[..].try_into().unwrap()),
@@ -175,6 +207,9 @@ impl Checksums {
 			None => None,
 			Some(ChecksumAlgorithm::Crc32) => Some(ChecksumValue::Crc32(self.crc32.unwrap())),
 			Some(ChecksumAlgorithm::Crc32c) => Some(ChecksumValue::Crc32c(self.crc32c.unwrap())),
+			Some(ChecksumAlgorithm::Crc64Nvme) => {
+				Some(ChecksumValue::Crc64Nvme(self.crc64nvme.unwrap()))
+			}
 			Some(ChecksumAlgorithm::Sha1) => Some(ChecksumValue::Sha1(self.sha1.unwrap())),
 			Some(ChecksumAlgorithm::Sha256) => Some(ChecksumValue::Sha256(self.sha256.unwrap())),
 		}
@@ -183,153 +218,64 @@ impl Checksums {
 
 // ----
 
-#[derive(Default)]
-pub(crate) struct MultipartChecksummer {
-	pub md5: Md5,
-	pub extra: Option<MultipartExtraChecksummer>,
-}
-
-pub(crate) enum MultipartExtraChecksummer {
-	Crc32(Crc32),
-	Crc32c(Crc32c),
-	Sha1(Sha1),
-	Sha256(Sha256),
-}
-
-impl MultipartChecksummer {
-	pub(crate) fn init(algo: Option<ChecksumAlgorithm>) -> Self {
-		Self {
-			md5: Md5::new(),
-			extra: match algo {
-				None => None,
-				Some(ChecksumAlgorithm::Crc32) => {
-					Some(MultipartExtraChecksummer::Crc32(Crc32::new()))
-				}
-				Some(ChecksumAlgorithm::Crc32c) => {
-					Some(MultipartExtraChecksummer::Crc32c(Crc32c::default()))
-				}
-				Some(ChecksumAlgorithm::Sha1) => Some(MultipartExtraChecksummer::Sha1(Sha1::new())),
-				Some(ChecksumAlgorithm::Sha256) => {
-					Some(MultipartExtraChecksummer::Sha256(Sha256::new()))
-				}
-			},
-		}
-	}
-
-	pub(crate) fn update(
-		&mut self,
-		etag: &str,
-		checksum: Option<ChecksumValue>,
-	) -> Result<(), Error> {
-		self.md5
-			.update(&hex::decode(&etag).ok_or_message("invalid etag hex")?);
-		match (&mut self.extra, checksum) {
-			(None, _) => (),
-			(
-				Some(MultipartExtraChecksummer::Crc32(ref mut crc32)),
-				Some(ChecksumValue::Crc32(x)),
-			) => {
-				crc32.update(&x);
-			}
-			(
-				Some(MultipartExtraChecksummer::Crc32c(ref mut crc32c)),
-				Some(ChecksumValue::Crc32c(x)),
-			) => {
-				crc32c.write(&x);
-			}
-			(Some(MultipartExtraChecksummer::Sha1(ref mut sha1)), Some(ChecksumValue::Sha1(x))) => {
-				sha1.update(&x);
-			}
-			(
-				Some(MultipartExtraChecksummer::Sha256(ref mut sha256)),
-				Some(ChecksumValue::Sha256(x)),
-			) => {
-				sha256.update(&x);
-			}
-			(Some(_), b) => {
-				return Err(Error::internal_error(format!(
-					"part checksum was not computed correctly, got: {:?}",
-					b
-				)))
-			}
-		}
-		Ok(())
-	}
-
-	pub(crate) fn finalize(self) -> (Md5Checksum, Option<ChecksumValue>) {
-		let md5 = self.md5.finalize()[..].try_into().unwrap();
-		let extra = match self.extra {
-			None => None,
-			Some(MultipartExtraChecksummer::Crc32(crc32)) => {
-				Some(ChecksumValue::Crc32(u32::to_be_bytes(crc32.finalize())))
-			}
-			Some(MultipartExtraChecksummer::Crc32c(crc32c)) => Some(ChecksumValue::Crc32c(
-				u32::to_be_bytes(u32::try_from(crc32c.finish()).unwrap()),
-			)),
-			Some(MultipartExtraChecksummer::Sha1(sha1)) => {
-				Some(ChecksumValue::Sha1(sha1.finalize()[..].try_into().unwrap()))
-			}
-			Some(MultipartExtraChecksummer::Sha256(sha256)) => Some(ChecksumValue::Sha256(
-				sha256.finalize()[..].try_into().unwrap(),
-			)),
-		};
-		(md5, extra)
+pub fn parse_checksum_algorithm(algo: &str) -> Result<ChecksumAlgorithm, Error> {
+	match algo {
+		"CRC32" => Ok(ChecksumAlgorithm::Crc32),
+		"CRC32C" => Ok(ChecksumAlgorithm::Crc32c),
+		"CRC64NVME" => Ok(ChecksumAlgorithm::Crc64Nvme),
+		"SHA1" => Ok(ChecksumAlgorithm::Sha1),
+		"SHA256" => Ok(ChecksumAlgorithm::Sha256),
+		_ => Err(Error::bad_request("invalid checksum algorithm")),
 	}
 }
 
-// ----
-
 /// Extract the value of the x-amz-checksum-algorithm header
-pub(crate) fn request_checksum_algorithm(
+pub fn request_checksum_algorithm(
 	headers: &HeaderMap<HeaderValue>,
 ) -> Result<Option<ChecksumAlgorithm>, Error> {
 	match headers.get(X_AMZ_CHECKSUM_ALGORITHM) {
 		None => Ok(None),
-		Some(x) if x == "CRC32" => Ok(Some(ChecksumAlgorithm::Crc32)),
-		Some(x) if x == "CRC32C" => Ok(Some(ChecksumAlgorithm::Crc32c)),
-		Some(x) if x == "SHA1" => Ok(Some(ChecksumAlgorithm::Sha1)),
-		Some(x) if x == "SHA256" => Ok(Some(ChecksumAlgorithm::Sha256)),
+		Some(x) => parse_checksum_algorithm(x.to_str()?).map(Some),
+	}
+}
+
+pub fn request_trailer_checksum_algorithm(
+	headers: &HeaderMap<HeaderValue>,
+) -> Result<Option<ChecksumAlgorithm>, Error> {
+	match headers.get(X_AMZ_TRAILER).map(|x| x.to_str()).transpose()? {
+		None => Ok(None),
+		Some(x) if x == X_AMZ_CHECKSUM_CRC32 => Ok(Some(ChecksumAlgorithm::Crc32)),
+		Some(x) if x == X_AMZ_CHECKSUM_CRC32C => Ok(Some(ChecksumAlgorithm::Crc32c)),
+		Some(x) if x == X_AMZ_CHECKSUM_CRC64NVME => Ok(Some(ChecksumAlgorithm::Crc64Nvme)),
+		Some(x) if x == X_AMZ_CHECKSUM_SHA1 => Ok(Some(ChecksumAlgorithm::Sha1)),
+		Some(x) if x == X_AMZ_CHECKSUM_SHA256 => Ok(Some(ChecksumAlgorithm::Sha256)),
 		_ => Err(Error::bad_request("invalid checksum algorithm")),
 	}
 }
 
 /// Extract the value of any of the x-amz-checksum-* headers
-pub(crate) fn request_checksum_value(
+pub fn request_checksum_value(
 	headers: &HeaderMap<HeaderValue>,
 ) -> Result<Option<ChecksumValue>, Error> {
 	let mut ret = vec![];
 
-	if let Some(crc32_str) = headers.get(X_AMZ_CHECKSUM_CRC32) {
-		let crc32 = BASE64_STANDARD
-			.decode(&crc32_str)
-			.ok()
-			.and_then(|x| x.try_into().ok())
-			.ok_or_bad_request("invalid x-amz-checksum-crc32 header")?;
-		ret.push(ChecksumValue::Crc32(crc32))
+	if headers.contains_key(X_AMZ_CHECKSUM_CRC32) {
+		ret.push(extract_checksum_value(headers, ChecksumAlgorithm::Crc32)?);
 	}
-	if let Some(crc32c_str) = headers.get(X_AMZ_CHECKSUM_CRC32C) {
-		let crc32c = BASE64_STANDARD
-			.decode(&crc32c_str)
-			.ok()
-			.and_then(|x| x.try_into().ok())
-			.ok_or_bad_request("invalid x-amz-checksum-crc32c header")?;
-		ret.push(ChecksumValue::Crc32c(crc32c))
+	if headers.contains_key(X_AMZ_CHECKSUM_CRC32C) {
+		ret.push(extract_checksum_value(headers, ChecksumAlgorithm::Crc32c)?);
 	}
-	if let Some(sha1_str) = headers.get(X_AMZ_CHECKSUM_SHA1) {
-		let sha1 = BASE64_STANDARD
-			.decode(&sha1_str)
-			.ok()
-			.and_then(|x| x.try_into().ok())
-			.ok_or_bad_request("invalid x-amz-checksum-sha1 header")?;
-		ret.push(ChecksumValue::Sha1(sha1))
+	if headers.contains_key(X_AMZ_CHECKSUM_CRC64NVME) {
+		ret.push(extract_checksum_value(
+			headers,
+			ChecksumAlgorithm::Crc64Nvme,
+		)?);
 	}
-	if let Some(sha256_str) = headers.get(X_AMZ_CHECKSUM_SHA256) {
-		let sha256 = BASE64_STANDARD
-			.decode(&sha256_str)
-			.ok()
-			.and_then(|x| x.try_into().ok())
-			.ok_or_bad_request("invalid x-amz-checksum-sha256 header")?;
-		ret.push(ChecksumValue::Sha256(sha256))
+	if headers.contains_key(X_AMZ_CHECKSUM_SHA1) {
+		ret.push(extract_checksum_value(headers, ChecksumAlgorithm::Sha1)?);
+	}
+	if headers.contains_key(X_AMZ_CHECKSUM_SHA256) {
+		ret.push(extract_checksum_value(headers, ChecksumAlgorithm::Sha256)?);
 	}
 
 	if ret.len() > 1 {
@@ -342,48 +288,55 @@ pub(crate) fn request_checksum_value(
 
 /// Checks for the presence of x-amz-checksum-algorithm
 /// if so extract the corresponding x-amz-checksum-* value
-pub(crate) fn request_checksum_algorithm_value(
+pub fn extract_checksum_value(
 	headers: &HeaderMap<HeaderValue>,
-) -> Result<Option<ChecksumValue>, Error> {
-	match headers.get(X_AMZ_CHECKSUM_ALGORITHM) {
-		Some(x) if x == "CRC32" => {
+	algo: ChecksumAlgorithm,
+) -> Result<ChecksumValue, Error> {
+	match algo {
+		ChecksumAlgorithm::Crc32 => {
 			let crc32 = headers
 				.get(X_AMZ_CHECKSUM_CRC32)
 				.and_then(|x| BASE64_STANDARD.decode(&x).ok())
 				.and_then(|x| x.try_into().ok())
 				.ok_or_bad_request("invalid x-amz-checksum-crc32 header")?;
-			Ok(Some(ChecksumValue::Crc32(crc32)))
+			Ok(ChecksumValue::Crc32(crc32))
 		}
-		Some(x) if x == "CRC32C" => {
+		ChecksumAlgorithm::Crc32c => {
 			let crc32c = headers
 				.get(X_AMZ_CHECKSUM_CRC32C)
 				.and_then(|x| BASE64_STANDARD.decode(&x).ok())
 				.and_then(|x| x.try_into().ok())
 				.ok_or_bad_request("invalid x-amz-checksum-crc32c header")?;
-			Ok(Some(ChecksumValue::Crc32c(crc32c)))
+			Ok(ChecksumValue::Crc32c(crc32c))
 		}
-		Some(x) if x == "SHA1" => {
+		ChecksumAlgorithm::Crc64Nvme => {
+			let crc64nvme = headers
+				.get(X_AMZ_CHECKSUM_CRC64NVME)
+				.and_then(|x| BASE64_STANDARD.decode(&x).ok())
+				.and_then(|x| x.try_into().ok())
+				.ok_or_bad_request("invalid x-amz-checksum-crc64nvme header")?;
+			Ok(ChecksumValue::Crc64Nvme(crc64nvme))
+		}
+		ChecksumAlgorithm::Sha1 => {
 			let sha1 = headers
 				.get(X_AMZ_CHECKSUM_SHA1)
 				.and_then(|x| BASE64_STANDARD.decode(&x).ok())
 				.and_then(|x| x.try_into().ok())
 				.ok_or_bad_request("invalid x-amz-checksum-sha1 header")?;
-			Ok(Some(ChecksumValue::Sha1(sha1)))
+			Ok(ChecksumValue::Sha1(sha1))
 		}
-		Some(x) if x == "SHA256" => {
+		ChecksumAlgorithm::Sha256 => {
 			let sha256 = headers
 				.get(X_AMZ_CHECKSUM_SHA256)
 				.and_then(|x| BASE64_STANDARD.decode(&x).ok())
 				.and_then(|x| x.try_into().ok())
 				.ok_or_bad_request("invalid x-amz-checksum-sha256 header")?;
-			Ok(Some(ChecksumValue::Sha256(sha256)))
+			Ok(ChecksumValue::Sha256(sha256))
 		}
-		Some(_) => Err(Error::bad_request("invalid x-amz-checksum-algorithm")),
-		None => Ok(None),
 	}
 }
 
-pub(crate) fn add_checksum_response_headers(
+pub fn add_checksum_response_headers(
 	checksum: &Option<ChecksumValue>,
 	mut resp: http::response::Builder,
 ) -> http::response::Builder {
@@ -394,6 +347,9 @@ pub(crate) fn add_checksum_response_headers(
 		Some(ChecksumValue::Crc32c(crc32c)) => {
 			resp = resp.header(X_AMZ_CHECKSUM_CRC32C, BASE64_STANDARD.encode(&crc32c));
 		}
+		Some(ChecksumValue::Crc64Nvme(crc64nvme)) => {
+			resp = resp.header(X_AMZ_CHECKSUM_CRC64NVME, BASE64_STANDARD.encode(&crc64nvme));
+		}
 		Some(ChecksumValue::Sha1(sha1)) => {
 			resp = resp.header(X_AMZ_CHECKSUM_SHA1, BASE64_STANDARD.encode(&sha1));
 		}

src/api/common/signature/error.rs

@@ -18,6 +18,10 @@ pub enum Error {
 	/// The request contained an invalid UTF-8 sequence in its path or in other parameters
 	#[error(display = "Invalid UTF-8: {}", _0)]
 	InvalidUtf8Str(#[error(source)] std::str::Utf8Error),
+
+	/// The provided digest (checksum) value was invalid
+	#[error(display = "Invalid digest: {}", _0)]
+	InvalidDigest(String),
 }
 
 impl<T> From<T> for Error

src/api/common/signature/mod.rs

@@ -2,6 +2,7 @@ use chrono::{DateTime, Utc};
 use hmac::{Hmac, Mac};
 use sha2::Sha256;
 
+use hyper::header::HeaderName;
 use hyper::{body::Incoming as IncomingBody, Request};
 
 use garage_model::garage::Garage;
@@ -10,6 +11,8 @@ use garage_util::data::{sha256sum, Hash};
 
 use error::*;
 
+pub mod body;
+pub mod checksum;
 pub mod error;
 pub mod payload;
 pub mod streaming;
@@ -17,36 +20,73 @@ pub mod streaming;
 pub const SHORT_DATE: &str = "%Y%m%d";
 pub const LONG_DATETIME: &str = "%Y%m%dT%H%M%SZ";
 
+// ---- Constants used in AWSv4 signatures ----
+
+pub const X_AMZ_ALGORITHM: HeaderName = HeaderName::from_static("x-amz-algorithm");
+pub const X_AMZ_CREDENTIAL: HeaderName = HeaderName::from_static("x-amz-credential");
+pub const X_AMZ_DATE: HeaderName = HeaderName::from_static("x-amz-date");
+pub const X_AMZ_EXPIRES: HeaderName = HeaderName::from_static("x-amz-expires");
+pub const X_AMZ_SIGNEDHEADERS: HeaderName = HeaderName::from_static("x-amz-signedheaders");
+pub const X_AMZ_SIGNATURE: HeaderName = HeaderName::from_static("x-amz-signature");
+pub const X_AMZ_CONTENT_SHA256: HeaderName = HeaderName::from_static("x-amz-content-sha256");
+pub const X_AMZ_TRAILER: HeaderName = HeaderName::from_static("x-amz-trailer");
+
+/// Result of `sha256("")`
+pub(crate) const EMPTY_STRING_HEX_DIGEST: &str =
+	"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855";
+
+// Signature calculation algorithm
+pub const AWS4_HMAC_SHA256: &str = "AWS4-HMAC-SHA256";
 type HmacSha256 = Hmac<Sha256>;
 
-pub async fn verify_request(
+// Possible values for x-amz-content-sha256, in addition to the actual sha256
+pub const UNSIGNED_PAYLOAD: &str = "UNSIGNED-PAYLOAD";
+pub const STREAMING_UNSIGNED_PAYLOAD_TRAILER: &str = "STREAMING-UNSIGNED-PAYLOAD-TRAILER";
+pub const STREAMING_AWS4_HMAC_SHA256_PAYLOAD: &str = "STREAMING-AWS4-HMAC-SHA256-PAYLOAD";
+
+// Used in the computation of StringToSign
+pub const AWS4_HMAC_SHA256_PAYLOAD: &str = "AWS4-HMAC-SHA256-PAYLOAD";
+
+// ---- enums to describe stuff going on in signature calculation ----
+
+#[derive(Debug)]
+pub enum ContentSha256Header {
+	UnsignedPayload,
+	Sha256Checksum(Hash),
+	StreamingPayload { trailer: bool, signed: bool },
+}
+
+// ---- top-level functions ----
+
+pub struct VerifiedRequest {
+	pub request: Request<streaming::ReqBody>,
+	pub access_key: Key,
+	pub content_sha256_header: ContentSha256Header,
+}
+
+pub fn verify_request(
 	garage: &Garage,
 	mut req: Request<IncomingBody>,
 	service: &'static str,
-) -> Result<(Request<streaming::ReqBody>, Key, Option<Hash>), Error> {
-	let (api_key, mut content_sha256) =
-		payload::check_payload_signature(&garage, &mut req, service).await?;
-	let api_key =
-		api_key.ok_or_else(|| Error::forbidden("Garage does not support anonymous access yet"))?;
+) -> Result<VerifiedRequest, Error> {
+	let checked_signature = payload::check_payload_signature(&garage, &mut req, service)?;
 
-	let req = streaming::parse_streaming_body(
-		&api_key,
+	let request = streaming::parse_streaming_body(
 		req,
-		&mut content_sha256,
+		&checked_signature,
 		&garage.config.s3_api.s3_region,
 		service,
 	)?;
 
-	Ok((req, api_key, content_sha256))
-}
+	let access_key = checked_signature
+		.key
+		.ok_or_else(|| Error::forbidden("Garage does not support anonymous access yet"))?;
 
-pub fn verify_signed_content(expected_sha256: Hash, body: &[u8]) -> Result<(), Error> {
-	if expected_sha256 != sha256sum(body) {
-		return Err(Error::bad_request(
-			"Request content hash does not match signed hash".to_string(),
-		));
-	}
-	Ok(())
+	Ok(VerifiedRequest {
+		request,
+		access_key,
+		content_sha256_header: checked_signature.content_sha256_header,
+	})
 }
 
 pub fn signing_hmac(

src/api/common/signature/payload.rs

@@ -9,27 +9,14 @@ use sha2::{Digest, Sha256};
 
 use garage_table::*;
 use garage_util::data::Hash;
+use garage_util::time::now_msec;
 
 use garage_model::garage::Garage;
 use garage_model::key_table::*;
 
-use super::LONG_DATETIME;
-use super::{compute_scope, signing_hmac};
+use super::*;
 
 use crate::encoding::uri_encode;
-use crate::signature::error::*;
-
-pub const X_AMZ_ALGORITHM: HeaderName = HeaderName::from_static("x-amz-algorithm");
-pub const X_AMZ_CREDENTIAL: HeaderName = HeaderName::from_static("x-amz-credential");
-pub const X_AMZ_DATE: HeaderName = HeaderName::from_static("x-amz-date");
-pub const X_AMZ_EXPIRES: HeaderName = HeaderName::from_static("x-amz-expires");
-pub const X_AMZ_SIGNEDHEADERS: HeaderName = HeaderName::from_static("x-amz-signedheaders");
-pub const X_AMZ_SIGNATURE: HeaderName = HeaderName::from_static("x-amz-signature");
-pub const X_AMZ_CONTENT_SH256: HeaderName = HeaderName::from_static("x-amz-content-sha256");
-
-pub const AWS4_HMAC_SHA256: &str = "AWS4-HMAC-SHA256";
-pub const UNSIGNED_PAYLOAD: &str = "UNSIGNED-PAYLOAD";
-pub const STREAMING_AWS4_HMAC_SHA256_PAYLOAD: &str = "STREAMING-AWS4-HMAC-SHA256-PAYLOAD";
 
 pub type QueryMap = HeaderMap<QueryValue>;
 pub struct QueryValue {
@@ -39,44 +26,80 @@ pub struct QueryValue {
 	value: String,
 }
 
-pub async fn check_payload_signature(
+#[derive(Debug)]
+pub struct CheckedSignature {
+	pub key: Option<Key>,
+	pub content_sha256_header: ContentSha256Header,
+	pub signature_header: Option<String>,
+}
+
+pub fn check_payload_signature(
 	garage: &Garage,
 	request: &mut Request<IncomingBody>,
 	service: &'static str,
-) -> Result<(Option<Key>, Option<Hash>), Error> {
+) -> Result<CheckedSignature, Error> {
 	let query = parse_query_map(request.uri())?;
 
 	if query.contains_key(&X_AMZ_ALGORITHM) {
 		// We check for presigned-URL-style authentication first, because
 		// the browser or something else could inject an Authorization header
 		// that is totally unrelated to AWS signatures.
-		check_presigned_signature(garage, service, request, query).await
+		check_presigned_signature(garage, service, request, query)
 	} else if request.headers().contains_key(AUTHORIZATION) {
-		check_standard_signature(garage, service, request, query).await
+		check_standard_signature(garage, service, request, query)
 	} else {
 		// Unsigned (anonymous) request
 		let content_sha256 = request
 			.headers()
-			.get("x-amz-content-sha256")
-			.filter(|c| c.as_bytes() != UNSIGNED_PAYLOAD.as_bytes());
-		if let Some(content_sha256) = content_sha256 {
-			let sha256 = hex::decode(content_sha256)
-				.ok()
-				.and_then(|bytes| Hash::try_from(&bytes))
-				.ok_or_bad_request("Invalid content sha256 hash")?;
-			Ok((None, Some(sha256)))
-		} else {
-			Ok((None, None))
-		}
+			.get(X_AMZ_CONTENT_SHA256)
+			.map(|x| x.to_str())
+			.transpose()?;
+		Ok(CheckedSignature {
+			key: None,
+			content_sha256_header: parse_x_amz_content_sha256(content_sha256)?,
+			signature_header: None,
+		})
 	}
 }
 
-async fn check_standard_signature(
+fn parse_x_amz_content_sha256(header: Option<&str>) -> Result<ContentSha256Header, Error> {
+	let header = match header {
+		Some(x) => x,
+		None => return Ok(ContentSha256Header::UnsignedPayload),
+	};
+	if header == UNSIGNED_PAYLOAD {
+		Ok(ContentSha256Header::UnsignedPayload)
+	} else if let Some(rest) = header.strip_prefix("STREAMING-") {
+		let (trailer, algo) = if let Some(rest2) = rest.strip_suffix("-TRAILER") {
+			(true, rest2)
+		} else {
+			(false, rest)
+		};
+		let signed = match algo {
+			AWS4_HMAC_SHA256_PAYLOAD => true,
+			UNSIGNED_PAYLOAD => false,
+			_ => {
+				return Err(Error::bad_request(
+					"invalid or unsupported x-amz-content-sha256",
+				))
+			}
+		};
+		Ok(ContentSha256Header::StreamingPayload { trailer, signed })
+	} else {
+		let sha256 = hex::decode(header)
+			.ok()
+			.and_then(|bytes| Hash::try_from(&bytes))
+			.ok_or_bad_request("Invalid content sha256 hash")?;
+		Ok(ContentSha256Header::Sha256Checksum(sha256))
+	}
+}
+
+fn check_standard_signature(
 	garage: &Garage,
 	service: &'static str,
 	request: &Request<IncomingBody>,
 	query: QueryMap,
-) -> Result<(Option<Key>, Option<Hash>), Error> {
+) -> Result<CheckedSignature, Error> {
 	let authorization = Authorization::parse_header(request.headers())?;
 
 	// Verify that all necessary request headers are included in signed_headers
@@ -106,28 +129,23 @@ async fn check_standard_signature(
 	trace!("canonical request:\n{}", canonical_request);
 	trace!("string to sign:\n{}", string_to_sign);
 
-	let key = verify_v4(garage, service, &authorization, string_to_sign.as_bytes()).await?;
+	let key = verify_v4(garage, service, &authorization, string_to_sign.as_bytes())?;
 
-	let content_sha256 = if authorization.content_sha256 == UNSIGNED_PAYLOAD {
-		None
-	} else if authorization.content_sha256 == STREAMING_AWS4_HMAC_SHA256_PAYLOAD {
-		let bytes = hex::decode(authorization.signature).ok_or_bad_request("Invalid signature")?;
-		Some(Hash::try_from(&bytes).ok_or_bad_request("Invalid signature")?)
-	} else {
-		let bytes = hex::decode(authorization.content_sha256)
-			.ok_or_bad_request("Invalid content sha256 hash")?;
-		Some(Hash::try_from(&bytes).ok_or_bad_request("Invalid content sha256 hash")?)
-	};
+	let content_sha256_header = parse_x_amz_content_sha256(Some(&authorization.content_sha256))?;
 
-	Ok((Some(key), content_sha256))
+	Ok(CheckedSignature {
+		key: Some(key),
+		content_sha256_header,
+		signature_header: Some(authorization.signature),
+	})
 }
 
-async fn check_presigned_signature(
+fn check_presigned_signature(
 	garage: &Garage,
 	service: &'static str,
 	request: &mut Request<IncomingBody>,
 	mut query: QueryMap,
-) -> Result<(Option<Key>, Option<Hash>), Error> {
+) -> Result<CheckedSignature, Error> {
 	let algorithm = query.get(&X_AMZ_ALGORITHM).unwrap();
 	let authorization = Authorization::parse_presigned(&algorithm.value, &query)?;
 
@@ -161,7 +179,7 @@ async fn check_presigned_signature(
 	trace!("canonical request (presigned url):\n{}", canonical_request);
 	trace!("string to sign (presigned url):\n{}", string_to_sign);
 
-	let key = verify_v4(garage, service, &authorization, string_to_sign.as_bytes()).await?;
+	let key = verify_v4(garage, service, &authorization, string_to_sign.as_bytes())?;
 
 	// In the page on presigned URLs, AWS specifies that if a signed query
 	// parameter and a signed header of the same name have different values,
@@ -193,7 +211,11 @@ async fn check_presigned_signature(
 
 	// Presigned URLs always use UNSIGNED-PAYLOAD,
 	// so there is no sha256 hash to return.
-	Ok((Some(key), None))
+	Ok(CheckedSignature {
+		key: Some(key),
+		content_sha256_header: ContentSha256Header::UnsignedPayload,
+		signature_header: Some(authorization.signature),
+	})
 }
 
 pub fn parse_query_map(uri: &http::uri::Uri) -> Result<QueryMap, Error> {
@@ -357,7 +379,7 @@ pub fn parse_date(date: &str) -> Result<DateTime<Utc>, Error> {
 	Ok(Utc.from_utc_datetime(&date))
 }
 
-pub async fn verify_v4(
+pub fn verify_v4(
 	garage: &Garage,
 	service: &str,
 	auth: &Authorization,
@@ -370,12 +392,18 @@ pub async fn verify_v4(
 
 	let key = garage
 		.key_table
-		.get(&EmptyKey, &auth.key_id)
-		.await?
+		.get_local(&EmptyKey, &auth.key_id)?
 		.filter(|k| !k.state.is_deleted())
 		.ok_or_else(|| Error::forbidden(format!("No such key: {}", &auth.key_id)))?;
 	let key_p = key.params().unwrap();
 
+	if key_p.is_expired(now_msec()) {
+		return Err(Error::forbidden(format!(
+			"Access key {} has expired",
+			key.key_id
+		)));
+	}
+
 	let mut hmac = signing_hmac(
 		&auth.date,
 		&key_p.secret_key,
@@ -442,7 +470,7 @@ impl Authorization {
 			.to_string();
 
 		let content_sha256 = headers
-			.get(X_AMZ_CONTENT_SH256)
+			.get(X_AMZ_CONTENT_SHA256)
 			.ok_or_bad_request("Missing X-Amz-Content-Sha256 field")?;
 
 		let date = headers

src/api/common/signature/streaming.rs

@@ -1,84 +1,157 @@
 use std::pin::Pin;
+use std::sync::Mutex;
 
 use chrono::{DateTime, NaiveDateTime, TimeZone, Utc};
 use futures::prelude::*;
 use futures::task;
-use garage_model::key_table::Key;
 use hmac::Mac;
-use http_body_util::StreamBody;
-use hyper::body::{Bytes, Incoming as IncomingBody};
+use http::header::{HeaderMap, HeaderValue, CONTENT_ENCODING};
+use hyper::body::{Bytes, Frame, Incoming as IncomingBody};
 use hyper::Request;
 
 use garage_util::data::Hash;
 
-use super::{compute_scope, sha256sum, HmacSha256, LONG_DATETIME};
+use super::*;
 
-use crate::helpers::*;
-use crate::signature::error::*;
-use crate::signature::payload::{
-	STREAMING_AWS4_HMAC_SHA256_PAYLOAD, X_AMZ_CONTENT_SH256, X_AMZ_DATE,
-};
+use crate::helpers::body_stream;
+use crate::signature::checksum::*;
+use crate::signature::payload::CheckedSignature;
 
-pub const AWS4_HMAC_SHA256_PAYLOAD: &str = "AWS4-HMAC-SHA256-PAYLOAD";
-
-pub type ReqBody = BoxBody<Error>;
+pub use crate::signature::body::ReqBody;
 
 pub fn parse_streaming_body(
-	api_key: &Key,
-	req: Request<IncomingBody>,
-	content_sha256: &mut Option<Hash>,
+	mut req: Request<IncomingBody>,
+	checked_signature: &CheckedSignature,
 	region: &str,
 	service: &str,
 ) -> Result<Request<ReqBody>, Error> {
-	match req.headers().get(X_AMZ_CONTENT_SH256) {
-		Some(header) if header == STREAMING_AWS4_HMAC_SHA256_PAYLOAD => {
-			let signature = content_sha256
-				.take()
-				.ok_or_bad_request("No signature provided")?;
+	debug!(
+		"Content signature mode: {:?}",
+		checked_signature.content_sha256_header
+	);
 
-			let secret_key = &api_key
-				.state
-				.as_option()
-				.ok_or_internal_error("Deleted key state")?
-				.secret_key;
+	match checked_signature.content_sha256_header {
+		ContentSha256Header::StreamingPayload { signed, trailer } => {
+			// Sanity checks
+			if !signed && !trailer {
+				return Err(Error::bad_request(
+					"STREAMING-UNSIGNED-PAYLOAD without trailer is not a valid combination",
+				));
+			}
 
-			let date = req
-				.headers()
-				.get(X_AMZ_DATE)
-				.ok_or_bad_request("Missing X-Amz-Date field")?
-				.to_str()?;
-			let date: NaiveDateTime = NaiveDateTime::parse_from_str(date, LONG_DATETIME)
-				.ok_or_bad_request("Invalid date")?;
-			let date: DateTime<Utc> = Utc.from_utc_datetime(&date);
+			// Remove the aws-chunked component in the content-encoding: header
+			// Note: this header is not properly sent by minio client, so don't fail
+			// if it is absent from the request.
+			if let Some(content_encoding) = req.headers_mut().remove(CONTENT_ENCODING) {
+				if let Some(rest) = content_encoding.as_bytes().strip_prefix(b"aws-chunked,") {
+					req.headers_mut()
+						.insert(CONTENT_ENCODING, HeaderValue::from_bytes(rest).unwrap());
+				} else if content_encoding != "aws-chunked" {
+					return Err(Error::bad_request(
+						"content-encoding does not contain aws-chunked for STREAMING-*-PAYLOAD",
+					));
+				}
+			}
 
-			let scope = compute_scope(&date, region, service);
-			let signing_hmac = crate::signature::signing_hmac(&date, secret_key, region, service)
-				.ok_or_internal_error("Unable to build signing HMAC")?;
+			// If trailer header is announced, add the calculation of the requested checksum
+			let mut checksummer = Checksummer::init(&Default::default(), false);
+			let trailer_algorithm = if trailer {
+				let algo = Some(
+					request_trailer_checksum_algorithm(req.headers())?
+						.ok_or_bad_request("Missing x-amz-trailer header")?,
+				);
+				checksummer = checksummer.add(algo);
+				algo
+			} else {
+				None
+			};
+
+			// For signed variants, determine signing parameters
+			let sign_params = if signed {
+				let signature = checked_signature
+					.signature_header
+					.clone()
+					.ok_or_bad_request("No signature provided")?;
+				let signature = hex::decode(signature)
+					.ok()
+					.and_then(|bytes| Hash::try_from(&bytes))
+					.ok_or_bad_request("Invalid signature")?;
+
+				let secret_key = checked_signature
+					.key
+					.as_ref()
+					.ok_or_bad_request("Cannot sign streaming payload without signing key")?
+					.state
+					.as_option()
+					.ok_or_internal_error("Deleted key state")?
+					.secret_key
+					.to_string();
+
+				let date = req
+					.headers()
+					.get(X_AMZ_DATE)
+					.ok_or_bad_request("Missing X-Amz-Date field")?
+					.to_str()?;
+				let date: NaiveDateTime = NaiveDateTime::parse_from_str(date, LONG_DATETIME)
+					.ok_or_bad_request("Invalid date")?;
+				let date: DateTime<Utc> = Utc.from_utc_datetime(&date);
+
+				let scope = compute_scope(&date, region, service);
+				let signing_hmac =
+					crate::signature::signing_hmac(&date, &secret_key, region, service)
+						.ok_or_internal_error("Unable to build signing HMAC")?;
+
+				Some(SignParams {
+					datetime: date,
+					scope,
+					signing_hmac,
+					previous_signature: signature,
+				})
+			} else {
+				None
+			};
 
 			Ok(req.map(move |body| {
 				let stream = body_stream::<_, Error>(body);
+
 				let signed_payload_stream =
-					SignedPayloadStream::new(stream, signing_hmac, date, &scope, signature)
-						.map(|x| x.map(hyper::body::Frame::data))
-						.map_err(Error::from);
-				ReqBody::new(StreamBody::new(signed_payload_stream))
+					StreamingPayloadStream::new(stream, sign_params, trailer).map_err(Error::from);
+				ReqBody {
+					stream: Mutex::new(signed_payload_stream.boxed()),
+					checksummer,
+					expected_checksums: Default::default(),
+					trailer_algorithm,
+				}
 			}))
 		}
-		_ => Ok(req.map(|body| ReqBody::new(http_body_util::BodyExt::map_err(body, Error::from)))),
+		_ => Ok(req.map(|body| {
+			let expected_checksums = ExpectedChecksums {
+				sha256: match &checked_signature.content_sha256_header {
+					ContentSha256Header::Sha256Checksum(sha256) => Some(*sha256),
+					_ => None,
+				},
+				..Default::default()
+			};
+			let checksummer = Checksummer::init(&expected_checksums, false);
+
+			let stream = http_body_util::BodyStream::new(body).map_err(Error::from);
+			ReqBody {
+				stream: Mutex::new(stream.boxed()),
+				checksummer,
+				expected_checksums,
+				trailer_algorithm: None,
+			}
+		})),
 	}
 }
 
-/// Result of `sha256("")`
-const EMPTY_STRING_HEX_DIGEST: &str =
-	"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855";
-
 fn compute_streaming_payload_signature(
 	signing_hmac: &HmacSha256,
 	date: DateTime<Utc>,
 	scope: &str,
 	previous_signature: Hash,
 	content_sha256: Hash,
-) -> Result<Hash, Error> {
+) -> Result<Hash, StreamingPayloadError> {
 	let string_to_sign = [
 		AWS4_HMAC_SHA256_PAYLOAD,
 		&date.format(LONG_DATETIME).to_string(),
@@ -92,12 +165,49 @@ fn compute_streaming_payload_signature(
 	let mut hmac = signing_hmac.clone();
 	hmac.update(string_to_sign.as_bytes());
 
-	Ok(Hash::try_from(&hmac.finalize().into_bytes()).ok_or_internal_error("Invalid signature")?)
+	Hash::try_from(&hmac.finalize().into_bytes())
+		.ok_or_else(|| StreamingPayloadError::Message("Could not build signature".into()))
+}
+
+fn compute_streaming_trailer_signature(
+	signing_hmac: &HmacSha256,
+	date: DateTime<Utc>,
+	scope: &str,
+	previous_signature: Hash,
+	trailer_sha256: Hash,
+) -> Result<Hash, StreamingPayloadError> {
+	let string_to_sign = [
+		AWS4_HMAC_SHA256_PAYLOAD,
+		&date.format(LONG_DATETIME).to_string(),
+		scope,
+		&hex::encode(previous_signature),
+		&hex::encode(trailer_sha256),
+	]
+	.join("\n");
+
+	let mut hmac = signing_hmac.clone();
+	hmac.update(string_to_sign.as_bytes());
+
+	Hash::try_from(&hmac.finalize().into_bytes())
+		.ok_or_else(|| StreamingPayloadError::Message("Could not build signature".into()))
 }
 
 mod payload {
+	use http::{HeaderName, HeaderValue};
+
 	use garage_util::data::Hash;
 
+	use nom::bytes::streaming::{tag, take_while};
+	use nom::character::streaming::hex_digit1;
+	use nom::combinator::{map_res, opt};
+	use nom::number::streaming::hex_u32;
+
+	macro_rules! try_parse {
+		($expr:expr) => {
+			$expr.map_err(|e| e.map(Error::Parser))?
+		};
+	}
+
 	pub enum Error<I> {
 		Parser(nom::error::Error<I>),
 		BadSignature,
@@ -113,24 +223,13 @@ mod payload {
 	}
 
 	#[derive(Debug, Clone)]
-	pub struct Header {
+	pub struct ChunkHeader {
 		pub size: usize,
-		pub signature: Hash,
+		pub signature: Option<Hash>,
 	}
 
-	impl Header {
-		pub fn parse(input: &[u8]) -> nom::IResult<&[u8], Self, Error<&[u8]>> {
-			use nom::bytes::streaming::tag;
-			use nom::character::streaming::hex_digit1;
-			use nom::combinator::map_res;
-			use nom::number::streaming::hex_u32;
-
-			macro_rules! try_parse {
-				($expr:expr) => {
-					$expr.map_err(|e| e.map(Error::Parser))?
-				};
-			}
-
+	impl ChunkHeader {
+		pub fn parse_signed(input: &[u8]) -> nom::IResult<&[u8], Self, Error<&[u8]>> {
 			let (input, size) = try_parse!(hex_u32(input));
 			let (input, _) = try_parse!(tag(";")(input));
 
@@ -140,96 +239,172 @@ mod payload {
 
 			let (input, _) = try_parse!(tag("\r\n")(input));
 
-			let header = Header {
+			let header = ChunkHeader {
 				size: size as usize,
-				signature,
+				signature: Some(signature),
+			};
+
+			Ok((input, header))
+		}
+
+		pub fn parse_unsigned(input: &[u8]) -> nom::IResult<&[u8], Self, Error<&[u8]>> {
+			let (input, size) = try_parse!(hex_u32(input));
+			let (input, _) = try_parse!(tag("\r\n")(input));
+
+			let header = ChunkHeader {
+				size: size as usize,
+				signature: None,
 			};
 
 			Ok((input, header))
 		}
 	}
+
+	#[derive(Debug, Clone)]
+	pub struct TrailerChunk {
+		pub header_name: HeaderName,
+		pub header_value: HeaderValue,
+		pub signature: Option<Hash>,
+	}
+
+	impl TrailerChunk {
+		fn parse_content(input: &[u8]) -> nom::IResult<&[u8], Self, Error<&[u8]>> {
+			let (input, header_name) = try_parse!(map_res(
+				take_while(|c: u8| c.is_ascii_alphanumeric() || c == b'-'),
+				HeaderName::from_bytes
+			)(input));
+			let (input, _) = try_parse!(tag(b":")(input));
+			let (input, header_value) = try_parse!(map_res(
+				take_while(|c: u8| c.is_ascii_alphanumeric() || b"+/=".contains(&c)),
+				HeaderValue::from_bytes
+			)(input));
+
+			// Possible '\n' after the header value, depends on clients
+			// https://docs.aws.amazon.com/AmazonS3/latest/userguide/checking-object-integrity.html
+			let (input, _) = try_parse!(opt(tag(b"\n"))(input));
+
+			let (input, _) = try_parse!(tag(b"\r\n")(input));
+
+			Ok((
+				input,
+				TrailerChunk {
+					header_name,
+					header_value,
+					signature: None,
+				},
+			))
+		}
+		pub fn parse_signed(input: &[u8]) -> nom::IResult<&[u8], Self, Error<&[u8]>> {
+			let (input, trailer) = Self::parse_content(input)?;
+
+			let (input, _) = try_parse!(tag(b"x-amz-trailer-signature:")(input));
+			let (input, data) = try_parse!(map_res(hex_digit1, hex::decode)(input));
+			let signature = Hash::try_from(&data).ok_or(nom::Err::Failure(Error::BadSignature))?;
+			let (input, _) = try_parse!(tag(b"\r\n")(input));
+
+			Ok((
+				input,
+				TrailerChunk {
+					signature: Some(signature),
+					..trailer
+				},
+			))
+		}
+		pub fn parse_unsigned(input: &[u8]) -> nom::IResult<&[u8], Self, Error<&[u8]>> {
+			let (input, trailer) = Self::parse_content(input)?;
+			let (input, _) = try_parse!(tag(b"\r\n")(input));
+
+			Ok((input, trailer))
+		}
+	}
 }
 
 #[derive(Debug)]
-pub enum SignedPayloadStreamError {
+pub enum StreamingPayloadError {
 	Stream(Error),
 	InvalidSignature,
 	Message(String),
 }
 
-impl SignedPayloadStreamError {
+impl StreamingPayloadError {
 	fn message(msg: &str) -> Self {
-		SignedPayloadStreamError::Message(msg.into())
+		StreamingPayloadError::Message(msg.into())
 	}
 }
 
-impl From<SignedPayloadStreamError> for Error {
-	fn from(err: SignedPayloadStreamError) -> Self {
+impl From<StreamingPayloadError> for Error {
+	fn from(err: StreamingPayloadError) -> Self {
 		match err {
-			SignedPayloadStreamError::Stream(e) => e,
-			SignedPayloadStreamError::InvalidSignature => {
+			StreamingPayloadError::Stream(e) => e,
+			StreamingPayloadError::InvalidSignature => {
 				Error::bad_request("Invalid payload signature")
 			}
-			SignedPayloadStreamError::Message(e) => {
+			StreamingPayloadError::Message(e) => {
 				Error::bad_request(format!("Chunk format error: {}", e))
 			}
 		}
 	}
 }
 
-impl<I> From<payload::Error<I>> for SignedPayloadStreamError {
+impl<I> From<payload::Error<I>> for StreamingPayloadError {
 	fn from(err: payload::Error<I>) -> Self {
 		Self::message(err.description())
 	}
 }
 
-impl<I> From<nom::error::Error<I>> for SignedPayloadStreamError {
+impl<I> From<nom::error::Error<I>> for StreamingPayloadError {
 	fn from(err: nom::error::Error<I>) -> Self {
 		Self::message(err.code.description())
 	}
 }
 
-struct SignedPayload {
-	header: payload::Header,
-	data: Bytes,
+enum StreamingPayloadChunk {
+	Chunk {
+		header: payload::ChunkHeader,
+		data: Bytes,
+	},
+	Trailer(payload::TrailerChunk),
 }
 
-#[pin_project::pin_project]
-pub struct SignedPayloadStream<S>
-where
-	S: Stream<Item = Result<Bytes, Error>>,
-{
-	#[pin]
-	stream: S,
-	buf: bytes::BytesMut,
+struct SignParams {
 	datetime: DateTime<Utc>,
 	scope: String,
 	signing_hmac: HmacSha256,
 	previous_signature: Hash,
 }
 
-impl<S> SignedPayloadStream<S>
+#[pin_project::pin_project]
+pub struct StreamingPayloadStream<S>
 where
 	S: Stream<Item = Result<Bytes, Error>>,
 {
-	pub fn new(
-		stream: S,
-		signing_hmac: HmacSha256,
-		datetime: DateTime<Utc>,
-		scope: &str,
-		seed_signature: Hash,
-	) -> Self {
+	#[pin]
+	stream: S,
+	buf: bytes::BytesMut,
+	signing: Option<SignParams>,
+	has_trailer: bool,
+	done: bool,
+}
+
+impl<S> StreamingPayloadStream<S>
+where
+	S: Stream<Item = Result<Bytes, Error>>,
+{
+	fn new(stream: S, signing: Option<SignParams>, has_trailer: bool) -> Self {
 		Self {
 			stream,
 			buf: bytes::BytesMut::new(),
-			datetime,
-			scope: scope.into(),
-			signing_hmac,
-			previous_signature: seed_signature,
+			signing,
+			has_trailer,
+			done: false,
 		}
 	}
 
-	fn parse_next(input: &[u8]) -> nom::IResult<&[u8], SignedPayload, SignedPayloadStreamError> {
+	fn parse_next(
+		input: &[u8],
+		is_signed: bool,
+		has_trailer: bool,
+	) -> nom::IResult<&[u8], StreamingPayloadChunk, StreamingPayloadError> {
 		use nom::bytes::streaming::{tag, take};
 
 		macro_rules! try_parse {
@@ -238,17 +413,30 @@ where
 			};
 		}
 
-		let (input, header) = try_parse!(payload::Header::parse(input));
+		let (input, header) = if is_signed {
+			try_parse!(payload::ChunkHeader::parse_signed(input))
+		} else {
+			try_parse!(payload::ChunkHeader::parse_unsigned(input))
+		};
 
 		// 0-sized chunk is the last
 		if header.size == 0 {
-			return Ok((
-				input,
-				SignedPayload {
-					header,
-					data: Bytes::new(),
-				},
-			));
+			if has_trailer {
+				let (input, trailer) = if is_signed {
+					try_parse!(payload::TrailerChunk::parse_signed(input))
+				} else {
+					try_parse!(payload::TrailerChunk::parse_unsigned(input))
+				};
+				return Ok((input, StreamingPayloadChunk::Trailer(trailer)));
+			} else {
+				return Ok((
+					input,
+					StreamingPayloadChunk::Chunk {
+						header,
+						data: Bytes::new(),
+					},
+				));
+			}
 		}
 
 		let (input, data) = try_parse!(take::<_, _, nom::error::Error<_>>(header.size)(input));
@@ -256,15 +444,15 @@ where
 
 		let data = Bytes::from(data.to_vec());
 
-		Ok((input, SignedPayload { header, data }))
+		Ok((input, StreamingPayloadChunk::Chunk { header, data }))
 	}
 }
 
-impl<S> Stream for SignedPayloadStream<S>
+impl<S> Stream for StreamingPayloadStream<S>
 where
 	S: Stream<Item = Result<Bytes, Error>> + Unpin,
 {
-	type Item = Result<Bytes, SignedPayloadStreamError>;
+	type Item = Result<Frame<Bytes>, StreamingPayloadError>;
 
 	fn poll_next(
 		self: Pin<&mut Self>,
@@ -274,56 +462,105 @@ where
 
 		let mut this = self.project();
 
+		if *this.done {
+			return Poll::Ready(None);
+		}
+
 		loop {
-			let (input, payload) = match Self::parse_next(this.buf) {
-				Ok(res) => res,
-				Err(nom::Err::Incomplete(_)) => {
-					match futures::ready!(this.stream.as_mut().poll_next(cx)) {
-						Some(Ok(bytes)) => {
-							this.buf.extend(bytes);
-							continue;
-						}
-						Some(Err(e)) => {
-							return Poll::Ready(Some(Err(SignedPayloadStreamError::Stream(e))))
-						}
-						None => {
-							return Poll::Ready(Some(Err(SignedPayloadStreamError::message(
-								"Unexpected EOF",
-							))));
+			let (input, payload) =
+				match Self::parse_next(this.buf, this.signing.is_some(), *this.has_trailer) {
+					Ok(res) => res,
+					Err(nom::Err::Incomplete(_)) => {
+						match futures::ready!(this.stream.as_mut().poll_next(cx)) {
+							Some(Ok(bytes)) => {
+								this.buf.extend(bytes);
+								continue;
+							}
+							Some(Err(e)) => {
+								return Poll::Ready(Some(Err(StreamingPayloadError::Stream(e))))
+							}
+							None => {
+								return Poll::Ready(Some(Err(StreamingPayloadError::message(
+									"Unexpected EOF",
+								))));
+							}
 						}
 					}
+					Err(nom::Err::Error(e)) | Err(nom::Err::Failure(e)) => {
+						return Poll::Ready(Some(Err(e)))
+					}
+				};
+
+			match payload {
+				StreamingPayloadChunk::Chunk { data, header } => {
+					if let Some(signing) = this.signing.as_mut() {
+						let data_sha256sum = sha256sum(&data);
+
+						let expected_signature = compute_streaming_payload_signature(
+							&signing.signing_hmac,
+							signing.datetime,
+							&signing.scope,
+							signing.previous_signature,
+							data_sha256sum,
+						)?;
+
+						if header.signature.unwrap() != expected_signature {
+							return Poll::Ready(Some(Err(StreamingPayloadError::InvalidSignature)));
+						}
+
+						signing.previous_signature = header.signature.unwrap();
+					}
+
+					*this.buf = input.into();
+
+					// 0-sized chunk is the last
+					if data.is_empty() {
+						// if there was a trailer, it would have been returned by the parser
+						assert!(!*this.has_trailer);
+						*this.done = true;
+						return Poll::Ready(None);
+					}
+
+					return Poll::Ready(Some(Ok(Frame::data(data))));
 				}
-				Err(nom::Err::Error(e)) | Err(nom::Err::Failure(e)) => {
-					return Poll::Ready(Some(Err(e)))
+				StreamingPayloadChunk::Trailer(trailer) => {
+					trace!(
+						"In StreamingPayloadStream::poll_next: got trailer {:?}",
+						trailer
+					);
+
+					if let Some(signing) = this.signing.as_mut() {
+						let data = [
+							trailer.header_name.as_ref(),
+							&b":"[..],
+							trailer.header_value.as_ref(),
+							&b"\n"[..],
+						]
+						.concat();
+						let trailer_sha256sum = sha256sum(&data);
+
+						let expected_signature = compute_streaming_trailer_signature(
+							&signing.signing_hmac,
+							signing.datetime,
+							&signing.scope,
+							signing.previous_signature,
+							trailer_sha256sum,
+						)?;
+
+						if trailer.signature.unwrap() != expected_signature {
+							return Poll::Ready(Some(Err(StreamingPayloadError::InvalidSignature)));
+						}
+					}
+
+					*this.buf = input.into();
+					*this.done = true;
+
+					let mut trailers_map = HeaderMap::new();
+					trailers_map.insert(trailer.header_name, trailer.header_value);
+
+					return Poll::Ready(Some(Ok(Frame::trailers(trailers_map))));
 				}
-			};
-
-			// 0-sized chunk is the last
-			if payload.data.is_empty() {
-				return Poll::Ready(None);
 			}
-
-			let data_sha256sum = sha256sum(&payload.data);
-
-			let expected_signature = compute_streaming_payload_signature(
-				this.signing_hmac,
-				*this.datetime,
-				this.scope,
-				*this.previous_signature,
-				data_sha256sum,
-			)
-			.map_err(|e| {
-				SignedPayloadStreamError::Message(format!("Could not build signature: {}", e))
-			})?;
-
-			if payload.header.signature != expected_signature {
-				return Poll::Ready(Some(Err(SignedPayloadStreamError::InvalidSignature)));
-			}
-
-			*this.buf = input.into();
-			*this.previous_signature = payload.header.signature;
-
-			return Poll::Ready(Some(Ok(payload.data)));
 		}
 	}
 
@@ -336,7 +573,7 @@ where
 mod tests {
 	use futures::prelude::*;
 
-	use super::{SignedPayloadStream, SignedPayloadStreamError};
+	use super::{SignParams, StreamingPayloadError, StreamingPayloadStream};
 
 	#[tokio::test]
 	async fn test_interrupted_signed_payload_stream() {
@@ -358,12 +595,20 @@ mod tests {
 
 		let seed_signature = Hash::default();
 
-		let mut stream =
-			SignedPayloadStream::new(body, signing_hmac, datetime, &scope, seed_signature);
+		let mut stream = StreamingPayloadStream::new(
+			body,
+			Some(SignParams {
+				signing_hmac,
+				datetime,
+				scope,
+				previous_signature: seed_signature,
+			}),
+			false,
+		);
 
 		assert!(stream.try_next().await.is_err());
 		match stream.try_next().await {
-			Err(SignedPayloadStreamError::Message(msg)) if msg == "Unexpected EOF" => {}
+			Err(StreamingPayloadError::Message(msg)) if msg == "Unexpected EOF" => {}
 			item => panic!(
 				"Unexpected result, expected early EOF error, got {:?}",
 				item

src/api/k2v/Cargo.toml

@@ -1,12 +1,12 @@
 [package]
 name = "garage_api_k2v"
-version = "1.0.1"
+version = "2.0.0"
 authors = ["Alex Auvolat <alex@adnab.me>"]
 edition = "2018"
 license = "AGPL-3.0"
 description = "K2V API server crate for the Garage object store"
 repository = "https://git.deuxfleurs.fr/Deuxfleurs/garage"
-readme = "../../README.md"
+readme = "../../../README.md"
 
 [lib]
 path = "lib.rs"

src/api/k2v/api_server.rs

@@ -1,3 +1,4 @@
+use std::borrow::Cow;
 use std::sync::Arc;
 
 use hyper::{body::Incoming as IncomingBody, Method, Request, Response};
@@ -76,23 +77,19 @@ impl ApiHandler for K2VApiServer {
 		// The OPTIONS method is processed early, before we even check for an API key
 		if let Endpoint::Options = endpoint {
 			let options_res = handle_options_api(garage, &req, Some(bucket_name))
-				.await
 				.ok_or_bad_request("Error handling OPTIONS")?;
 			return Ok(options_res.map(|_empty_body: EmptyBody| empty_body()));
 		}
 
-		let (req, api_key, _content_sha256) = verify_request(&garage, req, "k2v").await?;
+		let verified_request = verify_request(&garage, req, "k2v")?;
+		let req = verified_request.request;
+		let api_key = verified_request.access_key;
 
-		let bucket_id = garage
-			.bucket_helper()
-			.resolve_bucket(&bucket_name, &api_key)
-			.await
-			.map_err(pass_helper_error)?;
 		let bucket = garage
 			.bucket_helper()
-			.get_existing_bucket(bucket_id)
-			.await
-			.map_err(helper_error_as_internal)?;
+			.resolve_bucket_fast(&bucket_name, &api_key)
+			.map_err(pass_helper_error)?;
+		let bucket_id = bucket.id;
 		let bucket_params = bucket.state.into_option().unwrap();
 
 		let allowed = match endpoint.authorization_type() {
@@ -177,8 +174,8 @@ impl ApiHandler for K2VApiServer {
 }
 
 impl ApiEndpoint for K2VApiEndpoint {
-	fn name(&self) -> &'static str {
-		self.endpoint.name()
+	fn name(&self) -> Cow<'static, str> {
+		Cow::Borrowed(self.endpoint.name())
 	}
 
 	fn add_span_attributes(&self, span: SpanRef<'_>) {

src/api/k2v/batch.rs

@@ -20,7 +20,7 @@ pub async fn handle_insert_batch(
 	let ReqCtx {
 		garage, bucket_id, ..
 	} = &ctx;
-	let items = parse_json_body::<Vec<InsertBatchItem>, _, Error>(req).await?;
+	let items = req.into_body().json::<Vec<InsertBatchItem>>().await?;
 
 	let mut items2 = vec![];
 	for it in items {
@@ -47,7 +47,7 @@ pub async fn handle_read_batch(
 	ctx: ReqCtx,
 	req: Request<ReqBody>,
 ) -> Result<Response<ResBody>, Error> {
-	let queries = parse_json_body::<Vec<ReadBatchQuery>, _, Error>(req).await?;
+	let queries = req.into_body().json::<Vec<ReadBatchQuery>>().await?;
 
 	let resp_results = futures::future::join_all(
 		queries
@@ -141,7 +141,7 @@ pub async fn handle_delete_batch(
 	ctx: ReqCtx,
 	req: Request<ReqBody>,
 ) -> Result<Response<ResBody>, Error> {
-	let queries = parse_json_body::<Vec<DeleteBatchQuery>, _, Error>(req).await?;
+	let queries = req.into_body().json::<Vec<DeleteBatchQuery>>().await?;
 
 	let resp_results = futures::future::join_all(
 		queries
@@ -262,7 +262,7 @@ pub(crate) async fn handle_poll_range(
 	} = ctx;
 	use garage_model::k2v::sub::PollRange;
 
-	let query = parse_json_body::<PollRangeQuery, _, Error>(req).await?;
+	let query = req.into_body().json::<PollRangeQuery>().await?;
 
 	let timeout_msec = query.timeout.unwrap_or(300).clamp(1, 600) * 1000;
 

src/api/k2v/error.rs

@@ -2,8 +2,8 @@ use err_derive::Error;
 use hyper::header::HeaderValue;
 use hyper::{HeaderMap, StatusCode};
 
+pub(crate) use garage_api_common::common_error::pass_helper_error;
 use garage_api_common::common_error::{commonErrorDerivative, CommonError};
-pub(crate) use garage_api_common::common_error::{helper_error_as_internal, pass_helper_error};
 pub use garage_api_common::common_error::{
 	CommonErrorDerivative, OkOrBadRequest, OkOrInternalError,
 };
@@ -23,6 +23,10 @@ pub enum Error {
 	#[error(display = "Authorization header malformed, unexpected scope: {}", _0)]
 	AuthorizationHeaderMalformed(String),
 
+	/// The provided digest (checksum) value was invalid
+	#[error(display = "Invalid digest: {}", _0)]
+	InvalidDigest(String),
+
 	/// The object requested don't exists
 	#[error(display = "Key not found")]
 	NoSuchKey,
@@ -54,6 +58,7 @@ impl From<SignatureError> for Error {
 				Self::AuthorizationHeaderMalformed(c)
 			}
 			SignatureError::InvalidUtf8Str(i) => Self::InvalidUtf8Str(i),
+			SignatureError::InvalidDigest(d) => Self::InvalidDigest(d),
 		}
 	}
 }
@@ -71,6 +76,7 @@ impl Error {
 			Error::InvalidBase64(_) => "InvalidBase64",
 			Error::InvalidUtf8Str(_) => "InvalidUtf8String",
 			Error::InvalidCausalityToken => "CausalityToken",
+			Error::InvalidDigest(_) => "InvalidDigest",
 		}
 	}
 }
@@ -85,6 +91,7 @@ impl ApiError for Error {
 			Error::AuthorizationHeaderMalformed(_)
 			| Error::InvalidBase64(_)
 			| Error::InvalidUtf8Str(_)
+			| Error::InvalidDigest(_)
 			| Error::InvalidCausalityToken => StatusCode::BAD_REQUEST,
 		}
 	}

src/api/k2v/item.rs

@@ -144,9 +144,7 @@ pub async fn handle_insert_item(
 		.map(parse_causality_token)
 		.transpose()?;
 
-	let body = http_body_util::BodyExt::collect(req.into_body())
-		.await?
-		.to_bytes();
+	let body = req.into_body().collect().await?;
 
 	let value = DvvsValue::Value(body.to_vec());
 

src/api/s3/Cargo.toml

@@ -1,12 +1,12 @@
 [package]
 name = "garage_api_s3"
-version = "1.0.1"
+version = "2.0.0"
 authors = ["Alex Auvolat <alex@adnab.me>"]
 edition = "2018"
 license = "AGPL-3.0"
 description = "S3 API server crate for the Garage object store"
 repository = "https://git.deuxfleurs.fr/Deuxfleurs/garage"
-readme = "../../README.md"
+readme = "../../../README.md"
 
 [lib]
 path = "lib.rs"
@@ -29,8 +29,10 @@ bytes.workspace = true
 chrono.workspace = true
 crc32fast.workspace = true
 crc32c.workspace = true
+crc64fast-nvme.workspace = true
 err-derive.workspace = true
 hex.workspace = true
+hmac.workspace = true
 tracing.workspace = true
 md-5.workspace = true
 pin-project.workspace = true

src/api/s3/api_server.rs

@@ -1,3 +1,4 @@
+use std::borrow::Cow;
 use std::sync::Arc;
 
 use hyper::header;
@@ -117,11 +118,13 @@ impl ApiHandler for S3ApiServer {
 			return handle_post_object(garage, req, bucket_name.unwrap()).await;
 		}
 		if let Endpoint::Options = endpoint {
-			let options_res = handle_options_api(garage, &req, bucket_name).await?;
+			let options_res = handle_options_api(garage, &req, bucket_name)?;
 			return Ok(options_res.map(|_empty_body: EmptyBody| empty_body()));
 		}
 
-		let (req, api_key, content_sha256) = verify_request(&garage, req, "s3").await?;
+		let verified_request = verify_request(&garage, req, "s3")?;
+		let req = verified_request.request;
+		let api_key = verified_request.access_key;
 
 		let bucket_name = match bucket_name {
 			None => {
@@ -134,25 +137,14 @@ impl ApiHandler for S3ApiServer {
 
 		// Special code path for CreateBucket API endpoint
 		if let Endpoint::CreateBucket {} = endpoint {
-			return handle_create_bucket(
-				&garage,
-				req,
-				content_sha256,
-				&api_key.key_id,
-				bucket_name,
-			)
-			.await;
+			return handle_create_bucket(&garage, req, &api_key.key_id, bucket_name).await;
 		}
 
-		let bucket_id = garage
-			.bucket_helper()
-			.resolve_bucket(&bucket_name, &api_key)
-			.await
-			.map_err(pass_helper_error)?;
 		let bucket = garage
 			.bucket_helper()
-			.get_existing_bucket(bucket_id)
-			.await?;
+			.resolve_bucket_fast(&bucket_name, &api_key)
+			.map_err(pass_helper_error)?;
+		let bucket_id = bucket.id;
 		let bucket_params = bucket.state.into_option().unwrap();
 
 		let allowed = match endpoint.authorization_type() {
@@ -179,7 +171,7 @@ impl ApiHandler for S3ApiServer {
 		let resp = match endpoint {
 			Endpoint::HeadObject {
 				key, part_number, ..
-			} => handle_head(ctx, &req, &key, part_number).await,
+			} => handle_head(ctx, &req.map(|_| ()), &key, part_number).await,
 			Endpoint::GetObject {
 				key,
 				part_number,
@@ -199,20 +191,20 @@ impl ApiHandler for S3ApiServer {
 					response_content_type,
 					response_expires,
 				};
-				handle_get(ctx, &req, &key, part_number, overrides).await
+				handle_get(ctx, &req.map(|_| ()), &key, part_number, overrides).await
 			}
 			Endpoint::UploadPart {
 				key,
 				part_number,
 				upload_id,
-			} => handle_put_part(ctx, req, &key, part_number, &upload_id, content_sha256).await,
+			} => handle_put_part(ctx, req, &key, part_number, &upload_id).await,
 			Endpoint::CopyObject { key } => handle_copy(ctx, &req, &key).await,
 			Endpoint::UploadPartCopy {
 				key,
 				part_number,
 				upload_id,
 			} => handle_upload_part_copy(ctx, &req, &key, part_number, &upload_id).await,
-			Endpoint::PutObject { key } => handle_put(ctx, req, &key, content_sha256).await,
+			Endpoint::PutObject { key } => handle_put(ctx, req, &key).await,
 			Endpoint::AbortMultipartUpload { key, upload_id } => {
 				handle_abort_multipart_upload(ctx, &key, &upload_id).await
 			}
@@ -221,7 +213,7 @@ impl ApiHandler for S3ApiServer {
 				handle_create_multipart_upload(ctx, &req, &key).await
 			}
 			Endpoint::CompleteMultipartUpload { key, upload_id } => {
-				handle_complete_multipart_upload(ctx, req, &key, &upload_id, content_sha256).await
+				handle_complete_multipart_upload(ctx, req, &key, &upload_id).await
 			}
 			Endpoint::CreateBucket {} => unreachable!(),
 			Endpoint::HeadBucket {} => {
@@ -324,17 +316,15 @@ impl ApiHandler for S3ApiServer {
 				};
 				handle_list_parts(ctx, req, &query).await
 			}
-			Endpoint::DeleteObjects {} => handle_delete_objects(ctx, req, content_sha256).await,
+			Endpoint::DeleteObjects {} => handle_delete_objects(ctx, req).await,
 			Endpoint::GetBucketWebsite {} => handle_get_website(ctx).await,
-			Endpoint::PutBucketWebsite {} => handle_put_website(ctx, req, content_sha256).await,
+			Endpoint::PutBucketWebsite {} => handle_put_website(ctx, req).await,
 			Endpoint::DeleteBucketWebsite {} => handle_delete_website(ctx).await,
 			Endpoint::GetBucketCors {} => handle_get_cors(ctx).await,
-			Endpoint::PutBucketCors {} => handle_put_cors(ctx, req, content_sha256).await,
+			Endpoint::PutBucketCors {} => handle_put_cors(ctx, req).await,
 			Endpoint::DeleteBucketCors {} => handle_delete_cors(ctx).await,
 			Endpoint::GetBucketLifecycleConfiguration {} => handle_get_lifecycle(ctx).await,
-			Endpoint::PutBucketLifecycleConfiguration {} => {
-				handle_put_lifecycle(ctx, req, content_sha256).await
-			}
+			Endpoint::PutBucketLifecycleConfiguration {} => handle_put_lifecycle(ctx, req).await,
 			Endpoint::DeleteBucketLifecycle {} => handle_delete_lifecycle(ctx).await,
 			endpoint => Err(Error::NotImplemented(endpoint.name().to_owned())),
 		};
@@ -352,8 +342,8 @@ impl ApiHandler for S3ApiServer {
 }
 
 impl ApiEndpoint for S3ApiEndpoint {
-	fn name(&self) -> &'static str {
-		self.endpoint.name()
+	fn name(&self) -> Cow<'static, str> {
+		Cow::Borrowed(self.endpoint.name())
 	}
 
 	fn add_span_attributes(&self, span: SpanRef<'_>) {

src/api/s3/bucket.rs

@@ -1,6 +1,5 @@
 use std::collections::HashMap;
 
-use http_body_util::BodyExt;
 use hyper::{Request, Response, StatusCode};
 
 use garage_model::bucket_alias_table::*;
@@ -10,12 +9,10 @@ use garage_model::key_table::Key;
 use garage_model::permission::BucketKeyPerm;
 use garage_table::util::*;
 use garage_util::crdt::*;
-use garage_util::data::*;
 use garage_util::time::*;
 
 use garage_api_common::common_error::CommonError;
 use garage_api_common::helpers::*;
-use garage_api_common::signature::verify_signed_content;
 
 use crate::api_server::{ReqBody, ResBody};
 use crate::error::*;
@@ -122,15 +119,10 @@ pub async fn handle_list_buckets(
 pub async fn handle_create_bucket(
 	garage: &Garage,
 	req: Request<ReqBody>,
-	content_sha256: Option<Hash>,
 	api_key_id: &String,
 	bucket_name: String,
 ) -> Result<Response<ResBody>, Error> {
-	let body = BodyExt::collect(req.into_body()).await?.to_bytes();
-
-	if let Some(content_sha256) = content_sha256 {
-		verify_signed_content(content_sha256, &body[..])?;
-	}
+	let body = req.into_body().collect().await?;
 
 	let cmd =
 		parse_create_bucket_xml(&body[..]).ok_or_bad_request("Invalid create bucket XML query")?;
@@ -151,21 +143,16 @@ pub async fn handle_create_bucket(
 	let api_key = helper.key().get_existing_key(api_key_id).await?;
 	let key_params = api_key.params().unwrap();
 
-	let existing_bucket = if let Some(Some(bucket_id)) = key_params.local_aliases.get(&bucket_name)
-	{
-		Some(*bucket_id)
-	} else {
-		helper
-			.bucket()
-			.resolve_global_bucket_name(&bucket_name)
-			.await?
-	};
+	let existing_bucket = helper
+		.bucket()
+		.resolve_bucket(&bucket_name, &api_key.key_id)
+		.await?;
 
-	if let Some(bucket_id) = existing_bucket {
+	if let Some(bucket) = existing_bucket {
 		// Check we have write or owner permission on the bucket,
 		// in that case it's fine, return 200 OK, bucket exists;
 		// otherwise return a forbidden error.
-		let kp = api_key.bucket_permissions(&bucket_id);
+		let kp = api_key.bucket_permissions(&bucket.id);
 		if !(kp.allow_write || kp.allow_owner) {
 			return Err(CommonError::BucketAlreadyExists.into());
 		}

src/api/s3/copy.rs

@@ -1,9 +1,9 @@
 use std::pin::Pin;
-use std::time::{Duration, SystemTime, UNIX_EPOCH};
 
 use futures::{stream, stream::Stream, StreamExt, TryStreamExt};
 
 use bytes::Bytes;
+use http::header::HeaderName;
 use hyper::{Request, Response};
 use serde::Serialize;
 
@@ -21,16 +21,25 @@ use garage_model::s3::object_table::*;
 use garage_model::s3::version_table::*;
 
 use garage_api_common::helpers::*;
+use garage_api_common::signature::checksum::*;
 
 use crate::api_server::{ReqBody, ResBody};
-use crate::checksum::*;
-use crate::encryption::EncryptionParams;
+use crate::encryption::{EncryptionParams, OekDerivationInfo};
 use crate::error::*;
-use crate::get::full_object_byte_stream;
+use crate::get::{full_object_byte_stream, PreconditionHeaders};
 use crate::multipart;
-use crate::put::{get_headers, save_stream, ChecksumMode, SaveStreamResult};
+use crate::put::{extract_metadata_headers, save_stream, ChecksumMode, SaveStreamResult};
 use crate::xml::{self as s3_xml, xmlns_tag};
 
+pub const X_AMZ_COPY_SOURCE_IF_MATCH: HeaderName =
+	HeaderName::from_static("x-amz-copy-source-if-match");
+pub const X_AMZ_COPY_SOURCE_IF_NONE_MATCH: HeaderName =
+	HeaderName::from_static("x-amz-copy-source-if-none-match");
+pub const X_AMZ_COPY_SOURCE_IF_MODIFIED_SINCE: HeaderName =
+	HeaderName::from_static("x-amz-copy-source-if-modified-since");
+pub const X_AMZ_COPY_SOURCE_IF_UNMODIFIED_SINCE: HeaderName =
+	HeaderName::from_static("x-amz-copy-source-if-unmodified-since");
+
 // -------- CopyObject ---------
 
 pub async fn handle_copy(
@@ -38,7 +47,7 @@ pub async fn handle_copy(
 	req: &Request<ReqBody>,
 	dest_key: &str,
 ) -> Result<Response<ResBody>, Error> {
-	let copy_precondition = CopyPreconditionHeaders::parse(req)?;
+	let copy_precondition = PreconditionHeaders::parse_copy_source(req)?;
 
 	let checksum_algorithm = request_checksum_algorithm(req.headers())?;
 
@@ -48,7 +57,7 @@ pub async fn handle_copy(
 		extract_source_info(&source_object)?;
 
 	// Check precondition, e.g. x-amz-copy-source-if-match
-	copy_precondition.check(source_version, &source_version_meta.etag)?;
+	copy_precondition.check_copy_source(source_version, &source_version_meta.etag)?;
 
 	// Determine encryption parameters
 	let (source_encryption, source_object_meta_inner) =
@@ -56,8 +65,18 @@ pub async fn handle_copy(
 			&ctx.garage,
 			req.headers(),
 			&source_version_meta.encryption,
+			OekDerivationInfo::for_object(&source_object, source_version),
 		)?;
-	let dest_encryption = EncryptionParams::new_from_headers(&ctx.garage, req.headers())?;
+	let dest_uuid = gen_uuid();
+	let dest_encryption = EncryptionParams::new_from_headers(
+		&ctx.garage,
+		req.headers(),
+		OekDerivationInfo {
+			bucket_id: ctx.bucket_id,
+			version_id: dest_uuid,
+			object_key: dest_key,
+		},
+	)?;
 
 	// Extract source checksum info before source_object_meta_inner is consumed
 	let source_checksum = source_object_meta_inner.checksum;
@@ -73,7 +92,7 @@ pub async fn handle_copy(
 	let dest_object_meta = ObjectVersionMetaInner {
 		headers: match req.headers().get("x-amz-metadata-directive") {
 			Some(v) if v == hyper::header::HeaderValue::from_static("REPLACE") => {
-				get_headers(req.headers())?
+				extract_metadata_headers(req.headers())?
 			}
 			_ => source_object_meta_inner.into_owned().headers,
 		},
@@ -106,6 +125,7 @@ pub async fn handle_copy(
 		handle_copy_metaonly(
 			ctx,
 			dest_key,
+			dest_uuid,
 			dest_object_meta,
 			dest_encryption,
 			source_version,
@@ -129,6 +149,7 @@ pub async fn handle_copy(
 		handle_copy_reencrypt(
 			ctx,
 			dest_key,
+			dest_uuid,
 			dest_object_meta,
 			dest_encryption,
 			source_version,
@@ -160,6 +181,7 @@ pub async fn handle_copy(
 async fn handle_copy_metaonly(
 	ctx: ReqCtx,
 	dest_key: &str,
+	dest_uuid: Uuid,
 	dest_object_meta: ObjectVersionMetaInner,
 	dest_encryption: EncryptionParams,
 	source_version: &ObjectVersion,
@@ -173,7 +195,6 @@ async fn handle_copy_metaonly(
 	} = ctx;
 
 	// Generate parameters for copied object
-	let new_uuid = gen_uuid();
 	let new_timestamp = now_msec();
 
 	let new_meta = ObjectVersionMeta {
@@ -183,7 +204,7 @@ async fn handle_copy_metaonly(
 	};
 
 	let res = SaveStreamResult {
-		version_uuid: new_uuid,
+		version_uuid: dest_uuid,
 		version_timestamp: new_timestamp,
 		etag: new_meta.etag.clone(),
 	};
@@ -195,7 +216,7 @@ async fn handle_copy_metaonly(
 			// bytes is either plaintext before&after or encrypted with the
 			// same keys, so it's ok to just copy it as is
 			let dest_object_version = ObjectVersion {
-				uuid: new_uuid,
+				uuid: dest_uuid,
 				timestamp: new_timestamp,
 				state: ObjectVersionState::Complete(ObjectVersionData::Inline(
 					new_meta,
@@ -221,7 +242,7 @@ async fn handle_copy_metaonly(
 			// This holds a reference to the object in the Version table
 			// so that it won't be deleted, e.g. by repair_versions.
 			let tmp_dest_object_version = ObjectVersion {
-				uuid: new_uuid,
+				uuid: dest_uuid,
 				timestamp: new_timestamp,
 				state: ObjectVersionState::Uploading {
 					encryption: new_meta.encryption.clone(),
@@ -241,7 +262,7 @@ async fn handle_copy_metaonly(
 			// marked as deleted (they are marked as deleted only if the Version
 			// doesn't exist or is marked as deleted).
 			let mut dest_version = Version::new(
-				new_uuid,
+				dest_uuid,
 				VersionBacklink::Object {
 					bucket_id: dest_bucket_id,
 					key: dest_key.to_string(),
@@ -260,7 +281,7 @@ async fn handle_copy_metaonly(
 				.iter()
 				.map(|b| BlockRef {
 					block: b.1.hash,
-					version: new_uuid,
+					version: dest_uuid,
 					deleted: false.into(),
 				})
 				.collect::<Vec<_>>();
@@ -276,7 +297,7 @@ async fn handle_copy_metaonly(
 			// with the stuff before, the block's reference counts could be decremented before
 			// they are incremented again for the new version, leading to data being deleted.
 			let dest_object_version = ObjectVersion {
-				uuid: new_uuid,
+				uuid: dest_uuid,
 				timestamp: new_timestamp,
 				state: ObjectVersionState::Complete(ObjectVersionData::FirstBlock(
 					new_meta,
@@ -298,6 +319,7 @@ async fn handle_copy_metaonly(
 async fn handle_copy_reencrypt(
 	ctx: ReqCtx,
 	dest_key: &str,
+	dest_uuid: Uuid,
 	dest_object_meta: ObjectVersionMetaInner,
 	dest_encryption: EncryptionParams,
 	source_version: &ObjectVersion,
@@ -317,6 +339,7 @@ async fn handle_copy_reencrypt(
 
 	save_stream(
 		&ctx,
+		dest_uuid,
 		dest_object_meta,
 		dest_encryption,
 		source_stream.map_err(|e| Error::from(GarageError::from(e))),
@@ -335,12 +358,12 @@ pub async fn handle_upload_part_copy(
 	part_number: u64,
 	upload_id: &str,
 ) -> Result<Response<ResBody>, Error> {
-	let copy_precondition = CopyPreconditionHeaders::parse(req)?;
+	let copy_precondition = PreconditionHeaders::parse_copy_source(req)?;
 
 	let dest_upload_id = multipart::decode_upload_id(upload_id)?;
 
 	let dest_key = dest_key.to_string();
-	let (source_object, (_, dest_version, mut dest_mpu)) = futures::try_join!(
+	let (source_object, (dest_object, dest_version, mut dest_mpu)) = futures::try_join!(
 		get_copy_source(&ctx, req),
 		multipart::get_upload(&ctx, &dest_key, &dest_upload_id)
 	)?;
@@ -351,14 +374,17 @@ pub async fn handle_upload_part_copy(
 		extract_source_info(&source_object)?;
 
 	// Check precondition on source, e.g. x-amz-copy-source-if-match
-	copy_precondition.check(source_object_version, &source_version_meta.etag)?;
+	copy_precondition.check_copy_source(source_object_version, &source_version_meta.etag)?;
 
 	// Determine encryption parameters
 	let (source_encryption, _) = EncryptionParams::check_decrypt_for_copy_source(
 		&garage,
 		req.headers(),
 		&source_version_meta.encryption,
+		OekDerivationInfo::for_object(&source_object, source_object_version),
 	)?;
+
+	let dest_oek_params = OekDerivationInfo::for_object(&dest_object, &dest_version);
 	let (dest_object_encryption, dest_object_checksum_algorithm) = match dest_version.state {
 		ObjectVersionState::Uploading {
 			encryption,
@@ -367,8 +393,12 @@ pub async fn handle_upload_part_copy(
 		} => (encryption, checksum_algorithm),
 		_ => unreachable!(),
 	};
-	let (dest_encryption, _) =
-		EncryptionParams::check_decrypt(&garage, req.headers(), &dest_object_encryption)?;
+	let (dest_encryption, _) = EncryptionParams::check_decrypt(
+		&garage,
+		req.headers(),
+		&dest_object_encryption,
+		dest_oek_params,
+	)?;
 	let same_encryption = EncryptionParams::is_same(&source_encryption, &dest_encryption);
 
 	// Check source range is valid
@@ -653,16 +683,15 @@ async fn get_copy_source(ctx: &ReqCtx, req: &Request<ReqBody>) -> Result<Object,
 	let copy_source = percent_encoding::percent_decode_str(copy_source).decode_utf8()?;
 
 	let (source_bucket, source_key) = parse_bucket_key(&copy_source, None)?;
-	let source_bucket_id = garage
+	let source_bucket = garage
 		.bucket_helper()
-		.resolve_bucket(&source_bucket.to_string(), api_key)
-		.await
+		.resolve_bucket_fast(&source_bucket.to_string(), api_key)
 		.map_err(pass_helper_error)?;
 
-	if !api_key.allow_read(&source_bucket_id) {
+	if !api_key.allow_read(&source_bucket.id) {
 		return Err(Error::forbidden(format!(
-			"Reading from bucket {} not allowed for this key",
-			source_bucket
+			"Reading from bucket {:?} not allowed for this key",
+			source_bucket.id
 		)));
 	}
 
@@ -670,7 +699,7 @@ async fn get_copy_source(ctx: &ReqCtx, req: &Request<ReqBody>) -> Result<Object,
 
 	let source_object = garage
 		.object_table
-		.get(&source_bucket_id, &source_key.to_string())
+		.get(&source_bucket.id, &source_key.to_string())
 		.await?
 		.ok_or(Error::NoSuchKey)?;
 
@@ -703,97 +732,6 @@ fn extract_source_info(
 	Ok((source_version, source_version_data, source_version_meta))
 }
 
-struct CopyPreconditionHeaders {
-	copy_source_if_match: Option<Vec<String>>,
-	copy_source_if_modified_since: Option<SystemTime>,
-	copy_source_if_none_match: Option<Vec<String>>,
-	copy_source_if_unmodified_since: Option<SystemTime>,
-}
-
-impl CopyPreconditionHeaders {
-	fn parse(req: &Request<ReqBody>) -> Result<Self, Error> {
-		Ok(Self {
-			copy_source_if_match: req
-				.headers()
-				.get("x-amz-copy-source-if-match")
-				.map(|x| x.to_str())
-				.transpose()?
-				.map(|x| {
-					x.split(',')
-						.map(|m| m.trim().trim_matches('"').to_string())
-						.collect::<Vec<_>>()
-				}),
-			copy_source_if_modified_since: req
-				.headers()
-				.get("x-amz-copy-source-if-modified-since")
-				.map(|x| x.to_str())
-				.transpose()?
-				.map(httpdate::parse_http_date)
-				.transpose()
-				.ok_or_bad_request("Invalid date in x-amz-copy-source-if-modified-since")?,
-			copy_source_if_none_match: req
-				.headers()
-				.get("x-amz-copy-source-if-none-match")
-				.map(|x| x.to_str())
-				.transpose()?
-				.map(|x| {
-					x.split(',')
-						.map(|m| m.trim().trim_matches('"').to_string())
-						.collect::<Vec<_>>()
-				}),
-			copy_source_if_unmodified_since: req
-				.headers()
-				.get("x-amz-copy-source-if-unmodified-since")
-				.map(|x| x.to_str())
-				.transpose()?
-				.map(httpdate::parse_http_date)
-				.transpose()
-				.ok_or_bad_request("Invalid date in x-amz-copy-source-if-unmodified-since")?,
-		})
-	}
-
-	fn check(&self, v: &ObjectVersion, etag: &str) -> Result<(), Error> {
-		let v_date = UNIX_EPOCH + Duration::from_millis(v.timestamp);
-
-		let ok = match (
-			&self.copy_source_if_match,
-			&self.copy_source_if_unmodified_since,
-			&self.copy_source_if_none_match,
-			&self.copy_source_if_modified_since,
-		) {
-			// TODO I'm not sure all of the conditions are evaluated correctly here
-
-			// If we have both if-match and if-unmodified-since,
-			// basically we don't care about if-unmodified-since,
-			// because in the spec it says that if if-match evaluates to
-			// true but if-unmodified-since evaluates to false,
-			// the copy is still done.
-			(Some(im), _, None, None) => im.iter().any(|x| x == etag || x == "*"),
-			(None, Some(ius), None, None) => v_date <= *ius,
-
-			// If we have both if-none-match and if-modified-since,
-			// then both of the two conditions must evaluate to true
-			(None, None, Some(inm), Some(ims)) => {
-				!inm.iter().any(|x| x == etag || x == "*") && v_date > *ims
-			}
-			(None, None, Some(inm), None) => !inm.iter().any(|x| x == etag || x == "*"),
-			(None, None, None, Some(ims)) => v_date > *ims,
-			(None, None, None, None) => true,
-			_ => {
-				return Err(Error::bad_request(
-					"Invalid combination of x-amz-copy-source-if-xxxxx headers",
-				))
-			}
-		};
-
-		if ok {
-			Ok(())
-		} else {
-			Err(Error::PreconditionFailed)
-		}
-	}
-}
-
 type BlockStreamItemOk = (Bytes, Option<Hash>);
 type BlockStreamItem = Result<BlockStreamItemOk, garage_util::error::Error>;
 

src/api/s3/cors.rs

@@ -2,15 +2,11 @@ use quick_xml::de::from_reader;
 
 use hyper::{header::HeaderName, Method, Request, Response, StatusCode};
 
-use http_body_util::BodyExt;
-
 use serde::{Deserialize, Serialize};
 
 use garage_model::bucket_table::{Bucket, CorsRule as GarageCorsRule};
-use garage_util::data::*;
 
 use garage_api_common::helpers::*;
-use garage_api_common::signature::verify_signed_content;
 
 use crate::api_server::{ReqBody, ResBody};
 use crate::error::*;
@@ -59,7 +55,6 @@ pub async fn handle_delete_cors(ctx: ReqCtx) -> Result<Response<ResBody>, Error>
 pub async fn handle_put_cors(
 	ctx: ReqCtx,
 	req: Request<ReqBody>,
-	content_sha256: Option<Hash>,
 ) -> Result<Response<ResBody>, Error> {
 	let ReqCtx {
 		garage,
@@ -68,11 +63,7 @@ pub async fn handle_put_cors(
 		..
 	} = ctx;
 
-	let body = BodyExt::collect(req.into_body()).await?.to_bytes();
-
-	if let Some(content_sha256) = content_sha256 {
-		verify_signed_content(content_sha256, &body[..])?;
-	}
+	let body = req.into_body().collect().await?;
 
 	let conf: CorsConfiguration = from_reader(&body as &[u8])?;
 	conf.validate()?;

src/api/s3/delete.rs

@@ -1,4 +1,3 @@
-use http_body_util::BodyExt;
 use hyper::{Request, Response, StatusCode};
 
 use garage_util::data::*;
@@ -6,7 +5,6 @@ use garage_util::data::*;
 use garage_model::s3::object_table::*;
 
 use garage_api_common::helpers::*;
-use garage_api_common::signature::verify_signed_content;
 
 use crate::api_server::{ReqBody, ResBody};
 use crate::error::*;
@@ -68,13 +66,8 @@ pub async fn handle_delete(ctx: ReqCtx, key: &str) -> Result<Response<ResBody>,
 pub async fn handle_delete_objects(
 	ctx: ReqCtx,
 	req: Request<ReqBody>,
-	content_sha256: Option<Hash>,
 ) -> Result<Response<ResBody>, Error> {
-	let body = BodyExt::collect(req.into_body()).await?.to_bytes();
-
-	if let Some(content_sha256) = content_sha256 {
-		verify_signed_content(content_sha256, &body[..])?;
-	}
+	let body = req.into_body().collect().await?;
 
 	let cmd_xml = roxmltree::Document::parse(std::str::from_utf8(&body)?)?;
 	let cmd = parse_delete_objects_xml(&cmd_xml).ok_or_bad_request("Invalid delete XML query")?;

src/api/s3/encryption.rs

@@ -11,6 +11,7 @@ use aes_gcm::{
 };
 use base64::prelude::*;
 use bytes::Bytes;
+use sha2::Sha256;
 
 use futures::stream::Stream;
 use futures::task;
@@ -21,16 +22,16 @@ use http::header::{HeaderMap, HeaderName, HeaderValue};
 use garage_net::bytes_buf::BytesBuf;
 use garage_net::stream::{stream_asyncread, ByteStream};
 use garage_rpc::rpc_helper::OrderTag;
-use garage_util::data::Hash;
+use garage_util::data::{Hash, Uuid};
 use garage_util::error::Error as GarageError;
 use garage_util::migrate::Migrate;
 
 use garage_model::garage::Garage;
-use garage_model::s3::object_table::{ObjectVersionEncryption, ObjectVersionMetaInner};
+use garage_model::s3::object_table::*;
 
 use garage_api_common::common_error::*;
+use garage_api_common::signature::checksum::Md5Checksum;
 
-use crate::checksum::Md5Checksum;
 use crate::error::Error;
 
 const X_AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_ALGORITHM: HeaderName =
@@ -64,32 +65,45 @@ const STREAM_ENC_CYPER_CHUNK_SIZE: usize = STREAM_ENC_PLAIN_CHUNK_SIZE + 16;
 pub enum EncryptionParams {
 	Plaintext,
 	SseC {
+		/// the value of x-amz-server-side-encryption-customer-key
 		client_key: Key<Aes256Gcm>,
+		/// the value of x-amz-server-side-encryption-customer-key-md5
 		client_key_md5: Md5Output,
+		/// the object encryption key, for uploads created in garage v2+
+		object_key: Option<Key<Aes256Gcm>>,
+		/// the compression level used for compressing data blocks
 		compression_level: Option<i32>,
 	},
 }
 
+#[derive(Clone, Copy)]
+pub struct OekDerivationInfo<'a> {
+	pub bucket_id: Uuid,
+	pub version_id: Uuid,
+	pub object_key: &'a str,
+}
+
 impl EncryptionParams {
 	pub fn is_encrypted(&self) -> bool {
 		!matches!(self, Self::Plaintext)
 	}
 
 	pub fn is_same(a: &Self, b: &Self) -> bool {
-		let relevant_info = |x: &Self| match x {
-			Self::Plaintext => None,
-			Self::SseC {
-				client_key,
-				compression_level,
-				..
-			} => Some((*client_key, compression_level.is_some())),
-		};
-		relevant_info(a) == relevant_info(b)
+		// This function is used in CopyObject and UploadPartCopy to determine
+		// whether the object must be re-encrypted. If this returns true,
+		// data blocks are reused as-is. Since Garage v2, we are using
+		// object-specific encryption keys, so we know that if both source
+		// and destination are encrypted, it can't be with the same key.
+		match (a, b) {
+			(Self::Plaintext, Self::Plaintext) => true,
+			_ => false,
+		}
 	}
 
 	pub fn new_from_headers(
 		garage: &Garage,
 		headers: &HeaderMap,
+		oek_info: OekDerivationInfo<'_>,
 	) -> Result<EncryptionParams, Error> {
 		let key = parse_request_headers(
 			headers,
@@ -101,6 +115,7 @@ impl EncryptionParams {
 			Some((client_key, client_key_md5)) => Ok(EncryptionParams::SseC {
 				client_key,
 				client_key_md5,
+				object_key: Some(oek_info.derive_oek(&client_key)),
 				compression_level: garage.config.compression_level,
 			}),
 			None => Ok(EncryptionParams::Plaintext),
@@ -126,6 +141,7 @@ impl EncryptionParams {
 		garage: &Garage,
 		headers: &HeaderMap,
 		obj_enc: &'a ObjectVersionEncryption,
+		oek_info: OekDerivationInfo<'_>,
 	) -> Result<(Self, Cow<'a, ObjectVersionMetaInner>), Error> {
 		let key = parse_request_headers(
 			headers,
@@ -133,13 +149,14 @@ impl EncryptionParams {
 			&X_AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY,
 			&X_AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY_MD5,
 		)?;
-		Self::check_decrypt_common(garage, key, obj_enc)
+		Self::check_decrypt_common(garage, key, obj_enc, oek_info)
 	}
 
 	pub fn check_decrypt_for_copy_source<'a>(
 		garage: &Garage,
 		headers: &HeaderMap,
 		obj_enc: &'a ObjectVersionEncryption,
+		oek_info: OekDerivationInfo<'_>,
 	) -> Result<(Self, Cow<'a, ObjectVersionMetaInner>), Error> {
 		let key = parse_request_headers(
 			headers,
@@ -147,22 +164,32 @@ impl EncryptionParams {
 			&X_AMZ_COPY_SOURCE_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY,
 			&X_AMZ_COPY_SOURCE_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY_MD5,
 		)?;
-		Self::check_decrypt_common(garage, key, obj_enc)
+		Self::check_decrypt_common(garage, key, obj_enc, oek_info)
 	}
 
 	fn check_decrypt_common<'a>(
 		garage: &Garage,
 		key: Option<(Key<Aes256Gcm>, Md5Output)>,
 		obj_enc: &'a ObjectVersionEncryption,
+		oek_info: OekDerivationInfo<'_>,
 	) -> Result<(Self, Cow<'a, ObjectVersionMetaInner>), Error> {
 		match (key, &obj_enc) {
 			(
 				Some((client_key, client_key_md5)),
-				ObjectVersionEncryption::SseC { inner, compressed },
+				ObjectVersionEncryption::SseC {
+					inner,
+					compressed,
+					use_oek,
+				},
 			) => {
 				let enc = Self::SseC {
 					client_key,
 					client_key_md5,
+					object_key: if *use_oek {
+						Some(oek_info.derive_oek(&client_key))
+					} else {
+						None
+					},
 					compression_level: if *compressed {
 						Some(garage.config.compression_level.unwrap_or(1))
 					} else {
@@ -193,13 +220,16 @@ impl EncryptionParams {
 	) -> Result<ObjectVersionEncryption, Error> {
 		match self {
 			Self::SseC {
-				compression_level, ..
+				compression_level,
+				object_key,
+				..
 			} => {
 				let plaintext = meta.encode().map_err(GarageError::from)?;
 				let ciphertext = self.encrypt_blob(&plaintext)?;
 				Ok(ObjectVersionEncryption::SseC {
 					inner: ciphertext.into_owned(),
 					compressed: compression_level.is_some(),
+					use_oek: object_key.is_some(),
 				})
 			}
 			Self::Plaintext => Ok(ObjectVersionEncryption::Plaintext { inner: meta }),
@@ -228,24 +258,37 @@ impl EncryptionParams {
 	// This is used for encrypting object metadata and inlined data for small objects.
 	// This does not compress anything.
 
-	pub fn encrypt_blob<'a>(&self, blob: &'a [u8]) -> Result<Cow<'a, [u8]>, Error> {
+	fn cipher(&self) -> Option<Aes256Gcm> {
 		match self {
-			Self::SseC { client_key, .. } => {
-				let cipher = Aes256Gcm::new(&client_key);
+			Self::SseC {
+				object_key: Some(oek),
+				..
+			} => Some(Aes256Gcm::new(&oek)),
+			Self::SseC {
+				client_key,
+				object_key: None,
+				..
+			} => Some(Aes256Gcm::new(&client_key)),
+			Self::Plaintext => None,
+		}
+	}
+
+	pub fn encrypt_blob<'a>(&self, blob: &'a [u8]) -> Result<Cow<'a, [u8]>, Error> {
+		match self.cipher() {
+			Some(cipher) => {
 				let nonce = Aes256Gcm::generate_nonce(&mut OsRng);
 				let ciphertext = cipher
 					.encrypt(&nonce, blob)
 					.ok_or_internal_error("Encryption failed")?;
 				Ok(Cow::Owned([nonce.to_vec(), ciphertext].concat()))
 			}
-			Self::Plaintext => Ok(Cow::Borrowed(blob)),
+			None => Ok(Cow::Borrowed(blob)),
 		}
 	}
 
 	pub fn decrypt_blob<'a>(&self, blob: &'a [u8]) -> Result<Cow<'a, [u8]>, Error> {
-		match self {
-			Self::SseC { client_key, .. } => {
-				let cipher = Aes256Gcm::new(&client_key);
+		match self.cipher() {
+			Some(cipher) => {
 				let nonce_size = <Aes256Gcm as AeadCore>::NonceSize::to_usize();
 				let nonce = Nonce::from_slice(
 					blob.get(..nonce_size)
@@ -258,7 +301,7 @@ impl EncryptionParams {
 					)?;
 				Ok(Cow::Owned(plaintext))
 			}
-			Self::Plaintext => Ok(Cow::Borrowed(blob)),
+			None => Ok(Cow::Borrowed(blob)),
 		}
 	}
 
@@ -284,10 +327,12 @@ impl EncryptionParams {
 			Self::Plaintext => stream,
 			Self::SseC {
 				client_key,
+				object_key,
 				compression_level,
 				..
 			} => {
-				let plaintext = DecryptStream::new(stream, *client_key);
+				let key = object_key.as_ref().unwrap_or(client_key);
+				let plaintext = DecryptStream::new(stream, *key);
 				if compression_level.is_some() {
 					let reader = stream_asyncread(Box::pin(plaintext));
 					let reader = BufReader::new(reader);
@@ -307,9 +352,12 @@ impl EncryptionParams {
 			Self::Plaintext => Ok(block),
 			Self::SseC {
 				client_key,
+				object_key,
 				compression_level,
 				..
 			} => {
+				let key = object_key.as_ref().unwrap_or(client_key);
+
 				let block = if let Some(level) = compression_level {
 					Cow::Owned(
 						garage_block::zstd_encode(block.as_ref(), *level)
@@ -325,7 +373,7 @@ impl EncryptionParams {
 				OsRng.fill_bytes(&mut nonce);
 				ret.extend_from_slice(nonce.as_slice());
 
-				let mut cipher = EncryptorLE31::<Aes256Gcm>::new(&client_key, &nonce);
+				let mut cipher = EncryptorLE31::<Aes256Gcm>::new(key, &nonce);
 				let mut iter = block.chunks(STREAM_ENC_PLAIN_CHUNK_SIZE).peekable();
 
 				if iter.peek().is_none() {
@@ -361,6 +409,13 @@ impl EncryptionParams {
 	}
 }
 
+pub fn has_encryption_header(headers: &HeaderMap) -> bool {
+	match headers.get(X_AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_ALGORITHM) {
+		Some(h) => h.as_bytes() == CUSTOMER_ALGORITHM_AES256,
+		None => false,
+	}
+}
+
 fn parse_request_headers(
 	headers: &HeaderMap,
 	alg_header: &HeaderName,
@@ -420,6 +475,30 @@ fn parse_request_headers(
 	}
 }
 
+impl<'a> OekDerivationInfo<'a> {
+	pub fn for_object<'b>(object: &'a Object, version: &'b ObjectVersion) -> Self {
+		Self {
+			bucket_id: object.bucket_id,
+			version_id: version.uuid,
+			object_key: &object.key,
+		}
+	}
+
+	fn derive_oek(&self, client_key: &Key<Aes256Gcm>) -> Key<Aes256Gcm> {
+		use hmac::{Hmac, Mac};
+
+		// info = bucket_id + object_name + version_uuid + "garage-object-encryption-key"
+		// oek = hmac_sha256(ssec_key, info)
+		let mut hmac = <Hmac<Sha256> as Mac>::new_from_slice(client_key.as_slice())
+			.expect("create hmac-sha256");
+		hmac.update(b"garage-object-encryption-key");
+		hmac.update(self.bucket_id.as_slice());
+		hmac.update(self.version_id.as_slice());
+		hmac.update(self.object_key.as_bytes());
+		hmac.finalize().into_bytes()
+	}
+}
+
 // ---- encrypt & decrypt streams ----
 
 #[pin_project::pin_project]
@@ -569,6 +648,7 @@ mod tests {
 		let enc = EncryptionParams::SseC {
 			client_key: Aes256Gcm::generate_key(&mut OsRng),
 			client_key_md5: Default::default(), // not needed
+			object_key: Some(Aes256Gcm::generate_key(&mut OsRng)),
 			compression_level,
 		};
 

src/api/s3/error.rs

@@ -80,7 +80,7 @@ pub enum Error {
 	#[error(display = "Invalid encryption algorithm: {:?}, should be AES256", _0)]
 	InvalidEncryptionAlgorithm(String),
 
-	/// The client sent invalid XML data
+	/// The provided digest (checksum) value was invalid
 	#[error(display = "Invalid digest: {}", _0)]
 	InvalidDigest(String),
 
@@ -119,6 +119,7 @@ impl From<SignatureError> for Error {
 				Self::AuthorizationHeaderMalformed(c)
 			}
 			SignatureError::InvalidUtf8Str(i) => Self::InvalidUtf8Str(i),
+			SignatureError::InvalidDigest(d) => Self::InvalidDigest(d),
 		}
 	}
 }

src/api/s3/get.rs

@@ -2,17 +2,17 @@
 use std::collections::BTreeMap;
 use std::convert::TryInto;
 use std::sync::Arc;
-use std::time::{Duration, UNIX_EPOCH};
+use std::time::{Duration, SystemTime, UNIX_EPOCH};
 
 use bytes::Bytes;
 use futures::future;
 use futures::stream::{self, Stream, StreamExt};
 use http::header::{
-	ACCEPT_RANGES, CACHE_CONTROL, CONTENT_DISPOSITION, CONTENT_ENCODING, CONTENT_LANGUAGE,
-	CONTENT_LENGTH, CONTENT_RANGE, CONTENT_TYPE, ETAG, EXPIRES, IF_MODIFIED_SINCE, IF_NONE_MATCH,
-	LAST_MODIFIED, RANGE,
+	HeaderMap, HeaderName, ACCEPT_RANGES, CACHE_CONTROL, CONTENT_DISPOSITION, CONTENT_ENCODING,
+	CONTENT_LANGUAGE, CONTENT_LENGTH, CONTENT_RANGE, CONTENT_TYPE, ETAG, EXPIRES, IF_MATCH,
+	IF_MODIFIED_SINCE, IF_NONE_MATCH, IF_UNMODIFIED_SINCE, LAST_MODIFIED, RANGE,
 };
-use hyper::{body::Body, Request, Response, StatusCode};
+use hyper::{Request, Response, StatusCode};
 use tokio::sync::mpsc;
 
 use garage_net::stream::ByteStream;
@@ -26,13 +26,14 @@ use garage_model::s3::object_table::*;
 use garage_model::s3::version_table::*;
 
 use garage_api_common::helpers::*;
+use garage_api_common::signature::checksum::{add_checksum_response_headers, X_AMZ_CHECKSUM_MODE};
 
 use crate::api_server::ResBody;
-use crate::checksum::{add_checksum_response_headers, X_AMZ_CHECKSUM_MODE};
-use crate::encryption::EncryptionParams;
+use crate::copy::*;
+use crate::encryption::{EncryptionParams, OekDerivationInfo};
 use crate::error::*;
 
-const X_AMZ_MP_PARTS_COUNT: &str = "x-amz-mp-parts-count";
+const X_AMZ_MP_PARTS_COUNT: HeaderName = HeaderName::from_static("x-amz-mp-parts-count");
 
 #[derive(Default)]
 pub struct GetObjectOverrides {
@@ -115,49 +116,29 @@ fn getobject_override_headers(
 	Ok(())
 }
 
-fn try_answer_cached(
+fn handle_http_precondition(
 	version: &ObjectVersion,
 	version_meta: &ObjectVersionMeta,
-	req: &Request<impl Body>,
-) -> Option<Response<ResBody>> {
-	// <trinity> It is possible, and is even usually the case, [that both If-None-Match and
-	// If-Modified-Since] are present in a request. In this situation If-None-Match takes
-	// precedence and If-Modified-Since is ignored (as per 6.Precedence from rfc7232). The rational
-	// being that etag based matching is more accurate, it has no issue with sub-second precision
-	// for instance (in case of very fast updates)
-	let cached = if let Some(none_match) = req.headers().get(IF_NONE_MATCH) {
-		let none_match = none_match.to_str().ok()?;
-		let expected = format!("\"{}\"", version_meta.etag);
-		let found = none_match
-			.split(',')
-			.map(str::trim)
-			.any(|etag| etag == expected || etag == "\"*\"");
-		found
-	} else if let Some(modified_since) = req.headers().get(IF_MODIFIED_SINCE) {
-		let modified_since = modified_since.to_str().ok()?;
-		let client_date = httpdate::parse_http_date(modified_since).ok()?;
-		let server_date = UNIX_EPOCH + Duration::from_millis(version.timestamp);
-		client_date >= server_date
-	} else {
-		false
-	};
+	req: &Request<()>,
+) -> Result<Option<Response<ResBody>>, Error> {
+	let precondition_headers = PreconditionHeaders::parse(req)?;
 
-	if cached {
-		Some(
+	if let Some(status_code) = precondition_headers.check(&version, &version_meta.etag)? {
+		Ok(Some(
 			Response::builder()
-				.status(StatusCode::NOT_MODIFIED)
+				.status(status_code)
 				.body(empty_body())
 				.unwrap(),
-		)
+		))
 	} else {
-		None
+		Ok(None)
 	}
 }
 
 /// Handle HEAD request
 pub async fn handle_head(
 	ctx: ReqCtx,
-	req: &Request<impl Body>,
+	req: &Request<()>,
 	key: &str,
 	part_number: Option<u64>,
 ) -> Result<Response<ResBody>, Error> {
@@ -167,7 +148,7 @@ pub async fn handle_head(
 /// Handle HEAD request for website
 pub async fn handle_head_without_ctx(
 	garage: Arc<Garage>,
-	req: &Request<impl Body>,
+	req: &Request<()>,
 	bucket_id: Uuid,
 	key: &str,
 	part_number: Option<u64>,
@@ -196,12 +177,16 @@ pub async fn handle_head_without_ctx(
 		_ => unreachable!(),
 	};
 
-	if let Some(cached) = try_answer_cached(object_version, version_meta, req) {
-		return Ok(cached);
+	if let Some(res) = handle_http_precondition(object_version, version_meta, req)? {
+		return Ok(res);
 	}
 
-	let (encryption, headers) =
-		EncryptionParams::check_decrypt(&garage, req.headers(), &version_meta.encryption)?;
+	let (encryption, headers) = EncryptionParams::check_decrypt(
+		&garage,
+		req.headers(),
+		&version_meta.encryption,
+		OekDerivationInfo::for_object(&object, object_version),
+	)?;
 
 	let checksum_mode = checksum_mode(&req);
 
@@ -278,7 +263,7 @@ pub async fn handle_head_without_ctx(
 /// Handle GET request
 pub async fn handle_get(
 	ctx: ReqCtx,
-	req: &Request<impl Body>,
+	req: &Request<()>,
 	key: &str,
 	part_number: Option<u64>,
 	overrides: GetObjectOverrides,
@@ -289,7 +274,7 @@ pub async fn handle_get(
 /// Handle GET request
 pub async fn handle_get_without_ctx(
 	garage: Arc<Garage>,
-	req: &Request<impl Body>,
+	req: &Request<()>,
 	bucket_id: Uuid,
 	key: &str,
 	part_number: Option<u64>,
@@ -318,12 +303,16 @@ pub async fn handle_get_without_ctx(
 		ObjectVersionData::FirstBlock(meta, _) => meta,
 	};
 
-	if let Some(cached) = try_answer_cached(last_v, last_v_meta, req) {
-		return Ok(cached);
+	if let Some(res) = handle_http_precondition(last_v, last_v_meta, req)? {
+		return Ok(res);
 	}
 
-	let (enc, headers) =
-		EncryptionParams::check_decrypt(&garage, req.headers(), &last_v_meta.encryption)?;
+	let (enc, headers) = EncryptionParams::check_decrypt(
+		&garage,
+		req.headers(),
+		&last_v_meta.encryption,
+		OekDerivationInfo::for_object(&object, last_v),
+	)?;
 
 	let checksum_mode = checksum_mode(&req);
 
@@ -340,7 +329,12 @@ pub async fn handle_get_without_ctx(
 				enc,
 				&headers,
 				pn,
-				checksum_mode,
+				ChecksumMode {
+					// TODO: for multipart uploads, checksums of each part should be stored
+					// so that we can return the corresponding checksum here
+					// https://docs.aws.amazon.com/AmazonS3/latest/userguide/checking-object-integrity.html
+					enabled: false,
+				},
 			)
 			.await
 		}
@@ -354,7 +348,12 @@ pub async fn handle_get_without_ctx(
 				&headers,
 				range.start,
 				range.start + range.length,
-				checksum_mode,
+				ChecksumMode {
+					// TODO: for range queries that align with part boundaries,
+					// we should return the saved checksum of the part
+					// https://docs.aws.amazon.com/AmazonS3/latest/userguide/checking-object-integrity.html
+					enabled: false,
+				},
 			)
 			.await
 		}
@@ -577,7 +576,7 @@ async fn handle_get_part(
 }
 
 fn parse_range_header(
-	req: &Request<impl Body>,
+	req: &Request<()>,
 	total_size: u64,
 ) -> Result<Option<http_range::HttpRange>, Error> {
 	let range = match req.headers().get(RANGE) {
@@ -618,7 +617,7 @@ struct ChecksumMode {
 	enabled: bool,
 }
 
-fn checksum_mode(req: &Request<impl Body>) -> ChecksumMode {
+fn checksum_mode(req: &Request<()>) -> ChecksumMode {
 	ChecksumMode {
 		enabled: req
 			.headers()
@@ -751,3 +750,116 @@ fn std_error_from_read_error<E: std::fmt::Display>(e: E) -> std::io::Error {
 		format!("Error while reading object data: {}", e),
 	)
 }
+
+// ----
+
+pub struct PreconditionHeaders {
+	if_match: Option<Vec<String>>,
+	if_modified_since: Option<SystemTime>,
+	if_none_match: Option<Vec<String>>,
+	if_unmodified_since: Option<SystemTime>,
+}
+
+impl PreconditionHeaders {
+	fn parse<B>(req: &Request<B>) -> Result<Self, Error> {
+		Self::parse_with(
+			req.headers(),
+			&IF_MATCH,
+			&IF_NONE_MATCH,
+			&IF_MODIFIED_SINCE,
+			&IF_UNMODIFIED_SINCE,
+		)
+	}
+
+	pub(crate) fn parse_copy_source<B>(req: &Request<B>) -> Result<Self, Error> {
+		Self::parse_with(
+			req.headers(),
+			&X_AMZ_COPY_SOURCE_IF_MATCH,
+			&X_AMZ_COPY_SOURCE_IF_NONE_MATCH,
+			&X_AMZ_COPY_SOURCE_IF_MODIFIED_SINCE,
+			&X_AMZ_COPY_SOURCE_IF_UNMODIFIED_SINCE,
+		)
+	}
+
+	fn parse_with(
+		headers: &HeaderMap,
+		hdr_if_match: &HeaderName,
+		hdr_if_none_match: &HeaderName,
+		hdr_if_modified_since: &HeaderName,
+		hdr_if_unmodified_since: &HeaderName,
+	) -> Result<Self, Error> {
+		Ok(Self {
+			if_match: headers
+				.get(hdr_if_match)
+				.map(|x| x.to_str())
+				.transpose()?
+				.map(|x| {
+					x.split(',')
+						.map(|m| m.trim().trim_matches('"').to_string())
+						.collect::<Vec<_>>()
+				}),
+			if_none_match: headers
+				.get(hdr_if_none_match)
+				.map(|x| x.to_str())
+				.transpose()?
+				.map(|x| {
+					x.split(',')
+						.map(|m| m.trim().trim_matches('"').to_string())
+						.collect::<Vec<_>>()
+				}),
+			if_modified_since: headers
+				.get(hdr_if_modified_since)
+				.map(|x| x.to_str())
+				.transpose()?
+				.map(httpdate::parse_http_date)
+				.transpose()
+				.ok_or_bad_request("Invalid date in if-modified-since")?,
+			if_unmodified_since: headers
+				.get(hdr_if_unmodified_since)
+				.map(|x| x.to_str())
+				.transpose()?
+				.map(httpdate::parse_http_date)
+				.transpose()
+				.ok_or_bad_request("Invalid date in if-unmodified-since")?,
+		})
+	}
+
+	fn check(&self, v: &ObjectVersion, etag: &str) -> Result<Option<StatusCode>, Error> {
+		let v_date = UNIX_EPOCH + Duration::from_millis(v.timestamp);
+
+		// Implemented from https://datatracker.ietf.org/doc/html/rfc7232#section-6
+
+		if let Some(im) = &self.if_match {
+			// Step 1: if-match is present
+			if !im.iter().any(|x| x == etag || x == "*") {
+				return Ok(Some(StatusCode::PRECONDITION_FAILED));
+			}
+		} else if let Some(ius) = &self.if_unmodified_since {
+			// Step 2: if-unmodified-since is present, and if-match is absent
+			if v_date > *ius {
+				return Ok(Some(StatusCode::PRECONDITION_FAILED));
+			}
+		}
+
+		if let Some(inm) = &self.if_none_match {
+			// Step 3: if-none-match is present
+			if inm.iter().any(|x| x == etag || x == "*") {
+				return Ok(Some(StatusCode::NOT_MODIFIED));
+			}
+		} else if let Some(ims) = &self.if_modified_since {
+			// Step 4: if-modified-since is present, and if-none-match is absent
+			if v_date <= *ims {
+				return Ok(Some(StatusCode::NOT_MODIFIED));
+			}
+		}
+
+		Ok(None)
+	}
+
+	pub(crate) fn check_copy_source(&self, v: &ObjectVersion, etag: &str) -> Result<(), Error> {
+		match self.check(v, etag)? {
+			Some(_) => Err(Error::PreconditionFailed),
+			None => Ok(()),
+		}
+	}
+}

src/api/s3/lib.rs

@@ -14,9 +14,8 @@ mod list;
 mod multipart;
 mod post_object;
 mod put;
-mod website;
+pub mod website;
 
-mod checksum;
 mod encryption;
 mod router;
 pub mod xml;

src/api/s3/lifecycle.rs

@@ -1,12 +1,10 @@
 use quick_xml::de::from_reader;
 
-use http_body_util::BodyExt;
 use hyper::{Request, Response, StatusCode};
 
 use serde::{Deserialize, Serialize};
 
 use garage_api_common::helpers::*;
-use garage_api_common::signature::verify_signed_content;
 
 use crate::api_server::{ReqBody, ResBody};
 use crate::error::*;
@@ -16,7 +14,6 @@ use garage_model::bucket_table::{
 	parse_lifecycle_date, Bucket, LifecycleExpiration as GarageLifecycleExpiration,
 	LifecycleFilter as GarageLifecycleFilter, LifecycleRule as GarageLifecycleRule,
 };
-use garage_util::data::*;
 
 pub async fn handle_get_lifecycle(ctx: ReqCtx) -> Result<Response<ResBody>, Error> {
 	let ReqCtx { bucket_params, .. } = ctx;
@@ -56,7 +53,6 @@ pub async fn handle_delete_lifecycle(ctx: ReqCtx) -> Result<Response<ResBody>, E
 pub async fn handle_put_lifecycle(
 	ctx: ReqCtx,
 	req: Request<ReqBody>,
-	content_sha256: Option<Hash>,
 ) -> Result<Response<ResBody>, Error> {
 	let ReqCtx {
 		garage,
@@ -65,11 +61,7 @@ pub async fn handle_put_lifecycle(
 		..
 	} = ctx;
 
-	let body = BodyExt::collect(req.into_body()).await?.to_bytes();
-
-	if let Some(content_sha256) = content_sha256 {
-		verify_signed_content(content_sha256, &body[..])?;
-	}
+	let body = req.into_body().collect().await?;
 
 	let conf: LifecycleConfiguration = from_reader(&body as &[u8])?;
 	let config = conf

src/api/s3/list.rs

@@ -17,7 +17,7 @@ use garage_api_common::encoding::*;
 use garage_api_common::helpers::*;
 
 use crate::api_server::{ReqBody, ResBody};
-use crate::encryption::EncryptionParams;
+use crate::encryption::{EncryptionParams, OekDerivationInfo};
 use crate::error::*;
 use crate::multipart as s3_multipart;
 use crate::xml as s3_xml;
@@ -285,8 +285,16 @@ pub async fn handle_list_parts(
 		ObjectVersionState::Uploading { encryption, .. } => encryption,
 		_ => unreachable!(),
 	};
-	let encryption_res =
-		EncryptionParams::check_decrypt(&ctx.garage, req.headers(), &object_encryption);
+	let encryption_res = EncryptionParams::check_decrypt(
+		&ctx.garage,
+		req.headers(),
+		&object_encryption,
+		OekDerivationInfo {
+			bucket_id: ctx.bucket_id,
+			version_id: upload_id,
+			object_key: &query.key,
+		},
+	);
 
 	let (info, next) = fetch_part_info(query, &mpu)?;
 
@@ -326,6 +334,12 @@ pub async fn handle_list_parts(
 						}
 						_ => None,
 					},
+					checksum_crc64nvme: match &checksum {
+						Some(ChecksumValue::Crc64Nvme(x)) => {
+							Some(s3_xml::Value(BASE64_STANDARD.encode(&x)))
+						}
+						_ => None,
+					},
 					checksum_sha1: match &checksum {
 						Some(ChecksumValue::Sha1(x)) => {
 							Some(s3_xml::Value(BASE64_STANDARD.encode(&x)))

src/api/s3/multipart.rs

@@ -1,13 +1,21 @@
 use std::collections::HashMap;
-use std::convert::TryInto;
+use std::convert::{TryFrom, TryInto};
+use std::hash::Hasher;
 use std::sync::Arc;
 
 use base64::prelude::*;
+use crc32c::Crc32cHasher as Crc32c;
+use crc32fast::Hasher as Crc32;
+use crc64fast_nvme::Digest as Crc64Nvme;
 use futures::prelude::*;
 use hyper::{Request, Response};
+use md5::{Digest, Md5};
+use sha1::Sha1;
+use sha2::Sha256;
 
 use garage_table::*;
 use garage_util::data::*;
+use garage_util::error::OkOrMessage;
 
 use garage_model::garage::Garage;
 use garage_model::s3::block_ref_table::*;
@@ -16,11 +24,10 @@ use garage_model::s3::object_table::*;
 use garage_model::s3::version_table::*;
 
 use garage_api_common::helpers::*;
-use garage_api_common::signature::verify_signed_content;
+use garage_api_common::signature::checksum::*;
 
 use crate::api_server::{ReqBody, ResBody};
-use crate::checksum::*;
-use crate::encryption::EncryptionParams;
+use crate::encryption::{has_encryption_header, EncryptionParams, OekDerivationInfo};
 use crate::error::*;
 use crate::put::*;
 use crate::xml as s3_xml;
@@ -43,14 +50,22 @@ pub async fn handle_create_multipart_upload(
 	let upload_id = gen_uuid();
 	let timestamp = next_timestamp(existing_object.as_ref());
 
-	let headers = get_headers(req.headers())?;
+	let headers = extract_metadata_headers(req.headers())?;
 	let meta = ObjectVersionMetaInner {
 		headers,
 		checksum: None,
 	};
 
 	// Determine whether object should be encrypted, and if so the key
-	let encryption = EncryptionParams::new_from_headers(&garage, req.headers())?;
+	let encryption = EncryptionParams::new_from_headers(
+		&garage,
+		req.headers(),
+		OekDerivationInfo {
+			bucket_id: *bucket_id,
+			version_id: upload_id,
+			object_key: &key,
+		},
+	)?;
 	let object_encryption = encryption.encrypt_meta(meta)?;
 
 	let checksum_algorithm = request_checksum_algorithm(req.headers())?;
@@ -94,7 +109,6 @@ pub async fn handle_put_part(
 	key: &str,
 	part_number: u64,
 	upload_id: &str,
-	content_sha256: Option<Hash>,
 ) -> Result<Response<ResBody>, Error> {
 	let ReqCtx { garage, .. } = &ctx;
 
@@ -105,21 +119,34 @@ pub async fn handle_put_part(
 			Some(x) => Some(x.to_str()?.to_string()),
 			None => None,
 		},
-		sha256: content_sha256,
+		sha256: None,
 		extra: request_checksum_value(req.headers())?,
 	};
 
-	// Read first chuck, and at the same time try to get object to see if it exists
 	let key = key.to_string();
 
-	let (req_head, req_body) = req.into_parts();
-	let stream = body_stream(req_body);
+	let (req_head, mut req_body) = req.into_parts();
+
+	// Before we stream the body, configure the needed checksums.
+	req_body.add_expected_checksums(expected_checksums.clone());
+	if !has_encryption_header(&req_head.headers) {
+		// For non-encrypted objects, we need to compute the md5sum in all cases
+		// (even if content-md5 is not set), because it is used as an etag of the
+		// part, which is in turn used in the etag computation of the whole object
+		req_body.add_md5();
+	}
+
+	let (stream, stream_checksums) = req_body.streaming_with_checksums();
+	let stream = stream.map_err(Error::from);
+
 	let mut chunker = StreamChunker::new(stream, garage.config.block_size);
 
-	let ((_, object_version, mut mpu), first_block) =
+	// Read first chuck, and at the same time try to get object to see if it exists
+	let ((object, object_version, mut mpu), first_block) =
 		futures::try_join!(get_upload(&ctx, &key, &upload_id), chunker.next(),)?;
 
 	// Check encryption params
+	let oek_params = OekDerivationInfo::for_object(&object, &object_version);
 	let (object_encryption, checksum_algorithm) = match object_version.state {
 		ObjectVersionState::Uploading {
 			encryption,
@@ -128,8 +155,12 @@ pub async fn handle_put_part(
 		} => (encryption, checksum_algorithm),
 		_ => unreachable!(),
 	};
-	let (encryption, _) =
-		EncryptionParams::check_decrypt(&garage, &req_head.headers, &object_encryption)?;
+	let (encryption, _) = EncryptionParams::check_decrypt(
+		&garage,
+		&req_head.headers,
+		&object_encryption,
+		oek_params,
+	)?;
 
 	// Check object is valid and part can be accepted
 	let first_block = first_block.ok_or_bad_request("Empty body")?;
@@ -172,21 +203,21 @@ pub async fn handle_put_part(
 	garage.version_table.insert(&version).await?;
 
 	// Copy data to version
-	let checksummer =
-		Checksummer::init(&expected_checksums, !encryption.is_encrypted()).add(checksum_algorithm);
-	let (total_size, checksums, _) = read_and_put_blocks(
+	let (total_size, _, _) = read_and_put_blocks(
 		&ctx,
 		&version,
 		encryption,
 		part_number,
 		first_block,
-		&mut chunker,
-		checksummer,
+		chunker,
+		Checksummer::new(),
 	)
 	.await?;
 
-	// Verify that checksums map
-	checksums.verify(&expected_checksums)?;
+	// Verify that checksums match
+	let checksums = stream_checksums
+		.await
+		.ok_or_internal_error("checksum calculation")??;
 
 	// Store part etag in version
 	let etag = encryption.etag_from_md5(&checksums.md5);
@@ -248,7 +279,6 @@ pub async fn handle_complete_multipart_upload(
 	req: Request<ReqBody>,
 	key: &str,
 	upload_id: &str,
-	content_sha256: Option<Hash>,
 ) -> Result<Response<ResBody>, Error> {
 	let ReqCtx {
 		garage,
@@ -260,11 +290,7 @@ pub async fn handle_complete_multipart_upload(
 
 	let expected_checksum = request_checksum_value(&req_head.headers)?;
 
-	let body = http_body_util::BodyExt::collect(req_body).await?.to_bytes();
-
-	if let Some(content_sha256) = content_sha256 {
-		verify_signed_content(content_sha256, &body[..])?;
-	}
+	let body = req_body.collect().await?;
 
 	let body_xml = roxmltree::Document::parse(std::str::from_utf8(&body)?)?;
 	let body_list_of_parts = parse_complete_multipart_upload_body(&body_xml)
@@ -284,6 +310,7 @@ pub async fn handle_complete_multipart_upload(
 		return Err(Error::bad_request("No data was uploaded"));
 	}
 
+	let oek_params = OekDerivationInfo::for_object(&object, &object_version);
 	let (object_encryption, checksum_algorithm) = match object_version.state {
 		ObjectVersionState::Uploading {
 			encryption,
@@ -404,8 +431,12 @@ pub async fn handle_complete_multipart_upload(
 	let object_encryption = match checksum_algorithm {
 		None => object_encryption,
 		Some(_) => {
-			let (encryption, meta) =
-				EncryptionParams::check_decrypt(&garage, &req_head.headers, &object_encryption)?;
+			let (encryption, meta) = EncryptionParams::check_decrypt(
+				&garage,
+				&req_head.headers,
+				&object_encryption,
+				oek_params,
+			)?;
 			let new_meta = ObjectVersionMetaInner {
 				headers: meta.into_owned().headers,
 				checksum: checksum_extra,
@@ -451,6 +482,10 @@ pub async fn handle_complete_multipart_upload(
 			Some(ChecksumValue::Crc32c(x)) => Some(s3_xml::Value(BASE64_STANDARD.encode(&x))),
 			_ => None,
 		},
+		checksum_crc64nvme: match &checksum_extra {
+			Some(ChecksumValue::Crc64Nvme(x)) => Some(s3_xml::Value(BASE64_STANDARD.encode(&x))),
+			_ => None,
+		},
 		checksum_sha1: match &checksum_extra {
 			Some(ChecksumValue::Sha1(x)) => Some(s3_xml::Value(BASE64_STANDARD.encode(&x))),
 			_ => None,
@@ -574,6 +609,15 @@ fn parse_complete_multipart_upload_body(
 						.try_into()
 						.ok()?,
 				))
+			} else if let Some(crc64nvme) = item
+				.children()
+				.find(|e| e.has_tag_name("ChecksumCRC64NVME"))
+			{
+				Some(ChecksumValue::Crc64Nvme(
+					BASE64_STANDARD.decode(crc64nvme.text()?).ok()?[..]
+						.try_into()
+						.ok()?,
+				))
 			} else if let Some(sha1) = item.children().find(|e| e.has_tag_name("ChecksumSHA1")) {
 				Some(ChecksumValue::Sha1(
 					BASE64_STANDARD.decode(sha1.text()?).ok()?[..]
@@ -602,3 +646,112 @@ fn parse_complete_multipart_upload_body(
 
 	Some(parts)
 }
+
+// ====== checksummer ====
+
+#[derive(Default)]
+pub(crate) struct MultipartChecksummer {
+	pub md5: Md5,
+	pub extra: Option<MultipartExtraChecksummer>,
+}
+
+pub(crate) enum MultipartExtraChecksummer {
+	Crc32(Crc32),
+	Crc32c(Crc32c),
+	Crc64Nvme(Crc64Nvme),
+	Sha1(Sha1),
+	Sha256(Sha256),
+}
+
+impl MultipartChecksummer {
+	pub(crate) fn init(algo: Option<ChecksumAlgorithm>) -> Self {
+		Self {
+			md5: Md5::new(),
+			extra: match algo {
+				None => None,
+				Some(ChecksumAlgorithm::Crc32) => {
+					Some(MultipartExtraChecksummer::Crc32(Crc32::new()))
+				}
+				Some(ChecksumAlgorithm::Crc32c) => {
+					Some(MultipartExtraChecksummer::Crc32c(Crc32c::default()))
+				}
+				Some(ChecksumAlgorithm::Crc64Nvme) => {
+					Some(MultipartExtraChecksummer::Crc64Nvme(Crc64Nvme::default()))
+				}
+				Some(ChecksumAlgorithm::Sha1) => Some(MultipartExtraChecksummer::Sha1(Sha1::new())),
+				Some(ChecksumAlgorithm::Sha256) => {
+					Some(MultipartExtraChecksummer::Sha256(Sha256::new()))
+				}
+			},
+		}
+	}
+
+	pub(crate) fn update(
+		&mut self,
+		etag: &str,
+		checksum: Option<ChecksumValue>,
+	) -> Result<(), Error> {
+		self.md5
+			.update(&hex::decode(&etag).ok_or_message("invalid etag hex")?);
+		match (&mut self.extra, checksum) {
+			(None, _) => (),
+			(
+				Some(MultipartExtraChecksummer::Crc32(ref mut crc32)),
+				Some(ChecksumValue::Crc32(x)),
+			) => {
+				crc32.update(&x);
+			}
+			(
+				Some(MultipartExtraChecksummer::Crc32c(ref mut crc32c)),
+				Some(ChecksumValue::Crc32c(x)),
+			) => {
+				crc32c.write(&x);
+			}
+			(
+				Some(MultipartExtraChecksummer::Crc64Nvme(ref mut crc64nvme)),
+				Some(ChecksumValue::Crc64Nvme(x)),
+			) => {
+				crc64nvme.write(&x);
+			}
+			(Some(MultipartExtraChecksummer::Sha1(ref mut sha1)), Some(ChecksumValue::Sha1(x))) => {
+				sha1.update(&x);
+			}
+			(
+				Some(MultipartExtraChecksummer::Sha256(ref mut sha256)),
+				Some(ChecksumValue::Sha256(x)),
+			) => {
+				sha256.update(&x);
+			}
+			(Some(_), b) => {
+				return Err(Error::internal_error(format!(
+					"part checksum was not computed correctly, got: {:?}",
+					b
+				)))
+			}
+		}
+		Ok(())
+	}
+
+	pub(crate) fn finalize(self) -> (Md5Checksum, Option<ChecksumValue>) {
+		let md5 = self.md5.finalize()[..].try_into().unwrap();
+		let extra = match self.extra {
+			None => None,
+			Some(MultipartExtraChecksummer::Crc32(crc32)) => {
+				Some(ChecksumValue::Crc32(u32::to_be_bytes(crc32.finalize())))
+			}
+			Some(MultipartExtraChecksummer::Crc32c(crc32c)) => Some(ChecksumValue::Crc32c(
+				u32::to_be_bytes(u32::try_from(crc32c.finish()).unwrap()),
+			)),
+			Some(MultipartExtraChecksummer::Crc64Nvme(crc64nvme)) => Some(
+				ChecksumValue::Crc64Nvme(u64::to_be_bytes(crc64nvme.sum64())),
+			),
+			Some(MultipartExtraChecksummer::Sha1(sha1)) => {
+				Some(ChecksumValue::Sha1(sha1.finalize()[..].try_into().unwrap()))
+			}
+			Some(MultipartExtraChecksummer::Sha256(sha256)) => Some(ChecksumValue::Sha256(
+				sha256.finalize()[..].try_into().unwrap(),
+			)),
+		};
+		(md5, extra)
+	}
+}

src/api/s3/post_object.rs

@@ -15,16 +15,17 @@ use serde::Deserialize;
 
 use garage_model::garage::Garage;
 use garage_model::s3::object_table::*;
+use garage_util::data::gen_uuid;
 
 use garage_api_common::cors::*;
 use garage_api_common::helpers::*;
+use garage_api_common::signature::checksum::*;
 use garage_api_common::signature::payload::{verify_v4, Authorization};
 
 use crate::api_server::ResBody;
-use crate::checksum::*;
-use crate::encryption::EncryptionParams;
+use crate::encryption::{EncryptionParams, OekDerivationInfo};
 use crate::error::*;
-use crate::put::{get_headers, save_stream, ChecksumMode};
+use crate::put::{extract_metadata_headers, save_stream, ChecksumMode};
 use crate::xml as s3_xml;
 
 pub async fn handle_post_object(
@@ -103,22 +104,18 @@ pub async fn handle_post_object(
 		key.to_owned()
 	};
 
-	let api_key = verify_v4(&garage, "s3", &authorization, policy.as_bytes()).await?;
+	let api_key = verify_v4(&garage, "s3", &authorization, policy.as_bytes())?;
 
-	let bucket_id = garage
+	let bucket = garage
 		.bucket_helper()
-		.resolve_bucket(&bucket_name, &api_key)
-		.await
+		.resolve_bucket_fast(&bucket_name, &api_key)
 		.map_err(pass_helper_error)?;
+	let bucket_id = bucket.id;
 
 	if !api_key.allow_write(&bucket_id) {
 		return Err(Error::forbidden("Operation is not allowed for this key."));
 	}
 
-	let bucket = garage
-		.bucket_helper()
-		.get_existing_bucket(bucket_id)
-		.await?;
 	let bucket_params = bucket.state.into_option().unwrap();
 	let matching_cors_rule = find_matching_cors_rule(
 		&bucket_params,
@@ -216,8 +213,9 @@ pub async fn handle_post_object(
 
 	// if we ever start supporting ACLs, we likely want to map "acl" to x-amz-acl" somewhere
 	// around here to make sure the rest of the machinery takes our acl into account.
-	let headers = get_headers(&params)?;
+	let headers = extract_metadata_headers(&params)?;
 
+	let checksum_algorithm = request_checksum_algorithm(&params)?;
 	let expected_checksums = ExpectedChecksums {
 		md5: params
 			.get("content-md5")
@@ -225,15 +223,27 @@ pub async fn handle_post_object(
 			.transpose()?
 			.map(str::to_string),
 		sha256: None,
-		extra: request_checksum_algorithm_value(&params)?,
+		extra: checksum_algorithm
+			.map(|algo| extract_checksum_value(&params, algo))
+			.transpose()?,
 	};
 
+	let version_uuid = gen_uuid();
+
 	let meta = ObjectVersionMetaInner {
 		headers,
 		checksum: expected_checksums.extra,
 	};
 
-	let encryption = EncryptionParams::new_from_headers(&garage, &params)?;
+	let encryption = EncryptionParams::new_from_headers(
+		&garage,
+		&params,
+		OekDerivationInfo {
+			bucket_id,
+			version_id: version_uuid,
+			object_key: &key,
+		},
+	)?;
 
 	let stream = file_field.map(|r| r.map_err(Into::into));
 	let ctx = ReqCtx {
@@ -246,6 +256,7 @@ pub async fn handle_post_object(
 
 	let res = save_stream(
 		&ctx,
+		version_uuid,
 		meta,
 		encryption,
 		StreamLimiter::new(stream, conditions.content_length),

src/api/s3/put.rs

@@ -31,11 +31,13 @@ use garage_model::s3::object_table::*;
 use garage_model::s3::version_table::*;
 
 use garage_api_common::helpers::*;
+use garage_api_common::signature::body::StreamingChecksumReceiver;
+use garage_api_common::signature::checksum::*;
 
 use crate::api_server::{ReqBody, ResBody};
-use crate::checksum::*;
-use crate::encryption::EncryptionParams;
+use crate::encryption::{EncryptionParams, OekDerivationInfo};
 use crate::error::*;
+use crate::website::X_AMZ_WEBSITE_REDIRECT_LOCATION;
 
 const PUT_BLOCKS_MAX_PARALLEL: usize = 3;
 
@@ -48,6 +50,10 @@ pub(crate) struct SaveStreamResult {
 
 pub(crate) enum ChecksumMode<'a> {
 	Verify(&'a ExpectedChecksums),
+	VerifyFrom {
+		checksummer: StreamingChecksumReceiver,
+		trailer_algo: Option<ChecksumAlgorithm>,
+	},
 	Calculate(Option<ChecksumAlgorithm>),
 }
 
@@ -55,10 +61,13 @@ pub async fn handle_put(
 	ctx: ReqCtx,
 	req: Request<ReqBody>,
 	key: &String,
-	content_sha256: Option<Hash>,
 ) -> Result<Response<ResBody>, Error> {
+	// Generate version uuid now, because it is necessary to compute SSE-C
+	// encryption parameters
+	let version_uuid = gen_uuid();
+
 	// Retrieve interesting headers from request
-	let headers = get_headers(req.headers())?;
+	let headers = extract_metadata_headers(req.headers())?;
 	debug!("Object headers: {:?}", headers);
 
 	let expected_checksums = ExpectedChecksums {
@@ -66,9 +75,10 @@ pub async fn handle_put(
 			Some(x) => Some(x.to_str()?.to_string()),
 			None => None,
 		},
-		sha256: content_sha256,
+		sha256: None,
 		extra: request_checksum_value(req.headers())?,
 	};
+	let trailer_checksum_algorithm = request_trailer_checksum_algorithm(req.headers())?;
 
 	let meta = ObjectVersionMetaInner {
 		headers,
@@ -76,17 +86,41 @@ pub async fn handle_put(
 	};
 
 	// Determine whether object should be encrypted, and if so the key
-	let encryption = EncryptionParams::new_from_headers(&ctx.garage, req.headers())?;
+	let encryption = EncryptionParams::new_from_headers(
+		&ctx.garage,
+		req.headers(),
+		OekDerivationInfo {
+			bucket_id: ctx.bucket_id,
+			version_id: version_uuid,
+			object_key: &key,
+		},
+	)?;
 
-	let stream = body_stream(req.into_body());
+	// The request body is a special ReqBody object (see garage_api_common::signature::body)
+	// which supports calculating checksums while streaming the data.
+	// Before we start streaming, we configure it to calculate all the checksums we need.
+	let mut req_body = req.into_body();
+	req_body.add_expected_checksums(expected_checksums.clone());
+	if !encryption.is_encrypted() {
+		// For non-encrypted objects, we need to compute the md5sum in all cases
+		// (even if content-md5 is not set), because it is used as the object etag
+		req_body.add_md5();
+	}
+
+	let (stream, checksummer) = req_body.streaming_with_checksums();
+	let stream = stream.map_err(Error::from);
 
 	let res = save_stream(
 		&ctx,
+		version_uuid,
 		meta,
 		encryption,
 		stream,
 		key,
-		ChecksumMode::Verify(&expected_checksums),
+		ChecksumMode::VerifyFrom {
+			checksummer,
+			trailer_algo: trailer_checksum_algorithm,
+		},
 	)
 	.await?;
 
@@ -100,6 +134,7 @@ pub async fn handle_put(
 
 pub(crate) async fn save_stream<S: Stream<Item = Result<Bytes, Error>> + Unpin>(
 	ctx: &ReqCtx,
+	version_uuid: Uuid,
 	mut meta: ObjectVersionMetaInner,
 	encryption: EncryptionParams,
 	body: S,
@@ -119,13 +154,17 @@ pub(crate) async fn save_stream<S: Stream<Item = Result<Bytes, Error>> + Unpin>(
 	let first_block = first_block_opt.unwrap_or_default();
 
 	// Generate identity of new version
-	let version_uuid = gen_uuid();
 	let version_timestamp = next_timestamp(existing_object.as_ref());
 
-	let mut checksummer = match checksum_mode {
+	let mut checksummer = match &checksum_mode {
 		ChecksumMode::Verify(expected) => Checksummer::init(expected, !encryption.is_encrypted()),
 		ChecksumMode::Calculate(algo) => {
-			Checksummer::init(&Default::default(), !encryption.is_encrypted()).add(algo)
+			Checksummer::init(&Default::default(), !encryption.is_encrypted()).add(*algo)
+		}
+		ChecksumMode::VerifyFrom { .. } => {
+			// Checksums are calculated by the garage_api_common::signature module
+			// so here we can just have an empty checksummer that does nothing
+			Checksummer::new()
 		}
 	};
 
@@ -133,7 +172,7 @@ pub(crate) async fn save_stream<S: Stream<Item = Result<Bytes, Error>> + Unpin>(
 	// as "inline data". We can then return immediately.
 	if first_block.len() < INLINE_THRESHOLD {
 		checksummer.update(&first_block);
-		let checksums = checksummer.finalize();
+		let mut checksums = checksummer.finalize();
 
 		match checksum_mode {
 			ChecksumMode::Verify(expected) => {
@@ -142,6 +181,18 @@ pub(crate) async fn save_stream<S: Stream<Item = Result<Bytes, Error>> + Unpin>(
 			ChecksumMode::Calculate(algo) => {
 				meta.checksum = checksums.extract(algo);
 			}
+			ChecksumMode::VerifyFrom {
+				checksummer,
+				trailer_algo,
+			} => {
+				drop(chunker);
+				checksums = checksummer
+					.await
+					.ok_or_internal_error("checksum calculation")??;
+				if let Some(algo) = trailer_algo {
+					meta.checksum = checksums.extract(Some(algo));
+				}
+			}
 		};
 
 		let size = first_block.len() as u64;
@@ -213,13 +264,13 @@ pub(crate) async fn save_stream<S: Stream<Item = Result<Bytes, Error>> + Unpin>(
 	garage.version_table.insert(&version).await?;
 
 	// Transfer data
-	let (total_size, checksums, first_block_hash) = read_and_put_blocks(
+	let (total_size, mut checksums, first_block_hash) = read_and_put_blocks(
 		ctx,
 		&version,
 		encryption,
 		1,
 		first_block,
-		&mut chunker,
+		chunker,
 		checksummer,
 	)
 	.await?;
@@ -232,6 +283,17 @@ pub(crate) async fn save_stream<S: Stream<Item = Result<Bytes, Error>> + Unpin>(
 		ChecksumMode::Calculate(algo) => {
 			meta.checksum = checksums.extract(algo);
 		}
+		ChecksumMode::VerifyFrom {
+			checksummer,
+			trailer_algo,
+		} => {
+			checksums = checksummer
+				.await
+				.ok_or_internal_error("checksum calculation")??;
+			if let Some(algo) = trailer_algo {
+				meta.checksum = checksums.extract(Some(algo));
+			}
+		}
 	};
 
 	// Verify quotas are respsected
@@ -332,7 +394,7 @@ pub(crate) async fn read_and_put_blocks<S: Stream<Item = Result<Bytes, Error>> +
 	encryption: EncryptionParams,
 	part_number: u64,
 	first_block: Bytes,
-	chunker: &mut StreamChunker<S>,
+	mut chunker: StreamChunker<S>,
 	checksummer: Checksummer,
 ) -> Result<(u64, Checksums, Hash), Error> {
 	let tracer = opentelemetry::global::tracer("garage");
@@ -601,7 +663,9 @@ impl Drop for InterruptedCleanup {
 
 // ============ helpers ============
 
-pub(crate) fn get_headers(headers: &HeaderMap<HeaderValue>) -> Result<HeaderList, Error> {
+pub(crate) fn extract_metadata_headers(
+	headers: &HeaderMap<HeaderValue>,
+) -> Result<HeaderList, Error> {
 	let mut ret = Vec::new();
 
 	// Preserve standard headers
@@ -627,6 +691,18 @@ pub(crate) fn get_headers(headers: &HeaderMap<HeaderValue>) -> Result<HeaderList
 				std::str::from_utf8(value.as_bytes())?.to_string(),
 			));
 		}
+		if name == X_AMZ_WEBSITE_REDIRECT_LOCATION {
+			let value = std::str::from_utf8(value.as_bytes())?.to_string();
+			if !(value.starts_with("/")
+				|| value.starts_with("http://")
+				|| value.starts_with("https://"))
+			{
+				return Err(Error::bad_request(format!(
+					"Invalid {X_AMZ_WEBSITE_REDIRECT_LOCATION} header",
+				)));
+			}
+			ret.push((X_AMZ_WEBSITE_REDIRECT_LOCATION.to_string(), value));
+		}
 	}
 
 	Ok(ret)

src/api/s3/website.rs

@@ -1,19 +1,19 @@
 use quick_xml::de::from_reader;
 
-use http_body_util::BodyExt;
-use hyper::{Request, Response, StatusCode};
+use hyper::{header::HeaderName, Request, Response, StatusCode};
 use serde::{Deserialize, Serialize};
 
-use garage_model::bucket_table::*;
-use garage_util::data::*;
+use garage_model::bucket_table::{self, *};
 
 use garage_api_common::helpers::*;
-use garage_api_common::signature::verify_signed_content;
 
 use crate::api_server::{ReqBody, ResBody};
 use crate::error::*;
 use crate::xml::{to_xml_with_header, xmlns_tag, IntValue, Value};
 
+pub const X_AMZ_WEBSITE_REDIRECT_LOCATION: HeaderName =
+	HeaderName::from_static("x-amz-website-redirect-location");
+
 pub async fn handle_get_website(ctx: ReqCtx) -> Result<Response<ResBody>, Error> {
 	let ReqCtx { bucket_params, .. } = ctx;
 	if let Some(website) = bucket_params.website_config.get() {
@@ -26,7 +26,28 @@ pub async fn handle_get_website(ctx: ReqCtx) -> Result<Response<ResBody>, Error>
 				suffix: Value(website.index_document.to_string()),
 			}),
 			redirect_all_requests_to: None,
-			routing_rules: None,
+			routing_rules: RoutingRules {
+				rules: website
+					.routing_rules
+					.clone()
+					.into_iter()
+					.map(|rule| RoutingRule {
+						condition: rule.condition.map(|cond| Condition {
+							http_error_code: cond.http_error_code.map(|c| IntValue(c as i64)),
+							prefix: cond.prefix.map(Value),
+						}),
+						redirect: Redirect {
+							hostname: rule.redirect.hostname.map(Value),
+							http_redirect_code: Some(IntValue(
+								rule.redirect.http_redirect_code as i64,
+							)),
+							protocol: rule.redirect.protocol.map(Value),
+							replace_full: rule.redirect.replace_key.map(Value),
+							replace_prefix: rule.redirect.replace_key_prefix.map(Value),
+						},
+					})
+					.collect(),
+			},
 		};
 		let xml = to_xml_with_header(&wc)?;
 		Ok(Response::builder()
@@ -61,7 +82,6 @@ pub async fn handle_delete_website(ctx: ReqCtx) -> Result<Response<ResBody>, Err
 pub async fn handle_put_website(
 	ctx: ReqCtx,
 	req: Request<ReqBody>,
-	content_sha256: Option<Hash>,
 ) -> Result<Response<ResBody>, Error> {
 	let ReqCtx {
 		garage,
@@ -70,11 +90,7 @@ pub async fn handle_put_website(
 		..
 	} = ctx;
 
-	let body = BodyExt::collect(req.into_body()).await?.to_bytes();
-
-	if let Some(content_sha256) = content_sha256 {
-		verify_signed_content(content_sha256, &body[..])?;
-	}
+	let body = req.into_body().collect().await?;
 
 	let conf: WebsiteConfiguration = from_reader(&body as &[u8])?;
 	conf.validate()?;
@@ -102,18 +118,28 @@ pub struct WebsiteConfiguration {
 	pub index_document: Option<Suffix>,
 	#[serde(rename = "RedirectAllRequestsTo")]
 	pub redirect_all_requests_to: Option<Target>,
-	#[serde(rename = "RoutingRules")]
-	pub routing_rules: Option<Vec<RoutingRule>>,
+	#[serde(
+		rename = "RoutingRules",
+		default,
+		skip_serializing_if = "RoutingRules::is_empty"
+	)]
+	pub routing_rules: RoutingRules,
+}
+
+#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord, Default)]
+pub struct RoutingRules {
+	#[serde(rename = "RoutingRule")]
+	pub rules: Vec<RoutingRule>,
+}
+
+impl RoutingRules {
+	fn is_empty(&self) -> bool {
+		self.rules.is_empty()
+	}
 }
 
 #[derive(Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)]
 pub struct RoutingRule {
-	#[serde(rename = "RoutingRule")]
-	pub inner: RoutingRuleInner,
-}
-
-#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)]
-pub struct RoutingRuleInner {
 	#[serde(rename = "Condition")]
 	pub condition: Option<Condition>,
 	#[serde(rename = "Redirect")]
@@ -167,7 +193,7 @@ impl WebsiteConfiguration {
 		if self.redirect_all_requests_to.is_some()
 			&& (self.error_document.is_some()
 				|| self.index_document.is_some()
-				|| self.routing_rules.is_some())
+				|| !self.routing_rules.is_empty())
 		{
 			return Err(Error::bad_request(
 				"Bad XML: can't have RedirectAllRequestsTo and other fields",
@@ -182,10 +208,15 @@ impl WebsiteConfiguration {
 		if let Some(ref rart) = self.redirect_all_requests_to {
 			rart.validate()?;
 		}
-		if let Some(ref rrs) = self.routing_rules {
-			for rr in rrs {
-				rr.inner.validate()?;
-			}
+		for rr in &self.routing_rules.rules {
+			rr.validate()?;
+		}
+		if self.routing_rules.rules.len() > 1000 {
+			// we will do linear scans, best to avoid overly long configuration. The
+			// limit was choosen arbitrarily
+			return Err(Error::bad_request(
+				"Bad XML: RoutingRules can't have more than 1000 child elements",
+			));
 		}
 
 		Ok(())
@@ -194,11 +225,7 @@ impl WebsiteConfiguration {
 	pub fn into_garage_website_config(self) -> Result<WebsiteConfig, Error> {
 		if self.redirect_all_requests_to.is_some() {
 			Err(Error::NotImplemented(
-				"S3 website redirects are not currently implemented in Garage.".into(),
-			))
-		} else if self.routing_rules.map(|x| !x.is_empty()).unwrap_or(false) {
-			Err(Error::NotImplemented(
-				"S3 routing rules are not currently implemented in Garage.".into(),
+				"RedirectAllRequestsTo is not currently implemented in Garage, however its effect can be emulated using a single inconditional RoutingRule.".into(),
 			))
 		} else {
 			Ok(WebsiteConfig {
@@ -207,6 +234,36 @@ impl WebsiteConfiguration {
 					.map(|x| x.suffix.0)
 					.unwrap_or_else(|| "index.html".to_string()),
 				error_document: self.error_document.map(|x| x.key.0),
+				redirect_all: None,
+				routing_rules: self
+					.routing_rules
+					.rules
+					.into_iter()
+					.map(|rule| {
+						bucket_table::RoutingRule {
+							condition: rule.condition.map(|condition| {
+								bucket_table::RedirectCondition {
+									http_error_code: condition.http_error_code.map(|c| c.0 as u16),
+									prefix: condition.prefix.map(|p| p.0),
+								}
+							}),
+							redirect: bucket_table::Redirect {
+								hostname: rule.redirect.hostname.map(|h| h.0),
+								protocol: rule.redirect.protocol.map(|p| p.0),
+								// aws default to 301, which i find punitive in case of
+								// missconfiguration (can be permanently cached on the
+								// user agent)
+								http_redirect_code: rule
+									.redirect
+									.http_redirect_code
+									.map(|c| c.0 as u16)
+									.unwrap_or(302),
+								replace_key_prefix: rule.redirect.replace_prefix.map(|k| k.0),
+								replace_key: rule.redirect.replace_full.map(|k| k.0),
+							},
+						}
+					})
+					.collect(),
 			})
 		}
 	}
@@ -247,37 +304,69 @@ impl Target {
 	}
 }
 
-impl RoutingRuleInner {
+impl RoutingRule {
 	pub fn validate(&self) -> Result<(), Error> {
-		let has_prefix = self
-			.condition
-			.as_ref()
-			.and_then(|c| c.prefix.as_ref())
-			.is_some();
-		self.redirect.validate(has_prefix)
+		if let Some(condition) = &self.condition {
+			condition.validate()?;
+		}
+		self.redirect.validate()
+	}
+}
+
+impl Condition {
+	pub fn validate(&self) -> Result<bool, Error> {
+		if let Some(ref error_code) = self.http_error_code {
+			// TODO do other error codes make sense? Aws only allows 4xx and 5xx
+			if error_code.0 != 404 {
+				return Err(Error::bad_request(
+					"Bad XML: HttpErrorCodeReturnedEquals must be 404 or absent",
+				));
+			}
+		}
+		Ok(self.prefix.is_some())
 	}
 }
 
 impl Redirect {
-	pub fn validate(&self, has_prefix: bool) -> Result<(), Error> {
-		if self.replace_prefix.is_some() {
-			if self.replace_full.is_some() {
-				return Err(Error::bad_request(
-					"Bad XML: both ReplaceKeyPrefixWith and ReplaceKeyWith are set",
-				));
-			}
-			if !has_prefix {
-				return Err(Error::bad_request(
-					"Bad XML: ReplaceKeyPrefixWith is set, but  KeyPrefixEquals isn't",
-				));
-			}
+	pub fn validate(&self) -> Result<(), Error> {
+		if self.replace_prefix.is_some() && self.replace_full.is_some() {
+			return Err(Error::bad_request(
+				"Bad XML: both ReplaceKeyPrefixWith and ReplaceKeyWith are set",
+			));
 		}
 		if let Some(ref protocol) = self.protocol {
 			if protocol.0 != "http" && protocol.0 != "https" {
 				return Err(Error::bad_request("Bad XML: invalid protocol"));
 			}
 		}
-		// TODO there are probably more invalid cases, but which ones?
+		if let Some(ref http_redirect_code) = self.http_redirect_code {
+			match http_redirect_code.0 {
+				// aws allows all 3xx except 300, but some are non-sensical (not modified,
+				// use proxy...)
+				301 | 302 | 303 | 307 | 308 => {
+					if self.hostname.is_none() && self.protocol.is_some() {
+						return Err(Error::bad_request(
+							"Bad XML: HostName must be set if Protocol is set",
+						));
+					}
+				}
+				// aws doesn't allow these codes, but netlify does, and it seems like a
+				// cool feature (change the page seen without changing the url shown by the
+				// user agent)
+				200 | 404 => {
+					if self.hostname.is_some() || self.protocol.is_some() {
+						// hostname would mean different bucket, protocol doesn't make
+						// sense
+						return Err(Error::bad_request(
+                                        "Bad XML: an HttpRedirectCode of 200 is not acceptable alongside HostName or Protocol",
+                                ));
+					}
+				}
+				_ => {
+					return Err(Error::bad_request("Bad XML: invalid HttpRedirectCode"));
+				}
+			}
+		}
 		Ok(())
 	}
 }
@@ -316,6 +405,15 @@ mod tests {
             <ReplaceKeyWith>fullkey</ReplaceKeyWith>
          </Redirect>
       </RoutingRule>
+      <RoutingRule>
+         <Condition>
+            <KeyPrefixEquals></KeyPrefixEquals>
+         </Condition>
+         <Redirect>
+            <HttpRedirectCode>404</HttpRedirectCode>
+            <ReplaceKeyWith>missing</ReplaceKeyWith>
+         </Redirect>
+      </RoutingRule>
    </RoutingRules>
 </WebsiteConfiguration>"#;
 		let conf: WebsiteConfiguration = from_str(message).unwrap();
@@ -331,21 +429,36 @@ mod tests {
 				hostname: Value("garage.tld".to_owned()),
 				protocol: Some(Value("https".to_owned())),
 			}),
-			routing_rules: Some(vec![RoutingRule {
-				inner: RoutingRuleInner {
-					condition: Some(Condition {
-						http_error_code: Some(IntValue(404)),
-						prefix: Some(Value("prefix1".to_owned())),
-					}),
-					redirect: Redirect {
-						hostname: Some(Value("gara.ge".to_owned())),
-						protocol: Some(Value("http".to_owned())),
-						http_redirect_code: Some(IntValue(303)),
-						replace_prefix: Some(Value("prefix2".to_owned())),
-						replace_full: Some(Value("fullkey".to_owned())),
+			routing_rules: RoutingRules {
+				rules: vec![
+					RoutingRule {
+						condition: Some(Condition {
+							http_error_code: Some(IntValue(404)),
+							prefix: Some(Value("prefix1".to_owned())),
+						}),
+						redirect: Redirect {
+							hostname: Some(Value("gara.ge".to_owned())),
+							protocol: Some(Value("http".to_owned())),
+							http_redirect_code: Some(IntValue(303)),
+							replace_prefix: Some(Value("prefix2".to_owned())),
+							replace_full: Some(Value("fullkey".to_owned())),
+						},
 					},
-				},
-			}]),
+					RoutingRule {
+						condition: Some(Condition {
+							http_error_code: None,
+							prefix: Some(Value("".to_owned())),
+						}),
+						redirect: Redirect {
+							hostname: None,
+							protocol: None,
+							http_redirect_code: Some(IntValue(404)),
+							replace_prefix: None,
+							replace_full: Some(Value("missing".to_owned())),
+						},
+					},
+				],
+			},
 		};
 		assert_eq! {
 			ref_value,

src/api/s3/xml.rs

@@ -135,6 +135,8 @@ pub struct CompleteMultipartUploadResult {
 	pub checksum_crc32: Option<Value>,
 	#[serde(rename = "ChecksumCRC32C")]
 	pub checksum_crc32c: Option<Value>,
+	#[serde(rename = "ChecksumCR64NVME")]
+	pub checksum_crc64nvme: Option<Value>,
 	#[serde(rename = "ChecksumSHA1")]
 	pub checksum_sha1: Option<Value>,
 	#[serde(rename = "ChecksumSHA256")]
@@ -209,6 +211,8 @@ pub struct PartItem {
 	pub checksum_crc32: Option<Value>,
 	#[serde(rename = "ChecksumCRC32C")]
 	pub checksum_crc32c: Option<Value>,
+	#[serde(rename = "ChecksumCRC64NVME")]
+	pub checksum_crc64nvme: Option<Value>,
 	#[serde(rename = "ChecksumSHA1")]
 	pub checksum_sha1: Option<Value>,
 	#[serde(rename = "ChecksumSHA256")]
@@ -518,6 +522,7 @@ mod tests {
 			etag: Value("\"3858f62230ac3c915f300c664312c11f-9\"".to_string()),
 			checksum_crc32: None,
 			checksum_crc32c: None,
+			checksum_crc64nvme: None,
 			checksum_sha1: Some(Value("ZJAnHyG8PeKz9tI8UTcHrJos39A=".into())),
 			checksum_sha256: None,
 		};
@@ -803,6 +808,7 @@ mod tests {
 					size: IntValue(10485760),
 					checksum_crc32: None,
 					checksum_crc32c: None,
+					checksum_crc64nvme: None,
 					checksum_sha256: Some(Value(
 						"5RQ3A5uk0w7ojNjvegohch4JRBBGN/cLhsNrPzfv/hA=".into(),
 					)),
@@ -816,6 +822,7 @@ mod tests {
 					checksum_sha256: None,
 					checksum_crc32c: None,
 					checksum_crc32: Some(Value("ZJAnHyG8=".into())),
+					checksum_crc64nvme: None,
 					checksum_sha1: None,
 				},
 			],

src/block/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "garage_block"
-version = "1.0.1"
+version = "2.0.0"
 authors = ["Alex Auvolat <alex@adnab.me>"]
 edition = "2018"
 license = "AGPL-3.0"
@@ -18,7 +18,6 @@ garage_db.workspace = true
 garage_net.workspace = true
 garage_rpc.workspace = true
 garage_util.workspace = true
-garage_table.workspace = true
 
 opentelemetry.workspace = true
 

src/block/manager.rs

@@ -33,8 +33,6 @@ use garage_rpc::rpc_helper::OrderTag;
 use garage_rpc::system::System;
 use garage_rpc::*;
 
-use garage_table::replication::{TableReplication, TableShardedReplication};
-
 use crate::block::*;
 use crate::layout::*;
 use crate::metrics::*;
@@ -74,8 +72,8 @@ impl Rpc for BlockRpc {
 
 /// The block manager, handling block exchange between nodes, and block storage on local node
 pub struct BlockManager {
-	/// Replication strategy, allowing to find on which node blocks should be located
-	pub replication: TableShardedReplication,
+	/// Quorum of nodes for write operations
+	pub write_quorum: usize,
 
 	/// Data layout
 	pub(crate) data_layout: ArcSwap<DataLayout>,
@@ -122,7 +120,7 @@ impl BlockManager {
 	pub fn new(
 		db: &db::Db,
 		config: &Config,
-		replication: TableShardedReplication,
+		write_quorum: usize,
 		system: Arc<System>,
 	) -> Result<Arc<Self>, Error> {
 		// Load or compute layout, i.e. assignment of data blocks to the different data directories
@@ -166,7 +164,7 @@ impl BlockManager {
 		let scrub_persister = PersisterShared::new(&system.metadata_dir, "scrub_info");
 
 		let block_manager = Arc::new(Self {
-			replication,
+			write_quorum,
 			data_layout: ArcSwap::new(Arc::new(data_layout)),
 			data_layout_persister,
 			data_fsync: config.data_fsync,
@@ -338,6 +336,19 @@ impl BlockManager {
 		Err(err)
 	}
 
+	/// Returns the set of nodes that should store a copy of a given block.
+	/// These are the nodes assigned to the block's hash in the current
+	/// layout version only: since blocks are immutable, we don't need to
+	/// do complex logic when several layour versions are active at once,
+	/// just move them directly to the new nodes.
+	pub(crate) fn storage_nodes_of(&self, hash: &Hash) -> Vec<Uuid> {
+		self.system
+			.cluster_layout()
+			.current()
+			.nodes_of(hash)
+			.collect()
+	}
+
 	// ---- Public interface ----
 
 	/// Ask nodes that might have a block for it, return it as a stream
@@ -370,7 +381,7 @@ impl BlockManager {
 		prevent_compression: bool,
 		order_tag: Option<OrderTag>,
 	) -> Result<(), Error> {
-		let who = self.system.cluster_layout().current_storage_nodes_of(&hash);
+		let who = self.storage_nodes_of(&hash);
 
 		let compression_level = self.compression_level.filter(|_| !prevent_compression);
 		let (header, bytes) = DataBlock::from_buffer(data, compression_level)
@@ -400,7 +411,7 @@ impl BlockManager {
 				put_block_rpc,
 				RequestStrategy::with_priority(PRIO_NORMAL | PRIO_SECONDARY)
 					.with_drop_on_completion(permit)
-					.with_quorum(self.replication.write_quorum()),
+					.with_quorum(self.write_quorum),
 			)
 			.await?;
 

src/block/resync.rs

@@ -27,8 +27,6 @@ use garage_util::tranquilizer::Tranquilizer;
 use garage_rpc::system::System;
 use garage_rpc::*;
 
-use garage_table::replication::TableReplication;
-
 use crate::manager::*;
 
 // The delay between the time where a resync operation fails
@@ -377,11 +375,8 @@ impl BlockResyncManager {
 			info!("Resync block {:?}: offloading and deleting", hash);
 			let existing_path = existing_path.unwrap();
 
-			let mut who = manager
-				.system
-				.cluster_layout()
-				.current_storage_nodes_of(hash);
-			if who.len() < manager.replication.write_quorum() {
+			let mut who = manager.storage_nodes_of(hash);
+			if who.len() < manager.write_quorum {
 				return Err(Error::Message("Not trying to offload block because we don't have a quorum of nodes to write to".to_string()));
 			}
 			who.retain(|id| *id != manager.system.id);
@@ -463,10 +458,7 @@ impl BlockResyncManager {
 
 			// First, check whether we are still supposed to store that
 			// block in the latest cluster layout version.
-			let storage_nodes = manager
-				.system
-				.cluster_layout()
-				.current_storage_nodes_of(&hash);
+			let storage_nodes = manager.storage_nodes_of(&hash);
 
 			if !storage_nodes.contains(&manager.system.id) {
 				info!(

src/db/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "garage_db"
-version = "1.0.1"
+version = "2.0.0"
 authors = ["Alex Auvolat <alex@adnab.me>"]
 edition = "2018"
 license = "AGPL-3.0"

src/garage/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "garage"
-version = "1.0.1"
+version = "2.0.0"
 authors = ["Alex Auvolat <alex@adnab.me>"]
 edition = "2018"
 license = "AGPL-3.0"
@@ -26,6 +26,7 @@ garage_db.workspace = true
 garage_api_admin.workspace = true
 garage_api_s3.workspace = true
 garage_api_k2v = { workspace = true, optional = true }
+garage_api_common.workspace = true
 garage_block.workspace = true
 garage_model.workspace = true
 garage_net.workspace = true
@@ -37,6 +38,7 @@ garage_web.workspace = true
 backtrace.workspace = true
 bytes.workspace = true
 bytesize.workspace = true
+chrono.workspace = true
 timeago.workspace = true
 parse_duration.workspace = true
 hex.workspace = true
@@ -47,8 +49,8 @@ sha1.workspace = true
 sodiumoxide.workspace = true
 structopt.workspace = true
 git-version.workspace = true
-
-serde.workspace = true
+utoipa.workspace = true
+serde_json.workspace = true
 
 futures.workspace = true
 tokio.workspace = true
@@ -62,6 +64,7 @@ syslog-tracing = { workspace = true, optional = true }
 garage_api_common.workspace = true
 
 aws-sdk-s3.workspace = true
+aws-smithy-runtime.workspace = true
 chrono.workspace = true
 http.workspace = true
 hmac.workspace = true
@@ -71,10 +74,12 @@ hyper-util.workspace = true
 mktemp.workspace = true
 sha2.workspace = true
 
+
 static_init.workspace = true
 assert-json-diff.workspace = true
 serde_json.workspace = true
 base64.workspace = true
+crc32fast.workspace = true
 
 k2v-client.workspace = true
 
@@ -82,7 +87,7 @@ k2v-client.workspace = true
 [features]
 default = [ "bundled-libs", "metrics", "lmdb", "sqlite", "k2v" ]
 
-k2v = [ "garage_util/k2v", "garage_api_k2v" ]
+k2v = [ "garage_util/k2v", "garage_api_k2v", "garage_api_admin/k2v" ]
 
 # Database engines
 lmdb = [ "garage_model/lmdb" ]

src/garage/admin/block.rs

@@ -1,235 +0,0 @@
-use garage_util::data::*;
-
-use garage_table::*;
-
-use garage_model::helper::error::{Error, OkOrBadRequest};
-use garage_model::s3::object_table::*;
-use garage_model::s3::version_table::*;
-
-use crate::cli::*;
-
-use super::*;
-
-impl AdminRpcHandler {
-	pub(super) async fn handle_block_cmd(&self, cmd: &BlockOperation) -> Result<AdminRpc, Error> {
-		match cmd {
-			BlockOperation::ListErrors => Ok(AdminRpc::BlockErrorList(
-				self.garage.block_manager.list_resync_errors()?,
-			)),
-			BlockOperation::Info { hash } => self.handle_block_info(hash).await,
-			BlockOperation::RetryNow { all, blocks } => {
-				self.handle_block_retry_now(*all, blocks).await
-			}
-			BlockOperation::Purge { yes, blocks } => self.handle_block_purge(*yes, blocks).await,
-		}
-	}
-
-	async fn handle_block_info(&self, hash: &String) -> Result<AdminRpc, Error> {
-		let hash = self.find_block_hash_by_prefix(hash)?;
-		let refcount = self.garage.block_manager.get_block_rc(&hash)?;
-		let block_refs = self
-			.garage
-			.block_ref_table
-			.get_range(&hash, None, None, 10000, Default::default())
-			.await?;
-		let mut versions = vec![];
-		let mut uploads = vec![];
-		for br in block_refs {
-			if let Some(v) = self
-				.garage
-				.version_table
-				.get(&br.version, &EmptyKey)
-				.await?
-			{
-				if let VersionBacklink::MultipartUpload { upload_id } = &v.backlink {
-					if let Some(u) = self.garage.mpu_table.get(upload_id, &EmptyKey).await? {
-						uploads.push(u);
-					}
-				}
-				versions.push(Ok(v));
-			} else {
-				versions.push(Err(br.version));
-			}
-		}
-		Ok(AdminRpc::BlockInfo {
-			hash,
-			refcount,
-			versions,
-			uploads,
-		})
-	}
-
-	async fn handle_block_retry_now(
-		&self,
-		all: bool,
-		blocks: &[String],
-	) -> Result<AdminRpc, Error> {
-		if all {
-			if !blocks.is_empty() {
-				return Err(Error::BadRequest(
-					"--all was specified, cannot also specify blocks".into(),
-				));
-			}
-			let blocks = self.garage.block_manager.list_resync_errors()?;
-			for b in blocks.iter() {
-				self.garage.block_manager.resync.clear_backoff(&b.hash)?;
-			}
-			Ok(AdminRpc::Ok(format!(
-				"{} blocks returned in queue for a retry now (check logs to see results)",
-				blocks.len()
-			)))
-		} else {
-			for hash in blocks {
-				let hash = hex::decode(hash).ok_or_bad_request("invalid hash")?;
-				let hash = Hash::try_from(&hash).ok_or_bad_request("invalid hash")?;
-				self.garage.block_manager.resync.clear_backoff(&hash)?;
-			}
-			Ok(AdminRpc::Ok(format!(
-				"{} blocks returned in queue for a retry now (check logs to see results)",
-				blocks.len()
-			)))
-		}
-	}
-
-	async fn handle_block_purge(&self, yes: bool, blocks: &[String]) -> Result<AdminRpc, Error> {
-		if !yes {
-			return Err(Error::BadRequest(
-				"Pass the --yes flag to confirm block purge operation.".into(),
-			));
-		}
-
-		let mut obj_dels = 0;
-		let mut mpu_dels = 0;
-		let mut ver_dels = 0;
-
-		for hash in blocks {
-			let hash = hex::decode(hash).ok_or_bad_request("invalid hash")?;
-			let hash = Hash::try_from(&hash).ok_or_bad_request("invalid hash")?;
-			let block_refs = self
-				.garage
-				.block_ref_table
-				.get_range(&hash, None, None, 10000, Default::default())
-				.await?;
-
-			for br in block_refs {
-				if let Some(version) = self
-					.garage
-					.version_table
-					.get(&br.version, &EmptyKey)
-					.await?
-				{
-					self.handle_block_purge_version_backlink(
-						&version,
-						&mut obj_dels,
-						&mut mpu_dels,
-					)
-					.await?;
-
-					if !version.deleted.get() {
-						let deleted_version = Version::new(version.uuid, version.backlink, true);
-						self.garage.version_table.insert(&deleted_version).await?;
-						ver_dels += 1;
-					}
-				}
-			}
-		}
-
-		Ok(AdminRpc::Ok(format!(
-			"Purged {} blocks, {} versions, {} objects, {} multipart uploads",
-			blocks.len(),
-			ver_dels,
-			obj_dels,
-			mpu_dels,
-		)))
-	}
-
-	async fn handle_block_purge_version_backlink(
-		&self,
-		version: &Version,
-		obj_dels: &mut usize,
-		mpu_dels: &mut usize,
-	) -> Result<(), Error> {
-		let (bucket_id, key, ov_id) = match &version.backlink {
-			VersionBacklink::Object { bucket_id, key } => (*bucket_id, key.clone(), version.uuid),
-			VersionBacklink::MultipartUpload { upload_id } => {
-				if let Some(mut mpu) = self.garage.mpu_table.get(upload_id, &EmptyKey).await? {
-					if !mpu.deleted.get() {
-						mpu.parts.clear();
-						mpu.deleted.set();
-						self.garage.mpu_table.insert(&mpu).await?;
-						*mpu_dels += 1;
-					}
-					(mpu.bucket_id, mpu.key.clone(), *upload_id)
-				} else {
-					return Ok(());
-				}
-			}
-		};
-
-		if let Some(object) = self.garage.object_table.get(&bucket_id, &key).await? {
-			let ov = object.versions().iter().rev().find(|v| v.is_complete());
-			if let Some(ov) = ov {
-				if ov.uuid == ov_id {
-					let del_uuid = gen_uuid();
-					let deleted_object = Object::new(
-						bucket_id,
-						key,
-						vec![ObjectVersion {
-							uuid: del_uuid,
-							timestamp: ov.timestamp + 1,
-							state: ObjectVersionState::Complete(ObjectVersionData::DeleteMarker),
-						}],
-					);
-					self.garage.object_table.insert(&deleted_object).await?;
-					*obj_dels += 1;
-				}
-			}
-		}
-
-		Ok(())
-	}
-
-	// ---- helper function ----
-	fn find_block_hash_by_prefix(&self, prefix: &str) -> Result<Hash, Error> {
-		if prefix.len() < 4 {
-			return Err(Error::BadRequest(
-				"Please specify at least 4 characters of the block hash".into(),
-			));
-		}
-
-		let prefix_bin =
-			hex::decode(&prefix[..prefix.len() & !1]).ok_or_bad_request("invalid hash")?;
-
-		let iter = self
-			.garage
-			.block_ref_table
-			.data
-			.store
-			.range(&prefix_bin[..]..)
-			.map_err(GarageError::from)?;
-		let mut found = None;
-		for item in iter {
-			let (k, _v) = item.map_err(GarageError::from)?;
-			let hash = Hash::try_from(&k[..32]).unwrap();
-			if &hash.as_slice()[..prefix_bin.len()] != prefix_bin {
-				break;
-			}
-			if hex::encode(hash.as_slice()).starts_with(prefix) {
-				match &found {
-					Some(x) if *x == hash => (),
-					Some(_) => {
-						return Err(Error::BadRequest(format!(
-							"Several blocks match prefix `{}`",
-							prefix
-						)));
-					}
-					None => {
-						found = Some(hash);
-					}
-				}
-			}
-		}
-
-		found.ok_or_else(|| Error::BadRequest("No matching block found".into()))
-	}
-}

src/garage/admin/bucket.rs

@@ -1,500 +0,0 @@
-use std::collections::HashMap;
-use std::fmt::Write;
-
-use garage_util::crdt::*;
-use garage_util::time::*;
-
-use garage_table::*;
-
-use garage_model::bucket_alias_table::*;
-use garage_model::bucket_table::*;
-use garage_model::helper::error::{Error, OkOrBadRequest};
-use garage_model::permission::*;
-
-use crate::cli::*;
-
-use super::*;
-
-impl AdminRpcHandler {
-	pub(super) async fn handle_bucket_cmd(&self, cmd: &BucketOperation) -> Result<AdminRpc, Error> {
-		match cmd {
-			BucketOperation::List => self.handle_list_buckets().await,
-			BucketOperation::Info(query) => self.handle_bucket_info(query).await,
-			BucketOperation::Create(query) => self.handle_create_bucket(&query.name).await,
-			BucketOperation::Delete(query) => self.handle_delete_bucket(query).await,
-			BucketOperation::Alias(query) => self.handle_alias_bucket(query).await,
-			BucketOperation::Unalias(query) => self.handle_unalias_bucket(query).await,
-			BucketOperation::Allow(query) => self.handle_bucket_allow(query).await,
-			BucketOperation::Deny(query) => self.handle_bucket_deny(query).await,
-			BucketOperation::Website(query) => self.handle_bucket_website(query).await,
-			BucketOperation::SetQuotas(query) => self.handle_bucket_set_quotas(query).await,
-			BucketOperation::CleanupIncompleteUploads(query) => {
-				self.handle_bucket_cleanup_incomplete_uploads(query).await
-			}
-		}
-	}
-
-	async fn handle_list_buckets(&self) -> Result<AdminRpc, Error> {
-		let buckets = self
-			.garage
-			.bucket_table
-			.get_range(
-				&EmptyKey,
-				None,
-				Some(DeletedFilter::NotDeleted),
-				10000,
-				EnumerationOrder::Forward,
-			)
-			.await?;
-
-		Ok(AdminRpc::BucketList(buckets))
-	}
-
-	async fn handle_bucket_info(&self, query: &BucketOpt) -> Result<AdminRpc, Error> {
-		let bucket_id = self
-			.garage
-			.bucket_helper()
-			.admin_get_existing_matching_bucket(&query.name)
-			.await?;
-
-		let bucket = self
-			.garage
-			.bucket_helper()
-			.get_existing_bucket(bucket_id)
-			.await?;
-
-		let counters = self
-			.garage
-			.object_counter_table
-			.table
-			.get(&bucket_id, &EmptyKey)
-			.await?
-			.map(|x| x.filtered_values(&self.garage.system.cluster_layout()))
-			.unwrap_or_default();
-
-		let mpu_counters = self
-			.garage
-			.mpu_counter_table
-			.table
-			.get(&bucket_id, &EmptyKey)
-			.await?
-			.map(|x| x.filtered_values(&self.garage.system.cluster_layout()))
-			.unwrap_or_default();
-
-		let mut relevant_keys = HashMap::new();
-		for (k, _) in bucket
-			.state
-			.as_option()
-			.unwrap()
-			.authorized_keys
-			.items()
-			.iter()
-		{
-			if let Some(key) = self
-				.garage
-				.key_table
-				.get(&EmptyKey, k)
-				.await?
-				.filter(|k| !k.is_deleted())
-			{
-				relevant_keys.insert(k.clone(), key);
-			}
-		}
-		for ((k, _), _, _) in bucket
-			.state
-			.as_option()
-			.unwrap()
-			.local_aliases
-			.items()
-			.iter()
-		{
-			if relevant_keys.contains_key(k) {
-				continue;
-			}
-			if let Some(key) = self.garage.key_table.get(&EmptyKey, k).await? {
-				relevant_keys.insert(k.clone(), key);
-			}
-		}
-
-		Ok(AdminRpc::BucketInfo {
-			bucket,
-			relevant_keys,
-			counters,
-			mpu_counters,
-		})
-	}
-
-	#[allow(clippy::ptr_arg)]
-	async fn handle_create_bucket(&self, name: &String) -> Result<AdminRpc, Error> {
-		if !is_valid_bucket_name(name) {
-			return Err(Error::BadRequest(format!(
-				"{}: {}",
-				name, INVALID_BUCKET_NAME_MESSAGE
-			)));
-		}
-
-		let helper = self.garage.locked_helper().await;
-
-		if let Some(alias) = self.garage.bucket_alias_table.get(&EmptyKey, name).await? {
-			if alias.state.get().is_some() {
-				return Err(Error::BadRequest(format!("Bucket {} already exists", name)));
-			}
-		}
-
-		// ---- done checking, now commit ----
-
-		let bucket = Bucket::new();
-		self.garage.bucket_table.insert(&bucket).await?;
-
-		helper.set_global_bucket_alias(bucket.id, name).await?;
-
-		Ok(AdminRpc::Ok(format!("Bucket {} was created.", name)))
-	}
-
-	async fn handle_delete_bucket(&self, query: &DeleteBucketOpt) -> Result<AdminRpc, Error> {
-		let helper = self.garage.locked_helper().await;
-
-		let bucket_id = helper
-			.bucket()
-			.admin_get_existing_matching_bucket(&query.name)
-			.await?;
-
-		// Get the alias, but keep in minde here the bucket name
-		// given in parameter can also be directly the bucket's ID.
-		// In that case bucket_alias will be None, and
-		// we can still delete the bucket if it has zero aliases
-		// (a condition which we try to prevent but that could still happen somehow).
-		// We just won't try to delete an alias entry because there isn't one.
-		let bucket_alias = self
-			.garage
-			.bucket_alias_table
-			.get(&EmptyKey, &query.name)
-			.await?;
-
-		// Check bucket doesn't have other aliases
-		let mut bucket = helper.bucket().get_existing_bucket(bucket_id).await?;
-		let bucket_state = bucket.state.as_option().unwrap();
-		if bucket_state
-			.aliases
-			.items()
-			.iter()
-			.filter(|(_, _, active)| *active)
-			.any(|(name, _, _)| name != &query.name)
-		{
-			return Err(Error::BadRequest(format!("Bucket {} still has other global aliases. Use `bucket unalias` to delete them one by one.", query.name)));
-		}
-		if bucket_state
-			.local_aliases
-			.items()
-			.iter()
-			.any(|(_, _, active)| *active)
-		{
-			return Err(Error::BadRequest(format!("Bucket {} still has other local aliases. Use `bucket unalias` to delete them one by one.", query.name)));
-		}
-
-		// Check bucket is empty
-		if !helper.bucket().is_bucket_empty(bucket_id).await? {
-			return Err(Error::BadRequest(format!(
-				"Bucket {} is not empty",
-				query.name
-			)));
-		}
-
-		if !query.yes {
-			return Err(Error::BadRequest(
-				"Add --yes flag to really perform this operation".to_string(),
-			));
-		}
-
-		// --- done checking, now commit ---
-		// 1. delete authorization from keys that had access
-		for (key_id, _) in bucket.authorized_keys() {
-			helper
-				.set_bucket_key_permissions(bucket.id, key_id, BucketKeyPerm::NO_PERMISSIONS)
-				.await?;
-		}
-
-		// 2. delete bucket alias
-		if bucket_alias.is_some() {
-			helper
-				.purge_global_bucket_alias(bucket_id, &query.name)
-				.await?;
-		}
-
-		// 3. delete bucket
-		bucket.state = Deletable::delete();
-		self.garage.bucket_table.insert(&bucket).await?;
-
-		Ok(AdminRpc::Ok(format!("Bucket {} was deleted.", query.name)))
-	}
-
-	async fn handle_alias_bucket(&self, query: &AliasBucketOpt) -> Result<AdminRpc, Error> {
-		let helper = self.garage.locked_helper().await;
-
-		let bucket_id = helper
-			.bucket()
-			.admin_get_existing_matching_bucket(&query.existing_bucket)
-			.await?;
-
-		if let Some(key_pattern) = &query.local {
-			let key = helper.key().get_existing_matching_key(key_pattern).await?;
-
-			helper
-				.set_local_bucket_alias(bucket_id, &key.key_id, &query.new_name)
-				.await?;
-			Ok(AdminRpc::Ok(format!(
-				"Alias {} now points to bucket {:?} in namespace of key {}",
-				query.new_name, bucket_id, key.key_id
-			)))
-		} else {
-			helper
-				.set_global_bucket_alias(bucket_id, &query.new_name)
-				.await?;
-			Ok(AdminRpc::Ok(format!(
-				"Alias {} now points to bucket {:?}",
-				query.new_name, bucket_id
-			)))
-		}
-	}
-
-	async fn handle_unalias_bucket(&self, query: &UnaliasBucketOpt) -> Result<AdminRpc, Error> {
-		let helper = self.garage.locked_helper().await;
-
-		if let Some(key_pattern) = &query.local {
-			let key = helper.key().get_existing_matching_key(key_pattern).await?;
-
-			let bucket_id = key
-				.state
-				.as_option()
-				.unwrap()
-				.local_aliases
-				.get(&query.name)
-				.cloned()
-				.flatten()
-				.ok_or_bad_request("Bucket not found")?;
-
-			helper
-				.unset_local_bucket_alias(bucket_id, &key.key_id, &query.name)
-				.await?;
-
-			Ok(AdminRpc::Ok(format!(
-				"Alias {} no longer points to bucket {:?} in namespace of key {}",
-				&query.name, bucket_id, key.key_id
-			)))
-		} else {
-			let bucket_id = helper
-				.bucket()
-				.resolve_global_bucket_name(&query.name)
-				.await?
-				.ok_or_bad_request("Bucket not found")?;
-
-			helper
-				.unset_global_bucket_alias(bucket_id, &query.name)
-				.await?;
-
-			Ok(AdminRpc::Ok(format!(
-				"Alias {} no longer points to bucket {:?}",
-				&query.name, bucket_id
-			)))
-		}
-	}
-
-	async fn handle_bucket_allow(&self, query: &PermBucketOpt) -> Result<AdminRpc, Error> {
-		let helper = self.garage.locked_helper().await;
-
-		let bucket_id = helper
-			.bucket()
-			.admin_get_existing_matching_bucket(&query.bucket)
-			.await?;
-		let key = helper
-			.key()
-			.get_existing_matching_key(&query.key_pattern)
-			.await?;
-
-		let allow_read = query.read || key.allow_read(&bucket_id);
-		let allow_write = query.write || key.allow_write(&bucket_id);
-		let allow_owner = query.owner || key.allow_owner(&bucket_id);
-
-		helper
-			.set_bucket_key_permissions(
-				bucket_id,
-				&key.key_id,
-				BucketKeyPerm {
-					timestamp: now_msec(),
-					allow_read,
-					allow_write,
-					allow_owner,
-				},
-			)
-			.await?;
-
-		Ok(AdminRpc::Ok(format!(
-			"New permissions for {} on {}: read {}, write {}, owner {}.",
-			&key.key_id, &query.bucket, allow_read, allow_write, allow_owner
-		)))
-	}
-
-	async fn handle_bucket_deny(&self, query: &PermBucketOpt) -> Result<AdminRpc, Error> {
-		let helper = self.garage.locked_helper().await;
-
-		let bucket_id = helper
-			.bucket()
-			.admin_get_existing_matching_bucket(&query.bucket)
-			.await?;
-		let key = helper
-			.key()
-			.get_existing_matching_key(&query.key_pattern)
-			.await?;
-
-		let allow_read = !query.read && key.allow_read(&bucket_id);
-		let allow_write = !query.write && key.allow_write(&bucket_id);
-		let allow_owner = !query.owner && key.allow_owner(&bucket_id);
-
-		helper
-			.set_bucket_key_permissions(
-				bucket_id,
-				&key.key_id,
-				BucketKeyPerm {
-					timestamp: now_msec(),
-					allow_read,
-					allow_write,
-					allow_owner,
-				},
-			)
-			.await?;
-
-		Ok(AdminRpc::Ok(format!(
-			"New permissions for {} on {}: read {}, write {}, owner {}.",
-			&key.key_id, &query.bucket, allow_read, allow_write, allow_owner
-		)))
-	}
-
-	async fn handle_bucket_website(&self, query: &WebsiteOpt) -> Result<AdminRpc, Error> {
-		let bucket_id = self
-			.garage
-			.bucket_helper()
-			.admin_get_existing_matching_bucket(&query.bucket)
-			.await?;
-
-		let mut bucket = self
-			.garage
-			.bucket_helper()
-			.get_existing_bucket(bucket_id)
-			.await?;
-		let bucket_state = bucket.state.as_option_mut().unwrap();
-
-		if !(query.allow ^ query.deny) {
-			return Err(Error::BadRequest(
-				"You must specify exactly one flag, either --allow or --deny".to_string(),
-			));
-		}
-
-		let website = if query.allow {
-			Some(WebsiteConfig {
-				index_document: query.index_document.clone(),
-				error_document: query.error_document.clone(),
-			})
-		} else {
-			None
-		};
-
-		bucket_state.website_config.update(website);
-		self.garage.bucket_table.insert(&bucket).await?;
-
-		let msg = if query.allow {
-			format!("Website access allowed for {}", &query.bucket)
-		} else {
-			format!("Website access denied for {}", &query.bucket)
-		};
-
-		Ok(AdminRpc::Ok(msg))
-	}
-
-	async fn handle_bucket_set_quotas(&self, query: &SetQuotasOpt) -> Result<AdminRpc, Error> {
-		let bucket_id = self
-			.garage
-			.bucket_helper()
-			.admin_get_existing_matching_bucket(&query.bucket)
-			.await?;
-
-		let mut bucket = self
-			.garage
-			.bucket_helper()
-			.get_existing_bucket(bucket_id)
-			.await?;
-		let bucket_state = bucket.state.as_option_mut().unwrap();
-
-		if query.max_size.is_none() && query.max_objects.is_none() {
-			return Err(Error::BadRequest(
-				"You must specify either --max-size or --max-objects (or both) for this command to do something.".to_string(),
-			));
-		}
-
-		let mut quotas = bucket_state.quotas.get().clone();
-
-		match query.max_size.as_ref().map(String::as_ref) {
-			Some("none") => quotas.max_size = None,
-			Some(v) => {
-				let bs = v
-					.parse::<bytesize::ByteSize>()
-					.ok_or_bad_request(format!("Invalid size specified: {}", v))?;
-				quotas.max_size = Some(bs.as_u64());
-			}
-			_ => (),
-		}
-
-		match query.max_objects.as_ref().map(String::as_ref) {
-			Some("none") => quotas.max_objects = None,
-			Some(v) => {
-				let mo = v
-					.parse::<u64>()
-					.ok_or_bad_request(format!("Invalid number specified: {}", v))?;
-				quotas.max_objects = Some(mo);
-			}
-			_ => (),
-		}
-
-		bucket_state.quotas.update(quotas);
-		self.garage.bucket_table.insert(&bucket).await?;
-
-		Ok(AdminRpc::Ok(format!(
-			"Quotas updated for {}",
-			&query.bucket
-		)))
-	}
-
-	async fn handle_bucket_cleanup_incomplete_uploads(
-		&self,
-		query: &CleanupIncompleteUploadsOpt,
-	) -> Result<AdminRpc, Error> {
-		let mut bucket_ids = vec![];
-		for b in query.buckets.iter() {
-			bucket_ids.push(
-				self.garage
-					.bucket_helper()
-					.admin_get_existing_matching_bucket(b)
-					.await?,
-			);
-		}
-
-		let duration = parse_duration::parse::parse(&query.older_than)
-			.ok_or_bad_request("Invalid duration passed for --older-than parameter")?;
-
-		let mut ret = String::new();
-		for bucket in bucket_ids {
-			let count = self
-				.garage
-				.bucket_helper()
-				.cleanup_incomplete_uploads(&bucket, duration)
-				.await?;
-			writeln!(
-				&mut ret,
-				"Bucket {:?}: {} incomplete uploads aborted",
-				bucket, count
-			)
-			.unwrap();
-		}
-
-		Ok(AdminRpc::Ok(ret))
-	}
-}

src/garage/admin/key.rs

@@ -1,161 +0,0 @@
-use std::collections::HashMap;
-
-use garage_table::*;
-
-use garage_model::helper::error::*;
-use garage_model::key_table::*;
-
-use crate::cli::*;
-
-use super::*;
-
-impl AdminRpcHandler {
-	pub(super) async fn handle_key_cmd(&self, cmd: &KeyOperation) -> Result<AdminRpc, Error> {
-		match cmd {
-			KeyOperation::List => self.handle_list_keys().await,
-			KeyOperation::Info(query) => self.handle_key_info(query).await,
-			KeyOperation::Create(query) => self.handle_create_key(query).await,
-			KeyOperation::Rename(query) => self.handle_rename_key(query).await,
-			KeyOperation::Delete(query) => self.handle_delete_key(query).await,
-			KeyOperation::Allow(query) => self.handle_allow_key(query).await,
-			KeyOperation::Deny(query) => self.handle_deny_key(query).await,
-			KeyOperation::Import(query) => self.handle_import_key(query).await,
-		}
-	}
-
-	async fn handle_list_keys(&self) -> Result<AdminRpc, Error> {
-		let key_ids = self
-			.garage
-			.key_table
-			.get_range(
-				&EmptyKey,
-				None,
-				Some(KeyFilter::Deleted(DeletedFilter::NotDeleted)),
-				10000,
-				EnumerationOrder::Forward,
-			)
-			.await?
-			.iter()
-			.map(|k| (k.key_id.to_string(), k.params().unwrap().name.get().clone()))
-			.collect::<Vec<_>>();
-		Ok(AdminRpc::KeyList(key_ids))
-	}
-
-	async fn handle_key_info(&self, query: &KeyInfoOpt) -> Result<AdminRpc, Error> {
-		let mut key = self
-			.garage
-			.key_helper()
-			.get_existing_matching_key(&query.key_pattern)
-			.await?;
-
-		if !query.show_secret {
-			key.state.as_option_mut().unwrap().secret_key = "(redacted)".into();
-		}
-
-		self.key_info_result(key).await
-	}
-
-	async fn handle_create_key(&self, query: &KeyNewOpt) -> Result<AdminRpc, Error> {
-		let key = Key::new(&query.name);
-		self.garage.key_table.insert(&key).await?;
-		self.key_info_result(key).await
-	}
-
-	async fn handle_rename_key(&self, query: &KeyRenameOpt) -> Result<AdminRpc, Error> {
-		let mut key = self
-			.garage
-			.key_helper()
-			.get_existing_matching_key(&query.key_pattern)
-			.await?;
-		key.params_mut()
-			.unwrap()
-			.name
-			.update(query.new_name.clone());
-		self.garage.key_table.insert(&key).await?;
-		self.key_info_result(key).await
-	}
-
-	async fn handle_delete_key(&self, query: &KeyDeleteOpt) -> Result<AdminRpc, Error> {
-		let helper = self.garage.locked_helper().await;
-
-		let mut key = helper
-			.key()
-			.get_existing_matching_key(&query.key_pattern)
-			.await?;
-
-		if !query.yes {
-			return Err(Error::BadRequest(
-				"Add --yes flag to really perform this operation".to_string(),
-			));
-		}
-
-		helper.delete_key(&mut key).await?;
-
-		Ok(AdminRpc::Ok(format!(
-			"Key {} was deleted successfully.",
-			key.key_id
-		)))
-	}
-
-	async fn handle_allow_key(&self, query: &KeyPermOpt) -> Result<AdminRpc, Error> {
-		let mut key = self
-			.garage
-			.key_helper()
-			.get_existing_matching_key(&query.key_pattern)
-			.await?;
-		if query.create_bucket {
-			key.params_mut().unwrap().allow_create_bucket.update(true);
-		}
-		self.garage.key_table.insert(&key).await?;
-		self.key_info_result(key).await
-	}
-
-	async fn handle_deny_key(&self, query: &KeyPermOpt) -> Result<AdminRpc, Error> {
-		let mut key = self
-			.garage
-			.key_helper()
-			.get_existing_matching_key(&query.key_pattern)
-			.await?;
-		if query.create_bucket {
-			key.params_mut().unwrap().allow_create_bucket.update(false);
-		}
-		self.garage.key_table.insert(&key).await?;
-		self.key_info_result(key).await
-	}
-
-	async fn handle_import_key(&self, query: &KeyImportOpt) -> Result<AdminRpc, Error> {
-		if !query.yes {
-			return Err(Error::BadRequest("This command is intended to re-import keys that were previously generated by Garage. If you want to create a new key, use `garage key new` instead. Add the --yes flag if you really want to re-import a key.".to_string()));
-		}
-
-		let prev_key = self.garage.key_table.get(&EmptyKey, &query.key_id).await?;
-		if prev_key.is_some() {
-			return Err(Error::BadRequest(format!("Key {} already exists in data store. Even if it is deleted, we can't let you create a new key with the same ID. Sorry.", query.key_id)));
-		}
-
-		let imported_key = Key::import(&query.key_id, &query.secret_key, &query.name)
-			.ok_or_bad_request("Invalid key format")?;
-		self.garage.key_table.insert(&imported_key).await?;
-
-		self.key_info_result(imported_key).await
-	}
-
-	async fn key_info_result(&self, key: Key) -> Result<AdminRpc, Error> {
-		let mut relevant_buckets = HashMap::new();
-
-		for (id, _) in key
-			.state
-			.as_option()
-			.unwrap()
-			.authorized_buckets
-			.items()
-			.iter()
-		{
-			if let Some(b) = self.garage.bucket_table.get(&EmptyKey, id).await? {
-				relevant_buckets.insert(*id, b);
-			}
-		}
-
-		Ok(AdminRpc::KeyInfo(key, relevant_buckets))
-	}
-}

src/garage/admin/mod.rs

@@ -1,540 +0,0 @@
-mod block;
-mod bucket;
-mod key;
-
-use std::collections::HashMap;
-use std::fmt::Write;
-use std::future::Future;
-use std::sync::Arc;
-
-use futures::future::FutureExt;
-
-use serde::{Deserialize, Serialize};
-
-use format_table::format_table_to_string;
-
-use garage_util::background::BackgroundRunner;
-use garage_util::data::*;
-use garage_util::error::Error as GarageError;
-
-use garage_table::replication::*;
-use garage_table::*;
-
-use garage_rpc::layout::PARTITION_BITS;
-use garage_rpc::*;
-
-use garage_block::manager::BlockResyncErrorInfo;
-
-use garage_model::bucket_table::*;
-use garage_model::garage::Garage;
-use garage_model::helper::error::{Error, OkOrBadRequest};
-use garage_model::key_table::*;
-use garage_model::s3::mpu_table::MultipartUpload;
-use garage_model::s3::version_table::Version;
-
-use crate::cli::*;
-use crate::repair::online::launch_online_repair;
-
-pub const ADMIN_RPC_PATH: &str = "garage/admin_rpc.rs/Rpc";
-
-#[derive(Debug, Serialize, Deserialize)]
-#[allow(clippy::large_enum_variant)]
-pub enum AdminRpc {
-	BucketOperation(BucketOperation),
-	KeyOperation(KeyOperation),
-	LaunchRepair(RepairOpt),
-	Stats(StatsOpt),
-	Worker(WorkerOperation),
-	BlockOperation(BlockOperation),
-	MetaOperation(MetaOperation),
-
-	// Replies
-	Ok(String),
-	BucketList(Vec<Bucket>),
-	BucketInfo {
-		bucket: Bucket,
-		relevant_keys: HashMap<String, Key>,
-		counters: HashMap<String, i64>,
-		mpu_counters: HashMap<String, i64>,
-	},
-	KeyList(Vec<(String, String)>),
-	KeyInfo(Key, HashMap<Uuid, Bucket>),
-	WorkerList(
-		HashMap<usize, garage_util::background::WorkerInfo>,
-		WorkerListOpt,
-	),
-	WorkerVars(Vec<(Uuid, String, String)>),
-	WorkerInfo(usize, garage_util::background::WorkerInfo),
-	BlockErrorList(Vec<BlockResyncErrorInfo>),
-	BlockInfo {
-		hash: Hash,
-		refcount: u64,
-		versions: Vec<Result<Version, Uuid>>,
-		uploads: Vec<MultipartUpload>,
-	},
-}
-
-impl Rpc for AdminRpc {
-	type Response = Result<AdminRpc, Error>;
-}
-
-pub struct AdminRpcHandler {
-	garage: Arc<Garage>,
-	background: Arc<BackgroundRunner>,
-	endpoint: Arc<Endpoint<AdminRpc, Self>>,
-}
-
-impl AdminRpcHandler {
-	pub fn new(garage: Arc<Garage>, background: Arc<BackgroundRunner>) -> Arc<Self> {
-		let endpoint = garage.system.netapp.endpoint(ADMIN_RPC_PATH.into());
-		let admin = Arc::new(Self {
-			garage,
-			background,
-			endpoint,
-		});
-		admin.endpoint.set_handler(admin.clone());
-		admin
-	}
-
-	// ================ REPAIR COMMANDS ====================
-
-	async fn handle_launch_repair(self: &Arc<Self>, opt: RepairOpt) -> Result<AdminRpc, Error> {
-		if !opt.yes {
-			return Err(Error::BadRequest(
-				"Please provide the --yes flag to initiate repair operations.".to_string(),
-			));
-		}
-		if opt.all_nodes {
-			let mut opt_to_send = opt.clone();
-			opt_to_send.all_nodes = false;
-
-			let mut failures = vec![];
-			let all_nodes = self.garage.system.cluster_layout().all_nodes().to_vec();
-			for node in all_nodes.iter() {
-				let node = (*node).into();
-				let resp = self
-					.endpoint
-					.call(
-						&node,
-						AdminRpc::LaunchRepair(opt_to_send.clone()),
-						PRIO_NORMAL,
-					)
-					.await;
-				if !matches!(resp, Ok(Ok(_))) {
-					failures.push(node);
-				}
-			}
-			if failures.is_empty() {
-				Ok(AdminRpc::Ok("Repair launched on all nodes".to_string()))
-			} else {
-				Err(Error::BadRequest(format!(
-					"Could not launch repair on nodes: {:?} (launched successfully on other nodes)",
-					failures
-				)))
-			}
-		} else {
-			launch_online_repair(&self.garage, &self.background, opt).await?;
-			Ok(AdminRpc::Ok(format!(
-				"Repair launched on {:?}",
-				self.garage.system.id
-			)))
-		}
-	}
-
-	// ================ STATS COMMANDS ====================
-
-	async fn handle_stats(&self, opt: StatsOpt) -> Result<AdminRpc, Error> {
-		if opt.all_nodes {
-			let mut ret = String::new();
-			let mut all_nodes = self.garage.system.cluster_layout().all_nodes().to_vec();
-			for node in self.garage.system.get_known_nodes().iter() {
-				if node.is_up && !all_nodes.contains(&node.id) {
-					all_nodes.push(node.id);
-				}
-			}
-
-			for node in all_nodes.iter() {
-				let mut opt = opt.clone();
-				opt.all_nodes = false;
-				opt.skip_global = true;
-
-				writeln!(&mut ret, "\n======================").unwrap();
-				writeln!(&mut ret, "Stats for node {:?}:", node).unwrap();
-
-				let node_id = (*node).into();
-				match self
-					.endpoint
-					.call(&node_id, AdminRpc::Stats(opt), PRIO_NORMAL)
-					.await
-				{
-					Ok(Ok(AdminRpc::Ok(s))) => writeln!(&mut ret, "{}", s).unwrap(),
-					Ok(Ok(x)) => writeln!(&mut ret, "Bad answer: {:?}", x).unwrap(),
-					Ok(Err(e)) => writeln!(&mut ret, "Remote error: {}", e).unwrap(),
-					Err(e) => writeln!(&mut ret, "Network error: {}", e).unwrap(),
-				}
-			}
-
-			writeln!(&mut ret, "\n======================").unwrap();
-			write!(
-				&mut ret,
-				"Cluster statistics:\n\n{}",
-				self.gather_cluster_stats()
-			)
-			.unwrap();
-
-			Ok(AdminRpc::Ok(ret))
-		} else {
-			Ok(AdminRpc::Ok(self.gather_stats_local(opt)?))
-		}
-	}
-
-	fn gather_stats_local(&self, opt: StatsOpt) -> Result<String, Error> {
-		let mut ret = String::new();
-		writeln!(
-			&mut ret,
-			"\nGarage version: {} [features: {}]\nRust compiler version: {}",
-			garage_util::version::garage_version(),
-			garage_util::version::garage_features()
-				.map(|list| list.join(", "))
-				.unwrap_or_else(|| "(unknown)".into()),
-			garage_util::version::rust_version(),
-		)
-		.unwrap();
-
-		writeln!(&mut ret, "\nDatabase engine: {}", self.garage.db.engine()).unwrap();
-
-		// Gather table statistics
-		let mut table = vec!["  Table\tItems\tMklItems\tMklTodo\tGcTodo".into()];
-		table.push(self.gather_table_stats(&self.garage.bucket_table)?);
-		table.push(self.gather_table_stats(&self.garage.key_table)?);
-		table.push(self.gather_table_stats(&self.garage.object_table)?);
-		table.push(self.gather_table_stats(&self.garage.version_table)?);
-		table.push(self.gather_table_stats(&self.garage.block_ref_table)?);
-		write!(
-			&mut ret,
-			"\nTable stats:\n{}",
-			format_table_to_string(table)
-		)
-		.unwrap();
-
-		// Gather block manager statistics
-		writeln!(&mut ret, "\nBlock manager stats:").unwrap();
-		let rc_len = self.garage.block_manager.rc_len()?.to_string();
-
-		writeln!(
-			&mut ret,
-			"  number of RC entries (~= number of blocks): {}",
-			rc_len
-		)
-		.unwrap();
-		writeln!(
-			&mut ret,
-			"  resync queue length: {}",
-			self.garage.block_manager.resync.queue_len()?
-		)
-		.unwrap();
-		writeln!(
-			&mut ret,
-			"  blocks with resync errors: {}",
-			self.garage.block_manager.resync.errors_len()?
-		)
-		.unwrap();
-
-		if !opt.skip_global {
-			write!(&mut ret, "\n{}", self.gather_cluster_stats()).unwrap();
-		}
-
-		Ok(ret)
-	}
-
-	fn gather_cluster_stats(&self) -> String {
-		let mut ret = String::new();
-
-		// Gather storage node and free space statistics for current nodes
-		let layout = &self.garage.system.cluster_layout();
-		let mut node_partition_count = HashMap::<Uuid, u64>::new();
-		for short_id in layout.current().ring_assignment_data.iter() {
-			let id = layout.current().node_id_vec[*short_id as usize];
-			*node_partition_count.entry(id).or_default() += 1;
-		}
-		let node_info = self
-			.garage
-			.system
-			.get_known_nodes()
-			.into_iter()
-			.map(|n| (n.id, n))
-			.collect::<HashMap<_, _>>();
-
-		let mut table = vec!["  ID\tHostname\tZone\tCapacity\tPart.\tDataAvail\tMetaAvail".into()];
-		for (id, parts) in node_partition_count.iter() {
-			let info = node_info.get(id);
-			let status = info.map(|x| &x.status);
-			let role = layout.current().roles.get(id).and_then(|x| x.0.as_ref());
-			let hostname = status.and_then(|x| x.hostname.as_deref()).unwrap_or("?");
-			let zone = role.map(|x| x.zone.as_str()).unwrap_or("?");
-			let capacity = role
-				.map(|x| x.capacity_string())
-				.unwrap_or_else(|| "?".into());
-			let avail_str = |x| match x {
-				Some((avail, total)) => {
-					let pct = (avail as f64) / (total as f64) * 100.;
-					let avail = bytesize::ByteSize::b(avail);
-					let total = bytesize::ByteSize::b(total);
-					format!("{}/{} ({:.1}%)", avail, total, pct)
-				}
-				None => "?".into(),
-			};
-			let data_avail = avail_str(status.and_then(|x| x.data_disk_avail));
-			let meta_avail = avail_str(status.and_then(|x| x.meta_disk_avail));
-			table.push(format!(
-				"  {:?}\t{}\t{}\t{}\t{}\t{}\t{}",
-				id, hostname, zone, capacity, parts, data_avail, meta_avail
-			));
-		}
-		write!(
-			&mut ret,
-			"Storage nodes:\n{}",
-			format_table_to_string(table)
-		)
-		.unwrap();
-
-		let meta_part_avail = node_partition_count
-			.iter()
-			.filter_map(|(id, parts)| {
-				node_info
-					.get(id)
-					.and_then(|x| x.status.meta_disk_avail)
-					.map(|c| c.0 / *parts)
-			})
-			.collect::<Vec<_>>();
-		let data_part_avail = node_partition_count
-			.iter()
-			.filter_map(|(id, parts)| {
-				node_info
-					.get(id)
-					.and_then(|x| x.status.data_disk_avail)
-					.map(|c| c.0 / *parts)
-			})
-			.collect::<Vec<_>>();
-		if !meta_part_avail.is_empty() && !data_part_avail.is_empty() {
-			let meta_avail =
-				bytesize::ByteSize(meta_part_avail.iter().min().unwrap() * (1 << PARTITION_BITS));
-			let data_avail =
-				bytesize::ByteSize(data_part_avail.iter().min().unwrap() * (1 << PARTITION_BITS));
-			writeln!(
-				&mut ret,
-				"\nEstimated available storage space cluster-wide (might be lower in practice):"
-			)
-			.unwrap();
-			if meta_part_avail.len() < node_partition_count.len()
-				|| data_part_avail.len() < node_partition_count.len()
-			{
-				writeln!(&mut ret, "  data: < {}", data_avail).unwrap();
-				writeln!(&mut ret, "  metadata: < {}", meta_avail).unwrap();
-				writeln!(&mut ret, "A precise estimate could not be given as information is missing for some storage nodes.").unwrap();
-			} else {
-				writeln!(&mut ret, "  data: {}", data_avail).unwrap();
-				writeln!(&mut ret, "  metadata: {}", meta_avail).unwrap();
-			}
-		}
-
-		ret
-	}
-
-	fn gather_table_stats<F, R>(&self, t: &Arc<Table<F, R>>) -> Result<String, Error>
-	where
-		F: TableSchema + 'static,
-		R: TableReplication + 'static,
-	{
-		let data_len = t.data.store.len().map_err(GarageError::from)?.to_string();
-		let mkl_len = t.merkle_updater.merkle_tree_len()?.to_string();
-
-		Ok(format!(
-			"  {}\t{}\t{}\t{}\t{}",
-			F::TABLE_NAME,
-			data_len,
-			mkl_len,
-			t.merkle_updater.todo_len()?,
-			t.data.gc_todo_len()?
-		))
-	}
-
-	// ================ WORKER COMMANDS ====================
-
-	async fn handle_worker_cmd(&self, cmd: &WorkerOperation) -> Result<AdminRpc, Error> {
-		match cmd {
-			WorkerOperation::List { opt } => {
-				let workers = self.background.get_worker_info();
-				Ok(AdminRpc::WorkerList(workers, *opt))
-			}
-			WorkerOperation::Info { tid } => {
-				let info = self
-					.background
-					.get_worker_info()
-					.get(tid)
-					.ok_or_bad_request(format!("No worker with TID {}", tid))?
-					.clone();
-				Ok(AdminRpc::WorkerInfo(*tid, info))
-			}
-			WorkerOperation::Get {
-				all_nodes,
-				variable,
-			} => self.handle_get_var(*all_nodes, variable).await,
-			WorkerOperation::Set {
-				all_nodes,
-				variable,
-				value,
-			} => self.handle_set_var(*all_nodes, variable, value).await,
-		}
-	}
-
-	async fn handle_get_var(
-		&self,
-		all_nodes: bool,
-		variable: &Option<String>,
-	) -> Result<AdminRpc, Error> {
-		if all_nodes {
-			let mut ret = vec![];
-			let all_nodes = self.garage.system.cluster_layout().all_nodes().to_vec();
-			for node in all_nodes.iter() {
-				let node = (*node).into();
-				match self
-					.endpoint
-					.call(
-						&node,
-						AdminRpc::Worker(WorkerOperation::Get {
-							all_nodes: false,
-							variable: variable.clone(),
-						}),
-						PRIO_NORMAL,
-					)
-					.await??
-				{
-					AdminRpc::WorkerVars(v) => ret.extend(v),
-					m => return Err(GarageError::unexpected_rpc_message(m).into()),
-				}
-			}
-			Ok(AdminRpc::WorkerVars(ret))
-		} else {
-			#[allow(clippy::collapsible_else_if)]
-			if let Some(v) = variable {
-				Ok(AdminRpc::WorkerVars(vec![(
-					self.garage.system.id,
-					v.clone(),
-					self.garage.bg_vars.get(v)?,
-				)]))
-			} else {
-				let mut vars = self.garage.bg_vars.get_all();
-				vars.sort();
-				Ok(AdminRpc::WorkerVars(
-					vars.into_iter()
-						.map(|(k, v)| (self.garage.system.id, k.to_string(), v))
-						.collect(),
-				))
-			}
-		}
-	}
-
-	async fn handle_set_var(
-		&self,
-		all_nodes: bool,
-		variable: &str,
-		value: &str,
-	) -> Result<AdminRpc, Error> {
-		if all_nodes {
-			let mut ret = vec![];
-			let all_nodes = self.garage.system.cluster_layout().all_nodes().to_vec();
-			for node in all_nodes.iter() {
-				let node = (*node).into();
-				match self
-					.endpoint
-					.call(
-						&node,
-						AdminRpc::Worker(WorkerOperation::Set {
-							all_nodes: false,
-							variable: variable.to_string(),
-							value: value.to_string(),
-						}),
-						PRIO_NORMAL,
-					)
-					.await??
-				{
-					AdminRpc::WorkerVars(v) => ret.extend(v),
-					m => return Err(GarageError::unexpected_rpc_message(m).into()),
-				}
-			}
-			Ok(AdminRpc::WorkerVars(ret))
-		} else {
-			self.garage.bg_vars.set(variable, value)?;
-			Ok(AdminRpc::WorkerVars(vec![(
-				self.garage.system.id,
-				variable.to_string(),
-				value.to_string(),
-			)]))
-		}
-	}
-
-	// ================ META DB COMMANDS ====================
-
-	async fn handle_meta_cmd(self: &Arc<Self>, mo: &MetaOperation) -> Result<AdminRpc, Error> {
-		match mo {
-			MetaOperation::Snapshot { all: true } => {
-				let to = self.garage.system.cluster_layout().all_nodes().to_vec();
-
-				let resps = futures::future::join_all(to.iter().map(|to| async move {
-					let to = (*to).into();
-					self.endpoint
-						.call(
-							&to,
-							AdminRpc::MetaOperation(MetaOperation::Snapshot { all: false }),
-							PRIO_NORMAL,
-						)
-						.await?
-				}))
-				.await;
-
-				let mut ret = vec![];
-				for (to, resp) in to.iter().zip(resps.iter()) {
-					let res_str = match resp {
-						Ok(_) => "ok".to_string(),
-						Err(e) => format!("error: {}", e),
-					};
-					ret.push(format!("{:?}\t{}", to, res_str));
-				}
-
-				if resps.iter().any(Result::is_err) {
-					Err(GarageError::Message(format_table_to_string(ret)).into())
-				} else {
-					Ok(AdminRpc::Ok(format_table_to_string(ret)))
-				}
-			}
-			MetaOperation::Snapshot { all: false } => {
-				garage_model::snapshot::async_snapshot_metadata(&self.garage).await?;
-				Ok(AdminRpc::Ok("Snapshot has been saved.".into()))
-			}
-		}
-	}
-}
-
-impl EndpointHandler<AdminRpc> for AdminRpcHandler {
-	fn handle(
-		self: &Arc<Self>,
-		message: &AdminRpc,
-		_from: NodeID,
-	) -> impl Future<Output = Result<AdminRpc, Error>> + Send {
-		let self2 = self.clone();
-		async move {
-			match message {
-				AdminRpc::BucketOperation(bo) => self2.handle_bucket_cmd(bo).await,
-				AdminRpc::KeyOperation(ko) => self2.handle_key_cmd(ko).await,
-				AdminRpc::LaunchRepair(opt) => self2.handle_launch_repair(opt.clone()).await,
-				AdminRpc::Stats(opt) => self2.handle_stats(opt.clone()).await,
-				AdminRpc::Worker(wo) => self2.handle_worker_cmd(wo).await,
-				AdminRpc::BlockOperation(bo) => self2.handle_block_cmd(bo).await,
-				AdminRpc::MetaOperation(mo) => self2.handle_meta_cmd(mo).await,
-				m => Err(GarageError::unexpected_rpc_message(m).into()),
-			}
-		}
-		.boxed()
-	}
-}

src/garage/cli/cmd.rs

@@ -1,280 +0,0 @@
-use std::collections::{HashMap, HashSet};
-use std::time::Duration;
-
-use format_table::format_table;
-use garage_util::error::*;
-
-use garage_rpc::layout::*;
-use garage_rpc::system::*;
-use garage_rpc::*;
-
-use garage_model::helper::error::Error as HelperError;
-
-use crate::admin::*;
-use crate::cli::*;
-
-pub async fn cli_command_dispatch(
-	cmd: Command,
-	system_rpc_endpoint: &Endpoint<SystemRpc, ()>,
-	admin_rpc_endpoint: &Endpoint<AdminRpc, ()>,
-	rpc_host: NodeID,
-) -> Result<(), HelperError> {
-	match cmd {
-		Command::Status => Ok(cmd_status(system_rpc_endpoint, rpc_host).await?),
-		Command::Node(NodeOperation::Connect(connect_opt)) => {
-			Ok(cmd_connect(system_rpc_endpoint, rpc_host, connect_opt).await?)
-		}
-		Command::Layout(layout_opt) => {
-			Ok(cli_layout_command_dispatch(layout_opt, system_rpc_endpoint, rpc_host).await?)
-		}
-		Command::Bucket(bo) => {
-			cmd_admin(admin_rpc_endpoint, rpc_host, AdminRpc::BucketOperation(bo)).await
-		}
-		Command::Key(ko) => {
-			cmd_admin(admin_rpc_endpoint, rpc_host, AdminRpc::KeyOperation(ko)).await
-		}
-		Command::Repair(ro) => {
-			cmd_admin(admin_rpc_endpoint, rpc_host, AdminRpc::LaunchRepair(ro)).await
-		}
-		Command::Stats(so) => cmd_admin(admin_rpc_endpoint, rpc_host, AdminRpc::Stats(so)).await,
-		Command::Worker(wo) => cmd_admin(admin_rpc_endpoint, rpc_host, AdminRpc::Worker(wo)).await,
-		Command::Block(bo) => {
-			cmd_admin(admin_rpc_endpoint, rpc_host, AdminRpc::BlockOperation(bo)).await
-		}
-		Command::Meta(mo) => {
-			cmd_admin(admin_rpc_endpoint, rpc_host, AdminRpc::MetaOperation(mo)).await
-		}
-		_ => unreachable!(),
-	}
-}
-
-pub async fn cmd_status(rpc_cli: &Endpoint<SystemRpc, ()>, rpc_host: NodeID) -> Result<(), Error> {
-	let status = fetch_status(rpc_cli, rpc_host).await?;
-	let layout = fetch_layout(rpc_cli, rpc_host).await?;
-
-	println!("==== HEALTHY NODES ====");
-	let mut healthy_nodes =
-		vec!["ID\tHostname\tAddress\tTags\tZone\tCapacity\tDataAvail".to_string()];
-	for adv in status.iter().filter(|adv| adv.is_up) {
-		let host = adv.status.hostname.as_deref().unwrap_or("?");
-		let addr = match adv.addr {
-			Some(addr) => addr.to_string(),
-			None => "N/A".to_string(),
-		};
-		if let Some(NodeRoleV(Some(cfg))) = layout.current().roles.get(&adv.id) {
-			let data_avail = match &adv.status.data_disk_avail {
-				_ if cfg.capacity.is_none() => "N/A".into(),
-				Some((avail, total)) => {
-					let pct = (*avail as f64) / (*total as f64) * 100.;
-					let avail = bytesize::ByteSize::b(*avail);
-					format!("{} ({:.1}%)", avail, pct)
-				}
-				None => "?".into(),
-			};
-			healthy_nodes.push(format!(
-				"{id:?}\t{host}\t{addr}\t[{tags}]\t{zone}\t{capacity}\t{data_avail}",
-				id = adv.id,
-				host = host,
-				addr = addr,
-				tags = cfg.tags.join(","),
-				zone = cfg.zone,
-				capacity = cfg.capacity_string(),
-				data_avail = data_avail,
-			));
-		} else {
-			let prev_role = layout
-				.versions
-				.iter()
-				.rev()
-				.find_map(|x| match x.roles.get(&adv.id) {
-					Some(NodeRoleV(Some(cfg))) => Some(cfg),
-					_ => None,
-				});
-			if let Some(cfg) = prev_role {
-				healthy_nodes.push(format!(
-					"{id:?}\t{host}\t{addr}\t[{tags}]\t{zone}\tdraining metadata...",
-					id = adv.id,
-					host = host,
-					addr = addr,
-					tags = cfg.tags.join(","),
-					zone = cfg.zone,
-				));
-			} else {
-				let new_role = match layout.staging.get().roles.get(&adv.id) {
-					Some(NodeRoleV(Some(_))) => "pending...",
-					_ => "NO ROLE ASSIGNED",
-				};
-				healthy_nodes.push(format!(
-					"{id:?}\t{h}\t{addr}\t\t\t{new_role}",
-					id = adv.id,
-					h = host,
-					addr = addr,
-					new_role = new_role,
-				));
-			}
-		}
-	}
-	format_table(healthy_nodes);
-
-	// Determine which nodes are unhealthy and print that to stdout
-	let status_map = status
-		.iter()
-		.map(|adv| (adv.id, adv))
-		.collect::<HashMap<_, _>>();
-
-	let tf = timeago::Formatter::new();
-	let mut drain_msg = false;
-	let mut failed_nodes = vec!["ID\tHostname\tTags\tZone\tCapacity\tLast seen".to_string()];
-	let mut listed = HashSet::new();
-	for ver in layout.versions.iter().rev() {
-		for (node, _, role) in ver.roles.items().iter() {
-			let cfg = match role {
-				NodeRoleV(Some(role)) if role.capacity.is_some() => role,
-				_ => continue,
-			};
-
-			if listed.contains(node) {
-				continue;
-			}
-			listed.insert(*node);
-
-			let adv = status_map.get(node);
-			if adv.map(|x| x.is_up).unwrap_or(false) {
-				continue;
-			}
-
-			// Node is in a layout version, is not a gateway node, and is not up:
-			// it is in a failed state, add proper line to the output
-			let (host, last_seen) = match adv {
-				Some(adv) => (
-					adv.status.hostname.as_deref().unwrap_or("?"),
-					adv.last_seen_secs_ago
-						.map(|s| tf.convert(Duration::from_secs(s)))
-						.unwrap_or_else(|| "never seen".into()),
-				),
-				None => ("??", "never seen".into()),
-			};
-			let capacity = if ver.version == layout.current().version {
-				cfg.capacity_string()
-			} else {
-				drain_msg = true;
-				"draining metadata...".to_string()
-			};
-			failed_nodes.push(format!(
-				"{id:?}\t{host}\t[{tags}]\t{zone}\t{capacity}\t{last_seen}",
-				id = node,
-				host = host,
-				tags = cfg.tags.join(","),
-				zone = cfg.zone,
-				capacity = capacity,
-				last_seen = last_seen,
-			));
-		}
-	}
-
-	if failed_nodes.len() > 1 {
-		println!("\n==== FAILED NODES ====");
-		format_table(failed_nodes);
-		if drain_msg {
-			println!();
-			println!("Your cluster is expecting to drain data from nodes that are currently unavailable.");
-			println!("If these nodes are definitely dead, please review the layout history with");
-			println!(
-				"`garage layout history` and use `garage layout skip-dead-nodes` to force progress."
-			);
-		}
-	}
-
-	if print_staging_role_changes(&layout) {
-		println!();
-		println!("Please use `garage layout show` to check the proposed new layout and apply it.");
-		println!();
-	}
-
-	Ok(())
-}
-
-pub async fn cmd_connect(
-	rpc_cli: &Endpoint<SystemRpc, ()>,
-	rpc_host: NodeID,
-	args: ConnectNodeOpt,
-) -> Result<(), Error> {
-	match rpc_cli
-		.call(&rpc_host, SystemRpc::Connect(args.node), PRIO_NORMAL)
-		.await??
-	{
-		SystemRpc::Ok => {
-			println!("Success.");
-			Ok(())
-		}
-		m => Err(Error::unexpected_rpc_message(m)),
-	}
-}
-
-pub async fn cmd_admin(
-	rpc_cli: &Endpoint<AdminRpc, ()>,
-	rpc_host: NodeID,
-	args: AdminRpc,
-) -> Result<(), HelperError> {
-	match rpc_cli.call(&rpc_host, args, PRIO_NORMAL).await?? {
-		AdminRpc::Ok(msg) => {
-			println!("{}", msg);
-		}
-		AdminRpc::BucketList(bl) => {
-			print_bucket_list(bl);
-		}
-		AdminRpc::BucketInfo {
-			bucket,
-			relevant_keys,
-			counters,
-			mpu_counters,
-		} => {
-			print_bucket_info(&bucket, &relevant_keys, &counters, &mpu_counters);
-		}
-		AdminRpc::KeyList(kl) => {
-			print_key_list(kl);
-		}
-		AdminRpc::KeyInfo(key, rb) => {
-			print_key_info(&key, &rb);
-		}
-		AdminRpc::WorkerList(wi, wlo) => {
-			print_worker_list(wi, wlo);
-		}
-		AdminRpc::WorkerVars(wv) => {
-			print_worker_vars(wv);
-		}
-		AdminRpc::WorkerInfo(tid, wi) => {
-			print_worker_info(tid, wi);
-		}
-		AdminRpc::BlockErrorList(el) => {
-			print_block_error_list(el);
-		}
-		AdminRpc::BlockInfo {
-			hash,
-			refcount,
-			versions,
-			uploads,
-		} => {
-			print_block_info(hash, refcount, versions, uploads);
-		}
-		r => {
-			error!("Unexpected response: {:?}", r);
-		}
-	}
-	Ok(())
-}
-
-// ---- utility ----
-
-pub async fn fetch_status(
-	rpc_cli: &Endpoint<SystemRpc, ()>,
-	rpc_host: NodeID,
-) -> Result<Vec<KnownNodeInfo>, Error> {
-	match rpc_cli
-		.call(&rpc_host, SystemRpc::GetKnownNodes, PRIO_NORMAL)
-		.await??
-	{
-		SystemRpc::ReturnKnownNodes(nodes) => Ok(nodes),
-		resp => Err(Error::unexpected_rpc_message(resp)),
-	}
-}

src/garage/cli/layout.rs

@@ -1,584 +0,0 @@
-use bytesize::ByteSize;
-
-use format_table::format_table;
-use garage_util::crdt::Crdt;
-use garage_util::error::*;
-
-use garage_rpc::layout::*;
-use garage_rpc::system::*;
-use garage_rpc::*;
-
-use crate::cli::*;
-
-pub async fn cli_layout_command_dispatch(
-	cmd: LayoutOperation,
-	system_rpc_endpoint: &Endpoint<SystemRpc, ()>,
-	rpc_host: NodeID,
-) -> Result<(), Error> {
-	match cmd {
-		LayoutOperation::Assign(assign_opt) => {
-			cmd_assign_role(system_rpc_endpoint, rpc_host, assign_opt).await
-		}
-		LayoutOperation::Remove(remove_opt) => {
-			cmd_remove_role(system_rpc_endpoint, rpc_host, remove_opt).await
-		}
-		LayoutOperation::Show => cmd_show_layout(system_rpc_endpoint, rpc_host).await,
-		LayoutOperation::Apply(apply_opt) => {
-			cmd_apply_layout(system_rpc_endpoint, rpc_host, apply_opt).await
-		}
-		LayoutOperation::Revert(revert_opt) => {
-			cmd_revert_layout(system_rpc_endpoint, rpc_host, revert_opt).await
-		}
-		LayoutOperation::Config(config_opt) => {
-			cmd_config_layout(system_rpc_endpoint, rpc_host, config_opt).await
-		}
-		LayoutOperation::History => cmd_layout_history(system_rpc_endpoint, rpc_host).await,
-		LayoutOperation::SkipDeadNodes(assume_sync_opt) => {
-			cmd_layout_skip_dead_nodes(system_rpc_endpoint, rpc_host, assume_sync_opt).await
-		}
-	}
-}
-
-pub async fn cmd_assign_role(
-	rpc_cli: &Endpoint<SystemRpc, ()>,
-	rpc_host: NodeID,
-	args: AssignRoleOpt,
-) -> Result<(), Error> {
-	let status = match rpc_cli
-		.call(&rpc_host, SystemRpc::GetKnownNodes, PRIO_NORMAL)
-		.await??
-	{
-		SystemRpc::ReturnKnownNodes(nodes) => nodes,
-		resp => return Err(Error::Message(format!("Invalid RPC response: {:?}", resp))),
-	};
-
-	let mut layout = fetch_layout(rpc_cli, rpc_host).await?;
-	let all_nodes = layout.get_all_nodes();
-
-	let added_nodes = args
-		.node_ids
-		.iter()
-		.map(|node_id| {
-			find_matching_node(
-				status
-					.iter()
-					.map(|adv| adv.id)
-					.chain(all_nodes.iter().cloned()),
-				node_id,
-			)
-		})
-		.collect::<Result<Vec<_>, _>>()?;
-
-	let mut roles = layout.current().roles.clone();
-	roles.merge(&layout.staging.get().roles);
-
-	for replaced in args.replace.iter() {
-		let replaced_node = find_matching_node(all_nodes.iter().cloned(), replaced)?;
-		match roles.get(&replaced_node) {
-			Some(NodeRoleV(Some(_))) => {
-				layout
-					.staging
-					.get_mut()
-					.roles
-					.merge(&roles.update_mutator(replaced_node, NodeRoleV(None)));
-			}
-			_ => {
-				return Err(Error::Message(format!(
-					"Cannot replace node {:?} as it is not currently in planned layout",
-					replaced_node
-				)));
-			}
-		}
-	}
-
-	if args.capacity.is_some() && args.gateway {
-		return Err(Error::Message(
-				"-c and -g are mutually exclusive, please configure node either with c>0 to act as a storage node or with -g to act as a gateway node".into()));
-	}
-	if args.capacity == Some(ByteSize::b(0)) {
-		return Err(Error::Message("Invalid capacity value: 0".into()));
-	}
-
-	for added_node in added_nodes {
-		let new_entry = match roles.get(&added_node) {
-			Some(NodeRoleV(Some(old))) => {
-				let capacity = match args.capacity {
-					Some(c) => Some(c.as_u64()),
-					None if args.gateway => None,
-					None => old.capacity,
-				};
-				let tags = if args.tags.is_empty() {
-					old.tags.clone()
-				} else {
-					args.tags.clone()
-				};
-				NodeRole {
-					zone: args.zone.clone().unwrap_or_else(|| old.zone.to_string()),
-					capacity,
-					tags,
-				}
-			}
-			_ => {
-				let capacity = match args.capacity {
-					Some(c) => Some(c.as_u64()),
-					None if args.gateway => None,
-					None => return Err(Error::Message(
-							"Please specify a capacity with the -c flag, or set node explicitly as gateway with -g".into())),
-				};
-				NodeRole {
-					zone: args
-						.zone
-						.clone()
-						.ok_or("Please specify a zone with the -z flag")?,
-					capacity,
-					tags: args.tags.clone(),
-				}
-			}
-		};
-
-		layout
-			.staging
-			.get_mut()
-			.roles
-			.merge(&roles.update_mutator(added_node, NodeRoleV(Some(new_entry))));
-	}
-
-	send_layout(rpc_cli, rpc_host, layout).await?;
-
-	println!("Role changes are staged but not yet committed.");
-	println!("Use `garage layout show` to view staged role changes,");
-	println!("and `garage layout apply` to enact staged changes.");
-	Ok(())
-}
-
-pub async fn cmd_remove_role(
-	rpc_cli: &Endpoint<SystemRpc, ()>,
-	rpc_host: NodeID,
-	args: RemoveRoleOpt,
-) -> Result<(), Error> {
-	let mut layout = fetch_layout(rpc_cli, rpc_host).await?;
-
-	let mut roles = layout.current().roles.clone();
-	roles.merge(&layout.staging.get().roles);
-
-	let deleted_node =
-		find_matching_node(roles.items().iter().map(|(id, _, _)| *id), &args.node_id)?;
-
-	layout
-		.staging
-		.get_mut()
-		.roles
-		.merge(&roles.update_mutator(deleted_node, NodeRoleV(None)));
-
-	send_layout(rpc_cli, rpc_host, layout).await?;
-
-	println!("Role removal is staged but not yet committed.");
-	println!("Use `garage layout show` to view staged role changes,");
-	println!("and `garage layout apply` to enact staged changes.");
-	Ok(())
-}
-
-pub async fn cmd_show_layout(
-	rpc_cli: &Endpoint<SystemRpc, ()>,
-	rpc_host: NodeID,
-) -> Result<(), Error> {
-	let layout = fetch_layout(rpc_cli, rpc_host).await?;
-
-	println!("==== CURRENT CLUSTER LAYOUT ====");
-	print_cluster_layout(layout.current(), "No nodes currently have a role in the cluster.\nSee `garage status` to view available nodes.");
-	println!();
-	println!(
-		"Current cluster layout version: {}",
-		layout.current().version
-	);
-
-	let has_role_changes = print_staging_role_changes(&layout);
-	if has_role_changes {
-		let v = layout.current().version;
-		let res_apply = layout.apply_staged_changes(Some(v + 1));
-
-		// this will print the stats of what partitions
-		// will move around when we apply
-		match res_apply {
-			Ok((layout, msg)) => {
-				println!();
-				println!("==== NEW CLUSTER LAYOUT AFTER APPLYING CHANGES ====");
-				print_cluster_layout(layout.current(), "No nodes have a role in the new layout.");
-				println!();
-
-				for line in msg.iter() {
-					println!("{}", line);
-				}
-				println!("To enact the staged role changes, type:");
-				println!();
-				println!("    garage layout apply --version {}", v + 1);
-				println!();
-				println!("You can also revert all proposed changes with: garage layout revert");
-			}
-			Err(e) => {
-				println!("Error while trying to compute the assignment: {}", e);
-				println!("This new layout cannot yet be applied.");
-				println!("You can also revert all proposed changes with: garage layout revert");
-			}
-		}
-	}
-
-	Ok(())
-}
-
-pub async fn cmd_apply_layout(
-	rpc_cli: &Endpoint<SystemRpc, ()>,
-	rpc_host: NodeID,
-	apply_opt: ApplyLayoutOpt,
-) -> Result<(), Error> {
-	let layout = fetch_layout(rpc_cli, rpc_host).await?;
-
-	let (layout, msg) = layout.apply_staged_changes(apply_opt.version)?;
-	for line in msg.iter() {
-		println!("{}", line);
-	}
-
-	send_layout(rpc_cli, rpc_host, layout).await?;
-
-	println!("New cluster layout with updated role assignment has been applied in cluster.");
-	println!("Data will now be moved around between nodes accordingly.");
-
-	Ok(())
-}
-
-pub async fn cmd_revert_layout(
-	rpc_cli: &Endpoint<SystemRpc, ()>,
-	rpc_host: NodeID,
-	revert_opt: RevertLayoutOpt,
-) -> Result<(), Error> {
-	if !revert_opt.yes {
-		return Err(Error::Message(
-			"Please add the --yes flag to run the layout revert operation".into(),
-		));
-	}
-
-	let layout = fetch_layout(rpc_cli, rpc_host).await?;
-
-	let layout = layout.revert_staged_changes()?;
-
-	send_layout(rpc_cli, rpc_host, layout).await?;
-
-	println!("All proposed role changes in cluster layout have been canceled.");
-	Ok(())
-}
-
-pub async fn cmd_config_layout(
-	rpc_cli: &Endpoint<SystemRpc, ()>,
-	rpc_host: NodeID,
-	config_opt: ConfigLayoutOpt,
-) -> Result<(), Error> {
-	let mut layout = fetch_layout(rpc_cli, rpc_host).await?;
-
-	let mut did_something = false;
-	match config_opt.redundancy {
-		None => (),
-		Some(r_str) => {
-			let r = r_str
-				.parse::<ZoneRedundancy>()
-				.ok_or_message("invalid zone redundancy value")?;
-			if let ZoneRedundancy::AtLeast(r_int) = r {
-				if r_int > layout.current().replication_factor {
-					return Err(Error::Message(format!(
-						"The zone redundancy must be smaller or equal to the \
-                    replication factor ({}).",
-						layout.current().replication_factor
-					)));
-				} else if r_int < 1 {
-					return Err(Error::Message(
-						"The zone redundancy must be at least 1.".into(),
-					));
-				}
-			}
-
-			layout
-				.staging
-				.get_mut()
-				.parameters
-				.update(LayoutParameters { zone_redundancy: r });
-			println!("The zone redundancy parameter has been set to '{}'.", r);
-			did_something = true;
-		}
-	}
-
-	if !did_something {
-		return Err(Error::Message(
-			"Please specify an action for `garage layout config`".into(),
-		));
-	}
-
-	send_layout(rpc_cli, rpc_host, layout).await?;
-	Ok(())
-}
-
-pub async fn cmd_layout_history(
-	rpc_cli: &Endpoint<SystemRpc, ()>,
-	rpc_host: NodeID,
-) -> Result<(), Error> {
-	let layout = fetch_layout(rpc_cli, rpc_host).await?;
-	let min_stored = layout.min_stored();
-
-	println!("==== LAYOUT HISTORY ====");
-	let mut table = vec!["Version\tStatus\tStorage nodes\tGateway nodes".to_string()];
-	for ver in layout
-		.versions
-		.iter()
-		.rev()
-		.chain(layout.old_versions.iter().rev())
-	{
-		let status = if ver.version == layout.current().version {
-			"current"
-		} else if ver.version >= min_stored {
-			"draining"
-		} else {
-			"historical"
-		};
-		table.push(format!(
-			"#{}\t{}\t{}\t{}",
-			ver.version,
-			status,
-			ver.roles
-				.items()
-				.iter()
-				.filter(|(_, _, x)| matches!(x, NodeRoleV(Some(c)) if c.capacity.is_some()))
-				.count(),
-			ver.roles
-				.items()
-				.iter()
-				.filter(|(_, _, x)| matches!(x, NodeRoleV(Some(c)) if c.capacity.is_none()))
-				.count(),
-		));
-	}
-	format_table(table);
-	println!();
-
-	if layout.versions.len() > 1 {
-		println!("==== UPDATE TRACKERS ====");
-		println!("Several layout versions are currently live in the cluster, and data is being migrated.");
-		println!(
-			"This is the internal data that Garage stores to know which nodes have what data."
-		);
-		println!();
-		let mut table = vec!["Node\tAck\tSync\tSync_ack".to_string()];
-		let all_nodes = layout.get_all_nodes();
-		for node in all_nodes.iter() {
-			table.push(format!(
-				"{:?}\t#{}\t#{}\t#{}",
-				node,
-				layout.update_trackers.ack_map.get(node, min_stored),
-				layout.update_trackers.sync_map.get(node, min_stored),
-				layout.update_trackers.sync_ack_map.get(node, min_stored),
-			));
-		}
-		table[1..].sort();
-		format_table(table);
-
-		let min_ack = layout
-			.update_trackers
-			.ack_map
-			.min_among(&all_nodes, layout.min_stored());
-
-		println!();
-		println!(
-			"If some nodes are not catching up to the latest layout version in the update trackers,"
-		);
-		println!("it might be because they are offline or unable to complete a sync successfully.");
-		if min_ack < layout.current().version {
-			println!(
-				"You may force progress using `garage layout skip-dead-nodes --version {}`",
-				layout.current().version
-			);
-		} else {
-			println!(
-				"You may force progress using `garage layout skip-dead-nodes --version {} --allow-missing-data`.",
-				layout.current().version
-			);
-		}
-	} else {
-		println!("Your cluster is currently in a stable state with a single live layout version.");
-		println!("No metadata migration is in progress. Note that the migration of data blocks is not tracked,");
-		println!(
-			"so you might want to keep old nodes online until their data directories become empty."
-		);
-	}
-
-	Ok(())
-}
-
-pub async fn cmd_layout_skip_dead_nodes(
-	rpc_cli: &Endpoint<SystemRpc, ()>,
-	rpc_host: NodeID,
-	opt: SkipDeadNodesOpt,
-) -> Result<(), Error> {
-	let status = fetch_status(rpc_cli, rpc_host).await?;
-	let mut layout = fetch_layout(rpc_cli, rpc_host).await?;
-
-	if layout.versions.len() == 1 {
-		return Err(Error::Message(
-			"This command cannot be called when there is only one live cluster layout version"
-				.into(),
-		));
-	}
-
-	let min_v = layout.min_stored();
-	if opt.version <= min_v || opt.version > layout.current().version {
-		return Err(Error::Message(format!(
-			"Invalid version, you may use the following version numbers: {}",
-			(min_v + 1..=layout.current().version)
-				.map(|x| x.to_string())
-				.collect::<Vec<_>>()
-				.join(" ")
-		)));
-	}
-
-	let all_nodes = layout.get_all_nodes();
-	let mut did_something = false;
-	for node in all_nodes.iter() {
-		// Update ACK tracker for dead nodes or for all nodes if --allow-missing-data
-		if opt.allow_missing_data || !status.iter().any(|x| x.id == *node && x.is_up) {
-			if layout.update_trackers.ack_map.set_max(*node, opt.version) {
-				println!("Increased the ACK tracker for node {:?}", node);
-				did_something = true;
-			}
-		}
-
-		// If --allow-missing-data, update SYNC tracker for all nodes.
-		if opt.allow_missing_data {
-			if layout.update_trackers.sync_map.set_max(*node, opt.version) {
-				println!("Increased the SYNC tracker for node {:?}", node);
-				did_something = true;
-			}
-		}
-	}
-
-	if did_something {
-		send_layout(rpc_cli, rpc_host, layout).await?;
-		println!("Success.");
-		Ok(())
-	} else if !opt.allow_missing_data {
-		Err(Error::Message("Nothing was done, try passing the `--allow-missing-data` flag to force progress even when not enough nodes can complete a metadata sync.".into()))
-	} else {
-		Err(Error::Message(
-			"Sorry, there is nothing I can do for you. Please wait patiently. If you ask for help, please send the output of the `garage layout history` command.".into(),
-		))
-	}
-}
-
-// --- utility ---
-
-pub async fn fetch_layout(
-	rpc_cli: &Endpoint<SystemRpc, ()>,
-	rpc_host: NodeID,
-) -> Result<LayoutHistory, Error> {
-	match rpc_cli
-		.call(&rpc_host, SystemRpc::PullClusterLayout, PRIO_NORMAL)
-		.await??
-	{
-		SystemRpc::AdvertiseClusterLayout(t) => Ok(t),
-		resp => Err(Error::unexpected_rpc_message(resp)),
-	}
-}
-
-pub async fn send_layout(
-	rpc_cli: &Endpoint<SystemRpc, ()>,
-	rpc_host: NodeID,
-	layout: LayoutHistory,
-) -> Result<(), Error> {
-	rpc_cli
-		.call(
-			&rpc_host,
-			SystemRpc::AdvertiseClusterLayout(layout),
-			PRIO_NORMAL,
-		)
-		.await??;
-	Ok(())
-}
-
-pub fn print_cluster_layout(layout: &LayoutVersion, empty_msg: &str) {
-	let mut table = vec!["ID\tTags\tZone\tCapacity\tUsable capacity".to_string()];
-	for (id, _, role) in layout.roles.items().iter() {
-		let role = match &role.0 {
-			Some(r) => r,
-			_ => continue,
-		};
-		let tags = role.tags.join(",");
-		let usage = layout.get_node_usage(id).unwrap_or(0);
-		let capacity = layout.get_node_capacity(id).unwrap_or(0);
-		if capacity > 0 {
-			table.push(format!(
-				"{:?}\t{}\t{}\t{}\t{} ({:.1}%)",
-				id,
-				tags,
-				role.zone,
-				role.capacity_string(),
-				ByteSize::b(usage as u64 * layout.partition_size).to_string_as(false),
-				(100.0 * usage as f32 * layout.partition_size as f32) / (capacity as f32)
-			));
-		} else {
-			table.push(format!(
-				"{:?}\t{}\t{}\t{}",
-				id,
-				tags,
-				role.zone,
-				role.capacity_string()
-			));
-		};
-	}
-	if table.len() > 1 {
-		format_table(table);
-		println!();
-		println!("Zone redundancy: {}", layout.parameters.zone_redundancy);
-	} else {
-		println!("{}", empty_msg);
-	}
-}
-
-pub fn print_staging_role_changes(layout: &LayoutHistory) -> bool {
-	let staging = layout.staging.get();
-	let has_role_changes = staging
-		.roles
-		.items()
-		.iter()
-		.any(|(k, _, v)| layout.current().roles.get(k) != Some(v));
-	let has_layout_changes = *staging.parameters.get() != layout.current().parameters;
-
-	if has_role_changes || has_layout_changes {
-		println!();
-		println!("==== STAGED ROLE CHANGES ====");
-		if has_role_changes {
-			let mut table = vec!["ID\tTags\tZone\tCapacity".to_string()];
-			for (id, _, role) in staging.roles.items().iter() {
-				if layout.current().roles.get(id) == Some(role) {
-					continue;
-				}
-				if let Some(role) = &role.0 {
-					let tags = role.tags.join(",");
-					table.push(format!(
-						"{:?}\t{}\t{}\t{}",
-						id,
-						tags,
-						role.zone,
-						role.capacity_string()
-					));
-				} else {
-					table.push(format!("{:?}\tREMOVED", id));
-				}
-			}
-			format_table(table);
-			println!();
-		}
-		if has_layout_changes {
-			println!(
-				"Zone redundancy: {}",
-				staging.parameters.get().zone_redundancy
-			);
-		}
-		true
-	} else {
-		false
-	}
-}

src/garage/cli/local/init.rs

@@ -36,16 +36,6 @@ pub fn node_id_command(config_file: PathBuf, quiet: bool) -> Result<(), Error> {
 		);
 		eprintln!("    garage [-c <config file path>] node connect {}", idstr);
 		eprintln!();
-		eprintln!("Or instruct them to connect from here by running:");
-		eprintln!(
-			"    garage -c {} -h <remote node> node connect {}",
-			config_file.to_string_lossy(),
-			idstr
-		);
-		eprintln!(
-			"where <remote_node> is their own node identifier in the format: <full-node-id>@<ip>:<port>"
-		);
-		eprintln!();
 		eprintln!("This node identifier can also be added as a bootstrap node in other node's garage.toml files:");
 		eprintln!("    bootstrap_peers = [");
 		eprintln!("        \"{}\",", idstr);

src/garage/cli/local/mod.rs

@@ -0,0 +1,3 @@
+pub(crate) mod convert_db;
+pub(crate) mod init;
+pub(crate) mod repair;

src/garage/cli/mod.rs

@@ -1,13 +1,4 @@
-pub(crate) mod cmd;
-pub(crate) mod init;
-pub(crate) mod layout;
-pub(crate) mod structs;
-pub(crate) mod util;
+pub mod structs;
 
-pub(crate) mod convert_db;
-
-pub(crate) use cmd::*;
-pub(crate) use init::*;
-pub(crate) use layout::*;
-pub(crate) use structs::*;
-pub(crate) use util::*;
+pub mod local;
+pub mod remote;

src/garage/cli/remote/admin_token.rs

@@ -0,0 +1,254 @@
+use format_table::format_table;
+
+use chrono::Local;
+
+use garage_util::error::*;
+
+use garage_api_admin::api::*;
+
+use crate::cli::remote::*;
+use crate::cli::structs::*;
+
+impl Cli {
+	pub async fn cmd_admin_token(&self, cmd: AdminTokenOperation) -> Result<(), Error> {
+		match cmd {
+			AdminTokenOperation::List => self.cmd_list_admin_tokens().await,
+			AdminTokenOperation::Info { api_token } => self.cmd_admin_token_info(api_token).await,
+			AdminTokenOperation::Create(opt) => self.cmd_create_admin_token(opt).await,
+			AdminTokenOperation::Rename {
+				api_token,
+				new_name,
+			} => self.cmd_rename_admin_token(api_token, new_name).await,
+			AdminTokenOperation::Set(opt) => self.cmd_update_admin_token(opt).await,
+			AdminTokenOperation::Delete { api_token, yes } => {
+				self.cmd_delete_admin_token(api_token, yes).await
+			}
+			AdminTokenOperation::DeleteExpired { yes } => {
+				self.cmd_delete_expired_admin_tokens(yes).await
+			}
+		}
+	}
+
+	pub async fn cmd_list_admin_tokens(&self) -> Result<(), Error> {
+		let mut list = self.api_request(ListAdminTokensRequest).await?;
+
+		list.0.sort_by_key(|x| x.created);
+
+		let mut table = vec!["ID\tCreated\tName\tExpiration\tScope".to_string()];
+		for tok in list.0.iter() {
+			let scope = if tok.expired {
+				String::new()
+			} else {
+				table_list_abbr(&tok.scope)
+			};
+			let exp = if tok.expired {
+				"expired".to_string()
+			} else {
+				tok.expiration
+					.map(|x| x.with_timezone(&Local).to_string())
+					.unwrap_or("never".into())
+			};
+			table.push(format!(
+				"{}\t{}\t{}\t{}\t{}",
+				tok.id.as_deref().unwrap_or("-"),
+				tok.created
+					.map(|x| x.with_timezone(&Local).date_naive().to_string())
+					.unwrap_or("-".into()),
+				tok.name,
+				exp,
+				scope,
+			));
+		}
+		format_table(table);
+
+		Ok(())
+	}
+
+	pub async fn cmd_admin_token_info(&self, search: String) -> Result<(), Error> {
+		let info = self
+			.api_request(GetAdminTokenInfoRequest {
+				id: None,
+				search: Some(search),
+			})
+			.await?;
+
+		print_token_info(&info);
+
+		Ok(())
+	}
+
+	pub async fn cmd_create_admin_token(&self, opt: AdminTokenCreateOp) -> Result<(), Error> {
+		let res = self
+			.api_request(CreateAdminTokenRequest(UpdateAdminTokenRequestBody {
+				name: opt.name,
+				expiration: parse_expires_in(&opt.expires_in)?,
+				never_expires: false,
+				scope: opt.scope.map(|s| {
+					s.split(",")
+						.map(|x| x.trim().to_string())
+						.collect::<Vec<_>>()
+				}),
+			}))
+			.await?;
+
+		if opt.quiet {
+			println!("{}", res.secret_token);
+		} else {
+			println!("This is your secret bearer token, it will not be shown again by Garage:");
+			println!("\n  {}\n", res.secret_token);
+			print_token_info(&res.info);
+		}
+
+		Ok(())
+	}
+
+	pub async fn cmd_rename_admin_token(&self, old: String, new: String) -> Result<(), Error> {
+		let token = self
+			.api_request(GetAdminTokenInfoRequest {
+				id: None,
+				search: Some(old),
+			})
+			.await?;
+
+		let info = self
+			.api_request(UpdateAdminTokenRequest {
+				id: token.id.unwrap(),
+				body: UpdateAdminTokenRequestBody {
+					name: Some(new),
+					expiration: None,
+					never_expires: false,
+					scope: None,
+				},
+			})
+			.await?;
+
+		print_token_info(&info.0);
+
+		Ok(())
+	}
+
+	pub async fn cmd_update_admin_token(&self, opt: AdminTokenSetOp) -> Result<(), Error> {
+		let token = self
+			.api_request(GetAdminTokenInfoRequest {
+				id: None,
+				search: Some(opt.api_token),
+			})
+			.await?;
+
+		let info = self
+			.api_request(UpdateAdminTokenRequest {
+				id: token.id.unwrap(),
+				body: UpdateAdminTokenRequestBody {
+					name: None,
+					expiration: parse_expires_in(&opt.expires_in)?,
+					never_expires: opt.never_expires,
+					scope: opt.scope.map({
+						let mut new_scope = token.scope;
+						|scope_str| {
+							if let Some(add) = scope_str.strip_prefix("+") {
+								for a in add.split(",").map(|x| x.trim().to_string()) {
+									if !new_scope.contains(&a) {
+										new_scope.push(a);
+									}
+								}
+								new_scope
+							} else if let Some(sub) = scope_str.strip_prefix("-") {
+								for r in sub.split(",").map(|x| x.trim()) {
+									new_scope.retain(|x| x != r);
+								}
+								new_scope
+							} else {
+								scope_str
+									.split(",")
+									.map(|x| x.trim().to_string())
+									.collect::<Vec<_>>()
+							}
+						}
+					}),
+				},
+			})
+			.await?;
+
+		print_token_info(&info.0);
+
+		Ok(())
+	}
+
+	pub async fn cmd_delete_admin_token(&self, token: String, yes: bool) -> Result<(), Error> {
+		let token = self
+			.api_request(GetAdminTokenInfoRequest {
+				id: None,
+				search: Some(token),
+			})
+			.await?;
+
+		let id = token.id.unwrap();
+
+		if !yes {
+			return Err(Error::Message(format!(
+				"Add the --yes flag to delete API token `{}` ({})",
+				token.name, id
+			)));
+		}
+
+		self.api_request(DeleteAdminTokenRequest { id }).await?;
+
+		println!("Admin API token has been deleted.");
+
+		Ok(())
+	}
+
+	pub async fn cmd_delete_expired_admin_tokens(&self, yes: bool) -> Result<(), Error> {
+		let mut list = self.api_request(ListAdminTokensRequest).await?.0;
+
+		list.retain(|tok| tok.expired);
+
+		if !yes {
+			return Err(Error::Message(format!(
+				"This would delete {} admin API tokens, add the --yes flag to proceed.",
+				list.len(),
+			)));
+		}
+
+		for token in list.iter() {
+			let id = token.id.clone().unwrap();
+			println!("Deleting token `{}` ({})", token.name, id);
+			self.api_request(DeleteAdminTokenRequest { id }).await?;
+		}
+
+		println!("{} admin API tokens have been deleted.", list.len());
+
+		Ok(())
+	}
+}
+
+fn print_token_info(token: &GetAdminTokenInfoResponse) {
+	println!("==== ADMINISTRATION TOKEN INFORMATION ====");
+	let mut table = vec![
+		format!("Token ID:\t{}", token.id.as_ref().unwrap()),
+		format!("Token name:\t{}", token.name),
+		format!("Created:\t{}", token.created.unwrap().with_timezone(&Local)),
+		format!(
+			"Validity:\t{}",
+			token.expired.then_some("EXPIRED").unwrap_or("valid")
+		),
+		format!(
+			"Expiration:\t{}",
+			token
+				.expiration
+				.map(|x| x.with_timezone(&Local).to_string())
+				.unwrap_or("never".into())
+		),
+		String::new(),
+	];
+
+	for (i, scope) in token.scope.iter().enumerate() {
+		if i == 0 {
+			table.push(format!("Scope:\t{}", scope));
+		} else {
+			table.push(format!("\t{}", scope));
+		}
+	}
+
+	format_table(table);
+}

src/garage/cli/remote/block.rs

@@ -0,0 +1,169 @@
+//use bytesize::ByteSize;
+use format_table::format_table;
+
+use garage_util::error::*;
+
+use garage_api_admin::api::*;
+
+use crate::cli::remote::*;
+use crate::cli::structs::*;
+
+impl Cli {
+	pub async fn cmd_block(&self, cmd: BlockOperation) -> Result<(), Error> {
+		match cmd {
+			BlockOperation::ListErrors => self.cmd_list_block_errors().await,
+			BlockOperation::Info { hash } => self.cmd_get_block_info(hash).await,
+			BlockOperation::RetryNow { all, blocks } => self.cmd_block_retry_now(all, blocks).await,
+			BlockOperation::Purge { yes, blocks } => self.cmd_block_purge(yes, blocks).await,
+		}
+	}
+
+	pub async fn cmd_list_block_errors(&self) -> Result<(), Error> {
+		let errors = self.local_api_request(LocalListBlockErrorsRequest).await?.0;
+
+		let tf = timeago::Formatter::new();
+		let mut tf2 = timeago::Formatter::new();
+		tf2.ago("");
+
+		let mut table = vec!["Hash\tRC\tErrors\tLast error\tNext try".into()];
+		for e in errors {
+			let next_try = if e.next_try_in_secs > 0 {
+				tf2.convert(Duration::from_secs(e.next_try_in_secs))
+			} else {
+				"asap".to_string()
+			};
+			table.push(format!(
+				"{}\t{}\t{}\t{}\tin {}",
+				e.block_hash,
+				e.refcount,
+				e.error_count,
+				tf.convert(Duration::from_secs(e.last_try_secs_ago)),
+				next_try
+			));
+		}
+		format_table(table);
+
+		Ok(())
+	}
+
+	pub async fn cmd_get_block_info(&self, hash: String) -> Result<(), Error> {
+		let info = self
+			.local_api_request(LocalGetBlockInfoRequest { block_hash: hash })
+			.await?;
+
+		println!("==== BLOCK INFORMATION ====");
+		format_table(vec![
+			format!("Block hash:\t{}", info.block_hash),
+			format!("Refcount:\t{}", info.refcount),
+		]);
+		println!();
+
+		println!("==== REFERENCES TO THIS BLOCK ====");
+		let mut table = vec!["Status\tVersion\tBucket\tKey\tMPU".into()];
+		let mut nondeleted_count = 0;
+		let mut inconsistent_refs = false;
+		for ver in info.versions.iter() {
+			match &ver.backlink {
+				Some(BlockVersionBacklink::Object { bucket_id, key }) => {
+					table.push(format!(
+						"{}\t{:.16}{}\t{:.16}\t{}",
+						ver.ref_deleted.then_some("deleted").unwrap_or("active"),
+						ver.version_id,
+						ver.version_deleted
+							.then_some(" (deleted)")
+							.unwrap_or_default(),
+						bucket_id,
+						key
+					));
+				}
+				Some(BlockVersionBacklink::Upload {
+					upload_id,
+					upload_deleted,
+					upload_garbage_collected: _,
+					bucket_id,
+					key,
+				}) => {
+					table.push(format!(
+						"{}\t{:.16}{}\t{:.16}\t{}\t{:.16}{}",
+						ver.ref_deleted.then_some("deleted").unwrap_or("active"),
+						ver.version_id,
+						ver.version_deleted
+							.then_some(" (deleted)")
+							.unwrap_or_default(),
+						bucket_id.as_deref().unwrap_or(""),
+						key.as_deref().unwrap_or(""),
+						upload_id,
+						upload_deleted.then_some(" (deleted)").unwrap_or_default(),
+					));
+				}
+				None => {
+					table.push(format!("{:.16}\t\t\tyes", ver.version_id));
+				}
+			}
+			if ver.ref_deleted != ver.version_deleted {
+				inconsistent_refs = true;
+			}
+			if !ver.ref_deleted {
+				nondeleted_count += 1;
+			}
+		}
+		format_table(table);
+
+		if inconsistent_refs {
+			println!();
+			println!("There are inconsistencies between the block_ref and the version tables.");
+			println!("Fix them by running `garage repair block-refs`");
+		}
+
+		if info.refcount != nondeleted_count {
+			println!();
+			println!(
+                "Warning: refcount does not match number of non-deleted versions, you should try `garage repair block-rc`."
+            );
+		}
+
+		Ok(())
+	}
+
+	pub async fn cmd_block_retry_now(&self, all: bool, blocks: Vec<String>) -> Result<(), Error> {
+		let req = match (all, blocks.len()) {
+			(true, 0) => LocalRetryBlockResyncRequest::All { all: true },
+			(false, n) if n > 0 => LocalRetryBlockResyncRequest::Blocks {
+				block_hashes: blocks,
+			},
+			_ => {
+				return Err(Error::Message(
+					"Please specify block hashes or --all (not both)".into(),
+				))
+			}
+		};
+
+		let res = self.local_api_request(req).await?;
+
+		println!(
+			"{} blocks returned in queue for a retry now (check logs to see results)",
+			res.count
+		);
+
+		Ok(())
+	}
+
+	pub async fn cmd_block_purge(&self, yes: bool, blocks: Vec<String>) -> Result<(), Error> {
+		if !yes {
+			return Err(Error::Message(
+				"Pass the --yes flag to confirm block purge operation.".into(),
+			));
+		}
+
+		let res = self
+			.local_api_request(LocalPurgeBlocksRequest(blocks))
+			.await?;
+
+		println!(
+			"Purged {} blocks: deleted {} versions, {} objects, {} multipart uploads",
+			res.blocks_purged, res.versions_deleted, res.objects_deleted, res.uploads_deleted,
+		);
+
+		Ok(())
+	}
+}

src/garage/cli/remote/bucket.rs

@@ -0,0 +1,588 @@
+//use bytesize::ByteSize;
+use format_table::format_table;
+
+use chrono::Local;
+
+use garage_util::error::*;
+
+use garage_api_admin::api::*;
+
+use crate::cli::remote::*;
+use crate::cli::structs::*;
+
+impl Cli {
+	pub async fn cmd_bucket(&self, cmd: BucketOperation) -> Result<(), Error> {
+		match cmd {
+			BucketOperation::List => self.cmd_list_buckets().await,
+			BucketOperation::Info(query) => self.cmd_bucket_info(query).await,
+			BucketOperation::Create(query) => self.cmd_create_bucket(query).await,
+			BucketOperation::Delete(query) => self.cmd_delete_bucket(query).await,
+			BucketOperation::Alias(query) => self.cmd_alias_bucket(query).await,
+			BucketOperation::Unalias(query) => self.cmd_unalias_bucket(query).await,
+			BucketOperation::Allow(query) => self.cmd_bucket_allow(query).await,
+			BucketOperation::Deny(query) => self.cmd_bucket_deny(query).await,
+			BucketOperation::Website(query) => self.cmd_bucket_website(query).await,
+			BucketOperation::SetQuotas(query) => self.cmd_bucket_set_quotas(query).await,
+			BucketOperation::CleanupIncompleteUploads(query) => {
+				self.cmd_cleanup_incomplete_uploads(query).await
+			}
+			BucketOperation::InspectObject(query) => self.cmd_inspect_object(query).await,
+		}
+	}
+
+	pub async fn cmd_list_buckets(&self) -> Result<(), Error> {
+		let mut buckets = self.api_request(ListBucketsRequest).await?;
+
+		buckets.0.sort_by_key(|x| x.created);
+
+		let mut table = vec!["ID\tCreated\tGlobal aliases\tLocal aliases".to_string()];
+		for bucket in buckets.0.iter() {
+			table.push(format!(
+				"{:.16}\t{}\t{}\t{}",
+				bucket.id,
+				bucket.created.with_timezone(&Local).date_naive(),
+				table_list_abbr(&bucket.global_aliases),
+				table_list_abbr(
+					bucket
+						.local_aliases
+						.iter()
+						.map(|x| format!("{}:{}", x.access_key_id, x.alias))
+				),
+			));
+		}
+		format_table(table);
+
+		Ok(())
+	}
+
+	pub async fn cmd_bucket_info(&self, opt: BucketOpt) -> Result<(), Error> {
+		let bucket = self
+			.api_request(GetBucketInfoRequest {
+				id: None,
+				global_alias: None,
+				search: Some(opt.name),
+			})
+			.await?;
+
+		print_bucket_info(&bucket);
+
+		Ok(())
+	}
+
+	pub async fn cmd_create_bucket(&self, opt: BucketOpt) -> Result<(), Error> {
+		let bucket = self
+			.api_request(CreateBucketRequest {
+				global_alias: Some(opt.name.clone()),
+				local_alias: None,
+			})
+			.await?;
+
+		print_bucket_info(&bucket.0);
+
+		Ok(())
+	}
+
+	pub async fn cmd_delete_bucket(&self, opt: DeleteBucketOpt) -> Result<(), Error> {
+		let bucket = self
+			.api_request(GetBucketInfoRequest {
+				id: None,
+				global_alias: None,
+				search: Some(opt.name.clone()),
+			})
+			.await?;
+
+		// CLI-only checks: the bucket must not have other aliases
+		if bucket
+			.global_aliases
+			.iter()
+			.find(|a| **a != opt.name)
+			.is_some()
+		{
+			return Err(Error::Message(format!("Bucket {} still has other global aliases. Use `bucket unalias` to delete them one by one.", opt.name)));
+		}
+
+		if bucket
+			.keys
+			.iter()
+			.any(|k| !k.bucket_local_aliases.is_empty())
+		{
+			return Err(Error::Message(format!("Bucket {} still has other local aliases. Use `bucket unalias` to delete them one by one.", opt.name)));
+		}
+
+		if !opt.yes {
+			println!("About to delete bucket {}.", bucket.id);
+			return Err(Error::Message(
+				"Add --yes flag to really perform this operation".to_string(),
+			));
+		}
+
+		self.api_request(DeleteBucketRequest {
+			id: bucket.id.clone(),
+		})
+		.await?;
+
+		println!("Bucket {} has been deleted.", bucket.id);
+
+		Ok(())
+	}
+
+	pub async fn cmd_alias_bucket(&self, opt: AliasBucketOpt) -> Result<(), Error> {
+		let bucket = self
+			.api_request(GetBucketInfoRequest {
+				id: None,
+				global_alias: None,
+				search: Some(opt.existing_bucket.clone()),
+			})
+			.await?;
+
+		let res = if let Some(key_pat) = &opt.local {
+			let key = self
+				.api_request(GetKeyInfoRequest {
+					search: Some(key_pat.clone()),
+					id: None,
+					show_secret_key: false,
+				})
+				.await?;
+
+			self.api_request(AddBucketAliasRequest {
+				bucket_id: bucket.id.clone(),
+				alias: BucketAliasEnum::Local {
+					local_alias: opt.new_name.clone(),
+					access_key_id: key.access_key_id.clone(),
+				},
+			})
+			.await?
+		} else {
+			self.api_request(AddBucketAliasRequest {
+				bucket_id: bucket.id.clone(),
+				alias: BucketAliasEnum::Global {
+					global_alias: opt.new_name.clone(),
+				},
+			})
+			.await?
+		};
+
+		print_bucket_info(&res.0);
+
+		Ok(())
+	}
+
+	pub async fn cmd_unalias_bucket(&self, opt: UnaliasBucketOpt) -> Result<(), Error> {
+		let res = if let Some(key_pat) = &opt.local {
+			let key = self
+				.api_request(GetKeyInfoRequest {
+					search: Some(key_pat.clone()),
+					id: None,
+					show_secret_key: false,
+				})
+				.await?;
+
+			let bucket = key
+				.buckets
+				.iter()
+				.find(|x| x.local_aliases.contains(&opt.name))
+				.ok_or_message(format!(
+					"No bucket called {} in namespace of key {}",
+					opt.name, key.access_key_id
+				))?;
+
+			self.api_request(RemoveBucketAliasRequest {
+				bucket_id: bucket.id.clone(),
+				alias: BucketAliasEnum::Local {
+					access_key_id: key.access_key_id.clone(),
+					local_alias: opt.name.clone(),
+				},
+			})
+			.await?
+		} else {
+			let bucket = self
+				.api_request(GetBucketInfoRequest {
+					id: None,
+					global_alias: Some(opt.name.clone()),
+					search: None,
+				})
+				.await?;
+
+			self.api_request(RemoveBucketAliasRequest {
+				bucket_id: bucket.id.clone(),
+				alias: BucketAliasEnum::Global {
+					global_alias: opt.name.clone(),
+				},
+			})
+			.await?
+		};
+
+		print_bucket_info(&res.0);
+
+		Ok(())
+	}
+
+	pub async fn cmd_bucket_allow(&self, opt: PermBucketOpt) -> Result<(), Error> {
+		let bucket = self
+			.api_request(GetBucketInfoRequest {
+				id: None,
+				global_alias: None,
+				search: Some(opt.bucket.clone()),
+			})
+			.await?;
+
+		let key = self
+			.api_request(GetKeyInfoRequest {
+				id: None,
+				search: Some(opt.key_pattern.clone()),
+				show_secret_key: false,
+			})
+			.await?;
+
+		let res = self
+			.api_request(AllowBucketKeyRequest(BucketKeyPermChangeRequest {
+				bucket_id: bucket.id.clone(),
+				access_key_id: key.access_key_id.clone(),
+				permissions: ApiBucketKeyPerm {
+					read: opt.read,
+					write: opt.write,
+					owner: opt.owner,
+				},
+			}))
+			.await?;
+
+		print_bucket_info(&res.0);
+
+		Ok(())
+	}
+
+	pub async fn cmd_bucket_deny(&self, opt: PermBucketOpt) -> Result<(), Error> {
+		let bucket = self
+			.api_request(GetBucketInfoRequest {
+				id: None,
+				global_alias: None,
+				search: Some(opt.bucket.clone()),
+			})
+			.await?;
+
+		let key = self
+			.api_request(GetKeyInfoRequest {
+				id: None,
+				search: Some(opt.key_pattern.clone()),
+				show_secret_key: false,
+			})
+			.await?;
+
+		let res = self
+			.api_request(DenyBucketKeyRequest(BucketKeyPermChangeRequest {
+				bucket_id: bucket.id.clone(),
+				access_key_id: key.access_key_id.clone(),
+				permissions: ApiBucketKeyPerm {
+					read: opt.read,
+					write: opt.write,
+					owner: opt.owner,
+				},
+			}))
+			.await?;
+
+		print_bucket_info(&res.0);
+
+		Ok(())
+	}
+
+	pub async fn cmd_bucket_website(&self, opt: WebsiteOpt) -> Result<(), Error> {
+		let bucket = self
+			.api_request(GetBucketInfoRequest {
+				id: None,
+				global_alias: None,
+				search: Some(opt.bucket.clone()),
+			})
+			.await?;
+
+		if !(opt.allow ^ opt.deny) {
+			return Err(Error::Message(
+				"You must specify exactly one flag, either --allow or --deny".to_string(),
+			));
+		}
+
+		let wa = if opt.allow {
+			UpdateBucketWebsiteAccess {
+				enabled: true,
+				index_document: Some(opt.index_document.clone()),
+				error_document: opt
+					.error_document
+					.or(bucket.website_config.and_then(|x| x.error_document.clone())),
+			}
+		} else {
+			UpdateBucketWebsiteAccess {
+				enabled: false,
+				index_document: None,
+				error_document: None,
+			}
+		};
+
+		let res = self
+			.api_request(UpdateBucketRequest {
+				id: bucket.id,
+				body: UpdateBucketRequestBody {
+					website_access: Some(wa),
+					quotas: None,
+				},
+			})
+			.await?;
+
+		print_bucket_info(&res.0);
+
+		Ok(())
+	}
+
+	pub async fn cmd_bucket_set_quotas(&self, opt: SetQuotasOpt) -> Result<(), Error> {
+		let bucket = self
+			.api_request(GetBucketInfoRequest {
+				id: None,
+				global_alias: None,
+				search: Some(opt.bucket.clone()),
+			})
+			.await?;
+
+		if opt.max_size.is_none() && opt.max_objects.is_none() {
+			return Err(Error::Message(
+				"You must specify either --max-size or --max-objects (or both) for this command to do something.".to_string(),
+			));
+		}
+
+		let new_quotas = ApiBucketQuotas {
+			max_size: match opt.max_size.as_deref() {
+				Some("none") => None,
+				Some(v) => Some(
+					v.parse::<bytesize::ByteSize>()
+						.ok_or_message(format!("Invalid size specified: {}", v))?
+						.as_u64(),
+				),
+				None => bucket.quotas.max_size,
+			},
+			max_objects: match opt.max_objects.as_deref() {
+				Some("none") => None,
+				Some(v) => Some(
+					v.parse::<u64>()
+						.ok_or_message(format!("Invalid number: {}", v))?,
+				),
+				None => bucket.quotas.max_objects,
+			},
+		};
+
+		let res = self
+			.api_request(UpdateBucketRequest {
+				id: bucket.id.clone(),
+				body: UpdateBucketRequestBody {
+					website_access: None,
+					quotas: Some(new_quotas),
+				},
+			})
+			.await?;
+
+		print_bucket_info(&res.0);
+
+		Ok(())
+	}
+
+	pub async fn cmd_cleanup_incomplete_uploads(
+		&self,
+		opt: CleanupIncompleteUploadsOpt,
+	) -> Result<(), Error> {
+		let older_than = parse_duration::parse::parse(&opt.older_than)
+			.ok_or_message("Invalid duration passed for --older-than parameter")?;
+
+		for b in opt.buckets.iter() {
+			let bucket = self
+				.api_request(GetBucketInfoRequest {
+					id: None,
+					global_alias: None,
+					search: Some(b.clone()),
+				})
+				.await?;
+
+			let res = self
+				.api_request(CleanupIncompleteUploadsRequest {
+					bucket_id: bucket.id.clone(),
+					older_than_secs: older_than.as_secs(),
+				})
+				.await?;
+
+			if res.uploads_deleted > 0 {
+				println!("{:.16}: {} uploads deleted", bucket.id, res.uploads_deleted);
+			} else {
+				println!("{:.16}: no uploads deleted", bucket.id);
+			}
+		}
+
+		Ok(())
+	}
+
+	pub async fn cmd_inspect_object(&self, opt: InspectObjectOpt) -> Result<(), Error> {
+		let bucket = self
+			.api_request(GetBucketInfoRequest {
+				id: None,
+				global_alias: None,
+				search: Some(opt.bucket),
+			})
+			.await?;
+
+		let info = self
+			.api_request(InspectObjectRequest {
+				bucket_id: bucket.id,
+				key: opt.key,
+			})
+			.await?;
+
+		for ver in info.versions {
+			println!("==== OBJECT VERSION ====");
+			let mut tab = vec![
+				format!("Bucket ID:\t{}", info.bucket_id),
+				format!("Key:\t{}", info.key),
+				format!("Version ID:\t{}", ver.uuid),
+				format!("Timestamp:\t{}", ver.timestamp),
+			];
+			if let Some(size) = ver.size {
+				let bs = bytesize::ByteSize::b(size);
+				tab.push(format!(
+					"Size:\t{} ({})",
+					bs.to_string_as(true),
+					bs.to_string_as(false)
+				));
+				tab.push(format!("Size (exact):\t{}", size));
+				if !ver.blocks.is_empty() {
+					tab.push(format!("Number of blocks:\t{:?}", ver.blocks.len()));
+				}
+			}
+			if let Some(etag) = ver.etag {
+				tab.push(format!("Etag:\t{}", etag));
+			}
+			tab.extend([
+				format!("Encrypted:\t{}", ver.encrypted),
+				format!("Uploading:\t{}", ver.uploading),
+				format!("Aborted:\t{}", ver.aborted),
+				format!("Delete marker:\t{}", ver.delete_marker),
+				format!("Inline data:\t{}", ver.inline),
+			]);
+			if !ver.headers.is_empty() {
+				tab.push(String::new());
+				tab.extend(ver.headers.iter().map(|(k, v)| format!("{}\t{}", k, v)));
+			}
+			format_table(tab);
+
+			if !ver.blocks.is_empty() {
+				let mut tab = vec!["Part#\tOffset\tBlock hash\tSize".to_string()];
+				tab.extend(ver.blocks.iter().map(|b| {
+					format!(
+						"{:4}\t{:9}\t{}\t{:9}",
+						b.part_number, b.offset, b.hash, b.size
+					)
+				}));
+				println!();
+				format_table(tab);
+			}
+			println!();
+		}
+
+		Ok(())
+	}
+}
+
+fn print_bucket_info(bucket: &GetBucketInfoResponse) {
+	println!("==== BUCKET INFORMATION ====");
+
+	let mut info = vec![
+		format!("Bucket:\t{}", bucket.id),
+		format!("Created:\t{}", bucket.created.with_timezone(&Local)),
+		String::new(),
+		{
+			let size = bytesize::ByteSize::b(bucket.bytes as u64);
+			format!(
+				"Size:\t{} ({})",
+				size.to_string_as(true),
+				size.to_string_as(false)
+			)
+		},
+		format!("Objects:\t{}", bucket.objects),
+	];
+
+	if bucket.unfinished_uploads > 0 {
+		info.extend([
+			format!(
+				"Unfinished uploads:\t{} multipart uploads",
+				bucket.unfinished_multipart_uploads
+			),
+			format!("\t{} including regular uploads", bucket.unfinished_uploads),
+			{
+				let mpu_size =
+					bytesize::ByteSize::b(bucket.unfinished_multipart_upload_bytes as u64);
+				format!(
+					"Size of unfinished multipart uploads:\t{} ({})",
+					mpu_size.to_string_as(true),
+					mpu_size.to_string_as(false),
+				)
+			},
+		]);
+	}
+
+	info.extend([
+		String::new(),
+		format!("Website access:\t{}", bucket.website_access),
+	]);
+
+	if let Some(wc) = &bucket.website_config {
+		info.extend([
+			format!("  index document:\t{}", wc.index_document),
+			format!(
+				"  error document:\t{}",
+				wc.error_document.as_deref().unwrap_or("(not defined)")
+			),
+		]);
+	}
+
+	if bucket.quotas.max_size.is_some() || bucket.quotas.max_objects.is_some() {
+		info.push(String::new());
+		info.push("Quotas:\tenabled".into());
+		if let Some(ms) = bucket.quotas.max_size {
+			let ms = bytesize::ByteSize::b(ms);
+			info.push(format!(
+				"  maximum size:\t{} ({})",
+				ms.to_string_as(true),
+				ms.to_string_as(false)
+			));
+		}
+		if let Some(mo) = bucket.quotas.max_objects {
+			info.push(format!("  maximum number of objects:\t{}", mo));
+		}
+	}
+
+	if !bucket.global_aliases.is_empty() {
+		info.push(String::new());
+		for (i, alias) in bucket.global_aliases.iter().enumerate() {
+			if i == 0 && bucket.global_aliases.len() > 1 {
+				info.push(format!("Global aliases:\t{}", alias));
+			} else if i == 0 {
+				info.push(format!("Global alias:\t{}", alias));
+			} else {
+				info.push(format!("\t{}", alias));
+			}
+		}
+	}
+
+	format_table(info);
+
+	println!("");
+	println!("==== KEYS FOR THIS BUCKET ====");
+	let mut key_info = vec!["Permissions\tAccess key\t\tLocal aliases".to_string()];
+	key_info.extend(bucket.keys.iter().map(|key| {
+		let rflag = if key.permissions.read { "R" } else { " " };
+		let wflag = if key.permissions.write { "W" } else { " " };
+		let oflag = if key.permissions.owner { "O" } else { " " };
+		format!(
+			"{}{}{}\t{}\t{}\t{}",
+			rflag,
+			wflag,
+			oflag,
+			key.access_key_id,
+			key.name,
+			key.bucket_local_aliases.to_vec().join(","),
+		)
+	}));
+	format_table(key_info);
+}

src/garage/cli/remote/cluster.rs

@@ -0,0 +1,160 @@
+use format_table::format_table;
+
+use garage_util::error::*;
+
+use garage_api_admin::api::*;
+
+use crate::cli::remote::layout::*;
+use crate::cli::remote::*;
+use crate::cli::structs::*;
+
+impl Cli {
+	pub async fn cmd_status(&self) -> Result<(), Error> {
+		let status = self.api_request(GetClusterStatusRequest).await?;
+		let layout = self.api_request(GetClusterLayoutRequest).await?;
+
+		println!("==== HEALTHY NODES ====");
+
+		let mut healthy_nodes =
+			vec!["ID\tHostname\tAddress\tTags\tZone\tCapacity\tDataAvail\tVersion".to_string()];
+
+		for adv in status.nodes.iter().filter(|adv| adv.is_up) {
+			let host = adv.hostname.as_deref().unwrap_or("?");
+			let addr = match adv.addr {
+				Some(addr) => addr.to_string(),
+				None => "N/A".to_string(),
+			};
+			if let Some(cfg) = &adv.role {
+				let data_avail = match &adv.data_partition {
+					_ if cfg.capacity.is_none() => "N/A".into(),
+					Some(FreeSpaceResp { available, total }) => {
+						let pct = (*available as f64) / (*total as f64) * 100.;
+						let avail_str = bytesize::ByteSize::b(*available);
+						format!("{} ({:.1}%)", avail_str, pct)
+					}
+					None => "?".into(),
+				};
+				healthy_nodes.push(format!(
+					"{id:.16}\t{host}\t{addr}\t[{tags}]\t{zone}\t{capacity}\t{data_avail}\t{version}",
+					id = adv.id,
+					host = host,
+					addr = addr,
+					tags = cfg.tags.join(","),
+					zone = cfg.zone,
+					capacity = capacity_string(cfg.capacity),
+					data_avail = data_avail,
+                    version = adv.garage_version.as_deref().unwrap_or_default(),
+				));
+			} else {
+				let status = match layout.staged_role_changes.iter().find(|x| x.id == adv.id) {
+					Some(NodeRoleChange {
+						action: NodeRoleChangeEnum::Update { .. },
+						..
+					}) => "pending...",
+					_ if adv.draining => "draining metadata..",
+					_ => "NO ROLE ASSIGNED",
+				};
+				healthy_nodes.push(format!(
+					"{id:.16}\t{h}\t{addr}\t\t\t{status}\t\t{version}",
+					id = adv.id,
+					h = host,
+					addr = addr,
+					status = status,
+					version = adv.garage_version.as_deref().unwrap_or_default(),
+				));
+			}
+		}
+		format_table(healthy_nodes);
+
+		let tf = timeago::Formatter::new();
+		let mut drain_msg = false;
+		let mut failed_nodes = vec!["ID\tHostname\tTags\tZone\tCapacity\tLast seen".to_string()];
+		for adv in status.nodes.iter().filter(|x| !x.is_up) {
+			let node = &adv.id;
+
+			let host = adv.hostname.as_deref().unwrap_or("?");
+			let last_seen = adv
+				.last_seen_secs_ago
+				.map(|s| tf.convert(Duration::from_secs(s)))
+				.unwrap_or_else(|| "never seen".into());
+
+			if let Some(cfg) = &adv.role {
+				let capacity = capacity_string(cfg.capacity);
+
+				failed_nodes.push(format!(
+					"{id:.16}\t{host}\t[{tags}]\t{zone}\t{capacity}\t{last_seen}",
+					id = node,
+					host = host,
+					tags = cfg.tags.join(","),
+					zone = cfg.zone,
+					capacity = capacity,
+					last_seen = last_seen,
+				));
+			} else {
+				let status = match layout.staged_role_changes.iter().find(|x| x.id == adv.id) {
+					Some(NodeRoleChange {
+						action: NodeRoleChangeEnum::Update { .. },
+						..
+					}) => "pending...",
+					_ if adv.draining => {
+						drain_msg = true;
+						"draining metadata.."
+					}
+					_ => unreachable!(),
+				};
+
+				failed_nodes.push(format!(
+					"{id:.16}\t{host}\t\t\t{status}\t{last_seen}",
+					id = node,
+					host = host,
+					status = status,
+					last_seen = last_seen,
+				));
+			}
+		}
+
+		if failed_nodes.len() > 1 {
+			println!("\n==== FAILED NODES ====");
+			format_table(failed_nodes);
+			if drain_msg {
+				println!();
+				println!("Your cluster is expecting to drain data from nodes that are currently unavailable.");
+				println!(
+					"If these nodes are definitely dead, please review the layout history with"
+				);
+				println!(
+                    "`garage layout history` and use `garage layout skip-dead-nodes` to force progress."
+                );
+			}
+		}
+
+		if print_staging_role_changes(&layout) {
+			println!();
+			println!(
+				"Please use `garage layout show` to check the proposed new layout and apply it."
+			);
+			println!();
+		}
+
+		Ok(())
+	}
+
+	pub async fn cmd_connect(&self, opt: ConnectNodeOpt) -> Result<(), Error> {
+		let res = self
+			.api_request(ConnectClusterNodesRequest(vec![opt.node]))
+			.await?;
+		if res.0.len() != 1 {
+			return Err(Error::Message(format!("unexpected response: {:?}", res)));
+		}
+		let res = res.0.into_iter().next().unwrap();
+		if res.success {
+			println!("Success.");
+			Ok(())
+		} else {
+			Err(Error::Message(format!(
+				"Failure: {}",
+				res.error.unwrap_or_default()
+			)))
+		}
+	}
+}

src/garage/cli/remote/key.rs

@@ -0,0 +1,318 @@
+use format_table::format_table;
+
+use chrono::Local;
+
+use garage_util::error::*;
+
+use garage_api_admin::api::*;
+
+use crate::cli::remote::*;
+use crate::cli::structs::*;
+
+impl Cli {
+	pub async fn cmd_key(&self, cmd: KeyOperation) -> Result<(), Error> {
+		match cmd {
+			KeyOperation::List => self.cmd_list_keys().await,
+			KeyOperation::Info(query) => self.cmd_key_info(query).await,
+			KeyOperation::Create(query) => self.cmd_create_key(query).await,
+			KeyOperation::Rename(query) => self.cmd_rename_key(query).await,
+			KeyOperation::Set(opt) => self.cmd_update_key(opt).await,
+			KeyOperation::Delete(query) => self.cmd_delete_key(query).await,
+			KeyOperation::Allow(query) => self.cmd_allow_key(query).await,
+			KeyOperation::Deny(query) => self.cmd_deny_key(query).await,
+			KeyOperation::Import(query) => self.cmd_import_key(query).await,
+			KeyOperation::DeleteExpired { yes } => self.cmd_delete_expired_keys(yes).await,
+		}
+	}
+
+	pub async fn cmd_list_keys(&self) -> Result<(), Error> {
+		let mut keys = self.api_request(ListKeysRequest).await?;
+
+		keys.0.sort_by_key(|x| x.created);
+
+		let mut table = vec!["ID\tCreated\tName\tExpiration".to_string()];
+		for key in keys.0.iter() {
+			let exp = if key.expired {
+				"expired".to_string()
+			} else {
+				key.expiration
+					.map(|x| x.with_timezone(&Local).to_string())
+					.unwrap_or("never".into())
+			};
+			table.push(format!(
+				"{}\t{}\t{}\t{}",
+				key.id,
+				key.created
+					.map(|x| x.with_timezone(&Local).date_naive().to_string())
+					.unwrap_or_default(),
+				key.name,
+				exp
+			));
+		}
+		format_table(table);
+
+		Ok(())
+	}
+
+	pub async fn cmd_key_info(&self, opt: KeyInfoOpt) -> Result<(), Error> {
+		let key = self
+			.api_request(GetKeyInfoRequest {
+				id: None,
+				search: Some(opt.key_pattern),
+				show_secret_key: opt.show_secret,
+			})
+			.await?;
+
+		print_key_info(&key);
+
+		Ok(())
+	}
+
+	pub async fn cmd_create_key(&self, opt: KeyNewOpt) -> Result<(), Error> {
+		let key = self
+			.api_request(CreateKeyRequest(UpdateKeyRequestBody {
+				name: Some(opt.name),
+				expiration: parse_expires_in(&opt.expires_in)?,
+				never_expires: false,
+				allow: None,
+				deny: None,
+			}))
+			.await?;
+
+		print_key_info(&key.0);
+
+		Ok(())
+	}
+
+	pub async fn cmd_rename_key(&self, opt: KeyRenameOpt) -> Result<(), Error> {
+		let key = self
+			.api_request(GetKeyInfoRequest {
+				id: None,
+				search: Some(opt.key_pattern),
+				show_secret_key: false,
+			})
+			.await?;
+
+		let new_key = self
+			.api_request(UpdateKeyRequest {
+				id: key.access_key_id,
+				body: UpdateKeyRequestBody {
+					name: Some(opt.new_name),
+					expiration: None,
+					never_expires: false,
+					allow: None,
+					deny: None,
+				},
+			})
+			.await?;
+
+		print_key_info(&new_key.0);
+
+		Ok(())
+	}
+
+	pub async fn cmd_update_key(&self, opt: KeySetOpt) -> Result<(), Error> {
+		let key = self
+			.api_request(GetKeyInfoRequest {
+				id: None,
+				search: Some(opt.key_pattern),
+				show_secret_key: false,
+			})
+			.await?;
+
+		let new_key = self
+			.api_request(UpdateKeyRequest {
+				id: key.access_key_id,
+				body: UpdateKeyRequestBody {
+					name: None,
+					expiration: parse_expires_in(&opt.expires_in)?,
+					never_expires: opt.never_expires,
+					allow: None,
+					deny: None,
+				},
+			})
+			.await?;
+
+		print_key_info(&new_key.0);
+
+		Ok(())
+	}
+
+	pub async fn cmd_delete_key(&self, opt: KeyDeleteOpt) -> Result<(), Error> {
+		let key = self
+			.api_request(GetKeyInfoRequest {
+				id: None,
+				search: Some(opt.key_pattern),
+				show_secret_key: false,
+			})
+			.await?;
+
+		if !opt.yes {
+			println!("About to delete key {}...", key.access_key_id);
+			return Err(Error::Message(
+				"Add --yes flag to really perform this operation".to_string(),
+			));
+		}
+
+		self.api_request(DeleteKeyRequest {
+			id: key.access_key_id.clone(),
+		})
+		.await?;
+
+		println!("Access key {} has been deleted.", key.access_key_id);
+
+		Ok(())
+	}
+
+	pub async fn cmd_allow_key(&self, opt: KeyPermOpt) -> Result<(), Error> {
+		let key = self
+			.api_request(GetKeyInfoRequest {
+				id: None,
+				search: Some(opt.key_pattern),
+				show_secret_key: false,
+			})
+			.await?;
+
+		let new_key = self
+			.api_request(UpdateKeyRequest {
+				id: key.access_key_id,
+				body: UpdateKeyRequestBody {
+					name: None,
+					expiration: None,
+					never_expires: false,
+					allow: Some(KeyPerm {
+						create_bucket: opt.create_bucket,
+					}),
+					deny: None,
+				},
+			})
+			.await?;
+
+		print_key_info(&new_key.0);
+
+		Ok(())
+	}
+
+	pub async fn cmd_deny_key(&self, opt: KeyPermOpt) -> Result<(), Error> {
+		let key = self
+			.api_request(GetKeyInfoRequest {
+				id: None,
+				search: Some(opt.key_pattern),
+				show_secret_key: false,
+			})
+			.await?;
+
+		let new_key = self
+			.api_request(UpdateKeyRequest {
+				id: key.access_key_id,
+				body: UpdateKeyRequestBody {
+					name: None,
+					expiration: None,
+					never_expires: false,
+					allow: None,
+					deny: Some(KeyPerm {
+						create_bucket: opt.create_bucket,
+					}),
+				},
+			})
+			.await?;
+
+		print_key_info(&new_key.0);
+
+		Ok(())
+	}
+
+	pub async fn cmd_import_key(&self, opt: KeyImportOpt) -> Result<(), Error> {
+		if !opt.yes {
+			return Err(Error::Message("This command is intended to re-import keys that were previously generated by Garage. If you want to create a new key, use `garage key new` instead. Add the --yes flag if you really want to re-import a key.".to_string()));
+		}
+
+		let new_key = self
+			.api_request(ImportKeyRequest {
+				name: Some(opt.name),
+				access_key_id: opt.key_id,
+				secret_access_key: opt.secret_key,
+			})
+			.await?;
+
+		print_key_info(&new_key.0);
+
+		Ok(())
+	}
+
+	pub async fn cmd_delete_expired_keys(&self, yes: bool) -> Result<(), Error> {
+		let mut list = self.api_request(ListKeysRequest).await?.0;
+
+		list.retain(|key| key.expired);
+
+		if !yes {
+			return Err(Error::Message(format!(
+				"This would delete {} access keys, add the --yes flag to proceed.",
+				list.len(),
+			)));
+		}
+
+		for key in list.iter() {
+			let id = key.id.clone();
+			println!("Deleting access key `{}` ({})", key.name, id);
+			self.api_request(DeleteKeyRequest { id }).await?;
+		}
+
+		println!("{} access keys have been deleted.", list.len());
+
+		Ok(())
+	}
+}
+
+fn print_key_info(key: &GetKeyInfoResponse) {
+	println!("==== ACCESS KEY INFORMATION ====");
+
+	let mut table = vec![
+		format!("Key ID:\t{}", key.access_key_id),
+		format!("Key name:\t{}", key.name),
+		format!(
+			"Secret key:\t{}",
+			key.secret_access_key.as_deref().unwrap_or("(redacted)")
+		),
+	];
+
+	if let Some(c) = key.created {
+		table.push(format!("Created:\t{}", c.with_timezone(&Local)));
+	}
+
+	table.extend([
+		format!(
+			"Validity:\t{}",
+			key.expired.then_some("EXPIRED").unwrap_or("valid")
+		),
+		format!(
+			"Expiration:\t{}",
+			key.expiration
+				.map(|x| x.with_timezone(&Local).to_string())
+				.unwrap_or("never".into())
+		),
+		String::new(),
+		format!("Can create buckets:\t{}", key.permissions.create_bucket),
+	]);
+	format_table(table);
+
+	println!("");
+	println!("==== BUCKETS FOR THIS KEY ====");
+	let mut bucket_info = vec!["Permissions\tID\tGlobal aliases\tLocal aliases".to_string()];
+	bucket_info.extend(key.buckets.iter().map(|bucket| {
+		let rflag = if bucket.permissions.read { "R" } else { " " };
+		let wflag = if bucket.permissions.write { "W" } else { " " };
+		let oflag = if bucket.permissions.owner { "O" } else { " " };
+		format!(
+			"{}{}{}\t{:.16}\t{}\t{}",
+			rflag,
+			wflag,
+			oflag,
+			bucket.id,
+			table_list_abbr(&bucket.global_aliases),
+			bucket.local_aliases.join(","),
+		)
+	}));
+
+	format_table(bucket_info);
+}

src/garage/cli/remote/layout.rs

@@ -0,0 +1,474 @@
+use bytesize::ByteSize;
+use format_table::format_table;
+
+use garage_util::error::*;
+
+use garage_api_admin::api::*;
+
+use crate::cli::remote::*;
+use crate::cli::structs::*;
+
+impl Cli {
+	pub async fn layout_command_dispatch(&self, cmd: LayoutOperation) -> Result<(), Error> {
+		match cmd {
+			LayoutOperation::Show => self.cmd_show_layout().await,
+			LayoutOperation::Assign(assign_opt) => self.cmd_assign_role(assign_opt).await,
+			LayoutOperation::Remove(remove_opt) => self.cmd_remove_role(remove_opt).await,
+			LayoutOperation::Config(config_opt) => self.cmd_config_layout(config_opt).await,
+			LayoutOperation::Apply(apply_opt) => self.cmd_apply_layout(apply_opt).await,
+			LayoutOperation::Revert(revert_opt) => self.cmd_revert_layout(revert_opt).await,
+			LayoutOperation::History => self.cmd_layout_history().await,
+			LayoutOperation::SkipDeadNodes(opt) => self.cmd_skip_dead_nodes(opt).await,
+		}
+	}
+
+	pub async fn cmd_show_layout(&self) -> Result<(), Error> {
+		let layout = self.api_request(GetClusterLayoutRequest).await?;
+
+		println!("==== CURRENT CLUSTER LAYOUT ====");
+		print_cluster_layout(&layout, "No nodes currently have a role in the cluster.\nSee `garage status` to view available nodes.");
+		println!();
+		println!("Current cluster layout version: {}", layout.version);
+
+		let has_role_changes = print_staging_role_changes(&layout);
+		if has_role_changes {
+			let res_apply = self.api_request(PreviewClusterLayoutChangesRequest).await?;
+
+			// this will print the stats of what partitions
+			// will move around when we apply
+			match res_apply {
+				PreviewClusterLayoutChangesResponse::Success {
+					message,
+					new_layout,
+				} => {
+					println!();
+					println!("==== NEW CLUSTER LAYOUT AFTER APPLYING CHANGES ====");
+					print_cluster_layout(&new_layout, "No nodes have a role in the new layout.");
+					println!();
+
+					for line in message.iter() {
+						println!("{}", line);
+					}
+					println!("To enact the staged role changes, type:");
+					println!();
+					println!("    garage layout apply --version {}", new_layout.version);
+					println!();
+					println!("You can also revert all proposed changes with: garage layout revert");
+				}
+				PreviewClusterLayoutChangesResponse::Error { error } => {
+					println!("Error while trying to compute the assignment: {}", error);
+					println!("This new layout cannot yet be applied.");
+					println!("You can also revert all proposed changes with: garage layout revert");
+				}
+			}
+		}
+
+		Ok(())
+	}
+
+	pub async fn cmd_assign_role(&self, opt: AssignRoleOpt) -> Result<(), Error> {
+		let status = self.api_request(GetClusterStatusRequest).await?;
+		let layout = self.api_request(GetClusterLayoutRequest).await?;
+
+		let mut actions = vec![];
+
+		for node in opt.replace.iter() {
+			let id = find_matching_node(&status, &layout, &node)?;
+
+			actions.push(NodeRoleChange {
+				id,
+				action: NodeRoleChangeEnum::Remove { remove: true },
+			});
+		}
+
+		for node in opt.node_ids.iter() {
+			let id = find_matching_node(&status, &layout, &node)?;
+
+			let current = get_staged_or_current_role(&id, &layout);
+
+			let zone = opt
+				.zone
+				.clone()
+				.or_else(|| current.as_ref().map(|c| c.zone.clone()))
+				.ok_or_message("Please specify a zone with the -z flag")?;
+
+			let capacity = if opt.gateway {
+				if opt.capacity.is_some() {
+					return Err(Error::Message("Please specify only -c or -g".into()));
+				}
+				None
+			} else if let Some(cap) = opt.capacity {
+				Some(cap.as_u64())
+			} else {
+				current.as_ref().ok_or_message("Please specify a capacity with the -c flag, or set node explicitly as gateway with -g")?.capacity
+			};
+
+			let tags = if !opt.tags.is_empty() {
+				opt.tags.clone()
+			} else if let Some(cur) = current.as_ref() {
+				cur.tags.clone()
+			} else {
+				vec![]
+			};
+
+			actions.push(NodeRoleChange {
+				id,
+				action: NodeRoleChangeEnum::Update(NodeAssignedRole {
+					zone,
+					capacity,
+					tags,
+				}),
+			});
+		}
+
+		self.api_request(UpdateClusterLayoutRequest {
+			roles: actions,
+			parameters: None,
+		})
+		.await?;
+
+		println!("Role changes are staged but not yet committed.");
+		println!("Use `garage layout show` to view staged role changes,");
+		println!("and `garage layout apply` to enact staged changes.");
+		Ok(())
+	}
+
+	pub async fn cmd_remove_role(&self, opt: RemoveRoleOpt) -> Result<(), Error> {
+		let status = self.api_request(GetClusterStatusRequest).await?;
+		let layout = self.api_request(GetClusterLayoutRequest).await?;
+
+		let id = find_matching_node(&status, &layout, &opt.node_id)?;
+
+		let actions = vec![NodeRoleChange {
+			id,
+			action: NodeRoleChangeEnum::Remove { remove: true },
+		}];
+
+		self.api_request(UpdateClusterLayoutRequest {
+			roles: actions,
+			parameters: None,
+		})
+		.await?;
+
+		println!("Role removal is staged but not yet committed.");
+		println!("Use `garage layout show` to view staged role changes,");
+		println!("and `garage layout apply` to enact staged changes.");
+		Ok(())
+	}
+
+	pub async fn cmd_config_layout(&self, config_opt: ConfigLayoutOpt) -> Result<(), Error> {
+		let mut did_something = false;
+		match config_opt.redundancy {
+			None => (),
+			Some(r_str) => {
+				let r = parse_zone_redundancy(&r_str)?;
+
+				self.api_request(UpdateClusterLayoutRequest {
+					roles: vec![],
+					parameters: Some(LayoutParameters { zone_redundancy: r }),
+				})
+				.await?;
+				println!(
+					"The zone redundancy parameter has been set to '{}'.",
+					display_zone_redundancy(r)
+				);
+				did_something = true;
+			}
+		}
+
+		if !did_something {
+			return Err(Error::Message(
+				"Please specify an action for `garage layout config`".into(),
+			));
+		}
+
+		Ok(())
+	}
+
+	pub async fn cmd_apply_layout(&self, apply_opt: ApplyLayoutOpt) -> Result<(), Error> {
+		let missing_version_error = r#"
+Please pass the new layout version number to ensure that you are writing the correct version of the cluster layout.
+To know the correct value of the new layout version, invoke `garage layout show` and review the proposed changes.
+        "#;
+
+		let req = ApplyClusterLayoutRequest {
+			version: apply_opt.version.ok_or_message(missing_version_error)?,
+		};
+		let res = self.api_request(req).await?;
+
+		for line in res.message.iter() {
+			println!("{}", line);
+		}
+
+		println!("New cluster layout with updated role assignment has been applied in cluster.");
+		println!("Data will now be moved around between nodes accordingly.");
+
+		Ok(())
+	}
+
+	pub async fn cmd_revert_layout(&self, revert_opt: RevertLayoutOpt) -> Result<(), Error> {
+		if !revert_opt.yes {
+			return Err(Error::Message(
+				"Please add the --yes flag to run the layout revert operation".into(),
+			));
+		}
+
+		self.api_request(RevertClusterLayoutRequest).await?;
+
+		println!("All proposed role changes in cluster layout have been canceled.");
+		Ok(())
+	}
+
+	pub async fn cmd_layout_history(&self) -> Result<(), Error> {
+		let history = self.api_request(GetClusterLayoutHistoryRequest).await?;
+
+		println!("==== LAYOUT HISTORY ====");
+		let mut table = vec!["Version\tStatus\tStorage nodes\tGateway nodes".to_string()];
+		for ver in history.versions.iter() {
+			table.push(format!(
+				"#{}\t{:?}\t{}\t{}",
+				ver.version, ver.status, ver.storage_nodes, ver.gateway_nodes,
+			));
+		}
+		format_table(table);
+		println!();
+
+		if let Some(update_trackers) = history.update_trackers {
+			println!("==== UPDATE TRACKERS ====");
+			println!("Several layout versions are currently live in the cluster, and data is being migrated.");
+			println!(
+				"This is the internal data that Garage stores to know which nodes have what data."
+			);
+			println!();
+			let mut table = vec!["Node\tAck\tSync\tSync_ack".to_string()];
+			for (node, trackers) in update_trackers.iter() {
+				table.push(format!(
+					"{:.16}\t#{}\t#{}\t#{}",
+					node, trackers.ack, trackers.sync, trackers.sync_ack,
+				));
+			}
+			table[1..].sort();
+			format_table(table);
+
+			println!();
+			println!(
+                "If some nodes are not catching up to the latest layout version in the update trackers,"
+            );
+			println!(
+				"it might be because they are offline or unable to complete a sync successfully."
+			);
+			if history.min_ack < history.current_version {
+				println!(
+					"You may force progress using `garage layout skip-dead-nodes --version {}`",
+					history.current_version
+				);
+			} else {
+				println!(
+                    "You may force progress using `garage layout skip-dead-nodes --version {} --allow-missing-data`.",
+                    history.current_version
+                );
+			}
+		} else {
+			println!(
+				"Your cluster is currently in a stable state with a single live layout version."
+			);
+			println!("No metadata migration is in progress. Note that the migration of data blocks is not tracked,");
+			println!(
+                "so you might want to keep old nodes online until their data directories become empty."
+            );
+		}
+
+		Ok(())
+	}
+
+	pub async fn cmd_skip_dead_nodes(&self, opt: SkipDeadNodesOpt) -> Result<(), Error> {
+		let res = self
+			.api_request(ClusterLayoutSkipDeadNodesRequest {
+				version: opt.version,
+				allow_missing_data: opt.allow_missing_data,
+			})
+			.await?;
+
+		if !res.sync_updated.is_empty() || !res.ack_updated.is_empty() {
+			for node in res.ack_updated.iter() {
+				println!("Increased the ACK tracker for node {:.16}", node);
+			}
+			for node in res.sync_updated.iter() {
+				println!("Increased the SYNC tracker for node {:.16}", node);
+			}
+			Ok(())
+		} else if !opt.allow_missing_data {
+			Err(Error::Message("Nothing was done, try passing the `--allow-missing-data` flag to force progress even when not enough nodes can complete a metadata sync.".into()))
+		} else {
+			Err(Error::Message(
+                "Sorry, there is nothing I can do for you. Please wait patiently. If you ask for help, please send the output of the `garage layout history` command.".into(),
+            ))
+		}
+	}
+}
+
+// --------------------------
+// ---- helper functions ----
+// --------------------------
+
+pub fn capacity_string(v: Option<u64>) -> String {
+	match v {
+		Some(c) => ByteSize::b(c).to_string_as(false),
+		None => "gateway".to_string(),
+	}
+}
+
+pub fn get_staged_or_current_role(
+	id: &str,
+	layout: &GetClusterLayoutResponse,
+) -> Option<NodeAssignedRole> {
+	for node in layout.staged_role_changes.iter() {
+		if node.id == id {
+			return match &node.action {
+				NodeRoleChangeEnum::Remove { .. } => None,
+				NodeRoleChangeEnum::Update(role) => Some(role.clone()),
+			};
+		}
+	}
+
+	for node in layout.roles.iter() {
+		if node.id == id {
+			return Some(NodeAssignedRole {
+				zone: node.zone.clone(),
+				capacity: node.capacity,
+				tags: node.tags.clone(),
+			});
+		}
+	}
+
+	None
+}
+
+pub fn find_matching_node<'a>(
+	status: &GetClusterStatusResponse,
+	layout: &GetClusterLayoutResponse,
+	pattern: &'a str,
+) -> Result<String, Error> {
+	let all_node_ids_iter = status
+		.nodes
+		.iter()
+		.map(|x| x.id.as_str())
+		.chain(layout.roles.iter().map(|x| x.id.as_str()));
+
+	let mut candidates = vec![];
+	for c in all_node_ids_iter {
+		if c.starts_with(pattern) && !candidates.contains(&c) {
+			candidates.push(c);
+		}
+	}
+	if candidates.len() != 1 {
+		Err(Error::Message(format!(
+			"{} nodes match '{}'",
+			candidates.len(),
+			pattern,
+		)))
+	} else {
+		Ok(candidates[0].to_string())
+	}
+}
+
+pub fn print_cluster_layout(layout: &GetClusterLayoutResponse, empty_msg: &str) {
+	let mut table = vec!["ID\tTags\tZone\tCapacity\tUsable capacity".to_string()];
+	for role in layout.roles.iter() {
+		let tags = role.tags.join(",");
+		if let (Some(capacity), Some(usable_capacity)) = (role.capacity, role.usable_capacity) {
+			table.push(format!(
+				"{:.16}\t[{}]\t{}\t{}\t{} ({:.1}%)",
+				role.id,
+				tags,
+				role.zone,
+				capacity_string(role.capacity),
+				ByteSize::b(usable_capacity).to_string_as(false),
+				(100.0 * usable_capacity as f32) / (capacity as f32)
+			));
+		} else {
+			table.push(format!(
+				"{:.16}\t[{}]\t{}\t{}",
+				role.id,
+				tags,
+				role.zone,
+				capacity_string(role.capacity),
+			));
+		};
+	}
+	if table.len() > 1 {
+		format_table(table);
+		println!();
+		println!(
+			"Zone redundancy: {}",
+			display_zone_redundancy(layout.parameters.zone_redundancy),
+		);
+	} else {
+		println!("{}", empty_msg);
+	}
+}
+
+pub fn print_staging_role_changes(layout: &GetClusterLayoutResponse) -> bool {
+	let has_role_changes = !layout.staged_role_changes.is_empty();
+
+	let has_layout_changes = layout.staged_parameters.is_some();
+
+	if has_role_changes || has_layout_changes {
+		println!();
+		println!("==== STAGED ROLE CHANGES ====");
+		if has_role_changes {
+			let mut table = vec!["ID\tTags\tZone\tCapacity".to_string()];
+			for change in layout.staged_role_changes.iter() {
+				match &change.action {
+					NodeRoleChangeEnum::Update(NodeAssignedRole {
+						tags,
+						zone,
+						capacity,
+					}) => {
+						let tags = tags.join(",");
+						table.push(format!(
+							"{:.16}\t[{}]\t{}\t{}",
+							change.id,
+							tags,
+							zone,
+							capacity_string(*capacity),
+						));
+					}
+					NodeRoleChangeEnum::Remove { .. } => {
+						table.push(format!("{:.16}\tREMOVED", change.id));
+					}
+				}
+			}
+			format_table(table);
+			println!();
+		}
+		if let Some(p) = layout.staged_parameters.as_ref() {
+			println!(
+				"Zone redundancy: {}",
+				display_zone_redundancy(p.zone_redundancy)
+			);
+		}
+		true
+	} else {
+		false
+	}
+}
+
+pub fn display_zone_redundancy(z: ZoneRedundancy) -> String {
+	match z {
+		ZoneRedundancy::Maximum => "maximum".into(),
+		ZoneRedundancy::AtLeast(x) => x.to_string(),
+	}
+}
+
+pub fn parse_zone_redundancy(s: &str) -> Result<ZoneRedundancy, Error> {
+	match s {
+		"none" | "max" | "maximum" => Ok(ZoneRedundancy::Maximum),
+		x => {
+			let v = x.parse::<usize>().map_err(|_| {
+				Error::Message("zone redundancy must be 'none'/'max' or an integer".into())
+			})?;
+			Ok(ZoneRedundancy::AtLeast(v))
+		}
+	}
+}

src/garage/cli/remote/mod.rs

@@ -0,0 +1,174 @@
+pub mod admin_token;
+pub mod bucket;
+pub mod cluster;
+pub mod key;
+pub mod layout;
+
+pub mod block;
+pub mod node;
+pub mod worker;
+
+use std::convert::TryFrom;
+use std::sync::Arc;
+use std::time::Duration;
+
+use chrono::{DateTime, Utc};
+
+use garage_util::error::*;
+
+use garage_rpc::*;
+
+use garage_api_admin::api::*;
+use garage_api_admin::api_server::{AdminRpc as ProxyRpc, AdminRpcResponse as ProxyRpcResponse};
+use garage_api_admin::RequestHandler;
+
+use crate::cli::structs::*;
+
+pub struct Cli {
+	pub proxy_rpc_endpoint: Arc<Endpoint<ProxyRpc, ()>>,
+	pub rpc_host: NodeID,
+}
+
+impl Cli {
+	pub async fn handle(&self, cmd: Command) -> Result<(), Error> {
+		match cmd {
+			Command::Status => self.cmd_status().await,
+			Command::Node(NodeOperation::Connect(connect_opt)) => {
+				self.cmd_connect(connect_opt).await
+			}
+			Command::Layout(layout_opt) => self.layout_command_dispatch(layout_opt).await,
+			Command::Bucket(bo) => self.cmd_bucket(bo).await,
+			Command::AdminToken(to) => self.cmd_admin_token(to).await,
+			Command::Key(ko) => self.cmd_key(ko).await,
+			Command::Worker(wo) => self.cmd_worker(wo).await,
+			Command::Block(bo) => self.cmd_block(bo).await,
+			Command::Meta(mo) => self.cmd_meta(mo).await,
+			Command::Stats(so) => self.cmd_stats(so).await,
+			Command::Repair(ro) => self.cmd_repair(ro).await,
+			Command::JsonApi { endpoint, payload } => self.cmd_json_api(endpoint, payload).await,
+
+			_ => unreachable!(),
+		}
+	}
+
+	pub async fn api_request<T>(&self, req: T) -> Result<<T as RequestHandler>::Response, Error>
+	where
+		T: RequestHandler,
+		AdminApiRequest: From<T>,
+		<T as RequestHandler>::Response: TryFrom<TaggedAdminApiResponse>,
+	{
+		let req = AdminApiRequest::from(req);
+		let req_name = req.name();
+		match self
+			.proxy_rpc_endpoint
+			.call(&self.rpc_host, ProxyRpc::Proxy(req), PRIO_NORMAL)
+			.await??
+		{
+			ProxyRpcResponse::ProxyApiOkResponse(resp) => {
+				<T as RequestHandler>::Response::try_from(resp).map_err(|_| {
+					Error::Message(format!("{} returned unexpected response", req_name))
+				})
+			}
+			ProxyRpcResponse::ApiErrorResponse {
+				http_code,
+				error_code,
+				message,
+			} => Err(Error::Message(format!(
+				"{} returned {} ({}): {}",
+				req_name, error_code, http_code, message
+			))),
+			m => Err(Error::unexpected_rpc_message(m)),
+		}
+	}
+
+	pub async fn local_api_request<T>(
+		&self,
+		req: T,
+	) -> Result<<T as RequestHandler>::Response, Error>
+	where
+		T: RequestHandler,
+		MultiRequest<T>: RequestHandler<Response = MultiResponse<<T as RequestHandler>::Response>>,
+		AdminApiRequest: From<MultiRequest<T>>,
+		<MultiRequest<T> as RequestHandler>::Response: TryFrom<TaggedAdminApiResponse>,
+	{
+		let req = MultiRequest {
+			node: hex::encode(self.rpc_host),
+			body: req,
+		};
+		let resp = self.api_request(req).await?;
+
+		if let Some((_, e)) = resp.error.into_iter().next() {
+			return Err(Error::Message(e));
+		}
+		if resp.success.len() != 1 {
+			return Err(Error::Message(format!(
+				"{} responses returned, expected 1",
+				resp.success.len()
+			)));
+		}
+		Ok(resp.success.into_iter().next().unwrap().1)
+	}
+
+	pub async fn cmd_json_api(&self, endpoint: String, payload: String) -> Result<(), Error> {
+		let payload: serde_json::Value = if payload == "-" {
+			serde_json::from_reader(&std::io::stdin())?
+		} else {
+			serde_json::from_str(&payload)?
+		};
+
+		let request: AdminApiRequest = serde_json::from_value(serde_json::json!({
+			endpoint.clone(): payload,
+		}))?;
+
+		let resp = match self
+			.proxy_rpc_endpoint
+			.call(&self.rpc_host, ProxyRpc::Proxy(request), PRIO_NORMAL)
+			.await??
+		{
+			ProxyRpcResponse::ProxyApiOkResponse(resp) => resp,
+			ProxyRpcResponse::ApiErrorResponse {
+				http_code,
+				error_code,
+				message,
+			} => {
+				return Err(Error::Message(format!(
+					"{} ({}): {}",
+					error_code, http_code, message
+				)))
+			}
+			m => return Err(Error::unexpected_rpc_message(m)),
+		};
+
+		if let serde_json::Value::Object(map) = serde_json::to_value(&resp)? {
+			if let Some(inner) = map.get(&endpoint) {
+				serde_json::to_writer_pretty(std::io::stdout(), &inner)?;
+				return Ok(());
+			}
+		}
+
+		Err(Error::Message(format!(
+			"Invalid response: {}",
+			serde_json::to_string(&resp)?
+		)))
+	}
+}
+
+pub fn table_list_abbr<T: IntoIterator<Item = S>, S: AsRef<str>>(values: T) -> String {
+	let mut iter = values.into_iter();
+
+	match iter.next() {
+		Some(first) => match iter.count() {
+			0 => first.as_ref().to_string(),
+			n => format!("{}, ... ({})", first.as_ref(), n + 1),
+		},
+		None => String::new(),
+	}
+}
+
+pub fn parse_expires_in(expires_in: &Option<String>) -> Result<Option<DateTime<Utc>>, Error> {
+	expires_in
+		.as_ref()
+		.map(|x| parse_duration::parse::parse(&x).map(|dur| Utc::now() + dur))
+		.transpose()
+		.ok_or_message("Invalid duration passed for --expires-in parameter")
+}