and again

Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/1295
Co-authored-by: Alex Auvolat <lx@deuxfleurs.fr>
Co-committed-by: Alex Auvolat <lx@deuxfleurs.fr>

.woodpecker/debug.yaml

@@ -1,3 +1,6 @@
+labels:
+  nix: "enabled"
+
 when:
   event:
   - push
@@ -9,27 +12,32 @@ when:
 
 steps:
   - name: check formatting
-    image: nixpkgs/nix:nixos-22.05
+    image: nixpkgs/nix:nixos-24.05
     commands:
-      - nix-shell --attr devShell --run "cargo fmt -- --check"
+      - nix-build -j4 --attr flakePackages.fmt
 
   - name: build
-    image: nixpkgs/nix:nixos-22.05
+    image: nixpkgs/nix:nixos-24.05
     commands:
       - nix-build -j4 --attr flakePackages.dev
 
   - name: unit + func tests (lmdb)
-    image: nixpkgs/nix:nixos-22.05
+    image: nixpkgs/nix:nixos-24.05
     commands:
       - nix-build -j4 --attr flakePackages.tests-lmdb
 
   - name: unit + func tests (sqlite)
-    image: nixpkgs/nix:nixos-22.05
+    image: nixpkgs/nix:nixos-24.05
     commands:
       - nix-build -j4 --attr flakePackages.tests-sqlite
 
+  - name: unit + func tests (fjall)
+    image: nixpkgs/nix:nixos-24.05
+    commands:
+      - nix-build -j4 --attr flakePackages.tests-fjall
+
   - name: integration tests
-    image: nixpkgs/nix:nixos-22.05
+    image: nixpkgs/nix:nixos-24.05
     commands:
       - nix-build -j4 --attr flakePackages.dev
       - nix-shell --attr ci --run ./script/test-smoke.sh || (cat /tmp/garage.log; false)

.woodpecker/publish.yaml

@@ -1,3 +1,6 @@
+labels:
+  nix: "enabled"
+
 when:
   event:
   - deployment
@@ -8,7 +11,7 @@ depends_on:
 
 steps:
   - name: refresh-index
-    image: nixpkgs/nix:nixos-22.05
+    image: nixpkgs/nix:nixos-24.05
     environment:
       AWS_ACCESS_KEY_ID:
         from_secret: garagehq_aws_access_key_id
@@ -19,7 +22,7 @@ steps:
       - nix-shell --attr ci --run "refresh_index"
 
   - name: multiarch-docker
-    image: nixpkgs/nix:nixos-22.05
+    image: nixpkgs/nix:nixos-24.05
     environment:
       DOCKER_AUTH:
         from_secret: docker_auth

.woodpecker/release.yaml

@@ -1,3 +1,6 @@
+labels:
+  nix: "enabled"
+
 when:
   event:
   - deployment
@@ -16,17 +19,17 @@ matrix:
 
 steps:
   - name: build
-    image: nixpkgs/nix:nixos-22.05
+    image: nixpkgs/nix:nixos-24.05
     commands:
       - nix-build --attr releasePackages.${ARCH} --argstr git_version ${CI_COMMIT_TAG:-$CI_COMMIT_SHA}
 
   - name: check is static binary
-    image: nixpkgs/nix:nixos-22.05
+    image: nixpkgs/nix:nixos-24.05
     commands:
       - nix-shell --attr ci --run "./script/not-dynamic.sh result/bin/garage"
 
   - name: integration tests
-    image: nixpkgs/nix:nixos-22.05
+    image: nixpkgs/nix:nixos-24.05
     commands:
       - nix-shell --attr ci --run ./script/test-smoke.sh || (cat /tmp/garage.log; false)
     when:
@@ -36,7 +39,7 @@ steps:
           ARCH: i386
 
   - name: upgrade tests from v1.0.0
-    image: nixpkgs/nix:nixos-22.05
+    image: nixpkgs/nix:nixos-24.05
     commands:
       - nix-shell --attr ci --run "./script/test-upgrade.sh v1.0.0 x86_64-unknown-linux-musl" || (cat /tmp/garage.log; false)
     when:
@@ -44,7 +47,7 @@ steps:
           ARCH: amd64
 
   - name: upgrade tests from v0.8.4
-    image: nixpkgs/nix:nixos-22.05
+    image: nixpkgs/nix:nixos-24.05
     commands:
       - nix-shell --attr ci --run "./script/test-upgrade.sh v0.8.4 x86_64-unknown-linux-musl" || (cat /tmp/garage.log; false)
     when:
@@ -52,7 +55,7 @@ steps:
           ARCH: amd64
 
   - name: push static binary
-    image: nixpkgs/nix:nixos-22.05
+    image: nixpkgs/nix:nixos-24.05
     environment:
       TARGET: "${TARGET}"
       AWS_ACCESS_KEY_ID:
@@ -63,7 +66,7 @@ steps:
       - nix-shell --attr ci --run "to_s3"
 
   - name: docker build and publish
-    image: nixpkgs/nix:nixos-22.05
+    image: nixpkgs/nix:nixos-24.05
     environment:
       DOCKER_PLATFORM: "linux/${ARCH}"
       CONTAINER_NAME: "dxflrs/${ARCH}_garage"

Cargo.toml

@@ -24,18 +24,18 @@ default-members = ["src/garage"]
 
 # Internal Garage crates
 format_table = { version = "0.1.1", path = "src/format-table" }
-garage_api_common = { version = "2.0.0", path = "src/api/common" }
-garage_api_admin = { version = "2.0.0", path = "src/api/admin" }
-garage_api_s3 = { version = "2.0.0", path = "src/api/s3" }
-garage_api_k2v = { version = "2.0.0", path = "src/api/k2v" }
-garage_block = { version = "2.0.0", path = "src/block" }
-garage_db = { version = "2.0.0", path = "src/db", default-features = false }
-garage_model = { version = "2.0.0", path = "src/model", default-features = false }
-garage_net = { version = "2.0.0", path = "src/net" }
-garage_rpc = { version = "2.0.0", path = "src/rpc" }
-garage_table = { version = "2.0.0", path = "src/table" }
-garage_util = { version = "2.0.0", path = "src/util" }
-garage_web = { version = "2.0.0", path = "src/web" }
+garage_api_common = { version = "2.2.0", path = "src/api/common" }
+garage_api_admin = { version = "2.2.0", path = "src/api/admin" }
+garage_api_s3 = { version = "2.2.0", path = "src/api/s3" }
+garage_api_k2v = { version = "2.2.0", path = "src/api/k2v" }
+garage_block = { version = "2.2.0", path = "src/block" }
+garage_db = { version = "2.2.0", path = "src/db", default-features = false }
+garage_model = { version = "2.2.0", path = "src/model", default-features = false }
+garage_net = { version = "2.2.0", path = "src/net" }
+garage_rpc = { version = "2.2.0", path = "src/rpc" }
+garage_table = { version = "2.2.0", path = "src/table" }
+garage_util = { version = "2.2.0", path = "src/util" }
+garage_web = { version = "2.2.0", path = "src/web" }
 k2v-client = { version = "0.0.4", path = "src/k2v-client" }
 
 # External crates from crates.io
@@ -49,9 +49,8 @@ bytes = "1.0"
 bytesize = "1.1"
 cfg-if = "1.0"
 chrono = { version = "0.4", features = ["serde"] }
-crc-fast = "1.1"
+crc-fast = "1.6"
 crypto-common = "0.1"
-err-derive = "0.3"
 gethostname = "0.4"
 git-version = "0.3.4"
 hex = "0.4"
@@ -64,6 +63,7 @@ md-5 = "0.10"
 mktemp = "0.5"
 nix = { version = "0.29", default-features = false, features = ["fs"] }
 nom = "7.1"
+parking_lot = "0.12"
 parse_duration = "2.1"
 paste = "1.0"
 pin-project = "1.0.12"
@@ -87,9 +87,10 @@ tracing-journald = "0.3.1"
 tracing-subscriber = { version = "0.3", features = ["env-filter"] }
 
 heed = { version = "0.11", default-features = false, features = ["lmdb"] }
-rusqlite = "0.31.0"
+rusqlite = "0.37"
 r2d2 = "0.8"
-r2d2_sqlite = "0.24"
+r2d2_sqlite = "0.31"
+fjall = "2.4"
 
 async-compression = { version = "0.4", features = ["tokio", "zstd"] }
 zstd = { version = "0.13", default-features = false }
@@ -136,7 +137,7 @@ prometheus = "0.13"
 aws-sigv4 = { version = "1.1", default-features = false }
 hyper-rustls = { version = "0.26", default-features = false, features = ["http1", "http2", "ring", "rustls-native-certs"] }
 log = "0.4"
-thiserror = "1.0"
+thiserror = "2.0"
 
 # ---- used only as build / dev dependencies ----
 assert-json-diff = "2.0"
@@ -153,5 +154,5 @@ lto = "off"
 [profile.release]
 lto = true
 codegen-units = 1
-opt-level = "s"
+opt-level = 3
 strip = true

doc/api/garage-admin-v2.json

@@ -12,7 +12,7 @@
       "name": "AGPL-3.0",
       "identifier": "AGPL-3.0"
     },
-    "version": "v2.0.0"
+    "version": "v2.2.0"
   },
   "servers": [
     {
@@ -31,9 +31,12 @@
         "parameters": [
           {
             "name": "domain",
-            "in": "path",
+            "in": "query",
             "description": "The domain name to check for",
-            "required": true
+            "required": true,
+            "schema": {
+              "type": "string"
+            }
           }
         ],
         "responses": {
@@ -100,7 +103,7 @@
           "content": {
             "application/json": {
               "schema": {
-                "$ref": "#/components/schemas/AddBucketAliasRequest"
+                "$ref": "#/components/schemas/BucketAliasEnum"
               }
             }
           },
@@ -405,9 +408,12 @@
         "parameters": [
           {
             "name": "node",
-            "in": "path",
+            "in": "query",
             "description": "Node ID to query, or `*` for all nodes, or `self` for the node responding to the request",
-            "required": true
+            "required": true,
+            "schema": {
+              "type": "string"
+            }
           }
         ],
         "responses": {
@@ -437,9 +443,12 @@
         "parameters": [
           {
             "name": "id",
-            "in": "path",
+            "in": "query",
             "description": "Admin API token ID",
-            "required": true
+            "required": true,
+            "schema": {
+              "type": "string"
+            }
           }
         ],
         "responses": {
@@ -462,9 +471,12 @@
         "parameters": [
           {
             "name": "id",
-            "in": "path",
+            "in": "query",
             "description": "ID of the bucket to delete",
-            "required": true
+            "required": true,
+            "schema": {
+              "type": "string"
+            }
           }
         ],
         "responses": {
@@ -493,9 +505,12 @@
         "parameters": [
           {
             "name": "id",
-            "in": "path",
+            "in": "query",
             "description": "Access key ID",
-            "required": true
+            "required": true,
+            "schema": {
+              "type": "string"
+            }
           }
         ],
         "responses": {
@@ -552,26 +567,20 @@
         "parameters": [
           {
             "name": "id",
-            "in": "path",
+            "in": "query",
             "description": "Admin API token ID",
-            "required": true,
+            "required": false,
             "schema": {
-              "type": [
-                "string",
-                "null"
-              ]
+              "type": "string"
             }
           },
           {
             "name": "search",
-            "in": "path",
+            "in": "query",
             "description": "Partial token ID or name to search for",
-            "required": true,
+            "required": false,
             "schema": {
-              "type": [
-                "string",
-                "null"
-              ]
+              "type": "string"
             }
           }
         ],
@@ -602,9 +611,12 @@
         "parameters": [
           {
             "name": "node",
-            "in": "path",
+            "in": "query",
             "description": "Node ID to query, or `*` for all nodes, or `self` for the node responding to the request",
-            "required": true
+            "required": true,
+            "schema": {
+              "type": "string"
+            }
           }
         ],
         "requestBody": {
@@ -644,38 +656,29 @@
         "parameters": [
           {
             "name": "id",
-            "in": "path",
+            "in": "query",
             "description": "Exact bucket ID to look up",
-            "required": true,
+            "required": false,
             "schema": {
-              "type": [
-                "string",
-                "null"
-              ]
+              "type": "string"
             }
           },
           {
             "name": "globalAlias",
-            "in": "path",
+            "in": "query",
             "description": "Global alias of bucket to look up",
-            "required": true,
+            "required": false,
             "schema": {
-              "type": [
-                "string",
-                "null"
-              ]
+              "type": "string"
             }
           },
           {
             "name": "search",
-            "in": "path",
+            "in": "query",
             "description": "Partial ID or alias to search for",
-            "required": true,
+            "required": false,
             "schema": {
-              "type": [
-                "string",
-                "null"
-              ]
+              "type": "string"
             }
           }
         ],
@@ -813,6 +816,30 @@
         }
       }
     },
+    "/v2/GetCurrentAdminTokenInfo": {
+      "get": {
+        "tags": [
+          "Admin API token"
+        ],
+        "description": "\nReturn information about the calling admin API token.\n    ",
+        "operationId": "GetCurrentAdminTokenInfo",
+        "responses": {
+          "200": {
+            "description": "Information about the admin token",
+            "content": {
+              "application/json": {
+                "schema": {
+                  "$ref": "#/components/schemas/GetCurrentAdminTokenInfoResponse"
+                }
+              }
+            }
+          },
+          "500": {
+            "description": "Internal server error"
+          }
+        }
+      }
+    },
     "/v2/GetKeyInfo": {
       "get": {
         "tags": [
@@ -823,33 +850,27 @@
         "parameters": [
           {
             "name": "id",
-            "in": "path",
+            "in": "query",
             "description": "Access key ID",
-            "required": true,
+            "required": false,
             "schema": {
-              "type": [
-                "string",
-                "null"
-              ]
+              "type": "string"
             }
           },
           {
             "name": "search",
-            "in": "path",
+            "in": "query",
             "description": "Partial key ID or name to search for",
-            "required": true,
+            "required": false,
             "schema": {
-              "type": [
-                "string",
-                "null"
-              ]
+              "type": "string"
             }
           },
           {
             "name": "showSecretKey",
-            "in": "path",
+            "in": "query",
             "description": "Whether to return the secret access key",
-            "required": true,
+            "required": false,
             "schema": {
               "type": "boolean"
             }
@@ -882,9 +903,12 @@
         "parameters": [
           {
             "name": "node",
-            "in": "path",
+            "in": "query",
             "description": "Node ID to query, or `*` for all nodes, or `self` for the node responding to the request",
-            "required": true
+            "required": true,
+            "schema": {
+              "type": "string"
+            }
           }
         ],
         "responses": {
@@ -914,9 +938,12 @@
         "parameters": [
           {
             "name": "node",
-            "in": "path",
+            "in": "query",
             "description": "Node ID to query, or `*` for all nodes, or `self` for the node responding to the request",
-            "required": true
+            "required": true,
+            "schema": {
+              "type": "string"
+            }
           }
         ],
         "responses": {
@@ -946,9 +973,12 @@
         "parameters": [
           {
             "name": "node",
-            "in": "path",
+            "in": "query",
             "description": "Node ID to query, or `*` for all nodes, or `self` for the node responding to the request",
-            "required": true
+            "required": true,
+            "schema": {
+              "type": "string"
+            }
           }
         ],
         "requestBody": {
@@ -988,9 +1018,12 @@
         "parameters": [
           {
             "name": "node",
-            "in": "path",
+            "in": "query",
             "description": "Node ID to query, or `*` for all nodes, or `self` for the node responding to the request",
-            "required": true
+            "required": true,
+            "schema": {
+              "type": "string"
+            }
           }
         ],
         "requestBody": {
@@ -1064,7 +1097,7 @@
         "parameters": [
           {
             "name": "bucketId",
-            "in": "path",
+            "in": "query",
             "required": true,
             "schema": {
               "type": "string"
@@ -1072,7 +1105,7 @@
           },
           {
             "name": "key",
-            "in": "path",
+            "in": "query",
             "required": true,
             "schema": {
               "type": "string"
@@ -1109,9 +1142,12 @@
         "parameters": [
           {
             "name": "node",
-            "in": "path",
+            "in": "query",
             "description": "Node ID to query, or `*` for all nodes, or `self` for the node responding to the request",
-            "required": true
+            "required": true,
+            "schema": {
+              "type": "string"
+            }
           }
         ],
         "requestBody": {
@@ -1175,9 +1211,12 @@
         "parameters": [
           {
             "name": "node",
-            "in": "path",
+            "in": "query",
             "description": "Node ID to query, or `*` for all nodes, or `self` for the node responding to the request",
-            "required": true
+            "required": true,
+            "schema": {
+              "type": "string"
+            }
           }
         ],
         "responses": {
@@ -1255,9 +1294,12 @@
         "parameters": [
           {
             "name": "node",
-            "in": "path",
+            "in": "query",
             "description": "Node ID to query, or `*` for all nodes, or `self` for the node responding to the request",
-            "required": true
+            "required": true,
+            "schema": {
+              "type": "string"
+            }
           }
         ],
         "requestBody": {
@@ -1321,9 +1363,12 @@
         "parameters": [
           {
             "name": "node",
-            "in": "path",
+            "in": "query",
             "description": "Node ID to query, or `*` for all nodes, or `self` for the node responding to the request",
-            "required": true
+            "required": true,
+            "schema": {
+              "type": "string"
+            }
           }
         ],
         "requestBody": {
@@ -1364,7 +1409,7 @@
           "content": {
             "application/json": {
               "schema": {
-                "$ref": "#/components/schemas/RemoveBucketAliasRequest"
+                "$ref": "#/components/schemas/BucketAliasEnum"
               }
             }
           },
@@ -1397,9 +1442,12 @@
         "parameters": [
           {
             "name": "node",
-            "in": "path",
+            "in": "query",
             "description": "Node ID to query, or `*` for all nodes, or `self` for the node responding to the request",
-            "required": true
+            "required": true,
+            "schema": {
+              "type": "string"
+            }
           }
         ],
         "requestBody": {
@@ -1463,9 +1511,12 @@
         "parameters": [
           {
             "name": "node",
-            "in": "path",
+            "in": "query",
             "description": "Node ID to query, or `*` for all nodes, or `self` for the node responding to the request",
-            "required": true
+            "required": true,
+            "schema": {
+              "type": "string"
+            }
           }
         ],
         "requestBody": {
@@ -1505,9 +1556,12 @@
         "parameters": [
           {
             "name": "id",
-            "in": "path",
+            "in": "query",
             "description": "Admin API token ID",
-            "required": true
+            "required": true,
+            "schema": {
+              "type": "string"
+            }
           }
         ],
         "requestBody": {
@@ -1547,9 +1601,12 @@
         "parameters": [
           {
             "name": "id",
-            "in": "path",
+            "in": "query",
             "description": "ID of the bucket to update",
-            "required": true
+            "required": true,
+            "schema": {
+              "type": "string"
+            }
           }
         ],
         "requestBody": {
@@ -1627,9 +1684,12 @@
         "parameters": [
           {
             "name": "id",
-            "in": "path",
+            "in": "query",
             "description": "Access key ID",
-            "required": true
+            "required": true,
+            "schema": {
+              "type": "string"
+            }
           }
         ],
         "requestBody": {
@@ -1662,24 +1722,6 @@
   },
   "components": {
     "schemas": {
-      "AddBucketAliasRequest": {
-        "allOf": [
-          {
-            "$ref": "#/components/schemas/BucketAliasEnum"
-          },
-          {
-            "type": "object",
-            "required": [
-              "bucketId"
-            ],
-            "properties": {
-              "bucketId": {
-                "type": "string"
-              }
-            }
-          }
-        ]
-      },
       "AddBucketAliasResponse": {
         "$ref": "#/components/schemas/GetBucketInfoResponse"
       },
@@ -1897,9 +1939,13 @@
           {
             "type": "object",
             "required": [
+              "bucketId",
               "globalAlias"
             ],
             "properties": {
+              "bucketId": {
+                "type": "string"
+              },
               "globalAlias": {
                 "type": "string"
               }
@@ -1908,6 +1954,7 @@
           {
             "type": "object",
             "required": [
+              "bucketId",
               "localAlias",
               "accessKeyId"
             ],
@@ -1915,6 +1962,9 @@
               "accessKeyId": {
                 "type": "string"
               },
+              "bucketId": {
+                "type": "string"
+              },
               "localAlias": {
                 "type": "string"
               }
@@ -2383,7 +2433,7 @@
           "knownNodes",
           "connectedNodes",
           "storageNodes",
-          "storageNodesOk",
+          "storageNodesUp",
           "partitions",
           "partitionsQuorum",
           "partitionsAllOk"
@@ -2423,7 +2473,7 @@
             "description": "the number of storage nodes currently registered in the cluster layout",
             "minimum": 0
           },
-          "storageNodesOk": {
+          "storageNodesUp": {
             "type": "integer",
             "description": "the number of storage nodes to which a connection is currently open",
             "minimum": 0
@@ -2558,6 +2608,9 @@
           }
         }
       },
+      "GetCurrentAdminTokenInfoResponse": {
+        "$ref": "#/components/schemas/GetAdminTokenInfoResponse"
+      },
       "GetKeyInfoResponse": {
         "type": "object",
         "required": [
@@ -3849,6 +3902,49 @@
           }
         ]
       },
+      "NodeRoleChangeRequest": {
+        "oneOf": [
+          {
+            "type": "object",
+            "required": [
+              "id",
+              "remove"
+            ],
+            "properties": {
+              "id": {
+                "type": "string",
+                "description": "ID of the node for which this change applies"
+              },
+              "remove": {
+                "type": "boolean",
+                "description": "Set `remove` to `true` to remove the node from the layout"
+              }
+            }
+          },
+          {
+            "$ref": "#/components/schemas/NodeRoleUpdate"
+          }
+        ]
+      },
+      "NodeRoleUpdate": {
+        "allOf": [
+          {
+            "$ref": "#/components/schemas/NodeAssignedRole"
+          },
+          {
+            "type": "object",
+            "required": [
+              "id"
+            ],
+            "properties": {
+              "id": {
+                "type": "string",
+                "description": "ID of the node for which this change applies"
+              }
+            }
+          }
+        ]
+      },
       "NodeUpdateTrackers": {
         "type": "object",
         "required": [
@@ -3910,24 +4006,6 @@
           }
         ]
       },
-      "RemoveBucketAliasRequest": {
-        "allOf": [
-          {
-            "$ref": "#/components/schemas/BucketAliasEnum"
-          },
-          {
-            "type": "object",
-            "required": [
-              "bucketId"
-            ],
-            "properties": {
-              "bucketId": {
-                "type": "string"
-              }
-            }
-          }
-        ]
-      },
       "RemoveBucketAliasResponse": {
         "$ref": "#/components/schemas/GetBucketInfoResponse"
       },
@@ -3991,6 +4069,12 @@
             "enum": [
               "aliases"
             ]
+          },
+          {
+            "type": "string",
+            "enum": [
+              "clearResyncQueue"
+            ]
           }
         ]
       },
@@ -4111,7 +4195,7 @@
           "roles": {
             "type": "array",
             "items": {
-              "$ref": "#/components/schemas/NodeRoleChange"
+              "$ref": "#/components/schemas/NodeRoleChangeRequest"
             },
             "description": "New node roles to assign or remove in the cluster layout"
           }
@@ -4269,43 +4353,39 @@
       "WorkerStateResp": {
         "oneOf": [
           {
-            "type": "string",
-            "enum": [
-              "busy"
-            ]
-          },
+            "$ref": "#/components/schemas/Yolo"
+	  },
           {
+            "$ref": "#/components/schemas/WorkerStateRespStrs"
+	  }
+        ]
+      },
+      "Yolo": {
+        "type": "object",
+        "required": [
+          "throttled"
+        ],
+        "properties": {
+          "throttled": {
             "type": "object",
             "required": [
-              "throttled"
+              "durationSecs"
             ],
             "properties": {
-              "throttled": {
-                "type": "object",
-                "required": [
-                  "durationSecs"
-                ],
-                "properties": {
-                  "durationSecs": {
-                    "type": "number",
-                    "format": "float"
-                  }
-                }
+              "durationSecs": {
+                "type": "number",
+                "format": "float"
               }
             }
-          },
-          {
-            "type": "string",
-            "enum": [
-              "idle"
-            ]
-          },
-          {
-            "type": "string",
-            "enum": [
-              "done"
-            ]
           }
+        }
+      },
+      "WorkerStateRespStrs": {
+        "type": "string",
+        "enum": [
+          "busy",
+          "idle",
+          "done"
         ]
       },
       "ZoneRedundancy": {

doc/book/connect/apps/index.md

@@ -12,8 +12,9 @@ In this section, we cover the following web applications:
 | [Mastodon](#mastodon)     | ✅       | Natively supported    |
 | [Matrix](#matrix)     | ✅       |  Tested with `synapse-s3-storage-provider`    |
 | [ejabberd](#ejabberd)     | ✅       |  `mod_s3_upload`    |
-| [Pixelfed](#pixelfed)     | ❓       |  Not yet tested    |
-| [Pleroma](#pleroma)     | ❓       |  Not yet tested    |
+| [Ente](#ente)     | ✅       |  Natively supported |
+| [Pixelfed](#pixelfed)     | ❓       |  Natively supported    |
+| [Pleroma](#pleroma)     | ✅        |  Natively supported    |
 | [Lemmy](#lemmy)     | ✅        |  Supported with pict-rs    |
 | [Funkwhale](#funkwhale)     | ❓       | Not yet tested     |
 | [Misskey](#misskey)     | ❓       | Not yet tested     |
@@ -191,10 +192,10 @@ garage key create peertube-key
 
 Keep the Key ID and the Secret key in a pad, they will be needed later.  
 
-We need two buckets, one for normal videos (named peertube-video) and one for webtorrent videos (named peertube-playlist).
+We need two buckets, one for normal videos (named peertube-videos) and one for webtorrent videos (named peertube-playlists).
 ```bash
 garage bucket create peertube-videos
-garage bucket create peertube-playlist
+garage bucket create peertube-playlists
 ```
 
 Now we allow our key to read and write on these buckets:
@@ -253,7 +254,7 @@ object_storage:
     proxify_private_files: false
 
   streaming_playlists:
-    bucket_name: 'peertube-playlist'
+    bucket_name: 'peertube-playlists'
 
     # Keep it empty for our example
     prefix: ''
@@ -567,13 +568,186 @@ The module can then be configured with:
 Other configuration options can be found in the
 [configuration YAML file](https://github.com/processone/ejabberd-contrib/blob/master/mod_s3_upload/conf/mod_s3_upload.yml).
 
+
+## Ente
+
+Ente is an alternative for Google Photos and Apple Photos. It [can be selfhosted](https://help.ente.io/self-hosting/) and is working fine with Garage as of May 2024.
+As a first step we need to create a bucket and a key for Ente:
+
+```bash
+garage bucket create ente
+garage key create ente-key
+# For the CORS setup to work, the key needs to be --owner as well, at least temporarily.
+garage bucket allow ente --read --write --owner --key ente-key
+```
+
+We also need to setup some CORS rules to allow the Ente frontend to access the bucket:
+
+```bash
+export CORS='{"CORSRules":[{"AllowedHeaders":["*"],"AllowedMethods":["GET", "PUT", "POST", "DELETE"],"AllowedOrigins":["*"], "ExposeHeaders":["ETag"]}]}'
+aws s3api put-bucket-cors --bucket ente --cors-configuration $CORS
+```
+
+Now we need to configure ente-server to use our bucket. This is explained [in the Ente S3 documentation](https://help.ente.io/self-hosting/guides/external-s3).
+Prepare a configuration file for ente's backend as `museum.yaml`:
+
+```yaml
+credentials-file: /credentials.yaml
+apps:
+  public-albums: https://albums.example.tld # If you want to use the share album feature
+    internal:
+hardcoded-ott:
+  local-domain-suffix: "@example.com" # Your domain
+  local-domain-value: 123456 # Custom One-Time Password since we are not sending mail by default
+key:
+  # WARNING -- You MUST CHANGE the values below
+  # Someone has made an image that can do it for you : https://github.com/EdyTheCow/ente-selfhost/blob/main/images/ente-server-tools/Dockerfile
+  # Simply build it yourself or run docker run --rm ghcr.io/edythecow/ente-server-tools go run tools/gen-random-keys/main.go
+  encryption: yvmG/RnzKrbCb9L3mgsmoxXr9H7i2Z4qlbT0mL3ln4w= # CHANGE THIS VALUE
+  hash: KXYiG07wC7GIgvCSdg+WmyWdXDAn6XKYJtp/wkEU7x573+byBRAYtpTP0wwvi8i/4l37uicX1dVTUzwH3sLZyw== # CHANGE THIS VALUE
+jwt:
+  secret: i2DecQmfGreG6q1vBj5tCokhlN41gcfS2cjOs9Po-u8= # CHANGE THIS VALUE
+```
+
+The full configuration file can be found [here](https://github.com/ente-io/ente/blob/main/server/configurations/local.yaml)
+Then prepare a credentials file as `credentials.yaml`
+
+```yaml
+db:
+    host: postgres
+    port: 5432
+    name: <ente_db_name>
+    user: <pguser>
+    password: <pgpass>
+
+s3:
+    # Override the primary and secondary hot storage. The commented out values
+    # are the defaults.
+    #
+    hot_storage:
+        primary: b2-eu-cen
+    #    secondary: wasabi-eu-central-2-v3
+
+    # If true, enable some workarounds to allow us to use a local minio instance
+    # for object storage.
+    #
+    # 1. Disable SSL.
+    # 2. Use "path" style S3 URLs (see `use_path_style_urls` below).
+    # 3. Directly download the file during replication instead of going via the
+    #    Cloudflare worker.
+    # 4. Do not specify storage classes when uploading objects (since minio does
+    #    not support them, specifically it doesn't support GLACIER).
+    are_local_buckets: true
+
+    # To use "path" style S3 URLs instead of DNS-based bucket access
+    # default to true if you set "are_local_buckets: true"
+    # use_path_style_urls: true
+
+    b2-eu-cen: # Don't change this key, it is hardcoded
+        key: <keyID> 
+        secret: <keySecret>
+        endpoint: garage:3900 # publically accessible endpoint of your garage instance
+        region: garage
+        bucket: <yourbucketName>
+        use_path_style: true
+    # you can specify secondary locations, names are hardcoded as well
+    # wasabi-eu-central-2-v3:
+    # scw-eu-fr-v3:
+    
+    # and you can also specify a bucket to be used for embeddings, preview etc..
+    # default to the first bucket 
+    # derived-storage: wasabi-eu-central-2-derived
+```
+
+Finally you can run it with Docker : 
+
+```bash
+docker run -d --name ente-server --restart unless-stopped -v /path/to/museum.yaml:/museum.yaml -v /path/to/credentials.yaml:/credentials.yaml -p 8080:8080 ghcr.io/ente-io/ente-server
+```
+
+For more information on deployment you can check the [ente documentation](https://help.ente.io/self-hosting/)
+
 ## Pixelfed
 
 [Pixelfed Technical Documentation > Configuration](https://docs.pixelfed.org/technical-documentation/env.html#filesystem)
 
 ## Pleroma
 
-[Pleroma Documentation > Pleroma.Uploaders.S3](https://docs-develop.pleroma.social/backend/configuration/cheatsheet/#pleromauploaderss3)
+### Creating your bucket
+
+This is the usual Garage setup:
+
+```bash
+garage key new --name pleroma-key
+garage bucket create pleroma
+garage bucket allow pleroma --read --write --owner --key pleroma-key
+```
+
+We also need to expose these buckets publicly to serve their content to users:
+
+```bash
+garage bucket website --allow pleroma
+```
+
+Note the Key ID and Secret Key.
+
+### Configure Pleroma
+
+Update your Pleroma configuration like that in `/etc/pleroma/config.exs`.
+
+```
+config :pleroma, Pleroma.Upload,
+    uploader: Pleroma.Uploaders.S3,
+    base_url: "https://pleroma.garage.example.tld"
+
+config :ex_aws, :s3,
+    access_key_id: "GW...",
+    secret_access_key: "XXX",
+    region: "garage",
+    host: "api.garage.example.tld"
+```
+
+And restart Pleroma.
+
+You can found more information in [Pleroma Documentation > Pleroma.Uploaders.S3](https://docs-develop.pleroma.social/backend/configuration/cheatsheet/#pleromauploaderss3)
+
+### Migrating your data
+
+Pleroma have an internal migration tool that can encounter some fatal error
+
+```
+** (EXIT from #PID<0.98.0>) an exception was raised:
+    ** (File.Error) could not stream "/var/lib/pleroma/uploads/09/f8": illegal operation on a directory
+        (elixir 1.17.3) lib/file/stream.ex:100: anonymous fn/3 in Enumerable.File.Stream.reduce/3
+        (elixir 1.17.3) lib/stream.ex:1675: anonymous fn/5 in Stream.resource/3
+        (elixir 1.17.3) lib/stream.ex:1891: Enumerable.Stream.do_each/4
+        (elixir 1.17.3) lib/task/supervised.ex:370: Task.Supervised.stream_reduce/7
+        (elixir 1.17.3) lib/enum.ex:4423: Enum.map/2
+        (ex_aws_s3 2.5.8) lib/ex_aws/s3/upload.ex:141: ExAws.Operation.ExAws.S3.Upload.perform/2
+        (pleroma 2.10.0) lib/pleroma/uploaders/s3.ex:60: Pleroma.Uploaders.S3.put_file/1
+        (pleroma 2.10.0) lib/pleroma/uploaders/uploader.ex:49: Pleroma.Uploaders.Uploader.put_file/2
+```
+
+So, use [your best tool](https://garagehq.deuxfleurs.fr/documentation/connect/cli/) to sync `/var/lib/pleroma/uploads/` in your S3.
+
+Then, to avoid some non existant problem (just in case of), run this command
+
+```bash
+while true
+do
+    rm -vr $(./bin/pleroma_ctl uploads migrate_local S3 2>&1 | grep "could not stream" | awk -F '"' '{print $2}')
+    sleep 5
+done
+```
+
+If you have many files, stop this command sometime and the command bellow (interactive) to delete local
+file after upload. Then restart the loop.
+
+```bash
+./bin/pleroma_ctl uploads migrate_local S3 --delete
+```
+
+And *voilà*
 
 ## Lemmy
 

doc/book/connect/backup.md

@@ -161,3 +161,59 @@ kopia repository validate-provider
 
 You can then run all the standard kopia commands: `kopia snapshot create`, `kopia mount`...
 Everything should work out-of-the-box.
+
+## Plakar
+
+Create your key and bucket on Garage server:
+
+```bash
+garage key create my-plakar-key
+garage bucket create plakar-backups
+garage bucket allow plakar-backups --read --write --key my-plakar-key
+…
+```
+
+On Plakar server, add your Garage as a storage location:
+```bash
+plakar store add garageS3 s3://my-garage.tld/plakar-backups \
+region=garage # Or as you've specified in garage.toml \
+access_key=<Key ID from "garage key info my-plakar-key"> \
+secret_access_key=<Secret key from "garage key info my-plakar-key">
+```
+
+Then create the repository.
+```bash
+plakar at @garageS3 create -plaintext # Unencrypted
+# or
+plakar at @garageS3 create #encrypted
+```
+
+If you encrypt your backups (Plakar default), you will need to define a strong passphrase. Do not forget to save your password safely. It will be needed to decrypt your backups.
+
+
+After the repository has been created, check that everything works as expected (that might give an empty result as no file has been added yet, but no error message):
+```bash
+plakar at @garageS3 check
+```
+
+Now that everything is configure, you can use Garage as your backups storage. For instance sync it with a local backup storage:
+```bash
+$ plakar at ~/backups sync to @garageS3
+```
+
+Or list the S3 storage content:
+```bash
+$ plakar at @garageS3 ls
+```
+
+More information in Plakar documentation: https://www.plakar.io/docs/main/quickstart/
+
+## Synology HyperBackup
+
+HyperBackup can be configured to upload backups to garage using a custom S3 destination. However, the HyperBackup client hardcodes the `us-east-1` region that is a critical input to the v4 signature process. If garage is not set to `us-east-1`, HyperBackup will recognize available buckets, but fail during the final setup stage.
+
+In garage.toml:
+```toml
+[s3_api]
+s3_region = "us-east-1"
+```

doc/book/connect/cli.md

@@ -149,6 +149,15 @@ rclone help
 This will tremendously accelerate operations such as `rclone sync` or `rclone ncdu` by reducing the number
 of ListObjects calls that are made.
 
+**Garage behind Cloudflare proxy:** when running Garage behind Cloudflare proxy, you might see `Response: error 403 Forbidden, Forbidden: Invalid signature` error in your garage logs or `AccessDenied: Forbidden: Invalid signature` error in rclone logs. Try adding `--s3-sign-accept-encoding=false` flag to your rclone command and see if the issue is resolved.
+
+```bash
+# this throws an error
+rclone lsd garage:
+
+# this should work
+rclone lsd --s3-sign-accept-encoding=false garage:
+```
 
 ## `s3cmd`
 
@@ -314,4 +323,3 @@ ls
 ```
 
 And through the web interface at http://[::1]:8080/web/client
-

doc/book/cookbook/ansible.md

@@ -8,7 +8,7 @@ have published Ansible roles.  We list them and compare them below.
 
 ## Comparison of Ansible roles
 
-| Feature                            | [ansible-role-garage](#zorun-ansible-role-garage) | [garage-docker-ansible-deploy](#moan0s-garage-docker-ansible-deploy) | [eddster ansible-role-garage](#eddster-ansible-role-garage) |
+| Feature                            | [ansible-role-garage](#zorun-ansible-role-garage) | [garage-docker-ansible-deploy](#moan0s-garage-docker-ansible-deploy) | [eddster2309 ansible-role-garage](#eddster2309-ansible-role-garage) |
 |------------------------------------|---------------------------------------------|---------------------------------------------------------------|---------------------------------|
 | **Runtime**                        | Systemd                                     | Docker                                                        | Systemd                         |
 | **Target OS**                      | Any Linux                                   | Any Linux                                                     | Any Linux                       |

doc/book/cookbook/binary-packages.md

@@ -15,9 +15,10 @@ Alpine Linux repositories (available since v3.17):
 apk add garage
 ```
 
-The default configuration file is installed to `/etc/garage.toml`. You can run
-Garage using: `rc-service garage start`. If you don't specify `rpc_secret`, it
-will be automatically replaced with a random string on the first start.
+The default configuration file is installed to `/etc/garage/garage.toml`. You can run
+Garage using: `rc-service garage start`.
+
+If you don't specify `rpc_secret`, it will be automatically replaced with a random string on the first start.
 
 Please note that this package is built without Consul discovery, Kubernetes
 discovery, OpenTelemetry exporter, and K2V features (K2V will be enabled once
@@ -26,7 +27,11 @@ it's stable).
 
 ## Arch Linux
 
-Garage is available in the [AUR](https://aur.archlinux.org/packages/garage).
+Garage is available in the official repositories under [extra](https://archlinux.org/packages/extra/x86_64/garage).
+
+```bash
+pacman -S garage
+```
 
 ## FreeBSD
 
@@ -39,3 +44,9 @@ pkg install garage
 ```bash
 nix-shell -p garage
 ```
+
+## conda-forge
+
+```bash
+pixi global install garage
+```

doc/book/cookbook/from-source.md

@@ -20,10 +20,10 @@ sudo apt-get update
 sudo apt-get install build-essential
 ```
 
-## Building from source from the Gitea repository
+## Building from source from the Forgejo repository
 
 The primary location for Garage's source code is the
-[Gitea repository](https://git.deuxfleurs.fr/Deuxfleurs/garage),
+[Forgejo repository](https://git.deuxfleurs.fr/Deuxfleurs/garage),
 which contains all of the released versions as well as the code
 for the developpement of the next version.
 
@@ -85,11 +85,14 @@ The following feature flags are available in v0.8.0:
 | Feature flag | Enabled | Description |
 | ------------ | ------- | ----------- |
 | `bundled-libs` | *by default* | Use bundled version of sqlite3, zstd, lmdb and libsodium |
-| `system-libs` | optional | Use system version of sqlite3, zstd, lmdb and libsodium<br>if available (exclusive with `bundled-libs`, build using<br>`cargo build --no-default-features --features system-libs`) |
+| `consul-discovery` | optional | Enable automatic registration and discovery<br>of cluster nodes through the Consul API |
+| `fjall` | experimental | Enable using Fjall to store Garage's metadata |
+| `journald` | optional | Enable logging to systemd-journald with<br>`GARAGE_LOG_TO_JOURNALD=true` environment variable set |
 | `k2v` | optional | Enable the experimental K2V API (if used, all nodes on your<br>Garage cluster must have it enabled as well) |
 | `kubernetes-discovery` | optional | Enable automatic registration and discovery<br>of cluster nodes through the Kubernetes API |
-| `metrics` | *by default* | Enable collection of metrics in Prometheus format on the admin API |
-| `telemetry-otlp` | optional | Enable collection of execution traces using OpenTelemetry |
-| `syslog` | optional | Enable logging to Syslog |
 | `lmdb` | *by default* | Enable using LMDB to store Garage's metadata |
+| `metrics` | *by default* | Enable collection of metrics in Prometheus format on the admin API |
 | `sqlite` | *by default* | Enable using Sqlite3 to store Garage's metadata |
+| `syslog` | optional | Enable logging to Syslog with<br>`GARAGE_LOG_TO_SYSLOG=true` environment variable set |
+| `system-libs` | optional | Use system version of sqlite3, zstd, lmdb and libsodium<br>if available (exclusive with `bundled-libs`, build using<br>`cargo build --no-default-features --features system-libs`) |
+| `telemetry-otlp` | optional | Enable collection of execution traces using OpenTelemetry |

doc/book/cookbook/kubernetes.md

@@ -11,7 +11,7 @@ Firstly clone the repository:
 
 ```bash
 git clone https://git.deuxfleurs.fr/Deuxfleurs/garage
-cd garage/scripts/helm
+cd garage/script/helm
 ```
 
 Deploy with default options:

doc/book/cookbook/real-world.md

@@ -96,14 +96,14 @@ to store 2 TB of data in total.
 ## Get a Docker image
 
 Our docker image is currently named `dxflrs/garage` and is stored on the [Docker Hub](https://hub.docker.com/r/dxflrs/garage/tags?page=1&ordering=last_updated).
-We encourage you to use a fixed tag (eg. `v2.0.0`) and not the `latest` tag.
-For this example, we will use the latest published version at the time of the writing which is `v2.0.0` but it's up to you
+We encourage you to use a fixed tag (eg. `v2.2.0`) and not the `latest` tag.
+For this example, we will use the latest published version at the time of the writing which is `v2.2.0` but it's up to you
 to check [the most recent versions on the Docker Hub](https://hub.docker.com/r/dxflrs/garage/tags?page=1&ordering=last_updated).
 
 For example:
 
 ```
-sudo docker pull dxflrs/garage:v2.0.0
+sudo docker pull dxflrs/garage:v2.2.0
 ```
 
 ## Deploying and configuring Garage
@@ -171,7 +171,7 @@ docker run \
   -v /etc/garage.toml:/etc/garage.toml \
   -v /var/lib/garage/meta:/var/lib/garage/meta \
   -v /var/lib/garage/data:/var/lib/garage/data \
-  dxflrs/garage:v2.0.0
+  dxflrs/garage:v2.2.0
 ```
 
 With this command line, Garage should be started automatically at each boot.
@@ -185,7 +185,7 @@ If you want to use `docker-compose`, you may use the following `docker-compose.y
 version: "3"
 services:
   garage:
-    image: dxflrs/garage:v2.0.0
+    image: dxflrs/garage:v2.2.0
     network_mode: "host"
     restart: unless-stopped
     volumes:

doc/book/cookbook/reverse-proxy.md

@@ -7,7 +7,7 @@ The main reason to add a reverse proxy in front of Garage is to provide TLS to y
 
 In production you will likely need your certificates signed by a certificate authority.
 The most automated way is to use a provider supporting the [ACME protocol](https://datatracker.ietf.org/doc/html/rfc8555) 
-such as [Let's Encrypt](https://letsencrypt.org/), [ZeroSSL](https://zerossl.com/) or [Buypass Go SSL](https://www.buypass.com/ssl/products/acme).
+such as [Let's Encrypt](https://letsencrypt.org/) or [ZeroSSL](https://zerossl.com/).
 
 If you are only testing Garage, you can generate a self-signed certificate to follow the documentation:
 
@@ -97,7 +97,7 @@ server {
   location / {
     proxy_pass http://s3_backend;
     proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
-    proxy_set_header Host $host;
+    proxy_set_header Host $http_host;
     # Disable buffering to a temporary file.
     proxy_max_temp_file_size 0;
   }

doc/book/design/goals.md

@@ -59,11 +59,13 @@ Garage themselves for the following tasks:
 
 - Hosting of their homepage, [privacyguides.org](https://www.privacyguides.org/), and various other static sites
 
-- As a Mastodon object storage backend for [mstdn.party](https://mstdn.party/) and [mstdn.plus](https://mstdn.plus/)
+- As a PowerDNS authoritative zone backend through [Lightning Stream](https://doc.powerdns.com/lightningstream/latest/index.html) and [LMDB](https://doc.powerdns.com/authoritative/backends/lmdb.html)
+
+- As a Mastodon media storage backend for [mstdn.party](https://mstdn.party/) and [mstdn.plus](https://mstdn.plus/)
 
 - As a PeerTube storage backend for [neat.tube](https://neat.tube/)
 
 - As a [Matrix media backend](https://github.com/matrix-org/synapse-s3-storage-provider)
 
 Triplebit's Garage cluster is a multi-site cluster currently composed of
-10 nodes in 3 physical locations.
+15 storage nodes in 3 physical locations.

doc/book/operations/durability-repairs.md

@@ -42,7 +42,7 @@ You may pause an ongoing scrub using `garage repair scrub pause`, but note that
 the scrub will resume automatically 24 hours later as Garage will not let your
 cluster run without a regular scrub. If the scrub procedure is too intensive
 for your servers and is slowing down your workload, the recommended solution
-is to increase the "scrub tranquility" using `garage repair scrub set-tranquility`.
+is to increase the "scrub tranquility" using `garage worker set scrub-tranquility`.
 A higher tranquility value will make Garage take longer pauses between two block
 verifications. Of course, scrubbing the entire data store will also take longer.
 

doc/book/operations/recovering.md

@@ -161,4 +161,7 @@ your recovery options are as follows:
 
 - **Option 3: restoring a filesystem-level snapshot.** If you are using ZFS or
   BTRFS to snapshot your metadata partition, refer to their specific
-  documentation on rolling back or copying files from an old snapshot.
+  documentation on rolling back or copying files from an old snapshot. 
+  Note that, depending on the properties of the filesystem and of the DB engine,
+  if these snapshots were taken during a write operation to the database, they may
+  also be corrupted and thus unfit for recovery.

doc/book/quick-start/_index.md

@@ -132,7 +132,7 @@ docker run \
   -v /path/to/garage.toml:/etc/garage.toml \
   -v /path/to/garage/meta:/var/lib/garage/meta \
   -v /path/to/garage/data:/var/lib/garage/data \
-  dxflrs/garage:v2.0.0
+  dxflrs/garage:v2.2.0
 ```
 
 Under Linux, you can substitute `--network host` for `-p 3900:3900 -p 3901:3901 -p 3902:3902 -p 3903:3903`

doc/book/reference-manual/admin-api.md

@@ -6,41 +6,167 @@ weight = 40
 The Garage administration API is accessible through a dedicated server whose
 listen address is specified in the `[admin]` section of the configuration
 file (see [configuration file
-reference](@/documentation/reference-manual/configuration.md))
+reference](@/documentation/reference-manual/configuration.md)).
 
-**WARNING.** At this point, there is no commitment to the stability of the APIs described in this document.
-We will bump the version numbers prefixed to each API endpoint each time the syntax
-or semantics change, meaning that code that relies on these endpoint will break
-when changes are introduced.
-
-Versions:
- - Before Garage 0.7.2 - no admin API
- - Garage 0.7.2 - admin APIv0
- - Garage 0.9.0 - admin APIv1, deprecate admin APIv0
+The current version of the admin API is v2.  No breaking changes to the Garage
+administration API will be published outside of a major release.
 
+History of previous versions:
 
+ - Before Garage v0.7.2 - no admin API
+ - Garage v0.7.2 - admin API v0
+ - Garage v0.9.0 - admin API v1, deprecate admin API v0
+ - Garage v2.0.0 - admin API v2, deprecate admin API v1
 
 ## Access control
 
-The admin API uses two different tokens for access control, that are specified in the config file's `[admin]` section:
+### Using an API token
 
-- `metrics_token`: the token for accessing the Metrics endpoint (if this token
-  is not set in the config file, the Metrics endpoint can be accessed without
-  access control);
-
-- `admin_token`: the token for accessing all of the other administration
-  endpoints (if this token is not set in the config file, access to these
-  endpoints is disabled entirely).
-
-These tokens are used as simple HTTP bearer tokens. In other words, to
-authenticate access to an admin API endpoint, add the following HTTP header
-to your request:
+Administration API tokens tokens are used as simple HTTP bearer tokens. In
+other words, to authenticate access to an admin API endpoint, add the following
+HTTP header to your request:
 
 ```
 Authorization: Bearer <token>
 ```
 
-## Administration API endpoints
+### User-defined API tokens
+
+Cluster administrators may dynamically define administration tokens using the CLI commands under `garage admin-token`.
+Such tokens may be limited in scope, meaning that they may enable access to only a subset of API calls.
+They may also have an expiration date to limit their use in time.
+
+Here is an example to create an administration token that is valid for 30 days
+and gives access to only a subset of API calls, allowing it to create buckets
+and access keys and give keys permissions on buckets:
+
+```bash
+$ garage admin-token create --expires-in 30d \
+    --scope ListBuckets,GetBucketInfo,ListKeys,GetKeyInfo,CreateBucket,CreateKey,AllowBucketKey,DenyBucketKey \
+    my-token
+This is your secret bearer token, it will not be shown again by Garage:
+
+  8ed1830b10a276ff57061950.kOSIpxWK9zSGbTO9Xadpv3YndSFWma0_snXcYHaORXk
+
+==== ADMINISTRATION TOKEN INFORMATION ====
+Token ID:    8ed1830b10a276ff57061950
+Token name:  my-token
+Created:     2025-06-15 15:12:44.160 +02:00
+Validity:    valid
+Expiration:  2025-07-15 15:12:44.117 +02:00
+
+Scope:       ListBuckets
+             GetBucketInfo
+             ListKeys
+             GetKeyInfo
+             CreateBucket
+             CreateKey
+             AllowBucketKey
+             DenyBucketKey
+```
+
+When running this command, your token will be shown only once and **will never
+be shown again by Garage**, so make sure to save it directly.  The token is
+hashed internally, and is identified by its prefix (32 hex digits followed by a
+dot) which is saved in clear.
+
+When running `garage admin-token list`, you might see something like this:
+
+```
+ID                        Created     Name                                       Expiration                      Scope
+-                         -           metrics_token (from daemon configuration)  never                           Metrics
+8ed1830b10a276ff57061950  2025-06-15  my-token                                   2025-07-15 15:12:44.117 +02:00  ListBuckets, ... (8)
+```
+
+### Master API tokens
+
+The admin API can also use two different master tokens for access control,
+specified in the config file's `[admin]` section:
+
+- `metrics_token`: the token for accessing the Metrics endpoint. If this token
+  is not set in the config file, the Metrics endpoint can be accessed without
+  access control.
+
+- `admin_token`: the token for accessing all of the other administration
+  endpoints. If this token is not set in the config file, access to these
+  endpoints is only possible with a user-defined admin token.
+
+With the introduction of multiple user-defined admin tokens, the use of master
+API tokens is now discouraged.
+
+
+## Using the admin API
+
+All of the admin API endpoints are described in the OpenAPI specification:
+
+ - APIv2 - [HTML spec](https://garagehq.deuxfleurs.fr/api/garage-admin-v2.html) - [OpenAPI JSON](https://garagehq.deuxfleurs.fr/api/garage-admin-v2.json)
+ - APIv1 (deprecated) - [HTML spec](https://garagehq.deuxfleurs.fr/api/garage-admin-v1.html) - [OpenAPI YAML](https://garagehq.deuxfleurs.fr/api/garage-admin-v1.yml)
+ - APIv0 (deprecated) - [HTML spec](https://garagehq.deuxfleurs.fr/api/garage-admin-v0.html) - [OpenAPI YAML](https://garagehq.deuxfleurs.fr/api/garage-admin-v0.yml)
+
+Making a request to the API from the command line can be as simple as running:
+
+```bash
+curl -H 'Authorization: Bearer s3cr3t' http://localhost:3903/v2/GetClusterStatus | jq
+```
+
+For more advanced use cases, we recommend using an SDK.  
+[Go to the "Build your own app" section to know how to use our SDKs](@/documentation/build/_index.md)
+
+### Making API calls from the `garage` CLI
+
+Since v2.0.0, the `garage` binary provides a subcommand `garage json-api` that
+allows you to invoke the API without making an HTTP request.  This can be
+useful for scripting Garage deployments.
+
+`garage json-api` proxies API calls through Garage's internal RPC protocol,
+therefore it does not require any form of authentication: RPC connection
+parameters are discovered automatically to contact the locally-running Garage
+instance (as when running any other `garage` CLI command).
+
+For simple calls that take no parameters, usage is as follows:
+
+```
+$ garage json-api GetClusterHealth
+{
+  "connectedNodes": 3,
+  "knownNodes": 3,
+  "partitions": 256,
+  "partitionsAllOk": 256,
+  "partitionsQuorum": 256,
+  "status": "healthy",
+  "storageNodes": 3,
+  "storageNodesOk": 3
+}
+```
+
+If you need to specify a JSON body for your call, you can add it directly after
+the name of the function you are calling:
+
+```
+$ garage json-api CreateAdminToken '{"name": "test"}'
+```
+
+Or you can feed it through stdin by adding a `-` as the last command parameter:
+
+```
+$ garage json-api CreateAdminToken -
+{"name": "test"}
+<EOF>
+```
+
+For admin API calls that would have taken query parameters in their HTTP version, these parameters can be passed in the JSON body object:
+
+```
+$ garage json-api GetAdminTokenInfo '{"id":"b0e6e0ace2c0b2aca4cdb2de"}'
+```
+
+For admin API calls that take both query parameters and a JSON body, combine them in the following fashion:
+
+```
+$ garage json-api UpdateAdminToken '{"id":"b0e6e0ace2c0b2aca4cdb2de", "body":{"name":"not a test"}}'
+```
+
+## Special administration API endpoints
 
 ### Metrics `GET /metrics`
 
@@ -83,7 +209,7 @@ content-length: 102
 date: Tue, 08 Aug 2023 07:22:38 GMT
 
 Garage is fully operational
-Consult the full health check API endpoint at /v0/health for more details
+Consult the full health check API endpoint at /v2/GetClusterHealth for more details
 ```
 
 ### On-demand TLS `GET /check`
@@ -126,23 +252,7 @@ $ curl -so /dev/null -w "%{http_code}" http://localhost:3903/check?domain=exampl
 200
 ```
 
-
 **References:**
  - [Using On-Demand TLS](https://caddyserver.com/docs/automatic-https#using-on-demand-tls)
  - [Add option for a backend check to approve use of on-demand TLS](https://github.com/caddyserver/caddy/pull/1939)
  - [Serving tens of thousands of domains over HTTPS with Caddy](https://caddy.community/t/serving-tens-of-thousands-of-domains-over-https-with-caddy/11179)
-
-### Cluster operations
-
-These endpoints have a dedicated OpenAPI spec.
- - APIv1 - [HTML spec](https://garagehq.deuxfleurs.fr/api/garage-admin-v1.html) - [OpenAPI YAML](https://garagehq.deuxfleurs.fr/api/garage-admin-v1.yml)
- - APIv0 (deprecated) - [HTML spec](https://garagehq.deuxfleurs.fr/api/garage-admin-v0.html) - [OpenAPI YAML](https://garagehq.deuxfleurs.fr/api/garage-admin-v0.yml)
-
-Requesting the API from the command line can be as simple as running:
-
-```bash
-curl -H 'Authorization: Bearer s3cr3t' http://localhost:3903/v0/status | jq
-```
-
-For more advanced use cases, we recommend using a SDK.  
-[Go to the "Build your own app" section to know how to use our SDKs](@/documentation/build/_index.md)

doc/book/reference-manual/configuration.md

@@ -24,7 +24,8 @@ db_engine = "lmdb"
 
 block_size = "1M"
 block_ram_buffer_max = "256MiB"
-
+block_max_concurrent_reads = 16
+block_max_concurrent_writes_per_request =10
 lmdb_map_size = "1T"
 
 compression_level = 1
@@ -50,17 +51,20 @@ allow_punycode = false
 
 [consul_discovery]
 api = "catalog"
-consul_http_addr = "http://127.0.0.1:8500"
+consul_http_addr = "https://127.0.0.1:8500"
+tls_skip_verify = false
 service_name = "garage-daemon"
+
 ca_cert = "/etc/consul/consul-ca.crt"
 client_cert = "/etc/consul/consul-client.crt"
 client_key = "/etc/consul/consul-key.crt"
+
 # for `agent` API mode, unset client_cert and client_key, and optionally enable `token`
 # token = "abcdef-01234-56789"
-tls_skip_verify = false
+
 tags = [ "dns-enabled" ]
 meta = { dns-acl = "allow trusted" }
-
+datacenters = ["dc1", "dc2", "dc3"]
 
 [kubernetes_discovery]
 namespace = "garage"
@@ -97,6 +101,8 @@ The following gives details about each available configuration option.
 Top-level configuration options, in alphabetical order:
 [`allow_punycode`](#allow_punycode),
 [`allow_world_readable_secrets`](#allow_world_readable_secrets),
+[`block_max_concurrent_reads`](#block_max_concurrent_reads),
+[`block_max_concurrent_writes_per_request`](#block_max_concurrent_writes_per_request),
 [`block_ram_buffer_max`](#block_ram_buffer_max),
 [`block_size`](#block_size),
 [`bootstrap_peers`](#bootstrap_peers),
@@ -125,12 +131,14 @@ The `[consul_discovery]` section:
 [`client_cert`](#consul_client_cert_and_key),
 [`client_key`](#consul_client_cert_and_key),
 [`consul_http_addr`](#consul_http_addr),
+[`datacenters`](#consul_datacenters)
 [`meta`](#consul_tags_and_meta),
 [`service_name`](#consul_service_name),
 [`tags`](#consul_tags_and_meta),
 [`tls_skip_verify`](#consul_tls_skip_verify),
 [`token`](#consul_token).
 
+
 The `[kubernetes_discovery]` section:
 [`namespace`](#kube_namespace),
 [`service_name`](#kube_service_name),
@@ -335,6 +343,7 @@ Since `v0.8.0`, Garage can use alternative storage backends as follows:
 | --------- | ----------------- | ------------- |
 | [LMDB](https://www.symas.com/lmdb) (since `v0.8.0`, default since `v0.9.0`) | `"lmdb"` | `<metadata_dir>/db.lmdb/` |
 | [Sqlite](https://sqlite.org) (since `v0.8.0`) | `"sqlite"` | `<metadata_dir>/db.sqlite` |
+| [Fjall](https://github.com/fjall-rs/fjall) (**experimental support** since `v1.3.0`/`v2.1.0`) | `"fjall"` | `<metadata_dir>/db.fjall/` |
 | [Sled](https://sled.rs) (old default, removed since `v1.0`) | `"sled"` | `<metadata_dir>/db/` |
 
 Sled was supported until Garage v0.9.x, and was removed in Garage v1.0.
@@ -343,8 +352,16 @@ old Sled metadata databases to another engine.
 
 Performance characteristics of the different DB engines are as follows:
 
-- LMDB: the recommended database engine for high-performance distributed clusters.
-LMDB works very well, but is known to have the following limitations:
+- **LMDB:** the recommended database engine for high-performance distributed clusters
+  with `replication_factor` ≥ 2.
+  LMDB works well, but is known to have the following limitations:
+
+  - LMDB is prone to database corruption after an unclean shutdown (e.g. a process kill
+    or a power outage).  It is recommended to configure
+    [`metadata_auto_snapshot_interval`](#metadata_auto_snapshot_interval) to be
+    able to easily recover from this situation. With `replication_factor` ≥ 2,
+    metadata can also be reconstructed from remote nodes upon corruption
+    (see [Recovering from failures](@/documentation/operations/recovering.md#corrupted_meta)).
 
   - The data format of LMDB is not portable between architectures, so for
     instance the Garage database of an x86-64 node cannot be moved to an ARM64
@@ -354,22 +371,21 @@ LMDB works very well, but is known to have the following limitations:
     node to very small database sizes due to how LMDB works; it is therefore
     not recommended.
 
-  - Several users have reported corrupted LMDB database files after an unclean
-    shutdown (e.g. a power outage). This situation can generally be recovered
-    from if your cluster is geo-replicated (by rebuilding your metadata db from
-    other nodes), or if you have saved regular snapshots at the filesystem
-    level.
-
   - Keys in LMDB are limited to 511 bytes. This limit translates to limits on
     object keys in S3 and sort keys in K2V that are limted to 479 bytes.
 
-- Sqlite: Garage supports Sqlite as an alternative storage backend for
-  metadata, which does not have the issues listed above for LMDB.
-  On versions 0.8.x and earlier, Sqlite should be avoided due to abysmal
-  performance, which was fixed with the addition of `metadata_fsync`.
-  Sqlite is still probably slower than LMDB due to the way we use it,
-  so it is not the best choice for high-performance storage clusters,
-  but it should work fine in many cases.
+- **Sqlite:** Garage supports Sqlite as an alternative storage backend for
+  metadata, which does not have the issues listed above for LMDB.  Sqlite is
+  slower than LMDB, so it is not the best choice for high-performance storage
+  clusters.
+
+- **Fjall:** a storage engine based on LSM trees, which theoretically allow for
+  higher write throughput than other storage engines that are based on B-trees.
+  Using Fjall could potentially improve Garage's performance significantly in
+  write-heavy workloads. **Support for Fjall is experimental at this point**,
+  we have added it to Garage for evaluation purposes only. **Use it only with
+  test data, and report any issues to our bug tracker. Do not use it for
+  production workloads.**
 
 It is possible to convert Garage's metadata directory from one format to another
 using the `garage convert-db` command, which should be used as follows:
@@ -408,6 +424,7 @@ Here is how this option impacts the different database engines:
 |----------|------------------------------------|-------------------------------|
 | Sqlite   | `PRAGMA synchronous = OFF`         | `PRAGMA synchronous = NORMAL` |
 | LMDB     | `MDB_NOMETASYNC` + `MDB_NOSYNC`    | `MDB_NOMETASYNC`              |
+| Fjall    | default options                    | not supported                 |
 
 Note that the Sqlite database is always ran in `WAL` mode (`PRAGMA journal_mode = WAL`).
 
@@ -427,7 +444,8 @@ if geographical replication is used.
 #### `metadata_auto_snapshot_interval` (since `v0.9.4`) {#metadata_auto_snapshot_interval}
 
 If this value is set, Garage will automatically take a snapshot of the metadata
-DB file at a regular interval and save it in the metadata directory.
+DB file at a regular interval and save it in the metadata directory,
+or in [`metadata_snapshots_dir`](#metadata_snapshots_dir) if it is set.
 This parameter can take any duration string that can be parsed by
 the [`parse_duration`](https://docs.rs/parse_duration/latest/parse_duration/#syntax) crate.
 
@@ -436,14 +454,19 @@ corrupted, for instance after an unclean shutdown.  See [this
 page](@/documentation/operations/recovering.md#corrupted_meta) for details.
 Garage keeps only the two most recent snapshots of the metadata DB and deletes
 older ones automatically.
+You can also create metadata snapshots manually at any point using the
+`garage meta snapshot` command.
+
+Using snapshots created by Garage is the best option to make snapshots of your
+node's metadata for potential recovery, as they are guaranteed to be clean and
+consistent, contrarily to filesystem-level snapshots that may be taken while
+some writes are in-flight and thus might be corrupted.
 
 Note that taking a metadata snapshot is a relatively intensive operation as the
 entire data file is copied. A snapshot being taken might have performance
 impacts on the Garage node while it is running. If the cluster is under heavy
 write load when a snapshot operation is running, this might also cause the
 database file to grow in size significantly as pages cannot be recycled easily.
-For this reason, it might be better to use filesystem-level snapshots instead
-if possible.
 
 #### `disable_scrub` {#disable_scrub}
 
@@ -514,6 +537,37 @@ node.
 
 The default value is 256MiB.
 
+#### `block_max_concurrent_reads` (since `v1.3.0` / `v2.1.0`) {#block_max_concurrent_reads}
+
+The maximum number of blocks (individual files in the data directory) open
+simultaneously for reading.
+
+Reducing this number does not limit the number of data blocks that can be
+transferred through the network simultaneously. This mechanism was just added
+as a backpressure mechanism for HDD read speed: it helps avoid a situation
+where too many requests are coming in and Garage is reading too many block
+files simultaneously, thus not making timely progress on any of the reads.
+
+When a request to read a data block comes in through the network, the requests
+awaits for one of the `block_max_concurrent_reads` slots to be available
+(internally implemented using a Semaphore object). Once it acquired a read
+slot, it reads the entire block file to RAM and frees the slot as soon as the
+block file is finished reading. Only after the slot is released will the
+block's data start being transferred over the network.  If the request fails to
+acquire a reading slot wihtin 15 seconds, it fails with a timeout error.
+Timeout events can be monitored through the `block_read_semaphore_timeouts`
+metric in Prometheus: a non-zero number of such events indicates an I/O
+bottleneck on HDD read speed.
+
+
+#### `block_max_concurrent_writes_per_request` (since `v1.3.1` / `v2.2.0`) {#block_max_concurrent_writes_per_request}
+
+This parameter is designed to adapt to the concurrent write performance of 
+different storage media. Maximum number of parallel block writes per put request.
+Higher values may improve throughput but increase memory usage.
+
+Default value: 3. Recommended values: 10-30 for NVMe, 3-10 for spinning HDD.
+
 #### `lmdb_map_size` {#lmdb_map_size}
 
 This parameters can be used to set the map size used by LMDB,
@@ -686,6 +740,18 @@ node_prefix "" {
 }
 ```
 
+
+#### `datacenters` {#consul_datacenters}
+
+Optional list of datacenters that allow garage to do service discovery when Consul is configured in WAN federation.
+
+Example: `datacenters = ["dc1", "dc2", "dc3"]`
+
+In a WAN configuration, by default the Consul services API only responds with
+local LAN services.  When a list of datacenters is specified using this option,
+Garage will query the consul server API by datacenter directly, allowing for
+Garage to discover nodes across the Consul WAN.
+
 #### `tags` and `meta` {#consul_tags_and_meta}
 
 Additional list of tags and map of service meta to add during service registration.

doc/book/reference-manual/features.md

@@ -129,5 +129,5 @@ related to objects stored in an S3 bucket.
 In the context of our research project, [Aérogramme](https://aerogramme.deuxfleurs.fr),
 K2V is used to provide metadata and log storage for operations on encrypted e-mail storage.
 
-Learn more on the specification of K2V [here](https://git.deuxfleurs.fr/Deuxfleurs/garage/src/branch/k2v/doc/drafts/k2v-spec.md)
+Learn more on the specification of K2V [here](https://git.deuxfleurs.fr/Deuxfleurs/garage/src/commit/f8be15c37db857e177d543de7be863692628d567/doc/drafts/k2v-spec.md)
 and on how to enable it in Garage [here](@/documentation/reference-manual/k2v.md).

doc/book/reference-manual/k2v.md

@@ -16,7 +16,7 @@ the `k2v` feature flag enabled can be obtained from our download page under
 with `-k2v` (example: `v0.7.2-k2v`).
 
 The specification of the K2V API can be found
-[here](https://git.deuxfleurs.fr/Deuxfleurs/garage/src/branch/main/doc/drafts/k2v-spec.md).
+[here](https://git.deuxfleurs.fr/Deuxfleurs/garage/src/commit/f8be15c37db857e177d543de7be863692628d567/doc/drafts/k2v-spec.md).
 This document also includes a high-level overview of K2V's design.
 
 The K2V API uses AWSv4 signatures for authentification, same as the S3 API.

doc/book/reference-manual/s3-compatibility.md

@@ -27,7 +27,7 @@ Feel free to open a PR to suggest fixes this table. Minio is missing because the
 
 | Feature                      | Garage                           | [Openstack Swift](https://docs.openstack.org/swift/latest/s3_compat.html) | [Ceph Object Gateway](https://docs.ceph.com/en/latest/radosgw/s3/) | [Riak CS](https://docs.riak.com/riak/cs/2.1.1/references/apis/storage/s3/index.html) | [OpenIO](https://docs.openio.io/latest/source/arch-design/s3_compliancy.html) |
 |------------------------------|----------------------------------|-----------------|---------------|---------|-----|
-| [signature v2](https://docs.aws.amazon.com/general/latest/gr/signature-version-2.html) (deprecated) | ❌ Missing | ✅ |  ✅ | ✅ | ✅ |
+| [signature v2](https://docs.aws.amazon.com/AmazonS3/latest/API/Appendix-Sigv2.html) (deprecated) | ❌ Missing | ✅ |  ✅ | ✅ | ✅ |
 | [signature v4](https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-authenticating-requests.html) |  ✅ Implemented |  ✅ |  ✅ | ❌ | ✅ |
 | [URL path-style](https://docs.aws.amazon.com/AmazonS3/latest/userguide/VirtualHosting.html#path-style-access) (eg. `host.tld/bucket/key`) |  ✅ Implemented | ✅ |  ✅ | ❓| ✅ |
 | [URL vhost-style](https://docs.aws.amazon.com/AmazonS3/latest/userguide/VirtualHosting.html#virtual-hosted-style-access) URL (eg. `bucket.host.tld/key`) |  ✅ Implemented | ❌| ✅| ✅ | ✅ |

doc/book/working-documents/migration-04.md

@@ -1,6 +1,6 @@
 +++
 title = "Migrating from 0.3 to 0.4"
-weight = 20
+weight = 80
 +++
 
 **Migrating from 0.3 to 0.4 is unsupported. This document is only intended to

doc/book/working-documents/migration-06.md

@@ -1,6 +1,6 @@
 +++
 title = "Migrating from 0.5 to 0.6"
-weight = 15
+weight = 75
 +++
 
 **This guide explains how to migrate to 0.6 if you have an existing 0.5 cluster.

doc/book/working-documents/migration-07.md

@@ -1,6 +1,6 @@
 +++
 title = "Migrating from 0.6 to 0.7"
-weight = 14
+weight = 74
 +++
 **This guide explains how to migrate to 0.7 if you have an existing 0.6 cluster.
 We don't recommend trying to migrate to 0.7 directly from 0.5 or older.**

doc/book/working-documents/migration-08.md

@@ -1,6 +1,6 @@
 +++
 title = "Migrating from 0.7 to 0.8"
-weight = 13
+weight = 73
 +++
 
 **This guide explains how to migrate to 0.8 if you have an existing 0.7 cluster.

doc/book/working-documents/migration-09.md

@@ -1,6 +1,6 @@
 +++
 title = "Migrating from 0.8 to 0.9"
-weight = 12
+weight = 72
 +++
 
 **This guide explains how to migrate to 0.9 if you have an existing 0.8 cluster.

doc/book/working-documents/migration-1.md

@@ -1,6 +1,6 @@
 +++
 title = "Migrating from 0.9 to 1.0"
-weight = 11
+weight = 71
 +++
 
 **This guide explains how to migrate to 1.0 if you have an existing 0.9 cluster.

doc/book/working-documents/migration-2.md

@@ -0,0 +1,70 @@
++++
+title = "Migrating from 1.0 to 2.0"
+weight = 70
++++
+
+**This guide explains how to migrate to v2.x if you have an existing v1.x.x cluster.
+We don't recommend trying to migrate to v2.x directly from v0.9.x or older.**
+
+This migration procedure has been tested on several clusters without issues.
+However, it is still a *critical procedure* that might cause issues.
+**Make sure to back up all your data before attempting it!**
+
+You might also want to read our [general documentation on upgrading Garage](@/documentation/operations/upgrading.md).
+
+## Changes introduced in v2.0
+
+The following are **breaking changes** in Garage v2.0 that require your attention when migrating:
+
+- The administration API has been completely reworked.
+  Some calls to the `/v1/` endpoints will still work but most will not.
+  New endpoints are prefixed by `/v2/`. **You will need to update all your code that makes use of the admin API.**
+
+- `replication_mode` is no longer a supported configuration parameter,
+  please use `replication_factor` and `consistency_mode` instead.
+
+## Migration procedure
+
+The migration to Garage v2.0 can be done with almost no downtime,
+by restarting all nodes at once in the new version.
+
+The migration steps are as follows:
+
+1. Do a `garage repair --all-nodes --yes tables`, check the logs and check that
+   all data seems to be synced correctly between nodes. If you have time, do
+   additional `garage repair` procedures (`blocks`, `versions`, `block_refs`,
+   etc.)
+
+2. Ensure you have a snapshot of your Garage installation that you can restore
+   to in case the upgrade goes wrong, with one of the following options:
+
+   - You may use the `garage meta snapshot --all` command
+     to make a backup snapshot of the metadata directories of your nodes
+     for backup purposes. Once this command has completed, copy the following
+     files and directories from the `metadata_dir` of all your nodes
+     to somewhere safe: `snapshots`, `cluster_layout`, `data_layout`,
+     `node_key`, `node_key.pub`. (If you have set the `metadata_snapshots_dir`
+     to a different value in your config file, back up that directory instead.)
+
+   - If you are running a filesystem such as ZFS or BTRFS that support
+     snapshotting, you can create a filesystem-level snapshot of the `metadata_dir`
+     of all your nodes to be used as a restoration point if needed.
+
+   - You may also make a back-up manually: turn off each node
+     individually; back up its metadata folder (for instance, use the following
+     command if your metadata directory is `/var/lib/garage/meta`: `cd
+     /var/lib/garage ; tar -acf meta-v1.0.tar.zst meta/`); turn it back on
+     again.  This will allow you to take a backup of all nodes without
+     impacting global cluster availability.  You can do all nodes of a single
+     zone at once as this does not impact the availability of Garage.
+
+3. Prepare your updated binaries and configuration files for Garage v2.0.
+   **Remember to update your configuration file to remove `replication_mode` and replace it by `replication_factor`.**
+
+4. Shut down all v1.0 nodes simultaneously, and restart them all simultaneously
+   in v2.0.  Use your favorite deployment tool (Ansible, Kubernetes, Nomad) to
+   achieve this as fast as possible.  Garage v2.0 should be in a working state
+   as soon as enough nodes have started.
+
+5. Monitor your cluster in the following hours to see if it works well under
+   your production load.

doc/book/working-documents/testing-strategy.md

@@ -1,6 +1,6 @@
 +++
 title = "Testing strategy"
-weight = 30
+weight = 100
 +++
 
 

doc/talks/2025-10-06-josy/.gitignore

@@ -0,0 +1,17 @@
+*
+
+!*.txt
+!*.md
+
+!assets
+
+!.gitignore
+!*.svg
+!*.png
+!*.jpg
+!*.tex
+!Makefile
+!.gitignore
+!assets/*.drawio.pdf
+
+!talk.pdf

doc/talks/2025-10-06-josy/Makefile

@@ -0,0 +1,19 @@
+ASSETS=../assets/lattice/lattice1.pdf_tex \
+	   ../assets/lattice/lattice2.pdf_tex \
+	   ../assets/lattice/lattice3.pdf_tex \
+	   ../assets/lattice/lattice4.pdf_tex \
+	   ../assets/lattice/lattice5.pdf_tex \
+	   ../assets/lattice/lattice6.pdf_tex \
+	   ../assets/lattice/lattice7.pdf_tex \
+	   ../assets/lattice/lattice8.pdf_tex \
+	   ../assets/logos/deuxfleurs.pdf \
+	   ../assets/timeline-22-24.pdf
+
+talk.pdf: talk.tex $(ASSETS)
+	pdflatex talk.tex
+
+%.pdf: %.svg
+	inkscape -D -z --file=$^ --export-pdf=$@
+
+%.pdf_tex: %.svg
+	inkscape -D -z --file=$^ --export-pdf=$@ --export-latex

doc/talks/2025-10-06-josy/talk.tex

@@ -0,0 +1,702 @@
+\nonstopmode
+\documentclass[aspectratio=169,xcolor={svgnames}]{beamer}
+\usepackage[utf8]{inputenc}
+% \usepackage[frenchb]{babel}
+\usepackage{amsmath}
+\usepackage{mathtools}
+\usepackage{breqn}
+\usepackage{multirow}
+\usetheme{boxes}
+\usepackage{graphicx}
+\usepackage{import}
+\usepackage{adjustbox}
+\usepackage[absolute,overlay]{textpos}
+%\useoutertheme[footline=authortitle,subsection=false]{miniframes}
+%\useoutertheme[footline=authorinstitute,subsection=false]{miniframes}
+\useoutertheme{infolines}
+\setbeamertemplate{headline}{}
+
+\beamertemplatenavigationsymbolsempty
+
+\definecolor{TitleOrange}{RGB}{255,137,0}
+\setbeamercolor{title}{fg=TitleOrange}
+\setbeamercolor{frametitle}{fg=TitleOrange}
+
+\definecolor{ListOrange}{RGB}{255,145,5}
+\setbeamertemplate{itemize item}{\color{ListOrange}$\blacktriangleright$}
+
+\definecolor{verygrey}{RGB}{70,70,70}
+\setbeamercolor{normal text}{fg=verygrey}
+
+
+\usepackage{tabu}
+\usepackage{multicol}
+\usepackage{vwcol}
+\usepackage{stmaryrd}
+\usepackage{graphicx}
+
+\usepackage[normalem]{ulem}
+
+\AtBeginSection[]{
+  \begin{frame}
+  \vfill
+  \centering
+  \begin{beamercolorbox}[sep=8pt,center,shadow=true,rounded=true]{title}
+    \usebeamerfont{title}\insertsectionhead\par%
+  \end{beamercolorbox}
+  \vfill
+  \end{frame}
+}
+
+\title{Garage, an S3 backend as reliable as possible}
+\author{Garage Authors}
+\date{JoSy S3, 2025-10-08}
+
+\begin{document}
+
+\begin{frame}
+	\centering
+	\includegraphics[width=.3\linewidth]{../../sticker/Garage.png}
+	\vspace{1em}
+
+	{\large\bf Garage, an S3 backend as reliable as possible}
+	\vspace{1em}
+
+		\url{https://garagehq.deuxfleurs.fr/}\\
+		\url{mailto:garagehq@deuxfleurs.fr}\\
+		\texttt{\#garage:deuxfleurs.fr} on Matrix
+\end{frame}
+
+
+\section{Meet Garage}
+
+\begin{frame}
+	\frametitle{A non-profit initiative}
+
+
+	\begin{columns}[t]
+		\begin{column}{.2\textwidth}
+			\centering
+			\adjincludegraphics[width=.5\linewidth, valign=t]{../assets/logos/deuxfleurs.pdf}
+		\end{column}
+		\begin{column}{.8\textwidth}
+			\textbf{Part of a degrowth initiative}\\
+			Garage has been created at Deuxfleurs where we experiment running Internet services without datacenter on commodity and refurbished hardware.
+		\end{column}
+
+	\end{columns}
+	\vspace{2em}
+	\begin{columns}[t]
+		\begin{column}{.2\textwidth}
+			\centering
+			\adjincludegraphics[width=.5\linewidth, valign=t]{../assets/community.png}
+		\end{column}
+		\begin{column}{.8\textwidth}
+			\textbf{Developed by a community}\\
+			{\small Some recent contributors: Arthur C, Charles H, dongdigua, Etienne L, Jonah A, Julien K, Lapineige, MagicRR, Milas B, Niklas M, RockWolf, Schwitzd, trinity-1686a, Xavier S,  babykart, Baptiste J, eddster2309, James O'C, Joker9944, Maximilien R, Renjaya RZ, Yureka...}
+		\end{column}
+
+	\end{columns}
+	\vspace{2em}
+	\begin{columns}[t]
+		\begin{column}{.2\textwidth}
+			\centering
+			\adjincludegraphics[width=.5\linewidth, valign=t]{../assets/logos/AGPLv3_Logo.png}
+		\end{column}
+		\begin{column}{.8\textwidth}
+			\textbf{Owned by nobody, open-core is impossible, zero VC money}\\
+			AGPL + no Contributor License Agreement = Garage ownership spreads among hundredth of contributors.
+		\end{column}
+
+	\end{columns}
+\end{frame}
+
+\begin{frame}
+	\frametitle{Getting support for Garage}
+	\begin{columns}[t]
+		\begin{column}{.2\textwidth}
+			\centering
+			\adjincludegraphics[width=.4\linewidth, valign=t]{../assets/alex.jpg}
+		\end{column}
+		\begin{column}{.4\textwidth}
+			\textbf{Alex Auvolat}\\
+			PhD; co-founder of Deuxfleurs\\
+			Garage maintainer, Freelance
+		\end{column}
+		\begin{column}{.3\textwidth}
+			\centering
+			\adjincludegraphics[width=.4\linewidth, valign=t]{../assets/support.png}
+		\end{column}
+		\begin{column}{.1\textwidth}
+			~
+		\end{column}
+	\end{columns}
+	\vspace{2em}
+	\begin{columns}[t]
+		\begin{column}{.2\textwidth}
+			\centering
+			\adjincludegraphics[width=.4\linewidth, valign=t]{../assets/quentin.jpg}
+		\end{column}
+		\begin{column}{.4\textwidth}
+			\textbf{Quentin Dufour}\\
+			PhD; co-founder of Deuxfleurs\\
+			Garage contributor, Freelance
+		\end{column}
+		\begin{column}{.4\textwidth}
+			For support requests, write at: \\
+			\url{garagehq@deuxfleurs.fr}
+		\end{column}
+	\end{columns}
+	\vspace{2em}
+	\begin{columns}[t]
+		\begin{column}{.2\textwidth}
+			\centering
+			\adjincludegraphics[width=.4\linewidth, valign=t]{../assets/armael.jpg}
+		\end{column}
+		\begin{column}{.4\textwidth}
+			\textbf{Armaël Guéneau}\\
+			PhD; member of Deuxfleurs\\
+			Garage contributor, Freelance
+		\end{column}
+		\begin{column}{.4\textwidth}
+			Eligible: email support, architecture design, specific feature development, etc.
+		\end{column}
+	\end{columns}
+
+
+\end{frame}
+
+\begin{frame}
+	\frametitle{Our initial goal}
+
+	\centering
+	\Large
+
+	Being a self-sovereign community to be free of our degrowth choice
+
+	$\big\downarrow$
+
+	As web citizens, datacenters are big black boxes. \\
+	We want to leave them to autonoumously manage our servers.
+
+	$\big\downarrow$
+
+	We want reliable services without relying on dedicated hardware or places.
+
+\end{frame}
+
+\begin{frame}
+	\frametitle{Building a resilient system with cheap stuff}
+
+	\only<1,4-7>{
+		\begin{itemize}
+			\item \textcolor<5->{gray}{Commodity hardware (e.g. old desktop PCs)\\
+				\vspace{.5em}
+				\visible<4->{{\footnotesize (can die at any time)}}}
+				\vspace{1.5em}
+			\item<5-> \textcolor<7->{gray}{Regular Internet (e.g. FTTB, FTTH) and power grid connections\\
+				\vspace{.5em}
+				\visible<6->{{\footnotesize (can be unavailable randomly)}}}
+				\vspace{1.5em}
+			\item<7-> \textbf{Geographical redundancy} (multi-site replication)
+		\end{itemize}
+	}
+	\only<2>{
+		\begin{center}
+			\includegraphics[width=.8\linewidth]{../assets/neptune.jpg}
+		\end{center}
+	}
+	\only<3>{
+		\begin{center}
+			\includegraphics[width=.8\linewidth]{../assets/atuin.jpg}
+		\end{center}
+	}
+	\only<8>{
+		\begin{center}
+			\includegraphics[width=.8\linewidth]{../assets/inframap_jdll2023.pdf}
+		\end{center}
+	}
+\end{frame}
+
+\begin{frame}
+	\frametitle{Object storage: a crucial component}
+		\begin{center}
+			\includegraphics[height=6em]{../assets/logos/Amazon-S3.jpg}
+			\hspace{3em}
+			\visible<2->{\includegraphics[height=5em]{../assets/logos/minio.png}}
+			\hspace{3em}
+			\visible<3>{\includegraphics[height=6em]{../../logo/garage_hires_crop.png}}
+		\end{center}
+		\vspace{1em}
+	S3: a de-facto standard, many compatible applications
+
+	\vspace{1em}
+	\visible<2->{MinIO is self-hostable but not suited for geo-distributed deployments}
+
+	\vspace{1em}
+	\visible<3->{\textbf{Garage is a self-hosted drop-in replacement for the Amazon S3 object store}}
+\end{frame}
+
+\begin{frame}
+	\frametitle{CRDTs / weak consistency instead of consensus}
+
+	\underline{Internally, Garage uses only CRDTs} (conflict-free replicated data types)
+
+	\vspace{2em}
+	Why not Raft, Paxos, ...? Issues of consensus algorithms:
+
+	\vspace{1em}
+	\begin{itemize}
+		\item<2-> \textbf{Software complexity}
+			\vspace{1em}
+		\item<3-> \textbf{Performance issues:}
+			\vspace{.5em}
+			\begin{itemize}
+				\item<4-> The leader is a \textbf{bottleneck} for all requests\\
+					\vspace{.5em}
+				\item<5-> \textbf{Sensitive to higher latency} between nodes
+					\vspace{.5em}
+				\item<6-> \textbf{Takes time to reconverge} when disrupted (e.g. node going down)
+			\end{itemize}
+	\end{itemize}
+\end{frame}
+
+\begin{frame}
+	\frametitle{The data model of object storage}
+	Object storage is basically a \textbf{key-value store}:
+	\vspace{.5em}
+
+	{\scriptsize
+		\begin{center}
+		\begin{tabular}{|l|p{7cm}|}
+			\hline
+			\textbf{Key: file path + name} & \textbf{Value: file data + metadata} \\
+			\hline
+			\hline
+			\texttt{index.html} &
+				\texttt{Content-Type: text/html; charset=utf-8} \newline
+				\texttt{Content-Length: 24929} \newline
+				\texttt{<binary blob>} \\ 
+			\hline
+			\texttt{img/logo.svg} &
+				\texttt{Content-Type: text/svg+xml} \newline
+				\texttt{Content-Length: 13429} \newline
+				\texttt{<binary blob>} \\ 
+			\hline
+			\texttt{download/index.html} &
+				\texttt{Content-Type: text/html; charset=utf-8} \newline
+				\texttt{Content-Length: 26563} \newline
+				\texttt{<binary blob>} \\ 
+			\hline
+		\end{tabular}
+		\end{center}
+		}
+
+	\vspace{1em}
+	\begin{itemize}
+		\item<2> Maps well to CRDT data types
+	\end{itemize}
+\end{frame}
+
+\begin{frame}
+	\frametitle{Performance gains in practice}
+	\begin{center}
+		\includegraphics[width=.8\linewidth]{../assets/perf/endpoint_latency_0.7_0.8_minio.png}
+	\end{center}
+\end{frame}
+
+% ======================================== OPERATING
+% ======================================== OPERATING
+% ======================================== OPERATING
+
+
+\section{Production clusters}
+
+\begin{frame}
+	\frametitle{Deployment kinds}
+
+	\includegraphics[width=.9\linewidth]{../assets/cluster_kind.png}
+	\vspace{1em}
+	
+\end{frame}
+
+\begin{frame}
+	\frametitle{How big they are?}
+
+	\includegraphics[width=.9\linewidth]{../assets/cluster_size.png}
+	\vspace{1em}
+
+	\textit{"Petabyte storage setup for a video site. Nginx as CDN in-front using garage-s3-website feature. Each storage node has ~64TB storage with raid10, no replication within garage. 25gbit nic. haproxy to loadbalance across 5 nodes. mostly reads with very few writes."}
+
+	\vspace{1em}
+	\textit{"We currently manage 7 Garage nodes, 28TB total storage, 6M blocks for 3M objects and 4TB of object data. We have been running Garage in production for 2.5 years."}
+
+\end{frame}
+
+\begin{frame}
+	\frametitle{Operating Garage}
+	\begin{center}
+		\only<1-2>{
+			\includegraphics[width=.9\linewidth]{../assets/screenshots/garage_status_0.10.png}
+			\\\vspace{1em}
+			\visible<2>{\includegraphics[width=.9\linewidth]{../assets/screenshots/garage_status_unhealthy_0.10.png}}
+		}
+	\end{center}
+\end{frame}
+
+\begin{frame}
+	\frametitle{Garage's architecture}
+	\begin{center}
+		\only<1>{\includegraphics[width=.45\linewidth]{../assets/garage.drawio.pdf}}%
+		\only<2>{\includegraphics[width=.6\linewidth]{../assets/garage_sync.drawio.pdf}}%
+	\end{center}
+\end{frame}
+
+\begin{frame}
+	\frametitle{Digging deeper}
+	\begin{center}
+		\only<1>{\includegraphics[width=.9\linewidth]{../assets/screenshots/garage_stats_0.10.png}}
+		\only<2>{\includegraphics[width=.5\linewidth]{../assets/screenshots/garage_worker_list_0.10.png}}
+		\only<3>{\includegraphics[width=.6\linewidth]{../assets/screenshots/garage_worker_param_0.10.png}}
+	\end{center}
+\end{frame}
+
+\begin{frame}
+	\frametitle{Potential limitations and bottlenecks}
+	\begin{itemize}
+		\item Global:
+			\begin{itemize}
+				\item Max. $\sim$100 nodes per cluster (excluding gateways)
+			\end{itemize}
+			\vspace{1em}
+		\item Metadata:
+			\begin{itemize}
+				\item One big bucket = bottleneck, object list on 3 nodes only
+			\end{itemize}
+			\vspace{1em}
+		\item Block manager:
+			\begin{itemize}
+				\item Lots of small files on disk
+				\item Processing the resync queue can be slow
+			\end{itemize}
+	\end{itemize}
+\end{frame}
+
+\begin{frame}
+	\frametitle{Deployment advice for very large clusters}
+	\begin{itemize}
+		\item Metadata storage:
+			\begin{itemize}
+				\item ZFS mirror (x2) on fast NVMe
+				\item Use LMDB storage engine
+			\end{itemize}
+			\vspace{.5em}
+		\item Data block storage:
+			\begin{itemize}
+				\item Use Garage's native multi-HDD support
+				\item XFS on individual drives
+				\item Increase block size (1MB $\to$ 10MB, requires more RAM and good networking)
+				\item Tune \texttt{resync-tranquility} and \texttt{resync-worker-count} dynamically
+			\end{itemize}
+			\vspace{.5em}
+		\item Other :
+			\begin{itemize}
+				\item Split data over several buckets
+				\item Use less than 100 storage nodes
+				\item Use gateway nodes
+			\end{itemize}
+			\vspace{.5em}
+	\end{itemize}
+\end{frame}
+
+
+\begin{frame}
+	\frametitle{Focus on Deuxfleurs}
+
+	Host institutional websites, partnership with a web agency.
+	Matrix media backend.
+
+	Plan to use it as an email backend for an internally developed email server.
+
+\end{frame}
+
+
+% ======================================== TIMELINE
+% ======================================== TIMELINE
+% ======================================== TIMELINE
+
+\section{Recent developments}
+
+% ====================== v0.7.0 ===============================
+
+\begin{frame}
+	\begin{center}
+		\includegraphics[width=.8\linewidth]{../assets/tl.drawio.png}
+	\end{center}
+\end{frame}
+
+\begin{frame}
+	\frametitle{April 2022 - Garage v0.7.0}
+	Focus on \underline{observability and ecosystem integration}
+	\vspace{2em}
+	\begin{itemize}
+		\item \textbf{Monitoring:} metrics and traces, using OpenTelemetry
+			\vspace{1em}
+		\item Replication modes with 1 or 2 copies / weaker consistency
+			\vspace{1em}
+		\item Kubernetes integration for node discovery
+			\vspace{1em}
+		\item Admin API (v0.7.2)
+	\end{itemize}
+\end{frame}
+
+\begin{frame}
+	\frametitle{Metrics (Prometheus + Grafana)}
+	\begin{center}
+		\includegraphics[width=.9\linewidth]{../assets/screenshots/grafana_dashboard.png}
+	\end{center}
+\end{frame}
+
+\begin{frame}
+	\frametitle{Traces (Jaeger)}
+	\begin{center}
+		\includegraphics[width=.8\linewidth]{../assets/screenshots/jaeger_listobjects.png}
+	\end{center}
+\end{frame}
+
+% ====================== v0.8.0 ===============================
+
+\begin{frame}
+	\begin{center}
+		\includegraphics[width=.8\linewidth]{../assets/tl.drawio.png}
+	\end{center}
+\end{frame}
+
+\begin{frame}
+	\frametitle{November 2022 - Garage v0.8.0}
+	Focus on \underline{performance}
+	\vspace{2em}
+	\begin{itemize}
+		\item \textbf{Alternative metadata DB engines} (LMDB, Sqlite)
+			\vspace{1em}
+		\item \textbf{Performance improvements:} block streaming, various optimizations...
+			\vspace{1em}
+		\item Bucket quotas (max size, max \#objects)
+			\vspace{1em}
+		\item Quality of life improvements, observability, etc.
+	\end{itemize}
+\end{frame}
+
+\begin{frame}
+	\frametitle{About metadata DB engines}
+	\textbf{Issues with Sled:}
+	\vspace{1em}
+	\begin{itemize}
+		\item Huge files on disk
+			\vspace{.5em}
+		\item Unpredictable performance, especially on HDD
+			\vspace{.5em}
+		\item API limitations
+			\vspace{.5em}
+		\item Not actively maintained
+	\end{itemize}
+
+	\vspace{2em}
+	\textbf{LMDB:} very stable, good performance, file size is reasonable\\
+	\textbf{Sqlite} also available as a second choice
+
+	\vspace{1em}
+	Sled will be removed in Garage v1.0
+\end{frame}
+
+\begin{frame}
+	\frametitle{DB engine performance comparison}
+	\begin{center}
+		\includegraphics[width=.6\linewidth]{../assets/perf/db_engine.png}
+	\end{center}
+	NB: Sqlite was slow due to synchronous mode, now configurable
+\end{frame}
+
+\begin{frame}
+	\frametitle{Block streaming}
+	\begin{center}
+		\only<1>{\includegraphics[width=.8\linewidth]{../assets/schema-streaming-1.png}}
+		\only<2>{\includegraphics[width=.8\linewidth]{../assets/schema-streaming-2.png}}
+	\end{center}
+\end{frame}
+
+\begin{frame}
+	\frametitle{TTFB benchmark}
+	\begin{center}
+		\includegraphics[width=.8\linewidth]{../assets/perf/ttfb.png}
+	\end{center}
+\end{frame}
+
+\begin{frame}
+	\frametitle{Throughput benchmark}
+	\begin{center}
+		\includegraphics[width=.7\linewidth]{../assets/perf/io-0.7-0.8-minio.png}
+	\end{center}
+\end{frame}
+
+% ====================== v0.9.0 ===============================
+
+\begin{frame}
+	\begin{center}
+		\includegraphics[width=.8\linewidth]{../assets/tl.drawio.png}
+	\end{center}
+\end{frame}
+
+\begin{frame}
+	\frametitle{October 2023 - Garage v0.9.0}
+	Focus on \underline{streamlining \& usability}
+	\vspace{2em}
+	\begin{itemize}
+		\item Support multiple HDDs per node
+			\vspace{1em}
+		\item S3 compatibility: 
+			\vspace{1em}
+			\begin{itemize}
+				\item support basic lifecycle configurations
+					\vspace{.5em}
+				\item allow for multipart upload part retries
+			\end{itemize}
+			\vspace{1em}
+		\item LMDB by default, deprecation of Sled
+			\vspace{1em}
+		\item New layout computation algorithm
+	\end{itemize}
+\end{frame}
+
+
+\begin{frame}
+	\frametitle{Layout computation}
+	\begin{overprint}
+		\onslide<1>
+		\begin{center}
+			\includegraphics[width=\linewidth, trim=0 0 0 -4cm]{../assets/screenshots/garage_status_0.9_prod_zonehl.png}
+		\end{center}
+		\onslide<2>
+		\begin{center}
+			\includegraphics[width=.7\linewidth]{../assets/map.png}
+		\end{center}
+	\end{overprint}
+	\vspace{1em}
+	Garage stores replicas on different zones when possible
+\end{frame}
+
+\begin{frame}
+	\frametitle{What a "layout" is}
+		\textbf{A layout is a precomputed index table:}
+		\vspace{1em}
+
+		{\footnotesize
+			\begin{center}
+			\begin{tabular}{|l|l|l|l|}
+				\hline
+				\textbf{Partition} & \textbf{Node 1} & \textbf{Node 2} & \textbf{Node 3} \\
+				\hline
+				\hline
+				Partition 0 & df-ymk (bespin) & Abricot (scorpio) & Courgette (neptune) \\
+				\hline
+				Partition 1 & Ananas (scorpio) & Courgette (neptune) & df-ykl (bespin) \\
+				\hline
+				Partition 2 & df-ymf (bespin) & Celeri (neptune) & Abricot (scorpio) \\
+				\hline
+				\hspace{1em}$\vdots$ & \hspace{1em}$\vdots$ & \hspace{1em}$\vdots$ & \hspace{1em}$\vdots$ \\
+				\hline
+				Partition 255 & Concombre (neptune) & df-ykl (bespin) & Abricot (scorpio) \\
+				\hline
+			\end{tabular}
+			\end{center}
+		}
+
+	\vspace{2em}
+	\visible<2->{
+		The index table is built centrally using an optimal algorithm,\\
+		then propagated to all nodes
+	}
+
+	\vspace{1em}
+	\visible<3->{
+		\footnotesize
+		Oulamara, M., \& Auvolat, A. (2023). \emph{An algorithm for geo-distributed and redundant storage in Garage}.\\ arXiv preprint arXiv:2302.13798.
+	}
+\end{frame}
+
+
+
+% ====================== v1.0.0 ===============================
+
+\begin{frame}
+	\begin{center}
+		\includegraphics[width=.8\linewidth]{../assets/tl.drawio.png}
+	\end{center}
+\end{frame}
+
+\begin{frame}
+	\frametitle{April 2024 - Garage v1.0.0}
+	Focus on \underline{consistency, security \& stability}
+	\vspace{2em}
+	\begin{itemize}
+		\item Fix consistency issues when reshuffling data (Jepsen testing)
+			\vspace{1em}
+		\item \textbf{Security audit} by Radically Open Security
+			\vspace{1em}
+		\item Misc. S3 features (SSE-C, checksums,  ...) and compatibility fixes
+	\end{itemize}
+\end{frame}
+
+% ====================== v2.0.0 ===============================
+
+\begin{frame}
+	\begin{center}
+		\includegraphics[width=.8\linewidth]{../assets/tl.drawio.png}
+	\end{center}
+\end{frame}
+
+\begin{frame}
+	\frametitle{Garage v2.0.0}
+	Focus on \underline{}
+	\vspace{2em}
+	\begin{itemize}
+		\item TODO
+	\end{itemize}
+\end{frame}
+
+
+\begin{frame}
+	\frametitle{Currently funding...}
+
+	\textit{...}
+\end{frame}
+
+\begin{frame}
+	\frametitle{We run community surveys}
+	\begin{center}
+		\includegraphics[width=.6\linewidth]{../assets/survey_requested_features.png}
+	\end{center}
+\end{frame}
+
+% ======================================== END
+% ======================================== END
+% ======================================== END
+
+\begin{frame}
+	\frametitle{Where to find us}
+	\begin{center}
+			\includegraphics[width=.25\linewidth]{../../logo/garage_hires.png}\\
+			\vspace{-1em}
+		\url{https://garagehq.deuxfleurs.fr/}\\
+		\url{mailto:garagehq@deuxfleurs.fr}\\
+		\texttt{\#garage:deuxfleurs.fr} on Matrix
+
+		\vspace{1.5em}
+			\includegraphics[width=.06\linewidth]{../assets/logos/rust_logo.png}
+			\includegraphics[width=.13\linewidth]{../assets/logos/AGPLv3_Logo.png}
+	\end{center}
+\end{frame}
+
+\end{document}
+
+%% vim: set ts=4 sw=4 tw=0 noet spelllang=en :

flake.lock

@@ -12,16 +12,17 @@
       "original": {
         "owner": "ipetkov",
         "repo": "crane",
+        "rev": "6fe74265bbb6d016d663b1091f015e2976c4a527",
         "type": "github"
       }
     },
     "flake-compat": {
       "locked": {
-        "lastModified": 1717312683,
-        "narHash": "sha256-FrlieJH50AuvagamEvWMIE6D2OAnERuDboFDYAED/dE=",
+        "lastModified": 1761640442,
+        "narHash": "sha256-AtrEP6Jmdvrqiv4x2xa5mrtaIp3OEe8uBYCDZDS+hu8=",
         "owner": "nix-community",
         "repo": "flake-compat",
-        "rev": "38fd3954cf65ce6faf3d0d45cd26059e059f07ea",
+        "rev": "4a56054d8ffc173222d09dad23adf4ba946c8884",
         "type": "github"
       },
       "original": {
@@ -50,17 +51,17 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1747825515,
-        "narHash": "sha256-BWpMQymVI73QoKZdcVCxUCCK3GNvr/xa2Dc4DM1o2BE=",
+        "lastModified": 1763977559,
+        "narHash": "sha256-g4MKqsIRy5yJwEsI+fYODqLUnAqIY4kZai0nldAP6EM=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "cd2812de55cf87df88a9e09bf3be1ce63d50c1a6",
+        "rev": "cfe2c7d5b5d3032862254e68c37a6576b633d632",
         "type": "github"
       },
       "original": {
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "cd2812de55cf87df88a9e09bf3be1ce63d50c1a6",
+        "rev": "cfe2c7d5b5d3032862254e68c37a6576b633d632",
         "type": "github"
       }
     },
@@ -80,17 +81,17 @@
         ]
       },
       "locked": {
-        "lastModified": 1738549608,
-        "narHash": "sha256-GdyT9QEUSx5k/n8kILuNy83vxxdyUfJ8jL5mMpQZWfw=",
+        "lastModified": 1763952169,
+        "narHash": "sha256-+PeDBD8P+NKauH+w7eO/QWCIp8Cx4mCfWnh9sJmy9CM=",
         "owner": "oxalica",
         "repo": "rust-overlay",
-        "rev": "35c6f8c4352f995ecd53896200769f80a3e8f22d",
+        "rev": "ab726555a9a72e6dc80649809147823a813fa95b",
         "type": "github"
       },
       "original": {
         "owner": "oxalica",
         "repo": "rust-overlay",
-        "rev": "35c6f8c4352f995ecd53896200769f80a3e8f22d",
+        "rev": "ab726555a9a72e6dc80649809147823a813fa95b",
         "type": "github"
       }
     },

flake.nix

@@ -2,16 +2,17 @@
   description =
     "Garage, an S3-compatible distributed object store for self-hosted deployments";
 
-  # Nixpkgs 25.05 as of 2025-05-22
+  # Nixpkgs 25.05 as of 2025-11-24
   inputs.nixpkgs.url =
-    "github:NixOS/nixpkgs/cd2812de55cf87df88a9e09bf3be1ce63d50c1a6";
+    "github:NixOS/nixpkgs/cfe2c7d5b5d3032862254e68c37a6576b633d632";
 
-  # Rust overlay as of 2025-02-03
+  # Rust overlay as of 2025-11-24
   inputs.rust-overlay.url =
-  "github:oxalica/rust-overlay/35c6f8c4352f995ecd53896200769f80a3e8f22d";
+  "github:oxalica/rust-overlay/ab726555a9a72e6dc80649809147823a813fa95b";
   inputs.rust-overlay.inputs.nixpkgs.follows = "nixpkgs";
 
-  inputs.crane.url = "github:ipetkov/crane";
+  # Crane as of 2025-01-24
+  inputs.crane.url = "github:ipetkov/crane/6fe74265bbb6d016d663b1091f015e2976c4a527";
 
   inputs.flake-compat.url = "github:nix-community/flake-compat";
   inputs.flake-utils.url = "github:numtide/flake-utils";
@@ -30,6 +31,10 @@
           inherit system nixpkgs crane rust-overlay extraTestEnv;
           release = false;
         }).garage-test;
+        lints = (compile {
+          inherit system nixpkgs crane rust-overlay;
+          release = false;
+        });
       in
       {
         packages = {
@@ -53,6 +58,13 @@
           tests-sqlite = testWith {
             GARAGE_TEST_INTEGRATION_DB_ENGINE = "sqlite";
           };
+          tests-fjall = testWith {
+            GARAGE_TEST_INTEGRATION_DB_ENGINE = "fjall";
+          };
+
+          # lints (fmt, clippy)
+          fmt = lints.garage-cargo-fmt;
+          clippy = lints.garage-cargo-clippy;
         };
 
         # ---- developpment shell, for making native builds only ----
@@ -78,6 +90,9 @@
                 cargo-outdated
                 cargo-machete
                 nixpkgs-fmt
+                openssl
+                socat
+                killall
               ];
             };
           };

nix/build_index.nix

@@ -167,7 +167,7 @@ let
                  </ul></p>
                  <p> Sources:
                  <ul>
-                  <li><a href="https://git.deuxfleurs.fr/Deuxfleurs/garage/src/${r.type}/${x.version}">gitea</a></li>
+                  <li><a href="https://git.deuxfleurs.fr/Deuxfleurs/garage/src/${r.type}/${x.version}">Forgejo</a></li>
                   <li><a href="https://git.deuxfleurs.fr/Deuxfleurs/garage/archive/${x.version}.zip">.zip</a></li>
                   <li><a href="https://git.deuxfleurs.fr/Deuxfleurs/garage/archive/${x.version}.tar.gz">.tar.gz</a></li>
                  </ul></p>

nix/compile.nix

@@ -48,7 +48,7 @@ let
 
   inherit (pkgs) lib stdenv;
 
-  toolchainFn = (p: p.rust-bin.stable."1.82.0".default.override {
+  toolchainFn = (p: p.rust-bin.stable."1.91.0".default.override {
     targets = lib.optionals (target != null) [ rustTarget ];
     extensions = [
       "rust-src"
@@ -68,7 +68,7 @@ let
   rootFeatures = if features != null then
     features
   else
-    ([ "bundled-libs" "lmdb" "sqlite" "k2v" ] ++ (lib.optionals release [
+    ([ "bundled-libs" "lmdb" "sqlite" "fjall" "k2v" ] ++ (lib.optionals release [
       "consul-discovery"
       "kubernetes-discovery"
       "metrics"
@@ -190,4 +190,15 @@ in rec {
       pkgs.cacert
     ];
   } // extraTestEnv);
+
+  # ---- source code linting ----
+
+  garage-cargo-fmt = craneLib.cargoFmt (commonArgs // {
+    cargoExtraArgs = "";
+  });
+
+  garage-cargo-clippy = craneLib.cargoClippy (commonArgs // {
+    cargoArtifacts = garage-deps;
+    cargoClippyExtraArgs = "--all-targets -- -D warnings";
+  });
 }

script/helm/garage/Chart.yaml

@@ -2,8 +2,8 @@ apiVersion: v2
 name: garage
 description: S3-compatible object store for small self-hosted geo-distributed deployments
 type: application
-version: 0.8.0
-appVersion: "v2.0.0"
+version: 0.9.2
+appVersion: "v2.2.0"
 home: https://garagehq.deuxfleurs.fr/
 icon: https://garagehq.deuxfleurs.fr/images/garage-logo.svg
 

script/helm/garage/README.md

@@ -1,6 +1,6 @@
 # garage
 
-![Version: 0.8.0](https://img.shields.io/badge/Version-0.8.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: v2.0.0](https://img.shields.io/badge/AppVersion-v2.0.0-informational?style=flat-square)
+![Version: 0.9.2](https://img.shields.io/badge/Version-0.9.2-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: v2.2.0](https://img.shields.io/badge/AppVersion-v2.2.0-informational?style=flat-square)
 
 S3-compatible object store for small self-hosted geo-distributed deployments
 
@@ -15,6 +15,7 @@ S3-compatible object store for small self-hosted geo-distributed deployments
 | Key | Type | Default | Description |
 |-----|------|---------|-------------|
 | affinity | object | `{}` |  |
+| commonLabels  | object | `{}` | Extra labels for all resources |
 | deployment.kind | string | `"StatefulSet"` | Switchable to DaemonSet |
 | deployment.podManagementPolicy | string | `"OrderedReady"` | If using statefulset, allow Parallel or OrderedReady (default) |
 | deployment.replicaCount | int | `3` | Number of StatefulSet replicas/garage nodes to start |

script/helm/garage/templates/_helpers.tpl

@@ -27,7 +27,7 @@ If release name contains chart name it will be used as a full name.
 Create the name of the rpc secret
 */}}
 {{- define "garage.rpcSecretName" -}}
-{{- printf "%s-rpc-secret" (include "garage.fullname" .) -}}
+{{- .Values.garage.existingRpcSecret | default (printf "%s-rpc-secret" (include "garage.fullname" .)) -}}
 {{- end }}
 
 {{/*
@@ -47,6 +47,9 @@ helm.sh/chart: {{ include "garage.chart" . }}
 app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
 {{- end }}
 app.kubernetes.io/managed-by: {{ .Release.Service }}
+{{- with .Values.commonLabels }}
+{{- toYaml . | nindent 0 }}
+{{- end }}
 {{- end }}
 
 {{/*

script/helm/garage/templates/configmap.yaml

@@ -28,8 +28,16 @@ data:
     # rpc_secret will be populated by the init container from a k8s secret object
     rpc_secret = "__RPC_SECRET_REPLACE__"
 
-    bootstrap_peers = {{ .Values.garage.bootstrapPeers }}
+    bootstrap_peers = [
+    {{- range $index, $peer := .Values.garage.bootstrapPeers }}
+    {{- if $index}}, {{ end }}{{ $peer | quote }}
+    {{ end }} 
+    ]
 
+    {{- if .Values.garage.additionalTopLevelConfig }}
+    {{ .Values.garage.additionalTopLevelConfig | nindent 4 }}
+    {{- end }}
+  
     [kubernetes_discovery]
     namespace = "{{ .Release.Namespace }}"
     service_name = "{{ include "garage.fullname" . }}"

script/helm/garage/templates/secret.yaml

@@ -1,3 +1,4 @@
+{{- if not .Values.garage.existingRpcSecret }}
 apiVersion: v1
 kind: Secret
 metadata:
@@ -12,3 +13,4 @@ data:
   {{- $prevRpcSecret := $prevSecretData.rpcSecret | default "" | b64dec }}
   {{/* Priority is: 1. from values, 2. previous value, 3. generate random */}}
   rpcSecret: {{ .Values.garage.rpcSecret | default $prevRpcSecret | default (include "jupyterhub.randHex" 64) | b64enc | quote }}
+{{- end }}

script/helm/garage/templates/service.yaml

@@ -4,6 +4,10 @@ metadata:
   name: {{ include "garage.fullname" . }}
   labels:
     {{- include "garage.labels" . | nindent 4 }}
+  {{- with .Values.service.annotations }}
+  annotations:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
 spec:
   type: {{ .Values.service.type }}
   ports:
@@ -37,4 +41,4 @@ spec:
       name: metrics
   selector:
     {{- include "garage.selectorLabels" . | nindent 4 }}
-{{- end }}
+{{- end }}

script/helm/garage/templates/workload.yaml

@@ -21,7 +21,7 @@ spec:
         {{- toYaml . | nindent 8 }}
       {{- end }}
       labels:
-        {{- include "garage.selectorLabels" . | nindent 8 }}
+        {{- include "garage.labels" . | nindent 8 }}
     spec:
       {{- with .Values.imagePullSecrets }}
       imagePullSecrets:

script/helm/garage/values.yaml

@@ -2,6 +2,11 @@
 # This is a YAML-formatted file.
 # Declare variables to be passed into your templates.
 
+# -- Additional labels to add to all resources created by this chart
+commonLabels: {}
+#   app.kubernetes.io/part-of: storage
+#   team: platform
+
 # Garage configuration. These values go to garage.toml
 garage:
   # -- Can be changed for better performance on certain systems
@@ -32,6 +37,10 @@ garage:
   rpcBindAddr: "[::]:3901"
   # -- If not given, a random secret will be generated and stored in a Secret object
   rpcSecret: ""
+  # -- If you want to provide an rpcSecret within an existing k8s secret,
+  # specify the secret name here, and store the value under the secret key `rpcSecret`
+  # the default secret will not be created
+  existingRpcSecret: ""
   # -- This is not required if you use the integrated kubernetes discovery
   bootstrapPeers: []
   # -- Set to true if you want to use k8s discovery but install the CRDs manually outside
@@ -45,6 +54,12 @@ garage:
       rootDomain: ".web.garage.tld"
       index: "index.html"
 
+  # -- Additional configuration to append to garage.toml. Use a multi-line string for custom config.
+  # Example:
+  #  additionalTopLevelConfig: |-
+  #    data_fsync = true
+  additionalTopLevelConfig: ""
+
   # -- if not empty string, allow using an existing ConfigMap for the garage.toml,
   # if set, ignores garage.toml
   existingConfigMap: ""
@@ -112,6 +127,7 @@ podSecurityContext:
   runAsUser: 1000
   runAsGroup: 1000
   fsGroup: 1000
+  fsGroupChangePolicy: "OnRootMismatch"
   runAsNonRoot: true
 
 securityContext:
@@ -128,6 +144,8 @@ service:
   # - NodePort (+ Ingress)
   # - LoadBalancer
   type: ClusterIP
+  # -- Annotations to add to the service
+  annotations: {}
   s3:
     api:
       port: 3900

script/telemetry/grafana-garage-dashboard-prometheus.json

@@ -694,32 +694,7 @@
             ]
           }
         },
-        "overrides": [
-          {
-            "__systemRef": "hideSeriesFrom",
-            "matcher": {
-              "id": "byNames",
-              "options": {
-                "mode": "exclude",
-                "names": [
-                  "10.83.2.3:3903"
-                ],
-                "prefix": "All except:",
-                "readOnly": true
-              }
-            },
-            "properties": [
-              {
-                "id": "custom.hideFrom",
-                "value": {
-                  "legend": false,
-                  "tooltip": false,
-                  "viz": true
-                }
-              }
-            ]
-          }
-        ]
+        "overrides": []
       },
       "gridPos": {
         "h": 8,

shell.nix

@@ -26,7 +26,7 @@ in
       s3cmd
       minio-client
       rclone
-      (python312.withPackages (ps: [ ps.boto3 ]))
+      (python313.withPackages (ps: [ ps.boto3 ]))
 
       socat
       psmisc
@@ -36,6 +36,8 @@ in
       jq
     ];
     shellHook = ''
+      export AWS_REQUEST_CHECKSUM_CALCULATION='when_required'
+
       function to_s3 {
         AWS_REQUEST_CHECKSUM_CALCULATION=WHEN_REQUIRED AWS_RESPONSE_CHECKSUM_VALIDATION=WHEN_REQUIRED \
         aws \

src/api/admin/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "garage_api_admin"
-version = "2.0.0"
+version = "2.2.0"
 authors = ["Alex Auvolat <alex@adnab.me>"]
 edition = "2018"
 license = "AGPL-3.0"
@@ -26,7 +26,7 @@ argon2.workspace = true
 async-trait.workspace = true
 bytesize.workspace = true
 chrono.workspace = true
-err-derive.workspace = true
+thiserror.workspace = true
 hex.workspace = true
 paste.workspace = true
 tracing.workspace = true

src/api/admin/admin_token.rs

@@ -175,6 +175,63 @@ impl RequestHandler for DeleteAdminTokenRequest {
 	}
 }
 
+impl RequestHandler for GetCurrentAdminTokenInfoRequest {
+	type Response = GetCurrentAdminTokenInfoResponse;
+
+	async fn handle(
+		self,
+		garage: &Arc<Garage>,
+		_admin: &Admin,
+	) -> Result<GetCurrentAdminTokenInfoResponse, Error> {
+		let now = now_msec();
+
+		if garage
+			.config
+			.admin
+			.metrics_token
+			.as_ref()
+			.is_some_and(|s| s == &self.admin_token)
+		{
+			return Ok(GetCurrentAdminTokenInfoResponse(
+				GetAdminTokenInfoResponse {
+					id: None,
+					created: None,
+					name: "metrics_token (from daemon configuration)".into(),
+					expiration: None,
+					expired: false,
+					scope: vec!["Metrics".into()],
+				},
+			));
+		}
+
+		if garage
+			.config
+			.admin
+			.admin_token
+			.as_ref()
+			.is_some_and(|s| s == &self.admin_token)
+		{
+			return Ok(GetCurrentAdminTokenInfoResponse(
+				GetAdminTokenInfoResponse {
+					id: None,
+					created: None,
+					name: "admin_token (from daemon configuration)".into(),
+					expiration: None,
+					expired: false,
+					scope: vec!["*".into()],
+				},
+			));
+		}
+
+		let (prefix, _) = self.admin_token.split_once('.').unwrap();
+		let token = get_existing_admin_token(&garage, &prefix.to_string()).await?;
+
+		Ok(GetCurrentAdminTokenInfoResponse(admin_token_info_results(
+			&token, now,
+		)))
+	}
+}
+
 // ---- helpers ----
 
 fn admin_token_info_results(token: &AdminApiToken, now: u64) -> GetAdminTokenInfoResponse {

src/api/admin/api.rs

@@ -12,7 +12,7 @@ use garage_rpc::*;
 
 use garage_model::garage::Garage;
 
-use garage_api_common::helpers::is_default;
+use garage_api_common::{common_error::CommonError, helpers::is_default};
 
 use crate::api_server::{find_matching_nodes, AdminRpc, AdminRpcResponse};
 use crate::error::Error;
@@ -56,6 +56,7 @@ admin_endpoints![
 	CreateAdminToken,
 	UpdateAdminToken,
 	DeleteAdminToken,
+	GetCurrentAdminTokenInfo,
 
 	// Layout operations
 	GetClusterLayout,
@@ -144,6 +145,13 @@ pub struct MultiResponse<RB> {
 	pub error: HashMap<String, String>,
 }
 
+#[derive(Debug, Clone, Serialize, Deserialize, IntoParams)]
+#[into_params(parameter_in = Query)]
+pub struct MultiRequestQueryParams {
+	/// Node ID to query, or `*` for all nodes, or `self` for the node responding to the request
+	pub node: String,
+}
+
 // **********************************************
 //      Special endpoints
 //
@@ -154,8 +162,10 @@ pub struct MultiResponse<RB> {
 #[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct OptionsRequest;
 
-#[derive(Debug, Clone, Serialize, Deserialize)]
+#[derive(Debug, Clone, Serialize, Deserialize, IntoParams)]
+#[into_params(parameter_in = Query)]
 pub struct CheckDomainRequest {
+	/// The domain name to check for
 	pub domain: String,
 }
 
@@ -257,7 +267,7 @@ pub struct GetClusterHealthResponse {
 	/// the number of storage nodes currently registered in the cluster layout
 	pub storage_nodes: usize,
 	/// the number of storage nodes to which a connection is currently open
-	pub storage_nodes_ok: usize,
+	pub storage_nodes_up: usize,
 	/// the total number of partitions of the data (currently always 256)
 	pub partitions: usize,
 	/// the number of partitions for which a quorum of write nodes is available
@@ -308,6 +318,7 @@ pub struct ListAdminTokensResponse(pub Vec<GetAdminTokenInfoResponse>);
 // ---- GetAdminTokenInfo ----
 
 #[derive(Debug, Clone, Serialize, Deserialize, IntoParams)]
+#[into_params(parameter_in = Query)]
 #[serde(rename_all = "camelCase")]
 pub struct GetAdminTokenInfoRequest {
 	/// Admin API token ID
@@ -353,9 +364,12 @@ pub struct CreateAdminTokenResponse {
 
 // ---- UpdateAdminToken ----
 
-#[derive(Debug, Clone, Serialize, Deserialize)]
+#[derive(Debug, Clone, Serialize, Deserialize, IntoParams)]
+#[into_params(parameter_in = Query)]
 pub struct UpdateAdminTokenRequest {
+	/// Admin API token ID
 	pub id: String,
+	#[param(ignore = true)]
 	pub body: UpdateAdminTokenRequestBody,
 }
 
@@ -382,14 +396,25 @@ pub struct UpdateAdminTokenResponse(pub GetAdminTokenInfoResponse);
 
 // ---- DeleteAdminToken ----
 
-#[derive(Debug, Clone, Serialize, Deserialize)]
+#[derive(Debug, Clone, Serialize, Deserialize, IntoParams)]
+#[into_params(parameter_in = Query)]
 pub struct DeleteAdminTokenRequest {
+	/// Admin API token ID
 	pub id: String,
 }
 
 #[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct DeleteAdminTokenResponse;
 
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct GetCurrentAdminTokenInfoRequest {
+	pub admin_token: String,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)]
+#[serde(rename_all = "camelCase")]
+pub struct GetCurrentAdminTokenInfoResponse(pub GetAdminTokenInfoResponse);
+
 // **********************************************
 //      Layout operations
 // **********************************************
@@ -648,6 +673,7 @@ pub struct ListKeysResponseItem {
 // ---- GetKeyInfo ----
 
 #[derive(Debug, Clone, Serialize, Deserialize, IntoParams)]
+#[into_params(parameter_in = Query)]
 #[serde(rename_all = "camelCase")]
 pub struct GetKeyInfoRequest {
 	/// Access key ID
@@ -724,9 +750,12 @@ pub struct ImportKeyResponse(pub GetKeyInfoResponse);
 
 // ---- UpdateKey ----
 
-#[derive(Debug, Clone, Serialize, Deserialize)]
+#[derive(Debug, Clone, Serialize, Deserialize, IntoParams)]
+#[into_params(parameter_in = Query)]
 pub struct UpdateKeyRequest {
+	/// Access key ID
 	pub id: String,
+	#[param(ignore = true)]
 	pub body: UpdateKeyRequestBody,
 }
 
@@ -751,8 +780,10 @@ pub struct UpdateKeyRequestBody {
 
 // ---- DeleteKey ----
 
-#[derive(Debug, Clone, Serialize, Deserialize)]
+#[derive(Debug, Clone, Serialize, Deserialize, IntoParams)]
+#[into_params(parameter_in = Query)]
 pub struct DeleteKeyRequest {
+	/// Access key ID
 	pub id: String,
 }
 
@@ -790,6 +821,7 @@ pub struct BucketLocalAlias {
 // ---- GetBucketInfo ----
 
 #[derive(Debug, Clone, Serialize, Deserialize, IntoParams)]
+#[into_params(parameter_in = Query)]
 #[serde(rename_all = "camelCase")]
 pub struct GetBucketInfoRequest {
 	/// Exact bucket ID to look up
@@ -878,9 +910,12 @@ pub struct CreateBucketLocalAlias {
 
 // ---- UpdateBucket ----
 
-#[derive(Debug, Clone, Serialize, Deserialize)]
+#[derive(Debug, Clone, Serialize, Deserialize, IntoParams)]
+#[into_params(parameter_in = Query)]
 pub struct UpdateBucketRequest {
+	/// ID of the bucket to update
 	pub id: String,
+	#[param(ignore = true)]
 	pub body: UpdateBucketRequestBody,
 }
 
@@ -904,8 +939,10 @@ pub struct UpdateBucketWebsiteAccess {
 
 // ---- DeleteBucket ----
 
-#[derive(Debug, Clone, Serialize, Deserialize)]
+#[derive(Debug, Clone, Serialize, Deserialize, IntoParams)]
+#[into_params(parameter_in = Query)]
 pub struct DeleteBucketRequest {
+	/// ID of the bucket to delete
 	pub id: String,
 }
 
@@ -928,6 +965,7 @@ pub struct CleanupIncompleteUploadsResponse {
 }
 
 #[derive(Debug, Clone, Serialize, Deserialize, IntoParams)]
+#[into_params(parameter_in = Query)]
 #[serde(rename_all = "camelCase")]
 pub struct InspectObjectRequest {
 	pub bucket_id: String,
@@ -1115,6 +1153,7 @@ pub enum RepairType {
 	Rebalance,
 	Scrub(ScrubCommand),
 	Aliases,
+	ClearResyncQueue,
 }
 
 #[derive(Debug, Clone, Serialize, Deserialize, ToSchema)]

src/api/admin/api_server.rs

@@ -217,6 +217,13 @@ impl ApiHandler for ArcAdminApiServer {
 	) -> Result<Response<ResBody>, Error> {
 		self.0.handle_http_api(req, endpoint).await
 	}
+
+	fn key_id_from_request(&self, req: &Request<IncomingBody>) -> Option<String> {
+		let auth_header = req.headers().get(AUTHORIZATION)?;
+		let token = parse_authorization(auth_header).ok()?;
+		let key_id = token.split_once('.')?.0;
+		Some(key_id.to_string())
+	}
 }
 
 impl ApiEndpoint for HttpEndpoint {
@@ -244,6 +251,15 @@ fn hash_bearer_token(token: &str) -> String {
 		.to_string()
 }
 
+fn parse_authorization(auth_header: &hyper::http::HeaderValue) -> Result<&str, Error> {
+	let token = auth_header
+		.to_str()?
+		.strip_prefix("Bearer ")
+		.ok_or_else(|| Error::forbidden("Invalid Authorization header"))?
+		.trim();
+	Ok(token)
+}
+
 fn verify_authorization(
 	garage: &Garage,
 	global_token_hash: Option<&str>,
@@ -260,11 +276,7 @@ fn verify_authorization(
 				"Bearer token must be provided in Authorization header",
 			))
 		}
-		Some(authorization) => authorization
-			.to_str()?
-			.strip_prefix("Bearer ")
-			.ok_or_else(|| Error::forbidden("Invalid Authorization header"))?
-			.trim(),
+		Some(authorization) => parse_authorization(authorization)?,
 	};
 
 	let token_hash_string = if let Some((prefix, _)) = token.split_once('.') {
@@ -273,7 +285,8 @@ fn verify_authorization(
 			.get_local(&EmptyKey, &prefix.to_string())?
 			.and_then(|k| k.state.into_option())
 			.filter(|p| !p.is_expired(now_msec()))
-			.filter(|p| p.has_scope(endpoint_name))
+			// GetCurrentAdminTokenInfo endpoint must be accessible even if it is not in the token scopes
+			.filter(|p| p.has_scope(endpoint_name) || endpoint_name == "GetCurrentAdminTokenInfo")
 			.ok_or_else(|| Error::forbidden(invalid_msg))?
 			.token_hash
 	} else {
@@ -293,38 +306,36 @@ fn verify_authorization(
 }
 
 pub(crate) fn find_matching_nodes(garage: &Garage, spec: &str) -> Result<Vec<Uuid>, Error> {
-	let mut res = vec![];
-	if spec == "*" {
-		res = garage.system.cluster_layout().all_nodes().to_vec();
+	if spec == "self" {
+		Ok(vec![garage.system.id])
+	} else {
+		// Collect all nodes currently up and/or in cluster layout
+		let mut res = vec![];
+		if let Ok(all_nodes) = garage.system.cluster_layout().all_nodes() {
+			res = all_nodes.to_vec();
+		}
 		for node in garage.system.get_known_nodes() {
 			if node.is_up && !res.contains(&node.id) {
 				res.push(node.id);
 			}
 		}
-	} else if spec == "self" {
-		res.push(garage.system.id);
-	} else {
-		let layout = garage.system.cluster_layout();
-		let known_nodes = garage.system.get_known_nodes();
-		let all_nodes = layout
-			.all_nodes()
-			.iter()
-			.copied()
-			.chain(known_nodes.iter().filter(|x| x.is_up).map(|x| x.id));
-		for node in all_nodes {
-			if !res.contains(&node) && hex::encode(node).starts_with(spec) {
-				res.push(node);
+
+		if spec == "*" {
+			// match all nodes
+			Ok(res)
+		} else {
+			// filter nodes that match spec
+			res.retain(|node| hex::encode(node).starts_with(spec));
+			if res.is_empty() {
+				Err(Error::bad_request(format!("No nodes matching {}", spec)))
+			} else if res.len() > 1 {
+				Err(Error::bad_request(format!(
+					"Multiple nodes matching {}: {:?}",
+					spec, res
+				)))
+			} else {
+				Ok(res)
 			}
 		}
-		if res.is_empty() {
-			return Err(Error::bad_request(format!("No nodes matching {}", spec)));
-		}
-		if res.len() > 1 {
-			return Err(Error::bad_request(format!(
-				"Multiple nodes matching {}: {:?}",
-				spec, res
-			)));
-		}
 	}
-	Ok(res)
 }

src/api/admin/bucket.rs

@@ -697,20 +697,21 @@ async fn bucket_info_results(
 		}),
 		keys: relevant_keys
 			.into_values()
-			.map(|key| {
+			.filter_map(|key| {
 				let p = key.state.as_option().unwrap();
-				GetBucketInfoKey {
+				let permissions = p
+					.authorized_buckets
+					.get(&bucket.id)
+					.filter(|p| p.is_any())
+					.map(|p| ApiBucketKeyPerm {
+						read: p.allow_read,
+						write: p.allow_write,
+						owner: p.allow_owner,
+					})?;
+				Some(GetBucketInfoKey {
 					access_key_id: key.key_id,
 					name: p.name.get().to_string(),
-					permissions: p
-						.authorized_buckets
-						.get(&bucket.id)
-						.map(|p| ApiBucketKeyPerm {
-							read: p.allow_read,
-							write: p.allow_write,
-							owner: p.allow_owner,
-						})
-						.unwrap_or_default(),
+					permissions,
 					bucket_local_aliases: p
 						.local_aliases
 						.items()
@@ -718,7 +719,7 @@ async fn bucket_info_results(
 						.filter(|(_, _, b)| *b == Some(bucket.id))
 						.map(|(n, _, _)| n.to_string())
 						.collect::<Vec<_>>(),
-				}
+				})
 			})
 			.collect::<Vec<_>>(),
 		objects: *counters.get(OBJECTS).unwrap_or(&0),

src/api/admin/cluster.rs

@@ -56,48 +56,52 @@ impl RequestHandler for GetClusterStatusRequest {
 			})
 			.collect::<HashMap<_, _>>();
 
-		for (id, _, role) in layout.current().roles.items().iter() {
-			if let layout::NodeRoleV(Some(r)) = role {
-				let role = NodeAssignedRole {
-					zone: r.zone.to_string(),
-					capacity: r.capacity,
-					tags: r.tags.clone(),
-				};
-				match nodes.get_mut(id) {
-					None => {
-						nodes.insert(
-							*id,
-							NodeResp {
-								id: hex::encode(id),
-								role: Some(role),
-								..Default::default()
-							},
-						);
-					}
-					Some(n) => {
-						n.role = Some(role);
+		if let Ok(current_layout) = layout.current() {
+			for (id, _, role) in current_layout.roles.items().iter() {
+				if let layout::NodeRoleV(Some(r)) = role {
+					let role = NodeAssignedRole {
+						zone: r.zone.to_string(),
+						capacity: r.capacity,
+						tags: r.tags.clone(),
+					};
+					match nodes.get_mut(id) {
+						None => {
+							nodes.insert(
+								*id,
+								NodeResp {
+									id: hex::encode(id),
+									role: Some(role),
+									..Default::default()
+								},
+							);
+						}
+						Some(n) => {
+							n.role = Some(role);
+						}
 					}
 				}
 			}
 		}
 
-		for ver in layout.versions().iter().rev().skip(1) {
-			for (id, _, role) in ver.roles.items().iter() {
-				if let layout::NodeRoleV(Some(r)) = role {
-					if r.capacity.is_some() {
-						if let Some(n) = nodes.get_mut(id) {
-							if n.role.is_none() {
-								n.draining = true;
+		if let Ok(layout_versions) = layout.versions() {
+			for ver in layout_versions.iter().rev().skip(1) {
+				for (id, _, role) in ver.roles.items().iter() {
+					if let layout::NodeRoleV(Some(r)) = role {
+						if r.capacity.is_some() {
+							if let Some(n) = nodes.get_mut(id) {
+								if n.role.is_none() {
+									n.draining = true;
+								}
+							} else {
+								nodes.insert(
+									*id,
+									NodeResp {
+										id: hex::encode(id),
+										draining: true,
+										..Default::default()
+									},
+								);
 							}
-						} else {
-							nodes.insert(
-								*id,
-								NodeResp {
-									id: hex::encode(id),
-									draining: true,
-									..Default::default()
-								},
-							);
 						}
 					}
 				}
@@ -108,7 +112,7 @@ impl RequestHandler for GetClusterStatusRequest {
 		nodes.sort_by(|x, y| x.id.cmp(&y.id));
 
 		Ok(GetClusterStatusResponse {
-			layout_version: layout.current().version,
+			layout_version: layout.inner().current().version,
 			nodes,
 		})
 	}
@@ -134,7 +138,9 @@ impl RequestHandler for GetClusterHealthRequest {
 			known_nodes: health.known_nodes,
 			connected_nodes: health.connected_nodes,
 			storage_nodes: health.storage_nodes,
-			storage_nodes_ok: health.storage_nodes_ok,
+			// Translating storage_nodes_up (admin API context) to storage_nodes_ok (metrics context)
+			// TODO: when releasing major release, consider renaming all the fields in the metrics to storage_nodes_up
+			storage_nodes_up: health.storage_nodes_ok,
 			partitions: health.partitions,
 			partitions_quorum: health.partitions_quorum,
 			partitions_all_ok: health.partitions_all_ok,
@@ -157,9 +163,11 @@ impl RequestHandler for GetClusterStatisticsRequest {
 		// Gather storage node and free space statistics for current nodes
 		let layout = &garage.system.cluster_layout();
 		let mut node_partition_count = HashMap::<Uuid, u64>::new();
-		for short_id in layout.current().ring_assignment_data.iter() {
-			let id = layout.current().node_id_vec[*short_id as usize];
-			*node_partition_count.entry(id).or_default() += 1;
+		if let Ok(current_layout) = layout.current() {
+			for short_id in current_layout.ring_assignment_data.iter() {
+				let id = current_layout.node_id_vec[*short_id as usize];
+				*node_partition_count.entry(id).or_default() += 1;
+			}
 		}
 		let node_info = garage
 			.system
@@ -172,7 +180,11 @@ impl RequestHandler for GetClusterStatisticsRequest {
 		for (id, parts) in node_partition_count.iter() {
 			let info = node_info.get(id);
 			let status = info.map(|x| &x.status);
-			let role = layout.current().roles.get(id).and_then(|x| x.0.as_ref());
+			let role = layout
+				.current()
+				.ok()
+				.and_then(|l| l.roles.get(id))
+				.and_then(|x| x.0.as_ref());
 			let hostname = status.and_then(|x| x.hostname.as_deref()).unwrap_or("?");
 			let zone = role.map(|x| x.zone.as_str()).unwrap_or("?");
 			let capacity = role

src/api/admin/error.rs

@@ -1,8 +1,8 @@
 use std::convert::TryFrom;
 
-use err_derive::Error;
 use hyper::header::HeaderValue;
 use hyper::{HeaderMap, StatusCode};
+use thiserror::Error;
 
 pub use garage_model::helper::error::Error as HelperError;
 
@@ -16,36 +16,33 @@ use garage_api_common::helpers::*;
 /// Errors of this crate
 #[derive(Debug, Error)]
 pub enum Error {
-	#[error(display = "{}", _0)]
+	#[error("{0}")]
 	/// Error from common error
-	Common(#[error(source)] CommonError),
+	Common(#[from] CommonError),
 
 	// Category: cannot process
 	/// The admin API token does not exist
-	#[error(display = "Admin token not found: {}", _0)]
+	#[error("Admin token not found: {0}")]
 	NoSuchAdminToken(String),
 
 	/// The API access key does not exist
-	#[error(display = "Access key not found: {}", _0)]
+	#[error("Access key not found: {0}")]
 	NoSuchAccessKey(String),
 
 	/// The requested block does not exist
-	#[error(display = "Block not found: {}", _0)]
+	#[error("Block not found: {0}")]
 	NoSuchBlock(String),
 
 	/// The requested worker does not exist
-	#[error(display = "Worker not found: {}", _0)]
+	#[error("Worker not found: {0}")]
 	NoSuchWorker(u64),
 
 	/// The object requested don't exists
-	#[error(display = "Key not found")]
+	#[error("Key not found")]
 	NoSuchKey,
 
 	/// In Import key, the key already exists
-	#[error(
-		display = "Key {} already exists in data store. Even if it is deleted, we can't let you create a new key with the same ID. Sorry.",
-		_0
-	)]
+	#[error("Key {0} already exists in data store. Even if it is deleted, we can't let you create a new key with the same ID. Sorry.")]
 	KeyAlreadyExists(String),
 }
 
@@ -95,6 +92,7 @@ impl ApiError for Error {
 	fn add_http_headers(&self, header_map: &mut HeaderMap<HeaderValue>) {
 		use hyper::header;
 		header_map.append(header::CONTENT_TYPE, "application/json".parse().unwrap());
+		header_map.append(header::ACCESS_CONTROL_ALLOW_ORIGIN, "*".parse().unwrap());
 	}
 
 	fn http_body(&self, garage_region: &str, path: &str) -> ErrorBody {

src/api/admin/key.rs

@@ -230,9 +230,18 @@ async fn key_info_results(
 		},
 		buckets: relevant_buckets
 			.into_values()
-			.map(|bucket| {
+			.filter_map(|bucket| {
 				let state = bucket.state.as_option().unwrap();
-				KeyInfoBucketResponse {
+				let permissions = key_state
+					.authorized_buckets
+					.get(&bucket.id)
+					.filter(|p| p.is_any())
+					.map(|p| ApiBucketKeyPerm {
+						read: p.allow_read,
+						write: p.allow_write,
+						owner: p.allow_owner,
+					})?;
+				Some(KeyInfoBucketResponse {
 					id: hex::encode(bucket.id),
 					global_aliases: state
 						.aliases
@@ -248,16 +257,8 @@ async fn key_info_results(
 						.filter(|((k, _), _, a)| *a && *k == key.key_id)
 						.map(|((_, n), _, _)| n.to_string())
 						.collect::<Vec<_>>(),
-					permissions: key_state
-						.authorized_buckets
-						.get(&bucket.id)
-						.map(|p| ApiBucketKeyPerm {
-							read: p.allow_read,
-							write: p.allow_write,
-							owner: p.allow_owner,
-						})
-						.unwrap_or_default(),
-				}
+					permissions,
+				})
 			})
 			.collect::<Vec<_>>(),
 	};

src/api/admin/macros.rs

@@ -74,16 +74,18 @@ macro_rules! admin_endpoints {
                 type Response = AdminApiResponse;
 
                 async fn handle(self, garage: &Arc<Garage>, admin: &Admin) -> Result<AdminApiResponse, Error> {
-                    Ok(match self {
+                    match self {
                         $(
-                            AdminApiRequest::$special_endpoint(_) => panic!(
-                                concat!(stringify!($special_endpoint), " needs to go through a special handler")
+                            AdminApiRequest::$special_endpoint(_) => Err(
+                                Error::Common(CommonError::BadRequest(
+                                    concat!(stringify!($special_endpoint), " cannot be used outside of the HTTP Admin API").into()
+                                ))
                             ),
                         )*
                         $(
-                            AdminApiRequest::$endpoint(req) => AdminApiResponse::$endpoint(req.handle(garage, admin).await?),
+                            AdminApiRequest::$endpoint(req) => Ok(AdminApiResponse::$endpoint(req.handle(garage, admin).await?)),
                         )*
-                    })
+                    }
                 }
             }
         }

src/api/admin/node.rs

@@ -106,17 +106,17 @@ impl RequestHandler for LocalGetNodeStatisticsRequest {
 
 		// Gather block manager statistics
 		writeln!(&mut ret, "\nBlock manager stats:").unwrap();
-		let rc_len = garage.block_manager.rc_len()?.to_string();
+		let rc_len = garage.block_manager.rc_approximate_len()?.to_string();
 
 		ret += &format_table_to_string(vec![
 			format!("  number of RC entries:\t{} (~= number of blocks)", rc_len),
 			format!(
 				"  resync queue length:\t{}",
-				garage.block_manager.resync.queue_len()?
+				garage.block_manager.resync.queue_approximate_len()?
 			),
 			format!(
 				"  blocks with resync errors:\t{}",
-				garage.block_manager.resync.errors_len()?
+				garage.block_manager.resync.errors_approximate_len()?
 			),
 		]);
 
@@ -129,16 +129,21 @@ where
 	F: TableSchema + 'static,
 	R: TableReplication + 'static,
 {
-	let data_len = t.data.store.len().map_err(GarageError::from)?.to_string();
-	let mkl_len = t.merkle_updater.merkle_tree_len()?.to_string();
+	let data_len = t
+		.data
+		.store
+		.approximate_len()
+		.map_err(GarageError::from)?
+		.to_string();
+	let mkl_len = t.merkle_updater.merkle_tree_approximate_len()?.to_string();
 
 	Ok(format!(
 		"  {}\t{}\t{}\t{}\t{}\t{}",
 		F::TABLE_NAME,
 		data_len,
 		mkl_len,
-		t.merkle_updater.todo_len()?,
-		t.data.insert_queue_len()?,
-		t.data.gc_todo_len()?
+		t.merkle_updater.todo_approximate_len()?,
+		t.data.insert_queue_approximate_len()?,
+		t.data.gc_todo_approximate_len()?
 	))
 }

src/api/admin/openapi.rs

@@ -1,7 +1,8 @@
 #![allow(dead_code)]
 #![allow(non_snake_case)]
 
-use utoipa::{Modify, OpenApi};
+use serde::{Deserialize, Serialize};
+use utoipa::{Modify, OpenApi, ToSchema};
 
 use crate::api::*;
 
@@ -46,9 +47,7 @@ a static website for the requested domain. This is used by reverse proxies such
 as Caddy or Tricot, to avoid requesting TLS certificates for domain names that
 do not correspond to an actual website.
     ",
-    params(
-        ("domain", description = "The domain name to check for"),
-    ),
+    params(CheckDomainRequest),
     security(()),
 	responses(
             (status = 200, description = "The domain name redirects to a static website bucket"),
@@ -167,9 +166,7 @@ fn CreateAdminToken() -> () {}
 Updates information about the specified admin API token.
     ",
     request_body = UpdateAdminTokenRequestBody,
-    params(
-        ("id", description = "Admin API token ID"),
-    ),
+    params(UpdateAdminTokenRequest),
 	responses(
             (status = 200, description = "Admin token has been updated", body = UpdateAdminTokenResponse),
             (status = 500, description = "Internal server error")
@@ -181,9 +178,7 @@ fn UpdateAdminToken() -> () {}
     path = "/v2/DeleteAdminToken",
     tag = "Admin API token",
     description = "Delete an admin API token from the cluster, revoking all its permissions.",
-    params(
-        ("id", description = "Admin API token ID"),
-    ),
+    params(DeleteAdminTokenRequest),
 	responses(
             (status = 200, description = "Admin token has been deleted"),
             (status = 500, description = "Internal server error")
@@ -191,6 +186,19 @@ fn UpdateAdminToken() -> () {}
 )]
 fn DeleteAdminToken() -> () {}
 
+#[utoipa::path(get,
+    path = "/v2/GetCurrentAdminTokenInfo",
+    tag = "Admin API token",
+    description = "
+Return information about the calling admin API token.
+    ",
+    responses(
+            (status = 200, description = "Information about the admin token", body = GetCurrentAdminTokenInfoResponse),
+            (status = 500, description = "Internal server error")
+    ),
+)]
+fn GetCurrentAdminTokenInfo() -> () {}
+
 // **********************************************
 //      Layout operations
 // **********************************************
@@ -239,7 +247,7 @@ For example to declare 100GB, you must set `capacity: 100000000000`.
 Garage uses internally the International System of Units (SI), it assumes that 1kB = 1000 bytes, and displays storage as kB, MB, GB (and not KiB, MiB, GiB that assume 1KiB = 1024 bytes).
     ",
     request_body(
-        content=UpdateClusterLayoutRequest,
+        content=UpdateClusterLayoutRequestOpenapi,
         description="
 To add a new node to the layout or to change the configuration of an existing node, simply set the values you want (`zone`, `capacity`, and `tags`).
 To remove a node, simply pass the `remove: true` field.
@@ -255,6 +263,48 @@ Contrary to the CLI that may update only a subset of the fields capacity, zone a
 )]
 fn UpdateClusterLayout() -> () {}
 
+// Hack: we cannot use the UpdateClusterLayoutRequest from api.rs,
+// as it contains (via NodeRoleChange) an untagged enum flattenned into
+// a struct, which breaks the openapi generator.
+// See issue #1249.
+// Instead, we use a rewritten version of the NodeRoleChange struct where
+// the struct fields are distributed into the enum variants (this is an equivalent
+// representation, but this way we avoid having to rewrite all uses of the original
+// struct in the Garage codebase).
+#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)]
+#[schema(as = UpdateClusterLayoutRequest)]
+pub struct UpdateClusterLayoutRequestOpenapi {
+	/// New node roles to assign or remove in the cluster layout
+	#[serde(default)]
+	pub roles: Vec<NodeRoleChangeOpenapi>,
+	/// New layout computation parameters to use
+	#[serde(default)]
+	pub parameters: Option<LayoutParameters>,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)]
+#[schema(as = NodeRoleChangeRequest)]
+#[serde(untagged)]
+pub enum NodeRoleChangeOpenapi {
+	#[serde(rename_all = "camelCase")]
+	Remove {
+		/// ID of the node for which this change applies
+		id: String,
+		/// Set `remove` to `true` to remove the node from the layout
+		remove: bool,
+	},
+	Update(NodeRoleUpdate),
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)]
+#[serde(rename_all = "camelCase")]
+pub struct NodeRoleUpdate {
+	/// ID of the node for which this change applies
+	id: String,
+	#[serde(flatten)]
+	role: NodeAssignedRole,
+}
+
 #[utoipa::path(post,
     path = "/v2/PreviewClusterLayoutChanges",
     tag = "Cluster layout",
@@ -378,9 +428,7 @@ Updates information about the specified API access key.
 *Note: the secret key is not returned in the response, `null` is sent instead.*
     ",
     request_body = UpdateKeyRequestBody,
-    params(
-        ("id", description = "Access key ID"),
-    ),
+    params(UpdateKeyRequest),
 	responses(
             (status = 200, description = "Access key has been updated", body = UpdateKeyResponse),
             (status = 500, description = "Internal server error")
@@ -392,9 +440,7 @@ fn UpdateKey() -> () {}
     path = "/v2/DeleteKey",
     tag = "Access key",
     description = "Delete a key from the cluster. Its access will be removed from all the buckets. Buckets are not automatically deleted and can be dangling. You should manually delete them before. ",
-    params(
-        ("id", description = "Access key ID"),
-    ),
+    params(DeleteKeyRequest),
 	responses(
             (status = 200, description = "Access key has been deleted"),
             (status = 500, description = "Internal server error")
@@ -465,9 +511,7 @@ In `quotas`: new values of `maxSize` and `maxObjects` must both be specified, or
 to remove the quotas. An absent value will be considered the same as a `null`. It is not possible
 to change only one of the two quotas.
     ",
-    params(
-        ("id", description = "ID of the bucket to update"),
-    ),
+    params(UpdateBucketRequest),
     request_body = UpdateBucketRequestBody,
 	responses(
             (status = 200, description = "Bucket has been updated", body = UpdateBucketResponse),
@@ -485,9 +529,7 @@ Deletes a storage bucket. A bucket cannot be deleted if it is not empty.
 
 **Warning:** this will delete all aliases associated with the bucket!
     ",
-    params(
-        ("id", description = "ID of the bucket to delete"),
-    ),
+    params(DeleteBucketRequest),
 	responses(
             (status = 200, description = "Bucket has been deleted"),
             (status = 400, description = "Bucket is not empty"),
@@ -587,7 +629,7 @@ fn DenyBucketKey() -> () {}
     path = "/v2/AddBucketAlias",
     tag = "Bucket alias",
     description = "Add an alias for the target bucket.  This can be either a global or a local alias, depending on which fields are specified.",
-    request_body = AddBucketAliasRequest,
+    request_body = BucketAliasEnumOpenapi,
 	responses(
             (status = 200, description = "Returns exhaustive information about the bucket", body = AddBucketAliasResponse),
             (status = 500, description = "Internal server error")
@@ -599,7 +641,7 @@ fn AddBucketAlias() -> () {}
     path = "/v2/RemoveBucketAlias",
     tag = "Bucket alias",
     description = "Remove an alias for the target bucket.  This can be either a global or a local alias, depending on which fields are specified.",
-    request_body = RemoveBucketAliasRequest,
+    request_body = BucketAliasEnumOpenapi,
 	responses(
             (status = 200, description = "Returns exhaustive information about the bucket", body = RemoveBucketAliasResponse),
             (status = 500, description = "Internal server error")
@@ -607,6 +649,24 @@ fn AddBucketAlias() -> () {}
 )]
 fn RemoveBucketAlias() -> () {}
 
+// Hack for issue #1249 (see UpdateClusterLayout)
+#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)]
+#[serde(untagged)]
+#[schema(as = BucketAliasEnum)]
+pub enum BucketAliasEnumOpenapi {
+	#[serde(rename_all = "camelCase")]
+	Global {
+		bucket_id: String,
+		global_alias: String,
+	},
+	#[serde(rename_all = "camelCase")]
+	Local {
+		bucket_id: String,
+		local_alias: String,
+		access_key_id: String,
+	},
+}
+
 // **********************************************
 //      Node operations
 // **********************************************
@@ -617,9 +677,7 @@ fn RemoveBucketAlias() -> () {}
     description = "
 Return information about the Garage daemon running on one or several nodes.
     ",
-    params(
-        ("node", description = "Node ID to query, or `*` for all nodes, or `self` for the node responding to the request"),
-    ),
+    params(MultiRequestQueryParams),
 	responses(
             (status = 200, description = "Responses from individual cluster nodes", body = MultiResponse<LocalGetNodeInfoResponse>),
             (status = 500, description = "Internal server error")
@@ -635,9 +693,7 @@ Fetch statistics for one or several Garage nodes.
 
 *Note: do not try to parse the `freeform` field of the response, it is given as a string specifically because its format is not stable.*
     ",
-    params(
-        ("node", description = "Node ID to query, or `*` for all nodes, or `self` for the node responding to the request"),
-    ),
+    params(MultiRequestQueryParams),
 	responses(
             (status = 200, description = "Responses from individual cluster nodes", body = MultiResponse<LocalGetNodeStatisticsResponse>),
             (status = 500, description = "Internal server error")
@@ -651,9 +707,7 @@ fn GetNodeStatistics() -> () {}
     description = "
 Instruct one or several nodes to take a snapshot of their metadata databases.
     ",
-    params(
-        ("node", description = "Node ID to query, or `*` for all nodes, or `self` for the node responding to the request"),
-    ),
+    params(MultiRequestQueryParams),
 	responses(
             (status = 200, description = "Responses from individual cluster nodes", body = MultiResponse<LocalCreateMetadataSnapshotResponse>),
             (status = 500, description = "Internal server error")
@@ -667,9 +721,7 @@ fn CreateMetadataSnapshot() -> () {}
     description = "
 Launch a repair operation on one or several cluster nodes.
     ",
-    params(
-        ("node", description = "Node ID to query, or `*` for all nodes, or `self` for the node responding to the request"),
-    ),
+    params(MultiRequestQueryParams),
     request_body = LocalLaunchRepairOperationRequest,
 	responses(
             (status = 200, description = "Responses from individual cluster nodes", body = MultiResponse<LocalLaunchRepairOperationResponse>),
@@ -688,9 +740,7 @@ fn LaunchRepairOperation() -> () {}
     description = "
 List background workers currently running on one or several cluster nodes.
     ",
-    params(
-        ("node", description = "Node ID to query, or `*` for all nodes, or `self` for the node responding to the request"),
-    ),
+    params(MultiRequestQueryParams),
     request_body = LocalListWorkersRequest,
 	responses(
             (status = 200, description = "Responses from individual cluster nodes", body = MultiResponse<LocalListWorkersResponse>),
@@ -705,9 +755,7 @@ fn ListWorkers() -> () {}
     description = "
 Get information about the specified background worker on one or several cluster nodes.
     ",
-    params(
-        ("node", description = "Node ID to query, or `*` for all nodes, or `self` for the node responding to the request"),
-    ),
+    params(MultiRequestQueryParams),
     request_body = LocalGetWorkerInfoRequest,
 	responses(
             (status = 200, description = "Responses from individual cluster nodes", body = MultiResponse<LocalGetWorkerInfoResponse>),
@@ -722,9 +770,7 @@ fn GetWorkerInfo() -> () {}
     description = "
 Fetch values of one or several worker variables, from one or several cluster nodes.
     ",
-    params(
-        ("node", description = "Node ID to query, or `*` for all nodes, or `self` for the node responding to the request"),
-    ),
+    params(MultiRequestQueryParams),
     request_body = LocalGetWorkerVariableRequest,
 	responses(
             (status = 200, description = "Responses from individual cluster nodes", body = MultiResponse<LocalGetWorkerVariableResponse>),
@@ -739,9 +785,7 @@ fn GetWorkerVariable() -> () {}
     description = "
 Set the value for a worker variable, on one or several cluster nodes.
     ",
-    params(
-        ("node", description = "Node ID to query, or `*` for all nodes, or `self` for the node responding to the request"),
-    ),
+    params(MultiRequestQueryParams),
     request_body = LocalSetWorkerVariableRequest,
 	responses(
             (status = 200, description = "Responses from individual cluster nodes", body = MultiResponse<LocalSetWorkerVariableResponse>),
@@ -760,9 +804,7 @@ fn SetWorkerVariable() -> () {}
     description = "
 List data blocks that are currently in an errored state on one or several Garage nodes.
     ",
-    params(
-        ("node", description = "Node ID to query, or `*` for all nodes, or `self` for the node responding to the request"),
-    ),
+    params(MultiRequestQueryParams),
 	responses(
             (status = 200, description = "Responses from individual cluster nodes", body = MultiResponse<LocalListBlockErrorsResponse>),
             (status = 500, description = "Internal server error")
@@ -776,9 +818,7 @@ fn ListBlockErrors() -> () {}
     description = "
 Get detailed information about a data block stored on a Garage node, including all object versions and in-progress multipart uploads that contain a reference to this block.
     ",
-    params(
-        ("node", description = "Node ID to query, or `*` for all nodes, or `self` for the node responding to the request"),
-    ),
+    params(MultiRequestQueryParams),
     request_body = LocalGetBlockInfoRequest,
 	responses(
             (status = 200, description = "Detailed block information", body = MultiResponse<LocalGetBlockInfoResponse>),
@@ -793,9 +833,7 @@ fn GetBlockInfo() -> () {}
     description = "
 Instruct Garage node(s) to retry the resynchronization of one or several missing data block(s).
     ",
-    params(
-        ("node", description = "Node ID to query, or `*` for all nodes, or `self` for the node responding to the request"),
-    ),
+    params(MultiRequestQueryParams),
     request_body = LocalRetryBlockResyncRequest,
 	responses(
             (status = 200, description = "Responses from individual cluster nodes", body = MultiResponse<LocalRetryBlockResyncResponse>),
@@ -812,9 +850,7 @@ Purge references to one or several missing data blocks.
 
 This will remove all objects and in-progress multipart uploads that contain the specified data block(s). The objects will be permanently deleted from the buckets in which they appear. Use with caution.
     ",
-    params(
-        ("node", description = "Node ID to query, or `*` for all nodes, or `self` for the node responding to the request"),
-    ),
+    params(MultiRequestQueryParams),
     request_body = LocalPurgeBlocksRequest,
 	responses(
             (status = 200, description = "Responses from individual cluster nodes", body = MultiResponse<LocalPurgeBlocksResponse>),
@@ -843,7 +879,7 @@ impl Modify for SecurityAddon {
 #[derive(OpenApi)]
 #[openapi(
     info(
-        version = "v2.0.0",
+        version = "v2.2.0",
         title = "Garage administration API",
         description = "Administrate your Garage cluster programatically, including status, layout, keys, buckets, and maintainance tasks.
 
@@ -872,6 +908,7 @@ impl Modify for SecurityAddon {
         CreateAdminToken,
         UpdateAdminToken,
         DeleteAdminToken,
+        GetCurrentAdminTokenInfo,
         // Layout operations
         GetClusterLayout,
         GetClusterLayoutHistory,

src/api/admin/repair.rs

@@ -93,6 +93,15 @@ impl RequestHandler for LocalLaunchRepairOperationRequest {
 				info!("Repairing bucket aliases (foreground)");
 				garage.locked_helper().await.repair_aliases().await?;
 			}
+			RepairType::ClearResyncQueue => {
+				info!("Clearing resync queue (foreground)");
+				let garage = garage.clone();
+				tokio::task::spawn_blocking(move || {
+					garage.block_manager.resync.clear_resync_queue()
+				})
+				.await
+				.map_err(garage_util::error::Error::from)??;
+			}
 		}
 		Ok(LocalLaunchRepairOperationResponse)
 	}

src/api/admin/router_v2.rs

@@ -40,6 +40,7 @@ impl AdminApiRequest {
 			POST CreateAdminToken (body),
 			POST UpdateAdminToken (body_field, query::id),
 			POST DeleteAdminToken (query::id),
+			GET GetCurrentAdminTokenInfo (admin_token),
 			// Layout endpoints
 			GET GetClusterLayout (),
 			GET GetClusterLayoutHistory (),

src/api/common/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "garage_api_common"
-version = "2.0.0"
+version = "2.2.0"
 authors = ["Alex Auvolat <alex@adnab.me>"]
 edition = "2018"
 license = "AGPL-3.0"
@@ -23,7 +23,7 @@ bytes.workspace = true
 chrono.workspace = true
 crc-fast.workspace = true
 crypto-common.workspace = true
-err-derive.workspace = true
+thiserror.workspace = true
 hex.workspace = true
 hmac.workspace = true
 md-5.workspace = true

src/api/common/common_error.rs

@@ -1,7 +1,7 @@
 use std::convert::TryFrom;
 
-use err_derive::Error;
 use hyper::StatusCode;
+use thiserror::Error;
 
 use garage_util::error::Error as GarageError;
 
@@ -12,48 +12,48 @@ use garage_model::helper::error::Error as HelperError;
 pub enum CommonError {
 	// ---- INTERNAL ERRORS ----
 	/// Error related to deeper parts of Garage
-	#[error(display = "Internal error: {}", _0)]
-	InternalError(#[error(source)] GarageError),
+	#[error("Internal error: {0}")]
+	InternalError(#[from] GarageError),
 
 	/// Error related to Hyper
-	#[error(display = "Internal error (Hyper error): {}", _0)]
-	Hyper(#[error(source)] hyper::Error),
+	#[error("Internal error (Hyper error): {0}")]
+	Hyper(#[from] hyper::Error),
 
 	/// Error related to HTTP
-	#[error(display = "Internal error (HTTP error): {}", _0)]
-	Http(#[error(source)] http::Error),
+	#[error("Internal error (HTTP error): {0}")]
+	Http(#[from] http::Error),
 
 	// ---- GENERIC CLIENT ERRORS ----
 	/// Proper authentication was not provided
-	#[error(display = "Forbidden: {}", _0)]
+	#[error("Forbidden: {0}")]
 	Forbidden(String),
 
 	/// Generic bad request response with custom message
-	#[error(display = "Bad request: {}", _0)]
+	#[error("Bad request: {0}")]
 	BadRequest(String),
 
 	/// The client sent a header with invalid value
-	#[error(display = "Invalid header value: {}", _0)]
-	InvalidHeader(#[error(source)] hyper::header::ToStrError),
+	#[error("Invalid header value: {0}")]
+	InvalidHeader(#[from] hyper::header::ToStrError),
 
 	// ---- SPECIFIC ERROR CONDITIONS ----
 	// These have to be error codes referenced in the S3 spec here:
 	// https://docs.aws.amazon.com/AmazonS3/latest/API/ErrorResponses.html#ErrorCodeList
 	/// The bucket requested don't exists
-	#[error(display = "Bucket not found: {}", _0)]
+	#[error("Bucket not found: {0}")]
 	NoSuchBucket(String),
 
 	/// Tried to create a bucket that already exist
-	#[error(display = "Bucket already exists")]
+	#[error("Bucket already exists")]
 	BucketAlreadyExists,
 
 	/// Tried to delete a non-empty bucket
-	#[error(display = "Tried to delete a non-empty bucket")]
+	#[error("Tried to delete a non-empty bucket")]
 	BucketNotEmpty,
 
 	// Category: bad request
 	/// Bucket name is not valid according to AWS S3 specs
-	#[error(display = "Invalid bucket name: {}", _0)]
+	#[error("Invalid bucket name: {0}")]
 	InvalidBucketName(String),
 }
 

src/api/common/generic_server.rs

@@ -59,6 +59,12 @@ pub trait ApiHandler: Send + Sync + 'static {
 		req: Request<IncomingBody>,
 		endpoint: Self::Endpoint,
 	) -> impl Future<Output = Result<Response<BoxBody<Self::Error>>, Self::Error>> + Send;
+
+	/// Returns the key id used to authenticate this request. The ID returned must be safe to
+	/// log.
+	fn key_id_from_request(&self, _req: &Request<IncomingBody>) -> Option<String> {
+		None
+	}
 }
 
 pub struct ApiServer<A: ApiHandler> {
@@ -143,19 +149,20 @@ impl<A: ApiHandler> ApiServer<A> {
 	) -> Result<Response<BoxBody<A::Error>>, http::Error> {
 		let uri = req.uri().clone();
 
-		if let Ok(forwarded_for_ip_addr) =
+		let source = if let Ok(forwarded_for_ip_addr) =
 			forwarded_headers::handle_forwarded_for_headers(req.headers())
 		{
-			info!(
-				"{} (via {}) {} {}",
-				forwarded_for_ip_addr,
-				addr,
-				req.method(),
-				uri
-			);
+			format!("{forwarded_for_ip_addr} (via {addr})")
 		} else {
-			info!("{} {} {}", addr, req.method(), uri);
-		}
+			format!("{addr}")
+		};
+		// we only do this to log the access key, so we can discard any error
+		let key = self
+			.api_handler
+			.key_id_from_request(&req)
+			.map(|k| format!("(key {k}) "))
+			.unwrap_or_default();
+		info!("{source} {key}{} {uri}", req.method());
 		debug!("{:?}", req);
 
 		let tracer = opentelemetry::global::tracer("garage");
@@ -344,7 +351,11 @@ where
 
 	while !*must_exit.borrow() {
 		let (stream, client_addr) = tokio::select! {
-			acc = listener.accept() => acc?,
+			acc = listener.accept() => match acc {
+							Ok(r) => r,
+							Err(e) if e.kind() == std::io::ErrorKind::ConnectionAborted => continue,
+							Err(e) => return Err(e.into()),
+						},
 			_ = must_exit.changed() => continue,
 		};
 

src/api/common/router_macros.rs

@@ -83,6 +83,21 @@ macro_rules! router_match {
             parse_json_body::< [<$api Request>], _, Error>($req).await?
         })
     }};
+   (@@gen_parse_request $api:ident, (admin_token), $query: expr, $req:expr) => {{
+        paste!({
+            let auth_header = $req.headers()
+                .get(hyper::header::AUTHORIZATION)
+                .ok_or_else(|| Error::bad_request("Missing Authorization header"))?
+                .to_str()
+                .map_err(|_| Error::bad_request("Invalid Authorization header"))?;
+
+            let admin_token = auth_header.strip_prefix("Bearer ")
+                .ok_or_else(|| Error::bad_request("Authorization header must be Bearer token"))?
+                .to_string();
+
+            [< $api Request >] { admin_token }
+        })
+    }};
     (@@gen_parse_request $api:ident, (body_field, $($conv:ident $(($conv_arg:expr))? :: $param:ident),*), $query: expr, $req:expr)
         =>
     {{

src/api/common/signature/error.rs

@@ -1,4 +1,4 @@
-use err_derive::Error;
+use thiserror::Error;
 
 use crate::common_error::CommonError;
 pub use crate::common_error::{CommonErrorDerivative, OkOrBadRequest, OkOrInternalError};
@@ -6,21 +6,21 @@ pub use crate::common_error::{CommonErrorDerivative, OkOrBadRequest, OkOrInterna
 /// Errors of this crate
 #[derive(Debug, Error)]
 pub enum Error {
-	#[error(display = "{}", _0)]
+	#[error("{0}")]
 	/// Error from common error
 	Common(CommonError),
 
 	/// Authorization Header Malformed
-	#[error(display = "Authorization header malformed, unexpected scope: {}", _0)]
+	#[error("Authorization header malformed, unexpected scope: {0}")]
 	AuthorizationHeaderMalformed(String),
 
 	// Category: bad request
 	/// The request contained an invalid UTF-8 sequence in its path or in other parameters
-	#[error(display = "Invalid UTF-8: {}", _0)]
-	InvalidUtf8Str(#[error(source)] std::str::Utf8Error),
+	#[error("Invalid UTF-8: {0}")]
+	InvalidUtf8Str(#[from] std::str::Utf8Error),
 
 	/// The provided digest (checksum) value was invalid
-	#[error(display = "Invalid digest: {}", _0)]
+	#[error("Invalid digest: {0}")]
 	InvalidDigest(String),
 }
 

src/api/common/signature/payload.rs

@@ -105,7 +105,7 @@ fn check_standard_signature(
 	// Verify that all necessary request headers are included in signed_headers
 	// The following must be included for all signatures:
 	// - the Host header (mandatory)
-	// - all x-amz-* headers used in the request
+	// - all x-amz-* headers used in the request (except x-amz-content-sha256)
 	// AWS also indicates that the Content-Type header should be signed if
 	// it is used, but Minio client doesn't sign it so we don't check it for compatibility.
 	let signed_headers = split_signed_headers(&authorization)?;
@@ -152,7 +152,7 @@ fn check_presigned_signature(
 	// Verify that all necessary request headers are included in signed_headers
 	// For AWSv4 pre-signed URLs, the following must be included:
 	// - the Host header (mandatory)
-	// - all x-amz-* headers used in the request
+	// - all x-amz-* headers used in the request (except x-amz-content-sha256)
 	let signed_headers = split_signed_headers(&authorization)?;
 	verify_signed_headers(request.headers(), &signed_headers)?;
 
@@ -269,7 +269,9 @@ fn verify_signed_headers(headers: &HeaderMap, signed_headers: &[HeaderName]) ->
 		return Err(Error::bad_request("Header `Host` should be signed"));
 	}
 	for (name, _) in headers.iter() {
-		if name.as_str().starts_with("x-amz-") {
+		// Enforce signature of all x-amz-* headers, except x-amz-content-sh256
+		// because it is included in the canonical request in all cases
+		if name.as_str().starts_with("x-amz-") && name != X_AMZ_CONTENT_SHA256 {
 			if !signed_headers.contains(name) {
 				return Err(Error::bad_request(format!(
 					"Header `{}` should be signed",
@@ -424,7 +426,7 @@ pub fn verify_v4(
 // ============ Authorization header, or X-Amz-* query params =========
 
 pub struct Authorization {
-	key_id: String,
+	pub key_id: String,
 	scope: String,
 	signed_headers: String,
 	signature: String,
@@ -433,7 +435,7 @@ pub struct Authorization {
 }
 
 impl Authorization {
-	fn parse_header(headers: &HeaderMap) -> Result<Self, Error> {
+	pub fn parse_header(headers: &HeaderMap) -> Result<Self, Error> {
 		let authorization = headers
 			.get(AUTHORIZATION)
 			.ok_or_bad_request("Missing authorization header")?
@@ -475,8 +477,7 @@ impl Authorization {
 
 		let date = headers
 			.get(X_AMZ_DATE)
-			.ok_or_bad_request("Missing X-Amz-Date field")
-			.map_err(Error::from)?
+			.ok_or_bad_request("Missing X-Amz-Date field")?
 			.to_str()?;
 		let date = parse_date(date)?;
 

src/api/k2v/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "garage_api_k2v"
-version = "2.0.0"
+version = "2.2.0"
 authors = ["Alex Auvolat <alex@adnab.me>"]
 edition = "2018"
 license = "AGPL-3.0"
@@ -20,7 +20,7 @@ garage_util = { workspace = true, features = [ "k2v" ] }
 garage_api_common.workspace = true
 
 base64.workspace = true
-err-derive.workspace = true
+thiserror.workspace = true
 tracing.workspace = true
 
 futures.workspace = true

src/api/k2v/api_server.rs

@@ -171,6 +171,12 @@ impl ApiHandler for K2VApiServer {
 
 		Ok(resp_ok)
 	}
+
+	fn key_id_from_request(&self, req: &Request<IncomingBody>) -> Option<String> {
+		garage_api_common::signature::payload::Authorization::parse_header(req.headers())
+			.map(|auth| auth.key_id)
+			.ok()
+	}
 }
 
 impl ApiEndpoint for K2VApiEndpoint {

src/api/k2v/error.rs

@@ -1,6 +1,6 @@
-use err_derive::Error;
 use hyper::header::HeaderValue;
 use hyper::{HeaderMap, StatusCode};
+use thiserror::Error;
 
 pub(crate) use garage_api_common::common_error::pass_helper_error;
 use garage_api_common::common_error::{commonErrorDerivative, CommonError};
@@ -14,38 +14,38 @@ use garage_api_common::signature::error::Error as SignatureError;
 /// Errors of this crate
 #[derive(Debug, Error)]
 pub enum Error {
-	#[error(display = "{}", _0)]
+	#[error("{0}")]
 	/// Error from common error
-	Common(#[error(source)] CommonError),
+	Common(#[from] CommonError),
 
 	// Category: cannot process
 	/// Authorization Header Malformed
-	#[error(display = "Authorization header malformed, unexpected scope: {}", _0)]
+	#[error("Authorization header malformed, unexpected scope: {0}")]
 	AuthorizationHeaderMalformed(String),
 
 	/// The provided digest (checksum) value was invalid
-	#[error(display = "Invalid digest: {}", _0)]
+	#[error("Invalid digest: {0}")]
 	InvalidDigest(String),
 
 	/// The object requested don't exists
-	#[error(display = "Key not found")]
+	#[error("Key not found")]
 	NoSuchKey,
 
 	/// Some base64 encoded data was badly encoded
-	#[error(display = "Invalid base64: {}", _0)]
-	InvalidBase64(#[error(source)] base64::DecodeError),
+	#[error("Invalid base64: {0}")]
+	InvalidBase64(#[from] base64::DecodeError),
 
 	/// Invalid causality token
-	#[error(display = "Invalid causality token")]
+	#[error("Invalid causality token")]
 	InvalidCausalityToken,
 
 	/// The client asked for an invalid return format (invalid Accept header)
-	#[error(display = "Not acceptable: {}", _0)]
+	#[error("Not acceptable: {0}")]
 	NotAcceptable(String),
 
 	/// The request contained an invalid UTF-8 sequence in its path or in other parameters
-	#[error(display = "Invalid UTF-8: {}", _0)]
-	InvalidUtf8Str(#[error(source)] std::str::Utf8Error),
+	#[error("Invalid UTF-8: {0}")]
+	InvalidUtf8Str(#[from] std::str::Utf8Error),
 }
 
 commonErrorDerivative!(Error);
@@ -99,6 +99,7 @@ impl ApiError for Error {
 	fn add_http_headers(&self, header_map: &mut HeaderMap<HeaderValue>) {
 		use hyper::header;
 		header_map.append(header::CONTENT_TYPE, "application/json".parse().unwrap());
+		header_map.append(header::ACCESS_CONTROL_ALLOW_ORIGIN, "*".parse().unwrap());
 	}
 
 	fn http_body(&self, garage_region: &str, path: &str) -> ErrorBody {

src/api/k2v/index.rs

@@ -28,7 +28,7 @@ pub async fn handle_read_index(
 	let node_id_vec = garage
 		.system
 		.cluster_layout()
-		.all_nongateway_nodes()
+		.all_nongateway_nodes()?
 		.to_vec();
 
 	let (partition_keys, more, next_start) = read_range(
@@ -66,7 +66,7 @@ pub async fn handle_read_index(
 					bytes: *vals.get(&s_bytes).unwrap_or(&0),
 				}
 			})
-			.collect::<Vec<_>>(),
+			.collect(),
 		more,
 		next_start,
 	};

src/api/s3/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "garage_api_s3"
-version = "2.0.0"
+version = "2.2.0"
 authors = ["Alex Auvolat <alex@adnab.me>"]
 edition = "2018"
 license = "AGPL-3.0"
@@ -28,7 +28,7 @@ base64.workspace = true
 bytes.workspace = true
 chrono.workspace = true
 crc-fast.workspace = true
-err-derive.workspace = true
+thiserror.workspace = true
 hex.workspace = true
 hmac.workspace = true
 tracing.workspace = true

src/api/s3/api_server.rs

@@ -340,6 +340,12 @@ impl ApiHandler for S3ApiServer {
 
 		Ok(resp_ok)
 	}
+
+	fn key_id_from_request(&self, req: &Request<IncomingBody>) -> Option<String> {
+		garage_api_common::signature::payload::Authorization::parse_header(req.headers())
+			.map(|auth| auth.key_id)
+			.ok()
+	}
 }
 
 impl ApiEndpoint for S3ApiEndpoint {

src/api/s3/cors.rs

@@ -29,7 +29,7 @@ pub async fn handle_get_cors(ctx: ReqCtx) -> Result<Response<ResBody>, Error> {
 			.body(string_body(xml))?)
 	} else {
 		Ok(Response::builder()
-			.status(StatusCode::NO_CONTENT)
+			.status(StatusCode::NOT_FOUND)
 			.body(empty_body())?)
 	}
 }

src/api/s3/error.rs

@@ -1,8 +1,8 @@
 use std::convert::TryInto;
 
-use err_derive::Error;
 use hyper::header::HeaderValue;
 use hyper::{HeaderMap, StatusCode};
+use thiserror::Error;
 
 use garage_model::helper::error::Error as HelperError;
 
@@ -25,67 +25,67 @@ use crate::xml as s3_xml;
 /// Errors of this crate
 #[derive(Debug, Error)]
 pub enum Error {
-	#[error(display = "{}", _0)]
+	#[error("{0}")]
 	/// Error from common error
-	Common(#[error(source)] CommonError),
+	Common(#[from] CommonError),
 
 	// Category: cannot process
 	/// Authorization Header Malformed
-	#[error(display = "Authorization header malformed, unexpected scope: {}", _0)]
+	#[error("Authorization header malformed, unexpected scope: {0}")]
 	AuthorizationHeaderMalformed(String),
 
 	/// The object requested don't exists
-	#[error(display = "Key not found")]
+	#[error("Key not found")]
 	NoSuchKey,
 
 	/// The multipart upload requested don't exists
-	#[error(display = "Upload not found")]
+	#[error("Upload not found")]
 	NoSuchUpload,
 
 	/// Precondition failed (e.g. x-amz-copy-source-if-match)
-	#[error(display = "At least one of the preconditions you specified did not hold")]
+	#[error("At least one of the preconditions you specified did not hold")]
 	PreconditionFailed,
 
 	/// Parts specified in CMU request do not match parts actually uploaded
-	#[error(display = "Parts given to CompleteMultipartUpload do not match uploaded parts")]
+	#[error("Parts given to CompleteMultipartUpload do not match uploaded parts")]
 	InvalidPart,
 
 	/// Parts given to CompleteMultipartUpload were not in ascending order
-	#[error(display = "Parts given to CompleteMultipartUpload were not in ascending order")]
+	#[error("Parts given to CompleteMultipartUpload were not in ascending order")]
 	InvalidPartOrder,
 
 	/// In CompleteMultipartUpload: not enough data
 	/// (here we are more lenient than AWS S3)
-	#[error(display = "Proposed upload is smaller than the minimum allowed object size")]
+	#[error("Proposed upload is smaller than the minimum allowed object size")]
 	EntityTooSmall,
 
 	// Category: bad request
 	/// The request contained an invalid UTF-8 sequence in its path or in other parameters
-	#[error(display = "Invalid UTF-8: {}", _0)]
-	InvalidUtf8Str(#[error(source)] std::str::Utf8Error),
+	#[error("Invalid UTF-8: {0}")]
+	InvalidUtf8Str(#[from] std::str::Utf8Error),
 
 	/// The request used an invalid path
-	#[error(display = "Invalid UTF-8: {}", _0)]
-	InvalidUtf8String(#[error(source)] std::string::FromUtf8Error),
+	#[error("Invalid UTF-8: {0}")]
+	InvalidUtf8String(#[from] std::string::FromUtf8Error),
 
 	/// The client sent invalid XML data
-	#[error(display = "Invalid XML: {}", _0)]
+	#[error("Invalid XML: {0}")]
 	InvalidXml(String),
 
 	/// The client sent a range header with invalid value
-	#[error(display = "Invalid HTTP range: {:?}", _0)]
-	InvalidRange(#[error(from)] (http_range::HttpRangeParseError, u64)),
+	#[error("Invalid HTTP range: {0:?}")]
+	InvalidRange((http_range::HttpRangeParseError, u64)),
 
 	/// The client sent a range header with invalid value
-	#[error(display = "Invalid encryption algorithm: {:?}, should be AES256", _0)]
+	#[error("Invalid encryption algorithm: {0:?}, should be AES256")]
 	InvalidEncryptionAlgorithm(String),
 
 	/// The provided digest (checksum) value was invalid
-	#[error(display = "Invalid digest: {}", _0)]
+	#[error("Invalid digest: {0}")]
 	InvalidDigest(String),
 
 	/// The client sent a request for an action not supported by garage
-	#[error(display = "Unimplemented action: {}", _0)]
+	#[error("Unimplemented action: {0}")]
 	NotImplemented(String),
 }
 
@@ -99,6 +99,12 @@ impl From<HelperError> for Error {
 	}
 }
 
+impl From<(http_range::HttpRangeParseError, u64)> for Error {
+	fn from(err: (http_range::HttpRangeParseError, u64)) -> Error {
+		Error::InvalidRange(err)
+	}
+}
+
 impl From<roxmltree::Error> for Error {
 	fn from(err: roxmltree::Error) -> Self {
 		Self::InvalidXml(format!("{}", err))
@@ -176,6 +182,7 @@ impl ApiError for Error {
 		use hyper::header;
 
 		header_map.append(header::CONTENT_TYPE, "application/xml".parse().unwrap());
+		header_map.append(header::ACCESS_CONTROL_ALLOW_ORIGIN, "*".parse().unwrap());
 
 		#[allow(clippy::single_match)]
 		match self {

src/api/s3/get.rs

@@ -853,7 +853,9 @@ impl PreconditionHeaders {
 	}
 
 	fn check(&self, v: &ObjectVersion, etag: &str) -> Result<Option<StatusCode>, Error> {
-		let v_date = UNIX_EPOCH + Duration::from_millis(v.timestamp);
+		// we store date with ms precision, but headers are precise to the second: truncate
+		// the timestamp to handle the same-second edge case
+		let v_date = UNIX_EPOCH + Duration::from_secs(v.timestamp / 1000);
 
 		// Implemented from https://datatracker.ietf.org/doc/html/rfc7232#section-6
 

src/api/s3/multipart.rs

@@ -5,6 +5,7 @@ use std::sync::Arc;
 use base64::prelude::*;
 use crc_fast::{CrcAlgorithm, Digest as CrcDigest};
 use futures::prelude::*;
+use http::StatusCode;
 use hyper::{header::HeaderValue, HeaderMap, Request, Response};
 use md5::{Digest, Md5};
 use sha1::Sha1;
@@ -302,8 +303,11 @@ pub async fn handle_complete_multipart_upload(
 	let body = req_body.collect().await?;
 
 	let body_xml = roxmltree::Document::parse(std::str::from_utf8(&body)?)?;
-	let body_list_of_parts = parse_complete_multipart_upload_body(&body_xml)
-		.ok_or_bad_request("Invalid CompleteMultipartUpload XML")?;
+	let body_list_of_parts =
+		parse_complete_multipart_upload_body(&body_xml).ok_or_bad_request(format!(
+			"Invalid CompleteMultipartUpload XML:\n{}",
+			String::from_utf8_lossy(&body)
+		))?;
 	debug!(
 		"CompleteMultipartUpload list of parts: {:?}",
 		body_list_of_parts
@@ -548,7 +552,9 @@ pub async fn handle_abort_multipart_upload(
 	let final_object = Object::new(*bucket_id, key.to_string(), vec![object_version]);
 	garage.object_table.insert(&final_object).await?;
 
-	Ok(Response::new(empty_body()))
+	Ok(Response::builder()
+		.status(StatusCode::NO_CONTENT)
+		.body(empty_body())?)
 }
 
 // ======== helpers ============
@@ -600,6 +606,32 @@ struct CompleteMultipartUploadPart {
 	checksum: Option<ChecksumValue>,
 }
 
+macro_rules! extract_checksum_from {
+	($node:ident { $($name:expr => $variant:ident),* $(,)? }) => {
+		if false { None }
+		$(
+			else if let Some(node) = $node.children().find(|e| e.has_tag_name($name)) {
+				match node.last_child().map(|x| x.text()) {
+					// Child is text but empty post-trim, ignore it.
+					Some(Some(text)) if text.trim().is_empty() => None,
+
+					// Child is non-empty text, parse it.
+					Some(Some(text)) => Some(ChecksumValue::$variant(
+						BASE64_STANDARD.decode(text).ok()?[..].try_into().ok()?
+					)),
+
+					// Child is not text, reject it.
+					Some(None) => return None,
+
+					// No child, ignore it.
+					None => None,
+				}
+			}
+		)*
+		else { None }
+	}
+}
+
 fn parse_complete_multipart_upload_body(
 	xml: &roxmltree::Document,
 ) -> Option<Vec<CompleteMultipartUploadPart>> {
@@ -623,46 +655,15 @@ fn parse_complete_multipart_upload_body(
 				.children()
 				.find(|e| e.has_tag_name("PartNumber"))?
 				.text()?;
-			let checksum = if let Some(crc32) =
-				item.children().find(|e| e.has_tag_name("ChecksumCRC32"))
-			{
-				Some(ChecksumValue::Crc32(
-					BASE64_STANDARD.decode(crc32.text()?).ok()?[..]
-						.try_into()
-						.ok()?,
-				))
-			} else if let Some(crc32c) = item.children().find(|e| e.has_tag_name("ChecksumCRC32C"))
-			{
-				Some(ChecksumValue::Crc32c(
-					BASE64_STANDARD.decode(crc32c.text()?).ok()?[..]
-						.try_into()
-						.ok()?,
-				))
-			} else if let Some(crc64nvme) = item
-				.children()
-				.find(|e| e.has_tag_name("ChecksumCRC64NVME"))
-			{
-				Some(ChecksumValue::Crc64Nvme(
-					BASE64_STANDARD.decode(crc64nvme.text()?).ok()?[..]
-						.try_into()
-						.ok()?,
-				))
-			} else if let Some(sha1) = item.children().find(|e| e.has_tag_name("ChecksumSHA1")) {
-				Some(ChecksumValue::Sha1(
-					BASE64_STANDARD.decode(sha1.text()?).ok()?[..]
-						.try_into()
-						.ok()?,
-				))
-			} else if let Some(sha256) = item.children().find(|e| e.has_tag_name("ChecksumSHA256"))
-			{
-				Some(ChecksumValue::Sha256(
-					BASE64_STANDARD.decode(sha256.text()?).ok()?[..]
-						.try_into()
-						.ok()?,
-				))
-			} else {
-				None
-			};
+
+			let checksum = extract_checksum_from!(item {
+				"ChecksumCRC32" => Crc32,
+				"ChecksumCRC32C" => Crc32c,
+				"ChecksumCRC64NVME" => Crc64Nvme,
+				"ChecksumSHA1" => Sha1,
+				"ChecksumSHA256" => Sha256,
+			});
+
 			parts.push(CompleteMultipartUploadPart {
 				etag: etag.trim_matches('"').to_string(),
 				part_number: part_number.parse().ok()?,

src/api/s3/post_object.rs

@@ -138,10 +138,26 @@ pub async fn handle_post_object(
 
 	let mut conditions = decoded_policy.into_conditions()?;
 
+	// If there are conditions on the bucket name, check these against the actual bucket_name rather
+	// than the one in params, which is allowed to be absent.
+	if let Some(conds) = conditions.params.remove("bucket") {
+		for cond in conds {
+			let ok = match cond {
+				Operation::Equal(s) => s.as_str() == bucket_name,
+				Operation::StartsWith(s) => bucket_name.starts_with(&s),
+			};
+			if !ok {
+				return Err(Error::bad_request(
+					"Key 'bucket' has value not allowed in policy",
+				));
+			}
+		}
+	}
+
 	for (param_key, value) in params.iter() {
 		let param_key = param_key.as_str();
 		match param_key {
-			"policy" | "x-amz-signature" => (), // this is always accepted, as it's required to validate other fields
+			"policy" | "x-amz-signature" | "bucket" => (), // this is always accepted, as it's required to validate other fields
 			"content-type" => {
 				let conds = conditions.params.remove("content-type").ok_or_else(|| {
 					Error::bad_request(format!("Key '{}' is not allowed in policy", param_key))

src/api/s3/put.rs

@@ -39,8 +39,6 @@ use crate::encryption::{EncryptionParams, OekDerivationInfo};
 use crate::error::*;
 use crate::website::X_AMZ_WEBSITE_REDIRECT_LOCATION;
 
-const PUT_BLOCKS_MAX_PARALLEL: usize = 3;
-
 pub(crate) struct SaveStreamResult {
 	pub(crate) version_uuid: Uuid,
 	pub(crate) version_timestamp: u64,
@@ -507,7 +505,7 @@ pub(crate) async fn read_and_put_blocks<S: Stream<Item = Result<Bytes, Error>> +
 			};
 			let recv_next = async {
 				// If more than a maximum number of writes are in progress, don't add more for now
-				if currently_running >= PUT_BLOCKS_MAX_PARALLEL {
+				if currently_running >= ctx.garage.config.block_max_concurrent_writes_per_request {
 					futures::future::pending().await
 				} else {
 					block_rx3.recv().await

src/block/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "garage_block"
-version = "2.0.0"
+version = "2.2.0"
 authors = ["Alex Auvolat <alex@adnab.me>"]
 edition = "2018"
 license = "AGPL-3.0"

src/block/manager.rs

@@ -48,6 +48,8 @@ pub const INLINE_THRESHOLD: usize = 3072;
 // to delete the block locally.
 pub(crate) const BLOCK_GC_DELAY: Duration = Duration::from_secs(600);
 
+const BLOCK_READ_SEMAPHORE_TIMEOUT: Duration = Duration::from_secs(15);
+
 /// RPC messages used to share blocks of data between nodes
 #[derive(Debug, Serialize, Deserialize)]
 pub enum BlockRpc {
@@ -85,6 +87,7 @@ pub struct BlockManager {
 	disable_scrub: bool,
 
 	mutation_lock: Vec<Mutex<BlockManagerLocked>>,
+	read_semaphore: Semaphore,
 
 	pub rc: BlockRc,
 	pub resync: BlockResyncManager,
@@ -174,6 +177,8 @@ impl BlockManager {
 				.iter()
 				.map(|_| Mutex::new(BlockManagerLocked()))
 				.collect::<Vec<_>>(),
+
+			read_semaphore: Semaphore::new(config.block_max_concurrent_reads),
 			rc,
 			resync,
 			system,
@@ -285,7 +290,7 @@ impl BlockManager {
 		let who = self
 			.system
 			.rpc_helper()
-			.block_read_nodes_of(hash, self.system.rpc_helper());
+			.block_read_nodes_of(hash, self.system.rpc_helper())?;
 
 		for node in who.iter() {
 			let node_id = NodeID::from(*node);
@@ -341,12 +346,9 @@ impl BlockManager {
 	/// layout version only: since blocks are immutable, we don't need to
 	/// do complex logic when several layour versions are active at once,
 	/// just move them directly to the new nodes.
-	pub(crate) fn storage_nodes_of(&self, hash: &Hash) -> Vec<Uuid> {
-		self.system
-			.cluster_layout()
-			.current()
-			.nodes_of(hash)
-			.collect()
+	pub(crate) fn storage_nodes_of(&self, hash: &Hash) -> Result<Vec<Uuid>, Error> {
+		let cluster_layout = self.system.cluster_layout();
+		Ok(cluster_layout.current()?.nodes_of(hash).collect())
 	}
 
 	// ---- Public interface ----
@@ -381,7 +383,7 @@ impl BlockManager {
 		prevent_compression: bool,
 		order_tag: Option<OrderTag>,
 	) -> Result<(), Error> {
-		let who = self.storage_nodes_of(&hash);
+		let who = self.storage_nodes_of(&hash)?;
 
 		let compression_level = self.compression_level.filter(|_| !prevent_compression);
 		let (header, bytes) = DataBlock::from_buffer(data, compression_level)
@@ -419,8 +421,8 @@ impl BlockManager {
 	}
 
 	/// Get number of items in the refcount table
-	pub fn rc_len(&self) -> Result<usize, Error> {
-		Ok(self.rc.rc_table.len()?)
+	pub fn rc_approximate_len(&self) -> Result<usize, Error> {
+		Ok(self.rc.rc_table.approximate_len()?)
 	}
 
 	/// Send command to start/stop/manager scrub worker
@@ -438,7 +440,7 @@ impl BlockManager {
 
 	/// List all resync errors
 	pub fn list_resync_errors(&self) -> Result<Vec<BlockResyncErrorInfo>, Error> {
-		let mut blocks = Vec::with_capacity(self.resync.errors.len()?);
+		let mut blocks = Vec::with_capacity(self.resync.errors.approximate_len()?);
 		for ent in self.resync.errors.iter()? {
 			let (hash, cnt) = ent?;
 			let cnt = ErrorCounter::decode(&cnt);
@@ -568,9 +570,6 @@ impl BlockManager {
 			match self.find_block(hash).await {
 				Some(p) => self.read_block_from(hash, &p).await,
 				None => {
-					// Not found but maybe we should have had it ??
-					self.resync
-						.put_to_resync(hash, 2 * self.system.rpc_helper().rpc_timeout())?;
 					return Err(Error::Message(format!(
 						"block {:?} not found on node",
 						hash
@@ -592,6 +591,15 @@ impl BlockManager {
 	) -> Result<DataBlock, Error> {
 		let (header, path) = block_path.as_parts_ref();
 
+		let permit = tokio::select! {
+			sem = self.read_semaphore.acquire() => sem.ok_or_message("acquire read semaphore")?,
+			_ = tokio::time::sleep(BLOCK_READ_SEMAPHORE_TIMEOUT) => {
+				self.metrics.block_read_semaphore_timeouts.add(1);
+				debug!("read block {:?}: read_semaphore acquire timeout", hash);
+				return Err(Error::Message("read block: read_semaphore acquire timeout".into()));
+			}
+		};
+
 		let mut f = fs::File::open(&path).await?;
 		let mut data = vec![];
 		f.read_to_end(&mut data).await?;
@@ -616,6 +624,8 @@ impl BlockManager {
 			return Err(Error::CorruptData(*hash));
 		}
 
+		drop(permit);
+
 		Ok(data)
 	}
 

src/block/metrics.rs

@@ -22,6 +22,7 @@ pub struct BlockManagerMetrics {
 
 	pub(crate) bytes_read: BoundCounter<u64>,
 	pub(crate) block_read_duration: BoundValueRecorder<f64>,
+	pub(crate) block_read_semaphore_timeouts: BoundCounter<u64>,
 	pub(crate) bytes_written: BoundCounter<u64>,
 	pub(crate) block_write_duration: BoundValueRecorder<f64>,
 	pub(crate) delete_counter: BoundCounter<u64>,
@@ -50,7 +51,7 @@ impl BlockManagerMetrics {
 				.init(),
 			_rc_size: meter
 				.u64_value_observer("block.rc_size", move |observer| {
-					if let Ok(value) = rc_tree.len() {
+					if let Ok(value) = rc_tree.approximate_len() {
 						observer.observe(value as u64, &[])
 					}
 				})
@@ -58,7 +59,7 @@ impl BlockManagerMetrics {
 				.init(),
 			_resync_queue_len: meter
 				.u64_value_observer("block.resync_queue_length", move |observer| {
-					if let Ok(value) = resync_queue.len() {
+					if let Ok(value) = resync_queue.approximate_len() {
 						observer.observe(value as u64, &[]);
 					}
 				})
@@ -68,7 +69,7 @@ impl BlockManagerMetrics {
 				.init(),
 			_resync_errored_blocks: meter
 				.u64_value_observer("block.resync_errored_blocks", move |observer| {
-					if let Ok(value) = resync_errors.len() {
+					if let Ok(value) = resync_errors.approximate_len() {
 						observer.observe(value as u64, &[]);
 					}
 				})
@@ -119,6 +120,11 @@ impl BlockManagerMetrics {
 				.with_description("Duration of block read operations")
 				.init()
 				.bind(&[]),
+			block_read_semaphore_timeouts: meter
+				.u64_counter("block.read_semaphore_timeouts")
+				.with_description("Number of block reads that failed due to semaphore acquire timeout")
+				.init()
+				.bind(&[]),
 			bytes_written: meter
 				.u64_counter("block.bytes_written")
 				.with_description("Number of bytes written to disk")

src/block/resync.rs

@@ -104,13 +104,13 @@ impl BlockResyncManager {
 	}
 
 	/// Get length of resync queue
-	pub fn queue_len(&self) -> Result<usize, Error> {
-		Ok(self.queue.len()?)
+	pub fn queue_approximate_len(&self) -> Result<usize, Error> {
+		Ok(self.queue.approximate_len()?)
 	}
 
 	/// Get number of blocks that have an error
-	pub fn errors_len(&self) -> Result<usize, Error> {
-		Ok(self.errors.len()?)
+	pub fn errors_approximate_len(&self) -> Result<usize, Error> {
+		Ok(self.errors.approximate_len()?)
 	}
 
 	/// Clear the error counter for a block and put it in queue immediately
@@ -131,6 +131,14 @@ impl BlockResyncManager {
 		)))
 	}
 
+	/// Clear the entire resync queue and list of errored blocks
+	/// Corresponds to `garage repair clear-resync-queue`
+	pub fn clear_resync_queue(&self) -> Result<(), Error> {
+		self.queue.clear()?;
+		self.errors.clear()?;
+		Ok(())
+	}
+
 	pub fn register_bg_vars(&self, vars: &mut vars::BgVars) {
 		let notify = self.notify.clone();
 		vars.register_rw(
@@ -375,7 +383,7 @@ impl BlockResyncManager {
 			info!("Resync block {:?}: offloading and deleting", hash);
 			let existing_path = existing_path.unwrap();
 
-			let mut who = manager.storage_nodes_of(hash);
+			let mut who = manager.storage_nodes_of(hash)?;
 			if who.len() < manager.write_quorum {
 				return Err(Error::Message("Not trying to offload block because we don't have a quorum of nodes to write to".to_string()));
 			}
@@ -458,7 +466,7 @@ impl BlockResyncManager {
 
 			// First, check whether we are still supposed to store that
 			// block in the latest cluster layout version.
-			let storage_nodes = manager.storage_nodes_of(&hash);
+			let storage_nodes = manager.storage_nodes_of(&hash)?;
 
 			if !storage_nodes.contains(&manager.system.id) {
 				info!(
@@ -540,9 +548,11 @@ impl Worker for ResyncWorker {
 		}
 
 		WorkerStatus {
-			queue_length: Some(self.manager.resync.queue_len().unwrap_or(0) as u64),
+			queue_length: Some(self.manager.resync.queue_approximate_len().unwrap_or(0) as u64),
 			tranquility: Some(tranquility),
-			persistent_errors: Some(self.manager.resync.errors_len().unwrap_or(0) as u64),
+			persistent_errors: Some(
+				self.manager.resync.errors_approximate_len().unwrap_or(0) as u64
+			),
 			..Default::default()
 		}
 	}

src/db/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "garage_db"
-version = "2.0.0"
+version = "2.2.0"
 authors = ["Alex Auvolat <alex@adnab.me>"]
 edition = "2018"
 license = "AGPL-3.0"
@@ -12,14 +12,18 @@ readme = "../../README.md"
 path = "lib.rs"
 
 [dependencies]
-err-derive.workspace = true
+thiserror.workspace = true
 tracing.workspace = true
 
 heed = { workspace = true, optional = true }
+
 rusqlite = { workspace = true, optional = true, features = ["backup"] }
 r2d2 = { workspace = true, optional = true }
 r2d2_sqlite = { workspace = true, optional = true }
 
+fjall = { workspace = true, optional = true }
+parking_lot = { workspace = true, optional = true }
+
 [dev-dependencies]
 mktemp.workspace = true
 
@@ -27,4 +31,5 @@ mktemp.workspace = true
 default = [ "lmdb", "sqlite" ]
 bundled-libs = [ "rusqlite?/bundled" ]
 lmdb = [ "heed" ]
+fjall = [ "dep:fjall", "dep:parking_lot" ]
 sqlite = [ "rusqlite", "r2d2", "r2d2_sqlite" ]

src/db/fjall_adapter.rs

@@ -0,0 +1,453 @@
+use core::ops::Bound;
+
+use std::path::PathBuf;
+use std::sync::Arc;
+
+use parking_lot::{MappedRwLockReadGuard, RwLock, RwLockReadGuard};
+
+use fjall::{
+	PartitionCreateOptions, PersistMode, TransactionalKeyspace, TransactionalPartitionHandle,
+	WriteTransaction,
+};
+
+use crate::{
+	open::{Engine, OpenOpt},
+	Db, Error, IDb, ITx, ITxFn, OnCommit, Result, TxError, TxFnResult, TxOpError, TxOpResult,
+	TxResult, TxValueIter, Value, ValueIter,
+};
+
+pub use fjall;
+
+// --
+
+pub(crate) fn open_db(path: &PathBuf, opt: &OpenOpt) -> Result<Db> {
+	info!("Opening Fjall database at: {}", path.display());
+	if opt.fsync {
+		return Err(Error(
+			"metadata_fsync is not supported with the Fjall database engine".into(),
+		));
+	}
+	let mut config = fjall::Config::new(path);
+	if let Some(block_cache_size) = opt.fjall_block_cache_size {
+		config = config.cache_size(block_cache_size as u64);
+	}
+	let keyspace = config.open_transactional()?;
+	Ok(FjallDb::init(keyspace))
+}
+
+// -- err
+
+impl From<fjall::Error> for Error {
+	fn from(e: fjall::Error) -> Error {
+		Error(format!("fjall: {}", e).into())
+	}
+}
+
+impl From<fjall::LsmError> for Error {
+	fn from(e: fjall::LsmError) -> Error {
+		Error(format!("fjall lsm_tree: {}", e).into())
+	}
+}
+
+impl From<fjall::Error> for TxOpError {
+	fn from(e: fjall::Error) -> TxOpError {
+		TxOpError(e.into())
+	}
+}
+
+// -- db
+
+pub struct FjallDb {
+	keyspace: TransactionalKeyspace,
+	trees: RwLock<Vec<(String, TransactionalPartitionHandle)>>,
+}
+
+type ByteRefRangeBound<'r> = (Bound<&'r [u8]>, Bound<&'r [u8]>);
+
+impl FjallDb {
+	pub fn init(keyspace: TransactionalKeyspace) -> Db {
+		let s = Self {
+			keyspace,
+			trees: RwLock::new(Vec::new()),
+		};
+		Db(Arc::new(s))
+	}
+
+	fn get_tree(
+		&self,
+		i: usize,
+	) -> Result<MappedRwLockReadGuard<'_, TransactionalPartitionHandle>> {
+		RwLockReadGuard::try_map(self.trees.read(), |trees: &Vec<_>| {
+			trees.get(i).map(|tup| &tup.1)
+		})
+		.map_err(|_| Error("invalid tree id".into()))
+	}
+}
+
+impl IDb for FjallDb {
+	fn engine(&self) -> String {
+		"Fjall (EXPERIMENTAL!)".into()
+	}
+
+	fn open_tree(&self, name: &str) -> Result<usize> {
+		let mut trees = self.trees.write();
+		let safe_name = encode_name(name)?;
+		if let Some(i) = trees.iter().position(|(name, _)| *name == safe_name) {
+			Ok(i)
+		} else {
+			let tree = self
+				.keyspace
+				.open_partition(&safe_name, PartitionCreateOptions::default())?;
+			let i = trees.len();
+			trees.push((safe_name, tree));
+			Ok(i)
+		}
+	}
+
+	fn list_trees(&self) -> Result<Vec<String>> {
+		Ok(self
+			.keyspace
+			.list_partitions()
+			.iter()
+			.map(|n| decode_name(&n))
+			.collect::<Result<Vec<_>>>()?)
+	}
+
+	fn snapshot(&self, base_path: &PathBuf) -> Result<()> {
+		std::fs::create_dir_all(base_path)?;
+		let path = Engine::Fjall.db_path(base_path);
+
+		let source_state = self.keyspace.read_tx();
+		let copy_keyspace = fjall::Config::new(path).open()?;
+
+		for partition_name in self.keyspace.list_partitions() {
+			let source_partition = self
+				.keyspace
+				.open_partition(&partition_name, PartitionCreateOptions::default())?;
+			let copy_partition =
+				copy_keyspace.open_partition(&partition_name, PartitionCreateOptions::default())?;
+
+			for entry in source_state.iter(&source_partition) {
+				let (key, value) = entry?;
+				copy_partition.insert(key, value)?;
+			}
+		}
+
+		copy_keyspace.persist(PersistMode::SyncAll)?;
+		Ok(())
+	}
+
+	// ----
+
+	fn get(&self, tree_idx: usize, key: &[u8]) -> Result<Option<Value>> {
+		let tree = self.get_tree(tree_idx)?;
+		let tx = self.keyspace.read_tx();
+		let val = tx.get(&tree, key)?;
+		match val {
+			None => Ok(None),
+			Some(v) => Ok(Some(v.to_vec())),
+		}
+	}
+
+	fn approximate_len(&self, tree_idx: usize) -> Result<usize> {
+		let tree = self.get_tree(tree_idx)?;
+		Ok(tree.approximate_len())
+	}
+	fn is_empty(&self, tree_idx: usize) -> Result<bool> {
+		let tree = self.get_tree(tree_idx)?;
+		let tx = self.keyspace.read_tx();
+		Ok(tx.is_empty(&tree)?)
+	}
+
+	fn insert(&self, tree_idx: usize, key: &[u8], value: &[u8]) -> Result<()> {
+		let tree = self.get_tree(tree_idx)?;
+		let mut tx = self.keyspace.write_tx();
+		tx.insert(&tree, key, value);
+		tx.commit()?;
+		Ok(())
+	}
+
+	fn remove(&self, tree_idx: usize, key: &[u8]) -> Result<()> {
+		let tree = self.get_tree(tree_idx)?;
+		let mut tx = self.keyspace.write_tx();
+		tx.remove(&tree, key);
+		tx.commit()?;
+		Ok(())
+	}
+
+	fn clear(&self, tree_idx: usize) -> Result<()> {
+		let mut trees = self.trees.write();
+
+		if tree_idx >= trees.len() {
+			return Err(Error("invalid tree id".into()));
+		}
+		let (name, tree) = trees.remove(tree_idx);
+
+		self.keyspace.delete_partition(tree)?;
+		let tree = self
+			.keyspace
+			.open_partition(&name, PartitionCreateOptions::default())?;
+		trees.insert(tree_idx, (name, tree));
+
+		Ok(())
+	}
+
+	fn iter(&self, tree_idx: usize) -> Result<ValueIter<'_>> {
+		let tree = self.get_tree(tree_idx)?;
+		let tx = self.keyspace.read_tx();
+		Ok(Box::new(tx.iter(&tree).map(iterator_remap)))
+	}
+
+	fn iter_rev(&self, tree_idx: usize) -> Result<ValueIter<'_>> {
+		let tree = self.get_tree(tree_idx)?;
+		let tx = self.keyspace.read_tx();
+		Ok(Box::new(tx.iter(&tree).rev().map(iterator_remap)))
+	}
+
+	fn range<'r>(
+		&self,
+		tree_idx: usize,
+		low: Bound<&'r [u8]>,
+		high: Bound<&'r [u8]>,
+	) -> Result<ValueIter<'_>> {
+		let tree = self.get_tree(tree_idx)?;
+		let tx = self.keyspace.read_tx();
+		Ok(Box::new(
+			tx.range::<&'r [u8], ByteRefRangeBound>(&tree, (low, high))
+				.map(iterator_remap),
+		))
+	}
+	fn range_rev<'r>(
+		&self,
+		tree_idx: usize,
+		low: Bound<&'r [u8]>,
+		high: Bound<&'r [u8]>,
+	) -> Result<ValueIter<'_>> {
+		let tree = self.get_tree(tree_idx)?;
+		let tx = self.keyspace.read_tx();
+		Ok(Box::new(
+			tx.range::<&'r [u8], ByteRefRangeBound>(&tree, (low, high))
+				.rev()
+				.map(iterator_remap),
+		))
+	}
+
+	// ----
+
+	fn transaction(&self, f: &dyn ITxFn) -> TxResult<OnCommit, ()> {
+		let trees = self.trees.read();
+		let mut tx = FjallTx {
+			trees: &trees[..],
+			tx: self.keyspace.write_tx(),
+		};
+
+		let res = f.try_on(&mut tx);
+		match res {
+			TxFnResult::Ok(on_commit) => {
+				tx.tx.commit().map_err(Error::from).map_err(TxError::Db)?;
+				Ok(on_commit)
+			}
+			TxFnResult::Abort => {
+				tx.tx.rollback();
+				Err(TxError::Abort(()))
+			}
+			TxFnResult::DbErr => {
+				tx.tx.rollback();
+				Err(TxError::Db(Error(
+					"(this message will be discarded)".into(),
+				)))
+			}
+		}
+	}
+}
+
+// ----
+
+struct FjallTx<'a> {
+	trees: &'a [(String, TransactionalPartitionHandle)],
+	tx: WriteTransaction<'a>,
+}
+
+impl<'a> FjallTx<'a> {
+	fn get_tree(&self, i: usize) -> TxOpResult<&TransactionalPartitionHandle> {
+		self.trees.get(i).map(|tup| &tup.1).ok_or_else(|| {
+			TxOpError(Error(
+				"invalid tree id (it might have been openned after the transaction started)".into(),
+			))
+		})
+	}
+}
+
+impl<'a> ITx for FjallTx<'a> {
+	fn get(&self, tree_idx: usize, key: &[u8]) -> TxOpResult<Option<Value>> {
+		let tree = self.get_tree(tree_idx)?;
+		match self.tx.get(tree, key)? {
+			Some(v) => Ok(Some(v.to_vec())),
+			None => Ok(None),
+		}
+	}
+	fn len(&self, tree_idx: usize) -> TxOpResult<usize> {
+		let tree = self.get_tree(tree_idx)?;
+		Ok(self.tx.len(tree)? as usize)
+	}
+
+	fn insert(&mut self, tree_idx: usize, key: &[u8], value: &[u8]) -> TxOpResult<()> {
+		let tree = self.get_tree(tree_idx)?.clone();
+		self.tx.insert(&tree, key, value);
+		Ok(())
+	}
+	fn remove(&mut self, tree_idx: usize, key: &[u8]) -> TxOpResult<()> {
+		let tree = self.get_tree(tree_idx)?.clone();
+		self.tx.remove(&tree, key);
+		Ok(())
+	}
+	fn clear(&mut self, _tree_idx: usize) -> TxOpResult<()> {
+		unimplemented!("LSM tree clearing in cross-partition transaction is not supported")
+	}
+
+	fn iter(&self, tree_idx: usize) -> TxOpResult<TxValueIter<'_>> {
+		let tree = self.get_tree(tree_idx)?.clone();
+		Ok(Box::new(self.tx.iter(&tree).map(iterator_remap_tx)))
+	}
+	fn iter_rev(&self, tree_idx: usize) -> TxOpResult<TxValueIter<'_>> {
+		let tree = self.get_tree(tree_idx)?.clone();
+		Ok(Box::new(self.tx.iter(&tree).rev().map(iterator_remap_tx)))
+	}
+
+	fn range<'r>(
+		&self,
+		tree_idx: usize,
+		low: Bound<&'r [u8]>,
+		high: Bound<&'r [u8]>,
+	) -> TxOpResult<TxValueIter<'_>> {
+		let tree = self.get_tree(tree_idx)?;
+		let low = clone_bound(low);
+		let high = clone_bound(high);
+		Ok(Box::new(
+			self.tx
+				.range::<Vec<u8>, ByteVecRangeBounds>(&tree, (low, high))
+				.map(iterator_remap_tx),
+		))
+	}
+	fn range_rev<'r>(
+		&self,
+		tree_idx: usize,
+		low: Bound<&'r [u8]>,
+		high: Bound<&'r [u8]>,
+	) -> TxOpResult<TxValueIter<'_>> {
+		let tree = self.get_tree(tree_idx)?;
+		let low = clone_bound(low);
+		let high = clone_bound(high);
+		Ok(Box::new(
+			self.tx
+				.range::<Vec<u8>, ByteVecRangeBounds>(&tree, (low, high))
+				.rev()
+				.map(iterator_remap_tx),
+		))
+	}
+}
+
+// -- maps fjall's (k, v) to ours
+
+fn iterator_remap(r: fjall::Result<(fjall::Slice, fjall::Slice)>) -> Result<(Value, Value)> {
+	r.map(|(k, v)| (k.to_vec(), v.to_vec()))
+		.map_err(|e| e.into())
+}
+
+fn iterator_remap_tx(r: fjall::Result<(fjall::Slice, fjall::Slice)>) -> TxOpResult<(Value, Value)> {
+	r.map(|(k, v)| (k.to_vec(), v.to_vec()))
+		.map_err(|e| e.into())
+}
+
+// -- utils to deal with Garage's tightness on Bound lifetimes
+
+type ByteVecBound = Bound<Vec<u8>>;
+type ByteVecRangeBounds = (ByteVecBound, ByteVecBound);
+
+fn clone_bound(bound: Bound<&[u8]>) -> ByteVecBound {
+	let value = match bound {
+		Bound::Excluded(v) | Bound::Included(v) => v.to_vec(),
+		Bound::Unbounded => vec![],
+	};
+
+	match bound {
+		Bound::Included(_) => Bound::Included(value),
+		Bound::Excluded(_) => Bound::Excluded(value),
+		Bound::Unbounded => Bound::Unbounded,
+	}
+}
+
+// -- utils to encode table names --
+
+fn encode_name(s: &str) -> Result<String> {
+	let base = 'A' as u32;
+
+	let mut ret = String::with_capacity(s.len() + 10);
+	for c in s.chars() {
+		if c.is_alphanumeric() || c == '_' || c == '-' || c == '#' {
+			ret.push(c);
+		} else if c <= u8::MAX as char {
+			ret.push('$');
+			let c_hi = c as u32 / 16;
+			let c_lo = c as u32 % 16;
+			ret.push(char::from_u32(base + c_hi).unwrap());
+			ret.push(char::from_u32(base + c_lo).unwrap());
+		} else {
+			return Err(Error(
+				format!("table name {} could not be safely encoded", s).into(),
+			));
+		}
+	}
+	Ok(ret)
+}
+
+fn decode_name(s: &str) -> Result<String> {
+	use std::convert::TryFrom;
+
+	let errfn = || Error(format!("encoded table name {} is invalid", s).into());
+	let c_map = |c: char| {
+		let c = c as u32;
+		let base = 'A' as u32;
+		if (base..base + 16).contains(&c) {
+			Some(c - base)
+		} else {
+			None
+		}
+	};
+
+	let mut ret = String::with_capacity(s.len());
+	let mut it = s.chars();
+	while let Some(c) = it.next() {
+		if c == '$' {
+			let c_hi = it.next().and_then(c_map).ok_or_else(errfn)?;
+			let c_lo = it.next().and_then(c_map).ok_or_else(errfn)?;
+			let c_dec = char::try_from(c_hi * 16 + c_lo).map_err(|_| errfn())?;
+			ret.push(c_dec);
+		} else {
+			ret.push(c);
+		}
+	}
+	Ok(ret)
+}
+
+#[cfg(test)]
+mod tests {
+	use super::*;
+
+	#[test]
+	fn test_encdec_name() {
+		for name in [
+			"testname",
+			"test_name",
+			"test name",
+			"test$name",
+			"test:name@help.me$get/this**right",
+		] {
+			let encname = encode_name(name).unwrap();
+			assert!(!encname.contains(' '));
+			assert!(!encname.contains('.'));
+			assert!(!encname.contains('*'));
+			assert_eq!(*name, decode_name(&encname).unwrap());
+		}
+	}
+}

src/db/lib.rs

@@ -1,6 +1,8 @@
 #[macro_use]
 extern crate tracing;
 
+#[cfg(feature = "fjall")]
+pub mod fjall_adapter;
 #[cfg(feature = "lmdb")]
 pub mod lmdb_adapter;
 #[cfg(feature = "sqlite")]
@@ -18,7 +20,7 @@ use std::cell::Cell;
 use std::path::PathBuf;
 use std::sync::Arc;
 
-use err_derive::Error;
+use thiserror::Error;
 
 pub use open::*;
 
@@ -42,7 +44,7 @@ pub type TxValueIter<'a> = Box<dyn std::iter::Iterator<Item = TxOpResult<(Value,
 // ----
 
 #[derive(Debug, Error)]
-#[error(display = "{}", _0)]
+#[error("{0}")]
 pub struct Error(pub Cow<'static, str>);
 
 impl From<std::io::Error> for Error {
@@ -54,7 +56,7 @@ impl From<std::io::Error> for Error {
 pub type Result<T> = std::result::Result<T, Error>;
 
 #[derive(Debug, Error)]
-#[error(display = "{}", _0)]
+#[error("{0}")]
 pub struct TxOpError(pub(crate) Error);
 pub type TxOpResult<T> = std::result::Result<T, TxOpError>;
 
@@ -104,32 +106,44 @@ impl Db {
 			result: Cell::new(None),
 		};
 		let tx_res = self.0.transaction(&f);
-		let ret = f
-			.result
-			.into_inner()
-			.expect("Transaction did not store result");
+		let fn_res = f.result.into_inner();
 
-		match tx_res {
-			Ok(on_commit) => match ret {
-				Ok(value) => {
-					on_commit.into_iter().for_each(|f| f());
-					Ok(value)
-				}
-				_ => unreachable!(),
-			},
-			Err(TxError::Abort(())) => match ret {
-				Err(TxError::Abort(e)) => Err(TxError::Abort(e)),
-				_ => unreachable!(),
-			},
-			Err(TxError::Db(e2)) => match ret {
-				// Ok was stored -> the error occurred when finalizing
-				// transaction
-				Ok(_) => Err(TxError::Db(e2)),
-				// An error was already stored: that's the one we want to
-				// return
-				Err(TxError::Db(e)) => Err(TxError::Db(e)),
-				_ => unreachable!(),
-			},
+		match (tx_res, fn_res) {
+			(Ok(on_commit), Some(Ok(value))) => {
+				// Transaction succeeded
+				// TxFn stored the value to return to the user in fn_res
+				// tx_res contains the on_commit list of callbacks, run them now
+				on_commit.into_iter().for_each(|f| f());
+				Ok(value)
+			}
+			(Err(TxError::Abort(())), Some(Err(TxError::Abort(e)))) => {
+				// Transaction was aborted by user code
+				// The abort error value is stored in fn_res
+				Err(TxError::Abort(e))
+			}
+			(Err(TxError::Db(_tx_e)), Some(Err(TxError::Db(fn_e)))) => {
+				// Transaction encountered a DB error in user code
+				// The error value encountered is the one in fn_res,
+				// tx_res contains only a dummy error message
+				Err(TxError::Db(fn_e))
+			}
+			(Err(TxError::Db(tx_e)), None) => {
+				// Transaction encounterred a DB error when initializing the transaction,
+				// before user code was called
+				Err(TxError::Db(tx_e))
+			}
+			(Err(TxError::Db(tx_e)), Some(Ok(_))) => {
+				// Transaction encounterred a DB error when commiting the transaction,
+				// after user code was called
+				Err(TxError::Db(tx_e))
+			}
+			(tx_res, fn_res) => {
+				panic!(
+					"unexpected error case: tx_res={:?}, fn_res={:?}",
+					tx_res.map(|_| "..."),
+					fn_res.map(|x| x.map(|_| "...").map_err(|_| "..."))
+				);
+			}
 		}
 	}
 
@@ -152,7 +166,7 @@ impl Db {
 		let tree_names = other.list_trees()?;
 		for name in tree_names {
 			let tree = self.open_tree(&name)?;
-			if tree.len()? > 0 {
+			if !tree.is_empty()? {
 				return Err(Error(format!("tree {} already contains data", name).into()));
 			}
 
@@ -194,8 +208,12 @@ impl Tree {
 		self.0.get(self.1, key.as_ref())
 	}
 	#[inline]
-	pub fn len(&self) -> Result<usize> {
-		self.0.len(self.1)
+	pub fn approximate_len(&self) -> Result<usize> {
+		self.0.approximate_len(self.1)
+	}
+	#[inline]
+	pub fn is_empty(&self) -> Result<bool> {
+		self.0.is_empty(self.1)
 	}
 
 	#[inline]
@@ -333,7 +351,8 @@ pub(crate) trait IDb: Send + Sync {
 	fn snapshot(&self, path: &PathBuf) -> Result<()>;
 
 	fn get(&self, tree: usize, key: &[u8]) -> Result<Option<Value>>;
-	fn len(&self, tree: usize) -> Result<usize>;
+	fn approximate_len(&self, tree: usize) -> Result<usize>;
+	fn is_empty(&self, tree: usize) -> Result<bool>;
 
 	fn insert(&self, tree: usize, key: &[u8], value: &[u8]) -> Result<()>;
 	fn remove(&self, tree: usize, key: &[u8]) -> Result<()>;

src/db/lmdb_adapter.rs

@@ -1,8 +1,8 @@
 use core::ops::Bound;
-use core::ptr::NonNull;
 
 use std::collections::HashMap;
 use std::convert::TryInto;
+use std::marker::PhantomPinned;
 use std::path::PathBuf;
 use std::pin::Pin;
 use std::sync::{Arc, RwLock};
@@ -11,12 +11,55 @@ use heed::types::ByteSlice;
 use heed::{BytesDecode, Env, RoTxn, RwTxn, UntypedDatabase as Database};
 
 use crate::{
+	open::{Engine, OpenOpt},
 	Db, Error, IDb, ITx, ITxFn, OnCommit, Result, TxError, TxFnResult, TxOpError, TxOpResult,
 	TxResult, TxValueIter, Value, ValueIter,
 };
 
 pub use heed;
 
+// ---- top-level open function
+
+pub(crate) fn open_db(path: &PathBuf, opt: &OpenOpt) -> Result<Db> {
+	info!("Opening LMDB database at: {}", path.display());
+	if let Err(e) = std::fs::create_dir_all(&path) {
+		return Err(Error(
+			format!("Unable to create LMDB data directory: {}", e).into(),
+		));
+	}
+
+	let map_size = match opt.lmdb_map_size {
+		None => recommended_map_size(),
+		Some(v) => v - (v % 4096),
+	};
+
+	let mut env_builder = heed::EnvOpenOptions::new();
+	env_builder.max_dbs(100);
+	env_builder.map_size(map_size);
+	env_builder.max_readers(2048);
+	unsafe {
+		env_builder.flag(heed::flags::Flags::MdbNoRdAhead);
+		env_builder.flag(heed::flags::Flags::MdbNoMetaSync);
+		if !opt.fsync {
+			env_builder.flag(heed::flags::Flags::MdbNoSync);
+		}
+	}
+	match env_builder.open(&path) {
+		Err(heed::Error::Io(e)) if e.kind() == std::io::ErrorKind::OutOfMemory => {
+			return Err(Error(
+				"OutOfMemory error while trying to open LMDB database. This can happen \
+                if your operating system is not allowing you to use sufficient virtual \
+                memory address space. Please check that no limit is set (ulimit -v). \
+                You may also try to set a smaller `lmdb_map_size` configuration parameter. \
+                On 32-bit machines, you should probably switch to another database engine."
+					.into(),
+			))
+		}
+		Err(e) => Err(Error(format!("Cannot open LMDB database: {}", e).into())),
+		Ok(db) => Ok(LmdbDb::init(db)),
+	}
+}
+
 // -- err
 
 impl From<heed::Error> for Error {
@@ -104,10 +147,9 @@ impl IDb for LmdbDb {
 		Ok(ret2)
 	}
 
-	fn snapshot(&self, to: &PathBuf) -> Result<()> {
-		std::fs::create_dir_all(to)?;
-		let mut path = to.clone();
-		path.push("data.mdb");
+	fn snapshot(&self, base_path: &PathBuf) -> Result<()> {
+		std::fs::create_dir_all(base_path)?;
+		let path = Engine::Lmdb.db_path(base_path);
 		self.db
 			.copy_to_path(path, heed::CompactionOption::Enabled)?;
 		Ok(())
@@ -126,11 +168,16 @@ impl IDb for LmdbDb {
 		}
 	}
 
-	fn len(&self, tree: usize) -> Result<usize> {
+	fn approximate_len(&self, tree: usize) -> Result<usize> {
 		let tree = self.get_tree(tree)?;
 		let tx = self.db.read_txn()?;
 		Ok(tree.len(&tx)?.try_into().unwrap())
 	}
+	fn is_empty(&self, tree: usize) -> Result<bool> {
+		let tree = self.get_tree(tree)?;
+		let tx = self.db.read_txn()?;
+		Ok(tree.is_empty(&tx)?)
+	}
 
 	fn insert(&self, tree: usize, key: &[u8], value: &[u8]) -> Result<()> {
 		let tree = self.get_tree(tree)?;
@@ -159,13 +206,15 @@ impl IDb for LmdbDb {
 	fn iter(&self, tree: usize) -> Result<ValueIter<'_>> {
 		let tree = self.get_tree(tree)?;
 		let tx = self.db.read_txn()?;
-		TxAndIterator::make(tx, |tx| Ok(tree.iter(tx)?))
+		// Safety: the cloture does not store its argument anywhere,
+		unsafe { TxAndIterator::make(tx, |tx| Ok(tree.iter(tx)?)) }
 	}
 
 	fn iter_rev(&self, tree: usize) -> Result<ValueIter<'_>> {
 		let tree = self.get_tree(tree)?;
 		let tx = self.db.read_txn()?;
-		TxAndIterator::make(tx, |tx| Ok(tree.rev_iter(tx)?))
+		// Safety: the cloture does not store its argument anywhere,
+		unsafe { TxAndIterator::make(tx, |tx| Ok(tree.rev_iter(tx)?)) }
 	}
 
 	fn range<'r>(
@@ -176,7 +225,8 @@ impl IDb for LmdbDb {
 	) -> Result<ValueIter<'_>> {
 		let tree = self.get_tree(tree)?;
 		let tx = self.db.read_txn()?;
-		TxAndIterator::make(tx, |tx| Ok(tree.range(tx, &(low, high))?))
+		// Safety: the cloture does not store its argument anywhere,
+		unsafe { TxAndIterator::make(tx, |tx| Ok(tree.range(tx, &(low, high))?)) }
 	}
 	fn range_rev<'r>(
 		&self,
@@ -186,7 +236,8 @@ impl IDb for LmdbDb {
 	) -> Result<ValueIter<'_>> {
 		let tree = self.get_tree(tree)?;
 		let tx = self.db.read_txn()?;
-		TxAndIterator::make(tx, |tx| Ok(tree.rev_range(tx, &(low, high))?))
+		// Safety: the cloture does not store its argument anywhere,
+		unsafe { TxAndIterator::make(tx, |tx| Ok(tree.rev_range(tx, &(low, high))?)) }
 	}
 
 	// ----
@@ -316,28 +367,41 @@ where
 {
 	tx: RoTxn<'a>,
 	iter: Option<I>,
+	_pin: PhantomPinned,
 }
 
 impl<'a, I> TxAndIterator<'a, I>
 where
 	I: Iterator<Item = IteratorItem<'a>> + 'a,
 {
-	fn make<F>(tx: RoTxn<'a>, iterfun: F) -> Result<ValueIter<'a>>
+	fn iter(self: Pin<&mut Self>) -> &mut Option<I> {
+		// Safety: iter is not structural
+		unsafe { &mut self.get_unchecked_mut().iter }
+	}
+
+	/// Safety: iterfun must not store its argument anywhere but in its result.
+	unsafe fn make<F>(tx: RoTxn<'a>, iterfun: F) -> Result<ValueIter<'a>>
 	where
 		F: FnOnce(&'a RoTxn<'a>) -> Result<I>,
 	{
-		let res = TxAndIterator { tx, iter: None };
+		let res = TxAndIterator {
+			tx,
+			iter: None,
+			_pin: PhantomPinned,
+		};
 		let mut boxed = Box::pin(res);
 
-		// This unsafe allows us to bypass lifetime checks
-		let tx = unsafe { NonNull::from(&boxed.tx).as_ref() };
-		let iter = iterfun(tx)?;
+		let tx_lifetime_overextended: &'a RoTxn<'a> = {
+			let tx = &boxed.tx;
+			// Safety: Artificially extending the lifetime because
+			// this reference will only be stored and accessed from the
+			// returned ValueIter which guarantees that it is destroyed
+			// before the tx it is pointing  to.
+			unsafe { &*&raw const *tx }
+		};
+		let iter = iterfun(&tx_lifetime_overextended)?;
 
-		let mut_ref = Pin::as_mut(&mut boxed);
-		// This unsafe allows us to write in a field of the pinned struct
-		unsafe {
-			Pin::get_unchecked_mut(mut_ref).iter = Some(iter);
-		}
+		*boxed.as_mut().iter() = Some(iter);
 
 		Ok(Box::new(TxAndIteratorPin(boxed)))
 	}
@@ -348,8 +412,10 @@ where
 	I: Iterator<Item = IteratorItem<'a>> + 'a,
 {
 	fn drop(&mut self) {
-		// ensure the iterator is dropped before the RoTxn it references
-		drop(self.iter.take());
+		// Safety: `new_unchecked` is okay because we know this value is never
+		// used again after being dropped.
+		let this = unsafe { Pin::new_unchecked(self) };
+		drop(this.iter().take());
 	}
 }
 
@@ -365,13 +431,12 @@ where
 
 	fn next(&mut self) -> Option<Self::Item> {
 		let mut_ref = Pin::as_mut(&mut self.0);
-		// This unsafe allows us to mutably access the iterator field
-		let next = unsafe { Pin::get_unchecked_mut(mut_ref).iter.as_mut()?.next() };
-		match next {
-			None => None,
-			Some(Err(e)) => Some(Err(e.into())),
-			Some(Ok((k, v))) => Some(Ok((k.to_vec(), v.to_vec()))),
-		}
+		let next = mut_ref.iter().as_mut()?.next()?;
+		let res = match next {
+			Err(e) => Err(e.into()),
+			Ok((k, v)) => Ok((k.to_vec(), v.to_vec())),
+		};
+		Some(res)
 	}
 }
 

src/db/open.rs

@@ -11,6 +11,7 @@ use crate::{Db, Error, Result};
 pub enum Engine {
 	Lmdb,
 	Sqlite,
+	Fjall,
 }
 
 impl Engine {
@@ -19,8 +20,26 @@ impl Engine {
 		match self {
 			Self::Lmdb => "lmdb",
 			Self::Sqlite => "sqlite",
+			Self::Fjall => "fjall",
 		}
 	}
+
+	/// Return engine-specific DB path from base path
+	pub fn db_path(&self, base_path: &PathBuf) -> PathBuf {
+		let mut ret = base_path.clone();
+		match self {
+			Self::Lmdb => {
+				ret.push("db.lmdb");
+			}
+			Self::Sqlite => {
+				ret.push("db.sqlite");
+			}
+			Self::Fjall => {
+				ret.push("db.fjall");
+			}
+		}
+		ret
+	}
 }
 
 impl std::fmt::Display for Engine {
@@ -36,10 +55,11 @@ impl std::str::FromStr for Engine {
 		match text {
 			"lmdb" | "heed" => Ok(Self::Lmdb),
 			"sqlite" | "sqlite3" | "rusqlite" => Ok(Self::Sqlite),
+            "fjall" => Ok(Self::Fjall),
 			"sled" => Err(Error("Sled is no longer supported as a database engine. Converting your old metadata db can be done using an older Garage binary (e.g. v0.9.4).".into())),
 			kind => Err(Error(
 				format!(
-					"Invalid DB engine: {} (options are: lmdb, sqlite)",
+					"Invalid DB engine: {} (options are: lmdb, sqlite, fjall)",
 					kind
 				)
 				.into(),
@@ -51,6 +71,7 @@ impl std::str::FromStr for Engine {
 pub struct OpenOpt {
 	pub fsync: bool,
 	pub lmdb_map_size: Option<usize>,
+	pub fjall_block_cache_size: Option<usize>,
 }
 
 impl Default for OpenOpt {
@@ -58,6 +79,7 @@ impl Default for OpenOpt {
 		Self {
 			fsync: false,
 			lmdb_map_size: None,
+			fjall_block_cache_size: None,
 		}
 	}
 }
@@ -66,53 +88,15 @@ pub fn open_db(path: &PathBuf, engine: Engine, opt: &OpenOpt) -> Result<Db> {
 	match engine {
 		// ---- Sqlite DB ----
 		#[cfg(feature = "sqlite")]
-		Engine::Sqlite => {
-			info!("Opening Sqlite database at: {}", path.display());
-			let manager = r2d2_sqlite::SqliteConnectionManager::file(path);
-			Ok(crate::sqlite_adapter::SqliteDb::new(manager, opt.fsync)?)
-		}
+		Engine::Sqlite => crate::sqlite_adapter::open_db(path, opt),
 
 		// ---- LMDB DB ----
 		#[cfg(feature = "lmdb")]
-		Engine::Lmdb => {
-			info!("Opening LMDB database at: {}", path.display());
-			if let Err(e) = std::fs::create_dir_all(&path) {
-				return Err(Error(
-					format!("Unable to create LMDB data directory: {}", e).into(),
-				));
-			}
+		Engine::Lmdb => crate::lmdb_adapter::open_db(path, opt),
 
-			let map_size = match opt.lmdb_map_size {
-				None => crate::lmdb_adapter::recommended_map_size(),
-				Some(v) => v - (v % 4096),
-			};
-
-			let mut env_builder = heed::EnvOpenOptions::new();
-			env_builder.max_dbs(100);
-			env_builder.map_size(map_size);
-			env_builder.max_readers(2048);
-			unsafe {
-				env_builder.flag(crate::lmdb_adapter::heed::flags::Flags::MdbNoRdAhead);
-				env_builder.flag(crate::lmdb_adapter::heed::flags::Flags::MdbNoMetaSync);
-				if !opt.fsync {
-					env_builder.flag(heed::flags::Flags::MdbNoSync);
-				}
-			}
-			match env_builder.open(&path) {
-				Err(heed::Error::Io(e)) if e.kind() == std::io::ErrorKind::OutOfMemory => {
-					return Err(Error(
-						"OutOfMemory error while trying to open LMDB database. This can happen \
-                        if your operating system is not allowing you to use sufficient virtual \
-                        memory address space. Please check that no limit is set (ulimit -v). \
-                        You may also try to set a smaller `lmdb_map_size` configuration parameter. \
-                        On 32-bit machines, you should probably switch to another database engine."
-							.into(),
-					))
-				}
-				Err(e) => Err(Error(format!("Cannot open LMDB database: {}", e).into())),
-				Ok(db) => Ok(crate::lmdb_adapter::LmdbDb::init(db)),
-			}
-		}
+		// ---- Fjall DB ----
+		#[cfg(feature = "fjall")]
+		Engine::Fjall => crate::fjall_adapter::open_db(path, opt),
 
 		// Pattern is unreachable when all supported DB engines are compiled into binary. The allow
 		// attribute is added so that we won't have to change this match in case stop building

src/db/sqlite_adapter.rs

@@ -11,12 +11,23 @@ use r2d2_sqlite::SqliteConnectionManager;
 use rusqlite::{params, Rows, Statement, Transaction};
 
 use crate::{
+	open::{Engine, OpenOpt},
 	Db, Error, IDb, ITx, ITxFn, OnCommit, Result, TxError, TxFnResult, TxOpError, TxOpResult,
 	TxResult, TxValueIter, Value, ValueIter,
 };
 
 pub use rusqlite;
 
+// ---- top-level open function
+
+pub(crate) fn open_db(path: &PathBuf, opt: &OpenOpt) -> Result<Db> {
+	info!("Opening Sqlite database at: {}", path.display());
+	let manager = r2d2_sqlite::SqliteConnectionManager::file(path);
+	Ok(SqliteDb::new(manager, opt.fsync)?)
+}
+
+// ----
+
 type Connection = r2d2::PooledConnection<SqliteConnectionManager>;
 
 // --- err
@@ -139,17 +150,18 @@ impl IDb for SqliteDb {
 		Ok(trees)
 	}
 
-	fn snapshot(&self, to: &PathBuf) -> Result<()> {
-		fn progress(p: rusqlite::backup::Progress) {
-			let percent = (p.pagecount - p.remaining) * 100 / p.pagecount;
-			info!("Sqlite snapshot progress: {}%", percent);
-		}
-		std::fs::create_dir_all(to)?;
-		let mut path = to.clone();
-		path.push("db.sqlite");
-		self.db
-			.get()?
-			.backup(rusqlite::DatabaseName::Main, path, Some(progress))?;
+	fn snapshot(&self, base_path: &PathBuf) -> Result<()> {
+		std::fs::create_dir_all(base_path)?;
+		let path = Engine::Sqlite
+			.db_path(&base_path)
+			.into_os_string()
+			.into_string()
+			.map_err(|_| Error("invalid sqlite path string".into()))?;
+
+		info!("Start sqlite VACUUM INTO `{}`", path);
+		self.db.get()?.execute("VACUUM INTO ?1", params![path])?;
+		info!("Finished sqlite VACUUM INTO `{}`", path);
+
 		Ok(())
 	}
 
@@ -160,7 +172,7 @@ impl IDb for SqliteDb {
 		self.internal_get(&self.db.get()?, &tree, key)
 	}
 
-	fn len(&self, tree: usize) -> Result<usize> {
+	fn approximate_len(&self, tree: usize) -> Result<usize> {
 		let tree = self.get_tree(tree)?;
 		let db = self.db.get()?;
 
@@ -172,6 +184,10 @@ impl IDb for SqliteDb {
 		}
 	}
 
+	fn is_empty(&self, tree: usize) -> Result<bool> {
+		Ok(self.approximate_len(tree)? == 0)
+	}
+
 	fn insert(&self, tree: usize, key: &[u8], value: &[u8]) -> Result<()> {
 		let tree = self.get_tree(tree)?;
 		let db = self.db.get()?;

src/db/test.rs

@@ -1,7 +1,7 @@
 use crate::*;
 
 fn test_suite(db: Db) {
-	let tree = db.open_tree("tree").unwrap();
+	let tree = db.open_tree("tree:this_is_a_tree").unwrap();
 
 	let ka: &[u8] = &b"test"[..];
 	let kb: &[u8] = &b"zwello"[..];
@@ -14,7 +14,7 @@ fn test_suite(db: Db) {
 
 	assert!(tree.insert(ka, va).is_ok());
 	assert_eq!(tree.get(ka).unwrap().unwrap(), va);
-	assert_eq!(tree.len().unwrap(), 1);
+	assert_eq!(tree.iter().unwrap().count(), 1);
 
 	// ---- test transaction logic ----
 
@@ -148,3 +148,15 @@ fn test_sqlite_db() {
 	let db = SqliteDb::new(manager, false).unwrap();
 	test_suite(db);
 }
+
+#[test]
+#[cfg(feature = "fjall")]
+fn test_fjall_db() {
+	use crate::fjall_adapter::{fjall, FjallDb};
+
+	let path = mktemp::Temp::new_dir().unwrap();
+	let config = fjall::Config::new(path).temporary(true);
+	let keyspace = config.open_transactional().unwrap();
+	let db = FjallDb::init(keyspace);
+	test_suite(db);
+}

src/garage/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "garage"
-version = "2.0.0"
+version = "2.2.0"
 authors = ["Alex Auvolat <alex@adnab.me>"]
 edition = "2018"
 license = "AGPL-3.0"
@@ -93,6 +93,7 @@ k2v = [ "garage_util/k2v", "garage_api_k2v", "garage_api_admin/k2v" ]
 # Database engines
 lmdb = [ "garage_model/lmdb" ]
 sqlite = [ "garage_model/sqlite" ]
+fjall = [ "garage_model/fjall" ]
 
 # Automatic registration and discovery via Consul API
 consul-discovery = [ "garage_rpc/consul-discovery" ]