wip formal algorithm

[RFC] Garbage Collector Elimination
Merge pull request 'Change optimization level to 3' (#1235 ) from majudev/garage:main-v2 into main-v2
2026-08-13 23:56:53 +00:00 · 2026-01-26 10:53:07 +01:00 · 2026-01-26 10:53:07 +01:00 · 2025-11-25 09:57:56 +00:00 · 2025-11-24 18:12:01 +01:00 · 2025-11-24 09:56:04 +00:00
130 changed files with 3396 additions and 1096 deletions
@@ -1,3 +1,6 @@
+labels:
+  nix: "enabled"
+
 when:
  event:
  - push
@@ -28,6 +31,11 @@ steps:
    commands:
      - nix-build -j4 --attr flakePackages.tests-sqlite

+  - name: unit + func tests (fjall)
+    image: nixpkgs/nix:nixos-22.05
+    commands:
+      - nix-build -j4 --attr flakePackages.tests-fjall
+
  - name: integration tests
    image: nixpkgs/nix:nixos-22.05
    commands:
@@ -1,3 +1,6 @@
+labels:
+  nix: "enabled"
+
 when:
  event:
  - deployment
@@ -1,3 +1,6 @@
+labels:
+  nix: "enabled"
+
 when:
  event:
  - deployment
@@ -687,6 +687,12 @@ version = "1.3.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2d2c12f985c78475a6b8d629afd0c360260ef34cfef52efccdcfd31972f81c2e"

+[[package]]
+name = "byteview"
+version = "0.6.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6236364b88b9b6d0bc181ba374cf1ab55ba3ef97a1cb6f8cddad48a273767fb5"
+
 [[package]]
 name = "cc"
 version = "1.2.16"
@@ -798,6 +804,12 @@ version = "1.0.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990"

+[[package]]
+name = "compare"
+version = "0.0.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ea0095f6103c2a8b44acd6fd15960c801dafebf02e21940360833e0673f48ba7"
+
 [[package]]
 name = "core-foundation"
 version = "0.9.4"
@@ -840,16 +852,15 @@ checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5"

 [[package]]
 name = "crc-fast"
-version = "1.2.0"
+version = "1.6.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bf0ab672ed7761a44410115d2e0e5ccd86b987ba186a78a4148fb02bb302a774"
+checksum = "6ddc2d09feefeee8bd78101665bd8645637828fa9317f9f292496dbbd8c65ff3"
 dependencies = [
- "cc",
 "crc",
 "digest",
- "libc",
 "rand 0.9.1",
 "regex",
+ "rustversion",
 ]

 [[package]]
@@ -888,6 +899,15 @@ dependencies = [
 "crossbeam-utils",
 ]

+[[package]]
+name = "crossbeam-epoch"
+version = "0.9.18"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e"
+dependencies = [
+ "crossbeam-utils",
+]
+
 [[package]]
 name = "crossbeam-queue"
 version = "0.3.12"
@@ -897,6 +917,16 @@ dependencies = [
 "crossbeam-utils",
 ]

+[[package]]
+name = "crossbeam-skiplist"
+version = "0.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "df29de440c58ca2cc6e587ec3d22347551a32435fbde9d2bff64e78a9ffa151b"
+dependencies = [
+ "crossbeam-epoch",
+ "crossbeam-utils",
+]
+
 [[package]]
 name = "crossbeam-utils"
 version = "0.8.21"
@@ -968,6 +998,20 @@ dependencies = [
 "num_cpus",
 ]

+[[package]]
+name = "dashmap"
+version = "6.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5041cc499144891f3790297212f32a74fb938e5136a14943f338ef9e0ae276cf"
+dependencies = [
+ "cfg-if",
+ "crossbeam-utils",
+ "hashbrown 0.14.5",
+ "lock_api",
+ "once_cell",
+ "parking_lot_core 0.9.10",
+]
+
 [[package]]
 name = "deranged"
 version = "0.4.0"
@@ -1010,6 +1054,12 @@ dependencies = [
 "syn 2.0.100",
 ]

+[[package]]
+name = "double-ended-peekable"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c0d05e1c0dbad51b52c38bda7adceef61b9efc2baf04acfe8726a8c4630a6f57"
+
 [[package]]
 name = "dyn-clone"
 version = "1.0.19"
@@ -1031,6 +1081,18 @@ dependencies = [
 "cfg-if",
 ]

+[[package]]
+name = "enum_dispatch"
+version = "0.3.13"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "aa18ce2bc66555b3218614519ac839ddb759a7d6720732f979ef8d13be147ecd"
+dependencies = [
+ "once_cell",
+ "proc-macro2",
+ "quote",
+ "syn 2.0.100",
+]
+
 [[package]]
 name = "env_logger"
 version = "0.10.2"
@@ -1050,20 +1112,6 @@ version = "1.0.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f"

-[[package]]
-name = "err-derive"
-version = "0.3.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c34a887c8df3ed90498c1c437ce21f211c8e27672921a8ffa293cb8d6d4caa9e"
-dependencies = [
- "proc-macro-error",
- "proc-macro2",
- "quote",
- "rustversion",
- "syn 1.0.109",
- "synstructure 0.12.6",
-]
-
 [[package]]
 name = "errno"
 version = "0.3.10"
@@ -1098,6 +1146,23 @@ version = "0.4.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80"

+[[package]]
+name = "fjall"
+version = "2.11.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0b25ad44cd4360a0448a9b5a0a6f1c7a621101cca4578706d43c9a821418aebc"
+dependencies = [
+ "byteorder",
+ "byteview",
+ "dashmap 6.1.0",
+ "log",
+ "lsm-tree",
+ "path-absolutize",
+ "std-semaphore",
+ "tempfile",
+ "xxhash-rust",
+]
+
 [[package]]
 name = "fnv"
 version = "1.0.7"
@@ -1214,7 +1279,7 @@ dependencies = [

 [[package]]
 name = "garage"
-version = "2.0.0"
+version = "2.1.0"
 dependencies = [
 "assert-json-diff",
 "async-trait",
@@ -1270,13 +1335,12 @@ dependencies = [

 [[package]]
 name = "garage_api_admin"
-version = "2.0.0"
+version = "2.1.0"
 dependencies = [
 "argon2",
 "async-trait",
 "bytesize",
 "chrono",
- "err-derive",
 "format_table",
 "futures",
 "garage_api_common",
@@ -1294,6 +1358,7 @@ dependencies = [
 "prometheus",
 "serde",
 "serde_json",
+ "thiserror 2.0.12",
 "tokio",
 "tracing",
 "url",
@@ -1302,14 +1367,13 @@ dependencies = [

 [[package]]
 name = "garage_api_common"
-version = "2.0.0"
+version = "2.1.0"
 dependencies = [
 "base64 0.21.7",
 "bytes",
 "chrono",
 "crc-fast",
 "crypto-common",
- "err-derive",
 "futures",
 "garage_model",
 "garage_table",
@@ -1328,6 +1392,7 @@ dependencies = [
 "serde_json",
 "sha1",
 "sha2",
+ "thiserror 2.0.12",
 "tokio",
 "tracing",
 "url",
@@ -1335,10 +1400,9 @@ dependencies = [

 [[package]]
 name = "garage_api_k2v"
-version = "2.0.0"
+version = "2.1.0"
 dependencies = [
 "base64 0.21.7",
- "err-derive",
 "futures",
 "garage_api_common",
 "garage_model",
@@ -1351,6 +1415,7 @@ dependencies = [
 "percent-encoding",
 "serde",
 "serde_json",
+ "thiserror 2.0.12",
 "tokio",
 "tracing",
 "url",
@@ -1358,7 +1423,7 @@ dependencies = [

 [[package]]
 name = "garage_api_s3"
-version = "2.0.0"
+version = "2.1.0"
 dependencies = [
 "aes-gcm",
 "async-compression",
@@ -1366,7 +1431,6 @@ dependencies = [
 "bytes",
 "chrono",
 "crc-fast",
- "err-derive",
 "form_urlencoded",
 "futures",
 "garage_api_common",
@@ -1394,6 +1458,7 @@ dependencies = [
 "serde_json",
 "sha1",
 "sha2",
+ "thiserror 2.0.12",
 "tokio",
 "tokio-stream",
 "tokio-util 0.7.14",
@@ -1403,7 +1468,7 @@ dependencies = [

 [[package]]
 name = "garage_block"
-version = "2.0.0"
+version = "2.1.0"
 dependencies = [
 "arc-swap",
 "async-compression",
@@ -1427,27 +1492,28 @@ dependencies = [

 [[package]]
 name = "garage_db"
-version = "2.0.0"
+version = "2.1.0"
 dependencies = [
- "err-derive",
+ "fjall",
 "heed",
 "mktemp",
+ "parking_lot 0.12.3",
 "r2d2",
 "r2d2_sqlite",
 "rusqlite",
+ "thiserror 2.0.12",
 "tracing",
 ]

 [[package]]
 name = "garage_model"
-version = "2.0.0"
+version = "2.1.0"
 dependencies = [
 "argon2",
 "async-trait",
 "base64 0.21.7",
 "blake2",
 "chrono",
- "err-derive",
 "futures",
 "garage_block",
 "garage_db",
@@ -1461,6 +1527,7 @@ dependencies = [
 "rand 0.8.5",
 "serde",
 "serde_bytes",
+ "thiserror 2.0.12",
 "tokio",
 "tracing",
 "zstd",
@@ -1468,12 +1535,11 @@ dependencies = [

 [[package]]
 name = "garage_net"
-version = "2.0.0"
+version = "2.1.0"
 dependencies = [
 "arc-swap",
 "bytes",
 "cfg-if",
- "err-derive",
 "futures",
 "hex",
 "kuska-handshake",
@@ -1486,6 +1552,7 @@ dependencies = [
 "rand 0.8.5",
 "rmp-serde",
 "serde",
+ "thiserror 2.0.12",
 "tokio",
 "tokio-stream",
 "tokio-util 0.7.14",
@@ -1493,12 +1560,11 @@ dependencies = [

 [[package]]
 name = "garage_rpc"
-version = "2.0.0"
+version = "2.1.0"
 dependencies = [
 "arc-swap",
 "async-trait",
 "bytesize",
- "err-derive",
 "format_table",
 "futures",
 "garage_net",
@@ -1519,13 +1585,14 @@ dependencies = [
 "serde",
 "serde_bytes",
 "serde_json",
+ "thiserror 2.0.12",
 "tokio",
 "tracing",
 ]

 [[package]]
 name = "garage_table"
-version = "2.0.0"
+version = "2.1.0"
 dependencies = [
 "arc-swap",
 "async-trait",
@@ -1546,14 +1613,13 @@ dependencies = [

 [[package]]
 name = "garage_util"
-version = "2.0.0"
+version = "2.1.0"
 dependencies = [
 "arc-swap",
 "async-trait",
 "blake2",
 "bytesize",
 "chrono",
- "err-derive",
 "futures",
 "garage_db",
 "garage_net",
@@ -1570,6 +1636,7 @@ dependencies = [
 "serde",
 "serde_json",
 "sha2",
+ "thiserror 2.0.12",
 "tokio",
 "toml",
 "tracing",
@@ -1578,9 +1645,8 @@ dependencies = [

 [[package]]
 name = "garage_web"
-version = "2.0.0"
+version = "2.1.0"
 dependencies = [
- "err-derive",
 "garage_api_common",
 "garage_api_s3",
 "garage_model",
@@ -1591,6 +1657,7 @@ dependencies = [
 "hyper 1.6.0",
 "opentelemetry",
 "percent-encoding",
+ "thiserror 2.0.12",
 "tokio",
 "tracing",
 ]
@@ -1674,6 +1741,12 @@ dependencies = [
 "syn 2.0.100",
 ]

+[[package]]
+name = "guardian"
+version = "1.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "17e2ac29387b1aa07a1e448f7bb4f35b500787971e965b02842b900afa5c8f6f"
+
 [[package]]
 name = "h2"
 version = "0.3.26"
@@ -1741,11 +1814,11 @@ dependencies = [

 [[package]]
 name = "hashlink"
-version = "0.9.1"
+version = "0.10.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6ba4ff7128dee98c7dc9794b6a411377e1404dba1c97deb8d1a55297bd25d8af"
+checksum = "7382cf6263419f2d8df38c55d7da83da5c18aef87fc7a7fc1fb1e344edfe14c1"
 dependencies = [
- "hashbrown 0.14.5",
+ "hashbrown 0.15.2",
 ]

 [[package]]
@@ -2249,6 +2322,15 @@ dependencies = [
 "cfg-if",
 ]

+[[package]]
+name = "interval-heap"
+version = "0.0.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "11274e5e8e89b8607cfedc2910b6626e998779b48a019151c7604d0adcb86ac6"
+dependencies = [
+ "compare",
+]
+
 [[package]]
 name = "ipnet"
 version = "2.11.0"
@@ -2368,7 +2450,7 @@ dependencies = [
 "serde",
 "serde_json",
 "sha2",
- "thiserror 1.0.69",
+ "thiserror 2.0.12",
 "tokio",
 "tracing-subscriber",
 ]
@@ -2542,9 +2624,9 @@ dependencies = [

 [[package]]
 name = "libsqlite3-sys"
-version = "0.28.0"
+version = "0.35.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0c10584274047cb335c23d3e61bcef8e323adae7c5c8c760540f73610177fc3f"
+checksum = "133c182a6a2c87864fe97778797e46c7e999672690dc9fa3ee8e241aa4a9c13f"
 dependencies = [
 "cc",
 "pkg-config",
@@ -2605,6 +2687,36 @@ dependencies = [
 "hashbrown 0.15.2",
 ]

+[[package]]
+name = "lsm-tree"
+version = "2.10.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ab73c02eadb3dc12c0024e5b61d6284e6d59064e67e74fbad77856caa56f62c7"
+dependencies = [
+ "byteorder",
+ "crossbeam-skiplist",
+ "double-ended-peekable",
+ "enum_dispatch",
+ "guardian",
+ "interval-heap",
+ "log",
+ "lz4_flex",
+ "path-absolutize",
+ "quick_cache",
+ "rustc-hash",
+ "self_cell",
+ "tempfile",
+ "value-log",
+ "varint-rs",
+ "xxhash-rust",
+]
+
+[[package]]
+name = "lz4_flex"
+version = "0.11.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "75761162ae2b0e580d7e7c390558127e5f01b4194debd6221fd8c207fc80e3f5"
+
 [[package]]
 name = "matchers"
 version = "0.1.0"
@@ -2859,7 +2971,7 @@ checksum = "6105e89802af13fdf48c49d7646d3b533a70e536d818aae7e78ba0433d01acb8"
 dependencies = [
 "async-trait",
 "crossbeam-channel",
- "dashmap",
+ "dashmap 4.0.2",
 "fnv",
 "futures-channel",
 "futures-executor",
@@ -3020,6 +3132,24 @@ version = "1.0.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a"

+[[package]]
+name = "path-absolutize"
+version = "3.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e4af381fe79fa195b4909485d99f73a80792331df0625188e707854f0b3383f5"
+dependencies = [
+ "path-dedot",
+]
+
+[[package]]
+name = "path-dedot"
+version = "3.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "07ba0ad7e047712414213ff67533e6dd477af0a4e1d14fb52343e53d30ea9397"
+dependencies = [
+ "once_cell",
+]
+
 [[package]]
 name = "pem"
 version = "3.0.5"
@@ -3315,6 +3445,16 @@ dependencies = [
 "serde",
 ]

+[[package]]
+name = "quick_cache"
+version = "0.6.16"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9ad6644cb07b7f3488b9f3d2fde3b4c0a7fa367cafefb39dff93a659f76eb786"
+dependencies = [
+ "equivalent",
+ "hashbrown 0.15.2",
+]
+
 [[package]]
 name = "quote"
 version = "1.0.40"
@@ -3343,9 +3483,9 @@ dependencies = [

 [[package]]
 name = "r2d2_sqlite"
-version = "0.24.0"
+version = "0.31.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6a982edf65c129796dba72f8775b292ef482b40d035e827a9825b3bc07ccc5f2"
+checksum = "63417e83dc891797eea3ad379f52a5986da4bca0d6ef28baf4d14034dd111b0c"
 dependencies = [
 "r2d2",
 "rusqlite",
@@ -3431,13 +3571,13 @@ dependencies = [

 [[package]]
 name = "regex"
-version = "1.11.1"
+version = "1.12.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191"
+checksum = "843bc0191f75f3e22651ae5f1e72939ab2f72a4bc30fa80a066bd66edefc24d4"
 dependencies = [
 "aho-corasick",
 "memchr",
- "regex-automata 0.4.9",
+ "regex-automata 0.4.13",
 "regex-syntax 0.8.5",
 ]

@@ -3452,9 +3592,9 @@ dependencies = [

 [[package]]
 name = "regex-automata"
-version = "0.4.9"
+version = "0.4.13"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908"
+checksum = "5276caf25ac86c8d810222b3dbb938e512c55c6831a10f3e6ed1c93b84041f1c"
 dependencies = [
 "aho-corasick",
 "memchr",
@@ -3563,9 +3703,9 @@ checksum = "3cd14fd5e3b777a7422cca79358c57a8f6e3a703d9ac187448d0daf220c2407f"

 [[package]]
 name = "rusqlite"
-version = "0.31.0"
+version = "0.37.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b838eba278d213a8beaf485bd313fd580ca4505a00d5871caeb1457c55322cae"
+checksum = "165ca6e57b20e1351573e3729b958bc62f0e48025386970b6e4d29e7a7e71f3f"
 dependencies = [
 "bitflags 2.9.0",
 "fallible-iterator",
@@ -3581,6 +3721,12 @@ version = "0.1.24"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f"

+[[package]]
+name = "rustc-hash"
+version = "2.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d"
+
 [[package]]
 name = "rustc_version"
 version = "0.4.1"
@@ -3823,6 +3969,12 @@ dependencies = [
 "libc",
 ]

+[[package]]
+name = "self_cell"
+version = "1.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0f7d95a54511e0c7be3f51e8867aa8cf35148d7b9445d44de2f943e2b206e749"
+
 [[package]]
 name = "semver"
 version = "1.0.26"
@@ -4036,6 +4188,12 @@ dependencies = [
 "syn 1.0.109",
 ]

+[[package]]
+name = "std-semaphore"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "33ae9eec00137a8eed469fb4148acd9fc6ac8c3f9b110f52cd34698c8b5bfa0e"
+
 [[package]]
 name = "strsim"
 version = "0.11.1"
@@ -4109,18 +4267,6 @@ dependencies = [
 "crossbeam-queue",
 ]

-[[package]]
-name = "synstructure"
-version = "0.12.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f36bdaa60a83aca3921b5259d5400cbf5e90fc51931376a9bd4a0eb79aa7210f"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn 1.0.109",
- "unicode-xid",
-]
-
 [[package]]
 name = "synstructure"
 version = "0.13.1"
@@ -4640,12 +4786,6 @@ version = "0.1.14"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af"

-[[package]]
-name = "unicode-xid"
-version = "0.2.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853"
-
 [[package]]
 name = "universal-hash"
 version = "0.5.1"
@@ -4736,6 +4876,29 @@ version = "0.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65"

+[[package]]
+name = "value-log"
+version = "1.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "62fc7c4ce161f049607ecea654dca3f2d727da5371ae85e2e4f14ce2b98ed67c"
+dependencies = [
+ "byteorder",
+ "byteview",
+ "interval-heap",
+ "log",
+ "path-absolutize",
+ "rustc-hash",
+ "tempfile",
+ "varint-rs",
+ "xxhash-rust",
+]
+
+[[package]]
+name = "varint-rs"
+version = "2.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8f54a172d0620933a27a4360d3db3e2ae0dd6cceae9730751a036bbf182c4b23"
+
 [[package]]
 name = "vcpkg"
 version = "0.2.15"
@@ -5148,7 +5311,7 @@ dependencies = [
 "proc-macro2",
 "quote",
 "syn 2.0.100",
- "synstructure 0.13.1",
+ "synstructure",
 ]

 [[package]]
@@ -5209,7 +5372,7 @@ dependencies = [
 "proc-macro2",
 "quote",
 "syn 2.0.100",
- "synstructure 0.13.1",
+ "synstructure",
 ]

 [[package]]
@@ -24,18 +24,18 @@ default-members = ["src/garage"]

 # Internal Garage crates
 format_table = { version = "0.1.1", path = "src/format-table" }
-garage_api_common = { version = "2.0.0", path = "src/api/common" }
-garage_api_admin = { version = "2.0.0", path = "src/api/admin" }
-garage_api_s3 = { version = "2.0.0", path = "src/api/s3" }
-garage_api_k2v = { version = "2.0.0", path = "src/api/k2v" }
-garage_block = { version = "2.0.0", path = "src/block" }
-garage_db = { version = "2.0.0", path = "src/db", default-features = false }
-garage_model = { version = "2.0.0", path = "src/model", default-features = false }
-garage_net = { version = "2.0.0", path = "src/net" }
-garage_rpc = { version = "2.0.0", path = "src/rpc" }
-garage_table = { version = "2.0.0", path = "src/table" }
-garage_util = { version = "2.0.0", path = "src/util" }
-garage_web = { version = "2.0.0", path = "src/web" }
+garage_api_common = { version = "2.1.0", path = "src/api/common" }
+garage_api_admin = { version = "2.1.0", path = "src/api/admin" }
+garage_api_s3 = { version = "2.1.0", path = "src/api/s3" }
+garage_api_k2v = { version = "2.1.0", path = "src/api/k2v" }
+garage_block = { version = "2.1.0", path = "src/block" }
+garage_db = { version = "2.1.0", path = "src/db", default-features = false }
+garage_model = { version = "2.1.0", path = "src/model", default-features = false }
+garage_net = { version = "2.1.0", path = "src/net" }
+garage_rpc = { version = "2.1.0", path = "src/rpc" }
+garage_table = { version = "2.1.0", path = "src/table" }
+garage_util = { version = "2.1.0", path = "src/util" }
+garage_web = { version = "2.1.0", path = "src/web" }
 k2v-client = { version = "0.0.4", path = "src/k2v-client" }

 # External crates from crates.io
@@ -49,9 +49,8 @@ bytes = "1.0"
 bytesize = "1.1"
 cfg-if = "1.0"
 chrono = { version = "0.4", features = ["serde"] }
-crc-fast = "1.1"
+crc-fast = "1.6"
 crypto-common = "0.1"
-err-derive = "0.3"
 gethostname = "0.4"
 git-version = "0.3.4"
 hex = "0.4"
@@ -64,6 +63,7 @@ md-5 = "0.10"
 mktemp = "0.5"
 nix = { version = "0.29", default-features = false, features = ["fs"] }
 nom = "7.1"
+parking_lot = "0.12"
 parse_duration = "2.1"
 paste = "1.0"
 pin-project = "1.0.12"
@@ -87,9 +87,10 @@ tracing-journald = "0.3.1"
 tracing-subscriber = { version = "0.3", features = ["env-filter"] }

 heed = { version = "0.11", default-features = false, features = ["lmdb"] }
-rusqlite = "0.31.0"
+rusqlite = "0.37"
 r2d2 = "0.8"
-r2d2_sqlite = "0.24"
+r2d2_sqlite = "0.31"
+fjall = "2.4"

 async-compression = { version = "0.4", features = ["tokio", "zstd"] }
 zstd = { version = "0.13", default-features = false }
@@ -136,7 +137,7 @@ prometheus = "0.13"
 aws-sigv4 = { version = "1.1", default-features = false }
 hyper-rustls = { version = "0.26", default-features = false, features = ["http1", "http2", "ring", "rustls-native-certs"] }
 log = "0.4"
-thiserror = "1.0"
+thiserror = "2.0"

 # ---- used only as build / dev dependencies ----
 assert-json-diff = "2.0"
@@ -153,5 +154,5 @@ lto = "off"
 [profile.release]
 lto = true
 codegen-units = 1
-opt-level = "s"
+opt-level = 3
 strip = true
@@ -12,7 +12,7 @@
      "name": "AGPL-3.0",
      "identifier": "AGPL-3.0"
    },
-    "version": "v2.0.0"
+    "version": "v2.1.0"
  },
  "servers": [
    {
@@ -31,9 +31,12 @@
        "parameters": [
          {
            "name": "domain",
-            "in": "path",
+            "in": "query",
            "description": "The domain name to check for",
-            "required": true
+            "required": true,
+            "schema": {
+              "type": "string"
+            }
          }
        ],
        "responses": {
@@ -405,9 +408,12 @@
        "parameters": [
          {
            "name": "node",
-            "in": "path",
+            "in": "query",
            "description": "Node ID to query, or `*` for all nodes, or `self` for the node responding to the request",
-            "required": true
+            "required": true,
+            "schema": {
+              "type": "string"
+            }
          }
        ],
        "responses": {
@@ -437,9 +443,12 @@
        "parameters": [
          {
            "name": "id",
-            "in": "path",
+            "in": "query",
            "description": "Admin API token ID",
-            "required": true
+            "required": true,
+            "schema": {
+              "type": "string"
+            }
          }
        ],
        "responses": {
@@ -462,9 +471,12 @@
        "parameters": [
          {
            "name": "id",
-            "in": "path",
+            "in": "query",
            "description": "ID of the bucket to delete",
-            "required": true
+            "required": true,
+            "schema": {
+              "type": "string"
+            }
          }
        ],
        "responses": {
@@ -493,9 +505,12 @@
        "parameters": [
          {
            "name": "id",
-            "in": "path",
+            "in": "query",
            "description": "Access key ID",
-            "required": true
+            "required": true,
+            "schema": {
+              "type": "string"
+            }
          }
        ],
        "responses": {
@@ -552,26 +567,20 @@
        "parameters": [
          {
            "name": "id",
-            "in": "path",
+            "in": "query",
            "description": "Admin API token ID",
-            "required": true,
+            "required": false,
            "schema": {
-              "type": [
-                "string",
-                "null"
-              ]
+              "type": "string"
            }
          },
          {
            "name": "search",
-            "in": "path",
+            "in": "query",
            "description": "Partial token ID or name to search for",
-            "required": true,
+            "required": false,
            "schema": {
-              "type": [
-                "string",
-                "null"
-              ]
+              "type": "string"
            }
          }
        ],
@@ -602,9 +611,12 @@
        "parameters": [
          {
            "name": "node",
-            "in": "path",
+            "in": "query",
            "description": "Node ID to query, or `*` for all nodes, or `self` for the node responding to the request",
-            "required": true
+            "required": true,
+            "schema": {
+              "type": "string"
+            }
          }
        ],
        "requestBody": {
@@ -644,38 +656,29 @@
        "parameters": [
          {
            "name": "id",
-            "in": "path",
+            "in": "query",
            "description": "Exact bucket ID to look up",
-            "required": true,
+            "required": false,
            "schema": {
-              "type": [
-                "string",
-                "null"
-              ]
+              "type": "string"
            }
          },
          {
            "name": "globalAlias",
-            "in": "path",
+            "in": "query",
            "description": "Global alias of bucket to look up",
-            "required": true,
+            "required": false,
            "schema": {
-              "type": [
-                "string",
-                "null"
-              ]
+              "type": "string"
            }
          },
          {
            "name": "search",
-            "in": "path",
+            "in": "query",
            "description": "Partial ID or alias to search for",
-            "required": true,
+            "required": false,
            "schema": {
-              "type": [
-                "string",
-                "null"
-              ]
+              "type": "string"
            }
          }
        ],
@@ -813,6 +816,30 @@
        }
      }
    },
+    "/v2/GetCurrentAdminTokenInfo": {
+      "get": {
+        "tags": [
+          "Admin API token"
+        ],
+        "description": "\nReturn information about the calling admin API token.\n    ",
+        "operationId": "GetCurrentAdminTokenInfo",
+        "responses": {
+          "200": {
+            "description": "Information about the admin token",
+            "content": {
+              "application/json": {
+                "schema": {
+                  "$ref": "#/components/schemas/GetCurrentAdminTokenInfoResponse"
+                }
+              }
+            }
+          },
+          "500": {
+            "description": "Internal server error"
+          }
+        }
+      }
+    },
    "/v2/GetKeyInfo": {
      "get": {
        "tags": [
@@ -823,33 +850,27 @@
        "parameters": [
          {
            "name": "id",
-            "in": "path",
+            "in": "query",
            "description": "Access key ID",
-            "required": true,
+            "required": false,
            "schema": {
-              "type": [
-                "string",
-                "null"
-              ]
+              "type": "string"
            }
          },
          {
            "name": "search",
-            "in": "path",
+            "in": "query",
            "description": "Partial key ID or name to search for",
-            "required": true,
+            "required": false,
            "schema": {
-              "type": [
-                "string",
-                "null"
-              ]
+              "type": "string"
            }
          },
          {
            "name": "showSecretKey",
-            "in": "path",
+            "in": "query",
            "description": "Whether to return the secret access key",
-            "required": true,
+            "required": false,
            "schema": {
              "type": "boolean"
            }
@@ -882,9 +903,12 @@
        "parameters": [
          {
            "name": "node",
-            "in": "path",
+            "in": "query",
            "description": "Node ID to query, or `*` for all nodes, or `self` for the node responding to the request",
-            "required": true
+            "required": true,
+            "schema": {
+              "type": "string"
+            }
          }
        ],
        "responses": {
@@ -914,9 +938,12 @@
        "parameters": [
          {
            "name": "node",
-            "in": "path",
+            "in": "query",
            "description": "Node ID to query, or `*` for all nodes, or `self` for the node responding to the request",
-            "required": true
+            "required": true,
+            "schema": {
+              "type": "string"
+            }
          }
        ],
        "responses": {
@@ -946,9 +973,12 @@
        "parameters": [
          {
            "name": "node",
-            "in": "path",
+            "in": "query",
            "description": "Node ID to query, or `*` for all nodes, or `self` for the node responding to the request",
-            "required": true
+            "required": true,
+            "schema": {
+              "type": "string"
+            }
          }
        ],
        "requestBody": {
@@ -988,9 +1018,12 @@
        "parameters": [
          {
            "name": "node",
-            "in": "path",
+            "in": "query",
            "description": "Node ID to query, or `*` for all nodes, or `self` for the node responding to the request",
-            "required": true
+            "required": true,
+            "schema": {
+              "type": "string"
+            }
          }
        ],
        "requestBody": {
@@ -1064,7 +1097,7 @@
        "parameters": [
          {
            "name": "bucketId",
-            "in": "path",
+            "in": "query",
            "required": true,
            "schema": {
              "type": "string"
@@ -1072,7 +1105,7 @@
          },
          {
            "name": "key",
-            "in": "path",
+            "in": "query",
            "required": true,
            "schema": {
              "type": "string"
@@ -1109,9 +1142,12 @@
        "parameters": [
          {
            "name": "node",
-            "in": "path",
+            "in": "query",
            "description": "Node ID to query, or `*` for all nodes, or `self` for the node responding to the request",
-            "required": true
+            "required": true,
+            "schema": {
+              "type": "string"
+            }
          }
        ],
        "requestBody": {
@@ -1175,9 +1211,12 @@
        "parameters": [
          {
            "name": "node",
-            "in": "path",
+            "in": "query",
            "description": "Node ID to query, or `*` for all nodes, or `self` for the node responding to the request",
-            "required": true
+            "required": true,
+            "schema": {
+              "type": "string"
+            }
          }
        ],
        "responses": {
@@ -1255,9 +1294,12 @@
        "parameters": [
          {
            "name": "node",
-            "in": "path",
+            "in": "query",
            "description": "Node ID to query, or `*` for all nodes, or `self` for the node responding to the request",
-            "required": true
+            "required": true,
+            "schema": {
+              "type": "string"
+            }
          }
        ],
        "requestBody": {
@@ -1321,9 +1363,12 @@
        "parameters": [
          {
            "name": "node",
-            "in": "path",
+            "in": "query",
            "description": "Node ID to query, or `*` for all nodes, or `self` for the node responding to the request",
-            "required": true
+            "required": true,
+            "schema": {
+              "type": "string"
+            }
          }
        ],
        "requestBody": {
@@ -1397,9 +1442,12 @@
        "parameters": [
          {
            "name": "node",
-            "in": "path",
+            "in": "query",
            "description": "Node ID to query, or `*` for all nodes, or `self` for the node responding to the request",
-            "required": true
+            "required": true,
+            "schema": {
+              "type": "string"
+            }
          }
        ],
        "requestBody": {
@@ -1463,9 +1511,12 @@
        "parameters": [
          {
            "name": "node",
-            "in": "path",
+            "in": "query",
            "description": "Node ID to query, or `*` for all nodes, or `self` for the node responding to the request",
-            "required": true
+            "required": true,
+            "schema": {
+              "type": "string"
+            }
          }
        ],
        "requestBody": {
@@ -1505,9 +1556,12 @@
        "parameters": [
          {
            "name": "id",
-            "in": "path",
+            "in": "query",
            "description": "Admin API token ID",
-            "required": true
+            "required": true,
+            "schema": {
+              "type": "string"
+            }
          }
        ],
        "requestBody": {
@@ -1547,9 +1601,12 @@
        "parameters": [
          {
            "name": "id",
-            "in": "path",
+            "in": "query",
            "description": "ID of the bucket to update",
-            "required": true
+            "required": true,
+            "schema": {
+              "type": "string"
+            }
          }
        ],
        "requestBody": {
@@ -1627,9 +1684,12 @@
        "parameters": [
          {
            "name": "id",
-            "in": "path",
+            "in": "query",
            "description": "Access key ID",
-            "required": true
+            "required": true,
+            "schema": {
+              "type": "string"
+            }
          }
        ],
        "requestBody": {
@@ -2383,7 +2443,7 @@
          "knownNodes",
          "connectedNodes",
          "storageNodes",
-          "storageNodesOk",
+          "storageNodesUp",
          "partitions",
          "partitionsQuorum",
          "partitionsAllOk"
@@ -2423,7 +2483,7 @@
            "description": "the number of storage nodes currently registered in the cluster layout",
            "minimum": 0
          },
-          "storageNodesOk": {
+          "storageNodesUp": {
            "type": "integer",
            "description": "the number of storage nodes to which a connection is currently open",
            "minimum": 0
@@ -2558,6 +2618,9 @@
          }
        }
      },
+      "GetCurrentAdminTokenInfoResponse": {
+        "$ref": "#/components/schemas/GetAdminTokenInfoResponse"
+      },
      "GetKeyInfoResponse": {
        "type": "object",
        "required": [
@@ -3991,6 +4054,12 @@
            "enum": [
              "aliases"
            ]
+          },
+          {
+            "type": "string",
+            "enum": [
+              "clearResyncQueue"
+            ]
          }
        ]
      },
@@ -12,7 +12,7 @@ In this section, we cover the following web applications:
 | [Mastodon](#mastodon)     | ✅       | Natively supported    |
 | [Matrix](#matrix)     | ✅       |  Tested with `synapse-s3-storage-provider`    |
 | [ejabberd](#ejabberd)     | ✅       |  `mod_s3_upload`    |
-| [Pixelfed](#pixelfed)     | ❓       |  Not yet tested    |
+| [Pixelfed](#pixelfed)     | ✅       |  Natively supported    |
 | [Pleroma](#pleroma)     | ❓       |  Not yet tested    |
 | [Lemmy](#lemmy)     | ✅        |  Supported with pict-rs    |
 | [Funkwhale](#funkwhale)     | ❓       | Not yet tested     |
@@ -191,10 +191,10 @@ garage key create peertube-key

 Keep the Key ID and the Secret key in a pad, they will be needed later.  

-We need two buckets, one for normal videos (named peertube-video) and one for webtorrent videos (named peertube-playlist).
+We need two buckets, one for normal videos (named peertube-videos) and one for webtorrent videos (named peertube-playlists).
 ```bash
 garage bucket create peertube-videos
-garage bucket create peertube-playlist
+garage bucket create peertube-playlists
 ```

 Now we allow our key to read and write on these buckets:
@@ -253,7 +253,7 @@ object_storage:
    proxify_private_files: false

  streaming_playlists:
-    bucket_name: 'peertube-playlist'
+    bucket_name: 'peertube-playlists'

    # Keep it empty for our example
    prefix: ''
@@ -161,3 +161,49 @@ kopia repository validate-provider

 You can then run all the standard kopia commands: `kopia snapshot create`, `kopia mount`...
 Everything should work out-of-the-box.
+
+## Plakar
+
+Create your key and bucket on Garage server:
+
+```bash
+garage key create my-plakar-key
+garage bucket create plakar-backups
+garage bucket allow plakar-backups --read --write --key my-plakar-key
+…
+```
+
+On Plakar server, add your Garage as a storage location:
+```bash
+plakar store add garageS3 s3://my-garage.tld/plakar-backups \
+region=garage # Or as you've specified in garage.toml \
+access_key=<Key ID from "garage key info my-plakar-key"> \
+secret_access_key=<Secret key from "garage key info my-plakar-key">
+```
+
+Then create the repository.
+```bash
+plakar at @garageS3 create -plaintext # Unencrypted
+# or
+plakar at @garageS3 create #encrypted
+```
+
+If you encrypt your backups (Plakar default), you will need to define a strong passphrase. Do not forget to save your password safely. It will be needed to decrypt your backups.
+
+
+After the repository has been created, check that everything works as expected (that might give an empty result as no file has been added yet, but no error message):
+```bash
+plakar at @garageS3 check
+```
+
+Now that everything is configure, you can use Garage as your backups storage. For instance sync it with a local backup storage:
+```bash
+$ plakar at ~/backups sync to @garageS3
+```
+
+Or list the S3 storage content:
+```bash
+$ plakar at @garageS3 ls
+```
+
+More information in Plakar documentation: https://www.plakar.io/docs/main/quickstart/
@@ -8,7 +8,7 @@ have published Ansible roles.  We list them and compare them below.

 ## Comparison of Ansible roles

-| Feature                            | [ansible-role-garage](#zorun-ansible-role-garage) | [garage-docker-ansible-deploy](#moan0s-garage-docker-ansible-deploy) | [eddster ansible-role-garage](#eddster-ansible-role-garage) |
+| Feature                            | [ansible-role-garage](#zorun-ansible-role-garage) | [garage-docker-ansible-deploy](#moan0s-garage-docker-ansible-deploy) | [eddster2309 ansible-role-garage](#eddster2309-ansible-role-garage) |
 |------------------------------------|---------------------------------------------|---------------------------------------------------------------|---------------------------------|
 | **Runtime**                        | Systemd                                     | Docker                                                        | Systemd                         |
 | **Target OS**                      | Any Linux                                   | Any Linux                                                     | Any Linux                       |
@@ -96,14 +96,14 @@ to store 2 TB of data in total.
 ## Get a Docker image

 Our docker image is currently named `dxflrs/garage` and is stored on the [Docker Hub](https://hub.docker.com/r/dxflrs/garage/tags?page=1&ordering=last_updated).
-We encourage you to use a fixed tag (eg. `v2.0.0`) and not the `latest` tag.
-For this example, we will use the latest published version at the time of the writing which is `v2.0.0` but it's up to you
+We encourage you to use a fixed tag (eg. `v2.1.0`) and not the `latest` tag.
+For this example, we will use the latest published version at the time of the writing which is `v2.1.0` but it's up to you
 to check [the most recent versions on the Docker Hub](https://hub.docker.com/r/dxflrs/garage/tags?page=1&ordering=last_updated).

 For example:

 ```
-sudo docker pull dxflrs/garage:v2.0.0
+sudo docker pull dxflrs/garage:v2.1.0
 ```

 ## Deploying and configuring Garage
@@ -171,7 +171,7 @@ docker run \
  -v /etc/garage.toml:/etc/garage.toml \
  -v /var/lib/garage/meta:/var/lib/garage/meta \
  -v /var/lib/garage/data:/var/lib/garage/data \
-  dxflrs/garage:v2.0.0
+  dxflrs/garage:v2.1.0
 ```

 With this command line, Garage should be started automatically at each boot.
@@ -185,7 +185,7 @@ If you want to use `docker-compose`, you may use the following `docker-compose.y
 version: "3"
 services:
  garage:
-    image: dxflrs/garage:v2.0.0
+    image: dxflrs/garage:v2.1.0
    network_mode: "host"
    restart: unless-stopped
    volumes:
@@ -97,7 +97,7 @@ server {
  location / {
    proxy_pass http://s3_backend;
    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
-    proxy_set_header Host $host;
+    proxy_set_header Host $http_host;
    # Disable buffering to a temporary file.
    proxy_max_temp_file_size 0;
  }
@@ -59,11 +59,13 @@ Garage themselves for the following tasks:

 - Hosting of their homepage, [privacyguides.org](https://www.privacyguides.org/), and various other static sites

- As a Mastodon object storage backend for [mstdn.party](https://mstdn.party/) and [mstdn.plus](https://mstdn.plus/)
+- As a PowerDNS authoritative zone backend through [Lightning Stream](https://doc.powerdns.com/lightningstream/latest/index.html) and [LMDB](https://doc.powerdns.com/authoritative/backends/lmdb.html)
+
+- As a Mastodon media storage backend for [mstdn.party](https://mstdn.party/) and [mstdn.plus](https://mstdn.plus/)

 - As a PeerTube storage backend for [neat.tube](https://neat.tube/)

 - As a [Matrix media backend](https://github.com/matrix-org/synapse-s3-storage-provider)

 Triplebit's Garage cluster is a multi-site cluster currently composed of
-10 nodes in 3 physical locations.
+15 storage nodes in 3 physical locations.
@@ -42,7 +42,7 @@ You may pause an ongoing scrub using `garage repair scrub pause`, but note that
 the scrub will resume automatically 24 hours later as Garage will not let your
 cluster run without a regular scrub. If the scrub procedure is too intensive
 for your servers and is slowing down your workload, the recommended solution
-is to increase the "scrub tranquility" using `garage repair scrub set-tranquility`.
+is to increase the "scrub tranquility" using `garage worker set scrub-tranquility`.
 A higher tranquility value will make Garage take longer pauses between two block
 verifications. Of course, scrubbing the entire data store will also take longer.

@@ -161,4 +161,7 @@ your recovery options are as follows:

 - **Option 3: restoring a filesystem-level snapshot.** If you are using ZFS or
  BTRFS to snapshot your metadata partition, refer to their specific
-  documentation on rolling back or copying files from an old snapshot.
+  documentation on rolling back or copying files from an old snapshot. 
+  Note that, depending on the properties of the filesystem and of the DB engine,
+  if these snapshots were taken during a write operation to the database, they may
+  also be corrupted and thus unfit for recovery.
@@ -132,7 +132,7 @@ docker run \
  -v /path/to/garage.toml:/etc/garage.toml \
  -v /path/to/garage/meta:/var/lib/garage/meta \
  -v /path/to/garage/data:/var/lib/garage/data \
-  dxflrs/garage:v2.0.0
+  dxflrs/garage:v2.1.0
 ```

 Under Linux, you can substitute `--network host` for `-p 3900:3900 -p 3901:3901 -p 3902:3902 -p 3903:3903`
@@ -6,41 +6,167 @@ weight = 40
 The Garage administration API is accessible through a dedicated server whose
 listen address is specified in the `[admin]` section of the configuration
 file (see [configuration file
-reference](@/documentation/reference-manual/configuration.md))
+reference](@/documentation/reference-manual/configuration.md)).

-**WARNING.** At this point, there is no commitment to the stability of the APIs described in this document.
-We will bump the version numbers prefixed to each API endpoint each time the syntax
-or semantics change, meaning that code that relies on these endpoint will break
-when changes are introduced.
-
-Versions:
- - Before Garage 0.7.2 - no admin API
- - Garage 0.7.2 - admin APIv0
- - Garage 0.9.0 - admin APIv1, deprecate admin APIv0
+The current version of the admin API is v2.  No breaking changes to the Garage
+administration API will be published outside of a major release.

+History of previous versions:

+ - Before Garage v0.7.2 - no admin API
+ - Garage v0.7.2 - admin API v0
+ - Garage v0.9.0 - admin API v1, deprecate admin API v0
+ - Garage v2.0.0 - admin API v2, deprecate admin API v1

 ## Access control

-The admin API uses two different tokens for access control, that are specified in the config file's `[admin]` section:
+### Using an API token

- `metrics_token`: the token for accessing the Metrics endpoint (if this token
-  is not set in the config file, the Metrics endpoint can be accessed without
-  access control);
-
- `admin_token`: the token for accessing all of the other administration
-  endpoints (if this token is not set in the config file, access to these
-  endpoints is disabled entirely).
-
-These tokens are used as simple HTTP bearer tokens. In other words, to
-authenticate access to an admin API endpoint, add the following HTTP header
-to your request:
+Administration API tokens tokens are used as simple HTTP bearer tokens. In
+other words, to authenticate access to an admin API endpoint, add the following
+HTTP header to your request:

 ```
 Authorization: Bearer <token>
 ```

-## Administration API endpoints
+### User-defined API tokens
+
+Cluster administrators may dynamically define administration tokens using the CLI commands under `garage admin-token`.
+Such tokens may be limited in scope, meaning that they may enable access to only a subset of API calls.
+They may also have an expiration date to limit their use in time.
+
+Here is an example to create an administration token that is valid for 30 days
+and gives access to only a subset of API calls, allowing it to create buckets
+and access keys and give keys permissions on buckets:
+
+```bash
+$ garage admin-token create --expires-in 30d \
+    --scope ListBuckets,GetBucketInfo,ListKeys,GetKeyInfo,CreateBucket,CreateKey,AllowBucketKey,DenyBucketKey \
+    my-token
+This is your secret bearer token, it will not be shown again by Garage:
+
+  8ed1830b10a276ff57061950.kOSIpxWK9zSGbTO9Xadpv3YndSFWma0_snXcYHaORXk
+
+==== ADMINISTRATION TOKEN INFORMATION ====
+Token ID:    8ed1830b10a276ff57061950
+Token name:  my-token
+Created:     2025-06-15 15:12:44.160 +02:00
+Validity:    valid
+Expiration:  2025-07-15 15:12:44.117 +02:00
+
+Scope:       ListBuckets
+             GetBucketInfo
+             ListKeys
+             GetKeyInfo
+             CreateBucket
+             CreateKey
+             AllowBucketKey
+             DenyBucketKey
+```
+
+When running this command, your token will be shown only once and **will never
+be shown again by Garage**, so make sure to save it directly.  The token is
+hashed internally, and is identified by its prefix (32 hex digits followed by a
+dot) which is saved in clear.
+
+When running `garage admin-token list`, you might see something like this:
+
+```
+ID                        Created     Name                                       Expiration                      Scope
+-                         -           metrics_token (from daemon configuration)  never                           Metrics
+8ed1830b10a276ff57061950  2025-06-15  my-token                                   2025-07-15 15:12:44.117 +02:00  ListBuckets, ... (8)
+```
+
+### Master API tokens
+
+The admin API can also use two different master tokens for access control,
+specified in the config file's `[admin]` section:
+
+- `metrics_token`: the token for accessing the Metrics endpoint. If this token
+  is not set in the config file, the Metrics endpoint can be accessed without
+  access control.
+
+- `admin_token`: the token for accessing all of the other administration
+  endpoints. If this token is not set in the config file, access to these
+  endpoints is only possible with a user-defined admin token.
+
+With the introduction of multiple user-defined admin tokens, the use of master
+API tokens is now discouraged.
+
+
+## Using the admin API
+
+All of the admin API endpoints are described in the OpenAPI specification:
+
+ - APIv2 - [HTML spec](https://garagehq.deuxfleurs.fr/api/garage-admin-v2.html) - [OpenAPI JSON](https://garagehq.deuxfleurs.fr/api/garage-admin-v2.json)
+ - APIv1 (deprecated) - [HTML spec](https://garagehq.deuxfleurs.fr/api/garage-admin-v1.html) - [OpenAPI YAML](https://garagehq.deuxfleurs.fr/api/garage-admin-v1.yml)
+ - APIv0 (deprecated) - [HTML spec](https://garagehq.deuxfleurs.fr/api/garage-admin-v0.html) - [OpenAPI YAML](https://garagehq.deuxfleurs.fr/api/garage-admin-v0.yml)
+
+Making a request to the API from the command line can be as simple as running:
+
+```bash
+curl -H 'Authorization: Bearer s3cr3t' http://localhost:3903/v2/GetClusterStatus | jq
+```
+
+For more advanced use cases, we recommend using an SDK.  
+[Go to the "Build your own app" section to know how to use our SDKs](@/documentation/build/_index.md)
+
+### Making API calls from the `garage` CLI
+
+Since v2.0.0, the `garage` binary provides a subcommand `garage json-api` that
+allows you to invoke the API without making an HTTP request.  This can be
+useful for scripting Garage deployments.
+
+`garage json-api` proxies API calls through Garage's internal RPC protocol,
+therefore it does not require any form of authentication: RPC connection
+parameters are discovered automatically to contact the locally-running Garage
+instance (as when running any other `garage` CLI command).
+
+For simple calls that take no parameters, usage is as follows:
+
+```
+$ garage json-api GetClusterHealth
+{
+  "connectedNodes": 3,
+  "knownNodes": 3,
+  "partitions": 256,
+  "partitionsAllOk": 256,
+  "partitionsQuorum": 256,
+  "status": "healthy",
+  "storageNodes": 3,
+  "storageNodesOk": 3
+}
+```
+
+If you need to specify a JSON body for your call, you can add it directly after
+the name of the function you are calling:
+
+```
+$ garage json-api CreateAdminToken '{"name": "test"}'
+```
+
+Or you can feed it through stdin by adding a `-` as the last command parameter:
+
+```
+$ garage json-api CreateAdminToken -
+{"name": "test"}
+<EOF>
+```
+
+For admin API calls that would have taken query parameters in their HTTP version, these parameters can be passed in the JSON body object:
+
+```
+$ garage json-api GetAdminTokenInfo '{"id":"b0e6e0ace2c0b2aca4cdb2de"}'
+```
+
+For admin API calls that take both query parameters and a JSON body, combine them in the following fashion:
+
+```
+$ garage json-api UpdateAdminToken '{"id":"b0e6e0ace2c0b2aca4cdb2de", "body":{"name":"not a test"}}'
+```
+
+## Special administration API endpoints

 ### Metrics `GET /metrics`

@@ -83,7 +209,7 @@ content-length: 102
 date: Tue, 08 Aug 2023 07:22:38 GMT

 Garage is fully operational
-Consult the full health check API endpoint at /v0/health for more details
+Consult the full health check API endpoint at /v2/GetClusterHealth for more details
 ```

 ### On-demand TLS `GET /check`
@@ -126,23 +252,7 @@ $ curl -so /dev/null -w "%{http_code}" http://localhost:3903/check?domain=exampl
 200
 ```

-
 **References:**
 - [Using On-Demand TLS](https://caddyserver.com/docs/automatic-https#using-on-demand-tls)
 - [Add option for a backend check to approve use of on-demand TLS](https://github.com/caddyserver/caddy/pull/1939)
 - [Serving tens of thousands of domains over HTTPS with Caddy](https://caddy.community/t/serving-tens-of-thousands-of-domains-over-https-with-caddy/11179)
-
-### Cluster operations
-
-These endpoints have a dedicated OpenAPI spec.
- - APIv1 - [HTML spec](https://garagehq.deuxfleurs.fr/api/garage-admin-v1.html) - [OpenAPI YAML](https://garagehq.deuxfleurs.fr/api/garage-admin-v1.yml)
- - APIv0 (deprecated) - [HTML spec](https://garagehq.deuxfleurs.fr/api/garage-admin-v0.html) - [OpenAPI YAML](https://garagehq.deuxfleurs.fr/api/garage-admin-v0.yml)
-
-Requesting the API from the command line can be as simple as running:
-
-```bash
-curl -H 'Authorization: Bearer s3cr3t' http://localhost:3903/v0/status | jq
-```
-
-For more advanced use cases, we recommend using a SDK.  
-[Go to the "Build your own app" section to know how to use our SDKs](@/documentation/build/_index.md)
@@ -24,6 +24,7 @@ db_engine = "lmdb"

 block_size = "1M"
 block_ram_buffer_max = "256MiB"
+block_max_concurrent_reads = 16

 lmdb_map_size = "1T"

@@ -97,6 +98,7 @@ The following gives details about each available configuration option.
 Top-level configuration options, in alphabetical order:
 [`allow_punycode`](#allow_punycode),
 [`allow_world_readable_secrets`](#allow_world_readable_secrets),
+[`block_max_concurrent_reads`](#block_max_concurrent_reads),
 [`block_ram_buffer_max`](#block_ram_buffer_max),
 [`block_size`](#block_size),
 [`bootstrap_peers`](#bootstrap_peers),
@@ -335,6 +337,7 @@ Since `v0.8.0`, Garage can use alternative storage backends as follows:
 | --------- | ----------------- | ------------- |
 | [LMDB](https://www.symas.com/lmdb) (since `v0.8.0`, default since `v0.9.0`) | `"lmdb"` | `<metadata_dir>/db.lmdb/` |
 | [Sqlite](https://sqlite.org) (since `v0.8.0`) | `"sqlite"` | `<metadata_dir>/db.sqlite` |
+| [Fjall](https://github.com/fjall-rs/fjall) (**experimental support** since `v1.3.0`/`v2.1.0`) | `"fjall"` | `<metadata_dir>/db.fjall/` |
 | [Sled](https://sled.rs) (old default, removed since `v1.0`) | `"sled"` | `<metadata_dir>/db/` |

 Sled was supported until Garage v0.9.x, and was removed in Garage v1.0.
@@ -343,8 +346,16 @@ old Sled metadata databases to another engine.

 Performance characteristics of the different DB engines are as follows:

- LMDB: the recommended database engine for high-performance distributed clusters.
-LMDB works very well, but is known to have the following limitations:
+- **LMDB:** the recommended database engine for high-performance distributed clusters
+  with `replication_factor` ≥ 2.
+  LMDB works well, but is known to have the following limitations:
+
+  - LMDB is prone to database corruption after an unclean shutdown (e.g. a process kill
+    or a power outage).  It is recommended to configure
+    [`metadata_auto_snapshot_interval`](#metadata_auto_snapshot_interval) to be
+    able to easily recover from this situation. With `replication_factor` ≥ 2,
+    metadata can also be reconstructed from remote nodes upon corruption
+    (see [Recovering from failures](@/documentation/operations/recovering.md#corrupted_meta)).

  - The data format of LMDB is not portable between architectures, so for
    instance the Garage database of an x86-64 node cannot be moved to an ARM64
@@ -354,22 +365,21 @@ LMDB works very well, but is known to have the following limitations:
    node to very small database sizes due to how LMDB works; it is therefore
    not recommended.

-  - Several users have reported corrupted LMDB database files after an unclean
-    shutdown (e.g. a power outage). This situation can generally be recovered
-    from if your cluster is geo-replicated (by rebuilding your metadata db from
-    other nodes), or if you have saved regular snapshots at the filesystem
-    level.
-
  - Keys in LMDB are limited to 511 bytes. This limit translates to limits on
    object keys in S3 and sort keys in K2V that are limted to 479 bytes.

- Sqlite: Garage supports Sqlite as an alternative storage backend for
-  metadata, which does not have the issues listed above for LMDB.
-  On versions 0.8.x and earlier, Sqlite should be avoided due to abysmal
-  performance, which was fixed with the addition of `metadata_fsync`.
-  Sqlite is still probably slower than LMDB due to the way we use it,
-  so it is not the best choice for high-performance storage clusters,
-  but it should work fine in many cases.
+- **Sqlite:** Garage supports Sqlite as an alternative storage backend for
+  metadata, which does not have the issues listed above for LMDB.  Sqlite is
+  slower than LMDB, so it is not the best choice for high-performance storage
+  clusters.
+
+- **Fjall:** a storage engine based on LSM trees, which theoretically allow for
+  higher write throughput than other storage engines that are based on B-trees.
+  Using Fjall could potentially improve Garage's performance significantly in
+  write-heavy workloads. **Support for Fjall is experimental at this point**,
+  we have added it to Garage for evaluation purposes only. **Use it only with
+  test data, and report any issues to our bug tracker. Do not use it for
+  production workloads.**

 It is possible to convert Garage's metadata directory from one format to another
 using the `garage convert-db` command, which should be used as follows:
@@ -408,6 +418,7 @@ Here is how this option impacts the different database engines:
 |----------|------------------------------------|-------------------------------|
 | Sqlite   | `PRAGMA synchronous = OFF`         | `PRAGMA synchronous = NORMAL` |
 | LMDB     | `MDB_NOMETASYNC` + `MDB_NOSYNC`    | `MDB_NOMETASYNC`              |
+| Fjall    | default options                    | not supported                 |

 Note that the Sqlite database is always ran in `WAL` mode (`PRAGMA journal_mode = WAL`).

@@ -427,7 +438,8 @@ if geographical replication is used.
 #### `metadata_auto_snapshot_interval` (since `v0.9.4`) {#metadata_auto_snapshot_interval}

 If this value is set, Garage will automatically take a snapshot of the metadata
-DB file at a regular interval and save it in the metadata directory.
+DB file at a regular interval and save it in the metadata directory,
+or in [`metadata_snapshots_dir`](#metadata_snapshots_dir) if it is set.
 This parameter can take any duration string that can be parsed by
 the [`parse_duration`](https://docs.rs/parse_duration/latest/parse_duration/#syntax) crate.

@@ -436,14 +448,19 @@ corrupted, for instance after an unclean shutdown.  See [this
 page](@/documentation/operations/recovering.md#corrupted_meta) for details.
 Garage keeps only the two most recent snapshots of the metadata DB and deletes
 older ones automatically.
+You can also create metadata snapshots manually at any point using the
+`garage meta snapshot` command.
+
+Using snapshots created by Garage is the best option to make snapshots of your
+node's metadata for potential recovery, as they are guaranteed to be clean and
+consistent, contrarily to filesystem-level snapshots that may be taken while
+some writes are in-flight and thus might be corrupted.

 Note that taking a metadata snapshot is a relatively intensive operation as the
 entire data file is copied. A snapshot being taken might have performance
 impacts on the Garage node while it is running. If the cluster is under heavy
 write load when a snapshot operation is running, this might also cause the
 database file to grow in size significantly as pages cannot be recycled easily.
-For this reason, it might be better to use filesystem-level snapshots instead
-if possible.

 #### `disable_scrub` {#disable_scrub}

@@ -514,6 +531,29 @@ node.

 The default value is 256MiB.

+#### `block_max_concurrent_reads` (since `v1.3.0` / `v2.1.0`) {#block_max_concurrent_reads}
+
+The maximum number of blocks (individual files in the data directory) open
+simultaneously for reading.
+
+Reducing this number does not limit the number of data blocks that can be
+transferred through the network simultaneously. This mechanism was just added
+as a backpressure mechanism for HDD read speed: it helps avoid a situation
+where too many requests are coming in and Garage is reading too many block
+files simultaneously, thus not making timely progress on any of the reads.
+
+When a request to read a data block comes in through the network, the requests
+awaits for one of the `block_max_concurrent_reads` slots to be available
+(internally implemented using a Semaphore object). Once it acquired a read
+slot, it reads the entire block file to RAM and frees the slot as soon as the
+block file is finished reading. Only after the slot is released will the
+block's data start being transferred over the network.  If the request fails to
+acquire a reading slot wihtin 15 seconds, it fails with a timeout error.
+Timeout events can be monitored through the `block_read_semaphore_timeouts`
+metric in Prometheus: a non-zero number of such events indicates an I/O
+bottleneck on HDD read speed.
+
+
 #### `lmdb_map_size` {#lmdb_map_size}

 This parameters can be used to set the map size used by LMDB,
@@ -129,5 +129,5 @@ related to objects stored in an S3 bucket.
 In the context of our research project, [Aérogramme](https://aerogramme.deuxfleurs.fr),
 K2V is used to provide metadata and log storage for operations on encrypted e-mail storage.

-Learn more on the specification of K2V [here](https://git.deuxfleurs.fr/Deuxfleurs/garage/src/branch/k2v/doc/drafts/k2v-spec.md)
+Learn more on the specification of K2V [here](https://git.deuxfleurs.fr/Deuxfleurs/garage/src/commit/f8be15c37db857e177d543de7be863692628d567/doc/drafts/k2v-spec.md)
 and on how to enable it in Garage [here](@/documentation/reference-manual/k2v.md).
@@ -16,7 +16,7 @@ the `k2v` feature flag enabled can be obtained from our download page under
 with `-k2v` (example: `v0.7.2-k2v`).

 The specification of the K2V API can be found
-[here](https://git.deuxfleurs.fr/Deuxfleurs/garage/src/branch/main/doc/drafts/k2v-spec.md).
+[here](https://git.deuxfleurs.fr/Deuxfleurs/garage/src/commit/f8be15c37db857e177d543de7be863692628d567/doc/drafts/k2v-spec.md).
 This document also includes a high-level overview of K2V's design.

 The K2V API uses AWSv4 signatures for authentification, same as the S3 API.
@@ -1,6 +1,6 @@
 +++
 title = "Migrating from 0.3 to 0.4"
-weight = 20
+weight = 80
 +++

 **Migrating from 0.3 to 0.4 is unsupported. This document is only intended to
@@ -1,6 +1,6 @@
 +++
 title = "Migrating from 0.5 to 0.6"
-weight = 15
+weight = 75
 +++

 **This guide explains how to migrate to 0.6 if you have an existing 0.5 cluster.
@@ -1,6 +1,6 @@
 +++
 title = "Migrating from 0.6 to 0.7"
-weight = 14
+weight = 74
 +++
 **This guide explains how to migrate to 0.7 if you have an existing 0.6 cluster.
 We don't recommend trying to migrate to 0.7 directly from 0.5 or older.**
@@ -1,6 +1,6 @@
 +++
 title = "Migrating from 0.7 to 0.8"
-weight = 13
+weight = 73
 +++

 **This guide explains how to migrate to 0.8 if you have an existing 0.7 cluster.
@@ -1,6 +1,6 @@
 +++
 title = "Migrating from 0.8 to 0.9"
-weight = 12
+weight = 72
 +++

 **This guide explains how to migrate to 0.9 if you have an existing 0.8 cluster.
@@ -1,6 +1,6 @@
 +++
 title = "Migrating from 0.9 to 1.0"
-weight = 11
+weight = 71
 +++

 **This guide explains how to migrate to 1.0 if you have an existing 0.9 cluster.
@@ -0,0 +1,70 @@
+++
+title = "Migrating from 1.0 to 2.0"
+weight = 70
+++
+
+**This guide explains how to migrate to v2.x if you have an existing v1.x.x cluster.
+We don't recommend trying to migrate to v2.x directly from v0.9.x or older.**
+
+This migration procedure has been tested on several clusters without issues.
+However, it is still a *critical procedure* that might cause issues.
+**Make sure to back up all your data before attempting it!**
+
+You might also want to read our [general documentation on upgrading Garage](@/documentation/operations/upgrading.md).
+
+## Changes introduced in v2.0
+
+The following are **breaking changes** in Garage v2.0 that require your attention when migrating:
+
+- The administration API has been completely reworked.
+  Some calls to the `/v1/` endpoints will still work but most will not.
+  New endpoints are prefixed by `/v2/`. **You will need to update all your code that makes use of the admin API.**
+
+- `replication_mode` is no longer a supported configuration parameter,
+  please use `replication_factor` and `consistency_mode` instead.
+
+## Migration procedure
+
+The migration to Garage v2.0 can be done with almost no downtime,
+by restarting all nodes at once in the new version.
+
+The migration steps are as follows:
+
+1. Do a `garage repair --all-nodes --yes tables`, check the logs and check that
+   all data seems to be synced correctly between nodes. If you have time, do
+   additional `garage repair` procedures (`blocks`, `versions`, `block_refs`,
+   etc.)
+
+2. Ensure you have a snapshot of your Garage installation that you can restore
+   to in case the upgrade goes wrong, with one of the following options:
+
+   - You may use the `garage meta snapshot --all` command
+     to make a backup snapshot of the metadata directories of your nodes
+     for backup purposes. Once this command has completed, copy the following
+     files and directories from the `metadata_dir` of all your nodes
+     to somewhere safe: `snapshots`, `cluster_layout`, `data_layout`,
+     `node_key`, `node_key.pub`. (If you have set the `metadata_snapshots_dir`
+     to a different value in your config file, back up that directory instead.)
+
+   - If you are running a filesystem such as ZFS or BTRFS that support
+     snapshotting, you can create a filesystem-level snapshot of the `metadata_dir`
+     of all your nodes to be used as a restoration point if needed.
+
+   - You may also make a back-up manually: turn off each node
+     individually; back up its metadata folder (for instance, use the following
+     command if your metadata directory is `/var/lib/garage/meta`: `cd
+     /var/lib/garage ; tar -acf meta-v1.0.tar.zst meta/`); turn it back on
+     again.  This will allow you to take a backup of all nodes without
+     impacting global cluster availability.  You can do all nodes of a single
+     zone at once as this does not impact the availability of Garage.
+
+3. Prepare your updated binaries and configuration files for Garage v2.0.
+   **Remember to update your configuration file to remove `replication_mode` and replace it by `replication_factor`.**
+
+4. Shut down all v1.0 nodes simultaneously, and restart them all simultaneously
+   in v2.0.  Use your favorite deployment tool (Ansible, Kubernetes, Nomad) to
+   achieve this as fast as possible.  Garage v2.0 should be in a working state
+   as soon as enough nodes have started.
+
+5. Monitor your cluster in the following hours to see if it works well under
+   your production load.
@@ -1,6 +1,6 @@
 +++
 title = "Testing strategy"
-weight = 30
+weight = 100
 +++


@@ -0,0 +1,142 @@
+# [RFC] Garbage Collector Elimination
+
+## Statement of problem and prior art
+
+Currently, Garage's garbage collector has a few identified issues.
+Namely, it can only be run if all nodes in a partition are currently online,
+it may not be correct in front of a rebalancing (this is partially mitigated by a 24h delay added to tombstone deletion),
+and it isn't resilient to a subset of nodes being restored from snapshots.
+It's not clear if it is possible to implement a garbage collection process that can eliminate tombstones, but also support
+a node rollback to a point in time where a key existed.
+
+This problematic, perhaps unsurprisingly, maps very well to the general abstraction of CRDTs for sets, where a whole partition
+would be a single CRDT. The semantic required in Garage demands reinsertion of deleted keys, which excludes the most simple
+forms of sets such as G-Sets and 2P-Sets. As the goal is to not handle garbage collection of tombstones, a standard ORSet
+is also unfitting. In fact it could be argued Garage already uses something akin to an ORSet with a garbage collector today.
+
+There exists CRDTs supporting this feature-set, one of which is an OptORSet[^1].
+It only needs a set-wide metadata proportional in the number of writers, and a per alive-key metadata proportional in the number
+of writers to that key (but no metadata for dead keys).
+These metadata are akin to DVVs[^2]. An element is considered new if its DVV comes causaly after the current DVV of the set.
+Correctness of this algorithm depends however on having causal delivery, which isn't given in Garage, neither in general nor in
+presence of snapshot restoration.
+
+## Proposal
+
+We devise a new kind of CRDT based on the core ideas of an OptORSet, but replacing each version inside its DVVs with a list of
+range of observed updates, which we name a Seen Vector (SV). Such metadata can in the worst case grow linearly with the number of
+insertion. In practice, assuming all elements are eventually known to every replica, the storage requirement of a SV is equivalent
+to that of a standard DVV. Under causal delivery, a SV degenerates into a standard DVV.
+
+To add an element to the set, a node increments its own version counter, and sends an update containing (element, replica, version).
+On receiving such an update, a node checks if it has already seen this particular (replica, version), and if so ignore it.
+If it hasn't seen that update, it saves the new element, and update its SV to include that new (replica, version).
+
+TODO: describe formaly the algorithm
+
+
+Algorithm 1, Seen Vector:
+```
+payload set S    -- S: set of triple (replica i, timestamp s, timestamp e)
+
+initial ∅
+query seen (replica i, timestamp c): boolean b
+  let b = (∃s <= c,e > c: (i,s,e) ∈ S)
+
+update increment ()
+  prepare ()
+    let r = myID()
+    let t = e|∀s,s', ∄e' > e: (r,s,e) ∈ S, (r,s',e') ∈ S    -- t is the maximum end bound for this replica
+  effect(r, t)
+    if ¬seen (r, t) then
+      if seen (r, t - 1) ∧ seen (r, t + 1) then
+        let R = {∃s: (r, s, t) ∈ S}
+        let R' = {∃e: (r, t + 1, e) ∈ S}
+        let M = S \ R
+        let M' = M \ R'
+        S := M' ∪ {(r, s, e)}
+      else if seen (r, t - 1)then
+        let R = {∃s: (r, s, t) ∈ S}
+        let M = S \ R
+        S := M ∪ {(r, s, t+1)}
+      else if seen (r, t + 1)then
+        let R = {∃e: (r, t + 1, e) ∈ S}
+        let M = S \ R
+        E := E ∪ {(r, t, e)}
+      else
+        E := E ∪ {(r, t, t+1)}
+
+merge (B)
+  # TODO this is correct, but largelly suboptimal. We should perform the increment.effect subroutine for all elements of B instead
+  S := S ∪ B.S
+```
+
+Algorithm 2, OptORSet with SV. This algorithm is largely copied and adapted from Figure 3 of [^1]
+```
+payload set E, SV sv  -- E: elements, set of triples (element e, timestamp c, replica i)
+                      -- sv: SeenVector of received triples
+
+initial ∅, ∅
+query contains (element e) : boolean b
+  let b = (∃c, i : (e, c, i) ∈ E)
+
+query elements () : set S
+  let S = {e|∃c, i : (e, c, i) ∈ E}
+
+update add (element e)
+  prepare (e)
+    let r = myID() -- r = source replica
+    let c = sv.increment.prepare().t
+  effect (e, c, r)
+    if ¬sv.seen(r, c) then
+    let O = {(e, c′, r) ∈ E|c′ < c}
+    sv.increment.effect(r, c)
+    E := E ∪ {(e, c, r)} \ O
+
+update remove (element e)
+  prepare (e) -- Collect all unique triples containing e
+    let R = {(e, c, i) ∈ E}
+  effect (R) -- Remove triples observed at source
+    pre causal delivery
+    E := E \ R
+
+merge (B)
+  let M = (E ∩ B.E)
+  let M ′ = {(e, c, i) ∈ E \ B.E| ¬B.sv.seen(i, c)}
+  let M ′′ = {(e, c, i) ∈ B.E \ E| ¬sv.seen(i, c)}
+  let U = M ∪ M ′ ∪ M ′′
+  let O = {(e, c, i) ∈ U |∃(e, c′, i) ∈ U : c < c′}
+  E := U \ O
+  sv := sv.merge(B.sv)
+```
+
+## Storage evaluation
+
+As stated, if all updates are received, the SV is similar in size to that of a standard DVV. It may be however that a node create and
+immediately delete an element, creating holes in its sequence from the point of view of other replicas. These holes could be filled
+through an interactive process where the replica observing holes asks the node to scan over the whole set, and for each version in these
+hole, reply if no element has that exact version number. Holes caused by existing elements should be eventually fixed by an anti-entropy
+process, so replying with these elements appears unnecessary.
+
+The per-element storage requirement is proportional to the number of replicas having modified that element, even under non steady-state.
+This happens because as we always exchange whole elements, we have causal delivery for individual keys.
+
+## Replica version rollback
+
+This scheme assumes the same node won't issue the same version twice, which isn't a given when a node might be rollback to a previous state.
+The author proposes that on initialization, a replica asks all other replicas for the highest version number they know for it.
+If all replicas reply with a number less than or equal to the current version, it is safe to reuse the currently known number.
+If some replicas reply with a number higher, the node increase its version to that number.
+If at least one replica doesn't reply, it can't make any assumption about its actual version number.
+The node then increment a generation number, which is made part of its replica id, starts a new sequence from zero.
+
+## Appendix: providing SV to the underlying elements
+
+Some more complexe elements may want to have access to a version id and the Seen Vector to perform their own internal merge operations.
+The author reckon this may help implementing S3 versioning, by giving a simple way for Objects to know if an ObjectVersion was yet
+unknown or is known and already deleted.
+
+## References
+ 	
+[^1]: An optimized conflict-free replicated set, https://doi.org/10.48550/arXiv.1210.3368
+[^2]: Dotted Version Vectors: Logical Clocks for Optimistic Replication, https://doi.org/10.48550/arXiv.1011.5808
@@ -0,0 +1,17 @@
+*
+
+!*.txt
+!*.md
+
+!assets
+
+!.gitignore
+!*.svg
+!*.png
+!*.jpg
+!*.tex
+!Makefile
+!.gitignore
+!assets/*.drawio.pdf
+
+!talk.pdf
@@ -0,0 +1,19 @@
+ASSETS=../assets/lattice/lattice1.pdf_tex \
+	   ../assets/lattice/lattice2.pdf_tex \
+	   ../assets/lattice/lattice3.pdf_tex \
+	   ../assets/lattice/lattice4.pdf_tex \
+	   ../assets/lattice/lattice5.pdf_tex \
+	   ../assets/lattice/lattice6.pdf_tex \
+	   ../assets/lattice/lattice7.pdf_tex \
+	   ../assets/lattice/lattice8.pdf_tex \
+	   ../assets/logos/deuxfleurs.pdf \
+	   ../assets/timeline-22-24.pdf
+
+talk.pdf: talk.tex $(ASSETS)
+	pdflatex talk.tex
+
+%.pdf: %.svg
+	inkscape -D -z --file=$^ --export-pdf=$@
+
+%.pdf_tex: %.svg
+	inkscape -D -z --file=$^ --export-pdf=$@ --export-latex
@@ -0,0 +1,702 @@
+\nonstopmode
+\documentclass[aspectratio=169,xcolor={svgnames}]{beamer}
+\usepackage[utf8]{inputenc}
+% \usepackage[frenchb]{babel}
+\usepackage{amsmath}
+\usepackage{mathtools}
+\usepackage{breqn}
+\usepackage{multirow}
+\usetheme{boxes}
+\usepackage{graphicx}
+\usepackage{import}
+\usepackage{adjustbox}
+\usepackage[absolute,overlay]{textpos}
+%\useoutertheme[footline=authortitle,subsection=false]{miniframes}
+%\useoutertheme[footline=authorinstitute,subsection=false]{miniframes}
+\useoutertheme{infolines}
+\setbeamertemplate{headline}{}
+
+\beamertemplatenavigationsymbolsempty
+
+\definecolor{TitleOrange}{RGB}{255,137,0}
+\setbeamercolor{title}{fg=TitleOrange}
+\setbeamercolor{frametitle}{fg=TitleOrange}
+
+\definecolor{ListOrange}{RGB}{255,145,5}
+\setbeamertemplate{itemize item}{\color{ListOrange}$\blacktriangleright$}
+
+\definecolor{verygrey}{RGB}{70,70,70}
+\setbeamercolor{normal text}{fg=verygrey}
+
+
+\usepackage{tabu}
+\usepackage{multicol}
+\usepackage{vwcol}
+\usepackage{stmaryrd}
+\usepackage{graphicx}
+
+\usepackage[normalem]{ulem}
+
+\AtBeginSection[]{
+  \begin{frame}
+  \vfill
+  \centering
+  \begin{beamercolorbox}[sep=8pt,center,shadow=true,rounded=true]{title}
+    \usebeamerfont{title}\insertsectionhead\par%
+  \end{beamercolorbox}
+  \vfill
+  \end{frame}
+}
+
+\title{Garage, an S3 backend as reliable as possible}
+\author{Garage Authors}
+\date{JoSy S3, 2025-10-08}
+
+\begin{document}
+
+\begin{frame}
+	\centering
+	\includegraphics[width=.3\linewidth]{../../sticker/Garage.png}
+	\vspace{1em}
+
+	{\large\bf Garage, an S3 backend as reliable as possible}
+	\vspace{1em}
+
+		\url{https://garagehq.deuxfleurs.fr/}\\
+		\url{mailto:garagehq@deuxfleurs.fr}\\
+		\texttt{\#garage:deuxfleurs.fr} on Matrix
+\end{frame}
+
+
+\section{Meet Garage}
+
+\begin{frame}
+	\frametitle{A non-profit initiative}
+
+
+	\begin{columns}[t]
+		\begin{column}{.2\textwidth}
+			\centering
+			\adjincludegraphics[width=.5\linewidth, valign=t]{../assets/logos/deuxfleurs.pdf}
+		\end{column}
+		\begin{column}{.8\textwidth}
+			\textbf{Part of a degrowth initiative}\\
+			Garage has been created at Deuxfleurs where we experiment running Internet services without datacenter on commodity and refurbished hardware.
+		\end{column}
+
+	\end{columns}
+	\vspace{2em}
+	\begin{columns}[t]
+		\begin{column}{.2\textwidth}
+			\centering
+			\adjincludegraphics[width=.5\linewidth, valign=t]{../assets/community.png}
+		\end{column}
+		\begin{column}{.8\textwidth}
+			\textbf{Developed by a community}\\
+			{\small Some recent contributors: Arthur C, Charles H, dongdigua, Etienne L, Jonah A, Julien K, Lapineige, MagicRR, Milas B, Niklas M, RockWolf, Schwitzd, trinity-1686a, Xavier S,  babykart, Baptiste J, eddster2309, James O'C, Joker9944, Maximilien R, Renjaya RZ, Yureka...}
+		\end{column}
+
+	\end{columns}
+	\vspace{2em}
+	\begin{columns}[t]
+		\begin{column}{.2\textwidth}
+			\centering
+			\adjincludegraphics[width=.5\linewidth, valign=t]{../assets/logos/AGPLv3_Logo.png}
+		\end{column}
+		\begin{column}{.8\textwidth}
+			\textbf{Owned by nobody, open-core is impossible, zero VC money}\\
+			AGPL + no Contributor License Agreement = Garage ownership spreads among hundredth of contributors.
+		\end{column}
+
+	\end{columns}
+\end{frame}
+
+\begin{frame}
+	\frametitle{Getting support for Garage}
+	\begin{columns}[t]
+		\begin{column}{.2\textwidth}
+			\centering
+			\adjincludegraphics[width=.4\linewidth, valign=t]{../assets/alex.jpg}
+		\end{column}
+		\begin{column}{.4\textwidth}
+			\textbf{Alex Auvolat}\\
+			PhD; co-founder of Deuxfleurs\\
+			Garage maintainer, Freelance
+		\end{column}
+		\begin{column}{.3\textwidth}
+			\centering
+			\adjincludegraphics[width=.4\linewidth, valign=t]{../assets/support.png}
+		\end{column}
+		\begin{column}{.1\textwidth}
+			~
+		\end{column}
+	\end{columns}
+	\vspace{2em}
+	\begin{columns}[t]
+		\begin{column}{.2\textwidth}
+			\centering
+			\adjincludegraphics[width=.4\linewidth, valign=t]{../assets/quentin.jpg}
+		\end{column}
+		\begin{column}{.4\textwidth}
+			\textbf{Quentin Dufour}\\
+			PhD; co-founder of Deuxfleurs\\
+			Garage contributor, Freelance
+		\end{column}
+		\begin{column}{.4\textwidth}
+			For support requests, write at: \\
+			\url{garagehq@deuxfleurs.fr}
+		\end{column}
+	\end{columns}
+	\vspace{2em}
+	\begin{columns}[t]
+		\begin{column}{.2\textwidth}
+			\centering
+			\adjincludegraphics[width=.4\linewidth, valign=t]{../assets/armael.jpg}
+		\end{column}
+		\begin{column}{.4\textwidth}
+			\textbf{Armaël Guéneau}\\
+			PhD; member of Deuxfleurs\\
+			Garage contributor, Freelance
+		\end{column}
+		\begin{column}{.4\textwidth}
+			Eligible: email support, architecture design, specific feature development, etc.
+		\end{column}
+	\end{columns}
+
+
+\end{frame}
+
+\begin{frame}
+	\frametitle{Our initial goal}
+
+	\centering
+	\Large
+
+	Being a self-sovereign community to be free of our degrowth choice
+
+	$\big\downarrow$
+
+	As web citizens, datacenters are big black boxes. \\
+	We want to leave them to autonoumously manage our servers.
+
+	$\big\downarrow$
+
+	We want reliable services without relying on dedicated hardware or places.
+
+\end{frame}
+
+\begin{frame}
+	\frametitle{Building a resilient system with cheap stuff}
+
+	\only<1,4-7>{
+		\begin{itemize}
+			\item \textcolor<5->{gray}{Commodity hardware (e.g. old desktop PCs)\\
+				\vspace{.5em}
+				\visible<4->{{\footnotesize (can die at any time)}}}
+				\vspace{1.5em}
+			\item<5-> \textcolor<7->{gray}{Regular Internet (e.g. FTTB, FTTH) and power grid connections\\
+				\vspace{.5em}
+				\visible<6->{{\footnotesize (can be unavailable randomly)}}}
+				\vspace{1.5em}
+			\item<7-> \textbf{Geographical redundancy} (multi-site replication)
+		\end{itemize}
+	}
+	\only<2>{
+		\begin{center}
+			\includegraphics[width=.8\linewidth]{../assets/neptune.jpg}
+		\end{center}
+	}
+	\only<3>{
+		\begin{center}
+			\includegraphics[width=.8\linewidth]{../assets/atuin.jpg}
+		\end{center}
+	}
+	\only<8>{
+		\begin{center}
+			\includegraphics[width=.8\linewidth]{../assets/inframap_jdll2023.pdf}
+		\end{center}
+	}
+\end{frame}
+
+\begin{frame}
+	\frametitle{Object storage: a crucial component}
+		\begin{center}
+			\includegraphics[height=6em]{../assets/logos/Amazon-S3.jpg}
+			\hspace{3em}
+			\visible<2->{\includegraphics[height=5em]{../assets/logos/minio.png}}
+			\hspace{3em}
+			\visible<3>{\includegraphics[height=6em]{../../logo/garage_hires_crop.png}}
+		\end{center}
+		\vspace{1em}
+	S3: a de-facto standard, many compatible applications
+
+	\vspace{1em}
+	\visible<2->{MinIO is self-hostable but not suited for geo-distributed deployments}
+
+	\vspace{1em}
+	\visible<3->{\textbf{Garage is a self-hosted drop-in replacement for the Amazon S3 object store}}
+\end{frame}
+
+\begin{frame}
+	\frametitle{CRDTs / weak consistency instead of consensus}
+
+	\underline{Internally, Garage uses only CRDTs} (conflict-free replicated data types)
+
+	\vspace{2em}
+	Why not Raft, Paxos, ...? Issues of consensus algorithms:
+
+	\vspace{1em}
+	\begin{itemize}
+		\item<2-> \textbf{Software complexity}
+			\vspace{1em}
+		\item<3-> \textbf{Performance issues:}
+			\vspace{.5em}
+			\begin{itemize}
+				\item<4-> The leader is a \textbf{bottleneck} for all requests\\
+					\vspace{.5em}
+				\item<5-> \textbf{Sensitive to higher latency} between nodes
+					\vspace{.5em}
+				\item<6-> \textbf{Takes time to reconverge} when disrupted (e.g. node going down)
+			\end{itemize}
+	\end{itemize}
+\end{frame}
+
+\begin{frame}
+	\frametitle{The data model of object storage}
+	Object storage is basically a \textbf{key-value store}:
+	\vspace{.5em}
+
+	{\scriptsize
+		\begin{center}
+		\begin{tabular}{|l|p{7cm}|}
+			\hline
+			\textbf{Key: file path + name} & \textbf{Value: file data + metadata} \\
+			\hline
+			\hline
+			\texttt{index.html} &
+				\texttt{Content-Type: text/html; charset=utf-8} \newline
+				\texttt{Content-Length: 24929} \newline
+				\texttt{<binary blob>} \\ 
+			\hline
+			\texttt{img/logo.svg} &
+				\texttt{Content-Type: text/svg+xml} \newline
+				\texttt{Content-Length: 13429} \newline
+				\texttt{<binary blob>} \\ 
+			\hline
+			\texttt{download/index.html} &
+				\texttt{Content-Type: text/html; charset=utf-8} \newline
+				\texttt{Content-Length: 26563} \newline
+				\texttt{<binary blob>} \\ 
+			\hline
+		\end{tabular}
+		\end{center}
+		}
+
+	\vspace{1em}
+	\begin{itemize}
+		\item<2> Maps well to CRDT data types
+	\end{itemize}
+\end{frame}
+
+\begin{frame}
+	\frametitle{Performance gains in practice}
+	\begin{center}
+		\includegraphics[width=.8\linewidth]{../assets/perf/endpoint_latency_0.7_0.8_minio.png}
+	\end{center}
+\end{frame}
+
+% ======================================== OPERATING
+% ======================================== OPERATING
+% ======================================== OPERATING
+
+
+\section{Production clusters}
+
+\begin{frame}
+	\frametitle{Deployment kinds}
+
+	\includegraphics[width=.9\linewidth]{../assets/cluster_kind.png}
+	\vspace{1em}
+	
+\end{frame}
+
+\begin{frame}
+	\frametitle{How big they are?}
+
+	\includegraphics[width=.9\linewidth]{../assets/cluster_size.png}
+	\vspace{1em}
+
+	\textit{"Petabyte storage setup for a video site. Nginx as CDN in-front using garage-s3-website feature. Each storage node has ~64TB storage with raid10, no replication within garage. 25gbit nic. haproxy to loadbalance across 5 nodes. mostly reads with very few writes."}
+
+	\vspace{1em}
+	\textit{"We currently manage 7 Garage nodes, 28TB total storage, 6M blocks for 3M objects and 4TB of object data. We have been running Garage in production for 2.5 years."}
+
+\end{frame}
+
+\begin{frame}
+	\frametitle{Operating Garage}
+	\begin{center}
+		\only<1-2>{
+			\includegraphics[width=.9\linewidth]{../assets/screenshots/garage_status_0.10.png}
+			\\\vspace{1em}
+			\visible<2>{\includegraphics[width=.9\linewidth]{../assets/screenshots/garage_status_unhealthy_0.10.png}}
+		}
+	\end{center}
+\end{frame}
+
+\begin{frame}
+	\frametitle{Garage's architecture}
+	\begin{center}
+		\only<1>{\includegraphics[width=.45\linewidth]{../assets/garage.drawio.pdf}}%
+		\only<2>{\includegraphics[width=.6\linewidth]{../assets/garage_sync.drawio.pdf}}%
+	\end{center}
+\end{frame}
+
+\begin{frame}
+	\frametitle{Digging deeper}
+	\begin{center}
+		\only<1>{\includegraphics[width=.9\linewidth]{../assets/screenshots/garage_stats_0.10.png}}
+		\only<2>{\includegraphics[width=.5\linewidth]{../assets/screenshots/garage_worker_list_0.10.png}}
+		\only<3>{\includegraphics[width=.6\linewidth]{../assets/screenshots/garage_worker_param_0.10.png}}
+	\end{center}
+\end{frame}
+
+\begin{frame}
+	\frametitle{Potential limitations and bottlenecks}
+	\begin{itemize}
+		\item Global:
+			\begin{itemize}
+				\item Max. $\sim$100 nodes per cluster (excluding gateways)
+			\end{itemize}
+			\vspace{1em}
+		\item Metadata:
+			\begin{itemize}
+				\item One big bucket = bottleneck, object list on 3 nodes only
+			\end{itemize}
+			\vspace{1em}
+		\item Block manager:
+			\begin{itemize}
+				\item Lots of small files on disk
+				\item Processing the resync queue can be slow
+			\end{itemize}
+	\end{itemize}
+\end{frame}
+
+\begin{frame}
+	\frametitle{Deployment advice for very large clusters}
+	\begin{itemize}
+		\item Metadata storage:
+			\begin{itemize}
+				\item ZFS mirror (x2) on fast NVMe
+				\item Use LMDB storage engine
+			\end{itemize}
+			\vspace{.5em}
+		\item Data block storage:
+			\begin{itemize}
+				\item Use Garage's native multi-HDD support
+				\item XFS on individual drives
+				\item Increase block size (1MB $\to$ 10MB, requires more RAM and good networking)
+				\item Tune \texttt{resync-tranquility} and \texttt{resync-worker-count} dynamically
+			\end{itemize}
+			\vspace{.5em}
+		\item Other :
+			\begin{itemize}
+				\item Split data over several buckets
+				\item Use less than 100 storage nodes
+				\item Use gateway nodes
+			\end{itemize}
+			\vspace{.5em}
+	\end{itemize}
+\end{frame}
+
+
+\begin{frame}
+	\frametitle{Focus on Deuxfleurs}
+
+	Host institutional websites, partnership with a web agency.
+	Matrix media backend.
+
+	Plan to use it as an email backend for an internally developed email server.
+
+\end{frame}
+
+
+% ======================================== TIMELINE
+% ======================================== TIMELINE
+% ======================================== TIMELINE
+
+\section{Recent developments}
+
+% ====================== v0.7.0 ===============================
+
+\begin{frame}
+	\begin{center}
+		\includegraphics[width=.8\linewidth]{../assets/tl.drawio.png}
+	\end{center}
+\end{frame}
+
+\begin{frame}
+	\frametitle{April 2022 - Garage v0.7.0}
+	Focus on \underline{observability and ecosystem integration}
+	\vspace{2em}
+	\begin{itemize}
+		\item \textbf{Monitoring:} metrics and traces, using OpenTelemetry
+			\vspace{1em}
+		\item Replication modes with 1 or 2 copies / weaker consistency
+			\vspace{1em}
+		\item Kubernetes integration for node discovery
+			\vspace{1em}
+		\item Admin API (v0.7.2)
+	\end{itemize}
+\end{frame}
+
+\begin{frame}
+	\frametitle{Metrics (Prometheus + Grafana)}
+	\begin{center}
+		\includegraphics[width=.9\linewidth]{../assets/screenshots/grafana_dashboard.png}
+	\end{center}
+\end{frame}
+
+\begin{frame}
+	\frametitle{Traces (Jaeger)}
+	\begin{center}
+		\includegraphics[width=.8\linewidth]{../assets/screenshots/jaeger_listobjects.png}
+	\end{center}
+\end{frame}
+
+% ====================== v0.8.0 ===============================
+
+\begin{frame}
+	\begin{center}
+		\includegraphics[width=.8\linewidth]{../assets/tl.drawio.png}
+	\end{center}
+\end{frame}
+
+\begin{frame}
+	\frametitle{November 2022 - Garage v0.8.0}
+	Focus on \underline{performance}
+	\vspace{2em}
+	\begin{itemize}
+		\item \textbf{Alternative metadata DB engines} (LMDB, Sqlite)
+			\vspace{1em}
+		\item \textbf{Performance improvements:} block streaming, various optimizations...
+			\vspace{1em}
+		\item Bucket quotas (max size, max \#objects)
+			\vspace{1em}
+		\item Quality of life improvements, observability, etc.
+	\end{itemize}
+\end{frame}
+
+\begin{frame}
+	\frametitle{About metadata DB engines}
+	\textbf{Issues with Sled:}
+	\vspace{1em}
+	\begin{itemize}
+		\item Huge files on disk
+			\vspace{.5em}
+		\item Unpredictable performance, especially on HDD
+			\vspace{.5em}
+		\item API limitations
+			\vspace{.5em}
+		\item Not actively maintained
+	\end{itemize}
+
+	\vspace{2em}
+	\textbf{LMDB:} very stable, good performance, file size is reasonable\\
+	\textbf{Sqlite} also available as a second choice
+
+	\vspace{1em}
+	Sled will be removed in Garage v1.0
+\end{frame}
+
+\begin{frame}
+	\frametitle{DB engine performance comparison}
+	\begin{center}
+		\includegraphics[width=.6\linewidth]{../assets/perf/db_engine.png}
+	\end{center}
+	NB: Sqlite was slow due to synchronous mode, now configurable
+\end{frame}
+
+\begin{frame}
+	\frametitle{Block streaming}
+	\begin{center}
+		\only<1>{\includegraphics[width=.8\linewidth]{../assets/schema-streaming-1.png}}
+		\only<2>{\includegraphics[width=.8\linewidth]{../assets/schema-streaming-2.png}}
+	\end{center}
+\end{frame}
+
+\begin{frame}
+	\frametitle{TTFB benchmark}
+	\begin{center}
+		\includegraphics[width=.8\linewidth]{../assets/perf/ttfb.png}
+	\end{center}
+\end{frame}
+
+\begin{frame}
+	\frametitle{Throughput benchmark}
+	\begin{center}
+		\includegraphics[width=.7\linewidth]{../assets/perf/io-0.7-0.8-minio.png}
+	\end{center}
+\end{frame}
+
+% ====================== v0.9.0 ===============================
+
+\begin{frame}
+	\begin{center}
+		\includegraphics[width=.8\linewidth]{../assets/tl.drawio.png}
+	\end{center}
+\end{frame}
+
+\begin{frame}
+	\frametitle{October 2023 - Garage v0.9.0}
+	Focus on \underline{streamlining \& usability}
+	\vspace{2em}
+	\begin{itemize}
+		\item Support multiple HDDs per node
+			\vspace{1em}
+		\item S3 compatibility: 
+			\vspace{1em}
+			\begin{itemize}
+				\item support basic lifecycle configurations
+					\vspace{.5em}
+				\item allow for multipart upload part retries
+			\end{itemize}
+			\vspace{1em}
+		\item LMDB by default, deprecation of Sled
+			\vspace{1em}
+		\item New layout computation algorithm
+	\end{itemize}
+\end{frame}
+
+
+\begin{frame}
+	\frametitle{Layout computation}
+	\begin{overprint}
+		\onslide<1>
+		\begin{center}
+			\includegraphics[width=\linewidth, trim=0 0 0 -4cm]{../assets/screenshots/garage_status_0.9_prod_zonehl.png}
+		\end{center}
+		\onslide<2>
+		\begin{center}
+			\includegraphics[width=.7\linewidth]{../assets/map.png}
+		\end{center}
+	\end{overprint}
+	\vspace{1em}
+	Garage stores replicas on different zones when possible
+\end{frame}
+
+\begin{frame}
+	\frametitle{What a "layout" is}
+		\textbf{A layout is a precomputed index table:}
+		\vspace{1em}
+
+		{\footnotesize
+			\begin{center}
+			\begin{tabular}{|l|l|l|l|}
+				\hline
+				\textbf{Partition} & \textbf{Node 1} & \textbf{Node 2} & \textbf{Node 3} \\
+				\hline
+				\hline
+				Partition 0 & df-ymk (bespin) & Abricot (scorpio) & Courgette (neptune) \\
+				\hline
+				Partition 1 & Ananas (scorpio) & Courgette (neptune) & df-ykl (bespin) \\
+				\hline
+				Partition 2 & df-ymf (bespin) & Celeri (neptune) & Abricot (scorpio) \\
+				\hline
+				\hspace{1em}$\vdots$ & \hspace{1em}$\vdots$ & \hspace{1em}$\vdots$ & \hspace{1em}$\vdots$ \\
+				\hline
+				Partition 255 & Concombre (neptune) & df-ykl (bespin) & Abricot (scorpio) \\
+				\hline
+			\end{tabular}
+			\end{center}
+		}
+
+	\vspace{2em}
+	\visible<2->{
+		The index table is built centrally using an optimal algorithm,\\
+		then propagated to all nodes
+	}
+
+	\vspace{1em}
+	\visible<3->{
+		\footnotesize
+		Oulamara, M., \& Auvolat, A. (2023). \emph{An algorithm for geo-distributed and redundant storage in Garage}.\\ arXiv preprint arXiv:2302.13798.
+	}
+\end{frame}
+
+
+
+% ====================== v1.0.0 ===============================
+
+\begin{frame}
+	\begin{center}
+		\includegraphics[width=.8\linewidth]{../assets/tl.drawio.png}
+	\end{center}
+\end{frame}
+
+\begin{frame}
+	\frametitle{April 2024 - Garage v1.0.0}
+	Focus on \underline{consistency, security \& stability}
+	\vspace{2em}
+	\begin{itemize}
+		\item Fix consistency issues when reshuffling data (Jepsen testing)
+			\vspace{1em}
+		\item \textbf{Security audit} by Radically Open Security
+			\vspace{1em}
+		\item Misc. S3 features (SSE-C, checksums,  ...) and compatibility fixes
+	\end{itemize}
+\end{frame}
+
+% ====================== v2.0.0 ===============================
+
+\begin{frame}
+	\begin{center}
+		\includegraphics[width=.8\linewidth]{../assets/tl.drawio.png}
+	\end{center}
+\end{frame}
+
+\begin{frame}
+	\frametitle{Garage v2.0.0}
+	Focus on \underline{}
+	\vspace{2em}
+	\begin{itemize}
+		\item TODO
+	\end{itemize}
+\end{frame}
+
+
+\begin{frame}
+	\frametitle{Currently funding...}
+
+	\textit{...}
+\end{frame}
+
+\begin{frame}
+	\frametitle{We run community surveys}
+	\begin{center}
+		\includegraphics[width=.6\linewidth]{../assets/survey_requested_features.png}
+	\end{center}
+\end{frame}
+
+% ======================================== END
+% ======================================== END
+% ======================================== END
+
+\begin{frame}
+	\frametitle{Where to find us}
+	\begin{center}
+			\includegraphics[width=.25\linewidth]{../../logo/garage_hires.png}\\
+			\vspace{-1em}
+		\url{https://garagehq.deuxfleurs.fr/}\\
+		\url{mailto:garagehq@deuxfleurs.fr}\\
+		\texttt{\#garage:deuxfleurs.fr} on Matrix
+
+		\vspace{1.5em}
+			\includegraphics[width=.06\linewidth]{../assets/logos/rust_logo.png}
+			\includegraphics[width=.13\linewidth]{../assets/logos/AGPLv3_Logo.png}
+	\end{center}
+\end{frame}
+
+\end{document}
+
+%% vim: set ts=4 sw=4 tw=0 noet spelllang=en :
@@ -53,6 +53,9 @@
          tests-sqlite = testWith {
            GARAGE_TEST_INTEGRATION_DB_ENGINE = "sqlite";
          };
+          tests-fjall = testWith {
+            GARAGE_TEST_INTEGRATION_DB_ENGINE = "fjall";
+          };
        };

        # ---- developpment shell, for making native builds only ----
@@ -78,6 +81,9 @@
                cargo-outdated
                cargo-machete
                nixpkgs-fmt
+                openssl
+                socat
+                killall
              ];
            };
          };
@@ -68,7 +68,7 @@ let
  rootFeatures = if features != null then
    features
  else
-    ([ "bundled-libs" "lmdb" "sqlite" "k2v" ] ++ (lib.optionals release [
+    ([ "bundled-libs" "lmdb" "sqlite" "fjall" "k2v" ] ++ (lib.optionals release [
      "consul-discovery"
      "kubernetes-discovery"
      "metrics"
@@ -2,8 +2,8 @@ apiVersion: v2
 name: garage
 description: S3-compatible object store for small self-hosted geo-distributed deployments
 type: application
-version: 0.8.0
-appVersion: "v2.0.0"
+version: 0.9.1
+appVersion: "v2.1.0"
 home: https://garagehq.deuxfleurs.fr/
 icon: https://garagehq.deuxfleurs.fr/images/garage-logo.svg

@@ -1,6 +1,6 @@
 # garage

-![Version: 0.8.0](https://img.shields.io/badge/Version-0.8.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: v2.0.0](https://img.shields.io/badge/AppVersion-v2.0.0-informational?style=flat-square)
+![Version: 0.9.1](https://img.shields.io/badge/Version-0.9.1-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: v2.1.0](https://img.shields.io/badge/AppVersion-v2.1.0-informational?style=flat-square)

 S3-compatible object store for small self-hosted geo-distributed deployments

@@ -15,6 +15,7 @@ S3-compatible object store for small self-hosted geo-distributed deployments
 | Key | Type | Default | Description |
 |-----|------|---------|-------------|
 | affinity | object | `{}` |  |
+| commonLabels  | object | `{}` | Extra labels for all resources |
 | deployment.kind | string | `"StatefulSet"` | Switchable to DaemonSet |
 | deployment.podManagementPolicy | string | `"OrderedReady"` | If using statefulset, allow Parallel or OrderedReady (default) |
 | deployment.replicaCount | int | `3` | Number of StatefulSet replicas/garage nodes to start |
@@ -27,7 +27,7 @@ If release name contains chart name it will be used as a full name.
 Create the name of the rpc secret
 */}}
 {{- define "garage.rpcSecretName" -}}
-{{- printf "%s-rpc-secret" (include "garage.fullname" .) -}}
+{{- .Values.garage.existingRpcSecret | default (printf "%s-rpc-secret" (include "garage.fullname" .)) -}}
 {{- end }}

 {{/*
@@ -47,6 +47,9 @@ helm.sh/chart: {{ include "garage.chart" . }}
 app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
 {{- end }}
 app.kubernetes.io/managed-by: {{ .Release.Service }}
+{{ with .Values.commonLabels }}
+{{- toYaml . }}
+{{- end }}
 {{- end }}

 {{/*
@@ -30,6 +30,10 @@ data:

    bootstrap_peers = {{ .Values.garage.bootstrapPeers }}

+    {{- if .Values.garage.additionalTopLevelConfig }}
+    {{ .Values.garage.additionalTopLevelConfig | nindent 4 }}
+    {{- end }}
+  
    [kubernetes_discovery]
    namespace = "{{ .Release.Namespace }}"
    service_name = "{{ include "garage.fullname" . }}"
@@ -1,3 +1,4 @@
+{{- if not .Values.garage.existingRpcSecret }}
 apiVersion: v1
 kind: Secret
 metadata:
@@ -12,3 +13,4 @@ data:
  {{- $prevRpcSecret := $prevSecretData.rpcSecret | default "" | b64dec }}
  {{/* Priority is: 1. from values, 2. previous value, 3. generate random */}}
  rpcSecret: {{ .Values.garage.rpcSecret | default $prevRpcSecret | default (include "jupyterhub.randHex" 64) | b64enc | quote }}
+{{- end }}
@@ -21,7 +21,7 @@ spec:
        {{- toYaml . | nindent 8 }}
      {{- end }}
      labels:
-        {{- include "garage.selectorLabels" . | nindent 8 }}
+        {{- include "garage.labels" . | nindent 8 }}
    spec:
      {{- with .Values.imagePullSecrets }}
      imagePullSecrets:
@@ -2,6 +2,11 @@
 # This is a YAML-formatted file.
 # Declare variables to be passed into your templates.

+# -- Additional labels to add to all resources created by this chart
+commonLabels: {}
+#   app.kubernetes.io/part-of: storage
+#   team: platform
+
 # Garage configuration. These values go to garage.toml
 garage:
  # -- Can be changed for better performance on certain systems
@@ -32,6 +37,10 @@ garage:
  rpcBindAddr: "[::]:3901"
  # -- If not given, a random secret will be generated and stored in a Secret object
  rpcSecret: ""
+  # -- If you want to provide an rpcSecret within an existing k8s secret,
+  # specify the secret name here, and store the value under the secret key `rpcSecret`
+  # the default secret will not be created
+  existingRpcSecret: ""
  # -- This is not required if you use the integrated kubernetes discovery
  bootstrapPeers: []
  # -- Set to true if you want to use k8s discovery but install the CRDs manually outside
@@ -45,6 +54,12 @@ garage:
      rootDomain: ".web.garage.tld"
      index: "index.html"

+  # -- Additional configuration to append to garage.toml. Use a multi-line string for custom config.
+  # Example:
+  #  additionalTopLevelConfig: |-
+  #    data_fsync = true
+  additionalTopLevelConfig: ""
+
  # -- if not empty string, allow using an existing ConfigMap for the garage.toml,
  # if set, ignores garage.toml
  existingConfigMap: ""
@@ -112,6 +127,7 @@ podSecurityContext:
  runAsUser: 1000
  runAsGroup: 1000
  fsGroup: 1000
+  fsGroupChangePolicy: "OnRootMismatch"
  runAsNonRoot: true

 securityContext:
@@ -694,32 +694,7 @@
            ]
          }
        },
-        "overrides": [
-          {
-            "__systemRef": "hideSeriesFrom",
-            "matcher": {
-              "id": "byNames",
-              "options": {
-                "mode": "exclude",
-                "names": [
-                  "10.83.2.3:3903"
-                ],
-                "prefix": "All except:",
-                "readOnly": true
-              }
-            },
-            "properties": [
-              {
-                "id": "custom.hideFrom",
-                "value": {
-                  "legend": false,
-                  "tooltip": false,
-                  "viz": true
-                }
-              }
-            ]
-          }
-        ]
+        "overrides": []
      },
      "gridPos": {
        "h": 8,
@@ -1,6 +1,6 @@
 [package]
 name = "garage_api_admin"
-version = "2.0.0"
+version = "2.1.0"
 authors = ["Alex Auvolat <alex@adnab.me>"]
 edition = "2018"
 license = "AGPL-3.0"
@@ -26,7 +26,7 @@ argon2.workspace = true
 async-trait.workspace = true
 bytesize.workspace = true
 chrono.workspace = true
-err-derive.workspace = true
+thiserror.workspace = true
 hex.workspace = true
 paste.workspace = true
 tracing.workspace = true
@@ -175,6 +175,63 @@ impl RequestHandler for DeleteAdminTokenRequest {
 	}
 }

+impl RequestHandler for GetCurrentAdminTokenInfoRequest {
+	type Response = GetCurrentAdminTokenInfoResponse;
+
+	async fn handle(
+		self,
+		garage: &Arc<Garage>,
+		_admin: &Admin,
+	) -> Result<GetCurrentAdminTokenInfoResponse, Error> {
+		let now = now_msec();
+
+		if garage
+			.config
+			.admin
+			.metrics_token
+			.as_ref()
+			.is_some_and(|s| s == &self.admin_token)
+		{
+			return Ok(GetCurrentAdminTokenInfoResponse(
+				GetAdminTokenInfoResponse {
+					id: None,
+					created: None,
+					name: "metrics_token (from daemon configuration)".into(),
+					expiration: None,
+					expired: false,
+					scope: vec!["Metrics".into()],
+				},
+			));
+		}
+
+		if garage
+			.config
+			.admin
+			.admin_token
+			.as_ref()
+			.is_some_and(|s| s == &self.admin_token)
+		{
+			return Ok(GetCurrentAdminTokenInfoResponse(
+				GetAdminTokenInfoResponse {
+					id: None,
+					created: None,
+					name: "admin_token (from daemon configuration)".into(),
+					expiration: None,
+					expired: false,
+					scope: vec!["*".into()],
+				},
+			));
+		}
+
+		let (prefix, _) = self.admin_token.split_once('.').unwrap();
+		let token = get_existing_admin_token(&garage, &prefix.to_string()).await?;
+
+		Ok(GetCurrentAdminTokenInfoResponse(admin_token_info_results(
+			&token, now,
+		)))
+	}
+}
+
 // ---- helpers ----

 fn admin_token_info_results(token: &AdminApiToken, now: u64) -> GetAdminTokenInfoResponse {
@@ -12,7 +12,7 @@ use garage_rpc::*;

 use garage_model::garage::Garage;

-use garage_api_common::helpers::is_default;
+use garage_api_common::{common_error::CommonError, helpers::is_default};

 use crate::api_server::{find_matching_nodes, AdminRpc, AdminRpcResponse};
 use crate::error::Error;
@@ -56,6 +56,7 @@ admin_endpoints![
 	CreateAdminToken,
 	UpdateAdminToken,
 	DeleteAdminToken,
+	GetCurrentAdminTokenInfo,

 	// Layout operations
 	GetClusterLayout,
@@ -144,6 +145,13 @@ pub struct MultiResponse<RB> {
 	pub error: HashMap<String, String>,
 }

+#[derive(Debug, Clone, Serialize, Deserialize, IntoParams)]
+#[into_params(parameter_in = Query)]
+pub struct MultiRequestQueryParams {
+	/// Node ID to query, or `*` for all nodes, or `self` for the node responding to the request
+	pub node: String,
+}
+
 // **********************************************
 //      Special endpoints
 //
@@ -154,8 +162,10 @@ pub struct MultiResponse<RB> {
 #[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct OptionsRequest;

-#[derive(Debug, Clone, Serialize, Deserialize)]
+#[derive(Debug, Clone, Serialize, Deserialize, IntoParams)]
+#[into_params(parameter_in = Query)]
 pub struct CheckDomainRequest {
+	/// The domain name to check for
 	pub domain: String,
 }

@@ -257,7 +267,7 @@ pub struct GetClusterHealthResponse {
 	/// the number of storage nodes currently registered in the cluster layout
 	pub storage_nodes: usize,
 	/// the number of storage nodes to which a connection is currently open
-	pub storage_nodes_ok: usize,
+	pub storage_nodes_up: usize,
 	/// the total number of partitions of the data (currently always 256)
 	pub partitions: usize,
 	/// the number of partitions for which a quorum of write nodes is available
@@ -308,6 +318,7 @@ pub struct ListAdminTokensResponse(pub Vec<GetAdminTokenInfoResponse>);
 // ---- GetAdminTokenInfo ----

 #[derive(Debug, Clone, Serialize, Deserialize, IntoParams)]
+#[into_params(parameter_in = Query)]
 #[serde(rename_all = "camelCase")]
 pub struct GetAdminTokenInfoRequest {
 	/// Admin API token ID
@@ -353,9 +364,12 @@ pub struct CreateAdminTokenResponse {

 // ---- UpdateAdminToken ----

-#[derive(Debug, Clone, Serialize, Deserialize)]
+#[derive(Debug, Clone, Serialize, Deserialize, IntoParams)]
+#[into_params(parameter_in = Query)]
 pub struct UpdateAdminTokenRequest {
+	/// Admin API token ID
 	pub id: String,
+	#[param(ignore = true)]
 	pub body: UpdateAdminTokenRequestBody,
 }

@@ -382,14 +396,25 @@ pub struct UpdateAdminTokenResponse(pub GetAdminTokenInfoResponse);

 // ---- DeleteAdminToken ----

-#[derive(Debug, Clone, Serialize, Deserialize)]
+#[derive(Debug, Clone, Serialize, Deserialize, IntoParams)]
+#[into_params(parameter_in = Query)]
 pub struct DeleteAdminTokenRequest {
+	/// Admin API token ID
 	pub id: String,
 }

 #[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct DeleteAdminTokenResponse;

+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct GetCurrentAdminTokenInfoRequest {
+	pub admin_token: String,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)]
+#[serde(rename_all = "camelCase")]
+pub struct GetCurrentAdminTokenInfoResponse(pub GetAdminTokenInfoResponse);
+
 // **********************************************
 //      Layout operations
 // **********************************************
@@ -648,6 +673,7 @@ pub struct ListKeysResponseItem {
 // ---- GetKeyInfo ----

 #[derive(Debug, Clone, Serialize, Deserialize, IntoParams)]
+#[into_params(parameter_in = Query)]
 #[serde(rename_all = "camelCase")]
 pub struct GetKeyInfoRequest {
 	/// Access key ID
@@ -724,9 +750,12 @@ pub struct ImportKeyResponse(pub GetKeyInfoResponse);

 // ---- UpdateKey ----

-#[derive(Debug, Clone, Serialize, Deserialize)]
+#[derive(Debug, Clone, Serialize, Deserialize, IntoParams)]
+#[into_params(parameter_in = Query)]
 pub struct UpdateKeyRequest {
+	/// Access key ID
 	pub id: String,
+	#[param(ignore = true)]
 	pub body: UpdateKeyRequestBody,
 }

@@ -751,8 +780,10 @@ pub struct UpdateKeyRequestBody {

 // ---- DeleteKey ----

-#[derive(Debug, Clone, Serialize, Deserialize)]
+#[derive(Debug, Clone, Serialize, Deserialize, IntoParams)]
+#[into_params(parameter_in = Query)]
 pub struct DeleteKeyRequest {
+	/// Access key ID
 	pub id: String,
 }

@@ -790,6 +821,7 @@ pub struct BucketLocalAlias {
 // ---- GetBucketInfo ----

 #[derive(Debug, Clone, Serialize, Deserialize, IntoParams)]
+#[into_params(parameter_in = Query)]
 #[serde(rename_all = "camelCase")]
 pub struct GetBucketInfoRequest {
 	/// Exact bucket ID to look up
@@ -878,9 +910,12 @@ pub struct CreateBucketLocalAlias {

 // ---- UpdateBucket ----

-#[derive(Debug, Clone, Serialize, Deserialize)]
+#[derive(Debug, Clone, Serialize, Deserialize, IntoParams)]
+#[into_params(parameter_in = Query)]
 pub struct UpdateBucketRequest {
+	/// ID of the bucket to update
 	pub id: String,
+	#[param(ignore = true)]
 	pub body: UpdateBucketRequestBody,
 }

@@ -904,8 +939,10 @@ pub struct UpdateBucketWebsiteAccess {

 // ---- DeleteBucket ----

-#[derive(Debug, Clone, Serialize, Deserialize)]
+#[derive(Debug, Clone, Serialize, Deserialize, IntoParams)]
+#[into_params(parameter_in = Query)]
 pub struct DeleteBucketRequest {
+	/// ID of the bucket to delete
 	pub id: String,
 }

@@ -928,6 +965,7 @@ pub struct CleanupIncompleteUploadsResponse {
 }

 #[derive(Debug, Clone, Serialize, Deserialize, IntoParams)]
+#[into_params(parameter_in = Query)]
 #[serde(rename_all = "camelCase")]
 pub struct InspectObjectRequest {
 	pub bucket_id: String,
@@ -1115,6 +1153,7 @@ pub enum RepairType {
 	Rebalance,
 	Scrub(ScrubCommand),
 	Aliases,
+	ClearResyncQueue,
 }

 #[derive(Debug, Clone, Serialize, Deserialize, ToSchema)]
@@ -217,6 +217,13 @@ impl ApiHandler for ArcAdminApiServer {
 	) -> Result<Response<ResBody>, Error> {
 		self.0.handle_http_api(req, endpoint).await
 	}
+
+	fn key_id_from_request(&self, req: &Request<IncomingBody>) -> Option<String> {
+		let auth_header = req.headers().get(AUTHORIZATION)?;
+		let token = parse_authorization(auth_header).ok()?;
+		let key_id = token.split_once('.')?.0;
+		Some(key_id.to_string())
+	}
 }

 impl ApiEndpoint for HttpEndpoint {
@@ -244,6 +251,15 @@ fn hash_bearer_token(token: &str) -> String {
 		.to_string()
 }

+fn parse_authorization(auth_header: &hyper::http::HeaderValue) -> Result<&str, Error> {
+	let token = auth_header
+		.to_str()?
+		.strip_prefix("Bearer ")
+		.ok_or_else(|| Error::forbidden("Invalid Authorization header"))?
+		.trim();
+	Ok(token)
+}
+
 fn verify_authorization(
 	garage: &Garage,
 	global_token_hash: Option<&str>,
@@ -260,11 +276,7 @@ fn verify_authorization(
 				"Bearer token must be provided in Authorization header",
 			))
 		}
-		Some(authorization) => authorization
-			.to_str()?
-			.strip_prefix("Bearer ")
-			.ok_or_else(|| Error::forbidden("Invalid Authorization header"))?
-			.trim(),
+		Some(authorization) => parse_authorization(authorization)?,
 	};

 	let token_hash_string = if let Some((prefix, _)) = token.split_once('.') {
@@ -273,7 +285,8 @@ fn verify_authorization(
 			.get_local(&EmptyKey, &prefix.to_string())?
 			.and_then(|k| k.state.into_option())
 			.filter(|p| !p.is_expired(now_msec()))
-			.filter(|p| p.has_scope(endpoint_name))
+			// GetCurrentAdminTokenInfo endpoint must be accessible even if it is not in the token scopes
+			.filter(|p| p.has_scope(endpoint_name) || endpoint_name == "GetCurrentAdminTokenInfo")
 			.ok_or_else(|| Error::forbidden(invalid_msg))?
 			.token_hash
 	} else {
@@ -293,38 +306,36 @@ fn verify_authorization(
 }

 pub(crate) fn find_matching_nodes(garage: &Garage, spec: &str) -> Result<Vec<Uuid>, Error> {
-	let mut res = vec![];
-	if spec == "*" {
-		res = garage.system.cluster_layout().all_nodes().to_vec();
+	if spec == "self" {
+		Ok(vec![garage.system.id])
+	} else {
+		// Collect all nodes currently up and/or in cluster layout
+		let mut res = vec![];
+		if let Ok(all_nodes) = garage.system.cluster_layout().all_nodes() {
+			res = all_nodes.to_vec();
+		}
 		for node in garage.system.get_known_nodes() {
 			if node.is_up && !res.contains(&node.id) {
 				res.push(node.id);
 			}
 		}
-	} else if spec == "self" {
-		res.push(garage.system.id);
-	} else {
-		let layout = garage.system.cluster_layout();
-		let known_nodes = garage.system.get_known_nodes();
-		let all_nodes = layout
-			.all_nodes()
-			.iter()
-			.copied()
-			.chain(known_nodes.iter().filter(|x| x.is_up).map(|x| x.id));
-		for node in all_nodes {
-			if !res.contains(&node) && hex::encode(node).starts_with(spec) {
-				res.push(node);
+
+		if spec == "*" {
+			// match all nodes
+			Ok(res)
+		} else {
+			// filter nodes that match spec
+			res.retain(|node| hex::encode(node).starts_with(spec));
+			if res.is_empty() {
+				Err(Error::bad_request(format!("No nodes matching {}", spec)))
+			} else if res.len() > 1 {
+				Err(Error::bad_request(format!(
+					"Multiple nodes matching {}: {:?}",
+					spec, res
+				)))
+			} else {
+				Ok(res)
 			}
 		}
-		if res.is_empty() {
-			return Err(Error::bad_request(format!("No nodes matching {}", spec)));
-		}
-		if res.len() > 1 {
-			return Err(Error::bad_request(format!(
-				"Multiple nodes matching {}: {:?}",
-				spec, res
-			)));
-		}
 	}
-	Ok(res)
 }
@@ -697,20 +697,21 @@ async fn bucket_info_results(
 		}),
 		keys: relevant_keys
 			.into_values()
-			.map(|key| {
+			.filter_map(|key| {
 				let p = key.state.as_option().unwrap();
-				GetBucketInfoKey {
+				let permissions = p
+					.authorized_buckets
+					.get(&bucket.id)
+					.filter(|p| p.is_any())
+					.map(|p| ApiBucketKeyPerm {
+						read: p.allow_read,
+						write: p.allow_write,
+						owner: p.allow_owner,
+					})?;
+				Some(GetBucketInfoKey {
 					access_key_id: key.key_id,
 					name: p.name.get().to_string(),
-					permissions: p
-						.authorized_buckets
-						.get(&bucket.id)
-						.map(|p| ApiBucketKeyPerm {
-							read: p.allow_read,
-							write: p.allow_write,
-							owner: p.allow_owner,
-						})
-						.unwrap_or_default(),
+					permissions,
 					bucket_local_aliases: p
 						.local_aliases
 						.items()
@@ -718,7 +719,7 @@ async fn bucket_info_results(
 						.filter(|(_, _, b)| *b == Some(bucket.id))
 						.map(|(n, _, _)| n.to_string())
 						.collect::<Vec<_>>(),
-				}
+				})
 			})
 			.collect::<Vec<_>>(),
 		objects: *counters.get(OBJECTS).unwrap_or(&0),
@@ -56,48 +56,52 @@ impl RequestHandler for GetClusterStatusRequest {
 			})
 			.collect::<HashMap<_, _>>();

-		for (id, _, role) in layout.current().roles.items().iter() {
-			if let layout::NodeRoleV(Some(r)) = role {
-				let role = NodeAssignedRole {
-					zone: r.zone.to_string(),
-					capacity: r.capacity,
-					tags: r.tags.clone(),
-				};
-				match nodes.get_mut(id) {
-					None => {
-						nodes.insert(
-							*id,
-							NodeResp {
-								id: hex::encode(id),
-								role: Some(role),
-								..Default::default()
-							},
-						);
-					}
-					Some(n) => {
-						n.role = Some(role);
+		if let Ok(current_layout) = layout.current() {
+			for (id, _, role) in current_layout.roles.items().iter() {
+				if let layout::NodeRoleV(Some(r)) = role {
+					let role = NodeAssignedRole {
+						zone: r.zone.to_string(),
+						capacity: r.capacity,
+						tags: r.tags.clone(),
+					};
+					match nodes.get_mut(id) {
+						None => {
+							nodes.insert(
+								*id,
+								NodeResp {
+									id: hex::encode(id),
+									role: Some(role),
+									..Default::default()
+								},
+							);
+						}
+						Some(n) => {
+							n.role = Some(role);
+						}
 					}
 				}
 			}
 		}

-		for ver in layout.versions().iter().rev().skip(1) {
-			for (id, _, role) in ver.roles.items().iter() {
-				if let layout::NodeRoleV(Some(r)) = role {
-					if r.capacity.is_some() {
-						if let Some(n) = nodes.get_mut(id) {
-							if n.role.is_none() {
-								n.draining = true;
+		if let Ok(layout_versions) = layout.versions() {
+			for ver in layout_versions.iter().rev().skip(1) {
+				for (id, _, role) in ver.roles.items().iter() {
+					if let layout::NodeRoleV(Some(r)) = role {
+						if r.capacity.is_some() {
+							if let Some(n) = nodes.get_mut(id) {
+								if n.role.is_none() {
+									n.draining = true;
+								}
+							} else {
+								nodes.insert(
+									*id,
+									NodeResp {
+										id: hex::encode(id),
+										draining: true,
+										..Default::default()
+									},
+								);
 							}
-						} else {
-							nodes.insert(
-								*id,
-								NodeResp {
-									id: hex::encode(id),
-									draining: true,
-									..Default::default()
-								},
-							);
 						}
 					}
 				}
@@ -108,7 +112,7 @@ impl RequestHandler for GetClusterStatusRequest {
 		nodes.sort_by(|x, y| x.id.cmp(&y.id));

 		Ok(GetClusterStatusResponse {
-			layout_version: layout.current().version,
+			layout_version: layout.inner().current().version,
 			nodes,
 		})
 	}
@@ -134,7 +138,9 @@ impl RequestHandler for GetClusterHealthRequest {
 			known_nodes: health.known_nodes,
 			connected_nodes: health.connected_nodes,
 			storage_nodes: health.storage_nodes,
-			storage_nodes_ok: health.storage_nodes_ok,
+			// Translating storage_nodes_up (admin API context) to storage_nodes_ok (metrics context)
+			// TODO: when releasing major release, consider renaming all the fields in the metrics to storage_nodes_up
+			storage_nodes_up: health.storage_nodes_ok,
 			partitions: health.partitions,
 			partitions_quorum: health.partitions_quorum,
 			partitions_all_ok: health.partitions_all_ok,
@@ -157,9 +163,11 @@ impl RequestHandler for GetClusterStatisticsRequest {
 		// Gather storage node and free space statistics for current nodes
 		let layout = &garage.system.cluster_layout();
 		let mut node_partition_count = HashMap::<Uuid, u64>::new();
-		for short_id in layout.current().ring_assignment_data.iter() {
-			let id = layout.current().node_id_vec[*short_id as usize];
-			*node_partition_count.entry(id).or_default() += 1;
+		if let Ok(current_layout) = layout.current() {
+			for short_id in current_layout.ring_assignment_data.iter() {
+				let id = current_layout.node_id_vec[*short_id as usize];
+				*node_partition_count.entry(id).or_default() += 1;
+			}
 		}
 		let node_info = garage
 			.system
@@ -172,7 +180,11 @@ impl RequestHandler for GetClusterStatisticsRequest {
 		for (id, parts) in node_partition_count.iter() {
 			let info = node_info.get(id);
 			let status = info.map(|x| &x.status);
-			let role = layout.current().roles.get(id).and_then(|x| x.0.as_ref());
+			let role = layout
+				.current()
+				.ok()
+				.and_then(|l| l.roles.get(id))
+				.and_then(|x| x.0.as_ref());
 			let hostname = status.and_then(|x| x.hostname.as_deref()).unwrap_or("?");
 			let zone = role.map(|x| x.zone.as_str()).unwrap_or("?");
 			let capacity = role
@@ -1,8 +1,8 @@
 use std::convert::TryFrom;

-use err_derive::Error;
 use hyper::header::HeaderValue;
 use hyper::{HeaderMap, StatusCode};
+use thiserror::Error;

 pub use garage_model::helper::error::Error as HelperError;

@@ -16,36 +16,33 @@ use garage_api_common::helpers::*;
 /// Errors of this crate
 #[derive(Debug, Error)]
 pub enum Error {
-	#[error(display = "{}", _0)]
+	#[error("{0}")]
 	/// Error from common error
-	Common(#[error(source)] CommonError),
+	Common(#[from] CommonError),

 	// Category: cannot process
 	/// The admin API token does not exist
-	#[error(display = "Admin token not found: {}", _0)]
+	#[error("Admin token not found: {0}")]
 	NoSuchAdminToken(String),

 	/// The API access key does not exist
-	#[error(display = "Access key not found: {}", _0)]
+	#[error("Access key not found: {00}")]
 	NoSuchAccessKey(String),

 	/// The requested block does not exist
-	#[error(display = "Block not found: {}", _0)]
+	#[error("Block not found: {0}")]
 	NoSuchBlock(String),

 	/// The requested worker does not exist
-	#[error(display = "Worker not found: {}", _0)]
+	#[error("Worker not found: {0}")]
 	NoSuchWorker(u64),

 	/// The object requested don't exists
-	#[error(display = "Key not found")]
+	#[error("Key not found")]
 	NoSuchKey,

 	/// In Import key, the key already exists
-	#[error(
-		display = "Key {} already exists in data store. Even if it is deleted, we can't let you create a new key with the same ID. Sorry.",
-		_0
-	)]
+	#[error("Key {0} already exists in data store. Even if it is deleted, we can't let you create a new key with the same ID. Sorry.")]
 	KeyAlreadyExists(String),
 }

@@ -95,6 +92,7 @@ impl ApiError for Error {
 	fn add_http_headers(&self, header_map: &mut HeaderMap<HeaderValue>) {
 		use hyper::header;
 		header_map.append(header::CONTENT_TYPE, "application/json".parse().unwrap());
+		header_map.append(header::ACCESS_CONTROL_ALLOW_ORIGIN, "*".parse().unwrap());
 	}

 	fn http_body(&self, garage_region: &str, path: &str) -> ErrorBody {
@@ -230,9 +230,18 @@ async fn key_info_results(
 		},
 		buckets: relevant_buckets
 			.into_values()
-			.map(|bucket| {
+			.filter_map(|bucket| {
 				let state = bucket.state.as_option().unwrap();
-				KeyInfoBucketResponse {
+				let permissions = key_state
+					.authorized_buckets
+					.get(&bucket.id)
+					.filter(|p| p.is_any())
+					.map(|p| ApiBucketKeyPerm {
+						read: p.allow_read,
+						write: p.allow_write,
+						owner: p.allow_owner,
+					})?;
+				Some(KeyInfoBucketResponse {
 					id: hex::encode(bucket.id),
 					global_aliases: state
 						.aliases
@@ -248,16 +257,8 @@ async fn key_info_results(
 						.filter(|((k, _), _, a)| *a && *k == key.key_id)
 						.map(|((_, n), _, _)| n.to_string())
 						.collect::<Vec<_>>(),
-					permissions: key_state
-						.authorized_buckets
-						.get(&bucket.id)
-						.map(|p| ApiBucketKeyPerm {
-							read: p.allow_read,
-							write: p.allow_write,
-							owner: p.allow_owner,
-						})
-						.unwrap_or_default(),
-				}
+					permissions,
+				})
 			})
 			.collect::<Vec<_>>(),
 	};
@@ -74,16 +74,18 @@ macro_rules! admin_endpoints {
                type Response = AdminApiResponse;

                async fn handle(self, garage: &Arc<Garage>, admin: &Admin) -> Result<AdminApiResponse, Error> {
-                    Ok(match self {
+                    match self {
                        $(
-                            AdminApiRequest::$special_endpoint(_) => panic!(
-                                concat!(stringify!($special_endpoint), " needs to go through a special handler")
+                            AdminApiRequest::$special_endpoint(_) => Err(
+                                Error::Common(CommonError::BadRequest(
+                                    concat!(stringify!($special_endpoint), " cannot be used outside of the HTTP Admin API").into()
+                                ))
                            ),
                        )*
                        $(
-                            AdminApiRequest::$endpoint(req) => AdminApiResponse::$endpoint(req.handle(garage, admin).await?),
+                            AdminApiRequest::$endpoint(req) => Ok(AdminApiResponse::$endpoint(req.handle(garage, admin).await?)),
                        )*
-                    })
+                    }
                }
            }
        }
@@ -106,17 +106,17 @@ impl RequestHandler for LocalGetNodeStatisticsRequest {

 		// Gather block manager statistics
 		writeln!(&mut ret, "\nBlock manager stats:").unwrap();
-		let rc_len = garage.block_manager.rc_len()?.to_string();
+		let rc_len = garage.block_manager.rc_approximate_len()?.to_string();

 		ret += &format_table_to_string(vec![
 			format!("  number of RC entries:\t{} (~= number of blocks)", rc_len),
 			format!(
 				"  resync queue length:\t{}",
-				garage.block_manager.resync.queue_len()?
+				garage.block_manager.resync.queue_approximate_len()?
 			),
 			format!(
 				"  blocks with resync errors:\t{}",
-				garage.block_manager.resync.errors_len()?
+				garage.block_manager.resync.errors_approximate_len()?
 			),
 		]);

@@ -129,16 +129,21 @@ where
 	F: TableSchema + 'static,
 	R: TableReplication + 'static,
 {
-	let data_len = t.data.store.len().map_err(GarageError::from)?.to_string();
-	let mkl_len = t.merkle_updater.merkle_tree_len()?.to_string();
+	let data_len = t
+		.data
+		.store
+		.approximate_len()
+		.map_err(GarageError::from)?
+		.to_string();
+	let mkl_len = t.merkle_updater.merkle_tree_approximate_len()?.to_string();

 	Ok(format!(
 		"  {}\t{}\t{}\t{}\t{}\t{}",
 		F::TABLE_NAME,
 		data_len,
 		mkl_len,
-		t.merkle_updater.todo_len()?,
-		t.data.insert_queue_len()?,
-		t.data.gc_todo_len()?
+		t.merkle_updater.todo_approximate_len()?,
+		t.data.insert_queue_approximate_len()?,
+		t.data.gc_todo_approximate_len()?
 	))
 }
@@ -46,9 +46,7 @@ a static website for the requested domain. This is used by reverse proxies such
 as Caddy or Tricot, to avoid requesting TLS certificates for domain names that
 do not correspond to an actual website.
    ",
-    params(
-        ("domain", description = "The domain name to check for"),
-    ),
+    params(CheckDomainRequest),
    security(()),
 	responses(
            (status = 200, description = "The domain name redirects to a static website bucket"),
@@ -167,9 +165,7 @@ fn CreateAdminToken() -> () {}
 Updates information about the specified admin API token.
    ",
    request_body = UpdateAdminTokenRequestBody,
-    params(
-        ("id", description = "Admin API token ID"),
-    ),
+    params(UpdateAdminTokenRequest),
 	responses(
            (status = 200, description = "Admin token has been updated", body = UpdateAdminTokenResponse),
            (status = 500, description = "Internal server error")
@@ -181,9 +177,7 @@ fn UpdateAdminToken() -> () {}
    path = "/v2/DeleteAdminToken",
    tag = "Admin API token",
    description = "Delete an admin API token from the cluster, revoking all its permissions.",
-    params(
-        ("id", description = "Admin API token ID"),
-    ),
+    params(DeleteAdminTokenRequest),
 	responses(
            (status = 200, description = "Admin token has been deleted"),
            (status = 500, description = "Internal server error")
@@ -191,6 +185,19 @@ fn UpdateAdminToken() -> () {}
 )]
 fn DeleteAdminToken() -> () {}

+#[utoipa::path(get,
+    path = "/v2/GetCurrentAdminTokenInfo",
+    tag = "Admin API token",
+    description = "
+Return information about the calling admin API token.
+    ",
+    responses(
+            (status = 200, description = "Information about the admin token", body = GetCurrentAdminTokenInfoResponse),
+            (status = 500, description = "Internal server error")
+    ),
+)]
+fn GetCurrentAdminTokenInfo() -> () {}
+
 // **********************************************
 //      Layout operations
 // **********************************************
@@ -378,9 +385,7 @@ Updates information about the specified API access key.
 *Note: the secret key is not returned in the response, `null` is sent instead.*
    ",
    request_body = UpdateKeyRequestBody,
-    params(
-        ("id", description = "Access key ID"),
-    ),
+    params(UpdateKeyRequest),
 	responses(
            (status = 200, description = "Access key has been updated", body = UpdateKeyResponse),
            (status = 500, description = "Internal server error")
@@ -392,9 +397,7 @@ fn UpdateKey() -> () {}
    path = "/v2/DeleteKey",
    tag = "Access key",
    description = "Delete a key from the cluster. Its access will be removed from all the buckets. Buckets are not automatically deleted and can be dangling. You should manually delete them before. ",
-    params(
-        ("id", description = "Access key ID"),
-    ),
+    params(DeleteKeyRequest),
 	responses(
            (status = 200, description = "Access key has been deleted"),
            (status = 500, description = "Internal server error")
@@ -465,9 +468,7 @@ In `quotas`: new values of `maxSize` and `maxObjects` must both be specified, or
 to remove the quotas. An absent value will be considered the same as a `null`. It is not possible
 to change only one of the two quotas.
    ",
-    params(
-        ("id", description = "ID of the bucket to update"),
-    ),
+    params(UpdateBucketRequest),
    request_body = UpdateBucketRequestBody,
 	responses(
            (status = 200, description = "Bucket has been updated", body = UpdateBucketResponse),
@@ -485,9 +486,7 @@ Deletes a storage bucket. A bucket cannot be deleted if it is not empty.

 **Warning:** this will delete all aliases associated with the bucket!
    ",
-    params(
-        ("id", description = "ID of the bucket to delete"),
-    ),
+    params(DeleteBucketRequest),
 	responses(
            (status = 200, description = "Bucket has been deleted"),
            (status = 400, description = "Bucket is not empty"),
@@ -617,9 +616,7 @@ fn RemoveBucketAlias() -> () {}
    description = "
 Return information about the Garage daemon running on one or several nodes.
    ",
-    params(
-        ("node", description = "Node ID to query, or `*` for all nodes, or `self` for the node responding to the request"),
-    ),
+    params(MultiRequestQueryParams),
 	responses(
            (status = 200, description = "Responses from individual cluster nodes", body = MultiResponse<LocalGetNodeInfoResponse>),
            (status = 500, description = "Internal server error")
@@ -635,9 +632,7 @@ Fetch statistics for one or several Garage nodes.

 *Note: do not try to parse the `freeform` field of the response, it is given as a string specifically because its format is not stable.*
    ",
-    params(
-        ("node", description = "Node ID to query, or `*` for all nodes, or `self` for the node responding to the request"),
-    ),
+    params(MultiRequestQueryParams),
 	responses(
            (status = 200, description = "Responses from individual cluster nodes", body = MultiResponse<LocalGetNodeStatisticsResponse>),
            (status = 500, description = "Internal server error")
@@ -651,9 +646,7 @@ fn GetNodeStatistics() -> () {}
    description = "
 Instruct one or several nodes to take a snapshot of their metadata databases.
    ",
-    params(
-        ("node", description = "Node ID to query, or `*` for all nodes, or `self` for the node responding to the request"),
-    ),
+    params(MultiRequestQueryParams),
 	responses(
            (status = 200, description = "Responses from individual cluster nodes", body = MultiResponse<LocalCreateMetadataSnapshotResponse>),
            (status = 500, description = "Internal server error")
@@ -667,9 +660,7 @@ fn CreateMetadataSnapshot() -> () {}
    description = "
 Launch a repair operation on one or several cluster nodes.
    ",
-    params(
-        ("node", description = "Node ID to query, or `*` for all nodes, or `self` for the node responding to the request"),
-    ),
+    params(MultiRequestQueryParams),
    request_body = LocalLaunchRepairOperationRequest,
 	responses(
            (status = 200, description = "Responses from individual cluster nodes", body = MultiResponse<LocalLaunchRepairOperationResponse>),
@@ -688,9 +679,7 @@ fn LaunchRepairOperation() -> () {}
    description = "
 List background workers currently running on one or several cluster nodes.
    ",
-    params(
-        ("node", description = "Node ID to query, or `*` for all nodes, or `self` for the node responding to the request"),
-    ),
+    params(MultiRequestQueryParams),
    request_body = LocalListWorkersRequest,
 	responses(
            (status = 200, description = "Responses from individual cluster nodes", body = MultiResponse<LocalListWorkersResponse>),
@@ -705,9 +694,7 @@ fn ListWorkers() -> () {}
    description = "
 Get information about the specified background worker on one or several cluster nodes.
    ",
-    params(
-        ("node", description = "Node ID to query, or `*` for all nodes, or `self` for the node responding to the request"),
-    ),
+    params(MultiRequestQueryParams),
    request_body = LocalGetWorkerInfoRequest,
 	responses(
            (status = 200, description = "Responses from individual cluster nodes", body = MultiResponse<LocalGetWorkerInfoResponse>),
@@ -722,9 +709,7 @@ fn GetWorkerInfo() -> () {}
    description = "
 Fetch values of one or several worker variables, from one or several cluster nodes.
    ",
-    params(
-        ("node", description = "Node ID to query, or `*` for all nodes, or `self` for the node responding to the request"),
-    ),
+    params(MultiRequestQueryParams),
    request_body = LocalGetWorkerVariableRequest,
 	responses(
            (status = 200, description = "Responses from individual cluster nodes", body = MultiResponse<LocalGetWorkerVariableResponse>),
@@ -739,9 +724,7 @@ fn GetWorkerVariable() -> () {}
    description = "
 Set the value for a worker variable, on one or several cluster nodes.
    ",
-    params(
-        ("node", description = "Node ID to query, or `*` for all nodes, or `self` for the node responding to the request"),
-    ),
+    params(MultiRequestQueryParams),
    request_body = LocalSetWorkerVariableRequest,
 	responses(
            (status = 200, description = "Responses from individual cluster nodes", body = MultiResponse<LocalSetWorkerVariableResponse>),
@@ -760,9 +743,7 @@ fn SetWorkerVariable() -> () {}
    description = "
 List data blocks that are currently in an errored state on one or several Garage nodes.
    ",
-    params(
-        ("node", description = "Node ID to query, or `*` for all nodes, or `self` for the node responding to the request"),
-    ),
+    params(MultiRequestQueryParams),
 	responses(
            (status = 200, description = "Responses from individual cluster nodes", body = MultiResponse<LocalListBlockErrorsResponse>),
            (status = 500, description = "Internal server error")
@@ -776,9 +757,7 @@ fn ListBlockErrors() -> () {}
    description = "
 Get detailed information about a data block stored on a Garage node, including all object versions and in-progress multipart uploads that contain a reference to this block.
    ",
-    params(
-        ("node", description = "Node ID to query, or `*` for all nodes, or `self` for the node responding to the request"),
-    ),
+    params(MultiRequestQueryParams),
    request_body = LocalGetBlockInfoRequest,
 	responses(
            (status = 200, description = "Detailed block information", body = MultiResponse<LocalGetBlockInfoResponse>),
@@ -793,9 +772,7 @@ fn GetBlockInfo() -> () {}
    description = "
 Instruct Garage node(s) to retry the resynchronization of one or several missing data block(s).
    ",
-    params(
-        ("node", description = "Node ID to query, or `*` for all nodes, or `self` for the node responding to the request"),
-    ),
+    params(MultiRequestQueryParams),
    request_body = LocalRetryBlockResyncRequest,
 	responses(
            (status = 200, description = "Responses from individual cluster nodes", body = MultiResponse<LocalRetryBlockResyncResponse>),
@@ -812,9 +789,7 @@ Purge references to one or several missing data blocks.

 This will remove all objects and in-progress multipart uploads that contain the specified data block(s). The objects will be permanently deleted from the buckets in which they appear. Use with caution.
    ",
-    params(
-        ("node", description = "Node ID to query, or `*` for all nodes, or `self` for the node responding to the request"),
-    ),
+    params(MultiRequestQueryParams),
    request_body = LocalPurgeBlocksRequest,
 	responses(
            (status = 200, description = "Responses from individual cluster nodes", body = MultiResponse<LocalPurgeBlocksResponse>),
@@ -843,7 +818,7 @@ impl Modify for SecurityAddon {
 #[derive(OpenApi)]
 #[openapi(
    info(
-        version = "v2.0.0",
+        version = "v2.1.0",
        title = "Garage administration API",
        description = "Administrate your Garage cluster programatically, including status, layout, keys, buckets, and maintainance tasks.

@@ -872,6 +847,7 @@ impl Modify for SecurityAddon {
        CreateAdminToken,
        UpdateAdminToken,
        DeleteAdminToken,
+        GetCurrentAdminTokenInfo,
        // Layout operations
        GetClusterLayout,
        GetClusterLayoutHistory,
@@ -93,6 +93,15 @@ impl RequestHandler for LocalLaunchRepairOperationRequest {
 				info!("Repairing bucket aliases (foreground)");
 				garage.locked_helper().await.repair_aliases().await?;
 			}
+			RepairType::ClearResyncQueue => {
+				info!("Clearing resync queue (foreground)");
+				let garage = garage.clone();
+				tokio::task::spawn_blocking(move || {
+					garage.block_manager.resync.clear_resync_queue()
+				})
+				.await
+				.map_err(garage_util::error::Error::from)??;
+			}
 		}
 		Ok(LocalLaunchRepairOperationResponse)
 	}
@@ -40,6 +40,7 @@ impl AdminApiRequest {
 			POST CreateAdminToken (body),
 			POST UpdateAdminToken (body_field, query::id),
 			POST DeleteAdminToken (query::id),
+			GET GetCurrentAdminTokenInfo (admin_token),
 			// Layout endpoints
 			GET GetClusterLayout (),
 			GET GetClusterLayoutHistory (),
@@ -1,6 +1,6 @@
 [package]
 name = "garage_api_common"
-version = "2.0.0"
+version = "2.1.0"
 authors = ["Alex Auvolat <alex@adnab.me>"]
 edition = "2018"
 license = "AGPL-3.0"
@@ -23,7 +23,7 @@ bytes.workspace = true
 chrono.workspace = true
 crc-fast.workspace = true
 crypto-common.workspace = true
-err-derive.workspace = true
+thiserror.workspace = true
 hex.workspace = true
 hmac.workspace = true
 md-5.workspace = true
@@ -1,7 +1,7 @@
 use std::convert::TryFrom;

-use err_derive::Error;
 use hyper::StatusCode;
+use thiserror::Error;

 use garage_util::error::Error as GarageError;

@@ -12,48 +12,48 @@ use garage_model::helper::error::Error as HelperError;
 pub enum CommonError {
 	// ---- INTERNAL ERRORS ----
 	/// Error related to deeper parts of Garage
-	#[error(display = "Internal error: {}", _0)]
-	InternalError(#[error(source)] GarageError),
+	#[error("Internal error: {0}")]
+	InternalError(#[from] GarageError),

 	/// Error related to Hyper
-	#[error(display = "Internal error (Hyper error): {}", _0)]
-	Hyper(#[error(source)] hyper::Error),
+	#[error("Internal error (Hyper error): {0}")]
+	Hyper(#[from] hyper::Error),

 	/// Error related to HTTP
-	#[error(display = "Internal error (HTTP error): {}", _0)]
-	Http(#[error(source)] http::Error),
+	#[error("Internal error (HTTP error): {0}")]
+	Http(#[from] http::Error),

 	// ---- GENERIC CLIENT ERRORS ----
 	/// Proper authentication was not provided
-	#[error(display = "Forbidden: {}", _0)]
+	#[error("Forbidden: {0}")]
 	Forbidden(String),

 	/// Generic bad request response with custom message
-	#[error(display = "Bad request: {}", _0)]
+	#[error("Bad request: {0}")]
 	BadRequest(String),

 	/// The client sent a header with invalid value
-	#[error(display = "Invalid header value: {}", _0)]
-	InvalidHeader(#[error(source)] hyper::header::ToStrError),
+	#[error("Invalid header value: {0}")]
+	InvalidHeader(#[from] hyper::header::ToStrError),

 	// ---- SPECIFIC ERROR CONDITIONS ----
 	// These have to be error codes referenced in the S3 spec here:
 	// https://docs.aws.amazon.com/AmazonS3/latest/API/ErrorResponses.html#ErrorCodeList
 	/// The bucket requested don't exists
-	#[error(display = "Bucket not found: {}", _0)]
+	#[error("Bucket not found: {0}")]
 	NoSuchBucket(String),

 	/// Tried to create a bucket that already exist
-	#[error(display = "Bucket already exists")]
+	#[error("Bucket already exists")]
 	BucketAlreadyExists,

 	/// Tried to delete a non-empty bucket
-	#[error(display = "Tried to delete a non-empty bucket")]
+	#[error("Tried to delete a non-empty bucket")]
 	BucketNotEmpty,

 	// Category: bad request
 	/// Bucket name is not valid according to AWS S3 specs
-	#[error(display = "Invalid bucket name: {}", _0)]
+	#[error("Invalid bucket name: {0}")]
 	InvalidBucketName(String),
 }

@@ -59,6 +59,12 @@ pub trait ApiHandler: Send + Sync + 'static {
 		req: Request<IncomingBody>,
 		endpoint: Self::Endpoint,
 	) -> impl Future<Output = Result<Response<BoxBody<Self::Error>>, Self::Error>> + Send;
+
+	/// Returns the key id used to authenticate this request. The ID returned must be safe to
+	/// log.
+	fn key_id_from_request(&self, _req: &Request<IncomingBody>) -> Option<String> {
+		None
+	}
 }

 pub struct ApiServer<A: ApiHandler> {
@@ -143,19 +149,20 @@ impl<A: ApiHandler> ApiServer<A> {
 	) -> Result<Response<BoxBody<A::Error>>, http::Error> {
 		let uri = req.uri().clone();

-		if let Ok(forwarded_for_ip_addr) =
+		let source = if let Ok(forwarded_for_ip_addr) =
 			forwarded_headers::handle_forwarded_for_headers(req.headers())
 		{
-			info!(
-				"{} (via {}) {} {}",
-				forwarded_for_ip_addr,
-				addr,
-				req.method(),
-				uri
-			);
+			format!("{forwarded_for_ip_addr} (via {addr})")
 		} else {
-			info!("{} {} {}", addr, req.method(), uri);
-		}
+			format!("{addr}")
+		};
+		// we only do this to log the access key, so we can discard any error
+		let key = self
+			.api_handler
+			.key_id_from_request(&req)
+			.map(|k| format!("(key {k}) "))
+			.unwrap_or_default();
+		info!("{source} {key}{} {uri}", req.method());
 		debug!("{:?}", req);

 		let tracer = opentelemetry::global::tracer("garage");
@@ -344,7 +351,11 @@ where

 	while !*must_exit.borrow() {
 		let (stream, client_addr) = tokio::select! {
-			acc = listener.accept() => acc?,
+			acc = listener.accept() => match acc {
+							Ok(r) => r,
+							Err(e) if e.kind() == std::io::ErrorKind::ConnectionAborted => continue,
+							Err(e) => return Err(e.into()),
+						},
 			_ = must_exit.changed() => continue,
 		};

@@ -83,6 +83,21 @@ macro_rules! router_match {
            parse_json_body::< [<$api Request>], _, Error>($req).await?
        })
    }};
+   (@@gen_parse_request $api:ident, (admin_token), $query: expr, $req:expr) => {{
+        paste!({
+            let auth_header = $req.headers()
+                .get(hyper::header::AUTHORIZATION)
+                .ok_or_else(|| Error::bad_request("Missing Authorization header"))?
+                .to_str()
+                .map_err(|_| Error::bad_request("Invalid Authorization header"))?;
+
+            let admin_token = auth_header.strip_prefix("Bearer ")
+                .ok_or_else(|| Error::bad_request("Authorization header must be Bearer token"))?
+                .to_string();
+
+            [< $api Request >] { admin_token }
+        })
+    }};
    (@@gen_parse_request $api:ident, (body_field, $($conv:ident $(($conv_arg:expr))? :: $param:ident),*), $query: expr, $req:expr)
        =>
    {{
@@ -1,4 +1,4 @@
-use err_derive::Error;
+use thiserror::Error;

 use crate::common_error::CommonError;
 pub use crate::common_error::{CommonErrorDerivative, OkOrBadRequest, OkOrInternalError};
@@ -6,21 +6,21 @@ pub use crate::common_error::{CommonErrorDerivative, OkOrBadRequest, OkOrInterna
 /// Errors of this crate
 #[derive(Debug, Error)]
 pub enum Error {
-	#[error(display = "{}", _0)]
+	#[error("{0}")]
 	/// Error from common error
 	Common(CommonError),

 	/// Authorization Header Malformed
-	#[error(display = "Authorization header malformed, unexpected scope: {}", _0)]
+	#[error("Authorization header malformed, unexpected scope: {0}")]
 	AuthorizationHeaderMalformed(String),

 	// Category: bad request
 	/// The request contained an invalid UTF-8 sequence in its path or in other parameters
-	#[error(display = "Invalid UTF-8: {}", _0)]
-	InvalidUtf8Str(#[error(source)] std::str::Utf8Error),
+	#[error("Invalid UTF-8: {0}")]
+	InvalidUtf8Str(#[from] std::str::Utf8Error),

 	/// The provided digest (checksum) value was invalid
-	#[error(display = "Invalid digest: {}", _0)]
+	#[error("Invalid digest: {0}")]
 	InvalidDigest(String),
 }

@@ -424,7 +424,7 @@ pub fn verify_v4(
 // ============ Authorization header, or X-Amz-* query params =========

 pub struct Authorization {
-	key_id: String,
+	pub key_id: String,
 	scope: String,
 	signed_headers: String,
 	signature: String,
@@ -433,7 +433,7 @@ pub struct Authorization {
 }

 impl Authorization {
-	fn parse_header(headers: &HeaderMap) -> Result<Self, Error> {
+	pub fn parse_header(headers: &HeaderMap) -> Result<Self, Error> {
 		let authorization = headers
 			.get(AUTHORIZATION)
 			.ok_or_bad_request("Missing authorization header")?
@@ -1,6 +1,6 @@
 [package]
 name = "garage_api_k2v"
-version = "2.0.0"
+version = "2.1.0"
 authors = ["Alex Auvolat <alex@adnab.me>"]
 edition = "2018"
 license = "AGPL-3.0"
@@ -20,7 +20,7 @@ garage_util = { workspace = true, features = [ "k2v" ] }
 garage_api_common.workspace = true

 base64.workspace = true
-err-derive.workspace = true
+thiserror.workspace = true
 tracing.workspace = true

 futures.workspace = true
@@ -171,6 +171,12 @@ impl ApiHandler for K2VApiServer {

 		Ok(resp_ok)
 	}
+
+	fn key_id_from_request(&self, req: &Request<IncomingBody>) -> Option<String> {
+		garage_api_common::signature::payload::Authorization::parse_header(req.headers())
+			.map(|auth| auth.key_id)
+			.ok()
+	}
 }

 impl ApiEndpoint for K2VApiEndpoint {
@@ -1,6 +1,6 @@
-use err_derive::Error;
 use hyper::header::HeaderValue;
 use hyper::{HeaderMap, StatusCode};
+use thiserror::Error;

 pub(crate) use garage_api_common::common_error::pass_helper_error;
 use garage_api_common::common_error::{commonErrorDerivative, CommonError};
@@ -14,38 +14,38 @@ use garage_api_common::signature::error::Error as SignatureError;
 /// Errors of this crate
 #[derive(Debug, Error)]
 pub enum Error {
-	#[error(display = "{}", _0)]
+	#[error("{0}")]
 	/// Error from common error
-	Common(#[error(source)] CommonError),
+	Common(#[from] CommonError),

 	// Category: cannot process
 	/// Authorization Header Malformed
-	#[error(display = "Authorization header malformed, unexpected scope: {}", _0)]
+	#[error("Authorization header malformed, unexpected scope: {0}")]
 	AuthorizationHeaderMalformed(String),

 	/// The provided digest (checksum) value was invalid
-	#[error(display = "Invalid digest: {}", _0)]
+	#[error("Invalid digest: {0}")]
 	InvalidDigest(String),

 	/// The object requested don't exists
-	#[error(display = "Key not found")]
+	#[error("Key not found")]
 	NoSuchKey,

 	/// Some base64 encoded data was badly encoded
-	#[error(display = "Invalid base64: {}", _0)]
-	InvalidBase64(#[error(source)] base64::DecodeError),
+	#[error("Invalid base64: {0}")]
+	InvalidBase64(#[from] base64::DecodeError),

 	/// Invalid causality token
-	#[error(display = "Invalid causality token")]
+	#[error("Invalid causality token")]
 	InvalidCausalityToken,

 	/// The client asked for an invalid return format (invalid Accept header)
-	#[error(display = "Not acceptable: {}", _0)]
+	#[error("Not acceptable: {0}")]
 	NotAcceptable(String),

 	/// The request contained an invalid UTF-8 sequence in its path or in other parameters
-	#[error(display = "Invalid UTF-8: {}", _0)]
-	InvalidUtf8Str(#[error(source)] std::str::Utf8Error),
+	#[error("Invalid UTF-8: {0}")]
+	InvalidUtf8Str(#[from] std::str::Utf8Error),
 }

 commonErrorDerivative!(Error);
@@ -99,6 +99,7 @@ impl ApiError for Error {
 	fn add_http_headers(&self, header_map: &mut HeaderMap<HeaderValue>) {
 		use hyper::header;
 		header_map.append(header::CONTENT_TYPE, "application/json".parse().unwrap());
+		header_map.append(header::ACCESS_CONTROL_ALLOW_ORIGIN, "*".parse().unwrap());
 	}

 	fn http_body(&self, garage_region: &str, path: &str) -> ErrorBody {
@@ -28,7 +28,7 @@ pub async fn handle_read_index(
 	let node_id_vec = garage
 		.system
 		.cluster_layout()
-		.all_nongateway_nodes()
+		.all_nongateway_nodes()?
 		.to_vec();

 	let (partition_keys, more, next_start) = read_range(
@@ -66,7 +66,7 @@ pub async fn handle_read_index(
 					bytes: *vals.get(&s_bytes).unwrap_or(&0),
 				}
 			})
-			.collect::<Vec<_>>(),
+			.collect(),
 		more,
 		next_start,
 	};
@@ -1,6 +1,6 @@
 [package]
 name = "garage_api_s3"
-version = "2.0.0"
+version = "2.1.0"
 authors = ["Alex Auvolat <alex@adnab.me>"]
 edition = "2018"
 license = "AGPL-3.0"
@@ -28,7 +28,7 @@ base64.workspace = true
 bytes.workspace = true
 chrono.workspace = true
 crc-fast.workspace = true
-err-derive.workspace = true
+thiserror.workspace = true
 hex.workspace = true
 hmac.workspace = true
 tracing.workspace = true
@@ -340,6 +340,12 @@ impl ApiHandler for S3ApiServer {

 		Ok(resp_ok)
 	}
+
+	fn key_id_from_request(&self, req: &Request<IncomingBody>) -> Option<String> {
+		garage_api_common::signature::payload::Authorization::parse_header(req.headers())
+			.map(|auth| auth.key_id)
+			.ok()
+	}
 }

 impl ApiEndpoint for S3ApiEndpoint {
@@ -29,7 +29,7 @@ pub async fn handle_get_cors(ctx: ReqCtx) -> Result<Response<ResBody>, Error> {
 			.body(string_body(xml))?)
 	} else {
 		Ok(Response::builder()
-			.status(StatusCode::NO_CONTENT)
+			.status(StatusCode::NOT_FOUND)
 			.body(empty_body())?)
 	}
 }
@@ -1,8 +1,8 @@
 use std::convert::TryInto;

-use err_derive::Error;
 use hyper::header::HeaderValue;
 use hyper::{HeaderMap, StatusCode};
+use thiserror::Error;

 use garage_model::helper::error::Error as HelperError;

@@ -25,67 +25,67 @@ use crate::xml as s3_xml;
 /// Errors of this crate
 #[derive(Debug, Error)]
 pub enum Error {
-	#[error(display = "{}", _0)]
+	#[error("{0}")]
 	/// Error from common error
-	Common(#[error(source)] CommonError),
+	Common(#[from] CommonError),

 	// Category: cannot process
 	/// Authorization Header Malformed
-	#[error(display = "Authorization header malformed, unexpected scope: {}", _0)]
+	#[error("Authorization header malformed, unexpected scope: {0}")]
 	AuthorizationHeaderMalformed(String),

 	/// The object requested don't exists
-	#[error(display = "Key not found")]
+	#[error("Key not found")]
 	NoSuchKey,

 	/// The multipart upload requested don't exists
-	#[error(display = "Upload not found")]
+	#[error("Upload not found")]
 	NoSuchUpload,

 	/// Precondition failed (e.g. x-amz-copy-source-if-match)
-	#[error(display = "At least one of the preconditions you specified did not hold")]
+	#[error("At least one of the preconditions you specified did not hold")]
 	PreconditionFailed,

 	/// Parts specified in CMU request do not match parts actually uploaded
-	#[error(display = "Parts given to CompleteMultipartUpload do not match uploaded parts")]
+	#[error("Parts given to CompleteMultipartUpload do not match uploaded parts")]
 	InvalidPart,

 	/// Parts given to CompleteMultipartUpload were not in ascending order
-	#[error(display = "Parts given to CompleteMultipartUpload were not in ascending order")]
+	#[error("Parts given to CompleteMultipartUpload were not in ascending order")]
 	InvalidPartOrder,

 	/// In CompleteMultipartUpload: not enough data
 	/// (here we are more lenient than AWS S3)
-	#[error(display = "Proposed upload is smaller than the minimum allowed object size")]
+	#[error("Proposed upload is smaller than the minimum allowed object size")]
 	EntityTooSmall,

 	// Category: bad request
 	/// The request contained an invalid UTF-8 sequence in its path or in other parameters
-	#[error(display = "Invalid UTF-8: {}", _0)]
-	InvalidUtf8Str(#[error(source)] std::str::Utf8Error),
+	#[error("Invalid UTF-8: {0}")]
+	InvalidUtf8Str(#[from] std::str::Utf8Error),

 	/// The request used an invalid path
-	#[error(display = "Invalid UTF-8: {}", _0)]
-	InvalidUtf8String(#[error(source)] std::string::FromUtf8Error),
+	#[error("Invalid UTF-8: {0}")]
+	InvalidUtf8String(#[from] std::string::FromUtf8Error),

 	/// The client sent invalid XML data
-	#[error(display = "Invalid XML: {}", _0)]
+	#[error("Invalid XML: {0}")]
 	InvalidXml(String),

 	/// The client sent a range header with invalid value
-	#[error(display = "Invalid HTTP range: {:?}", _0)]
-	InvalidRange(#[error(from)] (http_range::HttpRangeParseError, u64)),
+	#[error("Invalid HTTP range: {0:?}")]
+	InvalidRange((http_range::HttpRangeParseError, u64)),

 	/// The client sent a range header with invalid value
-	#[error(display = "Invalid encryption algorithm: {:?}, should be AES256", _0)]
+	#[error("Invalid encryption algorithm: {0:?}, should be AES256")]
 	InvalidEncryptionAlgorithm(String),

 	/// The provided digest (checksum) value was invalid
-	#[error(display = "Invalid digest: {}", _0)]
+	#[error("Invalid digest: {0}")]
 	InvalidDigest(String),

 	/// The client sent a request for an action not supported by garage
-	#[error(display = "Unimplemented action: {}", _0)]
+	#[error("Unimplemented action: {0}")]
 	NotImplemented(String),
 }

@@ -99,6 +99,12 @@ impl From<HelperError> for Error {
 	}
 }

+impl From<(http_range::HttpRangeParseError, u64)> for Error {
+	fn from(err: (http_range::HttpRangeParseError, u64)) -> Error {
+		Error::InvalidRange(err)
+	}
+}
+
 impl From<roxmltree::Error> for Error {
 	fn from(err: roxmltree::Error) -> Self {
 		Self::InvalidXml(format!("{}", err))
@@ -176,6 +182,7 @@ impl ApiError for Error {
 		use hyper::header;

 		header_map.append(header::CONTENT_TYPE, "application/xml".parse().unwrap());
+		header_map.append(header::ACCESS_CONTROL_ALLOW_ORIGIN, "*".parse().unwrap());

 		#[allow(clippy::single_match)]
 		match self {
@@ -5,6 +5,7 @@ use std::sync::Arc;
 use base64::prelude::*;
 use crc_fast::{CrcAlgorithm, Digest as CrcDigest};
 use futures::prelude::*;
+use http::StatusCode;
 use hyper::{header::HeaderValue, HeaderMap, Request, Response};
 use md5::{Digest, Md5};
 use sha1::Sha1;
@@ -302,8 +303,11 @@ pub async fn handle_complete_multipart_upload(
 	let body = req_body.collect().await?;

 	let body_xml = roxmltree::Document::parse(std::str::from_utf8(&body)?)?;
-	let body_list_of_parts = parse_complete_multipart_upload_body(&body_xml)
-		.ok_or_bad_request("Invalid CompleteMultipartUpload XML")?;
+	let body_list_of_parts =
+		parse_complete_multipart_upload_body(&body_xml).ok_or_bad_request(format!(
+			"Invalid CompleteMultipartUpload XML:\n{}",
+			String::from_utf8_lossy(&body)
+		))?;
 	debug!(
 		"CompleteMultipartUpload list of parts: {:?}",
 		body_list_of_parts
@@ -548,7 +552,9 @@ pub async fn handle_abort_multipart_upload(
 	let final_object = Object::new(*bucket_id, key.to_string(), vec![object_version]);
 	garage.object_table.insert(&final_object).await?;

-	Ok(Response::new(empty_body()))
+	Ok(Response::builder()
+		.status(StatusCode::NO_CONTENT)
+		.body(empty_body())?)
 }

 // ======== helpers ============
@@ -600,6 +606,32 @@ struct CompleteMultipartUploadPart {
 	checksum: Option<ChecksumValue>,
 }

+macro_rules! extract_checksum_from {
+	($node:ident { $($name:expr => $variant:ident),* $(,)? }) => {
+		if false { None }
+		$(
+			else if let Some(node) = $node.children().find(|e| e.has_tag_name($name)) {
+				match node.last_child().map(|x| x.text()) {
+					// Child is text but empty post-trim, ignore it.
+					Some(Some(text)) if text.trim().is_empty() => None,
+
+					// Child is non-empty text, parse it.
+					Some(Some(text)) => Some(ChecksumValue::$variant(
+						BASE64_STANDARD.decode(text).ok()?[..].try_into().ok()?
+					)),
+
+					// Child is not text, reject it.
+					Some(None) => return None,
+
+					// No child, ignore it.
+					None => None,
+				}
+			}
+		)*
+		else { None }
+	}
+}
+
 fn parse_complete_multipart_upload_body(
 	xml: &roxmltree::Document,
 ) -> Option<Vec<CompleteMultipartUploadPart>> {
@@ -623,46 +655,15 @@ fn parse_complete_multipart_upload_body(
 				.children()
 				.find(|e| e.has_tag_name("PartNumber"))?
 				.text()?;
-			let checksum = if let Some(crc32) =
-				item.children().find(|e| e.has_tag_name("ChecksumCRC32"))
-			{
-				Some(ChecksumValue::Crc32(
-					BASE64_STANDARD.decode(crc32.text()?).ok()?[..]
-						.try_into()
-						.ok()?,
-				))
-			} else if let Some(crc32c) = item.children().find(|e| e.has_tag_name("ChecksumCRC32C"))
-			{
-				Some(ChecksumValue::Crc32c(
-					BASE64_STANDARD.decode(crc32c.text()?).ok()?[..]
-						.try_into()
-						.ok()?,
-				))
-			} else if let Some(crc64nvme) = item
-				.children()
-				.find(|e| e.has_tag_name("ChecksumCRC64NVME"))
-			{
-				Some(ChecksumValue::Crc64Nvme(
-					BASE64_STANDARD.decode(crc64nvme.text()?).ok()?[..]
-						.try_into()
-						.ok()?,
-				))
-			} else if let Some(sha1) = item.children().find(|e| e.has_tag_name("ChecksumSHA1")) {
-				Some(ChecksumValue::Sha1(
-					BASE64_STANDARD.decode(sha1.text()?).ok()?[..]
-						.try_into()
-						.ok()?,
-				))
-			} else if let Some(sha256) = item.children().find(|e| e.has_tag_name("ChecksumSHA256"))
-			{
-				Some(ChecksumValue::Sha256(
-					BASE64_STANDARD.decode(sha256.text()?).ok()?[..]
-						.try_into()
-						.ok()?,
-				))
-			} else {
-				None
-			};
+
+			let checksum = extract_checksum_from!(item {
+				"ChecksumCRC32" => Crc32,
+				"ChecksumCRC32C" => Crc32c,
+				"ChecksumCRC64NVME" => Crc64Nvme,
+				"ChecksumSHA1" => Sha1,
+				"ChecksumSHA256" => Sha256,
+			});
+
 			parts.push(CompleteMultipartUploadPart {
 				etag: etag.trim_matches('"').to_string(),
 				part_number: part_number.parse().ok()?,
@@ -1,6 +1,6 @@
 [package]
 name = "garage_block"
-version = "2.0.0"
+version = "2.1.0"
 authors = ["Alex Auvolat <alex@adnab.me>"]
 edition = "2018"
 license = "AGPL-3.0"
@@ -48,6 +48,8 @@ pub const INLINE_THRESHOLD: usize = 3072;
 // to delete the block locally.
 pub(crate) const BLOCK_GC_DELAY: Duration = Duration::from_secs(600);

+const BLOCK_READ_SEMAPHORE_TIMEOUT: Duration = Duration::from_secs(15);
+
 /// RPC messages used to share blocks of data between nodes
 #[derive(Debug, Serialize, Deserialize)]
 pub enum BlockRpc {
@@ -85,6 +87,7 @@ pub struct BlockManager {
 	disable_scrub: bool,

 	mutation_lock: Vec<Mutex<BlockManagerLocked>>,
+	read_semaphore: Semaphore,

 	pub rc: BlockRc,
 	pub resync: BlockResyncManager,
@@ -174,6 +177,8 @@ impl BlockManager {
 				.iter()
 				.map(|_| Mutex::new(BlockManagerLocked()))
 				.collect::<Vec<_>>(),
+
+			read_semaphore: Semaphore::new(config.block_max_concurrent_reads),
 			rc,
 			resync,
 			system,
@@ -285,7 +290,7 @@ impl BlockManager {
 		let who = self
 			.system
 			.rpc_helper()
-			.block_read_nodes_of(hash, self.system.rpc_helper());
+			.block_read_nodes_of(hash, self.system.rpc_helper())?;

 		for node in who.iter() {
 			let node_id = NodeID::from(*node);
@@ -341,12 +346,9 @@ impl BlockManager {
 	/// layout version only: since blocks are immutable, we don't need to
 	/// do complex logic when several layour versions are active at once,
 	/// just move them directly to the new nodes.
-	pub(crate) fn storage_nodes_of(&self, hash: &Hash) -> Vec<Uuid> {
-		self.system
-			.cluster_layout()
-			.current()
-			.nodes_of(hash)
-			.collect()
+	pub(crate) fn storage_nodes_of(&self, hash: &Hash) -> Result<Vec<Uuid>, Error> {
+		let cluster_layout = self.system.cluster_layout();
+		Ok(cluster_layout.current()?.nodes_of(hash).collect())
 	}

 	// ---- Public interface ----
@@ -381,7 +383,7 @@ impl BlockManager {
 		prevent_compression: bool,
 		order_tag: Option<OrderTag>,
 	) -> Result<(), Error> {
-		let who = self.storage_nodes_of(&hash);
+		let who = self.storage_nodes_of(&hash)?;

 		let compression_level = self.compression_level.filter(|_| !prevent_compression);
 		let (header, bytes) = DataBlock::from_buffer(data, compression_level)
@@ -419,8 +421,8 @@ impl BlockManager {
 	}

 	/// Get number of items in the refcount table
-	pub fn rc_len(&self) -> Result<usize, Error> {
-		Ok(self.rc.rc_table.len()?)
+	pub fn rc_approximate_len(&self) -> Result<usize, Error> {
+		Ok(self.rc.rc_table.approximate_len()?)
 	}

 	/// Send command to start/stop/manager scrub worker
@@ -438,7 +440,7 @@ impl BlockManager {

 	/// List all resync errors
 	pub fn list_resync_errors(&self) -> Result<Vec<BlockResyncErrorInfo>, Error> {
-		let mut blocks = Vec::with_capacity(self.resync.errors.len()?);
+		let mut blocks = Vec::with_capacity(self.resync.errors.approximate_len()?);
 		for ent in self.resync.errors.iter()? {
 			let (hash, cnt) = ent?;
 			let cnt = ErrorCounter::decode(&cnt);
@@ -568,9 +570,6 @@ impl BlockManager {
 			match self.find_block(hash).await {
 				Some(p) => self.read_block_from(hash, &p).await,
 				None => {
-					// Not found but maybe we should have had it ??
-					self.resync
-						.put_to_resync(hash, 2 * self.system.rpc_helper().rpc_timeout())?;
 					return Err(Error::Message(format!(
 						"block {:?} not found on node",
 						hash
@@ -592,6 +591,15 @@ impl BlockManager {
 	) -> Result<DataBlock, Error> {
 		let (header, path) = block_path.as_parts_ref();

+		let permit = tokio::select! {
+			sem = self.read_semaphore.acquire() => sem.ok_or_message("acquire read semaphore")?,
+			_ = tokio::time::sleep(BLOCK_READ_SEMAPHORE_TIMEOUT) => {
+				self.metrics.block_read_semaphore_timeouts.add(1);
+				debug!("read block {:?}: read_semaphore acquire timeout", hash);
+				return Err(Error::Message("read block: read_semaphore acquire timeout".into()));
+			}
+		};
+
 		let mut f = fs::File::open(&path).await?;
 		let mut data = vec![];
 		f.read_to_end(&mut data).await?;
@@ -616,6 +624,8 @@ impl BlockManager {
 			return Err(Error::CorruptData(*hash));
 		}

+		drop(permit);
+
 		Ok(data)
 	}

@@ -22,6 +22,7 @@ pub struct BlockManagerMetrics {

 	pub(crate) bytes_read: BoundCounter<u64>,
 	pub(crate) block_read_duration: BoundValueRecorder<f64>,
+	pub(crate) block_read_semaphore_timeouts: BoundCounter<u64>,
 	pub(crate) bytes_written: BoundCounter<u64>,
 	pub(crate) block_write_duration: BoundValueRecorder<f64>,
 	pub(crate) delete_counter: BoundCounter<u64>,
@@ -50,7 +51,7 @@ impl BlockManagerMetrics {
 				.init(),
 			_rc_size: meter
 				.u64_value_observer("block.rc_size", move |observer| {
-					if let Ok(value) = rc_tree.len() {
+					if let Ok(value) = rc_tree.approximate_len() {
 						observer.observe(value as u64, &[])
 					}
 				})
@@ -58,7 +59,7 @@ impl BlockManagerMetrics {
 				.init(),
 			_resync_queue_len: meter
 				.u64_value_observer("block.resync_queue_length", move |observer| {
-					if let Ok(value) = resync_queue.len() {
+					if let Ok(value) = resync_queue.approximate_len() {
 						observer.observe(value as u64, &[]);
 					}
 				})
@@ -68,7 +69,7 @@ impl BlockManagerMetrics {
 				.init(),
 			_resync_errored_blocks: meter
 				.u64_value_observer("block.resync_errored_blocks", move |observer| {
-					if let Ok(value) = resync_errors.len() {
+					if let Ok(value) = resync_errors.approximate_len() {
 						observer.observe(value as u64, &[]);
 					}
 				})
@@ -119,6 +120,11 @@ impl BlockManagerMetrics {
 				.with_description("Duration of block read operations")
 				.init()
 				.bind(&[]),
+			block_read_semaphore_timeouts: meter
+				.u64_counter("block.read_semaphore_timeouts")
+				.with_description("Number of block reads that failed due to semaphore acquire timeout")
+				.init()
+				.bind(&[]),
 			bytes_written: meter
 				.u64_counter("block.bytes_written")
 				.with_description("Number of bytes written to disk")
@@ -104,13 +104,13 @@ impl BlockResyncManager {
 	}

 	/// Get length of resync queue
-	pub fn queue_len(&self) -> Result<usize, Error> {
-		Ok(self.queue.len()?)
+	pub fn queue_approximate_len(&self) -> Result<usize, Error> {
+		Ok(self.queue.approximate_len()?)
 	}

 	/// Get number of blocks that have an error
-	pub fn errors_len(&self) -> Result<usize, Error> {
-		Ok(self.errors.len()?)
+	pub fn errors_approximate_len(&self) -> Result<usize, Error> {
+		Ok(self.errors.approximate_len()?)
 	}

 	/// Clear the error counter for a block and put it in queue immediately
@@ -131,6 +131,14 @@ impl BlockResyncManager {
 		)))
 	}

+	/// Clear the entire resync queue and list of errored blocks
+	/// Corresponds to `garage repair clear-resync-queue`
+	pub fn clear_resync_queue(&self) -> Result<(), Error> {
+		self.queue.clear()?;
+		self.errors.clear()?;
+		Ok(())
+	}
+
 	pub fn register_bg_vars(&self, vars: &mut vars::BgVars) {
 		let notify = self.notify.clone();
 		vars.register_rw(
@@ -375,7 +383,7 @@ impl BlockResyncManager {
 			info!("Resync block {:?}: offloading and deleting", hash);
 			let existing_path = existing_path.unwrap();

-			let mut who = manager.storage_nodes_of(hash);
+			let mut who = manager.storage_nodes_of(hash)?;
 			if who.len() < manager.write_quorum {
 				return Err(Error::Message("Not trying to offload block because we don't have a quorum of nodes to write to".to_string()));
 			}
@@ -458,7 +466,7 @@ impl BlockResyncManager {

 			// First, check whether we are still supposed to store that
 			// block in the latest cluster layout version.
-			let storage_nodes = manager.storage_nodes_of(&hash);
+			let storage_nodes = manager.storage_nodes_of(&hash)?;

 			if !storage_nodes.contains(&manager.system.id) {
 				info!(
@@ -540,9 +548,11 @@ impl Worker for ResyncWorker {
 		}

 		WorkerStatus {
-			queue_length: Some(self.manager.resync.queue_len().unwrap_or(0) as u64),
+			queue_length: Some(self.manager.resync.queue_approximate_len().unwrap_or(0) as u64),
 			tranquility: Some(tranquility),
-			persistent_errors: Some(self.manager.resync.errors_len().unwrap_or(0) as u64),
+			persistent_errors: Some(
+				self.manager.resync.errors_approximate_len().unwrap_or(0) as u64
+			),
 			..Default::default()
 		}
 	}
@@ -1,6 +1,6 @@
 [package]
 name = "garage_db"
-version = "2.0.0"
+version = "2.1.0"
 authors = ["Alex Auvolat <alex@adnab.me>"]
 edition = "2018"
 license = "AGPL-3.0"
@@ -12,14 +12,18 @@ readme = "../../README.md"
 path = "lib.rs"

 [dependencies]
-err-derive.workspace = true
+thiserror.workspace = true
 tracing.workspace = true

 heed = { workspace = true, optional = true }
+
 rusqlite = { workspace = true, optional = true, features = ["backup"] }
 r2d2 = { workspace = true, optional = true }
 r2d2_sqlite = { workspace = true, optional = true }

+fjall = { workspace = true, optional = true }
+parking_lot = { workspace = true, optional = true }
+
 [dev-dependencies]
 mktemp.workspace = true

@@ -27,4 +31,5 @@ mktemp.workspace = true
 default = [ "lmdb", "sqlite" ]
 bundled-libs = [ "rusqlite?/bundled" ]
 lmdb = [ "heed" ]
+fjall = [ "dep:fjall", "dep:parking_lot" ]
 sqlite = [ "rusqlite", "r2d2", "r2d2_sqlite" ]
@@ -0,0 +1,453 @@
+use core::ops::Bound;
+
+use std::path::PathBuf;
+use std::sync::Arc;
+
+use parking_lot::{MappedRwLockReadGuard, RwLock, RwLockReadGuard};
+
+use fjall::{
+	PartitionCreateOptions, PersistMode, TransactionalKeyspace, TransactionalPartitionHandle,
+	WriteTransaction,
+};
+
+use crate::{
+	open::{Engine, OpenOpt},
+	Db, Error, IDb, ITx, ITxFn, OnCommit, Result, TxError, TxFnResult, TxOpError, TxOpResult,
+	TxResult, TxValueIter, Value, ValueIter,
+};
+
+pub use fjall;
+
+// --
+
+pub(crate) fn open_db(path: &PathBuf, opt: &OpenOpt) -> Result<Db> {
+	info!("Opening Fjall database at: {}", path.display());
+	if opt.fsync {
+		return Err(Error(
+			"metadata_fsync is not supported with the Fjall database engine".into(),
+		));
+	}
+	let mut config = fjall::Config::new(path);
+	if let Some(block_cache_size) = opt.fjall_block_cache_size {
+		config = config.cache_size(block_cache_size as u64);
+	}
+	let keyspace = config.open_transactional()?;
+	Ok(FjallDb::init(keyspace))
+}
+
+// -- err
+
+impl From<fjall::Error> for Error {
+	fn from(e: fjall::Error) -> Error {
+		Error(format!("fjall: {}", e).into())
+	}
+}
+
+impl From<fjall::LsmError> for Error {
+	fn from(e: fjall::LsmError) -> Error {
+		Error(format!("fjall lsm_tree: {}", e).into())
+	}
+}
+
+impl From<fjall::Error> for TxOpError {
+	fn from(e: fjall::Error) -> TxOpError {
+		TxOpError(e.into())
+	}
+}
+
+// -- db
+
+pub struct FjallDb {
+	keyspace: TransactionalKeyspace,
+	trees: RwLock<Vec<(String, TransactionalPartitionHandle)>>,
+}
+
+type ByteRefRangeBound<'r> = (Bound<&'r [u8]>, Bound<&'r [u8]>);
+
+impl FjallDb {
+	pub fn init(keyspace: TransactionalKeyspace) -> Db {
+		let s = Self {
+			keyspace,
+			trees: RwLock::new(Vec::new()),
+		};
+		Db(Arc::new(s))
+	}
+
+	fn get_tree(
+		&self,
+		i: usize,
+	) -> Result<MappedRwLockReadGuard<'_, TransactionalPartitionHandle>> {
+		RwLockReadGuard::try_map(self.trees.read(), |trees: &Vec<_>| {
+			trees.get(i).map(|tup| &tup.1)
+		})
+		.map_err(|_| Error("invalid tree id".into()))
+	}
+}
+
+impl IDb for FjallDb {
+	fn engine(&self) -> String {
+		"Fjall (EXPERIMENTAL!)".into()
+	}
+
+	fn open_tree(&self, name: &str) -> Result<usize> {
+		let mut trees = self.trees.write();
+		let safe_name = encode_name(name)?;
+		if let Some(i) = trees.iter().position(|(name, _)| *name == safe_name) {
+			Ok(i)
+		} else {
+			let tree = self
+				.keyspace
+				.open_partition(&safe_name, PartitionCreateOptions::default())?;
+			let i = trees.len();
+			trees.push((safe_name, tree));
+			Ok(i)
+		}
+	}
+
+	fn list_trees(&self) -> Result<Vec<String>> {
+		Ok(self
+			.keyspace
+			.list_partitions()
+			.iter()
+			.map(|n| decode_name(&n))
+			.collect::<Result<Vec<_>>>()?)
+	}
+
+	fn snapshot(&self, base_path: &PathBuf) -> Result<()> {
+		std::fs::create_dir_all(base_path)?;
+		let path = Engine::Fjall.db_path(base_path);
+
+		let source_state = self.keyspace.read_tx();
+		let copy_keyspace = fjall::Config::new(path).open()?;
+
+		for partition_name in self.keyspace.list_partitions() {
+			let source_partition = self
+				.keyspace
+				.open_partition(&partition_name, PartitionCreateOptions::default())?;
+			let copy_partition =
+				copy_keyspace.open_partition(&partition_name, PartitionCreateOptions::default())?;
+
+			for entry in source_state.iter(&source_partition) {
+				let (key, value) = entry?;
+				copy_partition.insert(key, value)?;
+			}
+		}
+
+		copy_keyspace.persist(PersistMode::SyncAll)?;
+		Ok(())
+	}
+
+	// ----
+
+	fn get(&self, tree_idx: usize, key: &[u8]) -> Result<Option<Value>> {
+		let tree = self.get_tree(tree_idx)?;
+		let tx = self.keyspace.read_tx();
+		let val = tx.get(&tree, key)?;
+		match val {
+			None => Ok(None),
+			Some(v) => Ok(Some(v.to_vec())),
+		}
+	}
+
+	fn approximate_len(&self, tree_idx: usize) -> Result<usize> {
+		let tree = self.get_tree(tree_idx)?;
+		Ok(tree.approximate_len())
+	}
+	fn is_empty(&self, tree_idx: usize) -> Result<bool> {
+		let tree = self.get_tree(tree_idx)?;
+		let tx = self.keyspace.read_tx();
+		Ok(tx.is_empty(&tree)?)
+	}
+
+	fn insert(&self, tree_idx: usize, key: &[u8], value: &[u8]) -> Result<()> {
+		let tree = self.get_tree(tree_idx)?;
+		let mut tx = self.keyspace.write_tx();
+		tx.insert(&tree, key, value);
+		tx.commit()?;
+		Ok(())
+	}
+
+	fn remove(&self, tree_idx: usize, key: &[u8]) -> Result<()> {
+		let tree = self.get_tree(tree_idx)?;
+		let mut tx = self.keyspace.write_tx();
+		tx.remove(&tree, key);
+		tx.commit()?;
+		Ok(())
+	}
+
+	fn clear(&self, tree_idx: usize) -> Result<()> {
+		let mut trees = self.trees.write();
+
+		if tree_idx >= trees.len() {
+			return Err(Error("invalid tree id".into()));
+		}
+		let (name, tree) = trees.remove(tree_idx);
+
+		self.keyspace.delete_partition(tree)?;
+		let tree = self
+			.keyspace
+			.open_partition(&name, PartitionCreateOptions::default())?;
+		trees.insert(tree_idx, (name, tree));
+
+		Ok(())
+	}
+
+	fn iter(&self, tree_idx: usize) -> Result<ValueIter<'_>> {
+		let tree = self.get_tree(tree_idx)?;
+		let tx = self.keyspace.read_tx();
+		Ok(Box::new(tx.iter(&tree).map(iterator_remap)))
+	}
+
+	fn iter_rev(&self, tree_idx: usize) -> Result<ValueIter<'_>> {
+		let tree = self.get_tree(tree_idx)?;
+		let tx = self.keyspace.read_tx();
+		Ok(Box::new(tx.iter(&tree).rev().map(iterator_remap)))
+	}
+
+	fn range<'r>(
+		&self,
+		tree_idx: usize,
+		low: Bound<&'r [u8]>,
+		high: Bound<&'r [u8]>,
+	) -> Result<ValueIter<'_>> {
+		let tree = self.get_tree(tree_idx)?;
+		let tx = self.keyspace.read_tx();
+		Ok(Box::new(
+			tx.range::<&'r [u8], ByteRefRangeBound>(&tree, (low, high))
+				.map(iterator_remap),
+		))
+	}
+	fn range_rev<'r>(
+		&self,
+		tree_idx: usize,
+		low: Bound<&'r [u8]>,
+		high: Bound<&'r [u8]>,
+	) -> Result<ValueIter<'_>> {
+		let tree = self.get_tree(tree_idx)?;
+		let tx = self.keyspace.read_tx();
+		Ok(Box::new(
+			tx.range::<&'r [u8], ByteRefRangeBound>(&tree, (low, high))
+				.rev()
+				.map(iterator_remap),
+		))
+	}
+
+	// ----
+
+	fn transaction(&self, f: &dyn ITxFn) -> TxResult<OnCommit, ()> {
+		let trees = self.trees.read();
+		let mut tx = FjallTx {
+			trees: &trees[..],
+			tx: self.keyspace.write_tx(),
+		};
+
+		let res = f.try_on(&mut tx);
+		match res {
+			TxFnResult::Ok(on_commit) => {
+				tx.tx.commit().map_err(Error::from).map_err(TxError::Db)?;
+				Ok(on_commit)
+			}
+			TxFnResult::Abort => {
+				tx.tx.rollback();
+				Err(TxError::Abort(()))
+			}
+			TxFnResult::DbErr => {
+				tx.tx.rollback();
+				Err(TxError::Db(Error(
+					"(this message will be discarded)".into(),
+				)))
+			}
+		}
+	}
+}
+
+// ----
+
+struct FjallTx<'a> {
+	trees: &'a [(String, TransactionalPartitionHandle)],
+	tx: WriteTransaction<'a>,
+}
+
+impl<'a> FjallTx<'a> {
+	fn get_tree(&self, i: usize) -> TxOpResult<&TransactionalPartitionHandle> {
+		self.trees.get(i).map(|tup| &tup.1).ok_or_else(|| {
+			TxOpError(Error(
+				"invalid tree id (it might have been openned after the transaction started)".into(),
+			))
+		})
+	}
+}
+
+impl<'a> ITx for FjallTx<'a> {
+	fn get(&self, tree_idx: usize, key: &[u8]) -> TxOpResult<Option<Value>> {
+		let tree = self.get_tree(tree_idx)?;
+		match self.tx.get(tree, key)? {
+			Some(v) => Ok(Some(v.to_vec())),
+			None => Ok(None),
+		}
+	}
+	fn len(&self, tree_idx: usize) -> TxOpResult<usize> {
+		let tree = self.get_tree(tree_idx)?;
+		Ok(self.tx.len(tree)? as usize)
+	}
+
+	fn insert(&mut self, tree_idx: usize, key: &[u8], value: &[u8]) -> TxOpResult<()> {
+		let tree = self.get_tree(tree_idx)?.clone();
+		self.tx.insert(&tree, key, value);
+		Ok(())
+	}
+	fn remove(&mut self, tree_idx: usize, key: &[u8]) -> TxOpResult<()> {
+		let tree = self.get_tree(tree_idx)?.clone();
+		self.tx.remove(&tree, key);
+		Ok(())
+	}
+	fn clear(&mut self, _tree_idx: usize) -> TxOpResult<()> {
+		unimplemented!("LSM tree clearing in cross-partition transaction is not supported")
+	}
+
+	fn iter(&self, tree_idx: usize) -> TxOpResult<TxValueIter<'_>> {
+		let tree = self.get_tree(tree_idx)?.clone();
+		Ok(Box::new(self.tx.iter(&tree).map(iterator_remap_tx)))
+	}
+	fn iter_rev(&self, tree_idx: usize) -> TxOpResult<TxValueIter<'_>> {
+		let tree = self.get_tree(tree_idx)?.clone();
+		Ok(Box::new(self.tx.iter(&tree).rev().map(iterator_remap_tx)))
+	}
+
+	fn range<'r>(
+		&self,
+		tree_idx: usize,
+		low: Bound<&'r [u8]>,
+		high: Bound<&'r [u8]>,
+	) -> TxOpResult<TxValueIter<'_>> {
+		let tree = self.get_tree(tree_idx)?;
+		let low = clone_bound(low);
+		let high = clone_bound(high);
+		Ok(Box::new(
+			self.tx
+				.range::<Vec<u8>, ByteVecRangeBounds>(&tree, (low, high))
+				.map(iterator_remap_tx),
+		))
+	}
+	fn range_rev<'r>(
+		&self,
+		tree_idx: usize,
+		low: Bound<&'r [u8]>,
+		high: Bound<&'r [u8]>,
+	) -> TxOpResult<TxValueIter<'_>> {
+		let tree = self.get_tree(tree_idx)?;
+		let low = clone_bound(low);
+		let high = clone_bound(high);
+		Ok(Box::new(
+			self.tx
+				.range::<Vec<u8>, ByteVecRangeBounds>(&tree, (low, high))
+				.rev()
+				.map(iterator_remap_tx),
+		))
+	}
+}
+
+// -- maps fjall's (k, v) to ours
+
+fn iterator_remap(r: fjall::Result<(fjall::Slice, fjall::Slice)>) -> Result<(Value, Value)> {
+	r.map(|(k, v)| (k.to_vec(), v.to_vec()))
+		.map_err(|e| e.into())
+}
+
+fn iterator_remap_tx(r: fjall::Result<(fjall::Slice, fjall::Slice)>) -> TxOpResult<(Value, Value)> {
+	r.map(|(k, v)| (k.to_vec(), v.to_vec()))
+		.map_err(|e| e.into())
+}
+
+// -- utils to deal with Garage's tightness on Bound lifetimes
+
+type ByteVecBound = Bound<Vec<u8>>;
+type ByteVecRangeBounds = (ByteVecBound, ByteVecBound);
+
+fn clone_bound(bound: Bound<&[u8]>) -> ByteVecBound {
+	let value = match bound {
+		Bound::Excluded(v) | Bound::Included(v) => v.to_vec(),
+		Bound::Unbounded => vec![],
+	};
+
+	match bound {
+		Bound::Included(_) => Bound::Included(value),
+		Bound::Excluded(_) => Bound::Excluded(value),
+		Bound::Unbounded => Bound::Unbounded,
+	}
+}
+
+// -- utils to encode table names --
+
+fn encode_name(s: &str) -> Result<String> {
+	let base = 'A' as u32;
+
+	let mut ret = String::with_capacity(s.len() + 10);
+	for c in s.chars() {
+		if c.is_alphanumeric() || c == '_' || c == '-' || c == '#' {
+			ret.push(c);
+		} else if c <= u8::MAX as char {
+			ret.push('$');
+			let c_hi = c as u32 / 16;
+			let c_lo = c as u32 % 16;
+			ret.push(char::from_u32(base + c_hi).unwrap());
+			ret.push(char::from_u32(base + c_lo).unwrap());
+		} else {
+			return Err(Error(
+				format!("table name {} could not be safely encoded", s).into(),
+			));
+		}
+	}
+	Ok(ret)
+}
+
+fn decode_name(s: &str) -> Result<String> {
+	use std::convert::TryFrom;
+
+	let errfn = || Error(format!("encoded table name {} is invalid", s).into());
+	let c_map = |c: char| {
+		let c = c as u32;
+		let base = 'A' as u32;
+		if (base..base + 16).contains(&c) {
+			Some(c - base)
+		} else {
+			None
+		}
+	};
+
+	let mut ret = String::with_capacity(s.len());
+	let mut it = s.chars();
+	while let Some(c) = it.next() {
+		if c == '$' {
+			let c_hi = it.next().and_then(c_map).ok_or_else(errfn)?;
+			let c_lo = it.next().and_then(c_map).ok_or_else(errfn)?;
+			let c_dec = char::try_from(c_hi * 16 + c_lo).map_err(|_| errfn())?;
+			ret.push(c_dec);
+		} else {
+			ret.push(c);
+		}
+	}
+	Ok(ret)
+}
+
+#[cfg(test)]
+mod tests {
+	use super::*;
+
+	#[test]
+	fn test_encdec_name() {
+		for name in [
+			"testname",
+			"test_name",
+			"test name",
+			"test$name",
+			"test:name@help.me$get/this**right",
+		] {
+			let encname = encode_name(name).unwrap();
+			assert!(!encname.contains(' '));
+			assert!(!encname.contains('.'));
+			assert!(!encname.contains('*'));
+			assert_eq!(*name, decode_name(&encname).unwrap());
+		}
+	}
+}
@@ -1,6 +1,8 @@
 #[macro_use]
 extern crate tracing;

+#[cfg(feature = "fjall")]
+pub mod fjall_adapter;
 #[cfg(feature = "lmdb")]
 pub mod lmdb_adapter;
 #[cfg(feature = "sqlite")]
@@ -18,7 +20,7 @@ use std::cell::Cell;
 use std::path::PathBuf;
 use std::sync::Arc;

-use err_derive::Error;
+use thiserror::Error;

 pub use open::*;

@@ -42,7 +44,7 @@ pub type TxValueIter<'a> = Box<dyn std::iter::Iterator<Item = TxOpResult<(Value,
 // ----

 #[derive(Debug, Error)]
-#[error(display = "{}", _0)]
+#[error("{0}")]
 pub struct Error(pub Cow<'static, str>);

 impl From<std::io::Error> for Error {
@@ -54,7 +56,7 @@ impl From<std::io::Error> for Error {
 pub type Result<T> = std::result::Result<T, Error>;

 #[derive(Debug, Error)]
-#[error(display = "{}", _0)]
+#[error("{0}")]
 pub struct TxOpError(pub(crate) Error);
 pub type TxOpResult<T> = std::result::Result<T, TxOpError>;

@@ -104,32 +106,44 @@ impl Db {
 			result: Cell::new(None),
 		};
 		let tx_res = self.0.transaction(&f);
-		let ret = f
-			.result
-			.into_inner()
-			.expect("Transaction did not store result");
+		let fn_res = f.result.into_inner();

-		match tx_res {
-			Ok(on_commit) => match ret {
-				Ok(value) => {
-					on_commit.into_iter().for_each(|f| f());
-					Ok(value)
-				}
-				_ => unreachable!(),
-			},
-			Err(TxError::Abort(())) => match ret {
-				Err(TxError::Abort(e)) => Err(TxError::Abort(e)),
-				_ => unreachable!(),
-			},
-			Err(TxError::Db(e2)) => match ret {
-				// Ok was stored -> the error occurred when finalizing
-				// transaction
-				Ok(_) => Err(TxError::Db(e2)),
-				// An error was already stored: that's the one we want to
-				// return
-				Err(TxError::Db(e)) => Err(TxError::Db(e)),
-				_ => unreachable!(),
-			},
+		match (tx_res, fn_res) {
+			(Ok(on_commit), Some(Ok(value))) => {
+				// Transaction succeeded
+				// TxFn stored the value to return to the user in fn_res
+				// tx_res contains the on_commit list of callbacks, run them now
+				on_commit.into_iter().for_each(|f| f());
+				Ok(value)
+			}
+			(Err(TxError::Abort(())), Some(Err(TxError::Abort(e)))) => {
+				// Transaction was aborted by user code
+				// The abort error value is stored in fn_res
+				Err(TxError::Abort(e))
+			}
+			(Err(TxError::Db(_tx_e)), Some(Err(TxError::Db(fn_e)))) => {
+				// Transaction encountered a DB error in user code
+				// The error value encountered is the one in fn_res,
+				// tx_res contains only a dummy error message
+				Err(TxError::Db(fn_e))
+			}
+			(Err(TxError::Db(tx_e)), None) => {
+				// Transaction encounterred a DB error when initializing the transaction,
+				// before user code was called
+				Err(TxError::Db(tx_e))
+			}
+			(Err(TxError::Db(tx_e)), Some(Ok(_))) => {
+				// Transaction encounterred a DB error when commiting the transaction,
+				// after user code was called
+				Err(TxError::Db(tx_e))
+			}
+			(tx_res, fn_res) => {
+				panic!(
+					"unexpected error case: tx_res={:?}, fn_res={:?}",
+					tx_res.map(|_| "..."),
+					fn_res.map(|x| x.map(|_| "...").map_err(|_| "..."))
+				);
+			}
 		}
 	}

@@ -152,7 +166,7 @@ impl Db {
 		let tree_names = other.list_trees()?;
 		for name in tree_names {
 			let tree = self.open_tree(&name)?;
-			if tree.len()? > 0 {
+			if !tree.is_empty()? {
 				return Err(Error(format!("tree {} already contains data", name).into()));
 			}

@@ -194,8 +208,12 @@ impl Tree {
 		self.0.get(self.1, key.as_ref())
 	}
 	#[inline]
-	pub fn len(&self) -> Result<usize> {
-		self.0.len(self.1)
+	pub fn approximate_len(&self) -> Result<usize> {
+		self.0.approximate_len(self.1)
+	}
+	#[inline]
+	pub fn is_empty(&self) -> Result<bool> {
+		self.0.is_empty(self.1)
 	}

 	#[inline]
@@ -333,7 +351,8 @@ pub(crate) trait IDb: Send + Sync {
 	fn snapshot(&self, path: &PathBuf) -> Result<()>;

 	fn get(&self, tree: usize, key: &[u8]) -> Result<Option<Value>>;
-	fn len(&self, tree: usize) -> Result<usize>;
+	fn approximate_len(&self, tree: usize) -> Result<usize>;
+	fn is_empty(&self, tree: usize) -> Result<bool>;

 	fn insert(&self, tree: usize, key: &[u8], value: &[u8]) -> Result<()>;
 	fn remove(&self, tree: usize, key: &[u8]) -> Result<()>;
@@ -1,8 +1,8 @@
 use core::ops::Bound;
-use core::ptr::NonNull;

 use std::collections::HashMap;
 use std::convert::TryInto;
+use std::marker::PhantomPinned;
 use std::path::PathBuf;
 use std::pin::Pin;
 use std::sync::{Arc, RwLock};
@@ -11,12 +11,55 @@ use heed::types::ByteSlice;
 use heed::{BytesDecode, Env, RoTxn, RwTxn, UntypedDatabase as Database};

 use crate::{
+	open::{Engine, OpenOpt},
 	Db, Error, IDb, ITx, ITxFn, OnCommit, Result, TxError, TxFnResult, TxOpError, TxOpResult,
 	TxResult, TxValueIter, Value, ValueIter,
 };

 pub use heed;

+// ---- top-level open function
+
+pub(crate) fn open_db(path: &PathBuf, opt: &OpenOpt) -> Result<Db> {
+	info!("Opening LMDB database at: {}", path.display());
+	if let Err(e) = std::fs::create_dir_all(&path) {
+		return Err(Error(
+			format!("Unable to create LMDB data directory: {}", e).into(),
+		));
+	}
+
+	let map_size = match opt.lmdb_map_size {
+		None => recommended_map_size(),
+		Some(v) => v - (v % 4096),
+	};
+
+	let mut env_builder = heed::EnvOpenOptions::new();
+	env_builder.max_dbs(100);
+	env_builder.map_size(map_size);
+	env_builder.max_readers(2048);
+	unsafe {
+		env_builder.flag(heed::flags::Flags::MdbNoRdAhead);
+		env_builder.flag(heed::flags::Flags::MdbNoMetaSync);
+		if !opt.fsync {
+			env_builder.flag(heed::flags::Flags::MdbNoSync);
+		}
+	}
+	match env_builder.open(&path) {
+		Err(heed::Error::Io(e)) if e.kind() == std::io::ErrorKind::OutOfMemory => {
+			return Err(Error(
+				"OutOfMemory error while trying to open LMDB database. This can happen \
+                if your operating system is not allowing you to use sufficient virtual \
+                memory address space. Please check that no limit is set (ulimit -v). \
+                You may also try to set a smaller `lmdb_map_size` configuration parameter. \
+                On 32-bit machines, you should probably switch to another database engine."
+					.into(),
+			))
+		}
+		Err(e) => Err(Error(format!("Cannot open LMDB database: {}", e).into())),
+		Ok(db) => Ok(LmdbDb::init(db)),
+	}
+}
+
 // -- err

 impl From<heed::Error> for Error {
@@ -104,10 +147,9 @@ impl IDb for LmdbDb {
 		Ok(ret2)
 	}

-	fn snapshot(&self, to: &PathBuf) -> Result<()> {
-		std::fs::create_dir_all(to)?;
-		let mut path = to.clone();
-		path.push("data.mdb");
+	fn snapshot(&self, base_path: &PathBuf) -> Result<()> {
+		std::fs::create_dir_all(base_path)?;
+		let path = Engine::Lmdb.db_path(base_path);
 		self.db
 			.copy_to_path(path, heed::CompactionOption::Enabled)?;
 		Ok(())
@@ -126,11 +168,16 @@ impl IDb for LmdbDb {
 		}
 	}

-	fn len(&self, tree: usize) -> Result<usize> {
+	fn approximate_len(&self, tree: usize) -> Result<usize> {
 		let tree = self.get_tree(tree)?;
 		let tx = self.db.read_txn()?;
 		Ok(tree.len(&tx)?.try_into().unwrap())
 	}
+	fn is_empty(&self, tree: usize) -> Result<bool> {
+		let tree = self.get_tree(tree)?;
+		let tx = self.db.read_txn()?;
+		Ok(tree.is_empty(&tx)?)
+	}

 	fn insert(&self, tree: usize, key: &[u8], value: &[u8]) -> Result<()> {
 		let tree = self.get_tree(tree)?;
@@ -159,13 +206,15 @@ impl IDb for LmdbDb {
 	fn iter(&self, tree: usize) -> Result<ValueIter<'_>> {
 		let tree = self.get_tree(tree)?;
 		let tx = self.db.read_txn()?;
-		TxAndIterator::make(tx, |tx| Ok(tree.iter(tx)?))
+		// Safety: the cloture does not store its argument anywhere,
+		unsafe { TxAndIterator::make(tx, |tx| Ok(tree.iter(tx)?)) }
 	}

 	fn iter_rev(&self, tree: usize) -> Result<ValueIter<'_>> {
 		let tree = self.get_tree(tree)?;
 		let tx = self.db.read_txn()?;
-		TxAndIterator::make(tx, |tx| Ok(tree.rev_iter(tx)?))
+		// Safety: the cloture does not store its argument anywhere,
+		unsafe { TxAndIterator::make(tx, |tx| Ok(tree.rev_iter(tx)?)) }
 	}

 	fn range<'r>(
@@ -176,7 +225,8 @@ impl IDb for LmdbDb {
 	) -> Result<ValueIter<'_>> {
 		let tree = self.get_tree(tree)?;
 		let tx = self.db.read_txn()?;
-		TxAndIterator::make(tx, |tx| Ok(tree.range(tx, &(low, high))?))
+		// Safety: the cloture does not store its argument anywhere,
+		unsafe { TxAndIterator::make(tx, |tx| Ok(tree.range(tx, &(low, high))?)) }
 	}
 	fn range_rev<'r>(
 		&self,
@@ -186,7 +236,8 @@ impl IDb for LmdbDb {
 	) -> Result<ValueIter<'_>> {
 		let tree = self.get_tree(tree)?;
 		let tx = self.db.read_txn()?;
-		TxAndIterator::make(tx, |tx| Ok(tree.rev_range(tx, &(low, high))?))
+		// Safety: the cloture does not store its argument anywhere,
+		unsafe { TxAndIterator::make(tx, |tx| Ok(tree.rev_range(tx, &(low, high))?)) }
 	}

 	// ----
@@ -316,28 +367,41 @@ where
 {
 	tx: RoTxn<'a>,
 	iter: Option<I>,
+	_pin: PhantomPinned,
 }

 impl<'a, I> TxAndIterator<'a, I>
 where
 	I: Iterator<Item = IteratorItem<'a>> + 'a,
 {
-	fn make<F>(tx: RoTxn<'a>, iterfun: F) -> Result<ValueIter<'a>>
+	fn iter(self: Pin<&mut Self>) -> &mut Option<I> {
+		// Safety: iter is not structural
+		unsafe { &mut self.get_unchecked_mut().iter }
+	}
+
+	/// Safety: iterfun must not store its argument anywhere but in its result.
+	unsafe fn make<F>(tx: RoTxn<'a>, iterfun: F) -> Result<ValueIter<'a>>
 	where
 		F: FnOnce(&'a RoTxn<'a>) -> Result<I>,
 	{
-		let res = TxAndIterator { tx, iter: None };
+		let res = TxAndIterator {
+			tx,
+			iter: None,
+			_pin: PhantomPinned,
+		};
 		let mut boxed = Box::pin(res);

-		// This unsafe allows us to bypass lifetime checks
-		let tx = unsafe { NonNull::from(&boxed.tx).as_ref() };
-		let iter = iterfun(tx)?;
+		let tx_lifetime_overextended: &'a RoTxn<'a> = {
+			let tx = &boxed.tx;
+			// Safety: Artificially extending the lifetime because
+			// this reference will only be stored and accessed from the
+			// returned ValueIter which guarantees that it is destroyed
+			// before the tx it is pointing  to.
+			unsafe { &*&raw const *tx }
+		};
+		let iter = iterfun(&tx_lifetime_overextended)?;

-		let mut_ref = Pin::as_mut(&mut boxed);
-		// This unsafe allows us to write in a field of the pinned struct
-		unsafe {
-			Pin::get_unchecked_mut(mut_ref).iter = Some(iter);
-		}
+		*boxed.as_mut().iter() = Some(iter);

 		Ok(Box::new(TxAndIteratorPin(boxed)))
 	}
@@ -348,8 +412,10 @@ where
 	I: Iterator<Item = IteratorItem<'a>> + 'a,
 {
 	fn drop(&mut self) {
-		// ensure the iterator is dropped before the RoTxn it references
-		drop(self.iter.take());
+		// Safety: `new_unchecked` is okay because we know this value is never
+		// used again after being dropped.
+		let this = unsafe { Pin::new_unchecked(self) };
+		drop(this.iter().take());
 	}
 }

@@ -365,13 +431,12 @@ where

 	fn next(&mut self) -> Option<Self::Item> {
 		let mut_ref = Pin::as_mut(&mut self.0);
-		// This unsafe allows us to mutably access the iterator field
-		let next = unsafe { Pin::get_unchecked_mut(mut_ref).iter.as_mut()?.next() };
-		match next {
-			None => None,
-			Some(Err(e)) => Some(Err(e.into())),
-			Some(Ok((k, v))) => Some(Ok((k.to_vec(), v.to_vec()))),
-		}
+		let next = mut_ref.iter().as_mut()?.next()?;
+		let res = match next {
+			Err(e) => Err(e.into()),
+			Ok((k, v)) => Ok((k.to_vec(), v.to_vec())),
+		};
+		Some(res)
 	}
 }

@@ -11,6 +11,7 @@ use crate::{Db, Error, Result};
 pub enum Engine {
 	Lmdb,
 	Sqlite,
+	Fjall,
 }

 impl Engine {
@@ -19,8 +20,26 @@ impl Engine {
 		match self {
 			Self::Lmdb => "lmdb",
 			Self::Sqlite => "sqlite",
+			Self::Fjall => "fjall",
 		}
 	}
+
+	/// Return engine-specific DB path from base path
+	pub fn db_path(&self, base_path: &PathBuf) -> PathBuf {
+		let mut ret = base_path.clone();
+		match self {
+			Self::Lmdb => {
+				ret.push("db.lmdb");
+			}
+			Self::Sqlite => {
+				ret.push("db.sqlite");
+			}
+			Self::Fjall => {
+				ret.push("db.fjall");
+			}
+		}
+		ret
+	}
 }

 impl std::fmt::Display for Engine {
@@ -36,10 +55,11 @@ impl std::str::FromStr for Engine {
 		match text {
 			"lmdb" | "heed" => Ok(Self::Lmdb),
 			"sqlite" | "sqlite3" | "rusqlite" => Ok(Self::Sqlite),
+            "fjall" => Ok(Self::Fjall),
 			"sled" => Err(Error("Sled is no longer supported as a database engine. Converting your old metadata db can be done using an older Garage binary (e.g. v0.9.4).".into())),
 			kind => Err(Error(
 				format!(
-					"Invalid DB engine: {} (options are: lmdb, sqlite)",
+					"Invalid DB engine: {} (options are: lmdb, sqlite, fjall)",
 					kind
 				)
 				.into(),
@@ -51,6 +71,7 @@ impl std::str::FromStr for Engine {
 pub struct OpenOpt {
 	pub fsync: bool,
 	pub lmdb_map_size: Option<usize>,
+	pub fjall_block_cache_size: Option<usize>,
 }

 impl Default for OpenOpt {
@@ -58,6 +79,7 @@ impl Default for OpenOpt {
 		Self {
 			fsync: false,
 			lmdb_map_size: None,
+			fjall_block_cache_size: None,
 		}
 	}
 }
@@ -66,53 +88,15 @@ pub fn open_db(path: &PathBuf, engine: Engine, opt: &OpenOpt) -> Result<Db> {
 	match engine {
 		// ---- Sqlite DB ----
 		#[cfg(feature = "sqlite")]
-		Engine::Sqlite => {
-			info!("Opening Sqlite database at: {}", path.display());
-			let manager = r2d2_sqlite::SqliteConnectionManager::file(path);
-			Ok(crate::sqlite_adapter::SqliteDb::new(manager, opt.fsync)?)
-		}
+		Engine::Sqlite => crate::sqlite_adapter::open_db(path, opt),

 		// ---- LMDB DB ----
 		#[cfg(feature = "lmdb")]
-		Engine::Lmdb => {
-			info!("Opening LMDB database at: {}", path.display());
-			if let Err(e) = std::fs::create_dir_all(&path) {
-				return Err(Error(
-					format!("Unable to create LMDB data directory: {}", e).into(),
-				));
-			}
+		Engine::Lmdb => crate::lmdb_adapter::open_db(path, opt),

-			let map_size = match opt.lmdb_map_size {
-				None => crate::lmdb_adapter::recommended_map_size(),
-				Some(v) => v - (v % 4096),
-			};
-
-			let mut env_builder = heed::EnvOpenOptions::new();
-			env_builder.max_dbs(100);
-			env_builder.map_size(map_size);
-			env_builder.max_readers(2048);
-			unsafe {
-				env_builder.flag(crate::lmdb_adapter::heed::flags::Flags::MdbNoRdAhead);
-				env_builder.flag(crate::lmdb_adapter::heed::flags::Flags::MdbNoMetaSync);
-				if !opt.fsync {
-					env_builder.flag(heed::flags::Flags::MdbNoSync);
-				}
-			}
-			match env_builder.open(&path) {
-				Err(heed::Error::Io(e)) if e.kind() == std::io::ErrorKind::OutOfMemory => {
-					return Err(Error(
-						"OutOfMemory error while trying to open LMDB database. This can happen \
-                        if your operating system is not allowing you to use sufficient virtual \
-                        memory address space. Please check that no limit is set (ulimit -v). \
-                        You may also try to set a smaller `lmdb_map_size` configuration parameter. \
-                        On 32-bit machines, you should probably switch to another database engine."
-							.into(),
-					))
-				}
-				Err(e) => Err(Error(format!("Cannot open LMDB database: {}", e).into())),
-				Ok(db) => Ok(crate::lmdb_adapter::LmdbDb::init(db)),
-			}
-		}
+		// ---- Fjall DB ----
+		#[cfg(feature = "fjall")]
+		Engine::Fjall => crate::fjall_adapter::open_db(path, opt),

 		// Pattern is unreachable when all supported DB engines are compiled into binary. The allow
 		// attribute is added so that we won't have to change this match in case stop building
@@ -11,12 +11,23 @@ use r2d2_sqlite::SqliteConnectionManager;
 use rusqlite::{params, Rows, Statement, Transaction};

 use crate::{
+	open::{Engine, OpenOpt},
 	Db, Error, IDb, ITx, ITxFn, OnCommit, Result, TxError, TxFnResult, TxOpError, TxOpResult,
 	TxResult, TxValueIter, Value, ValueIter,
 };

 pub use rusqlite;

+// ---- top-level open function
+
+pub(crate) fn open_db(path: &PathBuf, opt: &OpenOpt) -> Result<Db> {
+	info!("Opening Sqlite database at: {}", path.display());
+	let manager = r2d2_sqlite::SqliteConnectionManager::file(path);
+	Ok(SqliteDb::new(manager, opt.fsync)?)
+}
+
+// ----
+
 type Connection = r2d2::PooledConnection<SqliteConnectionManager>;

 // --- err
@@ -139,17 +150,18 @@ impl IDb for SqliteDb {
 		Ok(trees)
 	}

-	fn snapshot(&self, to: &PathBuf) -> Result<()> {
-		fn progress(p: rusqlite::backup::Progress) {
-			let percent = (p.pagecount - p.remaining) * 100 / p.pagecount;
-			info!("Sqlite snapshot progress: {}%", percent);
-		}
-		std::fs::create_dir_all(to)?;
-		let mut path = to.clone();
-		path.push("db.sqlite");
-		self.db
-			.get()?
-			.backup(rusqlite::DatabaseName::Main, path, Some(progress))?;
+	fn snapshot(&self, base_path: &PathBuf) -> Result<()> {
+		std::fs::create_dir_all(base_path)?;
+		let path = Engine::Sqlite
+			.db_path(&base_path)
+			.into_os_string()
+			.into_string()
+			.map_err(|_| Error("invalid sqlite path string".into()))?;
+
+		info!("Start sqlite VACUUM INTO `{}`", path);
+		self.db.get()?.execute("VACUUM INTO ?1", params![path])?;
+		info!("Finished sqlite VACUUM INTO `{}`", path);
+
 		Ok(())
 	}

@@ -160,7 +172,7 @@ impl IDb for SqliteDb {
 		self.internal_get(&self.db.get()?, &tree, key)
 	}

-	fn len(&self, tree: usize) -> Result<usize> {
+	fn approximate_len(&self, tree: usize) -> Result<usize> {
 		let tree = self.get_tree(tree)?;
 		let db = self.db.get()?;

@@ -172,6 +184,10 @@ impl IDb for SqliteDb {
 		}
 	}

+	fn is_empty(&self, tree: usize) -> Result<bool> {
+		Ok(self.approximate_len(tree)? == 0)
+	}
+
 	fn insert(&self, tree: usize, key: &[u8], value: &[u8]) -> Result<()> {
 		let tree = self.get_tree(tree)?;
 		let db = self.db.get()?;
@@ -1,7 +1,7 @@
 use crate::*;

 fn test_suite(db: Db) {
-	let tree = db.open_tree("tree").unwrap();
+	let tree = db.open_tree("tree:this_is_a_tree").unwrap();

 	let ka: &[u8] = &b"test"[..];
 	let kb: &[u8] = &b"zwello"[..];
@@ -14,7 +14,7 @@ fn test_suite(db: Db) {

 	assert!(tree.insert(ka, va).is_ok());
 	assert_eq!(tree.get(ka).unwrap().unwrap(), va);
-	assert_eq!(tree.len().unwrap(), 1);
+	assert_eq!(tree.iter().unwrap().count(), 1);

 	// ---- test transaction logic ----

@@ -148,3 +148,15 @@ fn test_sqlite_db() {
 	let db = SqliteDb::new(manager, false).unwrap();
 	test_suite(db);
 }
+
+#[test]
+#[cfg(feature = "fjall")]
+fn test_fjall_db() {
+	use crate::fjall_adapter::{fjall, FjallDb};
+
+	let path = mktemp::Temp::new_dir().unwrap();
+	let config = fjall::Config::new(path).temporary(true);
+	let keyspace = config.open_transactional().unwrap();
+	let db = FjallDb::init(keyspace);
+	test_suite(db);
+}
@@ -1,6 +1,6 @@
 [package]
 name = "garage"
-version = "2.0.0"
+version = "2.1.0"
 authors = ["Alex Auvolat <alex@adnab.me>"]
 edition = "2018"
 license = "AGPL-3.0"
@@ -93,6 +93,7 @@ k2v = [ "garage_util/k2v", "garage_api_k2v", "garage_api_admin/k2v" ]
 # Database engines
 lmdb = [ "garage_model/lmdb" ]
 sqlite = [ "garage_model/sqlite" ]
+fjall = [ "garage_model/fjall" ]

 # Automatic registration and discovery via Consul API
 consul-discovery = [ "garage_rpc/consul-discovery" ]
@@ -100,7 +100,7 @@ impl Cli {
 						drain_msg = true;
 						"draining metadata.."
 					}
-					_ => unreachable!(),
+					_ => continue,
 				};

 				failed_nodes.push(format!(
@@ -92,6 +92,7 @@ impl Cli {
 				ScrubCmd::Resume => ScrubCommand::Resume,
 			}),
 			RepairWhat::Aliases => RepairType::Aliases,
+			RepairWhat::ClearResyncQueue => RepairType::ClearResyncQueue,
 		};

 		let res = self
@@ -645,6 +645,10 @@ pub enum RepairWhat {
 	/// Repair (resync/rebalance) the set of stored blocks in the cluster
 	#[structopt(name = "blocks", version = garage_version())]
 	Blocks,
+	/// Clear the block resync queue. The list of blocks in errored state
+	/// is cleared as well. You MUST run `garage repair blocks` after invoking this.
+	#[structopt(name = "clear-resync-queue", version = garage_version())]
+	ClearResyncQueue,
 	/// Repropagate object deletions to the version table
 	#[structopt(name = "versions", version = garage_version())]
 	Versions,
@@ -180,10 +180,21 @@ fn watch_shutdown_signal() -> watch::Receiver<bool> {
 		let mut sigterm =
 			signal(SignalKind::terminate()).expect("Failed to install SIGTERM handler");
 		let mut sighup = signal(SignalKind::hangup()).expect("Failed to install SIGHUP handler");
-		tokio::select! {
-			_ = sigint.recv() => info!("Received SIGINT, shutting down."),
-			_ = sigterm.recv() => info!("Received SIGTERM, shutting down."),
-			_ = sighup.recv() => info!("Received SIGHUP, shutting down."),
+		loop {
+			tokio::select! {
+					_ = sigint.recv() => {
+						info!("Received SIGINT, shutting down.");
+						break
+					}
+					_ = sigterm.recv() => {
+						info!("Received SIGTERM, shutting down.");
+						break
+					}
+					_ = sighup.recv() => {
+						info!("Received SIGHUP, reload not supported.");
+						continue
+					}
+			}
 		}
 		send_cancel.send(true).unwrap();
 	});
@@ -573,24 +573,68 @@ async fn test_website_redirect_full_bucket() {
 		.await
 		.unwrap();

-	let req = Request::builder()
-		.method("GET")
-		.uri(format!("http://127.0.0.1:{}/my-path", ctx.garage.web_port))
-		.header("Host", format!("{}.web.garage", BCKT_NAME))
-		.body(Body::new(Bytes::new()))
-		.unwrap();
+	{
+		let req = Request::builder()
+			.method("GET")
+			.uri(format!("http://127.0.0.1:{}/my-path", ctx.garage.web_port))
+			.header("Host", format!("{}.web.garage", BCKT_NAME))
+			.body(Body::new(Bytes::new()))
+			.unwrap();

-	let client = Client::builder(TokioExecutor::new()).build_http();
-	let resp = client.request(req).await.unwrap();
-	assert_eq!(resp.status(), StatusCode::FOUND);
-	assert_eq!(
-		resp.headers()
-			.get(hyper::header::LOCATION)
-			.unwrap()
-			.to_str()
-			.unwrap(),
-		"https://other.tld/my-path"
-	);
+		let client = Client::builder(TokioExecutor::new()).build_http();
+		let resp = client.request(req).await.unwrap();
+		assert_eq!(resp.status(), StatusCode::FOUND);
+		assert_eq!(
+			resp.headers()
+				.get(hyper::header::LOCATION)
+				.unwrap()
+				.to_str()
+				.unwrap(),
+			"https://other.tld/my-path"
+		);
+	}
+
+	{
+		let req = Request::builder()
+			.method("GET")
+			.uri(format!("http://127.0.0.1:{}/my-path/", ctx.garage.web_port))
+			.header("Host", format!("{}.web.garage", BCKT_NAME))
+			.body(Body::new(Bytes::new()))
+			.unwrap();
+
+		let client = Client::builder(TokioExecutor::new()).build_http();
+		let resp = client.request(req).await.unwrap();
+		assert_eq!(resp.status(), StatusCode::FOUND);
+		assert_eq!(
+			resp.headers()
+				.get(hyper::header::LOCATION)
+				.unwrap()
+				.to_str()
+				.unwrap(),
+			"https://other.tld/my-path/"
+		);
+	}
+
+	{
+		let req = Request::builder()
+			.method("GET")
+			.uri(format!("http://127.0.0.1:{}/", ctx.garage.web_port))
+			.header("Host", format!("{}.web.garage", BCKT_NAME))
+			.body(Body::new(Bytes::new()))
+			.unwrap();
+
+		let client = Client::builder(TokioExecutor::new()).build_http();
+		let resp = client.request(req).await.unwrap();
+		assert_eq!(resp.status(), StatusCode::FOUND);
+		assert_eq!(
+			resp.headers()
+				.get(hyper::header::LOCATION)
+				.unwrap()
+				.to_str()
+				.unwrap(),
+			"https://other.tld/"
+		);
+	}
 }

 #[tokio::test]
@@ -72,6 +72,16 @@ impl K2vClient {
 			.enable_http2()
 			.build();
 		let client = HttpClient::builder(TokioExecutor::new()).build(connector);
+		Self::new_with_client(config, client)
+	}
+
+	/// Create a new K2V client with an external client.
+	/// Useful for example if you plan on creating many clients but you want to mutualize the
+	/// underlying thread pools & co.
+	pub fn new_with_client(
+		config: K2vClientConfig,
+		client: HttpClient<HttpsConnector<HttpConnector>, Body>,
+	) -> Result<Self, Error> {
 		let user_agent: std::borrow::Cow<str> = match &config.user_agent {
 			Some(ua) => ua.into(),
 			None => format!("k2v/{}", env!("CARGO_PKG_VERSION")).into(),
@@ -1,6 +1,6 @@
 [package]
 name = "garage_model"
-version = "2.0.0"
+version = "2.1.0"
 authors = ["Alex Auvolat <alex@adnab.me>"]
 edition = "2018"
 license = "AGPL-3.0"
@@ -25,7 +25,7 @@ argon2.workspace = true
 async-trait.workspace = true
 blake2.workspace = true
 chrono.workspace = true
-err-derive.workspace = true
+thiserror.workspace = true
 hex.workspace = true
 http.workspace = true
 base64.workspace = true
@@ -45,3 +45,4 @@ default = [ "lmdb", "sqlite" ]
 k2v = [ "garage_util/k2v" ]
 lmdb = [ "garage_db/lmdb" ]
 sqlite = [ "garage_db/sqlite" ]
+fjall = [ "garage_db/fjall" ]
@@ -119,21 +119,17 @@ impl Garage {
 		info!("Opening database...");
 		let db_engine = db::Engine::from_str(&config.db_engine)
 			.ok_or_message("Invalid `db_engine` value in configuration file")?;
-		let mut db_path = config.metadata_dir.clone();
-		match db_engine {
-			db::Engine::Sqlite => {
-				db_path.push("db.sqlite");
-			}
-			db::Engine::Lmdb => {
-				db_path.push("db.lmdb");
-			}
-		}
+		let db_path = db_engine.db_path(&config.metadata_dir);
 		let db_opt = db::OpenOpt {
 			fsync: config.metadata_fsync,
 			lmdb_map_size: match config.lmdb_map_size {
 				v if v == usize::default() => None,
 				v => Some(v),
 			},
+			fjall_block_cache_size: match config.fjall_block_cache_size {
+				v if v == usize::default() => None,
+				v => Some(v),
+			},
 		};
 		let db = db::open_db(&db_path, db_engine, &db_opt)
 			.ok_or_message("Unable to open metadata db")?;
@@ -161,6 +157,7 @@ impl Garage {

 		let control_rep_param = TableFullReplication {
 			system: system.clone(),
+			consistency_mode,
 		};

 		info!("Initialize block manager...");
@@ -324,15 +321,15 @@ impl Garage {
 		Ok(())
 	}

-	pub fn bucket_helper(&self) -> helper::bucket::BucketHelper {
+	pub fn bucket_helper(&self) -> helper::bucket::BucketHelper<'_> {
 		helper::bucket::BucketHelper(self)
 	}

-	pub fn key_helper(&self) -> helper::key::KeyHelper {
+	pub fn key_helper(&self) -> helper::key::KeyHelper<'_> {
 		helper::key::KeyHelper(self)
 	}

-	pub async fn locked_helper(&self) -> helper::locked::LockedHelper {
+	pub async fn locked_helper(&self) -> helper::locked::LockedHelper<'_> {
 		let lock = self.bucket_lock.lock().await;
 		helper::locked::LockedHelper(self, Some(lock))
 	}
@@ -227,7 +227,7 @@ impl<'a> BucketHelper<'a> {
 				.0
 				.system
 				.cluster_layout()
-				.all_nongateway_nodes()
+				.all_nongateway_nodes()?
 				.to_vec();
 			let k2vindexes = self
 				.0
@@ -1,24 +1,24 @@
-use err_derive::Error;
 use serde::{Deserialize, Serialize};
+use thiserror::Error;

 use garage_util::error::Error as GarageError;

 #[derive(Debug, Error, Serialize, Deserialize)]
 pub enum Error {
-	#[error(display = "Internal error: {}", _0)]
-	Internal(#[error(source)] GarageError),
+	#[error("Internal error: {0}")]
+	Internal(#[from] GarageError),

-	#[error(display = "Bad request: {}", _0)]
+	#[error("Bad request: {0}")]
 	BadRequest(String),

 	/// Bucket name is not valid according to AWS S3 specs
-	#[error(display = "Invalid bucket name: {}", _0)]
+	#[error("Invalid bucket name: {0}")]
 	InvalidBucketName(String),

-	#[error(display = "Access key not found: {}", _0)]
+	#[error("Access key not found: {0}")]
 	NoSuchAccessKey(String),

-	#[error(display = "Bucket not found: {}", _0)]
+	#[error("Bucket not found: {0}")]
 	NoSuchBucket(String),
 }

@@ -84,17 +84,16 @@ impl<T: CountedItem> Entry<T::CP, T::CS> for CounterEntry<T> {

 impl<T: CountedItem> CounterEntry<T> {
 	pub fn filtered_values(&self, layout: &LayoutHelper) -> HashMap<String, i64> {
-		let nodes = layout.all_nongateway_nodes();
-		self.filtered_values_with_nodes(&nodes)
+		self.filtered_values_internal(layout.all_nongateway_nodes().ok())
 	}

-	pub fn filtered_values_with_nodes(&self, nodes: &[Uuid]) -> HashMap<String, i64> {
+	fn filtered_values_internal(&self, nodes_opt: Option<&[Uuid]>) -> HashMap<String, i64> {
 		let mut ret = HashMap::new();
 		for (name, vals) in self.values.iter() {
 			let new_vals = vals
 				.node_values
 				.iter()
-				.filter(|(n, _)| nodes.contains(n))
+				.filter(|(n, _)| nodes_opt.map(|nodes| nodes.contains(n)).unwrap_or(true))
 				.map(|(_, (_, v))| *v)
 				.collect::<Vec<_>>();
 			if !new_vals.is_empty() {
@@ -153,7 +152,7 @@ impl<T: CountedItem> TableSchema for CounterTable<T> {
 		}

 		let is_tombstone = entry
-			.filtered_values_with_nodes(&filter.1[..])
+			.filtered_values_internal(Some(&filter.1[..]))
 			.iter()
 			.all(|(_, v)| *v == 0);
 		filter.0.apply(is_tombstone)
--- a/Show More
+++ b/Show More