Merge pull request 'Garage v0.9.4' (#798) from rel-0.9.4 into main

Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/798

.woodpecker/debug.yaml

@@ -33,7 +33,11 @@ steps:
       - ./result/bin/garage_util-*
       - ./result/bin/garage_web-*
       - ./result/bin/garage-*
-      - ./result/bin/integration-* || (cat tmp-garage-integration/stderr.log; false)
+      - GARAGE_TEST_INTEGRATION_DB_ENGINE=sled ./result/bin/integration-* || (cat tmp-garage-integration/stderr.log; false)
+      - nix-shell --attr ci --run "killall -9 garage" || true
+      - GARAGE_TEST_INTEGRATION_DB_ENGINE=lmdb ./result/bin/integration-* || (cat tmp-garage-integration/stderr.log; false)
+      - nix-shell --attr ci --run "killall -9 garage" || true
+      - GARAGE_TEST_INTEGRATION_DB_ENGINE=sqlite ./result/bin/integration-* || (cat tmp-garage-integration/stderr.log; false)
       - rm result
       - rm -rv tmp-garage-integration
 

Cargo.lock

@@ -1277,7 +1277,7 @@ dependencies = [
 
 [[package]]
 name = "garage"
-version = "0.9.2"
+version = "0.9.4"
 dependencies = [
  "assert-json-diff",
  "async-trait",
@@ -1322,6 +1322,7 @@ dependencies = [
  "sha2",
  "static_init",
  "structopt",
+ "syslog-tracing",
  "timeago",
  "tokio",
  "toml",
@@ -1331,7 +1332,7 @@ dependencies = [
 
 [[package]]
 name = "garage_api"
-version = "0.9.2"
+version = "0.9.4"
 dependencies = [
  "argon2",
  "async-trait",
@@ -1380,7 +1381,7 @@ dependencies = [
 
 [[package]]
 name = "garage_block"
-version = "0.9.2"
+version = "0.9.4"
 dependencies = [
  "arc-swap",
  "async-compression",
@@ -1407,12 +1408,14 @@ dependencies = [
 
 [[package]]
 name = "garage_db"
-version = "0.9.2"
+version = "0.9.4"
 dependencies = [
  "err-derive",
  "heed",
  "hexdump",
  "mktemp",
+ "r2d2",
+ "r2d2_sqlite",
  "rusqlite",
  "sled",
  "tracing",
@@ -1420,7 +1423,7 @@ dependencies = [
 
 [[package]]
 name = "garage_model"
-version = "0.9.2"
+version = "0.9.4"
 dependencies = [
  "arc-swap",
  "async-trait",
@@ -1438,6 +1441,7 @@ dependencies = [
  "garage_util",
  "hex",
  "opentelemetry",
+ "parse_duration",
  "rand",
  "serde",
  "serde_bytes",
@@ -1448,7 +1452,7 @@ dependencies = [
 
 [[package]]
 name = "garage_net"
-version = "0.9.2"
+version = "0.9.4"
 dependencies = [
  "arc-swap",
  "async-trait",
@@ -1474,7 +1478,7 @@ dependencies = [
 
 [[package]]
 name = "garage_rpc"
-version = "0.9.2"
+version = "0.9.4"
 dependencies = [
  "arc-swap",
  "async-trait",
@@ -1509,7 +1513,7 @@ dependencies = [
 
 [[package]]
 name = "garage_table"
-version = "0.9.2"
+version = "0.9.4"
 dependencies = [
  "arc-swap",
  "async-trait",
@@ -1531,7 +1535,7 @@ dependencies = [
 
 [[package]]
 name = "garage_util"
-version = "0.9.2"
+version = "0.9.4"
 dependencies = [
  "arc-swap",
  "async-trait",
@@ -1565,7 +1569,7 @@ dependencies = [
 
 [[package]]
 name = "garage_web"
-version = "0.9.2"
+version = "0.9.4"
 dependencies = [
  "err-derive",
  "futures",
@@ -1707,9 +1711,9 @@ dependencies = [
 
 [[package]]
 name = "hashlink"
-version = "0.8.4"
+version = "0.9.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e8094feaf31ff591f651a2664fb9cfd92bba7a60ce3197265e9482ebe753c8f7"
+checksum = "692eaaf7f7607518dd3cef090f1474b61edc5301d8012f09579920df68b725ee"
 dependencies = [
  "hashbrown 0.14.3",
 ]
@@ -2366,9 +2370,9 @@ dependencies = [
 
 [[package]]
 name = "libsqlite3-sys"
-version = "0.27.0"
+version = "0.28.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cf4e226dcd58b4be396f7bd3c20da8fdee2911400705297ba7d2d7cc2c30f716"
+checksum = "0c10584274047cb335c23d3e61bcef8e323adae7c5c8c760540f73610177fc3f"
 dependencies = [
  "cc",
  "pkg-config",
@@ -3128,6 +3132,28 @@ dependencies = [
  "proc-macro2",
 ]
 
+[[package]]
+name = "r2d2"
+version = "0.8.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "51de85fb3fb6524929c8a2eb85e6b6d363de4e8c48f9e2c2eac4944abc181c93"
+dependencies = [
+ "log",
+ "parking_lot 0.12.1",
+ "scheduled-thread-pool",
+]
+
+[[package]]
+name = "r2d2_sqlite"
+version = "0.24.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6a982edf65c129796dba72f8775b292ef482b40d035e827a9825b3bc07ccc5f2"
+dependencies = [
+ "r2d2",
+ "rusqlite",
+ "uuid",
+]
+
 [[package]]
 name = "rand"
 version = "0.8.5"
@@ -3321,9 +3347,9 @@ checksum = "3cd14fd5e3b777a7422cca79358c57a8f6e3a703d9ac187448d0daf220c2407f"
 
 [[package]]
 name = "rusqlite"
-version = "0.30.0"
+version = "0.31.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a78046161564f5e7cd9008aff3b2990b3850dc8e0349119b98e8f251e099f24d"
+checksum = "b838eba278d213a8beaf485bd313fd580ca4505a00d5871caeb1457c55322cae"
 dependencies = [
  "bitflags 2.4.2",
  "fallible-iterator",
@@ -3488,6 +3514,15 @@ dependencies = [
  "windows-sys 0.52.0",
 ]
 
+[[package]]
+name = "scheduled-thread-pool"
+version = "0.2.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3cbc66816425a074528352f5789333ecff06ca41b36b0b0efdfbb29edc391a19"
+dependencies = [
+ "parking_lot 0.12.1",
+]
+
 [[package]]
 name = "schemars"
 version = "0.8.16"
@@ -3896,6 +3931,17 @@ dependencies = [
  "unicode-xid",
 ]
 
+[[package]]
+name = "syslog-tracing"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "340b1540dcdb6b066bc2966e7974f977ab1a38f21b2be189014ffb0cc2405768"
+dependencies = [
+ "libc",
+ "tracing-core",
+ "tracing-subscriber",
+]
+
 [[package]]
 name = "system-configuration"
 version = "0.5.1"
@@ -4441,6 +4487,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "79daa5ed5740825c40b389c5e50312b9c86df53fccd33f281df655642b43869d"
 dependencies = [
  "getrandom",
+ "rand",
 ]
 
 [[package]]

Cargo.nix

@@ -34,7 +34,7 @@ args@{
   ignoreLockHash,
 }:
 let
-  nixifiedLockHash = "3e1e0730302ee7d1f4185a13ad0086392562eb7cb962b1212801847887beaa47";
+  nixifiedLockHash = "9ea4045dd09421583b69811f95797af9c1f16239ecce89d8a6f5a9319d7d8526";
   workspaceSrc = if args.workspaceSrc == null then ./. else args.workspaceSrc;
   currentLockHash = builtins.hashFile "sha256" (workspaceSrc + /Cargo.lock);
   lockHashIgnored = if ignoreLockHash
@@ -58,17 +58,17 @@ in
 {
   cargo2nixVersion = "0.11.0";
   workspace = {
-    garage_db = rustPackages.unknown.garage_db."0.9.2";
+    garage_db = rustPackages.unknown.garage_db."0.9.4";
-    garage_util = rustPackages.unknown.garage_util."0.9.2";
+    garage_util = rustPackages.unknown.garage_util."0.9.4";
-    garage_net = rustPackages.unknown.garage_net."0.9.2";
+    garage_net = rustPackages.unknown.garage_net."0.9.4";
-    garage_rpc = rustPackages.unknown.garage_rpc."0.9.2";
+    garage_rpc = rustPackages.unknown.garage_rpc."0.9.4";
     format_table = rustPackages.unknown.format_table."0.1.1";
-    garage_table = rustPackages.unknown.garage_table."0.9.2";
+    garage_table = rustPackages.unknown.garage_table."0.9.4";
-    garage_block = rustPackages.unknown.garage_block."0.9.2";
+    garage_block = rustPackages.unknown.garage_block."0.9.4";
-    garage_model = rustPackages.unknown.garage_model."0.9.2";
+    garage_model = rustPackages.unknown.garage_model."0.9.4";
-    garage_api = rustPackages.unknown.garage_api."0.9.2";
+    garage_api = rustPackages.unknown.garage_api."0.9.4";
-    garage_web = rustPackages.unknown.garage_web."0.9.2";
+    garage_web = rustPackages.unknown.garage_web."0.9.4";
-    garage = rustPackages.unknown.garage."0.9.2";
+    garage = rustPackages.unknown.garage."0.9.4";
     k2v-client = rustPackages.unknown.k2v-client."0.0.4";
   };
   "registry+https://github.com/rust-lang/crates.io-index".addr2line."0.21.0" = overridableMkRustCrate (profileName: rec {
@@ -100,13 +100,13 @@ in
       (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "std")
     ];
     dependencies = {
-      ${ if rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "cfg_if" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".cfg-if."1.0.0" { inherit profileName; }).out;
+      ${ if rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/r2d2_sqlite" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "cfg_if" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".cfg-if."1.0.0" { inherit profileName; }).out;
       ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "getrandom" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".getrandom."0.2.12" { inherit profileName; }).out;
-      ${ if (rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") && !((hostPlatform.parsed.cpu.name == "armv6l" || hostPlatform.parsed.cpu.name == "armv7l") && hostPlatform.parsed.kernel.name == "none") then "once_cell" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".once_cell."1.19.0" { inherit profileName; }).out;
+      ${ if (rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/r2d2_sqlite" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") && !((hostPlatform.parsed.cpu.name == "armv6l" || hostPlatform.parsed.cpu.name == "armv7l") && hostPlatform.parsed.kernel.name == "none") then "once_cell" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".once_cell."1.19.0" { inherit profileName; }).out;
-      ${ if rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "zerocopy" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".zerocopy."0.7.32" { inherit profileName; }).out;
+      ${ if rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/r2d2_sqlite" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "zerocopy" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".zerocopy."0.7.32" { inherit profileName; }).out;
     };
     buildDependencies = {
-      ${ if rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "version_check" else null } = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".version_check."0.9.4" { profileName = "__noProfile"; }).out;
+      ${ if rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/r2d2_sqlite" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "version_check" else null } = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".version_check."0.9.4" { profileName = "__noProfile"; }).out;
     };
   });
   
@@ -130,7 +130,7 @@ in
     registry = "registry+https://github.com/rust-lang/crates.io-index";
     src = fetchCratesIo { inherit name version; sha256 = "0942ffc6dcaadf03badf6e6a2d0228460359d5e34b57ccdc720b7382dfbd5ec5"; };
     features = builtins.concatLists [
-      (lib.optional (rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "alloc")
+      (lib.optional (rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/r2d2_sqlite" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "alloc")
     ];
   });
   
@@ -1624,8 +1624,8 @@ in
     registry = "registry+https://github.com/rust-lang/crates.io-index";
     src = fetchCratesIo { inherit name version; sha256 = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649"; };
     features = builtins.concatLists [
-      (lib.optional (rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite") "alloc")
+      (lib.optional (rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/r2d2_sqlite" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite") "alloc")
-      (lib.optional (rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite") "default")
+      (lib.optional (rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/r2d2_sqlite" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite") "default")
     ];
   });
   
@@ -1871,9 +1871,9 @@ in
     };
   });
   
-  "unknown".garage."0.9.2" = overridableMkRustCrate (profileName: rec {
+  "unknown".garage."0.9.4" = overridableMkRustCrate (profileName: rec {
     name = "garage";
-    version = "0.9.2";
+    version = "0.9.4";
     registry = "unknown";
     src = fetchCrateLocal (workspaceSrc + "/src/garage");
     features = builtins.concatLists [
@@ -1889,6 +1889,8 @@ in
       (lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/metrics" || rootFeatures' ? "garage/prometheus") "prometheus")
       (lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sled") "sled")
       (lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite") "sqlite")
+      (lib.optional (rootFeatures' ? "garage/syslog") "syslog")
+      (lib.optional (rootFeatures' ? "garage/syslog" || rootFeatures' ? "garage/syslog-tracing") "syslog-tracing")
       (lib.optional (rootFeatures' ? "garage/system-libs") "system-libs")
       (lib.optional (rootFeatures' ? "garage/telemetry-otlp") "telemetry-otlp")
     ];
@@ -1900,15 +1902,15 @@ in
       format_table = (rustPackages."unknown".format_table."0.1.1" { inherit profileName; }).out;
       futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.30" { inherit profileName; }).out;
       futures_util = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.30" { inherit profileName; }).out;
-      garage_api = (rustPackages."unknown".garage_api."0.9.2" { inherit profileName; }).out;
+      garage_api = (rustPackages."unknown".garage_api."0.9.4" { inherit profileName; }).out;
-      garage_block = (rustPackages."unknown".garage_block."0.9.2" { inherit profileName; }).out;
+      garage_block = (rustPackages."unknown".garage_block."0.9.4" { inherit profileName; }).out;
-      garage_db = (rustPackages."unknown".garage_db."0.9.2" { inherit profileName; }).out;
+      garage_db = (rustPackages."unknown".garage_db."0.9.4" { inherit profileName; }).out;
-      garage_model = (rustPackages."unknown".garage_model."0.9.2" { inherit profileName; }).out;
+      garage_model = (rustPackages."unknown".garage_model."0.9.4" { inherit profileName; }).out;
-      garage_net = (rustPackages."unknown".garage_net."0.9.2" { inherit profileName; }).out;
+      garage_net = (rustPackages."unknown".garage_net."0.9.4" { inherit profileName; }).out;
-      garage_rpc = (rustPackages."unknown".garage_rpc."0.9.2" { inherit profileName; }).out;
+      garage_rpc = (rustPackages."unknown".garage_rpc."0.9.4" { inherit profileName; }).out;
-      garage_table = (rustPackages."unknown".garage_table."0.9.2" { inherit profileName; }).out;
+      garage_table = (rustPackages."unknown".garage_table."0.9.4" { inherit profileName; }).out;
-      garage_util = (rustPackages."unknown".garage_util."0.9.2" { inherit profileName; }).out;
+      garage_util = (rustPackages."unknown".garage_util."0.9.4" { inherit profileName; }).out;
-      garage_web = (rustPackages."unknown".garage_web."0.9.2" { inherit profileName; }).out;
+      garage_web = (rustPackages."unknown".garage_web."0.9.4" { inherit profileName; }).out;
       git_version = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".git-version."0.3.9" { inherit profileName; }).out;
       hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out;
       sodiumoxide = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".kuska-sodiumoxide."0.2.5-0" { inherit profileName; }).out;
@@ -1921,6 +1923,7 @@ in
       serde = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde."1.0.196" { inherit profileName; }).out;
       serde_bytes = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde_bytes."0.11.14" { inherit profileName; }).out;
       structopt = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".structopt."0.3.26" { inherit profileName; }).out;
+      ${ if rootFeatures' ? "garage/syslog" || rootFeatures' ? "garage/syslog-tracing" then "syslog_tracing" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".syslog-tracing."0.3.0" { inherit profileName; }).out;
       timeago = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".timeago."0.4.2" { inherit profileName; }).out;
       tokio = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.36.0" { inherit profileName; }).out;
       toml = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".toml."0.8.10" { inherit profileName; }).out;
@@ -1946,9 +1949,9 @@ in
     };
   });
   
-  "unknown".garage_api."0.9.2" = overridableMkRustCrate (profileName: rec {
+  "unknown".garage_api."0.9.4" = overridableMkRustCrate (profileName: rec {
     name = "garage_api";
-    version = "0.9.2";
+    version = "0.9.4";
     registry = "unknown";
     src = fetchCrateLocal (workspaceSrc + "/src/api");
     features = builtins.concatLists [
@@ -1968,12 +1971,12 @@ in
       form_urlencoded = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".form_urlencoded."1.2.1" { inherit profileName; }).out;
       futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.30" { inherit profileName; }).out;
       futures_util = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.30" { inherit profileName; }).out;
-      garage_block = (rustPackages."unknown".garage_block."0.9.2" { inherit profileName; }).out;
+      garage_block = (rustPackages."unknown".garage_block."0.9.4" { inherit profileName; }).out;
-      garage_model = (rustPackages."unknown".garage_model."0.9.2" { inherit profileName; }).out;
+      garage_model = (rustPackages."unknown".garage_model."0.9.4" { inherit profileName; }).out;
-      garage_net = (rustPackages."unknown".garage_net."0.9.2" { inherit profileName; }).out;
+      garage_net = (rustPackages."unknown".garage_net."0.9.4" { inherit profileName; }).out;
-      garage_rpc = (rustPackages."unknown".garage_rpc."0.9.2" { inherit profileName; }).out;
+      garage_rpc = (rustPackages."unknown".garage_rpc."0.9.4" { inherit profileName; }).out;
-      garage_table = (rustPackages."unknown".garage_table."0.9.2" { inherit profileName; }).out;
+      garage_table = (rustPackages."unknown".garage_table."0.9.4" { inherit profileName; }).out;
-      garage_util = (rustPackages."unknown".garage_util."0.9.2" { inherit profileName; }).out;
+      garage_util = (rustPackages."unknown".garage_util."0.9.4" { inherit profileName; }).out;
       hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out;
       hmac = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hmac."0.12.1" { inherit profileName; }).out;
       http = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."1.0.0" { inherit profileName; }).out;
@@ -2004,9 +2007,9 @@ in
     };
   });
   
-  "unknown".garage_block."0.9.2" = overridableMkRustCrate (profileName: rec {
+  "unknown".garage_block."0.9.4" = overridableMkRustCrate (profileName: rec {
     name = "garage_block";
-    version = "0.9.2";
+    version = "0.9.4";
     registry = "unknown";
     src = fetchCrateLocal (workspaceSrc + "/src/block");
     features = builtins.concatLists [
@@ -2020,11 +2023,11 @@ in
       bytesize = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytesize."1.3.0" { inherit profileName; }).out;
       futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.30" { inherit profileName; }).out;
       futures_util = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.30" { inherit profileName; }).out;
-      garage_db = (rustPackages."unknown".garage_db."0.9.2" { inherit profileName; }).out;
+      garage_db = (rustPackages."unknown".garage_db."0.9.4" { inherit profileName; }).out;
-      garage_net = (rustPackages."unknown".garage_net."0.9.2" { inherit profileName; }).out;
+      garage_net = (rustPackages."unknown".garage_net."0.9.4" { inherit profileName; }).out;
-      garage_rpc = (rustPackages."unknown".garage_rpc."0.9.2" { inherit profileName; }).out;
+      garage_rpc = (rustPackages."unknown".garage_rpc."0.9.4" { inherit profileName; }).out;
-      garage_table = (rustPackages."unknown".garage_table."0.9.2" { inherit profileName; }).out;
+      garage_table = (rustPackages."unknown".garage_table."0.9.4" { inherit profileName; }).out;
-      garage_util = (rustPackages."unknown".garage_util."0.9.2" { inherit profileName; }).out;
+      garage_util = (rustPackages."unknown".garage_util."0.9.4" { inherit profileName; }).out;
       hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out;
       opentelemetry = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry."0.17.0" { inherit profileName; }).out;
       rand = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand."0.8.5" { inherit profileName; }).out;
@@ -2037,9 +2040,9 @@ in
     };
   });
   
-  "unknown".garage_db."0.9.2" = overridableMkRustCrate (profileName: rec {
+  "unknown".garage_db."0.9.4" = overridableMkRustCrate (profileName: rec {
     name = "garage_db";
-    version = "0.9.2";
+    version = "0.9.4";
     registry = "unknown";
     src = fetchCrateLocal (workspaceSrc + "/src/db");
     features = builtins.concatLists [
@@ -2047,6 +2050,8 @@ in
       (lib.optional (rootFeatures' ? "garage_db/default") "default")
       (lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/lmdb" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/heed" || rootFeatures' ? "garage_db/lmdb" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/lmdb") "heed")
       (lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/lmdb" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/lmdb" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/lmdb") "lmdb")
+      (lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/r2d2" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite") "r2d2")
+      (lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/r2d2_sqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite") "r2d2_sqlite")
       (lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite") "rusqlite")
       (lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sled" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/sled" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sled") "sled")
       (lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite") "sqlite")
@@ -2055,7 +2060,9 @@ in
       err_derive = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".err-derive."0.3.1" { profileName = "__noProfile"; }).out;
       ${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/lmdb" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/heed" || rootFeatures' ? "garage_db/lmdb" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/lmdb" then "heed" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".heed."0.11.0" { inherit profileName; }).out;
       hexdump = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hexdump."0.1.1" { inherit profileName; }).out;
-      ${ if rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" then "rusqlite" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rusqlite."0.30.0" { inherit profileName; }).out;
+      ${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/r2d2" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" then "r2d2" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".r2d2."0.8.10" { inherit profileName; }).out;
+      ${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/r2d2_sqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" then "r2d2_sqlite" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".r2d2_sqlite."0.24.0" { inherit profileName; }).out;
+      ${ if rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" then "rusqlite" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rusqlite."0.31.0" { inherit profileName; }).out;
       ${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sled" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/sled" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sled" then "sled" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".sled."0.34.7" { inherit profileName; }).out;
       tracing = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing."0.1.40" { inherit profileName; }).out;
     };
@@ -2064,9 +2071,9 @@ in
     };
   });
   
-  "unknown".garage_model."0.9.2" = overridableMkRustCrate (profileName: rec {
+  "unknown".garage_model."0.9.4" = overridableMkRustCrate (profileName: rec {
     name = "garage_model";
-    version = "0.9.2";
+    version = "0.9.4";
     registry = "unknown";
     src = fetchCrateLocal (workspaceSrc + "/src/model");
     features = builtins.concatLists [
@@ -2085,14 +2092,15 @@ in
       err_derive = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".err-derive."0.3.1" { profileName = "__noProfile"; }).out;
       futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.30" { inherit profileName; }).out;
       futures_util = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.30" { inherit profileName; }).out;
-      garage_block = (rustPackages."unknown".garage_block."0.9.2" { inherit profileName; }).out;
+      garage_block = (rustPackages."unknown".garage_block."0.9.4" { inherit profileName; }).out;
-      garage_db = (rustPackages."unknown".garage_db."0.9.2" { inherit profileName; }).out;
+      garage_db = (rustPackages."unknown".garage_db."0.9.4" { inherit profileName; }).out;
-      garage_net = (rustPackages."unknown".garage_net."0.9.2" { inherit profileName; }).out;
+      garage_net = (rustPackages."unknown".garage_net."0.9.4" { inherit profileName; }).out;
-      garage_rpc = (rustPackages."unknown".garage_rpc."0.9.2" { inherit profileName; }).out;
+      garage_rpc = (rustPackages."unknown".garage_rpc."0.9.4" { inherit profileName; }).out;
-      garage_table = (rustPackages."unknown".garage_table."0.9.2" { inherit profileName; }).out;
+      garage_table = (rustPackages."unknown".garage_table."0.9.4" { inherit profileName; }).out;
-      garage_util = (rustPackages."unknown".garage_util."0.9.2" { inherit profileName; }).out;
+      garage_util = (rustPackages."unknown".garage_util."0.9.4" { inherit profileName; }).out;
       hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out;
       opentelemetry = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry."0.17.0" { inherit profileName; }).out;
+      parse_duration = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".parse_duration."2.1.1" { inherit profileName; }).out;
       rand = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand."0.8.5" { inherit profileName; }).out;
       serde = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde."1.0.196" { inherit profileName; }).out;
       serde_bytes = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde_bytes."0.11.14" { inherit profileName; }).out;
@@ -2102,9 +2110,9 @@ in
     };
   });
   
-  "unknown".garage_net."0.9.2" = overridableMkRustCrate (profileName: rec {
+  "unknown".garage_net."0.9.4" = overridableMkRustCrate (profileName: rec {
     name = "garage_net";
-    version = "0.9.2";
+    version = "0.9.4";
     registry = "unknown";
     src = fetchCrateLocal (workspaceSrc + "/src/net");
     features = builtins.concatLists [
@@ -2139,9 +2147,9 @@ in
     };
   });
   
-  "unknown".garage_rpc."0.9.2" = overridableMkRustCrate (profileName: rec {
+  "unknown".garage_rpc."0.9.4" = overridableMkRustCrate (profileName: rec {
     name = "garage_rpc";
-    version = "0.9.2";
+    version = "0.9.4";
     registry = "unknown";
     src = fetchCrateLocal (workspaceSrc + "/src/rpc");
     features = builtins.concatLists [
@@ -2163,9 +2171,9 @@ in
       format_table = (rustPackages."unknown".format_table."0.1.1" { inherit profileName; }).out;
       futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.30" { inherit profileName; }).out;
       futures_util = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.30" { inherit profileName; }).out;
-      garage_db = (rustPackages."unknown".garage_db."0.9.2" { inherit profileName; }).out;
+      garage_db = (rustPackages."unknown".garage_db."0.9.4" { inherit profileName; }).out;
-      garage_net = (rustPackages."unknown".garage_net."0.9.2" { inherit profileName; }).out;
+      garage_net = (rustPackages."unknown".garage_net."0.9.4" { inherit profileName; }).out;
-      garage_util = (rustPackages."unknown".garage_util."0.9.2" { inherit profileName; }).out;
+      garage_util = (rustPackages."unknown".garage_util."0.9.4" { inherit profileName; }).out;
       gethostname = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".gethostname."0.4.3" { inherit profileName; }).out;
       hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out;
       itertools = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".itertools."0.12.1" { inherit profileName; }).out;
@@ -2187,9 +2195,9 @@ in
     };
   });
   
-  "unknown".garage_table."0.9.2" = overridableMkRustCrate (profileName: rec {
+  "unknown".garage_table."0.9.4" = overridableMkRustCrate (profileName: rec {
     name = "garage_table";
-    version = "0.9.2";
+    version = "0.9.4";
     registry = "unknown";
     src = fetchCrateLocal (workspaceSrc + "/src/table");
     dependencies = {
@@ -2198,9 +2206,9 @@ in
       bytes = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.5.0" { inherit profileName; }).out;
       futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.30" { inherit profileName; }).out;
       futures_util = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.30" { inherit profileName; }).out;
-      garage_db = (rustPackages."unknown".garage_db."0.9.2" { inherit profileName; }).out;
+      garage_db = (rustPackages."unknown".garage_db."0.9.4" { inherit profileName; }).out;
-      garage_rpc = (rustPackages."unknown".garage_rpc."0.9.2" { inherit profileName; }).out;
+      garage_rpc = (rustPackages."unknown".garage_rpc."0.9.4" { inherit profileName; }).out;
-      garage_util = (rustPackages."unknown".garage_util."0.9.2" { inherit profileName; }).out;
+      garage_util = (rustPackages."unknown".garage_util."0.9.4" { inherit profileName; }).out;
       hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out;
       hexdump = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hexdump."0.1.1" { inherit profileName; }).out;
       opentelemetry = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry."0.17.0" { inherit profileName; }).out;
@@ -2212,9 +2220,9 @@ in
     };
   });
   
-  "unknown".garage_util."0.9.2" = overridableMkRustCrate (profileName: rec {
+  "unknown".garage_util."0.9.4" = overridableMkRustCrate (profileName: rec {
     name = "garage_util";
-    version = "0.9.2";
+    version = "0.9.4";
     registry = "unknown";
     src = fetchCrateLocal (workspaceSrc + "/src/util");
     features = builtins.concatLists [
@@ -2230,8 +2238,8 @@ in
       digest = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".digest."0.10.7" { inherit profileName; }).out;
       err_derive = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".err-derive."0.3.1" { profileName = "__noProfile"; }).out;
       futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.30" { inherit profileName; }).out;
-      garage_db = (rustPackages."unknown".garage_db."0.9.2" { inherit profileName; }).out;
+      garage_db = (rustPackages."unknown".garage_db."0.9.4" { inherit profileName; }).out;
-      garage_net = (rustPackages."unknown".garage_net."0.9.2" { inherit profileName; }).out;
+      garage_net = (rustPackages."unknown".garage_net."0.9.4" { inherit profileName; }).out;
       hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out;
       hexdump = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hexdump."0.1.1" { inherit profileName; }).out;
       http = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."1.0.0" { inherit profileName; }).out;
@@ -2256,18 +2264,18 @@ in
     };
   });
   
-  "unknown".garage_web."0.9.2" = overridableMkRustCrate (profileName: rec {
+  "unknown".garage_web."0.9.4" = overridableMkRustCrate (profileName: rec {
     name = "garage_web";
-    version = "0.9.2";
+    version = "0.9.4";
     registry = "unknown";
     src = fetchCrateLocal (workspaceSrc + "/src/web");
     dependencies = {
       err_derive = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".err-derive."0.3.1" { profileName = "__noProfile"; }).out;
       futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.30" { inherit profileName; }).out;
-      garage_api = (rustPackages."unknown".garage_api."0.9.2" { inherit profileName; }).out;
+      garage_api = (rustPackages."unknown".garage_api."0.9.4" { inherit profileName; }).out;
-      garage_model = (rustPackages."unknown".garage_model."0.9.2" { inherit profileName; }).out;
+      garage_model = (rustPackages."unknown".garage_model."0.9.4" { inherit profileName; }).out;
-      garage_table = (rustPackages."unknown".garage_table."0.9.2" { inherit profileName; }).out;
+      garage_table = (rustPackages."unknown".garage_table."0.9.4" { inherit profileName; }).out;
-      garage_util = (rustPackages."unknown".garage_util."0.9.2" { inherit profileName; }).out;
+      garage_util = (rustPackages."unknown".garage_util."0.9.4" { inherit profileName; }).out;
       http = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."1.0.0" { inherit profileName; }).out;
       http_body_util = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http-body-util."0.1.0" { inherit profileName; }).out;
       hyper = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hyper."1.1.0" { inherit profileName; }).out;
@@ -2422,25 +2430,25 @@ in
     registry = "registry+https://github.com/rust-lang/crates.io-index";
     src = fetchCratesIo { inherit name version; sha256 = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604"; };
     features = builtins.concatLists [
-      (lib.optional (rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "ahash")
+      (lib.optional (rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/r2d2_sqlite" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "ahash")
-      (lib.optional (rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "allocator-api2")
+      (lib.optional (rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/r2d2_sqlite" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "allocator-api2")
-      (lib.optional (rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "default")
+      (lib.optional (rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/r2d2_sqlite" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "default")
-      (lib.optional (rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "inline-more")
+      (lib.optional (rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/r2d2_sqlite" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "inline-more")
       [ "raw" ]
     ];
     dependencies = {
-      ${ if rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "ahash" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".ahash."0.8.7" { inherit profileName; }).out;
+      ${ if rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/r2d2_sqlite" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "ahash" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".ahash."0.8.7" { inherit profileName; }).out;
-      ${ if rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "allocator_api2" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".allocator-api2."0.2.16" { inherit profileName; }).out;
+      ${ if rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/r2d2_sqlite" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "allocator_api2" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".allocator-api2."0.2.16" { inherit profileName; }).out;
     };
   });
   
-  "registry+https://github.com/rust-lang/crates.io-index".hashlink."0.8.4" = overridableMkRustCrate (profileName: rec {
+  "registry+https://github.com/rust-lang/crates.io-index".hashlink."0.9.0" = overridableMkRustCrate (profileName: rec {
     name = "hashlink";
-    version = "0.8.4";
+    version = "0.9.0";
     registry = "registry+https://github.com/rust-lang/crates.io-index";
-    src = fetchCratesIo { inherit name version; sha256 = "e8094feaf31ff591f651a2664fb9cfd92bba7a60ce3197265e9482ebe753c8f7"; };
+    src = fetchCratesIo { inherit name version; sha256 = "692eaaf7f7607518dd3cef090f1474b61edc5301d8012f09579920df68b725ee"; };
     dependencies = {
-      ${ if rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" then "hashbrown" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hashbrown."0.14.3" { inherit profileName; }).out;
+      ${ if rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/r2d2_sqlite" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" then "hashbrown" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hashbrown."0.14.3" { inherit profileName; }).out;
     };
   });
   
@@ -3354,24 +3362,24 @@ in
     };
   });
   
-  "registry+https://github.com/rust-lang/crates.io-index".libsqlite3-sys."0.27.0" = overridableMkRustCrate (profileName: rec {
+  "registry+https://github.com/rust-lang/crates.io-index".libsqlite3-sys."0.28.0" = overridableMkRustCrate (profileName: rec {
     name = "libsqlite3-sys";
-    version = "0.27.0";
+    version = "0.28.0";
     registry = "registry+https://github.com/rust-lang/crates.io-index";
-    src = fetchCratesIo { inherit name version; sha256 = "cf4e226dcd58b4be396f7bd3c20da8fdee2911400705297ba7d2d7cc2c30f716"; };
+    src = fetchCratesIo { inherit name version; sha256 = "0c10584274047cb335c23d3e61bcef8e323adae7c5c8c760540f73610177fc3f"; };
     features = builtins.concatLists [
       (lib.optional (rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage_db/bundled-libs") "bundled")
       (lib.optional (rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage_db/bundled-libs") "bundled_bindings")
       (lib.optional (rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage_db/bundled-libs") "cc")
-      (lib.optional (rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite") "default")
+      (lib.optional (rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/r2d2_sqlite" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite") "default")
-      (lib.optional (rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite") "min_sqlite_version_3_14_0")
+      (lib.optional (rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/r2d2_sqlite" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite") "min_sqlite_version_3_14_0")
-      (lib.optional (rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite") "pkg-config")
+      (lib.optional (rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/r2d2_sqlite" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite") "pkg-config")
-      (lib.optional (rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite") "vcpkg")
+      (lib.optional (rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/r2d2_sqlite" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite") "vcpkg")
     ];
     buildDependencies = {
       ${ if rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage_db/bundled-libs" then "cc" else null } = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".cc."1.0.83" { profileName = "__noProfile"; }).out;
-      ${ if rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" then "pkg_config" else null } = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".pkg-config."0.3.29" { profileName = "__noProfile"; }).out;
+      ${ if rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/r2d2_sqlite" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" then "pkg_config" else null } = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".pkg-config."0.3.29" { profileName = "__noProfile"; }).out;
-      ${ if rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" then "vcpkg" else null } = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".vcpkg."0.2.15" { profileName = "__noProfile"; }).out;
+      ${ if rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/r2d2_sqlite" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" then "vcpkg" else null } = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".vcpkg."0.2.15" { profileName = "__noProfile"; }).out;
     };
   });
   
@@ -3960,11 +3968,11 @@ in
     registry = "registry+https://github.com/rust-lang/crates.io-index";
     src = fetchCratesIo { inherit name version; sha256 = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f"; };
     features = builtins.concatLists [
-      (lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage/metrics" || rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/opentelemetry-prometheus" || rootFeatures' ? "garage/prometheus" || rootFeatures' ? "garage/telemetry-otlp" || rootFeatures' ? "garage_api/metrics" || rootFeatures' ? "garage_api/opentelemetry-prometheus" || rootFeatures' ? "garage_api/prometheus" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "default")
+      (lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage/metrics" || rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/opentelemetry-prometheus" || rootFeatures' ? "garage/prometheus" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage/telemetry-otlp" || rootFeatures' ? "garage_api/metrics" || rootFeatures' ? "garage_api/opentelemetry-prometheus" || rootFeatures' ? "garage_api/prometheus" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/r2d2" || rootFeatures' ? "garage_db/r2d2_sqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "default")
     ];
     dependencies = {
-      ${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage/metrics" || rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/opentelemetry-prometheus" || rootFeatures' ? "garage/prometheus" || rootFeatures' ? "garage/telemetry-otlp" || rootFeatures' ? "garage_api/metrics" || rootFeatures' ? "garage_api/opentelemetry-prometheus" || rootFeatures' ? "garage_api/prometheus" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "lock_api" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".lock_api."0.4.11" { inherit profileName; }).out;
+      ${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage/metrics" || rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/opentelemetry-prometheus" || rootFeatures' ? "garage/prometheus" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage/telemetry-otlp" || rootFeatures' ? "garage_api/metrics" || rootFeatures' ? "garage_api/opentelemetry-prometheus" || rootFeatures' ? "garage_api/prometheus" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/r2d2" || rootFeatures' ? "garage_db/r2d2_sqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "lock_api" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".lock_api."0.4.11" { inherit profileName; }).out;
-      ${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage/metrics" || rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/opentelemetry-prometheus" || rootFeatures' ? "garage/prometheus" || rootFeatures' ? "garage/telemetry-otlp" || rootFeatures' ? "garage_api/metrics" || rootFeatures' ? "garage_api/opentelemetry-prometheus" || rootFeatures' ? "garage_api/prometheus" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "parking_lot_core" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".parking_lot_core."0.9.9" { inherit profileName; }).out;
+      ${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage/metrics" || rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/opentelemetry-prometheus" || rootFeatures' ? "garage/prometheus" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage/telemetry-otlp" || rootFeatures' ? "garage_api/metrics" || rootFeatures' ? "garage_api/opentelemetry-prometheus" || rootFeatures' ? "garage_api/prometheus" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/r2d2" || rootFeatures' ? "garage_db/r2d2_sqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "parking_lot_core" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".parking_lot_core."0.9.9" { inherit profileName; }).out;
     };
   });
   
@@ -3989,11 +3997,11 @@ in
     registry = "registry+https://github.com/rust-lang/crates.io-index";
     src = fetchCratesIo { inherit name version; sha256 = "4c42a9226546d68acdd9c0a280d17ce19bfe27a46bf68784e4066115788d008e"; };
     dependencies = {
-      ${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage/metrics" || rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/opentelemetry-prometheus" || rootFeatures' ? "garage/prometheus" || rootFeatures' ? "garage/telemetry-otlp" || rootFeatures' ? "garage_api/metrics" || rootFeatures' ? "garage_api/opentelemetry-prometheus" || rootFeatures' ? "garage_api/prometheus" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "cfg_if" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".cfg-if."1.0.0" { inherit profileName; }).out;
+      ${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage/metrics" || rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/opentelemetry-prometheus" || rootFeatures' ? "garage/prometheus" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage/telemetry-otlp" || rootFeatures' ? "garage_api/metrics" || rootFeatures' ? "garage_api/opentelemetry-prometheus" || rootFeatures' ? "garage_api/prometheus" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/r2d2" || rootFeatures' ? "garage_db/r2d2_sqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "cfg_if" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".cfg-if."1.0.0" { inherit profileName; }).out;
-      ${ if (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage/metrics" || rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/opentelemetry-prometheus" || rootFeatures' ? "garage/prometheus" || rootFeatures' ? "garage/telemetry-otlp" || rootFeatures' ? "garage_api/metrics" || rootFeatures' ? "garage_api/opentelemetry-prometheus" || rootFeatures' ? "garage_api/prometheus" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") && hostPlatform.isUnix then "libc" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.153" { inherit profileName; }).out;
+      ${ if (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage/metrics" || rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/opentelemetry-prometheus" || rootFeatures' ? "garage/prometheus" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage/telemetry-otlp" || rootFeatures' ? "garage_api/metrics" || rootFeatures' ? "garage_api/opentelemetry-prometheus" || rootFeatures' ? "garage_api/prometheus" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/r2d2" || rootFeatures' ? "garage_db/r2d2_sqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") && hostPlatform.isUnix then "libc" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.153" { inherit profileName; }).out;
-      ${ if (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage/metrics" || rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/opentelemetry-prometheus" || rootFeatures' ? "garage/prometheus" || rootFeatures' ? "garage/telemetry-otlp" || rootFeatures' ? "garage_api/metrics" || rootFeatures' ? "garage_api/opentelemetry-prometheus" || rootFeatures' ? "garage_api/prometheus" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") && hostPlatform.parsed.kernel.name == "redox" then "syscall" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".redox_syscall."0.4.1" { inherit profileName; }).out;
+      ${ if (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage/metrics" || rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/opentelemetry-prometheus" || rootFeatures' ? "garage/prometheus" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage/telemetry-otlp" || rootFeatures' ? "garage_api/metrics" || rootFeatures' ? "garage_api/opentelemetry-prometheus" || rootFeatures' ? "garage_api/prometheus" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/r2d2" || rootFeatures' ? "garage_db/r2d2_sqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") && hostPlatform.parsed.kernel.name == "redox" then "syscall" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".redox_syscall."0.4.1" { inherit profileName; }).out;
-      ${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage/metrics" || rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/opentelemetry-prometheus" || rootFeatures' ? "garage/prometheus" || rootFeatures' ? "garage/telemetry-otlp" || rootFeatures' ? "garage_api/metrics" || rootFeatures' ? "garage_api/opentelemetry-prometheus" || rootFeatures' ? "garage_api/prometheus" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "smallvec" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".smallvec."1.13.1" { inherit profileName; }).out;
+      ${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage/metrics" || rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/opentelemetry-prometheus" || rootFeatures' ? "garage/prometheus" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage/telemetry-otlp" || rootFeatures' ? "garage_api/metrics" || rootFeatures' ? "garage_api/opentelemetry-prometheus" || rootFeatures' ? "garage_api/prometheus" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/r2d2" || rootFeatures' ? "garage_db/r2d2_sqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "smallvec" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".smallvec."1.13.1" { inherit profileName; }).out;
-      ${ if (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage/metrics" || rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/opentelemetry-prometheus" || rootFeatures' ? "garage/prometheus" || rootFeatures' ? "garage/telemetry-otlp" || rootFeatures' ? "garage_api/metrics" || rootFeatures' ? "garage_api/opentelemetry-prometheus" || rootFeatures' ? "garage_api/prometheus" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") && hostPlatform.isWindows then "windows_targets" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows-targets."0.48.5" { inherit profileName; }).out;
+      ${ if (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage/metrics" || rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/opentelemetry-prometheus" || rootFeatures' ? "garage/prometheus" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage/telemetry-otlp" || rootFeatures' ? "garage_api/metrics" || rootFeatures' ? "garage_api/opentelemetry-prometheus" || rootFeatures' ? "garage_api/prometheus" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/r2d2" || rootFeatures' ? "garage_db/r2d2_sqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") && hostPlatform.isWindows then "windows_targets" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows-targets."0.48.5" { inherit profileName; }).out;
     };
   });
   
@@ -4435,6 +4443,30 @@ in
     };
   });
   
+  "registry+https://github.com/rust-lang/crates.io-index".r2d2."0.8.10" = overridableMkRustCrate (profileName: rec {
+    name = "r2d2";
+    version = "0.8.10";
+    registry = "registry+https://github.com/rust-lang/crates.io-index";
+    src = fetchCratesIo { inherit name version; sha256 = "51de85fb3fb6524929c8a2eb85e6b6d363de4e8c48f9e2c2eac4944abc181c93"; };
+    dependencies = {
+      ${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/r2d2" || rootFeatures' ? "garage_db/r2d2_sqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" then "log" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".log."0.4.20" { inherit profileName; }).out;
+      ${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/r2d2" || rootFeatures' ? "garage_db/r2d2_sqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" then "parking_lot" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".parking_lot."0.12.1" { inherit profileName; }).out;
+      ${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/r2d2" || rootFeatures' ? "garage_db/r2d2_sqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" then "scheduled_thread_pool" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".scheduled-thread-pool."0.2.7" { inherit profileName; }).out;
+    };
+  });
+  
+  "registry+https://github.com/rust-lang/crates.io-index".r2d2_sqlite."0.24.0" = overridableMkRustCrate (profileName: rec {
+    name = "r2d2_sqlite";
+    version = "0.24.0";
+    registry = "registry+https://github.com/rust-lang/crates.io-index";
+    src = fetchCratesIo { inherit name version; sha256 = "6a982edf65c129796dba72f8775b292ef482b40d035e827a9825b3bc07ccc5f2"; };
+    dependencies = {
+      ${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/r2d2_sqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" then "r2d2" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".r2d2."0.8.10" { inherit profileName; }).out;
+      ${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/r2d2_sqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" then "rusqlite" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rusqlite."0.31.0" { inherit profileName; }).out;
+      ${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/r2d2_sqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" then "uuid" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".uuid."1.4.1" { inherit profileName; }).out;
+    };
+  });
+  
   "registry+https://github.com/rust-lang/crates.io-index".rand."0.8.5" = overridableMkRustCrate (profileName: rec {
     name = "rand";
     version = "0.8.5";
@@ -4502,7 +4534,7 @@ in
     registry = "registry+https://github.com/rust-lang/crates.io-index";
     src = fetchCratesIo { inherit name version; sha256 = "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa"; };
     dependencies = {
-      ${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage/metrics" || rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/opentelemetry-prometheus" || rootFeatures' ? "garage/prometheus" || rootFeatures' ? "garage/telemetry-otlp" || rootFeatures' ? "garage_api/metrics" || rootFeatures' ? "garage_api/opentelemetry-prometheus" || rootFeatures' ? "garage_api/prometheus" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "bitflags" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bitflags."1.3.2" { inherit profileName; }).out;
+      ${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage/metrics" || rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/opentelemetry-prometheus" || rootFeatures' ? "garage/prometheus" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage/telemetry-otlp" || rootFeatures' ? "garage_api/metrics" || rootFeatures' ? "garage_api/opentelemetry-prometheus" || rootFeatures' ? "garage_api/prometheus" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/r2d2" || rootFeatures' ? "garage_db/r2d2_sqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "bitflags" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bitflags."1.3.2" { inherit profileName; }).out;
     };
   });
   
@@ -4763,22 +4795,23 @@ in
     ];
   });
   
-  "registry+https://github.com/rust-lang/crates.io-index".rusqlite."0.30.0" = overridableMkRustCrate (profileName: rec {
+  "registry+https://github.com/rust-lang/crates.io-index".rusqlite."0.31.0" = overridableMkRustCrate (profileName: rec {
     name = "rusqlite";
-    version = "0.30.0";
+    version = "0.31.0";
     registry = "registry+https://github.com/rust-lang/crates.io-index";
-    src = fetchCratesIo { inherit name version; sha256 = "a78046161564f5e7cd9008aff3b2990b3850dc8e0349119b98e8f251e099f24d"; };
+    src = fetchCratesIo { inherit name version; sha256 = "b838eba278d213a8beaf485bd313fd580ca4505a00d5871caeb1457c55322cae"; };
     features = builtins.concatLists [
+      (lib.optional (rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite") "backup")
       (lib.optional (rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage_db/bundled-libs") "bundled")
       (lib.optional (rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage_db/bundled-libs") "modern_sqlite")
     ];
     dependencies = {
-      ${ if rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" then "bitflags" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bitflags."2.4.2" { inherit profileName; }).out;
+      ${ if rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/r2d2_sqlite" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" then "bitflags" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bitflags."2.4.2" { inherit profileName; }).out;
-      ${ if rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" then "fallible_iterator" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".fallible-iterator."0.3.0" { inherit profileName; }).out;
+      ${ if rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/r2d2_sqlite" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" then "fallible_iterator" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".fallible-iterator."0.3.0" { inherit profileName; }).out;
-      ${ if rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" then "fallible_streaming_iterator" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".fallible-streaming-iterator."0.1.9" { inherit profileName; }).out;
+      ${ if rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/r2d2_sqlite" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" then "fallible_streaming_iterator" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".fallible-streaming-iterator."0.1.9" { inherit profileName; }).out;
-      ${ if rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" then "hashlink" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hashlink."0.8.4" { inherit profileName; }).out;
+      ${ if rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/r2d2_sqlite" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" then "hashlink" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hashlink."0.9.0" { inherit profileName; }).out;
-      ${ if rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" then "libsqlite3_sys" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".libsqlite3-sys."0.27.0" { inherit profileName; }).out;
+      ${ if rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/r2d2_sqlite" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" then "libsqlite3_sys" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".libsqlite3-sys."0.28.0" { inherit profileName; }).out;
-      ${ if rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" then "smallvec" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".smallvec."1.13.1" { inherit profileName; }).out;
+      ${ if rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/r2d2_sqlite" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" then "smallvec" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".smallvec."1.13.1" { inherit profileName; }).out;
     };
   });
   
@@ -4992,6 +5025,16 @@ in
     };
   });
   
+  "registry+https://github.com/rust-lang/crates.io-index".scheduled-thread-pool."0.2.7" = overridableMkRustCrate (profileName: rec {
+    name = "scheduled-thread-pool";
+    version = "0.2.7";
+    registry = "registry+https://github.com/rust-lang/crates.io-index";
+    src = fetchCratesIo { inherit name version; sha256 = "3cbc66816425a074528352f5789333ecff06ca41b36b0b0efdfbb29edc391a19"; };
+    dependencies = {
+      ${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/r2d2" || rootFeatures' ? "garage_db/r2d2_sqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" then "parking_lot" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".parking_lot."0.12.1" { inherit profileName; }).out;
+    };
+  });
+  
   "registry+https://github.com/rust-lang/crates.io-index".schemars."0.8.16" = overridableMkRustCrate (profileName: rec {
     name = "schemars";
     version = "0.8.16";
@@ -5572,6 +5615,18 @@ in
     };
   });
   
+  "registry+https://github.com/rust-lang/crates.io-index".syslog-tracing."0.3.0" = overridableMkRustCrate (profileName: rec {
+    name = "syslog-tracing";
+    version = "0.3.0";
+    registry = "registry+https://github.com/rust-lang/crates.io-index";
+    src = fetchCratesIo { inherit name version; sha256 = "340b1540dcdb6b066bc2966e7974f977ab1a38f21b2be189014ffb0cc2405768"; };
+    dependencies = {
+      ${ if rootFeatures' ? "garage/syslog" || rootFeatures' ? "garage/syslog-tracing" then "libc" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.153" { inherit profileName; }).out;
+      ${ if rootFeatures' ? "garage/syslog" || rootFeatures' ? "garage/syslog-tracing" then "tracing_core" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing-core."0.1.32" { inherit profileName; }).out;
+      ${ if rootFeatures' ? "garage/syslog" || rootFeatures' ? "garage/syslog-tracing" then "tracing_subscriber" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing-subscriber."0.3.18" { inherit profileName; }).out;
+    };
+  });
+  
   "registry+https://github.com/rust-lang/crates.io-index".system-configuration."0.5.1" = overridableMkRustCrate (profileName: rec {
     name = "system-configuration";
     version = "0.5.1";
@@ -6367,13 +6422,16 @@ in
     src = fetchCratesIo { inherit name version; sha256 = "79daa5ed5740825c40b389c5e50312b9c86df53fccd33f281df655642b43869d"; };
     features = builtins.concatLists [
       [ "default" ]
+      (lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/r2d2_sqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite") "fast-rng")
       [ "getrandom" ]
+      (lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/r2d2_sqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite") "rand")
       [ "rng" ]
       [ "std" ]
       [ "v4" ]
     ];
     dependencies = {
       getrandom = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".getrandom."0.2.12" { inherit profileName; }).out;
+      ${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/r2d2_sqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" then "rand" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand."0.8.5" { inherit profileName; }).out;
     };
   });
   
@@ -6894,7 +6952,7 @@ in
     registry = "registry+https://github.com/rust-lang/crates.io-index";
     src = fetchCratesIo { inherit name version; sha256 = "74d4d3961e53fa4c9a25a8637fc2bfaf2595b3d3ae34875568a5cf64787716be"; };
     features = builtins.concatLists [
-      (lib.optional (rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "simd")
+      (lib.optional (rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/r2d2_sqlite" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "simd")
     ];
     dependencies = {
       ${ if false then "zerocopy_derive" else null } = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".zerocopy-derive."0.7.32" { profileName = "__noProfile"; }).out;
@@ -6907,9 +6965,9 @@ in
     registry = "registry+https://github.com/rust-lang/crates.io-index";
     src = fetchCratesIo { inherit name version; sha256 = "9ce1b18ccd8e73a9321186f97e46f9f04b778851177567b1975109d26a08d2a6"; };
     dependencies = {
-      ${ if rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "proc_macro2" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".proc-macro2."1.0.78" { inherit profileName; }).out;
+      ${ if rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/r2d2_sqlite" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "proc_macro2" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".proc-macro2."1.0.78" { inherit profileName; }).out;
-      ${ if rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "quote" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".quote."1.0.35" { inherit profileName; }).out;
+      ${ if rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/r2d2_sqlite" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "quote" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".quote."1.0.35" { inherit profileName; }).out;
-      ${ if rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "syn" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".syn."2.0.48" { inherit profileName; }).out;
+      ${ if rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/r2d2_sqlite" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "syn" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".syn."2.0.48" { inherit profileName; }).out;
     };
   });
   

Cargo.toml

@@ -21,15 +21,15 @@ default-members = ["src/garage"]
 
 # Internal Garage crates
 format_table = { version = "0.1.1", path = "src/format-table" }
-garage_api = { version = "0.9.2", path = "src/api" }
+garage_api = { version = "0.9.4", path = "src/api" }
-garage_block = { version = "0.9.2", path = "src/block" }
+garage_block = { version = "0.9.4", path = "src/block" }
-garage_db = { version = "0.9.2", path = "src/db", default-features = false }
+garage_db = { version = "0.9.4", path = "src/db", default-features = false }
-garage_model = { version = "0.9.2", path = "src/model", default-features = false }
+garage_model = { version = "0.9.4", path = "src/model", default-features = false }
-garage_net = { version = "0.9.2", path = "src/net" }
+garage_net = { version = "0.9.4", path = "src/net" }
-garage_rpc = { version = "0.9.2", path = "src/rpc" }
+garage_rpc = { version = "0.9.4", path = "src/rpc" }
-garage_table = { version = "0.9.2", path = "src/table" }
+garage_table = { version = "0.9.4", path = "src/table" }
-garage_util = { version = "0.9.2", path = "src/util" }
+garage_util = { version = "0.9.4", path = "src/util" }
-garage_web = { version = "0.9.2", path = "src/web" }
+garage_web = { version = "0.9.4", path = "src/web" }
 k2v-client = { version = "0.0.4", path = "src/k2v-client" }
 
 # External crates from crates.io
@@ -72,11 +72,14 @@ kuska-handshake = { version = "0.2.0", features = ["default", "async_std"] }
 clap = { version = "4.1", features = ["derive", "env"] }
 pretty_env_logger = "0.5"
 structopt = { version = "0.3", default-features = false }
+syslog-tracing = "0.3"
 tracing = "0.1"
 tracing-subscriber = { version = "0.3", features = ["env-filter"] }
 
 heed = { version = "0.11", default-features = false, features = ["lmdb"] }
-rusqlite = "0.30.0"
+rusqlite = "0.31.0"
+r2d2 = "0.8"
+r2d2_sqlite = "0.24"
 sled = "0.34"
 
 async-compression = { version = "0.4", features = ["tokio", "zstd"] }

doc/book/connect/backup.md

@@ -55,8 +55,8 @@ Create your key and bucket:
 
 ```bash
 garage key create my-key
-garage bucket create backup
+garage bucket create backups
-garage bucket allow backup --read --write --key my-key
+garage bucket allow backups --read --write --key my-key
 ```
 
 Then register your Key ID and Secret key in your environment:

doc/book/connect/cli.md

@@ -259,7 +259,7 @@ duck --delete garage:/my-files/an-object.txt
 
 ## WinSCP (libs3) {#winscp}
 
-*You can find instructions on how to use the GUI in french [in our wiki](https://wiki.deuxfleurs.fr/fr/Guide/Garage/WinSCP).*
+*You can find instructions on how to use the GUI in french [in our wiki](https://guide.deuxfleurs.fr/prise_en_main/winscp/).*
 
 How to use `winscp.com`, the CLI interface of WinSCP:
 

doc/book/cookbook/from-source.md

@@ -90,6 +90,7 @@ The following feature flags are available in v0.8.0:
 | `kubernetes-discovery` | optional | Enable automatic registration and discovery<br>of cluster nodes through the Kubernetes API |
 | `metrics` | *by default* | Enable collection of metrics in Prometheus format on the admin API |
 | `telemetry-otlp` | optional | Enable collection of execution traces using OpenTelemetry |
+| `syslog` | optional | Enable logging to Syslog |
 | `sled` | *by default* | Enable using Sled to store Garage's metadata |
-| `lmdb` | optional | Enable using LMDB to store Garage's metadata |
+| `lmdb` | *by default* | Enable using LMDB to store Garage's metadata |
-| `sqlite` | optional | Enable using Sqlite3 to store Garage's metadata |
+| `sqlite` | *by default* | Enable using Sqlite3 to store Garage's metadata |

doc/book/cookbook/real-world.md

@@ -27,7 +27,7 @@ To run a real-world deployment, make sure the following conditions are met:
   [Yggdrasil](https://yggdrasil-network.github.io/) are approaches to consider
   in addition to building out your own VPN tunneling.
 
-- This guide will assume you are using Docker containers to deploy Garage on each node. 
+- This guide will assume you are using Docker containers to deploy Garage on each node.
   Garage can also be run independently, for instance as a [Systemd service](@/documentation/cookbook/systemd.md).
   You can also use an orchestrator such as Nomad or Kubernetes to automatically manage
   Docker containers on a fleet of nodes.
@@ -53,9 +53,9 @@ to store 2 TB of data in total.
 
 ### Best practices
 
-- If you have fast dedicated networking between all your nodes, and are planing to store
+- If you have reasonably fast networking between all your nodes, and are planing to store
-  very large files, bump the `block_size` configuration parameter to 10 MB
+  mostly large files, bump the `block_size` configuration parameter to 10 MB
-  (`block_size = 10485760`).
+  (`block_size = "10M"`).
 
 - Garage stores its files in two locations: it uses a metadata directory to store frequently-accessed
   small metadata items, and a data directory to store data blocks of uploaded objects.
@@ -68,31 +68,41 @@ to store 2 TB of data in total.
   EXT4 is not recommended as it has more strict limitations on the number of inodes,
   which might cause issues with Garage when large numbers of objects are stored.
 
-- If you only have an HDD and no SSD, it's fine to put your metadata alongside the data
-  on the same drive. Having lots of RAM for your kernel to cache the metadata will
-  help a lot with performance.  Make sure to use the LMDB database engine,
-  instead of Sled, which suffers from quite bad performance degradation on HDDs.
-  Sled is still the default for legacy reasons, but is not recommended anymore.
-
-- For the metadata storage, Garage does not do checksumming and integrity
-  verification on its own. If you are afraid of bitrot/data corruption,
-  put your metadata directory on a ZFS or BTRFS partition. Otherwise, just use regular
-  EXT4 or XFS.
-
 - Servers with multiple HDDs are supported natively by Garage without resorting
   to RAID, see [our dedicated documentation page](@/documentation/operations/multi-hdd.md).
 
+- For the metadata storage, Garage does not do checksumming and integrity
+  verification on its own, so it is better to use a robust filesystem such as
+  BTRFS or ZFS. Users have reported that when using the LMDB database engine
+  (the default), database files have a tendency of becoming corrupted after an
+  unclean shutdown (e.g. a power outage), so you should take regular snapshots
+  to be able to recover from such a situation.  This can be done using Garage's
+  built-in automatic snapshotting (since v0.9.4), or by using filesystem level
+  snapshots. If you cannot do so, you might want to switch to Sqlite which is
+  more robust.
+
+- LMDB is the fastest and most tested database engine, but it has the following
+  weaknesses: 1/ data files are not architecture-independent, you cannot simply
+  move a Garage metadata directory between nodes running different architectures,
+  and 2/ LMDB is not suited for 32-bit platforms. Sqlite is a viable alternative
+  if any of these are of concern.
+
+- If you only have an HDD and no SSD, it's fine to put your metadata alongside
+  the data on the same drive, but then consider your filesystem choice wisely
+  (see above).  Having lots of RAM for your kernel to cache the metadata will
+  help a lot with performance.
+
 ## Get a Docker image
 
 Our docker image is currently named `dxflrs/garage` and is stored on the [Docker Hub](https://hub.docker.com/r/dxflrs/garage/tags?page=1&ordering=last_updated).
-We encourage you to use a fixed tag (eg. `v0.9.2`) and not the `latest` tag.
+We encourage you to use a fixed tag (eg. `v0.9.4`) and not the `latest` tag.
-For this example, we will use the latest published version at the time of the writing which is `v0.9.2` but it's up to you
+For this example, we will use the latest published version at the time of the writing which is `v0.9.4` but it's up to you
 to check [the most recent versions on the Docker Hub](https://hub.docker.com/r/dxflrs/garage/tags?page=1&ordering=last_updated).
 
 For example:
 
 ```
-sudo docker pull dxflrs/garage:v0.9.2
+sudo docker pull dxflrs/garage:v0.9.4
 ```
 
 ## Deploying and configuring Garage
@@ -115,6 +125,7 @@ A valid `/etc/garage.toml` for our cluster would look as follows:
 metadata_dir = "/var/lib/garage/meta"
 data_dir = "/var/lib/garage/data"
 db_engine = "lmdb"
+metadata_auto_snapshot_interval = "6h"
 
 replication_mode = "3"
 
@@ -157,7 +168,7 @@ docker run \
   -v /etc/garage.toml:/etc/garage.toml \
   -v /var/lib/garage/meta:/var/lib/garage/meta \
   -v /var/lib/garage/data:/var/lib/garage/data \
-  dxflrs/garage:v0.9.2
+  dxflrs/garage:v0.9.4
 ```
 
 With this command line, Garage should be started automatically at each boot.
@@ -171,7 +182,7 @@ If you want to use `docker-compose`, you may use the following `docker-compose.y
 version: "3"
 services:
   garage:
-    image: dxflrs/garage:v0.9.2
+    image: dxflrs/garage:v0.9.4
     network_mode: "host"
     restart: unless-stopped
     volumes:
@@ -187,7 +198,7 @@ upgrades.  With the containerized setup proposed here, the upgrade process
 will require stopping and removing the existing container, and re-creating it
 with the upgraded version.
 
-## Controling the daemon
+## Controlling the daemon
 
 The `garage` binary has two purposes:
   - it acts as a daemon when launched with `garage server`
@@ -245,7 +256,7 @@ You can then instruct nodes to connect to one another as follows:
 Venus$ garage node connect 563e1ac825ee3323aa441e72c26d1030d6d4414aeb3dd25287c531e7fc2bc95d@[fc00:1::1]:3901
 ```
 
-You don't nead to instruct all node to connect to all other nodes:
+You don't need to instruct all node to connect to all other nodes:
 nodes will discover one another transitively.
 
 Now if your run `garage status` on any node, you should have an output that looks as follows:
@@ -328,8 +339,8 @@ Given the information above, we will configure our cluster as follow:
 ```bash
 garage layout assign 563e -z par1 -c 1T -t mercury
 garage layout assign 86f0 -z par1 -c 2T -t venus
-garage layout assign 6814 -z lon1 -c 2T -t earth 
+garage layout assign 6814 -z lon1 -c 2T -t earth
-garage layout assign 212f -z bru1 -c 1.5T -t mars 
+garage layout assign 212f -z bru1 -c 1.5T -t mars
 ```
 
 At this point, the changes in the cluster layout have not yet been applied.

doc/book/operations/durability-repairs.md

@@ -19,7 +19,7 @@ connecting to. To run on all nodes, add the `-a` flag as follows:
 
 # Data block operations
 
-## Data store scrub
+## Data store scrub {#scrub}
 
 Scrubbing the data store means examining each individual data block to check that
 their content is correct, by verifying their hash. Any block found to be corrupted
@@ -104,6 +104,24 @@ operation will also move out all data from locations marked as read-only.
 
 # Metadata operations
 
+## Metadata snapshotting
+
+It is good practice to setup automatic snapshotting of your metadata database
+file, to recover from situations where it becomes corrupted on disk. This can
+be done at the filesystem level if you are using ZFS or BTRFS.
+
+Since Garage v0.9.4, Garage is able to take snapshots of the metadata database
+itself. This basically amounts to copying the database file, except that it can
+be run live while Garage is running without the risk of corruption or
+inconsistencies.  This can be setup to run automatically on a schedule using
+[`metadata_auto_snapshot_interval`](@/documentation/reference-manual/configuration.md#metadata_auto_snapshot_interval).
+A snapshot can also be triggered manually using the `garage meta snapshot`
+command. Note that taking a snapshot using this method is very intensive as it
+requires making a full copy of the database file, so you might prefer using
+filesystem-level snapshots if possible. To recover a corrupted node from such a
+snapshot, read the instructions
+[here](@/documentation/operations/recovering.md#corrupted_meta).
+
 ## Metadata table resync
 
 Garage automatically resyncs all entries stored in the metadata tables every hour,

doc/book/operations/recovering.md

@@ -108,3 +108,57 @@ garage layout apply   # once satisfied, apply the changes
 
 Garage will then start synchronizing all required data on the new node.
 This process can be monitored using the `garage stats -a` command.
+
+## Replacement scenario 3: corrupted metadata {#corrupted_meta}
+
+In some cases, your metadata DB file might become corrupted, for instance if
+your node suffered a power outage and did not shut down properly. In this case,
+you can recover without having to change the node ID and rebuilding a cluster
+layout. This means that data blocks will not need to be shuffled around, you
+must simply find a way to repair the metadata file. The best way is generally
+to discard the corrupted file and recover it from another source.
+
+First of all, start by locating the database file in your metadata directory,
+which [depends on your `db_engine`
+choice](@/documentation/reference-manual/configuration.md#db_engine).  Then,
+your recovery options are as follows:
+
+- **Option 1: resyncing from other nodes.** In case your cluster is replicated
+  with two or three copies, you can simply delete the database file, and Garage
+  will resync from other nodes. To do so, stop Garage, delete the database file
+  or directory, and restart Garage. Then, do a full table repair by calling
+  `garage repair -a --yes tables`.  This will take a bit of time to complete as
+  the new node will need to receive copies of the metadata tables from the
+  network.
+
+- **Option 2: restoring a snapshot taken by Garage.** Since v0.9.4, Garage can
+  [automatically take regular
+  snapshots](@/documentation/reference-manual/configuration.md#metadata_auto_snapshot_interval)
+  of your metadata DB file. This file or directory should be located under
+  `<metadata_dir>/snapshots`, and is named according to the UTC time at which it
+  was taken. Stop Garage, discard the database file/directory and replace it by the
+  snapshot you want to use. For instance, in the case of LMDB:
+
+  ```bash
+  cd $METADATA_DIR
+  mv db.lmdb db.lmdb.bak
+  cp -r snapshots/2024-03-15T12:13:52Z db.lmdb
+  ```
+
+  And for Sqlite:
+
+  ```bash
+  cd $METADATA_DIR
+  mv db.sqlite db.sqlite.bak
+  cp snapshots/2024-03-15T12:13:52Z db.sqlite
+  ```
+
+  Then, restart Garage and run a full table repair by calling `garage repair -a
+  --yes tables`.  This should run relatively fast as only the changes that
+  occurred since the snapshot was taken will need to be resynchronized. Of
+  course, if your cluster is not replicated, you will lose all changes that
+  occurred since the snapshot was taken.
+
+- **Option 3: restoring a filesystem-level snapshot.** If you are using ZFS or
+  BTRFS to snapshot your metadata partition, refer to their specific
+  documentation on rolling back or copying files from an old snapshot.

doc/book/operations/upgrading.md

@@ -73,6 +73,18 @@ The entire procedure would look something like this:
   You can do all of the nodes in a single zone at once as that won't impact global cluster availability.
   Do not try to make a backup of the metadata folder of a running node.
 
+  **Since Garage v0.9.4,** you can use the `garage meta snapshot --all` command
+  to take a simultaneous snapshot of the metadata database files of all your
+  nodes.  This avoids the tedious process of having to take them down one by
+  one before upgrading. Be careful that if automatic snapshotting is enabled,
+  Garage only keeps the last two snapshots and deletes older ones, so you might
+  want to disable automatic snapshotting in your upgraded configuration file
+  until you have confirmed that the upgrade ran successfully.  In addition to
+  snapshotting the metadata databases of your nodes, you should back-up at
+  least the `cluster_layout` file of one of your Garage instances (this file
+  should be the same on all nodes and you can copy it safely while Garage is
+  running).
+
 3. Prepare your binaries and configuration files for the new Garage version
 
 4. Restart all nodes simultaneously in the new version

doc/book/quick-start/_index.md

@@ -57,7 +57,7 @@ to generate unique and private secrets for security reasons:
 cat > garage.toml <<EOF
 metadata_dir = "/tmp/meta"
 data_dir = "/tmp/data"
-db_engine = "lmdb"
+db_engine = "sqlite"
 
 replication_mode = "none"
 

doc/book/reference-manual/configuration.md

@@ -14,10 +14,13 @@ metadata_dir = "/var/lib/garage/meta"
 data_dir = "/var/lib/garage/data"
 metadata_fsync = true
 data_fsync = false
+disable_scrub = false
+metadata_auto_snapshot_interval = "6h"
 
 db_engine = "lmdb"
 
 block_size = "1M"
+block_ram_buffer_max = "256MiB"
 
 sled_cache_capacity = "128MiB"
 sled_flush_every_ms = 2000
@@ -30,6 +33,8 @@ rpc_bind_addr = "[::]:3901"
 rpc_bind_outgoing = false
 rpc_public_addr = "[fc00:1::1]:3901"
 
+allow_world_readable_secrets = false
+
 bootstrap_peers = [
     "563e1ac825ee3323aa441e72c26d1030d6d4414aeb3dd25287c531e7fc2bc95d@[fc00:1::1]:3901",
     "86f0f26ae4afbd59aaf9cfb059eefac844951efd5b8caeec0d53f4ed6c85f332@[fc00:1::2]:3901",
@@ -80,14 +85,20 @@ The following gives details about each available configuration option.
 
 ### Index
 
+[Environment variables](#env_variables).
+
 Top-level configuration options:
+[`allow_world_readable_secrets`](#allow_world_readable_secrets),
+[`block_ram_buffer_max`](#block_ram_buffer_max),
 [`block_size`](#block_size),
 [`bootstrap_peers`](#bootstrap_peers),
 [`compression_level`](#compression_level),
 [`data_dir`](#data_dir),
 [`data_fsync`](#data_fsync),
 [`db_engine`](#db_engine),
+[`disable_scrub`](#disable_scrub),
 [`lmdb_map_size`](#lmdb_map_size),
+[`metadata_auto_snapshot_interval`](#metadata_auto_snapshot_interval),
 [`metadata_dir`](#metadata_dir),
 [`metadata_fsync`](#metadata_fsync),
 [`replication_mode`](#replication_mode),
@@ -130,6 +141,23 @@ The `[admin]` section:
 [`admin_token`/`admin_token_file`](#admin_token),
 [`trace_sink`](#admin_trace_sink),
 
+### Environment variables {#env_variables}
+
+The following configuration parameter must be specified as an environment
+variable, it does not exist in the configuration file:
+
+- `GARAGE_LOG_TO_SYSLOG` (since v0.9.4): set this to `1` or `true` to make the
+  Garage daemon send its logs to `syslog` (using the libc `syslog` function)
+  instead of printing to stderr.
+
+The following environment variables can be used to override the corresponding
+values in the configuration file:
+
+- [`GARAGE_ALLOW_WORLD_READABLE_SECRETS`](#allow_world_readable_secrets)
+- [`GARAGE_RPC_SECRET` and `GARAGE_RPC_SECRET_FILE`](#rpc_secret)
+- [`GARAGE_ADMIN_TOKEN` and `GARAGE_ADMIN_TOKEN_FILE`](#admin_token)
+- [`GARAGE_METRICS_TOKEN` and `GARAGE_METRICS_TOKEN`](#admin_metrics_token)
+
 
 ### Top-level configuration options
 
@@ -264,18 +292,31 @@ Performance characteristics of the different DB engines are as follows:
 - Sled: tends to produce large data files and also has performance issues,
   especially when the metadata folder is on a traditional HDD and not on SSD.
 
-- LMDB: the recommended database engine on 64-bit systems, much more
+- LMDB: the recommended database engine for high-performance distributed
-  space-efficient and slightly faster. Note that the data format of LMDB is not
+  clusters, much more space-efficient and significantly faster. LMDB works very
-  portable between architectures, so for instance the Garage database of an
+  well, but is known to have the following limitations:
-  x86-64 node cannot be moved to an ARM64 node. Also note that, while LMDB can
+
-  technically be used on 32-bit systems, this will limit your node to very
+  - The data format of LMDB is not portable between architectures, so for
-  small database sizes due to how LMDB works; it is therefore not recommended.
+    instance the Garage database of an x86-64 node cannot be moved to an ARM64
+    node.
+
+  - While LMDB can technically be used on 32-bit systems, this will limit your
+    node to very small database sizes due to how LMDB works; it is therefore
+    not recommended.
+
+  - Several users have reported corrupted LMDB database files after an unclean
+    shutdown (e.g. a power outage). This situation can generally be recovered
+    from if your cluster is geo-replicated (by rebuilding your metadata db from
+    other nodes), or if you have saved regular snapshots at the filesystem
+    level.
 
 - Sqlite: Garage supports Sqlite as an alternative storage backend for
-  metadata, and although it has not been tested as much, it is expected to work
+  metadata, which does not have the issues listed above for LMDB.
-  satisfactorily.  Since Garage v0.9.0, performance issues have largely been
+  On versions 0.8.x and earlier, Sqlite should be avoided due to abysmal
-  fixed by allowing for a no-fsync mode (see `metadata_fsync`). Sqlite does not
+  performance, which was fixed with the addition of `metadata_fsync`.
-  have the database size limitation of LMDB on 32-bit systems.
+  Sqlite is still probably slower than LMDB due to the way we use it,
+  so it is not the best choice for high-performance storage clusters,
+  but it should work fine in many cases.
 
 It is possible to convert Garage's metadata directory from one format to another
 using the `garage convert-db` command, which should be used as follows:
@@ -302,7 +343,7 @@ Using this option reduces the risk of simultaneous metadata corruption on severa
 cluster nodes, which could lead to data loss.
 
 If multi-site replication is used, this option is most likely not necessary, as
-it is extremely unlikely that two nodes in different locations will have a 
+it is extremely unlikely that two nodes in different locations will have a
 power failure at the exact same time.
 
 (Metadata corruption on a single node is not an issue, the corrupted data file
@@ -331,6 +372,43 @@ at the cost of a moderate drop in write performance.
 Similarly to `metatada_fsync`, this is likely not necessary
 if geographical replication is used.
 
+#### `metadata_auto_snapshot_interval` (since Garage v0.9.4) {#metadata_auto_snapshot_interval}
+
+If this value is set, Garage will automatically take a snapshot of the metadata
+DB file at a regular interval and save it in the metadata directory.
+This parameter can take any duration string that can be parsed by
+the [`parse_duration`](https://docs.rs/parse_duration/latest/parse_duration/#syntax) crate.
+
+Snapshots can allow to recover from situations where the metadata DB file is
+corrupted, for instance after an unclean shutdown.  See [this
+page](@/documentation/operations/recovering.md#corrupted_meta) for details.
+Garage keeps only the two most recent snapshots of the metadata DB and deletes
+older ones automatically.
+
+Note that taking a metadata snapshot is a relatively intensive operation as the
+entire data file is copied. A snapshot being taken might have performance
+impacts on the Garage node while it is running. If the cluster is under heavy
+write load when a snapshot operation is running, this might also cause the
+database file to grow in size significantly as pages cannot be recycled easily.
+For this reason, it might be better to use filesystem-level snapshots instead
+if possible.
+
+#### `disable_scrub` {#disable_scrub}
+
+By default, Garage runs a scrub of the data directory approximately once per
+month, with a random delay to avoid all nodes running at the same time.  When
+it scrubs the data directory, Garage will read all of the data files stored on
+disk to check their integrity, and will rebuild any data files that it finds
+corrupted, using the remaining valid copies stored on other nodes.
+See [this page](@/documentation/operations/durability-repair.md#scrub) for details.
+
+Set the `disable_scrub` configuration value to `true` if you don't need Garage
+to scrub the data directory, for instance if you are already scrubbing at the
+filesystem level. Note that in this case, if you find a corrupted data file,
+you should delete it from the data directory and then call `garage repair
+blocks` on the node to ensure that it re-obtains a copy from another node on
+the network.
+
 #### `block_size` {#block_size}
 
 Garage splits stored objects in consecutive chunks of size `block_size`
@@ -346,6 +424,37 @@ files will remain available. This however means that chunks from existing files
 will not be deduplicated with chunks from newly uploaded files, meaning you
 might use more storage space that is optimally possible.
 
+#### `block_ram_buffer_max` (since v0.9.4) {#block_ram_buffer_max}
+
+A limit on the total size of data blocks kept in RAM by S3 API nodes awaiting
+to be sent to storage nodes asynchronously.
+
+Explanation: since Garage wants to tolerate node failures, it uses quorum
+writes to send data blocks to storage nodes: try to write the block to three
+nodes, and return ok as soon as two writes complete. So even if all three nodes
+are online, the third write always completes asynchronously.  In general, there
+are not many writes to a cluster, and the third asynchronous write can
+terminate early enough so as to not cause unbounded RAM growth.  However, if
+the S3 API node is continuously receiving large quantities of data and the
+third node is never able to catch up, many data blocks will be kept buffered in
+RAM as they are awaiting transfer to the third node.
+
+The `block_ram_buffer_max` sets a limit to the size of buffers that can be kept
+in RAM in this process.  When the limit is reached, backpressure is applied
+back to the S3 client.
+
+Note that this only counts buffers that have arrived to a certain stage of
+processing (received from the client + encrypted and/or compressed as
+necessary) and are ready to send to the storage nodes. Many other buffers will
+not be counted and this is not a hard limit on RAM consumption.  In particular,
+if many clients send requests simultaneously with large objects, the RAM
+consumption will always grow linearly with the number of concurrent requests,
+as each request will use a few buffers of size `block_size` for receiving and
+intermediate processing before even trying to send the data to the storage
+node.
+
+The default value is 256MiB.
+
 #### `sled_cache_capacity` {#sled_cache_capacity}
 
 This parameter can be used to tune the capacity of the cache used by
@@ -451,7 +560,7 @@ be obtained by running `garage node id` and then included directly in the
 key will be returned by `garage node id` and you will have to add the IP
 yourself.
 
-### `allow_world_readable_secrets`
+### `allow_world_readable_secrets` or `GARAGE_ALLOW_WORLD_READABLE_SECRETS` (env) {#allow_world_readable_secrets}
 
 Garage checks the permissions of your secret files to make sure they're not
 world-readable. In some cases, the check might fail and consider your files as

doc/book/reference-manual/monitoring.md

@@ -225,6 +225,17 @@ block_bytes_read 120586322022
 block_bytes_written 3386618077
 ```
 
+#### `block_ram_buffer_free_kb` (gauge)
+
+Kibibytes available for buffering blocks that have to be sent to remote nodes.
+When clients send too much data to this node and a storage node is not receiving
+data fast enough due to slower network conditions, this will decrease down to
+zero and backpressure will be applied.
+
+```
+block_ram_buffer_free_kb 219829
+```
+
 #### `block_compression_level` (counter)
 
 Exposes the block compression level configured for the Garage node.

doc/talks/2024-02-29-capitoul/.gitignore

@@ -0,0 +1,17 @@
+*
+
+!*.txt
+!*.md
+
+!assets
+
+!.gitignore
+!*.svg
+!*.png
+!*.jpg
+!*.tex
+!Makefile
+!.gitignore
+!assets/*.drawio.pdf
+
+!talk.pdf

doc/talks/2024-02-29-capitoul/Makefile

@@ -0,0 +1,10 @@
+ASSETS=../assets/logos/deuxfleurs.pdf
+
+talk.pdf: talk.tex $(ASSETS)
+	pdflatex talk.tex
+
+%.pdf: %.svg
+	inkscape -D -z --file=$^ --export-pdf=$@
+
+%.pdf_tex: %.svg
+	inkscape -D -z --file=$^ --export-pdf=$@ --export-latex

doc/talks/2024-02-29-capitoul/talk.tex

@@ -0,0 +1,543 @@
+\nonstopmode
+\documentclass[aspectratio=169,xcolor={svgnames}]{beamer}
+\usepackage[utf8]{inputenc}
+% \usepackage[frenchb]{babel}
+\usepackage{amsmath}
+\usepackage{mathtools}
+\usepackage{breqn}
+\usepackage{multirow}
+\usetheme{boxes}
+\usepackage{graphicx}
+\usepackage{import}
+\usepackage{adjustbox}
+\usepackage[absolute,overlay]{textpos}
+%\useoutertheme[footline=authortitle,subsection=false]{miniframes}
+%\useoutertheme[footline=authorinstitute,subsection=false]{miniframes}
+\useoutertheme{infolines}
+\setbeamertemplate{headline}{}
+
+\beamertemplatenavigationsymbolsempty
+
+\definecolor{TitleOrange}{RGB}{255,137,0}
+\setbeamercolor{title}{fg=TitleOrange}
+\setbeamercolor{frametitle}{fg=TitleOrange}
+
+\definecolor{ListOrange}{RGB}{255,145,5}
+\setbeamertemplate{itemize item}{\color{ListOrange}$\blacktriangleright$}
+
+\definecolor{verygrey}{RGB}{70,70,70}
+\setbeamercolor{normal text}{fg=verygrey}
+
+
+\usepackage{tabu}
+\usepackage{multicol}
+\usepackage{vwcol}
+\usepackage{stmaryrd}
+\usepackage{graphicx}
+
+\usepackage[normalem]{ulem}
+
+\AtBeginSection[]{
+  \begin{frame}
+  \vfill
+  \centering
+  \begin{beamercolorbox}[sep=8pt,center,shadow=true,rounded=true]{title}
+    \usebeamerfont{title}\insertsectionhead\par%
+  \end{beamercolorbox}
+  \vfill
+  \end{frame}
+}
+
+\title{Garage}
+\author{Alex Auvolat, Deuxfleurs}
+\date{Capitoul, 2024-02-29}
+
+\begin{document}
+
+\begin{frame}
+	\centering
+	\includegraphics[width=.3\linewidth]{../../sticker/Garage.png}
+	\vspace{1em}
+
+	{\large\bf Alex Auvolat, Deuxfleurs Association}
+	\vspace{1em}
+
+	\url{https://garagehq.deuxfleurs.fr/}
+
+	Matrix channel: \texttt{\#garage:deuxfleurs.fr}
+\end{frame}
+
+\begin{frame}
+	\frametitle{Who I am}
+	\begin{columns}[t]
+		\begin{column}{.2\textwidth}
+			\centering
+			\adjincludegraphics[width=.4\linewidth, valign=t]{../assets/alex.jpg}
+		\end{column}
+		\begin{column}{.6\textwidth}
+			\textbf{Alex Auvolat}\\
+			PhD; co-founder of Deuxfleurs
+		\end{column}
+		\begin{column}{.2\textwidth}
+			~
+		\end{column}
+	\end{columns}
+	\vspace{2em}
+
+	\begin{columns}[t]
+		\begin{column}{.2\textwidth}
+			\centering
+			\adjincludegraphics[width=.5\linewidth, valign=t]{../assets/logos/deuxfleurs.pdf}
+		\end{column}
+		\begin{column}{.6\textwidth}
+			\textbf{Deuxfleurs}\\
+			A non-profit self-hosting collective,\\
+			member of the CHATONS network
+		\end{column}
+		\begin{column}{.2\textwidth}
+			\centering
+			\adjincludegraphics[width=.7\linewidth, valign=t]{../assets/logos/logo_chatons.png}
+		\end{column}
+	\end{columns}
+
+\end{frame}
+
+\begin{frame}
+	\frametitle{Our objective at Deuxfleurs}
+	
+	\begin{center}
+		\textbf{Promote self-hosting and small-scale hosting\\
+			as an alternative to large cloud providers}
+	\end{center}
+	\vspace{2em}
+	\visible<2->{
+		Why is it hard?
+		\vspace{2em}
+		\begin{center}
+			\textbf{\underline{Resilience}}\\
+			{\footnotesize we want good uptime/availability with low supervision}
+		\end{center}
+	}
+\end{frame}
+
+\begin{frame}
+	\frametitle{Our very low-tech infrastructure}
+
+	\only<1,3-6>{
+		\begin{itemize}
+			\item \textcolor<4->{gray}{Commodity hardware (e.g. old desktop PCs)\\
+				\vspace{.5em}
+				\visible<3->{{\footnotesize (can die at any time)}}}
+				\vspace{1.5em}
+			\item<4-> \textcolor<6->{gray}{Regular Internet (e.g. FTTB, FTTH) and power grid connections\\
+				\vspace{.5em}
+				\visible<5->{{\footnotesize (can be unavailable randomly)}}}
+				\vspace{1.5em}
+			\item<6-> \textbf{Geographical redundancy} (multi-site replication)
+		\end{itemize}
+	}
+	\only<2>{
+		\begin{center}
+			\includegraphics[width=.8\linewidth]{../assets/neptune.jpg}
+		\end{center}
+	}
+	\only<7>{
+		\begin{center}
+			\includegraphics[width=.8\linewidth]{../assets/inframap_jdll2023.pdf}
+		\end{center}
+	}
+\end{frame}
+
+\begin{frame}
+	\frametitle{How to make this happen}
+	\begin{center}
+		\only<1>{\includegraphics[width=.8\linewidth]{../assets/intro/slide1.png}}%
+		\only<2>{\includegraphics[width=.8\linewidth]{../assets/intro/slide2.png}}%
+		\only<3>{\includegraphics[width=.8\linewidth]{../assets/intro/slide3.png}}%
+	\end{center}
+\end{frame}
+
+\begin{frame}
+	\frametitle{Distributed file systems are slow}
+	File systems are complex, for example:
+	\vspace{1em}
+	\begin{itemize}
+		\item Concurrent modification by several processes
+			\vspace{1em}
+		\item Folder hierarchies
+			\vspace{1em}
+		\item Other requirements of the POSIX spec (e.g.~locks)
+	\end{itemize}
+	\vspace{1em}
+	Coordination in a distributed system is costly
+
+	\vspace{1em}
+	Costs explode with commodity hardware / Internet connections\\
+	{\small (we experienced this!)}
+\end{frame}
+
+\begin{frame}
+	\frametitle{A simpler solution: object storage}
+	Only two operations:
+	\vspace{1em}
+	\begin{itemize}
+		\item Put an object at a key
+			\vspace{1em}
+		\item Retrieve an object from its key
+	\end{itemize}
+	\vspace{1em}
+	{\footnotesize (and a few others)}
+
+	\vspace{1em}
+	Sufficient for many applications!
+\end{frame}
+
+\begin{frame}
+	\frametitle{A simpler solution: object storage}
+		\begin{center}
+			\includegraphics[height=6em]{../assets/logos/Amazon-S3.jpg}
+			\hspace{3em}
+			\visible<2->{\includegraphics[height=5em]{../assets/logos/minio.png}}
+			\hspace{3em}
+			\visible<3>{\includegraphics[height=6em]{../../logo/garage_hires_crop.png}}
+		\end{center}
+		\vspace{1em}
+	S3: a de-facto standard, many compatible applications
+
+	\vspace{1em}
+	\visible<2->{MinIO is self-hostable but not suited for geo-distributed deployments}
+
+	\vspace{1em}
+	\visible<3->{\textbf{Garage is a self-hosted drop-in replacement for the Amazon S3 object store}}
+\end{frame}
+
+% --------- BASED ON CRDTS ----------
+
+\section{Principle 1: based on CRDTs}
+
+\begin{frame}
+	\frametitle{CRDTs / weak consistency instead of consensus}
+
+	\underline{Internally, Garage uses only CRDTs} (conflict-free replicated data types)
+
+	\vspace{2em}
+	Why not Raft, Paxos, ...? Issues of consensus algorithms:
+
+	\vspace{1em}
+	\begin{itemize}
+		\item<2-> \textbf{Software complexity}
+			\vspace{1em}
+		\item<3-> \textbf{Performance issues:}
+			\vspace{.5em}
+			\begin{itemize}
+				\item<4-> The leader is a \textbf{bottleneck} for all requests\\
+					\vspace{.5em}
+				\item<5-> \textbf{Sensitive to higher latency} between nodes
+					\vspace{.5em}
+				\item<6-> \textbf{Takes time to reconverge} when disrupted (e.g. node going down)
+			\end{itemize}
+	\end{itemize}
+\end{frame}
+
+\begin{frame}
+	\frametitle{The data model of object storage}
+	Object storage is basically a \textbf{key-value store}:
+	\vspace{.5em}
+
+	{\scriptsize
+		\begin{center}
+		\begin{tabular}{|l|p{7cm}|}
+			\hline
+			\textbf{Key: file path + name} & \textbf{Value: file data + metadata} \\
+			\hline
+			\hline
+			\texttt{index.html} &
+				\texttt{Content-Type: text/html; charset=utf-8} \newline
+				\texttt{Content-Length: 24929} \newline
+				\texttt{<binary blob>} \\ 
+			\hline
+			\texttt{img/logo.svg} &
+				\texttt{Content-Type: text/svg+xml} \newline
+				\texttt{Content-Length: 13429} \newline
+				\texttt{<binary blob>} \\ 
+			\hline
+			\texttt{download/index.html} &
+				\texttt{Content-Type: text/html; charset=utf-8} \newline
+				\texttt{Content-Length: 26563} \newline
+				\texttt{<binary blob>} \\ 
+			\hline
+		\end{tabular}
+		\end{center}
+		}
+
+	\vspace{.5em}
+	\begin{itemize}
+		\item<2-> Maps well to CRDT data types
+		\item<3> Read-after-write consistency with quorums
+	\end{itemize}
+\end{frame}
+
+
+\begin{frame}
+	\frametitle{Performance gains in practice}
+	\begin{center}
+		\includegraphics[width=.8\linewidth]{../assets/perf/endpoint_latency_0.7_0.8_minio.png}
+	\end{center}
+\end{frame}
+
+% --------- GEO-DISTRIBUTED MODEL ----------
+
+\section{Principle 2: geo-distributed data model}
+
+\begin{frame}
+	\frametitle{Key-value stores, upgraded: the Dynamo model}
+	\textbf{Two keys:}
+	\begin{itemize}
+		\item Partition key: used to divide data into partitions {\small (a.k.a.~shards)}
+		\item Sort key: used to identify items inside a partition
+	\end{itemize}
+
+	\vspace{1em}
+
+	\begin{center}
+	\begin{tabular}{|l|l|p{3cm}|}
+		\hline
+		\textbf{Partition key: bucket} & \textbf{Sort key: filename} & \textbf{Value} \\
+		\hline
+		\hline
+		\texttt{website} & \texttt{index.html} & (file data) \\
+		\hline
+		\texttt{website} & \texttt{img/logo.svg} & (file data) \\
+		\hline
+		\texttt{website} & \texttt{download/index.html} & (file data) \\
+		\hline
+		\hline
+		\texttt{backup} & \texttt{borg/index.2822} & (file data) \\
+		\hline
+		\texttt{backup} & \texttt{borg/data/2/2329} & (file data) \\
+		\hline
+		\texttt{backup} & \texttt{borg/data/2/2680} & (file data) \\
+		\hline
+		\hline
+		\texttt{private} & \texttt{qq3a2nbe1qjq0ebbvo6ocsp6co} & (file data) \\
+		\hline
+	\end{tabular}
+	\end{center}
+\end{frame}
+
+
+\begin{frame}
+	\frametitle{Layout computation}
+	\begin{overprint}
+		\onslide<1>
+		\begin{center}
+			\includegraphics[width=\linewidth, trim=0 0 0 -4cm]{../assets/screenshots/garage_status_0.9_prod_zonehl.png}
+		\end{center}
+		\onslide<2>
+		\begin{center}
+			\includegraphics[width=.7\linewidth]{../assets/map.png}
+		\end{center}
+	\end{overprint}
+	\vspace{1em}
+	Garage stores replicas on different zones when possible
+\end{frame}
+
+\begin{frame}
+	\frametitle{What a "layout" is}
+		\textbf{A layout is a precomputed index table:}
+		\vspace{1em}
+
+		{\footnotesize
+			\begin{center}
+			\begin{tabular}{|l|l|l|l|}
+				\hline
+				\textbf{Partition} & \textbf{Node 1} & \textbf{Node 2} & \textbf{Node 3} \\
+				\hline
+				\hline
+				Partition 0 & df-ymk (bespin) & Abricot (scorpio) & Courgette (neptune) \\
+				\hline
+				Partition 1 & Ananas (scorpio) & Courgette (neptune) & df-ykl (bespin) \\
+				\hline
+				Partition 2 & df-ymf (bespin) & Celeri (neptune) & Abricot (scorpio) \\
+				\hline
+				\hspace{1em}$\vdots$ & \hspace{1em}$\vdots$ & \hspace{1em}$\vdots$ & \hspace{1em}$\vdots$ \\
+				\hline
+				Partition 255 & Concombre (neptune) & df-ykl (bespin) & Abricot (scorpio) \\
+				\hline
+			\end{tabular}
+			\end{center}
+		}
+
+	\vspace{2em}
+	\visible<2->{
+		The index table is built centrally using an optimal algorithm,\\
+		then propagated to all nodes
+	}
+
+	\vspace{1em}
+	\visible<3->{
+		\footnotesize
+		Oulamara, M., \& Auvolat, A. (2023). \emph{An algorithm for geo-distributed and redundant storage in Garage}.\\ arXiv preprint arXiv:2302.13798.
+	}
+\end{frame}
+
+
+\begin{frame}
+	\frametitle{The relationship between \emph{partition} and \emph{partition key}}
+	\begin{center}
+	\begin{tabular}{|l|l|l|l|}
+		\hline
+		\textbf{Partition key} & \textbf{Partition} & \textbf{Sort key} & \textbf{Value} \\
+		\hline
+		\hline
+		\texttt{website} & Partition 12 & \texttt{index.html} & (file data) \\
+		\hline
+		\texttt{website} & Partition 12 & \texttt{img/logo.svg} & (file data) \\
+		\hline
+		\texttt{website} & Partition 12 &\texttt{download/index.html} & (file data) \\
+		\hline
+		\hline
+		\texttt{backup} & Partition 42 & \texttt{borg/index.2822} & (file data) \\
+		\hline
+		\texttt{backup} & Partition 42 & \texttt{borg/data/2/2329} & (file data) \\
+		\hline
+		\texttt{backup} & Partition 42 & \texttt{borg/data/2/2680} & (file data) \\
+		\hline
+		\hline
+		\texttt{private} & Partition 42 & \texttt{qq3a2nbe1qjq0ebbvo6ocsp6co} & (file data) \\
+		\hline
+	\end{tabular}
+	\end{center}
+	\vspace{1em}
+	\textbf{To read or write an item:} hash partition key
+	\\ \hspace{5cm} $\to$ determine partition number (first 8 bits)
+	\\ \hspace{5cm} $\to$ find associated nodes
+\end{frame}
+
+\begin{frame}
+	\frametitle{Garage's internal data structures}
+	\centering
+	\includegraphics[width=.75\columnwidth]{../assets/garage_tables.pdf}
+\end{frame}
+
+% ---------- OPERATING GARAGE ---------
+
+\section{Operating Garage clusters}
+
+\begin{frame}
+	\frametitle{Operating Garage}
+	\begin{center}
+		\only<1-2>{
+			\includegraphics[width=.9\linewidth]{../assets/screenshots/garage_status_0.10.png}
+			\\\vspace{1em}
+			\visible<2>{\includegraphics[width=.9\linewidth]{../assets/screenshots/garage_status_unhealthy_0.10.png}}
+		}
+	\end{center}
+\end{frame}
+
+\begin{frame}
+	\frametitle{Background synchronization}
+	\begin{center}
+		\includegraphics[width=.6\linewidth]{../assets/garage_sync.drawio.pdf}
+	\end{center}
+\end{frame}
+
+\begin{frame}
+	\frametitle{Digging deeper}
+	\begin{center}
+		\only<1>{\includegraphics[width=.9\linewidth]{../assets/screenshots/garage_stats_0.10.png}}
+		\only<2>{\includegraphics[width=.5\linewidth]{../assets/screenshots/garage_worker_list_0.10.png}}
+		\only<3>{\includegraphics[width=.6\linewidth]{../assets/screenshots/garage_worker_param_0.10.png}}
+	\end{center}
+\end{frame}
+
+\begin{frame}
+	\frametitle{Monitoring with Prometheus + Grafana}
+	\begin{center}
+		\includegraphics[width=.9\linewidth]{../assets/screenshots/grafana_dashboard.png}
+	\end{center}
+\end{frame}
+
+\begin{frame}
+	\frametitle{Debugging with traces}
+	\begin{center}
+		\includegraphics[width=.8\linewidth]{../assets/screenshots/jaeger_listobjects.png}
+	\end{center}
+\end{frame}
+
+% ---------- SCALING GARAGE ---------
+
+\section{Scaling Garage clusters}
+
+\begin{frame}
+	\frametitle{Potential limitations and bottlenecks}
+	\begin{itemize}
+		\item Global:
+			\begin{itemize}
+				\item Max. $\sim$100 nodes per cluster (excluding gateways)
+			\end{itemize}
+			\vspace{1em}
+		\item Metadata:
+			\begin{itemize}
+				\item One big bucket = bottleneck, object list on 3 nodes only
+			\end{itemize}
+			\vspace{1em}
+		\item Block manager:
+			\begin{itemize}
+				\item Lots of small files on disk
+				\item Processing the resync queue can be slow
+			\end{itemize}
+	\end{itemize}
+\end{frame}
+
+\begin{frame}
+	\frametitle{Deployment advice for very large clusters}
+	\begin{itemize}
+		\item Metadata storage:
+			\begin{itemize}
+				\item ZFS mirror (x2) on fast NVMe
+				\item Use LMDB storage engine
+			\end{itemize}
+			\vspace{.5em}
+		\item Data block storage:
+			\begin{itemize}
+				\item Use Garage's native multi-HDD support
+				\item XFS on individual drives
+				\item Increase block size (1MB $\to$ 10MB, requires more RAM and good networking)
+				\item Tune \texttt{resync-tranquility} and \texttt{resync-worker-count} dynamically
+			\end{itemize}
+			\vspace{.5em}
+		\item Other :
+			\begin{itemize}
+				\item Split data over several buckets
+				\item Use less than 100 storage nodes
+				\item Use gateway nodes
+			\end{itemize}
+			\vspace{.5em}
+	\end{itemize}
+	Our deployments: $< 10$ TB. Some people have done more!
+\end{frame}
+
+
+% ======================================== END
+% ======================================== END
+% ======================================== END
+
+\begin{frame}
+	\frametitle{Where to find us}
+	\begin{center}
+			\includegraphics[width=.25\linewidth]{../../logo/garage_hires.png}\\
+			\vspace{-1em}
+		\url{https://garagehq.deuxfleurs.fr/}\\
+		\url{mailto:garagehq@deuxfleurs.fr}\\
+		\texttt{\#garage:deuxfleurs.fr} on Matrix
+
+		\vspace{1.5em}
+			\includegraphics[width=.06\linewidth]{../assets/logos/rust_logo.png}
+			\includegraphics[width=.13\linewidth]{../assets/logos/AGPLv3_Logo.png}
+	\end{center}
+\end{frame}
+
+\end{document}
+
+%% vim: set ts=4 sw=4 tw=0 noet spelllang=en :

k2v_test.py

@@ -1,158 +0,0 @@
-#!/usr/bin/env python
-
-import os
-import requests
-from datetime import datetime
-
-# let's talk to our AWS Elasticsearch cluster
-#from requests_aws4auth import AWS4Auth
-#auth = AWS4Auth('GK31c2f218a2e44f485b94239e',
-#                       'b892c0665f0ada8a4755dae98baa3b133590e11dae3bcc1f9d769d67f16c3835',
-#                       'us-east-1',
-#                       's3')
-
-from aws_requests_auth.aws_auth import AWSRequestsAuth
-auth = AWSRequestsAuth(aws_access_key='GK31c2f218a2e44f485b94239e',
-        aws_secret_access_key='b892c0665f0ada8a4755dae98baa3b133590e11dae3bcc1f9d769d67f16c3835',
-        aws_host='localhost:3812',
-        aws_region='us-east-1',
-        aws_service='k2v')
-
-
-print("-- ReadIndex")
-response = requests.get('http://localhost:3812/alex',
-                        auth=auth)
-print(response.headers)
-print(response.text)
-
-
-sort_keys = ["a", "b", "c", "d"]
-
-for sk in sort_keys:
-    print("-- (%s) Put initial (no CT)"%sk)
-    response = requests.put('http://localhost:3812/alex/root?sort_key=%s'%sk,
-                            auth=auth,
-                            data='{}: Hello, world!'.format(datetime.timestamp(datetime.now())))
-    print(response.headers)
-    print(response.text)
-
-    print("-- Get")
-    response = requests.get('http://localhost:3812/alex/root?sort_key=%s'%sk,
-                            auth=auth)
-    print(response.headers)
-    print(response.text)
-    ct = response.headers["x-garage-causality-token"]
-
-    print("-- ReadIndex")
-    response = requests.get('http://localhost:3812/alex',
-                            auth=auth)
-    print(response.headers)
-    print(response.text)
-
-    print("-- Put with CT")
-    response = requests.put('http://localhost:3812/alex/root?sort_key=%s'%sk,
-                            auth=auth,
-                            headers={'x-garage-causality-token': ct},
-                            data='{}: Good bye, world!'.format(datetime.timestamp(datetime.now())))
-    print(response.headers)
-    print(response.text)
-
-    print("-- Get")
-    response = requests.get('http://localhost:3812/alex/root?sort_key=%s'%sk,
-                            auth=auth)
-    print(response.headers)
-    print(response.text)
-
-    print("-- Put again with same CT (concurrent)")
-    response = requests.put('http://localhost:3812/alex/root?sort_key=%s'%sk,
-                            auth=auth,
-                            headers={'x-garage-causality-token': ct},
-                            data='{}: Concurrent value, oops'.format(datetime.timestamp(datetime.now())))
-    print(response.headers)
-    print(response.text)
-
-for sk in sort_keys:
-    print("-- (%s) Get"%sk)
-    response = requests.get('http://localhost:3812/alex/root?sort_key=%s'%sk,
-                            auth=auth)
-    print(response.headers)
-    print(response.text)
-    ct = response.headers["x-garage-causality-token"]
-
-    print("-- Delete")
-    response = requests.delete('http://localhost:3812/alex/root?sort_key=%s'%sk,
-                            headers={'x-garage-causality-token': ct},
-                            auth=auth)
-    print(response.headers)
-    print(response.text)
-
-print("-- ReadIndex")
-response = requests.get('http://localhost:3812/alex',
-                        auth=auth)
-print(response.headers)
-print(response.text)
-
-print("-- InsertBatch")
-response = requests.post('http://localhost:3812/alex',
-                        auth=auth,
-                        data='''
-[
-    {"pk": "root", "sk": "a", "ct": null, "v": "aW5pdGlhbCB0ZXN0Cg=="},
-    {"pk": "root", "sk": "b", "ct": null, "v": "aW5pdGlhbCB0ZXN1Cg=="},
-    {"pk": "root", "sk": "c", "ct": null, "v": "aW5pdGlhbCB0ZXN2Cg=="}
-]
-''')
-print(response.headers)
-print(response.text)
-
-print("-- ReadIndex")
-response = requests.get('http://localhost:3812/alex',
-                        auth=auth)
-print(response.headers)
-print(response.text)
-
-for sk in sort_keys:
-    print("-- (%s) Get"%sk)
-    response = requests.get('http://localhost:3812/alex/root?sort_key=%s'%sk,
-                            auth=auth)
-    print(response.headers)
-    print(response.text)
-    ct = response.headers["x-garage-causality-token"]
-
-print("-- ReadBatch")
-response = requests.post('http://localhost:3812/alex?search',
-                        auth=auth,
-                        data='''
-[
-    {"partitionKey": "root"},
-    {"partitionKey": "root", "tombstones": true},
-    {"partitionKey": "root", "tombstones": true, "limit": 2},
-    {"partitionKey": "root", "start": "c", "singleItem": true},
-    {"partitionKey": "root", "start": "b", "end": "d", "tombstones": true}
-]
-''')
-print(response.headers)
-print(response.text)
-
-
-print("-- DeleteBatch")
-response = requests.post('http://localhost:3812/alex?delete',
-                        auth=auth,
-                        data='''
-[
-    {"partitionKey": "root", "start": "b", "end": "c"}
-]
-''')
-print(response.headers)
-print(response.text)
-
-print("-- ReadBatch")
-response = requests.post('http://localhost:3812/alex?search',
-                        auth=auth,
-                        data='''
-[
-    {"partitionKey": "root"}
-]
-''')
-print(response.headers)
-print(response.text)

nix/compile.nix

@@ -168,13 +168,12 @@ let
   rootFeatures = if features != null then
     features
   else
-    ([ "garage/bundled-libs" "garage/sled" "garage/lmdb" "garage/k2v" ] ++ (if release then [
+    ([ "garage/bundled-libs" "garage/sled" "garage/lmdb" "garage/sqlite" "garage/k2v" ] ++ (if release then [
       "garage/consul-discovery"
       "garage/kubernetes-discovery"
       "garage/metrics"
       "garage/telemetry-otlp"
-      "garage/lmdb"
+      "garage/syslog"
-      "garage/sqlite"
     ] else
       [ ]));
 

script/helm/garage/Chart.yaml

@@ -15,10 +15,10 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.4.1
+version: 0.4.2
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to
 # follow Semantic Versioning. They should reflect the version the application is using.
 # It is recommended to use it with quotes.
-appVersion: "v0.9.2"
+appVersion: "v0.9.4"

script/jepsen.garage/Vagrantfile

@@ -30,11 +30,11 @@ Vagrant.configure("2") do |config|
   config.vm.define "n6" do |config| vm(config, "n6", "192.168.56.26") end
   config.vm.define "n7" do |config| vm(config, "n7", "192.168.56.27") end
 
-  config.vm.define "n8" do |config| vm(config, "n8", "192.168.56.28") end
+  #config.vm.define "n8" do |config| vm(config, "n8", "192.168.56.28") end
-  config.vm.define "n9" do |config| vm(config, "n9", "192.168.56.29") end
+  #config.vm.define "n9" do |config| vm(config, "n9", "192.168.56.29") end
-  config.vm.define "n10" do |config| vm(config, "n10", "192.168.56.30") end
+  #config.vm.define "n10" do |config| vm(config, "n10", "192.168.56.30") end
-  config.vm.define "n11" do |config| vm(config, "n11", "192.168.56.31") end
+  #config.vm.define "n11" do |config| vm(config, "n11", "192.168.56.31") end
-  config.vm.define "n12" do |config| vm(config, "n12", "192.168.56.32") end
+  #config.vm.define "n12" do |config| vm(config, "n12", "192.168.56.32") end
-  config.vm.define "n13" do |config| vm(config, "n13", "192.168.56.33") end
+  #config.vm.define "n13" do |config| vm(config, "n13", "192.168.56.33") end
-  config.vm.define "n14" do |config| vm(config, "n14", "192.168.56.34") end
+  #config.vm.define "n14" do |config| vm(config, "n14", "192.168.56.34") end
 end

script/jepsen.garage/all_tests_1.sh

@@ -3,11 +3,10 @@
 set -x
 
 #for ppatch in task3c task3a tsfix2; do
-for ppatch in tsfix2; do
+for ppatch in v093 v1rc1; do
 	#for psc in c cp cdp r pr cpr dpr; do
-	for psc in cdp r pr cpr dpr; do
+	for ptsk in reg2 set2; do
-		#for ptsk in reg2 set1 set2; do
+		for psc in c cp cdp r pr cpr dpr; do
-		for ptsk in set1; do
 			for irun in $(seq 10); do
 				lein run test --nodes-file nodes.vagrant \
 					--time-limit 60 --rate 100  --concurrency 100 --ops-per-key 100 \

script/jepsen.garage/src/jepsen/garage.clj

@@ -38,7 +38,9 @@
    "tsfix2" "c82d91c6bccf307186332b6c5c6fc0b128b1b2b1"
    "task3a" "707442f5de416fdbed4681a33b739f0a787b7834"
    "task3b" "431b28e0cfdc9cac6c649193cf602108a8b02997"
-   "task3c" "0041b013a473e3ae72f50209d8f79db75a72848b"})
+   "task3c" "0041b013a473e3ae72f50209d8f79db75a72848b"
+   "v093" "v0.9.3"
+   "v1rc1" "v1.0.0-rc1"})
 
 (def cli-opts
   "Additional command line options."

script/jepsen.garage/src/jepsen/garage/daemon.clj

@@ -43,7 +43,7 @@
              "rpc_bind_addr = \"0.0.0.0:3901\"\n"
              "rpc_public_addr = \"" node ":3901\"\n"
              "db_engine = \"lmdb\"\n"
-             "replication_mode = \"2\"\n"
+             "replication_mode = \"3\"\n"
              "data_dir = \"" data-dir "\"\n"
              "metadata_dir = \"" meta-dir "\"\n"
              "[s3_api]\n"

src/api/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "garage_api"
-version = "0.9.2"
+version = "0.9.4"
 authors = ["Alex Auvolat <alex@adnab.me>"]
 edition = "2018"
 license = "AGPL-3.0"

src/api/helpers.rs

@@ -1,4 +1,5 @@
 use std::convert::Infallible;
+use std::sync::Arc;
 
 use futures::{Stream, StreamExt, TryStreamExt};
 
@@ -10,6 +11,10 @@ use hyper::{
 use idna::domain_to_unicode;
 use serde::{Deserialize, Serialize};
 
+use garage_model::bucket_table::BucketParams;
+use garage_model::garage::Garage;
+use garage_model::key_table::Key;
+use garage_util::data::Uuid;
 use garage_util::error::Error as GarageError;
 
 use crate::common_error::{CommonError as Error, *};
@@ -27,6 +32,15 @@ pub enum Authorization {
 	Owner,
 }
 
+/// The values which are known for each request related to a bucket
+pub struct ReqCtx {
+	pub garage: Arc<Garage>,
+	pub bucket_id: Uuid,
+	pub bucket_name: String,
+	pub bucket_params: BucketParams,
+	pub api_key: Key,
+}
+
 /// Host to bucket
 ///
 /// Convert a host, like "bucket.garage-site.tld" to the corresponding bucket "bucket",

src/api/k2v/api_server.rs

@@ -95,6 +95,7 @@ impl ApiHandler for K2VApiServer {
 			.bucket_helper()
 			.get_existing_bucket(bucket_id)
 			.await?;
+		let bucket_params = bucket.state.into_option().unwrap();
 
 		let allowed = match endpoint.authorization_type() {
 			Authorization::Read => api_key.allow_read(&bucket_id),
@@ -112,40 +113,42 @@ impl ApiHandler for K2VApiServer {
 		// are always preflighted, i.e. the browser should make
 		// an OPTIONS call before to check it is allowed
 		let matching_cors_rule = match *req.method() {
-			Method::GET | Method::HEAD | Method::POST => find_matching_cors_rule(&bucket, &req)
+			Method::GET | Method::HEAD | Method::POST => {
-				.ok_or_internal_error("Error looking up CORS rule")?,
+				find_matching_cors_rule(&bucket_params, &req)
+					.ok_or_internal_error("Error looking up CORS rule")?
+					.cloned()
+			}
 			_ => None,
 		};
 
+		let ctx = ReqCtx {
+			garage,
+			bucket_id,
+			bucket_name,
+			bucket_params,
+			api_key,
+		};
+
 		let resp = match endpoint {
 			Endpoint::DeleteItem {
 				partition_key,
 				sort_key,
-			} => handle_delete_item(garage, req, bucket_id, &partition_key, &sort_key).await,
+			} => handle_delete_item(ctx, req, &partition_key, &sort_key).await,
 			Endpoint::InsertItem {
 				partition_key,
 				sort_key,
-			} => handle_insert_item(garage, req, bucket_id, &partition_key, &sort_key).await,
+			} => handle_insert_item(ctx, req, &partition_key, &sort_key).await,
 			Endpoint::ReadItem {
 				partition_key,
 				sort_key,
-			} => handle_read_item(garage, &req, bucket_id, &partition_key, &sort_key).await,
+			} => handle_read_item(ctx, &req, &partition_key, &sort_key).await,
 			Endpoint::PollItem {
 				partition_key,
 				sort_key,
 				causality_token,
 				timeout,
 			} => {
-				handle_poll_item(
+				handle_poll_item(ctx, &req, partition_key, sort_key, causality_token, timeout).await
-					garage,
-					&req,
-					bucket_id,
-					partition_key,
-					sort_key,
-					causality_token,
-					timeout,
-				)
-				.await
 			}
 			Endpoint::ReadIndex {
 				prefix,
@@ -153,12 +156,12 @@ impl ApiHandler for K2VApiServer {
 				end,
 				limit,
 				reverse,
-			} => handle_read_index(garage, bucket_id, prefix, start, end, limit, reverse).await,
+			} => handle_read_index(ctx, prefix, start, end, limit, reverse).await,
-			Endpoint::InsertBatch {} => handle_insert_batch(garage, bucket_id, req).await,
+			Endpoint::InsertBatch {} => handle_insert_batch(ctx, req).await,
-			Endpoint::ReadBatch {} => handle_read_batch(garage, bucket_id, req).await,
+			Endpoint::ReadBatch {} => handle_read_batch(ctx, req).await,
-			Endpoint::DeleteBatch {} => handle_delete_batch(garage, bucket_id, req).await,
+			Endpoint::DeleteBatch {} => handle_delete_batch(ctx, req).await,
 			Endpoint::PollRange { partition_key } => {
-				handle_poll_range(garage, bucket_id, &partition_key, req).await
+				handle_poll_range(ctx, &partition_key, req).await
 			}
 			Endpoint::Options => unreachable!(),
 		};
@@ -167,7 +170,7 @@ impl ApiHandler for K2VApiServer {
 		// add the corresponding CORS headers to the response
 		let mut resp_ok = resp?;
 		if let Some(rule) = matching_cors_rule {
-			add_cors_headers(&mut resp_ok, rule)
+			add_cors_headers(&mut resp_ok, &rule)
 				.ok_or_internal_error("Invalid bucket CORS configuration")?;
 		}
 

src/api/k2v/batch.rs

@@ -1,14 +1,9 @@
-use std::sync::Arc;
-
 use base64::prelude::*;
 use hyper::{Request, Response, StatusCode};
 use serde::{Deserialize, Serialize};
 
-use garage_util::data::*;
-
 use garage_table::{EnumerationOrder, TableSchema};
 
-use garage_model::garage::Garage;
 use garage_model::k2v::causality::*;
 use garage_model::k2v::item_table::*;
 
@@ -18,10 +13,12 @@ use crate::k2v::error::*;
 use crate::k2v::range::read_range;
 
 pub async fn handle_insert_batch(
-	garage: Arc<Garage>,
+	ctx: ReqCtx,
-	bucket_id: Uuid,
 	req: Request<ReqBody>,
 ) -> Result<Response<ResBody>, Error> {
+	let ReqCtx {
+		garage, bucket_id, ..
+	} = &ctx;
 	let items = parse_json_body::<Vec<InsertBatchItem>, _, Error>(req).await?;
 
 	let mut items2 = vec![];
@@ -38,7 +35,7 @@ pub async fn handle_insert_batch(
 		items2.push((it.pk, it.sk, ct, v));
 	}
 
-	garage.k2v.rpc.insert_batch(bucket_id, items2).await?;
+	garage.k2v.rpc.insert_batch(*bucket_id, items2).await?;
 
 	Ok(Response::builder()
 		.status(StatusCode::NO_CONTENT)
@@ -46,8 +43,7 @@ pub async fn handle_insert_batch(
 }
 
 pub async fn handle_read_batch(
-	garage: Arc<Garage>,
+	ctx: ReqCtx,
-	bucket_id: Uuid,
 	req: Request<ReqBody>,
 ) -> Result<Response<ResBody>, Error> {
 	let queries = parse_json_body::<Vec<ReadBatchQuery>, _, Error>(req).await?;
@@ -55,7 +51,7 @@ pub async fn handle_read_batch(
 	let resp_results = futures::future::join_all(
 		queries
 			.into_iter()
-			.map(|q| handle_read_batch_query(&garage, bucket_id, q)),
+			.map(|q| handle_read_batch_query(&ctx, q)),
 	)
 	.await;
 
@@ -68,12 +64,15 @@ pub async fn handle_read_batch(
 }
 
 async fn handle_read_batch_query(
-	garage: &Arc<Garage>,
+	ctx: &ReqCtx,
-	bucket_id: Uuid,
 	query: ReadBatchQuery,
 ) -> Result<ReadBatchResponse, Error> {
+	let ReqCtx {
+		garage, bucket_id, ..
+	} = ctx;
+
 	let partition = K2VItemPartition {
-		bucket_id,
+		bucket_id: *bucket_id,
 		partition_key: query.partition_key.clone(),
 	};
 
@@ -138,8 +137,7 @@ async fn handle_read_batch_query(
 }
 
 pub async fn handle_delete_batch(
-	garage: Arc<Garage>,
+	ctx: ReqCtx,
-	bucket_id: Uuid,
 	req: Request<ReqBody>,
 ) -> Result<Response<ResBody>, Error> {
 	let queries = parse_json_body::<Vec<DeleteBatchQuery>, _, Error>(req).await?;
@@ -147,7 +145,7 @@ pub async fn handle_delete_batch(
 	let resp_results = futures::future::join_all(
 		queries
 			.into_iter()
-			.map(|q| handle_delete_batch_query(&garage, bucket_id, q)),
+			.map(|q| handle_delete_batch_query(&ctx, q)),
 	)
 	.await;
 
@@ -160,12 +158,15 @@ pub async fn handle_delete_batch(
 }
 
 async fn handle_delete_batch_query(
-	garage: &Arc<Garage>,
+	ctx: &ReqCtx,
-	bucket_id: Uuid,
 	query: DeleteBatchQuery,
 ) -> Result<DeleteBatchResponse, Error> {
+	let ReqCtx {
+		garage, bucket_id, ..
+	} = &ctx;
+
 	let partition = K2VItemPartition {
-		bucket_id,
+		bucket_id: *bucket_id,
 		partition_key: query.partition_key.clone(),
 	};
 
@@ -195,7 +196,7 @@ async fn handle_delete_batch_query(
 					.k2v
 					.rpc
 					.insert(
-						bucket_id,
+						*bucket_id,
 						i.partition.partition_key,
 						i.sort_key,
 						Some(cc),
@@ -235,7 +236,7 @@ async fn handle_delete_batch_query(
 			.collect::<Vec<_>>();
 		let n = items.len();
 
-		garage.k2v.rpc.insert_batch(bucket_id, items).await?;
+		garage.k2v.rpc.insert_batch(*bucket_id, items).await?;
 
 		n
 	};
@@ -251,11 +252,13 @@ async fn handle_delete_batch_query(
 }
 
 pub(crate) async fn handle_poll_range(
-	garage: Arc<Garage>,
+	ctx: ReqCtx,
-	bucket_id: Uuid,
 	partition_key: &str,
 	req: Request<ReqBody>,
 ) -> Result<Response<ResBody>, Error> {
+	let ReqCtx {
+		garage, bucket_id, ..
+	} = ctx;
 	use garage_model::k2v::sub::PollRange;
 
 	let query = parse_json_body::<PollRangeQuery, _, Error>(req).await?;

src/api/k2v/index.rs

@@ -3,12 +3,9 @@ use std::sync::Arc;
 use hyper::Response;
 use serde::Serialize;
 
-use garage_util::data::*;
-
 use garage_rpc::ring::Ring;
 use garage_table::util::*;
 
-use garage_model::garage::Garage;
 use garage_model::k2v::item_table::{BYTES, CONFLICTS, ENTRIES, VALUES};
 
 use crate::helpers::*;
@@ -17,14 +14,17 @@ use crate::k2v::error::*;
 use crate::k2v::range::read_range;
 
 pub async fn handle_read_index(
-	garage: Arc<Garage>,
+	ctx: ReqCtx,
-	bucket_id: Uuid,
 	prefix: Option<String>,
 	start: Option<String>,
 	end: Option<String>,
 	limit: Option<u64>,
 	reverse: Option<bool>,
 ) -> Result<Response<ResBody>, Error> {
+	let ReqCtx {
+		garage, bucket_id, ..
+	} = &ctx;
+
 	let reverse = reverse.unwrap_or(false);
 
 	let ring: Arc<Ring> = garage.system.ring.borrow().clone();

src/api/k2v/item.rs

@@ -1,13 +1,8 @@
-use std::sync::Arc;
-
 use base64::prelude::*;
 use http::header;
 
 use hyper::{Request, Response, StatusCode};
 
-use garage_util::data::*;
-
-use garage_model::garage::Garage;
 use garage_model::k2v::causality::*;
 use garage_model::k2v::item_table::*;
 
@@ -100,12 +95,15 @@ impl ReturnFormat {
 /// Handle ReadItem request
 #[allow(clippy::ptr_arg)]
 pub async fn handle_read_item(
-	garage: Arc<Garage>,
+	ctx: ReqCtx,
 	req: &Request<ReqBody>,
-	bucket_id: Uuid,
 	partition_key: &str,
 	sort_key: &String,
 ) -> Result<Response<ResBody>, Error> {
+	let ReqCtx {
+		garage, bucket_id, ..
+	} = &ctx;
+
 	let format = ReturnFormat::from(req)?;
 
 	let item = garage
@@ -113,7 +111,7 @@ pub async fn handle_read_item(
 		.item_table
 		.get(
 			&K2VItemPartition {
-				bucket_id,
+				bucket_id: *bucket_id,
 				partition_key: partition_key.to_string(),
 			},
 			sort_key,
@@ -125,12 +123,14 @@ pub async fn handle_read_item(
 }
 
 pub async fn handle_insert_item(
-	garage: Arc<Garage>,
+	ctx: ReqCtx,
 	req: Request<ReqBody>,
-	bucket_id: Uuid,
 	partition_key: &str,
 	sort_key: &str,
 ) -> Result<Response<ResBody>, Error> {
+	let ReqCtx {
+		garage, bucket_id, ..
+	} = &ctx;
 	let causal_context = req
 		.headers()
 		.get(X_GARAGE_CAUSALITY_TOKEN)
@@ -149,7 +149,7 @@ pub async fn handle_insert_item(
 		.k2v
 		.rpc
 		.insert(
-			bucket_id,
+			*bucket_id,
 			partition_key.to_string(),
 			sort_key.to_string(),
 			causal_context,
@@ -163,12 +163,14 @@ pub async fn handle_insert_item(
 }
 
 pub async fn handle_delete_item(
-	garage: Arc<Garage>,
+	ctx: ReqCtx,
 	req: Request<ReqBody>,
-	bucket_id: Uuid,
 	partition_key: &str,
 	sort_key: &str,
 ) -> Result<Response<ResBody>, Error> {
+	let ReqCtx {
+		garage, bucket_id, ..
+	} = &ctx;
 	let causal_context = req
 		.headers()
 		.get(X_GARAGE_CAUSALITY_TOKEN)
@@ -183,7 +185,7 @@ pub async fn handle_delete_item(
 		.k2v
 		.rpc
 		.insert(
-			bucket_id,
+			*bucket_id,
 			partition_key.to_string(),
 			sort_key.to_string(),
 			causal_context,
@@ -199,14 +201,16 @@ pub async fn handle_delete_item(
 /// Handle ReadItem request
 #[allow(clippy::ptr_arg)]
 pub async fn handle_poll_item(
-	garage: Arc<Garage>,
+	ctx: ReqCtx,
 	req: &Request<ReqBody>,
-	bucket_id: Uuid,
 	partition_key: String,
 	sort_key: String,
 	causality_token: String,
 	timeout_secs: Option<u64>,
 ) -> Result<Response<ResBody>, Error> {
+	let ReqCtx {
+		garage, bucket_id, ..
+	} = &ctx;
 	let format = ReturnFormat::from(req)?;
 
 	let causal_context =
@@ -218,7 +222,7 @@ pub async fn handle_poll_item(
 		.k2v
 		.rpc
 		.poll_item(
-			bucket_id,
+			*bucket_id,
 			partition_key,
 			sort_key,
 			causal_context,

src/api/s3/api_server.rs

@@ -155,6 +155,7 @@ impl ApiHandler for S3ApiServer {
 			.bucket_helper()
 			.get_existing_bucket(bucket_id)
 			.await?;
+		let bucket_params = bucket.state.into_option().unwrap();
 
 		let allowed = match endpoint.authorization_type() {
 			Authorization::Read => api_key.allow_read(&bucket_id),
@@ -167,12 +168,20 @@ impl ApiHandler for S3ApiServer {
 			return Err(Error::forbidden("Operation is not allowed for this key."));
 		}
 
-		let matching_cors_rule = find_matching_cors_rule(&bucket, &req)?;
+		let matching_cors_rule = find_matching_cors_rule(&bucket_params, &req)?.cloned();
+
+		let ctx = ReqCtx {
+			garage,
+			bucket_id,
+			bucket_name,
+			bucket_params,
+			api_key,
+		};
 
 		let resp = match endpoint {
 			Endpoint::HeadObject {
 				key, part_number, ..
-			} => handle_head(garage, &req, bucket_id, &key, part_number).await,
+			} => handle_head(ctx, &req, &key, part_number).await,
 			Endpoint::GetObject {
 				key,
 				part_number,
@@ -192,74 +201,37 @@ impl ApiHandler for S3ApiServer {
 					response_content_type,
 					response_expires,
 				};
-				handle_get(garage, &req, bucket_id, &key, part_number, overrides).await
+				handle_get(ctx, &req, &key, part_number, overrides).await
 			}
 			Endpoint::UploadPart {
 				key,
 				part_number,
 				upload_id,
-			} => {
+			} => handle_put_part(ctx, req, &key, part_number, &upload_id, content_sha256).await,
-				handle_put_part(
+			Endpoint::CopyObject { key } => handle_copy(ctx, &req, &key).await,
-					garage,
-					req,
-					bucket_id,
-					&key,
-					part_number,
-					&upload_id,
-					content_sha256,
-				)
-				.await
-			}
-			Endpoint::CopyObject { key } => {
-				handle_copy(garage, &api_key, &req, bucket_id, &key).await
-			}
 			Endpoint::UploadPartCopy {
 				key,
 				part_number,
 				upload_id,
-			} => {
+			} => handle_upload_part_copy(ctx, &req, &key, part_number, &upload_id).await,
-				handle_upload_part_copy(
+			Endpoint::PutObject { key } => handle_put(ctx, req, &key, content_sha256).await,
-					garage,
-					&api_key,
-					&req,
-					bucket_id,
-					&key,
-					part_number,
-					&upload_id,
-				)
-				.await
-			}
-			Endpoint::PutObject { key } => {
-				handle_put(garage, req, &bucket, &key, content_sha256).await
-			}
 			Endpoint::AbortMultipartUpload { key, upload_id } => {
-				handle_abort_multipart_upload(garage, bucket_id, &key, &upload_id).await
+				handle_abort_multipart_upload(ctx, &key, &upload_id).await
 			}
-			Endpoint::DeleteObject { key, .. } => handle_delete(garage, bucket_id, &key).await,
+			Endpoint::DeleteObject { key, .. } => handle_delete(ctx, &key).await,
 			Endpoint::CreateMultipartUpload { key } => {
-				handle_create_multipart_upload(garage, &req, &bucket_name, bucket_id, &key).await
+				handle_create_multipart_upload(ctx, &req, &key).await
 			}
 			Endpoint::CompleteMultipartUpload { key, upload_id } => {
-				handle_complete_multipart_upload(
+				handle_complete_multipart_upload(ctx, req, &key, &upload_id, content_sha256).await
-					garage,
-					req,
-					&bucket_name,
-					&bucket,
-					&key,
-					&upload_id,
-					content_sha256,
-				)
-				.await
 			}
 			Endpoint::CreateBucket {} => unreachable!(),
 			Endpoint::HeadBucket {} => {
 				let response = Response::builder().body(empty_body()).unwrap();
 				Ok(response)
 			}
-			Endpoint::DeleteBucket {} => {
+			Endpoint::DeleteBucket {} => handle_delete_bucket(ctx).await,
-				handle_delete_bucket(&garage, bucket_id, bucket_name, &api_key.key_id).await
+			Endpoint::GetBucketLocation {} => handle_get_bucket_location(ctx),
-			}
-			Endpoint::GetBucketLocation {} => handle_get_bucket_location(garage),
 			Endpoint::GetBucketVersioning {} => handle_get_bucket_versioning(),
 			Endpoint::ListObjects {
 				delimiter,
@@ -268,24 +240,21 @@ impl ApiHandler for S3ApiServer {
 				max_keys,
 				prefix,
 			} => {
-				handle_list(
+				let query = ListObjectsQuery {
-					garage,
+					common: ListQueryCommon {
-					&ListObjectsQuery {
+						bucket_name: ctx.bucket_name.clone(),
-						common: ListQueryCommon {
+						bucket_id,
-							bucket_name,
+						delimiter,
-							bucket_id,
+						page_size: max_keys.unwrap_or(1000).clamp(1, 1000),
-							delimiter,
+						prefix: prefix.unwrap_or_default(),
-							page_size: max_keys.unwrap_or(1000).clamp(1, 1000),
+						urlencode_resp: encoding_type.map(|e| e == "url").unwrap_or(false),
-							prefix: prefix.unwrap_or_default(),
-							urlencode_resp: encoding_type.map(|e| e == "url").unwrap_or(false),
-						},
-						is_v2: false,
-						marker,
-						continuation_token: None,
-						start_after: None,
 					},
-				)
+					is_v2: false,
-				.await
+					marker,
+					continuation_token: None,
+					start_after: None,
+				};
+				handle_list(ctx, &query).await
 			}
 			Endpoint::ListObjectsV2 {
 				delimiter,
@@ -298,24 +267,21 @@ impl ApiHandler for S3ApiServer {
 				..
 			} => {
 				if list_type == "2" {
-					handle_list(
+					let query = ListObjectsQuery {
-						garage,
+						common: ListQueryCommon {
-						&ListObjectsQuery {
+							bucket_name: ctx.bucket_name.clone(),
-							common: ListQueryCommon {
+							bucket_id,
-								bucket_name,
+							delimiter,
-								bucket_id,
+							page_size: max_keys.unwrap_or(1000).clamp(1, 1000),
-								delimiter,
+							urlencode_resp: encoding_type.map(|e| e == "url").unwrap_or(false),
-								page_size: max_keys.unwrap_or(1000).clamp(1, 1000),
+							prefix: prefix.unwrap_or_default(),
-								urlencode_resp: encoding_type.map(|e| e == "url").unwrap_or(false),
-								prefix: prefix.unwrap_or_default(),
-							},
-							is_v2: true,
-							marker: None,
-							continuation_token,
-							start_after,
 						},
-					)
+						is_v2: true,
-					.await
+						marker: None,
+						continuation_token,
+						start_after,
+					};
+					handle_list(ctx, &query).await
 				} else {
 					Err(Error::bad_request(format!(
 						"Invalid endpoint: list-type={}",
@@ -331,22 +297,19 @@ impl ApiHandler for S3ApiServer {
 				prefix,
 				upload_id_marker,
 			} => {
-				handle_list_multipart_upload(
+				let query = ListMultipartUploadsQuery {
-					garage,
+					common: ListQueryCommon {
-					&ListMultipartUploadsQuery {
+						bucket_name: ctx.bucket_name.clone(),
-						common: ListQueryCommon {
+						bucket_id,
-							bucket_name,
+						delimiter,
-							bucket_id,
+						page_size: max_uploads.unwrap_or(1000).clamp(1, 1000),
-							delimiter,
+						prefix: prefix.unwrap_or_default(),
-							page_size: max_uploads.unwrap_or(1000).clamp(1, 1000),
+						urlencode_resp: encoding_type.map(|e| e == "url").unwrap_or(false),
-							prefix: prefix.unwrap_or_default(),
-							urlencode_resp: encoding_type.map(|e| e == "url").unwrap_or(false),
-						},
-						key_marker,
-						upload_id_marker,
 					},
-				)
+					key_marker,
-				.await
+					upload_id_marker,
+				};
+				handle_list_multipart_upload(ctx, &query).await
 			}
 			Endpoint::ListParts {
 				key,
@@ -354,39 +317,28 @@ impl ApiHandler for S3ApiServer {
 				part_number_marker,
 				upload_id,
 			} => {
-				handle_list_parts(
+				let query = ListPartsQuery {
-					garage,
+					bucket_name: ctx.bucket_name.clone(),
-					&ListPartsQuery {
+					bucket_id,
-						bucket_name,
+					key,
-						bucket_id,
+					upload_id,
-						key,
+					part_number_marker: part_number_marker.map(|p| p.min(10000)),
-						upload_id,
+					max_parts: max_parts.unwrap_or(1000).clamp(1, 1000),
-						part_number_marker: part_number_marker.map(|p| p.min(10000)),
+				};
-						max_parts: max_parts.unwrap_or(1000).clamp(1, 1000),
+				handle_list_parts(ctx, &query).await
-					},
-				)
-				.await
 			}
-			Endpoint::DeleteObjects {} => {
+			Endpoint::DeleteObjects {} => handle_delete_objects(ctx, req, content_sha256).await,
-				handle_delete_objects(garage, bucket_id, req, content_sha256).await
+			Endpoint::GetBucketWebsite {} => handle_get_website(ctx).await,
-			}
+			Endpoint::PutBucketWebsite {} => handle_put_website(ctx, req, content_sha256).await,
-			Endpoint::GetBucketWebsite {} => handle_get_website(&bucket).await,
+			Endpoint::DeleteBucketWebsite {} => handle_delete_website(ctx).await,
-			Endpoint::PutBucketWebsite {} => {
+			Endpoint::GetBucketCors {} => handle_get_cors(ctx).await,
-				handle_put_website(garage, bucket.clone(), req, content_sha256).await
+			Endpoint::PutBucketCors {} => handle_put_cors(ctx, req, content_sha256).await,
-			}
+			Endpoint::DeleteBucketCors {} => handle_delete_cors(ctx).await,
-			Endpoint::DeleteBucketWebsite {} => handle_delete_website(garage, bucket.clone()).await,
+			Endpoint::GetBucketLifecycleConfiguration {} => handle_get_lifecycle(ctx).await,
-			Endpoint::GetBucketCors {} => handle_get_cors(&bucket).await,
-			Endpoint::PutBucketCors {} => {
-				handle_put_cors(garage, bucket.clone(), req, content_sha256).await
-			}
-			Endpoint::DeleteBucketCors {} => handle_delete_cors(garage, bucket.clone()).await,
-			Endpoint::GetBucketLifecycleConfiguration {} => handle_get_lifecycle(&bucket).await,
 			Endpoint::PutBucketLifecycleConfiguration {} => {
-				handle_put_lifecycle(garage, bucket.clone(), req, content_sha256).await
+				handle_put_lifecycle(ctx, req, content_sha256).await
-			}
-			Endpoint::DeleteBucketLifecycle {} => {
-				handle_delete_lifecycle(garage, bucket.clone()).await
 			}
+			Endpoint::DeleteBucketLifecycle {} => handle_delete_lifecycle(ctx).await,
 			endpoint => Err(Error::NotImplemented(endpoint.name().to_owned())),
 		};
 
@@ -394,7 +346,7 @@ impl ApiHandler for S3ApiServer {
 		// add the corresponding CORS headers to the response
 		let mut resp_ok = resp?;
 		if let Some(rule) = matching_cors_rule {
-			add_cors_headers(&mut resp_ok, rule)
+			add_cors_headers(&mut resp_ok, &rule)
 				.ok_or_internal_error("Invalid bucket CORS configuration")?;
 		}
 

src/api/s3/bucket.rs

@@ -1,5 +1,4 @@
 use std::collections::HashMap;
-use std::sync::Arc;
 
 use http_body_util::BodyExt;
 use hyper::{Request, Response, StatusCode};
@@ -21,7 +20,8 @@ use crate::s3::error::*;
 use crate::s3::xml as s3_xml;
 use crate::signature::verify_signed_content;
 
-pub fn handle_get_bucket_location(garage: Arc<Garage>) -> Result<Response<ResBody>, Error> {
+pub fn handle_get_bucket_location(ctx: ReqCtx) -> Result<Response<ResBody>, Error> {
+	let ReqCtx { garage, .. } = ctx;
 	let loc = s3_xml::LocationConstraint {
 		xmlns: (),
 		region: garage.config.s3_api.s3_region.to_string(),
@@ -204,21 +204,20 @@ pub async fn handle_create_bucket(
 		.unwrap())
 }
 
-pub async fn handle_delete_bucket(
+pub async fn handle_delete_bucket(ctx: ReqCtx) -> Result<Response<ResBody>, Error> {
-	garage: &Garage,
+	let ReqCtx {
-	bucket_id: Uuid,
+		garage,
-	bucket_name: String,
+		bucket_id,
-	api_key_id: &String,
+		bucket_name,
-) -> Result<Response<ResBody>, Error> {
+		bucket_params: bucket_state,
+		api_key,
+		..
+	} = &ctx;
 	let helper = garage.locked_helper().await;
 
-	let api_key = helper.key().get_existing_key(api_key_id).await?;
 	let key_params = api_key.params().unwrap();
 
-	let is_local_alias = matches!(key_params.local_aliases.get(&bucket_name), Some(Some(_)));
+	let is_local_alias = matches!(key_params.local_aliases.get(bucket_name), Some(Some(_)));
-
-	let mut bucket = helper.bucket().get_existing_bucket(bucket_id).await?;
-	let bucket_state = bucket.state.as_option().unwrap();
 
 	// If the bucket has no other aliases, this is a true deletion.
 	// Otherwise, it is just an alias removal.
@@ -228,20 +227,20 @@ pub async fn handle_delete_bucket(
 		.items()
 		.iter()
 		.filter(|(_, _, active)| *active)
-		.any(|(n, _, _)| is_local_alias || (*n != bucket_name));
+		.any(|(n, _, _)| is_local_alias || (*n != *bucket_name));
 
 	let has_other_local_aliases = bucket_state
 		.local_aliases
 		.items()
 		.iter()
 		.filter(|(_, _, active)| *active)
-		.any(|((k, n), _, _)| !is_local_alias || *n != bucket_name || *k != api_key.key_id);
+		.any(|((k, n), _, _)| !is_local_alias || *n != *bucket_name || *k != api_key.key_id);
 
 	if !has_other_global_aliases && !has_other_local_aliases {
 		// Delete bucket
 
 		// Check bucket is empty
-		if !helper.bucket().is_bucket_empty(bucket_id).await? {
+		if !helper.bucket().is_bucket_empty(*bucket_id).await? {
 			return Err(CommonError::BucketNotEmpty.into());
 		}
 
@@ -249,33 +248,36 @@ pub async fn handle_delete_bucket(
 		// 1. delete bucket alias
 		if is_local_alias {
 			helper
-				.unset_local_bucket_alias(bucket_id, &api_key.key_id, &bucket_name)
+				.unset_local_bucket_alias(*bucket_id, &api_key.key_id, bucket_name)
 				.await?;
 		} else {
 			helper
-				.unset_global_bucket_alias(bucket_id, &bucket_name)
+				.unset_global_bucket_alias(*bucket_id, bucket_name)
 				.await?;
 		}
 
 		// 2. delete authorization from keys that had access
-		for (key_id, _) in bucket.authorized_keys() {
+		for (key_id, _) in bucket_state.authorized_keys.items() {
 			helper
-				.set_bucket_key_permissions(bucket.id, key_id, BucketKeyPerm::NO_PERMISSIONS)
+				.set_bucket_key_permissions(*bucket_id, key_id, BucketKeyPerm::NO_PERMISSIONS)
 				.await?;
 		}
 
+		let bucket = Bucket {
+			id: *bucket_id,
+			state: Deletable::delete(),
+		};
 		// 3. delete bucket
-		bucket.state = Deletable::delete();
 		garage.bucket_table.insert(&bucket).await?;
 	} else if is_local_alias {
 		// Just unalias
 		helper
-			.unset_local_bucket_alias(bucket_id, &api_key.key_id, &bucket_name)
+			.unset_local_bucket_alias(*bucket_id, &api_key.key_id, bucket_name)
 			.await?;
 	} else {
 		// Just unalias (but from global namespace)
 		helper
-			.unset_global_bucket_alias(bucket_id, &bucket_name)
+			.unset_global_bucket_alias(*bucket_id, bucket_name)
 			.await?;
 	}
 

src/api/s3/copy.rs

@@ -1,5 +1,4 @@
 use std::pin::Pin;
-use std::sync::Arc;
 use std::time::{Duration, SystemTime, UNIX_EPOCH};
 
 use futures::{stream, stream::Stream, StreamExt};
@@ -15,8 +14,6 @@ use garage_table::*;
 use garage_util::data::*;
 use garage_util::time::*;
 
-use garage_model::garage::Garage;
-use garage_model::key_table::Key;
 use garage_model::s3::block_ref_table::*;
 use garage_model::s3::mpu_table::*;
 use garage_model::s3::object_table::*;
@@ -30,15 +27,19 @@ use crate::s3::put::get_headers;
 use crate::s3::xml::{self as s3_xml, xmlns_tag};
 
 pub async fn handle_copy(
-	garage: Arc<Garage>,
+	ctx: ReqCtx,
-	api_key: &Key,
 	req: &Request<ReqBody>,
-	dest_bucket_id: Uuid,
 	dest_key: &str,
 ) -> Result<Response<ResBody>, Error> {
 	let copy_precondition = CopyPreconditionHeaders::parse(req)?;
 
-	let source_object = get_copy_source(&garage, api_key, req).await?;
+	let source_object = get_copy_source(&ctx, req).await?;
+
+	let ReqCtx {
+		garage,
+		bucket_id: dest_bucket_id,
+		..
+	} = ctx;
 
 	let (source_version, source_version_data, source_version_meta) =
 		extract_source_info(&source_object)?;
@@ -181,10 +182,8 @@ pub async fn handle_copy(
 }
 
 pub async fn handle_upload_part_copy(
-	garage: Arc<Garage>,
+	ctx: ReqCtx,
-	api_key: &Key,
 	req: &Request<ReqBody>,
-	dest_bucket_id: Uuid,
 	dest_key: &str,
 	part_number: u64,
 	upload_id: &str,
@@ -195,10 +194,12 @@ pub async fn handle_upload_part_copy(
 
 	let dest_key = dest_key.to_string();
 	let (source_object, (_, _, mut dest_mpu)) = futures::try_join!(
-		get_copy_source(&garage, api_key, req),
+		get_copy_source(&ctx, req),
-		multipart::get_upload(&garage, &dest_bucket_id, &dest_key, &dest_upload_id)
+		multipart::get_upload(&ctx, &dest_key, &dest_upload_id)
 	)?;
 
+	let ReqCtx { garage, .. } = ctx;
+
 	let (source_object_version, source_version_data, source_version_meta) =
 		extract_source_info(&source_object)?;
 
@@ -439,11 +440,11 @@ pub async fn handle_upload_part_copy(
 		.body(string_body(resp_xml))?)
 }
 
-async fn get_copy_source(
+async fn get_copy_source(ctx: &ReqCtx, req: &Request<ReqBody>) -> Result<Object, Error> {
-	garage: &Garage,
+	let ReqCtx {
-	api_key: &Key,
+		garage, api_key, ..
-	req: &Request<ReqBody>,
+	} = ctx;
-) -> Result<Object, Error> {
+
 	let copy_source = req.headers().get("x-amz-copy-source").unwrap().to_str()?;
 	let copy_source = percent_encoding::percent_decode_str(copy_source).decode_utf8()?;
 

src/api/s3/cors.rs

@@ -21,16 +21,13 @@ use crate::s3::error::*;
 use crate::s3::xml::{to_xml_with_header, xmlns_tag, IntValue, Value};
 use crate::signature::verify_signed_content;
 
-use garage_model::bucket_table::{Bucket, CorsRule as GarageCorsRule};
+use garage_model::bucket_table::{Bucket, BucketParams, CorsRule as GarageCorsRule};
 use garage_model::garage::Garage;
 use garage_util::data::*;
 
-pub async fn handle_get_cors(bucket: &Bucket) -> Result<Response<ResBody>, Error> {
+pub async fn handle_get_cors(ctx: ReqCtx) -> Result<Response<ResBody>, Error> {
-	let param = bucket
+	let ReqCtx { bucket_params, .. } = ctx;
-		.params()
+	if let Some(cors) = bucket_params.cors_config.get() {
-		.ok_or_internal_error("Bucket should not be deleted at this point")?;
-
-	if let Some(cors) = param.cors_config.get() {
 		let wc = CorsConfiguration {
 			xmlns: (),
 			cors_rules: cors
@@ -50,16 +47,18 @@ pub async fn handle_get_cors(bucket: &Bucket) -> Result<Response<ResBody>, Error
 	}
 }
 
-pub async fn handle_delete_cors(
+pub async fn handle_delete_cors(ctx: ReqCtx) -> Result<Response<ResBody>, Error> {
-	garage: Arc<Garage>,
+	let ReqCtx {
-	mut bucket: Bucket,
+		garage,
-) -> Result<Response<ResBody>, Error> {
+		bucket_id,
-	let param = bucket
+		mut bucket_params,
-		.params_mut()
+		..
-		.ok_or_internal_error("Bucket should not be deleted at this point")?;
+	} = ctx;
-
+	bucket_params.cors_config.update(None);
-	param.cors_config.update(None);
+	garage
-	garage.bucket_table.insert(&bucket).await?;
+		.bucket_table
+		.insert(&Bucket::present(bucket_id, bucket_params))
+		.await?;
 
 	Ok(Response::builder()
 		.status(StatusCode::NO_CONTENT)
@@ -67,28 +66,33 @@ pub async fn handle_delete_cors(
 }
 
 pub async fn handle_put_cors(
-	garage: Arc<Garage>,
+	ctx: ReqCtx,
-	mut bucket: Bucket,
 	req: Request<ReqBody>,
 	content_sha256: Option<Hash>,
 ) -> Result<Response<ResBody>, Error> {
+	let ReqCtx {
+		garage,
+		bucket_id,
+		mut bucket_params,
+		..
+	} = ctx;
+
 	let body = BodyExt::collect(req.into_body()).await?.to_bytes();
 
 	if let Some(content_sha256) = content_sha256 {
 		verify_signed_content(content_sha256, &body[..])?;
 	}
 
-	let param = bucket
-		.params_mut()
-		.ok_or_internal_error("Bucket should not be deleted at this point")?;
-
 	let conf: CorsConfiguration = from_reader(&body as &[u8])?;
 	conf.validate()?;
 
-	param
+	bucket_params
 		.cors_config
 		.update(Some(conf.into_garage_cors_config()?));
-	garage.bucket_table.insert(&bucket).await?;
+	garage
+		.bucket_table
+		.insert(&Bucket::present(bucket_id, bucket_params))
+		.await?;
 
 	Ok(Response::builder()
 		.status(StatusCode::OK)
@@ -115,7 +119,8 @@ pub async fn handle_options_api(
 		let bucket_id = helper.resolve_global_bucket_name(&bn).await?;
 		if let Some(id) = bucket_id {
 			let bucket = garage.bucket_helper().get_existing_bucket(id).await?;
-			handle_options_for_bucket(req, &bucket)
+			let bucket_params = bucket.state.into_option().unwrap();
+			handle_options_for_bucket(req, &bucket_params)
 		} else {
 			// If there is a bucket name in the request, but that name
 			// does not correspond to a global alias for a bucket,
@@ -145,7 +150,7 @@ pub async fn handle_options_api(
 
 pub fn handle_options_for_bucket(
 	req: &Request<IncomingBody>,
-	bucket: &Bucket,
+	bucket_params: &BucketParams,
 ) -> Result<Response<EmptyBody>, CommonError> {
 	let origin = req
 		.headers()
@@ -162,7 +167,7 @@ pub fn handle_options_for_bucket(
 		None => vec![],
 	};
 
-	if let Some(cors_config) = bucket.params().unwrap().cors_config.get() {
+	if let Some(cors_config) = bucket_params.cors_config.get() {
 		let matching_rule = cors_config
 			.iter()
 			.find(|rule| cors_rule_matches(rule, origin, request_method, request_headers.iter()));
@@ -181,10 +186,10 @@ pub fn handle_options_for_bucket(
 }
 
 pub fn find_matching_cors_rule<'a>(
-	bucket: &'a Bucket,
+	bucket_params: &'a BucketParams,
 	req: &Request<impl Body>,
 ) -> Result<Option<&'a GarageCorsRule>, Error> {
-	if let Some(cors_config) = bucket.params().unwrap().cors_config.get() {
+	if let Some(cors_config) = bucket_params.cors_config.get() {
 		if let Some(origin) = req.headers().get("Origin") {
 			let origin = origin.to_str()?;
 			let request_headers = match req.headers().get(ACCESS_CONTROL_REQUEST_HEADERS) {

src/api/s3/delete.rs

@@ -1,11 +1,8 @@
-use std::sync::Arc;
-
 use http_body_util::BodyExt;
 use hyper::{Request, Response, StatusCode};
 
 use garage_util::data::*;
 
-use garage_model::garage::Garage;
 use garage_model::s3::object_table::*;
 
 use crate::helpers::*;
@@ -15,14 +12,13 @@ use crate::s3::put::next_timestamp;
 use crate::s3::xml as s3_xml;
 use crate::signature::verify_signed_content;
 
-async fn handle_delete_internal(
+async fn handle_delete_internal(ctx: &ReqCtx, key: &str) -> Result<(Uuid, Uuid), Error> {
-	garage: &Garage,
+	let ReqCtx {
-	bucket_id: Uuid,
+		garage, bucket_id, ..
-	key: &str,
+	} = ctx;
-) -> Result<(Uuid, Uuid), Error> {
 	let object = garage
 		.object_table
-		.get(&bucket_id, &key.to_string())
+		.get(bucket_id, &key.to_string())
 		.await?
 		.ok_or(Error::NoSuchKey)?; // No need to delete
 
@@ -44,7 +40,7 @@ async fn handle_delete_internal(
 	};
 
 	let object = Object::new(
-		bucket_id,
+		*bucket_id,
 		key.into(),
 		vec![ObjectVersion {
 			uuid: del_uuid,
@@ -58,12 +54,8 @@ async fn handle_delete_internal(
 	Ok((deleted_version, del_uuid))
 }
 
-pub async fn handle_delete(
+pub async fn handle_delete(ctx: ReqCtx, key: &str) -> Result<Response<ResBody>, Error> {
-	garage: Arc<Garage>,
+	match handle_delete_internal(&ctx, key).await {
-	bucket_id: Uuid,
-	key: &str,
-) -> Result<Response<ResBody>, Error> {
-	match handle_delete_internal(&garage, bucket_id, key).await {
 		Ok(_) | Err(Error::NoSuchKey) => Ok(Response::builder()
 			.status(StatusCode::NO_CONTENT)
 			.body(empty_body())
@@ -73,8 +65,7 @@ pub async fn handle_delete(
 }
 
 pub async fn handle_delete_objects(
-	garage: Arc<Garage>,
+	ctx: ReqCtx,
-	bucket_id: Uuid,
 	req: Request<ReqBody>,
 	content_sha256: Option<Hash>,
 ) -> Result<Response<ResBody>, Error> {
@@ -91,7 +82,7 @@ pub async fn handle_delete_objects(
 	let mut ret_errors = Vec::new();
 
 	for obj in cmd.objects.iter() {
-		match handle_delete_internal(&garage, bucket_id, &obj.key).await {
+		match handle_delete_internal(&ctx, &obj.key).await {
 			Ok((deleted_version, delete_marker_version)) => {
 				if cmd.quiet {
 					continue;

src/api/s3/get.rs

@@ -131,6 +131,16 @@ fn try_answer_cached(
 
 /// Handle HEAD request
 pub async fn handle_head(
+	ctx: ReqCtx,
+	req: &Request<impl Body>,
+	key: &str,
+	part_number: Option<u64>,
+) -> Result<Response<ResBody>, Error> {
+	handle_head_without_ctx(ctx.garage, req, ctx.bucket_id, key, part_number).await
+}
+
+/// Handle HEAD request for website
+pub async fn handle_head_without_ctx(
 	garage: Arc<Garage>,
 	req: &Request<impl Body>,
 	bucket_id: Uuid,
@@ -218,6 +228,17 @@ pub async fn handle_head(
 
 /// Handle GET request
 pub async fn handle_get(
+	ctx: ReqCtx,
+	req: &Request<impl Body>,
+	key: &str,
+	part_number: Option<u64>,
+	overrides: GetObjectOverrides,
+) -> Result<Response<ResBody>, Error> {
+	handle_get_without_ctx(ctx.garage, req, ctx.bucket_id, key, part_number, overrides).await
+}
+
+/// Handle GET request
+pub async fn handle_get_without_ctx(
 	garage: Arc<Garage>,
 	req: &Request<impl Body>,
 	bucket_id: Uuid,

src/api/s3/lifecycle.rs

@@ -1,5 +1,4 @@
 use quick_xml::de::from_reader;
-use std::sync::Arc;
 
 use http_body_util::BodyExt;
 use hyper::{Request, Response, StatusCode};
@@ -16,15 +15,12 @@ use garage_model::bucket_table::{
 	parse_lifecycle_date, Bucket, LifecycleExpiration as GarageLifecycleExpiration,
 	LifecycleFilter as GarageLifecycleFilter, LifecycleRule as GarageLifecycleRule,
 };
-use garage_model::garage::Garage;
 use garage_util::data::*;
 
-pub async fn handle_get_lifecycle(bucket: &Bucket) -> Result<Response<ResBody>, Error> {
+pub async fn handle_get_lifecycle(ctx: ReqCtx) -> Result<Response<ResBody>, Error> {
-	let param = bucket
+	let ReqCtx { bucket_params, .. } = ctx;
-		.params()
-		.ok_or_internal_error("Bucket should not be deleted at this point")?;
 
-	if let Some(lifecycle) = param.lifecycle_config.get() {
+	if let Some(lifecycle) = bucket_params.lifecycle_config.get() {
 		let wc = LifecycleConfiguration::from_garage_lifecycle_config(lifecycle);
 		let xml = to_xml_with_header(&wc)?;
 		Ok(Response::builder()
@@ -38,16 +34,18 @@ pub async fn handle_get_lifecycle(bucket: &Bucket) -> Result<Response<ResBody>,
 	}
 }
 
-pub async fn handle_delete_lifecycle(
+pub async fn handle_delete_lifecycle(ctx: ReqCtx) -> Result<Response<ResBody>, Error> {
-	garage: Arc<Garage>,
+	let ReqCtx {
-	mut bucket: Bucket,
+		garage,
-) -> Result<Response<ResBody>, Error> {
+		bucket_id,
-	let param = bucket
+		mut bucket_params,
-		.params_mut()
+		..
-		.ok_or_internal_error("Bucket should not be deleted at this point")?;
+	} = ctx;
-
+	bucket_params.lifecycle_config.update(None);
-	param.lifecycle_config.update(None);
+	garage
-	garage.bucket_table.insert(&bucket).await?;
+		.bucket_table
+		.insert(&Bucket::present(bucket_id, bucket_params))
+		.await?;
 
 	Ok(Response::builder()
 		.status(StatusCode::NO_CONTENT)
@@ -55,28 +53,33 @@ pub async fn handle_delete_lifecycle(
 }
 
 pub async fn handle_put_lifecycle(
-	garage: Arc<Garage>,
+	ctx: ReqCtx,
-	mut bucket: Bucket,
 	req: Request<ReqBody>,
 	content_sha256: Option<Hash>,
 ) -> Result<Response<ResBody>, Error> {
+	let ReqCtx {
+		garage,
+		bucket_id,
+		mut bucket_params,
+		..
+	} = ctx;
+
 	let body = BodyExt::collect(req.into_body()).await?.to_bytes();
 
 	if let Some(content_sha256) = content_sha256 {
 		verify_signed_content(content_sha256, &body[..])?;
 	}
 
-	let param = bucket
-		.params_mut()
-		.ok_or_internal_error("Bucket should not be deleted at this point")?;
-
 	let conf: LifecycleConfiguration = from_reader(&body as &[u8])?;
 	let config = conf
 		.validate_into_garage_lifecycle_config()
 		.ok_or_bad_request("Invalid lifecycle configuration")?;
 
-	param.lifecycle_config.update(Some(config));
+	bucket_params.lifecycle_config.update(Some(config));
-	garage.bucket_table.insert(&bucket).await?;
+	garage
+		.bucket_table
+		.insert(&Bucket::present(bucket_id, bucket_params))
+		.await?;
 
 	Ok(Response::builder()
 		.status(StatusCode::OK)

src/api/s3/list.rs

@@ -1,6 +1,5 @@
 use std::collections::{BTreeMap, BTreeSet};
 use std::iter::{Iterator, Peekable};
-use std::sync::Arc;
 
 use base64::prelude::*;
 use hyper::Response;
@@ -9,7 +8,6 @@ use garage_util::data::*;
 use garage_util::error::Error as GarageError;
 use garage_util::time::*;
 
-use garage_model::garage::Garage;
 use garage_model::s3::mpu_table::*;
 use garage_model::s3::object_table::*;
 
@@ -62,9 +60,10 @@ pub struct ListPartsQuery {
 }
 
 pub async fn handle_list(
-	garage: Arc<Garage>,
+	ctx: ReqCtx,
 	query: &ListObjectsQuery,
 ) -> Result<Response<ResBody>, Error> {
+	let ReqCtx { garage, .. } = &ctx;
 	let io = |bucket, key, count| {
 		let t = &garage.object_table;
 		async move {
@@ -167,9 +166,11 @@ pub async fn handle_list(
 }
 
 pub async fn handle_list_multipart_upload(
-	garage: Arc<Garage>,
+	ctx: ReqCtx,
 	query: &ListMultipartUploadsQuery,
 ) -> Result<Response<ResBody>, Error> {
+	let ReqCtx { garage, .. } = &ctx;
+
 	let io = |bucket, key, count| {
 		let t = &garage.object_table;
 		async move {
@@ -269,15 +270,14 @@ pub async fn handle_list_multipart_upload(
 }
 
 pub async fn handle_list_parts(
-	garage: Arc<Garage>,
+	ctx: ReqCtx,
 	query: &ListPartsQuery,
 ) -> Result<Response<ResBody>, Error> {
 	debug!("ListParts {:?}", query);
 
 	let upload_id = s3_multipart::decode_upload_id(&query.upload_id)?;
 
-	let (_, _, mpu) =
+	let (_, _, mpu) = s3_multipart::get_upload(&ctx, &query.key, &upload_id).await?;
-		s3_multipart::get_upload(&garage, &query.bucket_id, &query.key, &upload_id).await?;
 
 	let (info, next) = fetch_part_info(query, &mpu)?;
 

src/api/s3/multipart.rs

@@ -8,7 +8,6 @@ use md5::{Digest as Md5Digest, Md5};
 use garage_table::*;
 use garage_util::data::*;
 
-use garage_model::bucket_table::Bucket;
 use garage_model::garage::Garage;
 use garage_model::s3::block_ref_table::*;
 use garage_model::s3::mpu_table::*;
@@ -25,12 +24,16 @@ use crate::signature::verify_signed_content;
 // ----
 
 pub async fn handle_create_multipart_upload(
-	garage: Arc<Garage>,
+	ctx: ReqCtx,
 	req: &Request<ReqBody>,
-	bucket_name: &str,
-	bucket_id: Uuid,
 	key: &String,
 ) -> Result<Response<ResBody>, Error> {
+	let ReqCtx {
+		garage,
+		bucket_id,
+		bucket_name,
+		..
+	} = &ctx;
 	let existing_object = garage.object_table.get(&bucket_id, &key).await?;
 
 	let upload_id = gen_uuid();
@@ -47,13 +50,13 @@ pub async fn handle_create_multipart_upload(
 			headers,
 		},
 	};
-	let object = Object::new(bucket_id, key.to_string(), vec![object_version]);
+	let object = Object::new(*bucket_id, key.to_string(), vec![object_version]);
 	garage.object_table.insert(&object).await?;
 
 	// Create multipart upload in mpu table
 	// This multipart upload will hold references to uploaded parts
 	// (which are entries in the Version table)
-	let mpu = MultipartUpload::new(upload_id, timestamp, bucket_id, key.into(), false);
+	let mpu = MultipartUpload::new(upload_id, timestamp, *bucket_id, key.into(), false);
 	garage.mpu_table.insert(&mpu).await?;
 
 	// Send success response
@@ -69,14 +72,15 @@ pub async fn handle_create_multipart_upload(
 }
 
 pub async fn handle_put_part(
-	garage: Arc<Garage>,
+	ctx: ReqCtx,
 	req: Request<ReqBody>,
-	bucket_id: Uuid,
 	key: &str,
 	part_number: u64,
 	upload_id: &str,
 	content_sha256: Option<Hash>,
 ) -> Result<Response<ResBody>, Error> {
+	let ReqCtx { garage, .. } = &ctx;
+
 	let upload_id = decode_upload_id(upload_id)?;
 
 	let content_md5 = match req.headers().get("content-md5") {
@@ -90,10 +94,8 @@ pub async fn handle_put_part(
 	let stream = body_stream(req.into_body());
 	let mut chunker = StreamChunker::new(stream, garage.config.block_size);
 
-	let ((_, _, mut mpu), first_block) = futures::try_join!(
+	let ((_, _, mut mpu), first_block) =
-		get_upload(&garage, &bucket_id, &key, &upload_id),
+		futures::try_join!(get_upload(&ctx, &key, &upload_id), chunker.next(),)?;
-		chunker.next(),
-	)?;
 
 	// Check object is valid and part can be accepted
 	let first_block = first_block.ok_or_bad_request("Empty body")?;
@@ -135,7 +137,7 @@ pub async fn handle_put_part(
 
 	// Copy data to version
 	let (total_size, data_md5sum, data_sha256sum, _) =
-		read_and_put_blocks(&garage, &version, part_number, first_block, &mut chunker).await?;
+		read_and_put_blocks(&ctx, &version, part_number, first_block, &mut chunker).await?;
 
 	// Verify that checksums map
 	ensure_checksum_matches(
@@ -200,14 +202,19 @@ impl Drop for InterruptedCleanup {
 }
 
 pub async fn handle_complete_multipart_upload(
-	garage: Arc<Garage>,
+	ctx: ReqCtx,
 	req: Request<ReqBody>,
-	bucket_name: &str,
-	bucket: &Bucket,
 	key: &str,
 	upload_id: &str,
 	content_sha256: Option<Hash>,
 ) -> Result<Response<ResBody>, Error> {
+	let ReqCtx {
+		garage,
+		bucket_id,
+		bucket_name,
+		..
+	} = &ctx;
+
 	let body = http_body_util::BodyExt::collect(req.into_body())
 		.await?
 		.to_bytes();
@@ -228,8 +235,7 @@ pub async fn handle_complete_multipart_upload(
 
 	// Get object and multipart upload
 	let key = key.to_string();
-	let (object, mut object_version, mpu) =
+	let (object, mut object_version, mpu) = get_upload(&ctx, &key, &upload_id).await?;
-		get_upload(&garage, &bucket.id, &key, &upload_id).await?;
 
 	if mpu.parts.is_empty() {
 		return Err(Error::bad_request("No data was uploaded"));
@@ -283,7 +289,7 @@ pub async fn handle_complete_multipart_upload(
 	let mut final_version = Version::new(
 		upload_id,
 		VersionBacklink::Object {
-			bucket_id: bucket.id,
+			bucket_id: *bucket_id,
 			key: key.to_string(),
 		},
 		false,
@@ -327,9 +333,9 @@ pub async fn handle_complete_multipart_upload(
 	// Calculate total size of final object
 	let total_size = parts.iter().map(|x| x.size.unwrap()).sum();
 
-	if let Err(e) = check_quotas(&garage, bucket, total_size, Some(&object)).await {
+	if let Err(e) = check_quotas(&ctx, total_size, Some(&object)).await {
 		object_version.state = ObjectVersionState::Aborted;
-		let final_object = Object::new(bucket.id, key.clone(), vec![object_version]);
+		let final_object = Object::new(*bucket_id, key.clone(), vec![object_version]);
 		garage.object_table.insert(&final_object).await?;
 
 		return Err(e);
@@ -345,7 +351,7 @@ pub async fn handle_complete_multipart_upload(
 		final_version.blocks.items()[0].1.hash,
 	));
 
-	let final_object = Object::new(bucket.id, key.clone(), vec![object_version]);
+	let final_object = Object::new(*bucket_id, key.clone(), vec![object_version]);
 	garage.object_table.insert(&final_object).await?;
 
 	// Send response saying ok we're done
@@ -362,18 +368,20 @@ pub async fn handle_complete_multipart_upload(
 }
 
 pub async fn handle_abort_multipart_upload(
-	garage: Arc<Garage>,
+	ctx: ReqCtx,
-	bucket_id: Uuid,
 	key: &str,
 	upload_id: &str,
 ) -> Result<Response<ResBody>, Error> {
+	let ReqCtx {
+		garage, bucket_id, ..
+	} = &ctx;
+
 	let upload_id = decode_upload_id(upload_id)?;
 
-	let (_, mut object_version, _) =
+	let (_, mut object_version, _) = get_upload(&ctx, &key.to_string(), &upload_id).await?;
-		get_upload(&garage, &bucket_id, &key.to_string(), &upload_id).await?;
 
 	object_version.state = ObjectVersionState::Aborted;
-	let final_object = Object::new(bucket_id, key.to_string(), vec![object_version]);
+	let final_object = Object::new(*bucket_id, key.to_string(), vec![object_version]);
 	garage.object_table.insert(&final_object).await?;
 
 	Ok(Response::new(empty_body()))
@@ -383,11 +391,13 @@ pub async fn handle_abort_multipart_upload(
 
 #[allow(clippy::ptr_arg)]
 pub(crate) async fn get_upload(
-	garage: &Garage,
+	ctx: &ReqCtx,
-	bucket_id: &Uuid,
 	key: &String,
 	upload_id: &Uuid,
 ) -> Result<(Object, ObjectVersion, MultipartUpload), Error> {
+	let ReqCtx {
+		garage, bucket_id, ..
+	} = ctx;
 	let (object, mpu) = futures::try_join!(
 		garage.object_table.get(bucket_id, key).map_err(Error::from),
 		garage

src/api/s3/post_object.rs

@@ -120,6 +120,12 @@ pub async fn handle_post_object(
 		.bucket_helper()
 		.get_existing_bucket(bucket_id)
 		.await?;
+	let bucket_params = bucket.state.into_option().unwrap();
+	let matching_cors_rule = find_matching_cors_rule(
+		&bucket_params,
+		&Request::from_parts(head.clone(), empty_body::<Infallible>()),
+	)?
+	.cloned();
 
 	let decoded_policy = BASE64_STANDARD
 		.decode(policy)
@@ -213,11 +219,19 @@ pub async fn handle_post_object(
 	let headers = get_headers(&params)?;
 
 	let stream = field.map(|r| r.map_err(Into::into));
-	let (_, md5) = save_stream(
+
+	let ctx = ReqCtx {
 		garage,
+		bucket_id,
+		bucket_name,
+		bucket_params,
+		api_key,
+	};
+
+	let (_, md5) = save_stream(
+		&ctx,
 		headers,
 		StreamLimiter::new(stream, conditions.content_length),
-		&bucket,
 		&key,
 		None,
 		None,
@@ -234,7 +248,7 @@ pub async fn handle_post_object(
 	{
 		target
 			.query_pairs_mut()
-			.append_pair("bucket", &bucket_name)
+			.append_pair("bucket", &ctx.bucket_name)
 			.append_pair("key", &key)
 			.append_pair("etag", &etag);
 		let target = target.to_string();
@@ -278,7 +292,7 @@ pub async fn handle_post_object(
 				let xml = s3_xml::PostObject {
 					xmlns: (),
 					location: s3_xml::Value(location),
-					bucket: s3_xml::Value(bucket_name),
+					bucket: s3_xml::Value(ctx.bucket_name),
 					key: s3_xml::Value(key),
 					etag: s3_xml::Value(etag),
 				};
@@ -291,12 +305,8 @@ pub async fn handle_post_object(
 		}
 	};
 
-	let matching_cors_rule = find_matching_cors_rule(
-		&bucket,
-		&Request::from_parts(head, empty_body::<Infallible>()),
-	)?;
 	if let Some(rule) = matching_cors_rule {
-		add_cors_headers(&mut resp, rule)
+		add_cors_headers(&mut resp, &rule)
 			.ok_or_internal_error("Invalid bucket CORS configuration")?;
 	}
 

src/api/s3/put.rs

@@ -28,7 +28,6 @@ use garage_util::error::Error as GarageError;
 use garage_util::time::*;
 
 use garage_block::manager::INLINE_THRESHOLD;
-use garage_model::bucket_table::Bucket;
 use garage_model::garage::Garage;
 use garage_model::index_counter::CountedItem;
 use garage_model::s3::block_ref_table::*;
@@ -42,9 +41,8 @@ use crate::s3::error::*;
 const PUT_BLOCKS_MAX_PARALLEL: usize = 3;
 
 pub async fn handle_put(
-	garage: Arc<Garage>,
+	ctx: ReqCtx,
 	req: Request<ReqBody>,
-	bucket: &Bucket,
 	key: &String,
 	content_sha256: Option<Hash>,
 ) -> Result<Response<ResBody>, Error> {
@@ -59,35 +57,27 @@ pub async fn handle_put(
 
 	let stream = body_stream(req.into_body());
 
-	save_stream(
+	save_stream(&ctx, headers, stream, key, content_md5, content_sha256)
-		garage,
+		.await
-		headers,
+		.map(|(uuid, md5)| put_response(uuid, md5))
-		stream,
-		bucket,
-		key,
-		content_md5,
-		content_sha256,
-	)
-	.await
-	.map(|(uuid, md5)| put_response(uuid, md5))
 }
 
 pub(crate) async fn save_stream<S: Stream<Item = Result<Bytes, Error>> + Unpin>(
-	garage: Arc<Garage>,
+	ctx: &ReqCtx,
 	headers: ObjectVersionHeaders,
 	body: S,
-	bucket: &Bucket,
 	key: &String,
 	content_md5: Option<String>,
 	content_sha256: Option<FixedBytes32>,
 ) -> Result<(Uuid, String), Error> {
+	let ReqCtx {
+		garage, bucket_id, ..
+	} = ctx;
+
 	let mut chunker = StreamChunker::new(body, garage.config.block_size);
 	let (first_block_opt, existing_object) = try_join!(
 		chunker.next(),
-		garage
+		garage.object_table.get(bucket_id, key).map_err(Error::from),
-			.object_table
-			.get(&bucket.id, key)
-			.map_err(Error::from),
 	)?;
 
 	let first_block = first_block_opt.unwrap_or_default();
@@ -114,7 +104,7 @@ pub(crate) async fn save_stream<S: Stream<Item = Result<Bytes, Error>> + Unpin>(
 			content_sha256,
 		)?;
 
-		check_quotas(&garage, bucket, size, existing_object.as_ref()).await?;
+		check_quotas(ctx, size, existing_object.as_ref()).await?;
 
 		let object_version = ObjectVersion {
 			uuid: version_uuid,
@@ -129,7 +119,7 @@ pub(crate) async fn save_stream<S: Stream<Item = Result<Bytes, Error>> + Unpin>(
 			)),
 		};
 
-		let object = Object::new(bucket.id, key.into(), vec![object_version]);
+		let object = Object::new(*bucket_id, key.into(), vec![object_version]);
 		garage.object_table.insert(&object).await?;
 
 		return Ok((version_uuid, data_md5sum_hex));
@@ -140,7 +130,7 @@ pub(crate) async fn save_stream<S: Stream<Item = Result<Bytes, Error>> + Unpin>(
 	// before everything is finished (cleanup is done using the Drop trait).
 	let mut interrupted_cleanup = InterruptedCleanup(Some(InterruptedCleanupInner {
 		garage: garage.clone(),
-		bucket_id: bucket.id,
+		bucket_id: *bucket_id,
 		key: key.into(),
 		version_uuid,
 		version_timestamp,
@@ -156,7 +146,7 @@ pub(crate) async fn save_stream<S: Stream<Item = Result<Bytes, Error>> + Unpin>(
 			multipart: false,
 		},
 	};
-	let object = Object::new(bucket.id, key.into(), vec![object_version.clone()]);
+	let object = Object::new(*bucket_id, key.into(), vec![object_version.clone()]);
 	garage.object_table.insert(&object).await?;
 
 	// Initialize corresponding entry in version table
@@ -166,7 +156,7 @@ pub(crate) async fn save_stream<S: Stream<Item = Result<Bytes, Error>> + Unpin>(
 	let version = Version::new(
 		version_uuid,
 		VersionBacklink::Object {
-			bucket_id: bucket.id,
+			bucket_id: *bucket_id,
 			key: key.into(),
 		},
 		false,
@@ -175,7 +165,7 @@ pub(crate) async fn save_stream<S: Stream<Item = Result<Bytes, Error>> + Unpin>(
 
 	// Transfer data and verify checksum
 	let (total_size, data_md5sum, data_sha256sum, first_block_hash) =
-		read_and_put_blocks(&garage, &version, 1, first_block, &mut chunker).await?;
+		read_and_put_blocks(ctx, &version, 1, first_block, &mut chunker).await?;
 
 	ensure_checksum_matches(
 		data_md5sum.as_slice(),
@@ -184,7 +174,7 @@ pub(crate) async fn save_stream<S: Stream<Item = Result<Bytes, Error>> + Unpin>(
 		content_sha256,
 	)?;
 
-	check_quotas(&garage, bucket, total_size, existing_object.as_ref()).await?;
+	check_quotas(ctx, total_size, existing_object.as_ref()).await?;
 
 	// Save final object state, marked as Complete
 	let md5sum_hex = hex::encode(data_md5sum);
@@ -196,7 +186,7 @@ pub(crate) async fn save_stream<S: Stream<Item = Result<Bytes, Error>> + Unpin>(
 		},
 		first_block_hash,
 	));
-	let object = Object::new(bucket.id, key.into(), vec![object_version]);
+	let object = Object::new(*bucket_id, key.into(), vec![object_version]);
 	garage.object_table.insert(&object).await?;
 
 	// We were not interrupted, everything went fine.
@@ -235,12 +225,18 @@ pub(crate) fn ensure_checksum_matches(
 
 /// Check that inserting this object with this size doesn't exceed bucket quotas
 pub(crate) async fn check_quotas(
-	garage: &Arc<Garage>,
+	ctx: &ReqCtx,
-	bucket: &Bucket,
 	size: u64,
 	prev_object: Option<&Object>,
 ) -> Result<(), Error> {
-	let quotas = bucket.state.as_option().unwrap().quotas.get();
+	let ReqCtx {
+		garage,
+		bucket_id,
+		bucket_params,
+		..
+	} = ctx;
+
+	let quotas = bucket_params.quotas.get();
 	if quotas.max_objects.is_none() && quotas.max_size.is_none() {
 		return Ok(());
 	};
@@ -248,7 +244,7 @@ pub(crate) async fn check_quotas(
 	let counters = garage
 		.object_counter_table
 		.table
-		.get(&bucket.id, &EmptyKey)
+		.get(bucket_id, &EmptyKey)
 		.await?;
 
 	let counters = counters
@@ -292,7 +288,7 @@ pub(crate) async fn check_quotas(
 }
 
 pub(crate) async fn read_and_put_blocks<S: Stream<Item = Result<Bytes, Error>> + Unpin>(
-	garage: &Garage,
+	ctx: &ReqCtx,
 	version: &Version,
 	part_number: u64,
 	first_block: Bytes,
@@ -417,7 +413,7 @@ pub(crate) async fn read_and_put_blocks<S: Stream<Item = Result<Bytes, Error>> +
 			let offset = written_bytes;
 			written_bytes += block.len() as u64;
 			write_futs.push_back(put_block_and_meta(
-				garage,
+				ctx,
 				version,
 				part_number,
 				offset,
@@ -447,7 +443,7 @@ pub(crate) async fn read_and_put_blocks<S: Stream<Item = Result<Bytes, Error>> +
 }
 
 async fn put_block_and_meta(
-	garage: &Garage,
+	ctx: &ReqCtx,
 	version: &Version,
 	part_number: u64,
 	offset: u64,
@@ -455,6 +451,8 @@ async fn put_block_and_meta(
 	block: Bytes,
 	order_tag: OrderTag,
 ) -> Result<(), GarageError> {
+	let ReqCtx { garage, .. } = ctx;
+
 	let mut version = version.clone();
 	version.blocks.put(
 		VersionBlockKey {
@@ -597,7 +595,7 @@ pub(crate) fn get_headers(headers: &HeaderMap<HeaderValue>) -> Result<ObjectVers
 	// Preserve x-amz-meta- headers
 	for (k, v) in headers.iter() {
 		if k.as_str().starts_with("x-amz-meta-") {
-			match v.to_str() {
+			match std::str::from_utf8(v.as_bytes()) {
 				Ok(v_str) => {
 					other.insert(k.to_string(), v_str.to_string());
 				}

src/api/s3/website.rs

@@ -1,5 +1,4 @@
 use quick_xml::de::from_reader;
-use std::sync::Arc;
 
 use http_body_util::BodyExt;
 use hyper::{Request, Response, StatusCode};
@@ -12,15 +11,11 @@ use crate::s3::xml::{to_xml_with_header, xmlns_tag, IntValue, Value};
 use crate::signature::verify_signed_content;
 
 use garage_model::bucket_table::*;
-use garage_model::garage::Garage;
 use garage_util::data::*;
 
-pub async fn handle_get_website(bucket: &Bucket) -> Result<Response<ResBody>, Error> {
+pub async fn handle_get_website(ctx: ReqCtx) -> Result<Response<ResBody>, Error> {
-	let param = bucket
+	let ReqCtx { bucket_params, .. } = ctx;
-		.params()
+	if let Some(website) = bucket_params.website_config.get() {
-		.ok_or_internal_error("Bucket should not be deleted at this point")?;
-
-	if let Some(website) = param.website_config.get() {
 		let wc = WebsiteConfiguration {
 			xmlns: (),
 			error_document: website.error_document.as_ref().map(|v| Key {
@@ -44,16 +39,18 @@ pub async fn handle_get_website(bucket: &Bucket) -> Result<Response<ResBody>, Er
 	}
 }
 
-pub async fn handle_delete_website(
+pub async fn handle_delete_website(ctx: ReqCtx) -> Result<Response<ResBody>, Error> {
-	garage: Arc<Garage>,
+	let ReqCtx {
-	mut bucket: Bucket,
+		garage,
-) -> Result<Response<ResBody>, Error> {
+		bucket_id,
-	let param = bucket
+		mut bucket_params,
-		.params_mut()
+		..
-		.ok_or_internal_error("Bucket should not be deleted at this point")?;
+	} = ctx;
-
+	bucket_params.website_config.update(None);
-	param.website_config.update(None);
+	garage
-	garage.bucket_table.insert(&bucket).await?;
+		.bucket_table
+		.insert(&Bucket::present(bucket_id, bucket_params))
+		.await?;
 
 	Ok(Response::builder()
 		.status(StatusCode::NO_CONTENT)
@@ -61,28 +58,33 @@ pub async fn handle_delete_website(
 }
 
 pub async fn handle_put_website(
-	garage: Arc<Garage>,
+	ctx: ReqCtx,
-	mut bucket: Bucket,
 	req: Request<ReqBody>,
 	content_sha256: Option<Hash>,
 ) -> Result<Response<ResBody>, Error> {
+	let ReqCtx {
+		garage,
+		bucket_id,
+		mut bucket_params,
+		..
+	} = ctx;
+
 	let body = BodyExt::collect(req.into_body()).await?.to_bytes();
 
 	if let Some(content_sha256) = content_sha256 {
 		verify_signed_content(content_sha256, &body[..])?;
 	}
 
-	let param = bucket
-		.params_mut()
-		.ok_or_internal_error("Bucket should not be deleted at this point")?;
-
 	let conf: WebsiteConfiguration = from_reader(&body as &[u8])?;
 	conf.validate()?;
 
-	param
+	bucket_params
 		.website_config
 		.update(Some(conf.into_garage_website_config()?));
-	garage.bucket_table.insert(&bucket).await?;
+	garage
+		.bucket_table
+		.insert(&Bucket::present(bucket_id, bucket_params))
+		.await?;
 
 	Ok(Response::builder()
 		.status(StatusCode::OK)

src/api/signature/payload.rs

@@ -31,7 +31,13 @@ pub const AWS4_HMAC_SHA256: &str = "AWS4-HMAC-SHA256";
 pub const UNSIGNED_PAYLOAD: &str = "UNSIGNED-PAYLOAD";
 pub const STREAMING_AWS4_HMAC_SHA256_PAYLOAD: &str = "STREAMING-AWS4-HMAC-SHA256-PAYLOAD";
 
-pub type QueryMap = HashMap<String, String>;
+pub type QueryMap = HeaderMap<QueryValue>;
+pub struct QueryValue {
+	/// Original key with potential uppercase characters,
+	/// for use in signature calculation
+	key: String,
+	value: String,
+}
 
 pub async fn check_payload_signature(
 	garage: &Garage,
@@ -40,7 +46,7 @@ pub async fn check_payload_signature(
 ) -> Result<(Option<Key>, Option<Hash>), Error> {
 	let query = parse_query_map(request.uri())?;
 
-	if query.contains_key(X_AMZ_ALGORITHM.as_str()) {
+	if query.contains_key(&X_AMZ_ALGORITHM) {
 		// We check for presigned-URL-style authentification first, because
 		// the browser or someting else could inject an Authorization header
 		// that is totally unrelated to AWS signatures.
@@ -122,8 +128,8 @@ async fn check_presigned_signature(
 	request: &mut Request<IncomingBody>,
 	mut query: QueryMap,
 ) -> Result<(Option<Key>, Option<Hash>), Error> {
-	let algorithm = query.get(X_AMZ_ALGORITHM.as_str()).unwrap();
+	let algorithm = query.get(&X_AMZ_ALGORITHM).unwrap();
-	let authorization = Authorization::parse_presigned(algorithm, &query)?;
+	let authorization = Authorization::parse_presigned(&algorithm.value, &query)?;
 
 	// Verify that all necessary request headers are included in signed_headers
 	// For AWSv4 pre-signed URLs, the following must be incldued:
@@ -136,7 +142,7 @@ async fn check_presigned_signature(
 	// but the signature cannot be computed from a string that contains itself.
 	// AWS specifies that all query params except X-Amz-Signature are included
 	// in the canonical request.
-	query.remove(X_AMZ_SIGNATURE.as_str());
+	query.remove(&X_AMZ_SIGNATURE);
 	let canonical_request = canonical_request(
 		service,
 		request.method(),
@@ -162,10 +168,8 @@ async fn check_presigned_signature(
 	// then an InvalidRequest error is raised.
 	let headers_mut = request.headers_mut();
 	for (name, value) in query.iter() {
-		let name =
+		if let Some(existing) = headers_mut.get(name) {
-			HeaderName::from_bytes(name.as_bytes()).ok_or_bad_request("Invalid header name")?;
+			if signed_headers.contains(&name) && existing.as_bytes() != value.value.as_bytes() {
-		if let Some(existing) = headers_mut.get(&name) {
-			if signed_headers.contains(&name) && existing.as_bytes() != value.as_bytes() {
 				return Err(Error::bad_request(format!(
 					"Conflicting values for `{}` in query parameters and request headers",
 					name
@@ -181,7 +185,7 @@ async fn check_presigned_signature(
 			// that are not signed, however there is not much reason that this would happen)
 			headers_mut.insert(
 				name,
-				HeaderValue::from_bytes(value.as_bytes())
+				HeaderValue::from_bytes(value.value.as_bytes())
 					.ok_or_bad_request("invalid query parameter value")?,
 			);
 		}
@@ -193,11 +197,19 @@ async fn check_presigned_signature(
 }
 
 pub fn parse_query_map(uri: &http::uri::Uri) -> Result<QueryMap, Error> {
-	let mut query = QueryMap::new();
+	let mut query = QueryMap::with_capacity(0);
 	if let Some(query_str) = uri.query() {
 		let query_pairs = url::form_urlencoded::parse(query_str.as_bytes());
 		for (key, val) in query_pairs {
-			if query.insert(key.to_string(), val.into_owned()).is_some() {
+			let name =
+				HeaderName::from_bytes(key.as_bytes()).ok_or_bad_request("Invalid header name")?;
+
+			let value = QueryValue {
+				key: key.to_string(),
+				value: val.into_owned(),
+			};
+
+			if query.insert(name, value).is_some() {
 				return Err(Error::bad_request(format!(
 					"duplicate query parameter: `{}`",
 					key
@@ -306,7 +318,7 @@ pub fn canonical_request(
 	// Canonical query string from passed HeaderMap
 	let canonical_query_string = {
 		let mut items = Vec::with_capacity(query.len());
-		for (key, value) in query.iter() {
+		for (_, QueryValue { key, value }) in query.iter() {
 			items.push(uri_encode(&key, true) + "=" + &uri_encode(&value, true));
 		}
 		items.sort();
@@ -319,8 +331,8 @@ pub fn canonical_request(
 		.map(|name| {
 			let value = headers
 				.get(name)
-				.ok_or_bad_request(format!("signed header `{}` is not present", name))?
+				.ok_or_bad_request(format!("signed header `{}` is not present", name))?;
-				.to_str()?;
+			let value = std::str::from_utf8(value.as_bytes())?;
 			Ok(format!("{}:{}", name.as_str(), value.trim()))
 		})
 		.collect::<Result<Vec<String>, Error>>()?
@@ -464,18 +476,19 @@ impl Authorization {
 		}
 
 		let cred = query
-			.get(X_AMZ_CREDENTIAL.as_str())
+			.get(&X_AMZ_CREDENTIAL)
 			.ok_or_bad_request("X-Amz-Credential not found in query parameters")?;
 		let signed_headers = query
-			.get(X_AMZ_SIGNEDHEADERS.as_str())
+			.get(&X_AMZ_SIGNEDHEADERS)
 			.ok_or_bad_request("X-Amz-SignedHeaders not found in query parameters")?;
 		let signature = query
-			.get(X_AMZ_SIGNATURE.as_str())
+			.get(&X_AMZ_SIGNATURE)
 			.ok_or_bad_request("X-Amz-Signature not found in query parameters")?;
 
 		let duration = query
-			.get(X_AMZ_EXPIRES.as_str())
+			.get(&X_AMZ_EXPIRES)
 			.ok_or_bad_request("X-Amz-Expires not found in query parameters")?
+			.value
 			.parse()
 			.map_err(|_| Error::bad_request("X-Amz-Expires is not a number".to_string()))?;
 
@@ -486,20 +499,20 @@ impl Authorization {
 		}
 
 		let date = query
-			.get(X_AMZ_DATE.as_str())
+			.get(&X_AMZ_DATE)
 			.ok_or_bad_request("Missing X-Amz-Date field")?;
-		let date = parse_date(date)?;
+		let date = parse_date(&date.value)?;
 
 		if Utc::now() - date > Duration::seconds(duration) {
 			return Err(Error::bad_request("Date is too old".to_string()));
 		}
 
-		let (key_id, scope) = parse_credential(cred)?;
+		let (key_id, scope) = parse_credential(&cred.value)?;
 		Ok(Authorization {
 			key_id,
 			scope,
-			signed_headers: signed_headers.to_string(),
+			signed_headers: signed_headers.value.clone(),
-			signature: signature.to_string(),
+			signature: signature.value.clone(),
 			content_sha256: UNSIGNED_PAYLOAD.to_string(),
 			date,
 		})

src/block/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "garage_block"
-version = "0.9.2"
+version = "0.9.4"
 authors = ["Alex Auvolat <alex@adnab.me>"]
 edition = "2018"
 license = "AGPL-3.0"

src/block/layout.rs

@@ -1,3 +1,4 @@
+use std::collections::HashMap;
 use std::path::PathBuf;
 
 use serde::{Deserialize, Serialize};
@@ -13,9 +14,12 @@ const DRIVE_NPART: usize = 1024;
 
 const HASH_DRIVE_BYTES: (usize, usize) = (2, 3);
 
+const MARKER_FILE_NAME: &str = "garage-marker";
+
 #[derive(Serialize, Deserialize, Debug, Clone)]
 pub(crate) struct DataLayout {
 	pub(crate) data_dirs: Vec<DataDir>,
+	markers: HashMap<PathBuf, String>,
 
 	/// Primary storage location (index in data_dirs) for each partition
 	/// = the location where the data is supposed to be, blocks are always
@@ -75,16 +79,17 @@ impl DataLayout {
 
 		Ok(Self {
 			data_dirs,
+			markers: HashMap::new(),
 			part_prim,
 			part_sec,
 		})
 	}
 
-	pub(crate) fn update(&mut self, dirs: &DataDirEnum) -> Result<(), Error> {
+	pub(crate) fn update(self, dirs: &DataDirEnum) -> Result<Self, Error> {
 		// Make list of new data directories, exit if nothing changed
 		let data_dirs = make_data_dirs(dirs)?;
 		if data_dirs == self.data_dirs {
-			return Ok(());
+			return Ok(self);
 		}
 
 		let total_cap = data_dirs.iter().filter_map(|x| x.capacity()).sum::<u64>();
@@ -214,11 +219,43 @@ impl DataLayout {
 		}
 
 		// Apply newly generated config
-		*self = Self {
+		Ok(Self {
 			data_dirs,
+			markers: self.markers,
 			part_prim,
 			part_sec,
-		};
+		})
+	}
+
+	pub(crate) fn check_markers(&mut self) -> Result<(), Error> {
+		let data_dirs = &self.data_dirs;
+		self.markers
+			.retain(|k, _| data_dirs.iter().any(|x| x.path == *k));
+
+		for dir in self.data_dirs.iter() {
+			let mut marker_path = dir.path.clone();
+			marker_path.push(MARKER_FILE_NAME);
+			let existing_marker = std::fs::read_to_string(&marker_path).ok();
+			match (existing_marker, self.markers.get(&dir.path)) {
+				(Some(m1), Some(m2)) => {
+					if m1 != *m2 {
+						return Err(Error::Message(format!("Mismatched content for marker file `{}` in data directory `{}`. If you moved data directories or changed their mountpoints, you should remove the `data_layout` file in Garage's metadata directory and restart Garage.", MARKER_FILE_NAME, dir.path.display())));
+					}
+				}
+				(None, Some(_)) => {
+					return Err(Error::Message(format!("Could not find expected marker file `{}` in data directory `{}`, make sure this data directory is mounted correctly.", MARKER_FILE_NAME, dir.path.display())));
+				}
+				(Some(mkr), None) => {
+					self.markers.insert(dir.path.clone(), mkr);
+				}
+				(None, None) => {
+					let mkr = hex::encode(garage_util::data::gen_uuid().as_slice());
+					std::fs::write(&marker_path, &mkr)?;
+					self.markers.insert(dir.path.clone(), mkr);
+				}
+			}
+		}
+
 		Ok(())
 	}
 
@@ -255,6 +292,7 @@ impl DataLayout {
 	pub(crate) fn without_secondary_locations(&self) -> Self {
 		Self {
 			data_dirs: self.data_dirs.clone(),
+			markers: self.markers.clone(),
 			part_prim: self.part_prim.clone(),
 			part_sec: self.part_sec.iter().map(|_| vec![]).collect::<Vec<_>>(),
 		}
@@ -322,14 +360,12 @@ fn make_data_dirs(dirs: &DataDirEnum) -> Result<Vec<DataDir>, Error> {
 fn dir_not_empty(path: &PathBuf) -> Result<bool, Error> {
 	for entry in std::fs::read_dir(&path)? {
 		let dir = entry?;
-		if dir.file_type()?.is_dir()
+		let ft = dir.file_type()?;
-			&& dir
+		let name = dir.file_name().into_string().ok();
-				.file_name()
+		if ft.is_file() && name.as_deref() == Some(MARKER_FILE_NAME) {
-				.into_string()
+			return Ok(true);
-				.ok()
+		}
-				.and_then(|hex| hex::decode(&hex).ok())
+		if ft.is_dir() && name.and_then(|hex| hex::decode(&hex).ok()).is_some() {
-				.is_some()
-		{
 			return Ok(true);
 		}
 	}

src/block/manager.rs

@@ -1,3 +1,4 @@
+use std::convert::TryInto;
 use std::path::PathBuf;
 use std::sync::Arc;
 use std::time::Duration;
@@ -10,7 +11,7 @@ use serde::{Deserialize, Serialize};
 
 use tokio::fs;
 use tokio::io::{AsyncReadExt, AsyncWriteExt, BufReader};
-use tokio::sync::{mpsc, Mutex, MutexGuard};
+use tokio::sync::{mpsc, Mutex, MutexGuard, Semaphore};
 
 use opentelemetry::{
 	trace::{FutureExt as OtelFutureExt, TraceContextExt, Tracer},
@@ -22,7 +23,7 @@ use garage_net::stream::{read_stream_to_end, stream_asyncread, ByteStream};
 use garage_db as db;
 
 use garage_util::background::{vars, BackgroundRunner};
-use garage_util::config::DataDirEnum;
+use garage_util::config::Config;
 use garage_util::data::*;
 use garage_util::error::*;
 use garage_util::metrics::RecordDuration;
@@ -84,6 +85,7 @@ pub struct BlockManager {
 
 	data_fsync: bool,
 	compression_level: Option<i32>,
+	disable_scrub: bool,
 
 	mutation_lock: Vec<Mutex<BlockManagerLocked>>,
 
@@ -92,6 +94,7 @@ pub struct BlockManager {
 
 	pub(crate) system: Arc<System>,
 	pub(crate) endpoint: Arc<Endpoint<BlockRpc, Self>>,
+	buffer_kb_semaphore: Arc<Semaphore>,
 
 	pub(crate) metrics: BlockManagerMetrics,
 
@@ -119,24 +122,22 @@ struct BlockManagerLocked();
 impl BlockManager {
 	pub fn new(
 		db: &db::Db,
-		data_dir: DataDirEnum,
+		config: &Config,
-		data_fsync: bool,
-		compression_level: Option<i32>,
 		replication: TableShardedReplication,
 		system: Arc<System>,
 	) -> Result<Arc<Self>, Error> {
 		// Load or compute layout, i.e. assignment of data blocks to the different data directories
 		let data_layout_persister: Persister<DataLayout> =
 			Persister::new(&system.metadata_dir, "data_layout");
-		let data_layout = match data_layout_persister.load() {
+		let mut data_layout = match data_layout_persister.load() {
-			Ok(mut layout) => {
+			Ok(layout) => layout
-				layout
+				.update(&config.data_dir)
-					.update(&data_dir)
+				.ok_or_message("invalid data_dir config")?,
-					.ok_or_message("invalid data_dir config")?;
+			Err(_) => {
-				layout
+				DataLayout::initialize(&config.data_dir).ok_or_message("invalid data_dir config")?
 			}
-			Err(_) => DataLayout::initialize(&data_dir).ok_or_message("invalid data_dir config")?,
 		};
+		data_layout.check_markers()?;
 		data_layout_persister
 			.save(&data_layout)
 			.expect("cannot save data_layout");
@@ -153,11 +154,14 @@ impl BlockManager {
 			.netapp
 			.endpoint("garage_block/manager.rs/Rpc".to_string());
 
+		let buffer_kb_semaphore = Arc::new(Semaphore::new(config.block_ram_buffer_max / 1024));
+
 		let metrics = BlockManagerMetrics::new(
-			compression_level,
+			config.compression_level,
 			rc.rc.clone(),
 			resync.queue.clone(),
 			resync.errors.clone(),
+			buffer_kb_semaphore.clone(),
 		);
 
 		let scrub_persister = PersisterShared::new(&system.metadata_dir, "scrub_info");
@@ -166,8 +170,9 @@ impl BlockManager {
 			replication,
 			data_layout: ArcSwap::new(Arc::new(data_layout)),
 			data_layout_persister,
-			data_fsync,
+			data_fsync: config.data_fsync,
-			compression_level,
+			disable_scrub: config.disable_scrub,
+			compression_level: config.compression_level,
 			mutation_lock: vec![(); MUTEX_COUNT]
 				.iter()
 				.map(|_| Mutex::new(BlockManagerLocked()))
@@ -176,6 +181,7 @@ impl BlockManager {
 			resync,
 			system,
 			endpoint,
+			buffer_kb_semaphore,
 			metrics,
 			scrub_persister,
 			tx_scrub_command: ArcSwapOption::new(None),
@@ -194,33 +200,37 @@ impl BlockManager {
 		}
 
 		// Spawn scrub worker
-		let (scrub_tx, scrub_rx) = mpsc::channel(1);
+		if !self.disable_scrub {
-		self.tx_scrub_command.store(Some(Arc::new(scrub_tx)));
+			let (scrub_tx, scrub_rx) = mpsc::channel(1);
-		bg.spawn_worker(ScrubWorker::new(
+			self.tx_scrub_command.store(Some(Arc::new(scrub_tx)));
-			self.clone(),
+			bg.spawn_worker(ScrubWorker::new(
-			scrub_rx,
+				self.clone(),
-			self.scrub_persister.clone(),
+				scrub_rx,
-		));
+				self.scrub_persister.clone(),
+			));
+		}
 	}
 
 	pub fn register_bg_vars(&self, vars: &mut vars::BgVars) {
 		self.resync.register_bg_vars(vars);
 
-		vars.register_rw(
+		if !self.disable_scrub {
-			&self.scrub_persister,
+			vars.register_rw(
-			"scrub-tranquility",
+				&self.scrub_persister,
-			|p| p.get_with(|x| x.tranquility),
+				"scrub-tranquility",
-			|p, tranquility| p.set_with(|x| x.tranquility = tranquility),
+				|p| p.get_with(|x| x.tranquility),
-		);
+				|p, tranquility| p.set_with(|x| x.tranquility = tranquility),
-		vars.register_ro(&self.scrub_persister, "scrub-last-completed", |p| {
+			);
-			p.get_with(|x| msec_to_rfc3339(x.time_last_complete_scrub))
+			vars.register_ro(&self.scrub_persister, "scrub-last-completed", |p| {
-		});
+				p.get_with(|x| msec_to_rfc3339(x.time_last_complete_scrub))
-		vars.register_ro(&self.scrub_persister, "scrub-next-run", |p| {
+			});
-			p.get_with(|x| msec_to_rfc3339(x.time_next_run_scrub))
+			vars.register_ro(&self.scrub_persister, "scrub-next-run", |p| {
-		});
+				p.get_with(|x| msec_to_rfc3339(x.time_next_run_scrub))
-		vars.register_ro(&self.scrub_persister, "scrub-corruptions_detected", |p| {
+			});
-			p.get_with(|x| x.corruptions_detected)
+			vars.register_ro(&self.scrub_persister, "scrub-corruptions_detected", |p| {
-		});
+				p.get_with(|x| x.corruptions_detected)
+			});
+		}
 	}
 
 	/// Ask nodes that might have a (possibly compressed) block for it
@@ -228,10 +238,16 @@ impl BlockManager {
 	async fn rpc_get_raw_block_streaming(
 		&self,
 		hash: &Hash,
+		priority: RequestPriority,
 		order_tag: Option<OrderTag>,
 	) -> Result<DataBlockStream, Error> {
-		self.rpc_get_raw_block_internal(hash, order_tag, |stream| async move { Ok(stream) })
+		self.rpc_get_raw_block_internal(
-			.await
+			hash,
+			priority,
+			order_tag,
+			|stream| async move { Ok(stream) },
+		)
+		.await
 	}
 
 	/// Ask nodes that might have a (possibly compressed) block for it
@@ -239,9 +255,10 @@ impl BlockManager {
 	pub(crate) async fn rpc_get_raw_block(
 		&self,
 		hash: &Hash,
+		priority: RequestPriority,
 		order_tag: Option<OrderTag>,
 	) -> Result<DataBlock, Error> {
-		self.rpc_get_raw_block_internal(hash, order_tag, |block_stream| async move {
+		self.rpc_get_raw_block_internal(hash, priority, order_tag, |block_stream| async move {
 			let (header, stream) = block_stream.into_parts();
 			read_stream_to_end(stream)
 				.await
@@ -254,6 +271,7 @@ impl BlockManager {
 	async fn rpc_get_raw_block_internal<F, Fut, T>(
 		&self,
 		hash: &Hash,
+		priority: RequestPriority,
 		order_tag: Option<OrderTag>,
 		f: F,
 	) -> Result<T, Error>
@@ -269,7 +287,7 @@ impl BlockManager {
 			let rpc = self.endpoint.call_streaming(
 				&node_id,
 				BlockRpc::GetBlock(*hash, order_tag),
-				PRIO_NORMAL | PRIO_SECONDARY,
+				priority,
 			);
 			tokio::select! {
 				res = rpc => {
@@ -321,7 +339,9 @@ impl BlockManager {
 		hash: &Hash,
 		order_tag: Option<OrderTag>,
 	) -> Result<ByteStream, Error> {
-		let block_stream = self.rpc_get_raw_block_streaming(hash, order_tag).await?;
+		let block_stream = self
+			.rpc_get_raw_block_streaming(hash, PRIO_NORMAL | PRIO_SECONDARY, order_tag)
+			.await?;
 		let (header, stream) = block_stream.into_parts();
 		match header {
 			DataBlockHeader::Plain => Ok(stream),
@@ -357,6 +377,14 @@ impl BlockManager {
 		let (header, bytes) = DataBlock::from_buffer(data, self.compression_level)
 			.await
 			.into_parts();
+
+		let permit = self
+			.buffer_kb_semaphore
+			.clone()
+			.acquire_many_owned((bytes.len() / 1024).try_into().unwrap())
+			.await
+			.ok_or_message("could not reserve space for buffer of data to send to remote nodes")?;
+
 		let put_block_rpc =
 			Req::new(BlockRpc::PutBlock { hash, header })?.with_stream_from_buffer(bytes);
 		let put_block_rpc = if let Some(tag) = order_tag {
@@ -372,6 +400,7 @@ impl BlockManager {
 				&who[..],
 				put_block_rpc,
 				RequestStrategy::with_priority(PRIO_NORMAL | PRIO_SECONDARY)
+					.with_drop_on_completion(permit)
 					.with_quorum(self.replication.write_quorum()),
 			)
 			.await?;

src/block/metrics.rs

@@ -1,3 +1,7 @@
+use std::sync::Arc;
+
+use tokio::sync::Semaphore;
+
 use opentelemetry::{global, metrics::*};
 
 use garage_db as db;
@@ -9,6 +13,7 @@ pub struct BlockManagerMetrics {
 	pub(crate) _rc_size: ValueObserver<u64>,
 	pub(crate) _resync_queue_len: ValueObserver<u64>,
 	pub(crate) _resync_errored_blocks: ValueObserver<u64>,
+	pub(crate) _buffer_free_kb: ValueObserver<u64>,
 
 	pub(crate) resync_counter: BoundCounter<u64>,
 	pub(crate) resync_error_counter: BoundCounter<u64>,
@@ -31,6 +36,7 @@ impl BlockManagerMetrics {
 		rc_tree: db::Tree,
 		resync_queue: CountedTree,
 		resync_errors: CountedTree,
+		buffer_semaphore: Arc<Semaphore>,
 	) -> Self {
 		let meter = global::meter("garage_model/block");
 		Self {
@@ -66,6 +72,15 @@ impl BlockManagerMetrics {
 				.with_description("Number of block hashes whose last resync resulted in an error")
 				.init(),
 
+			_buffer_free_kb: meter
+				.u64_value_observer("block.ram_buffer_free_kb", move |observer| {
+					observer.observe(buffer_semaphore.available_permits() as u64, &[])
+				})
+				.with_description(
+					"Available RAM in KiB to use for buffering data blocks to be written to remote nodes",
+				)
+				.init(),
+
 			resync_counter: meter
 				.u64_counter("block.resync_counter")
 				.with_description("Number of calls to resync_block")

src/block/resync.rs

@@ -436,7 +436,7 @@ impl BlockResyncManager {
 						&manager.endpoint,
 						&need_nodes[..],
 						put_block_message,
-						RequestStrategy::with_priority(PRIO_BACKGROUND)
+						RequestStrategy::with_priority(PRIO_BACKGROUND | PRIO_SECONDARY)
 							.with_quorum(need_nodes.len()),
 					)
 					.await
@@ -460,7 +460,9 @@ impl BlockResyncManager {
 				hash
 			);
 
-			let block_data = manager.rpc_get_raw_block(hash, None).await?;
+			let block_data = manager
+				.rpc_get_raw_block(hash, PRIO_BACKGROUND | PRIO_SECONDARY, None)
+				.await?;
 
 			manager.metrics.resync_recv_counter.add(1);
 

src/db/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "garage_db"
-version = "0.9.2"
+version = "0.9.4"
 authors = ["Alex Auvolat <alex@adnab.me>"]
 edition = "2018"
 license = "AGPL-3.0"
@@ -17,7 +17,9 @@ hexdump.workspace = true
 tracing.workspace = true
 
 heed = { workspace = true, optional = true }
-rusqlite = { workspace = true, optional = true }
+rusqlite = { workspace = true, optional = true, features = ["backup"] }
+r2d2 = { workspace = true, optional = true }
+r2d2_sqlite = { workspace = true, optional = true }
 sled = { workspace = true, optional = true }
 
 [dev-dependencies]
@@ -27,4 +29,4 @@ mktemp.workspace = true
 default = [ "sled", "lmdb", "sqlite" ]
 bundled-libs = [ "rusqlite?/bundled" ]
 lmdb = [ "heed" ]
-sqlite = [ "rusqlite" ]
+sqlite = [ "rusqlite", "r2d2", "r2d2_sqlite" ]

src/db/lib.rs

@@ -1,5 +1,4 @@
 #[macro_use]
-#[cfg(feature = "sqlite")]
 extern crate tracing;
 
 #[cfg(feature = "lmdb")]
@@ -11,6 +10,8 @@ pub mod sqlite_adapter;
 
 pub mod counted_tree_hack;
 
+pub mod open;
+
 #[cfg(test)]
 pub mod test;
 
@@ -18,10 +19,13 @@ use core::ops::{Bound, RangeBounds};
 
 use std::borrow::Cow;
 use std::cell::Cell;
+use std::path::PathBuf;
 use std::sync::Arc;
 
 use err_derive::Error;
 
+pub use open::*;
+
 pub(crate) type OnCommit = Vec<Box<dyn FnOnce()>>;
 
 #[derive(Clone)]
@@ -45,6 +49,12 @@ pub type TxValueIter<'a> = Box<dyn std::iter::Iterator<Item = TxOpResult<(Value,
 #[error(display = "{}", _0)]
 pub struct Error(pub Cow<'static, str>);
 
+impl From<std::io::Error> for Error {
+	fn from(e: std::io::Error) -> Error {
+		Error(format!("IO: {}", e).into())
+	}
+}
+
 pub type Result<T> = std::result::Result<T, Error>;
 
 #[derive(Debug, Error)]
@@ -126,6 +136,10 @@ impl Db {
 		}
 	}
 
+	pub fn snapshot(&self, path: &PathBuf) -> Result<()> {
+		self.0.snapshot(path)
+	}
+
 	pub fn import(&self, other: &Db) -> Result<()> {
 		let existing_trees = self.list_trees()?;
 		if !existing_trees.is_empty() {
@@ -171,48 +185,6 @@ impl Db {
 	}
 }
 
-/// List of supported database engine types
-///
-/// The `enum` holds list of *all* database engines that are are be supported by crate, no matter
-/// if relevant feature is enabled or not. It allows us to distinguish between invalid engine
-/// and valid engine, whose support is not enabled via feature flag.
-#[derive(Clone, Copy, Debug, PartialEq, Eq)]
-pub enum Engine {
-	Lmdb,
-	Sqlite,
-	Sled,
-}
-
-impl Engine {
-	/// Return variant name as static `&str`
-	pub fn as_str(&self) -> &'static str {
-		match self {
-			Self::Lmdb => "lmdb",
-			Self::Sqlite => "sqlite",
-			Self::Sled => "sled",
-		}
-	}
-}
-
-impl std::fmt::Display for Engine {
-	fn fmt(&self, fmt: &mut std::fmt::Formatter) -> std::fmt::Result {
-		self.as_str().fmt(fmt)
-	}
-}
-
-impl std::str::FromStr for Engine {
-	type Err = Error;
-
-	fn from_str(text: &str) -> Result<Engine> {
-		match text {
-			"lmdb" | "heed" => Ok(Self::Lmdb),
-			"sqlite" | "sqlite3" | "rusqlite" => Ok(Self::Sqlite),
-			"sled" => Ok(Self::Sled),
-			kind => Err(Error(format!("Invalid DB engine: {}", kind).into())),
-		}
-	}
-}
-
 #[allow(clippy::len_without_is_empty)]
 impl Tree {
 	#[inline]
@@ -364,6 +336,7 @@ pub(crate) trait IDb: Send + Sync {
 	fn engine(&self) -> String;
 	fn open_tree(&self, name: &str) -> Result<usize>;
 	fn list_trees(&self) -> Result<Vec<String>>;
+	fn snapshot(&self, path: &PathBuf) -> Result<()>;
 
 	fn get(&self, tree: usize, key: &[u8]) -> Result<Option<Value>>;
 	fn len(&self, tree: usize) -> Result<usize>;

src/db/lmdb_adapter.rs

@@ -3,6 +3,7 @@ use core::ptr::NonNull;
 
 use std::collections::HashMap;
 use std::convert::TryInto;
+use std::path::PathBuf;
 use std::sync::{Arc, RwLock};
 
 use heed::types::ByteSlice;
@@ -102,6 +103,15 @@ impl IDb for LmdbDb {
 		Ok(ret2)
 	}
 
+	fn snapshot(&self, to: &PathBuf) -> Result<()> {
+		std::fs::create_dir_all(to)?;
+		let mut path = to.clone();
+		path.push("data.mdb");
+		self.db
+			.copy_to_path(path, heed::CompactionOption::Disabled)?;
+		Ok(())
+	}
+
 	// ----
 
 	fn get(&self, tree: usize, key: &[u8]) -> Result<Option<Value>> {

src/db/open.rs

@@ -0,0 +1,147 @@
+use std::path::PathBuf;
+
+use crate::{Db, Error, Result};
+
+/// List of supported database engine types
+///
+/// The `enum` holds list of *all* database engines that are are be supported by crate, no matter
+/// if relevant feature is enabled or not. It allows us to distinguish between invalid engine
+/// and valid engine, whose support is not enabled via feature flag.
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub enum Engine {
+	Lmdb,
+	Sqlite,
+	Sled,
+}
+
+impl Engine {
+	/// Return variant name as static `&str`
+	pub fn as_str(&self) -> &'static str {
+		match self {
+			Self::Lmdb => "lmdb",
+			Self::Sqlite => "sqlite",
+			Self::Sled => "sled",
+		}
+	}
+}
+
+impl std::fmt::Display for Engine {
+	fn fmt(&self, fmt: &mut std::fmt::Formatter) -> std::fmt::Result {
+		self.as_str().fmt(fmt)
+	}
+}
+
+impl std::str::FromStr for Engine {
+	type Err = Error;
+
+	fn from_str(text: &str) -> Result<Engine> {
+		match text {
+			"lmdb" | "heed" => Ok(Self::Lmdb),
+			"sqlite" | "sqlite3" | "rusqlite" => Ok(Self::Sqlite),
+			"sled" => Ok(Self::Sled),
+			kind => Err(Error(
+				format!(
+					"Invalid DB engine: {} (options are: lmdb, sled, sqlite)",
+					kind
+				)
+				.into(),
+			)),
+		}
+	}
+}
+
+pub struct OpenOpt {
+	pub fsync: bool,
+	pub lmdb_map_size: Option<usize>,
+	pub sled_cache_capacity: usize,
+	pub sled_flush_every_ms: u64,
+}
+
+impl Default for OpenOpt {
+	fn default() -> Self {
+		Self {
+			fsync: false,
+			lmdb_map_size: None,
+			sled_cache_capacity: 1024 * 1024 * 1024,
+			sled_flush_every_ms: 2000,
+		}
+	}
+}
+
+pub fn open_db(path: &PathBuf, engine: Engine, opt: &OpenOpt) -> Result<Db> {
+	match engine {
+		// ---- Sled DB ----
+		#[cfg(feature = "sled")]
+		Engine::Sled => {
+			if opt.fsync {
+				return Err(Error(
+					"`metadata_fsync = true` is not supported with the Sled database engine".into(),
+				));
+			}
+			info!("Opening Sled database at: {}", path.display());
+			let db = crate::sled_adapter::sled::Config::default()
+				.path(&path)
+				.cache_capacity(opt.sled_cache_capacity as u64)
+				.flush_every_ms(Some(opt.sled_flush_every_ms))
+				.open()?;
+			Ok(crate::sled_adapter::SledDb::init(db))
+		}
+
+		// ---- Sqlite DB ----
+		#[cfg(feature = "sqlite")]
+		Engine::Sqlite => {
+			info!("Opening Sqlite database at: {}", path.display());
+			let manager = r2d2_sqlite::SqliteConnectionManager::file(path);
+			Ok(crate::sqlite_adapter::SqliteDb::new(manager, opt.fsync)?)
+		}
+
+		// ---- LMDB DB ----
+		#[cfg(feature = "lmdb")]
+		Engine::Lmdb => {
+			info!("Opening LMDB database at: {}", path.display());
+			if let Err(e) = std::fs::create_dir_all(&path) {
+				return Err(Error(
+					format!("Unable to create LMDB data directory: {}", e).into(),
+				));
+			}
+
+			let map_size = match opt.lmdb_map_size {
+				None => crate::lmdb_adapter::recommended_map_size(),
+				Some(v) => v - (v % 4096),
+			};
+
+			let mut env_builder = heed::EnvOpenOptions::new();
+			env_builder.max_dbs(100);
+			env_builder.map_size(map_size);
+			env_builder.max_readers(2048);
+			unsafe {
+				env_builder.flag(crate::lmdb_adapter::heed::flags::Flags::MdbNoMetaSync);
+				if !opt.fsync {
+					env_builder.flag(heed::flags::Flags::MdbNoSync);
+				}
+			}
+			match env_builder.open(&path) {
+				Err(heed::Error::Io(e)) if e.kind() == std::io::ErrorKind::OutOfMemory => {
+					return Err(Error(
+						"OutOfMemory error while trying to open LMDB database. This can happen \
+                        if your operating system is not allowing you to use sufficient virtual \
+                        memory address space. Please check that no limit is set (ulimit -v). \
+                        You may also try to set a smaller `lmdb_map_size` configuration parameter. \
+                        On 32-bit machines, you should probably switch to another database engine."
+							.into(),
+					))
+				}
+				Err(e) => Err(Error(format!("Cannot open LMDB database: {}", e).into())),
+				Ok(db) => Ok(crate::lmdb_adapter::LmdbDb::init(db)),
+			}
+		}
+
+		// Pattern is unreachable when all supported DB engines are compiled into binary. The allow
+		// attribute is added so that we won't have to change this match in case stop building
+		// support for one or more engines by default.
+		#[allow(unreachable_patterns)]
+		engine => Err(Error(
+			format!("DB engine support not available in this build: {}", engine).into(),
+		)),
+	}
+}

src/db/sled_adapter.rs

@@ -2,6 +2,7 @@ use core::ops::Bound;
 
 use std::cell::Cell;
 use std::collections::HashMap;
+use std::path::PathBuf;
 use std::sync::{Arc, RwLock};
 
 use sled::transaction::{
@@ -96,6 +97,13 @@ impl IDb for SledDb {
 		Ok(trees)
 	}
 
+	fn snapshot(&self, to: &PathBuf) -> Result<()> {
+		let to_db = sled::open(to)?;
+		let export = self.db.export();
+		to_db.import(export);
+		Ok(())
+	}
+
 	// ----
 
 	fn get(&self, tree: usize, key: &[u8]) -> Result<Option<Value>> {

src/db/sqlite_adapter.rs

@@ -1,12 +1,14 @@
 use core::ops::Bound;
 
-use std::borrow::BorrowMut;
 use std::marker::PhantomPinned;
+use std::path::PathBuf;
 use std::pin::Pin;
 use std::ptr::NonNull;
-use std::sync::{Arc, Mutex, MutexGuard};
+use std::sync::{Arc, Mutex, RwLock};
 
-use rusqlite::{params, Connection, Rows, Statement, Transaction};
+use r2d2::Pool;
+use r2d2_sqlite::SqliteConnectionManager;
+use rusqlite::{params, Rows, Statement, Transaction};
 
 use crate::{
 	Db, Error, IDb, ITx, ITxFn, OnCommit, Result, TxError, TxFnResult, TxOpError, TxOpResult,
@@ -15,6 +17,8 @@ use crate::{
 
 pub use rusqlite;
 
+type Connection = r2d2::PooledConnection<SqliteConnectionManager>;
+
 // --- err
 
 impl From<rusqlite::Error> for Error {
@@ -23,6 +27,12 @@ impl From<rusqlite::Error> for Error {
 	}
 }
 
+impl From<r2d2::Error> for Error {
+	fn from(e: r2d2::Error) -> Error {
+		Error(format!("Sqlite: {}", e).into())
+	}
+}
+
 impl From<rusqlite::Error> for TxOpError {
 	fn from(e: rusqlite::Error) -> TxOpError {
 		TxOpError(e.into())
@@ -31,35 +41,47 @@ impl From<rusqlite::Error> for TxOpError {
 
 // -- db
 
-pub struct SqliteDb(Mutex<SqliteDbInner>);
+pub struct SqliteDb {
-
+	db: Pool<SqliteConnectionManager>,
-struct SqliteDbInner {
+	trees: RwLock<Vec<Arc<str>>>,
-	db: Connection,
+	// All operations that might write on the DB must take this lock first.
-	trees: Vec<String>,
+	// This emulates LMDB's approach where a single writer can be
+	// active at once.
+	write_lock: Mutex<()>,
 }
 
 impl SqliteDb {
-	pub fn init(db: rusqlite::Connection) -> Db {
+	pub fn new(manager: SqliteConnectionManager, sync_mode: bool) -> Result<Db> {
-		let s = Self(Mutex::new(SqliteDbInner {
+		let manager = manager.with_init(move |db| {
-			db,
+			db.pragma_update(None, "journal_mode", "WAL")?;
-			trees: Vec::new(),
+			if sync_mode {
-		}));
+				db.pragma_update(None, "synchronous", "NORMAL")?;
-		Db(Arc::new(s))
+			} else {
+				db.pragma_update(None, "synchronous", "OFF")?;
+			}
+			Ok(())
+		});
+		let s = Self {
+			db: Pool::builder().build(manager)?,
+			trees: RwLock::new(vec![]),
+			write_lock: Mutex::new(()),
+		};
+		Ok(Db(Arc::new(s)))
 	}
 }
 
-impl SqliteDbInner {
+impl SqliteDb {
-	fn get_tree(&self, i: usize) -> Result<&'_ str> {
+	fn get_tree(&self, i: usize) -> Result<Arc<str>> {
 		self.trees
+			.read()
+			.unwrap()
 			.get(i)
-			.map(String::as_str)
+			.cloned()
 			.ok_or_else(|| Error("invalid tree id".into()))
 	}
 
-	fn internal_get(&self, tree: &str, key: &[u8]) -> Result<Option<Value>> {
+	fn internal_get(&self, db: &Connection, tree: &str, key: &[u8]) -> Result<Option<Value>> {
-		let mut stmt = self
+		let mut stmt = db.prepare(&format!("SELECT v FROM {} WHERE k = ?1", tree))?;
-			.db
-			.prepare(&format!("SELECT v FROM {} WHERE k = ?1", tree))?;
 		let mut res_iter = stmt.query([key])?;
 		match res_iter.next()? {
 			None => Ok(None),
@@ -75,13 +97,14 @@ impl IDb for SqliteDb {
 
 	fn open_tree(&self, name: &str) -> Result<usize> {
 		let name = format!("tree_{}", name.replace(':', "_COLON_"));
-		let mut this = self.0.lock().unwrap();
+		let mut trees = self.trees.write().unwrap();
 
-		if let Some(i) = this.trees.iter().position(|x| x == &name) {
+		if let Some(i) = trees.iter().position(|x| x.as_ref() == &name) {
 			Ok(i)
 		} else {
+			let db = self.db.get()?;
 			trace!("create table {}", name);
-			this.db.execute(
+			db.execute(
 				&format!(
 					"CREATE TABLE IF NOT EXISTS {} (
 						k BLOB PRIMARY KEY,
@@ -93,8 +116,8 @@ impl IDb for SqliteDb {
 			)?;
 			trace!("table created: {}, unlocking", name);
 
-			let i = this.trees.len();
+			let i = trees.len();
-			this.trees.push(name.to_string());
+			trees.push(name.to_string().into_boxed_str().into());
 			Ok(i)
 		}
 	}
@@ -102,11 +125,8 @@ impl IDb for SqliteDb {
 	fn list_trees(&self) -> Result<Vec<String>> {
 		let mut trees = vec![];
 
-		trace!("list_trees: lock db");
+		let db = self.db.get()?;
-		let this = self.0.lock().unwrap();
+		let mut stmt = db.prepare(
-		trace!("list_trees: lock acquired");
-
-		let mut stmt = this.db.prepare(
 			"SELECT name FROM sqlite_schema WHERE type = 'table' AND name LIKE 'tree_%'",
 		)?;
 		let mut rows = stmt.query([])?;
@@ -119,24 +139,29 @@ impl IDb for SqliteDb {
 		Ok(trees)
 	}
 
+	fn snapshot(&self, to: &PathBuf) -> Result<()> {
+		fn progress(p: rusqlite::backup::Progress) {
+			let percent = (p.pagecount - p.remaining) * 100 / p.pagecount;
+			info!("Sqlite snapshot progres: {}%", percent);
+		}
+		self.db
+			.get()?
+			.backup(rusqlite::DatabaseName::Main, to, Some(progress))?;
+		Ok(())
+	}
+
 	// ----
 
 	fn get(&self, tree: usize, key: &[u8]) -> Result<Option<Value>> {
-		trace!("get {}: lock db", tree);
+		let tree = self.get_tree(tree)?;
-		let this = self.0.lock().unwrap();
+		self.internal_get(&self.db.get()?, &tree, key)
-		trace!("get {}: lock acquired", tree);
-
-		let tree = this.get_tree(tree)?;
-		this.internal_get(tree, key)
 	}
 
 	fn len(&self, tree: usize) -> Result<usize> {
-		trace!("len {}: lock db", tree);
+		let tree = self.get_tree(tree)?;
-		let this = self.0.lock().unwrap();
+		let db = self.db.get()?;
-		trace!("len {}: lock acquired", tree);
 
-		let tree = this.get_tree(tree)?;
+		let mut stmt = db.prepare(&format!("SELECT COUNT(*) FROM {}", tree))?;
-		let mut stmt = this.db.prepare(&format!("SELECT COUNT(*) FROM {}", tree))?;
 		let mut res_iter = stmt.query([])?;
 		match res_iter.next()? {
 			None => Ok(0),
@@ -149,69 +174,60 @@ impl IDb for SqliteDb {
 	}
 
 	fn insert(&self, tree: usize, key: &[u8], value: &[u8]) -> Result<Option<Value>> {
-		trace!("insert {}: lock db", tree);
+		let tree = self.get_tree(tree)?;
-		let this = self.0.lock().unwrap();
+		let db = self.db.get()?;
-		trace!("insert {}: lock acquired", tree);
+		let lock = self.write_lock.lock();
 
-		let tree = this.get_tree(tree)?;
+		let old_val = self.internal_get(&db, &tree, key)?;
-		let old_val = this.internal_get(tree, key)?;
 
 		let sql = match &old_val {
 			Some(_) => format!("UPDATE {} SET v = ?2 WHERE k = ?1", tree),
 			None => format!("INSERT INTO {} (k, v) VALUES (?1, ?2)", tree),
 		};
-		let n = this.db.execute(&sql, params![key, value])?;
+		let n = db.execute(&sql, params![key, value])?;
 		assert_eq!(n, 1);
 
+		drop(lock);
 		Ok(old_val)
 	}
 
 	fn remove(&self, tree: usize, key: &[u8]) -> Result<Option<Value>> {
-		trace!("remove {}: lock db", tree);
+		let tree = self.get_tree(tree)?;
-		let this = self.0.lock().unwrap();
+		let db = self.db.get()?;
-		trace!("remove {}: lock acquired", tree);
+		let lock = self.write_lock.lock();
 
-		let tree = this.get_tree(tree)?;
+		let old_val = self.internal_get(&db, &tree, key)?;
-		let old_val = this.internal_get(tree, key)?;
 
 		if old_val.is_some() {
-			let n = this
+			let n = db.execute(&format!("DELETE FROM {} WHERE k = ?1", tree), params![key])?;
-				.db
-				.execute(&format!("DELETE FROM {} WHERE k = ?1", tree), params![key])?;
 			assert_eq!(n, 1);
 		}
 
+		drop(lock);
 		Ok(old_val)
 	}
 
 	fn clear(&self, tree: usize) -> Result<()> {
-		trace!("clear {}: lock db", tree);
+		let tree = self.get_tree(tree)?;
-		let this = self.0.lock().unwrap();
+		let db = self.db.get()?;
-		trace!("clear {}: lock acquired", tree);
+		let lock = self.write_lock.lock();
 
-		let tree = this.get_tree(tree)?;
+		db.execute(&format!("DELETE FROM {}", tree), [])?;
-		this.db.execute(&format!("DELETE FROM {}", tree), [])?;
+
+		drop(lock);
 		Ok(())
 	}
 
 	fn iter(&self, tree: usize) -> Result<ValueIter<'_>> {
-		trace!("iter {}: lock db", tree);
+		let tree = self.get_tree(tree)?;
-		let this = self.0.lock().unwrap();
-		trace!("iter {}: lock acquired", tree);
-
-		let tree = this.get_tree(tree)?;
 		let sql = format!("SELECT k, v FROM {} ORDER BY k ASC", tree);
-		DbValueIterator::make(this, &sql, [])
+		DbValueIterator::make(self.db.get()?, &sql, [])
 	}
 
 	fn iter_rev(&self, tree: usize) -> Result<ValueIter<'_>> {
-		trace!("iter_rev {}: lock db", tree);
+		let tree = self.get_tree(tree)?;
-		let this = self.0.lock().unwrap();
-		trace!("iter_rev {}: lock acquired", tree);
-
-		let tree = this.get_tree(tree)?;
 		let sql = format!("SELECT k, v FROM {} ORDER BY k DESC", tree);
-		DbValueIterator::make(this, &sql, [])
+		DbValueIterator::make(self.db.get()?, &sql, [])
 	}
 
 	fn range<'r>(
@@ -220,11 +236,7 @@ impl IDb for SqliteDb {
 		low: Bound<&'r [u8]>,
 		high: Bound<&'r [u8]>,
 	) -> Result<ValueIter<'_>> {
-		trace!("range {}: lock db", tree);
+		let tree = self.get_tree(tree)?;
-		let this = self.0.lock().unwrap();
-		trace!("range {}: lock acquired", tree);
-
-		let tree = this.get_tree(tree)?;
 
 		let (bounds_sql, params) = bounds_sql(low, high);
 		let sql = format!("SELECT k, v FROM {} {} ORDER BY k ASC", tree, bounds_sql);
@@ -234,7 +246,7 @@ impl IDb for SqliteDb {
 			.map(|x| x as &dyn rusqlite::ToSql)
 			.collect::<Vec<_>>();
 
-		DbValueIterator::make::<&[&dyn rusqlite::ToSql]>(this, &sql, params.as_ref())
+		DbValueIterator::make::<&[&dyn rusqlite::ToSql]>(self.db.get()?, &sql, params.as_ref())
 	}
 	fn range_rev<'r>(
 		&self,
@@ -242,11 +254,7 @@ impl IDb for SqliteDb {
 		low: Bound<&'r [u8]>,
 		high: Bound<&'r [u8]>,
 	) -> Result<ValueIter<'_>> {
-		trace!("range_rev {}: lock db", tree);
+		let tree = self.get_tree(tree)?;
-		let this = self.0.lock().unwrap();
-		trace!("range_rev {}: lock acquired", tree);
-
-		let tree = this.get_tree(tree)?;
 
 		let (bounds_sql, params) = bounds_sql(low, high);
 		let sql = format!("SELECT k, v FROM {} {} ORDER BY k DESC", tree, bounds_sql);
@@ -256,25 +264,20 @@ impl IDb for SqliteDb {
 			.map(|x| x as &dyn rusqlite::ToSql)
 			.collect::<Vec<_>>();
 
-		DbValueIterator::make::<&[&dyn rusqlite::ToSql]>(this, &sql, params.as_ref())
+		DbValueIterator::make::<&[&dyn rusqlite::ToSql]>(self.db.get()?, &sql, params.as_ref())
 	}
 
 	// ----
 
 	fn transaction(&self, f: &dyn ITxFn) -> TxResult<OnCommit, ()> {
-		trace!("transaction: lock db");
+		let mut db = self.db.get().map_err(Error::from).map_err(TxError::Db)?;
-		let mut this = self.0.lock().unwrap();
+		let trees = self.trees.read().unwrap();
-		trace!("transaction: lock acquired");
+		let lock = self.write_lock.lock();
-
-		let this_mut_ref: &mut SqliteDbInner = this.borrow_mut();
 
+		trace!("trying transaction");
 		let mut tx = SqliteTx {
-			tx: this_mut_ref
+			tx: db.transaction().map_err(Error::from).map_err(TxError::Db)?,
-				.db
+			trees: &trees,
-				.transaction()
-				.map_err(Error::from)
-				.map_err(TxError::Db)?,
-			trees: &this_mut_ref.trees,
 		};
 		let res = match f.try_on(&mut tx) {
 			TxFnResult::Ok(on_commit) => {
@@ -294,7 +297,8 @@ impl IDb for SqliteDb {
 		};
 
 		trace!("transaction done");
-		res
+		drop(lock);
+		return res;
 	}
 }
 
@@ -302,12 +306,12 @@ impl IDb for SqliteDb {
 
 struct SqliteTx<'a> {
 	tx: Transaction<'a>,
-	trees: &'a [String],
+	trees: &'a [Arc<str>],
 }
 
 impl<'a> SqliteTx<'a> {
 	fn get_tree(&self, i: usize) -> TxOpResult<&'_ str> {
-		self.trees.get(i).map(String::as_ref).ok_or_else(|| {
+		self.trees.get(i).map(Arc::as_ref).ok_or_else(|| {
 			TxOpError(Error(
 				"invalid tree id (it might have been openned after the transaction started)".into(),
 			))
@@ -396,18 +400,14 @@ impl<'a> ITx for SqliteTx<'a> {
 // ----
 
 struct DbValueIterator<'a> {
-	db: MutexGuard<'a, SqliteDbInner>,
+	db: Connection,
 	stmt: Option<Statement<'a>>,
 	iter: Option<Rows<'a>>,
 	_pin: PhantomPinned,
 }
 
 impl<'a> DbValueIterator<'a> {
-	fn make<P: rusqlite::Params>(
+	fn make<P: rusqlite::Params>(db: Connection, sql: &str, args: P) -> Result<ValueIter<'a>> {
-		db: MutexGuard<'a, SqliteDbInner>,
-		sql: &str,
-		args: P,
-	) -> Result<ValueIter<'a>> {
 		let res = DbValueIterator {
 			db,
 			stmt: None,
@@ -419,7 +419,7 @@ impl<'a> DbValueIterator<'a> {
 
 		unsafe {
 			let db = NonNull::from(&boxed.db);
-			let stmt = db.as_ref().db.prepare(sql)?;
+			let stmt = db.as_ref().prepare(sql)?;
 
 			let mut_ref: Pin<&mut DbValueIterator<'a>> = Pin::as_mut(&mut boxed);
 			Pin::get_unchecked_mut(mut_ref).stmt = Some(stmt);

src/db/test.rs

@@ -106,6 +106,7 @@ fn test_sled_db() {
 fn test_sqlite_db() {
 	use crate::sqlite_adapter::SqliteDb;
 
-	let db = SqliteDb::init(rusqlite::Connection::open_in_memory().unwrap());
+	let manager = r2d2_sqlite::SqliteConnectionManager::memory();
+	let db = SqliteDb::new(manager, false).unwrap();
 	test_suite(db);
 }

src/garage/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "garage"
-version = "0.9.2"
+version = "0.9.4"
 authors = ["Alex Auvolat <alex@adnab.me>"]
 edition = "2018"
 license = "AGPL-3.0"
@@ -58,6 +58,7 @@ opentelemetry.workspace = true
 opentelemetry-prometheus = { workspace = true, optional = true }
 opentelemetry-otlp = { workspace = true, optional = true }
 prometheus = { workspace = true, optional = true }
+syslog-tracing = { workspace = true, optional = true }
 
 [dev-dependencies]
 aws-config.workspace = true
@@ -97,6 +98,8 @@ kubernetes-discovery = [ "garage_rpc/kubernetes-discovery" ]
 metrics = [ "garage_api/metrics", "opentelemetry-prometheus", "prometheus" ]
 # Exporter for the OpenTelemetry Collector.
 telemetry-otlp = [ "opentelemetry-otlp" ]
+# Logging to syslog
+syslog = [ "syslog-tracing" ]
 
 # NOTE: bundled-libs and system-libs should be treat as mutually exclusive;
 # exactly one of them should be enabled.

src/garage/admin/bucket.rs

@@ -54,9 +54,8 @@ impl AdminRpcHandler {
 		let bucket_id = self
 			.garage
 			.bucket_helper()
-			.resolve_global_bucket_name(&query.name)
+			.admin_get_existing_matching_bucket(&query.name)
-			.await?
+			.await?;
-			.ok_or_bad_request("Bucket not found")?;
 
 		let bucket = self
 			.garage
@@ -157,9 +156,8 @@ impl AdminRpcHandler {
 
 		let bucket_id = helper
 			.bucket()
-			.resolve_global_bucket_name(&query.name)
+			.admin_get_existing_matching_bucket(&query.name)
-			.await?
+			.await?;
-			.ok_or_bad_request("Bucket not found")?;
 
 		// Get the alias, but keep in minde here the bucket name
 		// given in parameter can also be directly the bucket's ID.
@@ -235,9 +233,8 @@ impl AdminRpcHandler {
 
 		let bucket_id = helper
 			.bucket()
-			.resolve_global_bucket_name(&query.existing_bucket)
+			.admin_get_existing_matching_bucket(&query.existing_bucket)
-			.await?
+			.await?;
-			.ok_or_bad_request("Bucket not found")?;
 
 		if let Some(key_pattern) = &query.local {
 			let key = helper.key().get_existing_matching_key(key_pattern).await?;
@@ -307,9 +304,8 @@ impl AdminRpcHandler {
 
 		let bucket_id = helper
 			.bucket()
-			.resolve_global_bucket_name(&query.bucket)
+			.admin_get_existing_matching_bucket(&query.bucket)
-			.await?
+			.await?;
-			.ok_or_bad_request("Bucket not found")?;
 		let key = helper
 			.key()
 			.get_existing_matching_key(&query.key_pattern)
@@ -343,9 +339,8 @@ impl AdminRpcHandler {
 
 		let bucket_id = helper
 			.bucket()
-			.resolve_global_bucket_name(&query.bucket)
+			.admin_get_existing_matching_bucket(&query.bucket)
-			.await?
+			.await?;
-			.ok_or_bad_request("Bucket not found")?;
 		let key = helper
 			.key()
 			.get_existing_matching_key(&query.key_pattern)
@@ -378,9 +373,8 @@ impl AdminRpcHandler {
 		let bucket_id = self
 			.garage
 			.bucket_helper()
-			.resolve_global_bucket_name(&query.bucket)
+			.admin_get_existing_matching_bucket(&query.bucket)
-			.await?
+			.await?;
-			.ok_or_bad_request("Bucket not found")?;
 
 		let mut bucket = self
 			.garage
@@ -420,9 +414,8 @@ impl AdminRpcHandler {
 		let bucket_id = self
 			.garage
 			.bucket_helper()
-			.resolve_global_bucket_name(&query.bucket)
+			.admin_get_existing_matching_bucket(&query.bucket)
-			.await?
+			.await?;
-			.ok_or_bad_request("Bucket not found")?;
 
 		let mut bucket = self
 			.garage
@@ -479,9 +472,8 @@ impl AdminRpcHandler {
 			bucket_ids.push(
 				self.garage
 					.bucket_helper()
-					.resolve_global_bucket_name(b)
+					.admin_get_existing_matching_bucket(b)
-					.await?
+					.await?,
-					.ok_or_bad_request(format!("Bucket not found: {}", b))?,
 			);
 		}
 

src/garage/admin/mod.rs

@@ -46,6 +46,7 @@ pub enum AdminRpc {
 	Stats(StatsOpt),
 	Worker(WorkerOperation),
 	BlockOperation(BlockOperation),
+	MetaOperation(MetaOperation),
 
 	// Replies
 	Ok(String),
@@ -518,6 +519,44 @@ impl AdminRpcHandler {
 			)]))
 		}
 	}
+
+	// ================ META DB COMMANDS ====================
+
+	async fn handle_meta_cmd(self: &Arc<Self>, mo: &MetaOperation) -> Result<AdminRpc, Error> {
+		match mo {
+			MetaOperation::Snapshot { all: true } => {
+				let ring = self.garage.system.ring.borrow().clone();
+				let to = ring.layout.node_ids().to_vec();
+
+				let resps = futures::future::join_all(to.iter().map(|to| async move {
+					let to = (*to).into();
+					self.endpoint
+						.call(
+							&to,
+							AdminRpc::MetaOperation(MetaOperation::Snapshot { all: false }),
+							PRIO_NORMAL,
+						)
+						.await
+				}))
+				.await;
+
+				let mut ret = vec![];
+				for (to, resp) in to.iter().zip(resps.iter()) {
+					let res_str = match resp {
+						Ok(_) => "ok".to_string(),
+						Err(e) => format!("error: {}", e),
+					};
+					ret.push(format!("{:?}\t{}", to, res_str));
+				}
+
+				Ok(AdminRpc::Ok(format_table_to_string(ret)))
+			}
+			MetaOperation::Snapshot { all: false } => {
+				garage_model::snapshot::async_snapshot_metadata(&self.garage).await?;
+				Ok(AdminRpc::Ok("Snapshot has been saved.".into()))
+			}
+		}
+	}
 }
 
 #[async_trait]
@@ -535,6 +574,7 @@ impl EndpointHandler<AdminRpc> for AdminRpcHandler {
 			AdminRpc::Stats(opt) => self.handle_stats(opt.clone()).await,
 			AdminRpc::Worker(wo) => self.handle_worker_cmd(wo).await,
 			AdminRpc::BlockOperation(bo) => self.handle_block_cmd(bo).await,
+			AdminRpc::MetaOperation(mo) => self.handle_meta_cmd(mo).await,
 			m => Err(GarageError::unexpected_rpc_message(m).into()),
 		}
 	}

src/garage/cli/cmd.rs

@@ -44,6 +44,9 @@ pub async fn cli_command_dispatch(
 		Command::Block(bo) => {
 			cmd_admin(admin_rpc_endpoint, rpc_host, AdminRpc::BlockOperation(bo)).await
 		}
+		Command::Meta(mo) => {
+			cmd_admin(admin_rpc_endpoint, rpc_host, AdminRpc::MetaOperation(mo)).await
+		}
 		_ => unreachable!(),
 	}
 }

src/garage/cli/convert_db.rs

@@ -51,53 +51,13 @@ pub(crate) fn do_conversion(args: ConvertDbOpt) -> Result<()> {
 		return Err(Error("input and output database engine must differ".into()));
 	}
 
-	let input = open_db(args.input_path, args.input_engine, &args.db_open)?;
+	let opt = OpenOpt {
-	let output = open_db(args.output_path, args.output_engine, &args.db_open)?;
+		lmdb_map_size: args.db_open.lmdb.map_size.map(|x| x.as_u64() as usize),
+		..Default::default()
+	};
+
+	let input = open_db(&args.input_path, args.input_engine, &opt)?;
+	let output = open_db(&args.output_path, args.output_engine, &opt)?;
 	output.import(&input)?;
 	Ok(())
 }
-
-fn open_db(path: PathBuf, engine: Engine, open: &OpenDbOpt) -> Result<Db> {
-	match engine {
-		#[cfg(feature = "sled")]
-		Engine::Sled => {
-			let db = sled_adapter::sled::Config::default().path(&path).open()?;
-			Ok(sled_adapter::SledDb::init(db))
-		}
-		#[cfg(feature = "sqlite")]
-		Engine::Sqlite => {
-			let db = sqlite_adapter::rusqlite::Connection::open(&path)?;
-			db.pragma_update(None, "journal_mode", "WAL")?;
-			db.pragma_update(None, "synchronous", "NORMAL")?;
-			Ok(sqlite_adapter::SqliteDb::init(db))
-		}
-		#[cfg(feature = "lmdb")]
-		Engine::Lmdb => {
-			std::fs::create_dir_all(&path).map_err(|e| {
-				Error(format!("Unable to create LMDB data directory: {}", e).into())
-			})?;
-
-			let map_size = match open.lmdb.map_size {
-				Some(c) => c.as_u64() as usize,
-				None => lmdb_adapter::recommended_map_size(),
-			};
-
-			let mut env_builder = lmdb_adapter::heed::EnvOpenOptions::new();
-			env_builder.max_dbs(100);
-			env_builder.map_size(map_size);
-			unsafe {
-				env_builder.flag(lmdb_adapter::heed::flags::Flags::MdbNoMetaSync);
-			}
-			let db = env_builder.open(&path)?;
-			Ok(lmdb_adapter::LmdbDb::init(db))
-		}
-
-		// Pattern is unreachable when all supported DB engines are compiled into binary. The allow
-		// attribute is added so that we won't have to change this match in case stop building
-		// support for one or more engines by default.
-		#[allow(unreachable_patterns)]
-		engine => Err(Error(
-			format!("Engine support not available in this build: {}", engine).into(),
-		)),
-	}
-}

src/garage/cli/structs.rs

@@ -57,6 +57,10 @@ pub enum Command {
 	#[structopt(name = "block", version = garage_version())]
 	Block(BlockOperation),
 
+	/// Operations on the metadata db
+	#[structopt(name = "meta", version = garage_version())]
+	Meta(MetaOperation),
+
 	/// Convert metadata db between database engine formats
 	#[structopt(name = "convert-db", version = garage_version())]
 	ConvertDb(convert_db::ConvertDbOpt),
@@ -617,3 +621,14 @@ pub enum BlockOperation {
 		blocks: Vec<String>,
 	},
 }
+
+#[derive(Serialize, Deserialize, StructOpt, Debug, Eq, PartialEq, Clone, Copy)]
+pub enum MetaOperation {
+	/// Save a snapshot of the metadata db file
+	#[structopt(name = "snapshot", version = garage_version())]
+	Snapshot {
+		/// Run on all nodes instead of only local node
+		#[structopt(long = "all")]
+		all: bool,
+	},
+}

src/garage/main.rs

@@ -140,17 +140,8 @@ async fn main() {
 	let opt = Opt::from_clap(&Opt::clap().version(version.as_str()).get_matches());
 
 	// Initialize logging as well as other libraries used in Garage
-	if std::env::var("RUST_LOG").is_err() {
+	init_logging(&opt);
-		let default_log = match &opt.cmd {
+
-			Command::Server => "netapp=info,garage=info",
-			_ => "netapp=warn,garage=warn",
-		};
-		std::env::set_var("RUST_LOG", default_log)
-	}
-	tracing_subscriber::fmt()
-		.with_writer(std::io::stderr)
-		.with_env_filter(tracing_subscriber::filter::EnvFilter::from_default_env())
-		.init();
 	sodiumoxide::init().expect("Unable to init sodiumoxide");
 
 	let res = match opt.cmd {
@@ -173,6 +164,58 @@ async fn main() {
 	}
 }
 
+fn init_logging(opt: &Opt) {
+	if std::env::var("RUST_LOG").is_err() {
+		let default_log = match &opt.cmd {
+			Command::Server => "netapp=info,garage=info",
+			_ => "netapp=warn,garage=warn",
+		};
+		std::env::set_var("RUST_LOG", default_log)
+	}
+
+	let env_filter = tracing_subscriber::filter::EnvFilter::from_default_env();
+
+	if std::env::var("GARAGE_LOG_TO_SYSLOG")
+		.map(|x| x == "1" || x == "true")
+		.unwrap_or(false)
+	{
+		#[cfg(feature = "syslog")]
+		{
+			use std::ffi::CStr;
+			use syslog_tracing::{Facility, Options, Syslog};
+
+			let syslog = Syslog::new(
+				CStr::from_bytes_with_nul(b"garage\0").unwrap(),
+				Options::LOG_PID | Options::LOG_PERROR,
+				Facility::Daemon,
+			)
+			.expect("Unable to init syslog");
+
+			tracing_subscriber::fmt()
+				.with_writer(syslog)
+				.with_env_filter(env_filter)
+				.with_ansi(false) // disable ANSI escape sequences (colours)
+				.with_file(false)
+				.with_level(false)
+				.without_time()
+				.compact()
+				.init();
+
+			return;
+		}
+		#[cfg(not(feature = "syslog"))]
+		{
+			eprintln!("Syslog support is not enabled in this build.");
+			std::process::exit(1);
+		}
+	}
+
+	tracing_subscriber::fmt()
+		.with_writer(std::io::stderr)
+		.with_env_filter(env_filter)
+		.init();
+}
+
 async fn cli_command(opt: Opt) -> Result<(), Error> {
 	let config = if (opt.secrets.rpc_secret.is_none() && opt.secrets.rpc_secret_file.is_none())
 		|| opt.rpc_host.is_none()

src/garage/server.rs

@@ -51,7 +51,7 @@ pub async fn run_server(config_file: PathBuf, secrets: Secrets) -> Result<(), Er
 	let (background, await_background_done) = BackgroundRunner::new(watch_cancel.clone());
 
 	info!("Spawning Garage workers...");
-	garage.spawn_workers(&background);
+	garage.spawn_workers(&background)?;
 
 	if config.admin.trace_sink.is_some() {
 		info!("Initialize tracing...");

src/garage/tests/common/custom_requester.rs

@@ -64,6 +64,10 @@ impl CustomRequester {
 			vhost_style: false,
 		}
 	}
+
+	pub fn client(&self) -> &Client<HttpConnector, Body> {
+		&self.client
+	}
 }
 
 pub struct RequestBuilder<'a> {

src/garage/tests/common/garage.rs

@@ -42,6 +42,10 @@ impl Instance {
 			.ok()
 			.unwrap_or_else(|| env::temp_dir().join(format!("garage-integ-test-{}", port)));
 
+		let db_engine = env::var("GARAGE_TEST_INTEGRATION_DB_ENGINE")
+			.ok()
+			.unwrap_or_else(|| "lmdb".into());
+
 		// Clean test runtime directory
 		if path.exists() {
 			fs::remove_dir_all(&path).expect("Could not clean test runtime directory");
@@ -52,7 +56,7 @@ impl Instance {
 			r#"
 metadata_dir = "{path}/meta"
 data_dir = "{path}/data"
-db_engine = "lmdb"
+db_engine = "{db_engine}"
 
 replication_mode = "1"
 

src/garage/tests/k2v/poll.rs

@@ -10,6 +10,7 @@ use crate::common;
 use crate::json_body;
 
 #[tokio::test]
+#[ignore = "currently broken"]
 async fn test_poll_item() {
 	let ctx = common::context();
 	let bucket = ctx.create_bucket("test-k2v-poll-item");
@@ -98,6 +99,7 @@ async fn test_poll_item() {
 }
 
 #[tokio::test]
+#[ignore = "currently broken"]
 async fn test_poll_range() {
 	let ctx = common::context();
 	let bucket = ctx.create_bucket("test-k2v-poll-range");

src/garage/tests/s3/mod.rs

@@ -1,6 +1,7 @@
 mod list;
 mod multipart;
 mod objects;
+mod presigned;
 mod simple;
 mod streaming_signature;
 mod website;

src/garage/tests/s3/objects.rs

@@ -185,8 +185,51 @@ async fn test_getobject() {
 		assert_eq!(o.content_range.unwrap().as_str(), "bytes 57-61/62");
 		assert_bytes_eq!(o.body, &BODY[57..]);
 	}
+}
+
+#[tokio::test]
+async fn test_metadata() {
+	let ctx = common::context();
+	let bucket = ctx.create_bucket("testmetadata");
+
+	let etag = "\"46cf18a9b447991b450cad3facf5937e\"";
+	let exp = aws_sdk_s3::primitives::DateTime::from_secs(10000000000);
+	let exp2 = aws_sdk_s3::primitives::DateTime::from_secs(10000500000);
+
 	{
-		let exp = aws_sdk_s3::primitives::DateTime::from_secs(10000000000);
+		// Note. The AWS client SDK adds a Content-Type header
+		// with value application/octet-stream if it is not given,
+		// so here we force it to a known different value.
+		let data = ByteStream::from_static(BODY);
+		let r = ctx
+			.client
+			.put_object()
+			.bucket(&bucket)
+			.key(STD_KEY)
+			.body(data)
+			.content_type("application/test")
+			.send()
+			.await
+			.unwrap();
+		assert_eq!(r.e_tag.unwrap().as_str(), etag);
+
+		let o = ctx
+			.client
+			.head_object()
+			.bucket(&bucket)
+			.key(STD_KEY)
+			.send()
+			.await
+			.unwrap();
+		assert_eq!(o.e_tag.unwrap().as_str(), etag);
+		assert_eq!(o.content_type.unwrap().as_str(), "application/test");
+		assert_eq!(o.cache_control, None);
+		assert_eq!(o.content_disposition, None);
+		assert_eq!(o.content_encoding, None);
+		assert_eq!(o.content_language, None);
+		assert_eq!(o.expires, None);
+		assert_eq!(o.metadata.unwrap_or_default().len(), 0);
+
 		let o = ctx
 			.client
 			.get_object()
@@ -201,13 +244,77 @@ async fn test_getobject() {
 			.send()
 			.await
 			.unwrap();
+		assert_eq!(o.e_tag.unwrap().as_str(), etag);
+		assert_eq!(o.content_type.unwrap().as_str(), "application/x-dummy-test");
+		assert_eq!(o.cache_control.unwrap().as_str(), "ccdummy");
+		assert_eq!(o.content_disposition.unwrap().as_str(), "cddummy");
+		assert_eq!(o.content_encoding.unwrap().as_str(), "cedummy");
+		assert_eq!(o.content_language.unwrap().as_str(), "cldummy");
+		assert_eq!(o.expires.unwrap(), exp);
+	}
+
+	{
+		let data = ByteStream::from_static(BODY);
+		let r = ctx
+			.client
+			.put_object()
+			.bucket(&bucket)
+			.key(STD_KEY)
+			.body(data)
+			.content_type("application/test")
+			.cache_control("cctest")
+			.content_disposition("cdtest")
+			.content_encoding("cetest")
+			.content_language("cltest")
+			.expires(exp2)
+			.metadata("testmeta", "hello people")
+			.metadata("nice-unicode-meta", "宅配便")
+			.send()
+			.await
+			.unwrap();
+		assert_eq!(r.e_tag.unwrap().as_str(), etag);
+
+		let o = ctx
+			.client
+			.head_object()
+			.bucket(&bucket)
+			.key(STD_KEY)
+			.send()
+			.await
+			.unwrap();
+		assert_eq!(o.e_tag.unwrap().as_str(), etag);
+		assert_eq!(o.content_type.unwrap().as_str(), "application/test");
+		assert_eq!(o.cache_control.unwrap().as_str(), "cctest");
+		assert_eq!(o.content_disposition.unwrap().as_str(), "cdtest");
+		assert_eq!(o.content_encoding.unwrap().as_str(), "cetest");
+		assert_eq!(o.content_language.unwrap().as_str(), "cltest");
+		assert_eq!(o.expires.unwrap(), exp2);
+		let mut meta = o.metadata.unwrap();
+		assert_eq!(meta.remove("testmeta").unwrap(), "hello people");
+		assert_eq!(meta.remove("nice-unicode-meta").unwrap(), "宅配便");
+		assert_eq!(meta.len(), 0);
+
+		let o = ctx
+			.client
+			.get_object()
+			.bucket(&bucket)
+			.key(STD_KEY)
+			.response_content_type("application/x-dummy-test")
+			.response_cache_control("ccdummy")
+			.response_content_disposition("cddummy")
+			.response_content_encoding("cedummy")
+			.response_content_language("cldummy")
+			.response_expires(exp)
+			.send()
+			.await
+			.unwrap();
+		assert_eq!(o.e_tag.unwrap().as_str(), etag);
 		assert_eq!(o.content_type.unwrap().as_str(), "application/x-dummy-test");
 		assert_eq!(o.cache_control.unwrap().as_str(), "ccdummy");
 		assert_eq!(o.content_disposition.unwrap().as_str(), "cddummy");
 		assert_eq!(o.content_encoding.unwrap().as_str(), "cedummy");
 		assert_eq!(o.content_language.unwrap().as_str(), "cldummy");
 		assert_eq!(o.expires.unwrap(), exp);
-		assert_bytes_eq!(o.body, &BODY[..]);
 	}
 }
 

src/garage/tests/s3/presigned.rs

@@ -0,0 +1,72 @@
+use std::time::{Duration, SystemTime};
+
+use crate::common;
+use aws_sdk_s3::presigning::PresigningConfig;
+use bytes::Bytes;
+use http_body_util::{BodyExt, Full};
+use hyper::Request;
+
+const STD_KEY: &str = "hello world";
+const BODY: &[u8; 62] = b"0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz";
+
+#[tokio::test]
+async fn test_presigned_url() {
+	let ctx = common::context();
+	let bucket = ctx.create_bucket("presigned");
+
+	let etag = "\"46cf18a9b447991b450cad3facf5937e\"";
+	let body = Bytes::from(BODY.to_vec());
+
+	let psc = PresigningConfig::builder()
+		.start_time(SystemTime::now() - Duration::from_secs(60))
+		.expires_in(Duration::from_secs(3600))
+		.build()
+		.unwrap();
+
+	{
+		// PutObject
+		let req = ctx
+			.client
+			.put_object()
+			.bucket(&bucket)
+			.key(STD_KEY)
+			.presigned(psc.clone())
+			.await
+			.unwrap();
+
+		let client = ctx.custom_request.client();
+		let req = Request::builder()
+			.method("PUT")
+			.uri(req.uri())
+			.body(Full::new(body.clone()))
+			.unwrap();
+		let res = client.request(req).await.unwrap();
+		assert_eq!(res.status(), 200);
+		assert_eq!(res.headers().get("etag").unwrap(), etag);
+	}
+
+	{
+		// GetObject
+		let req = ctx
+			.client
+			.get_object()
+			.bucket(&bucket)
+			.key(STD_KEY)
+			.presigned(psc)
+			.await
+			.unwrap();
+
+		let client = ctx.custom_request.client();
+		let req = Request::builder()
+			.method("GET")
+			.uri(req.uri())
+			.body(Full::new(Bytes::new()))
+			.unwrap();
+		let res = client.request(req).await.unwrap();
+		assert_eq!(res.status(), 200);
+		assert_eq!(res.headers().get("etag").unwrap(), etag);
+
+		let body2 = BodyExt::collect(res.into_body()).await.unwrap().to_bytes();
+		assert_eq!(body, body2);
+	}
+}

src/model/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "garage_model"
-version = "0.9.2"
+version = "0.9.4"
 authors = ["Alex Auvolat <alex@adnab.me>"]
 edition = "2018"
 license = "AGPL-3.0"
@@ -28,6 +28,7 @@ chrono.workspace = true
 err-derive.workspace = true
 hex.workspace = true
 base64.workspace = true
+parse_duration.workspace = true
 tracing.workspace = true
 rand.workspace = true
 zstd.workspace = true

src/model/bucket_table.rs

@@ -191,6 +191,13 @@ impl Bucket {
 		}
 	}
 
+	pub fn present(id: Uuid, params: BucketParams) -> Self {
+		Bucket {
+			id,
+			state: crdt::Deletable::present(params),
+		}
+	}
+
 	/// Returns true if this represents a deleted bucket
 	pub fn is_deleted(&self) -> bool {
 		self.state.is_deleted()

src/model/garage.rs

@@ -1,3 +1,4 @@
+use std::str::FromStr;
 use std::sync::Arc;
 
 use garage_net::NetworkKey;
@@ -113,108 +114,33 @@ impl Garage {
 		}
 
 		info!("Opening database...");
+		let db_engine = db::Engine::from_str(&config.db_engine)
+			.ok_or_message("Invalid `db_engine` value in configuration file")?;
 		let mut db_path = config.metadata_dir.clone();
-		let db = match config.db_engine.as_str() {
+		match db_engine {
-			// ---- Sled DB ----
+			db::Engine::Sled => {
-			#[cfg(feature = "sled")]
-			"sled" => {
-				if config.metadata_fsync {
-					return Err(Error::Message(format!(
-						"`metadata_fsync = true` is not supported with the Sled database engine"
-					)));
-				}
 				db_path.push("db");
-				info!("Opening Sled database at: {}", db_path.display());
-				let db = db::sled_adapter::sled::Config::default()
-					.path(&db_path)
-					.cache_capacity(config.sled_cache_capacity as u64)
-					.flush_every_ms(Some(config.sled_flush_every_ms))
-					.open()
-					.ok_or_message("Unable to open sled DB")?;
-				db::sled_adapter::SledDb::init(db)
 			}
-			#[cfg(not(feature = "sled"))]
+			db::Engine::Sqlite => {
-			"sled" => return Err(Error::Message("sled db not available in this build".into())),
-			// ---- Sqlite DB ----
-			#[cfg(feature = "sqlite")]
-			"sqlite" | "sqlite3" | "rusqlite" => {
 				db_path.push("db.sqlite");
-				info!("Opening Sqlite database at: {}", db_path.display());
-				let db = db::sqlite_adapter::rusqlite::Connection::open(db_path)
-					.and_then(|db| {
-						db.pragma_update(None, "journal_mode", &"WAL")?;
-						if config.metadata_fsync {
-							db.pragma_update(None, "synchronous", &"NORMAL")?;
-						} else {
-							db.pragma_update(None, "synchronous", &"OFF")?;
-						}
-						Ok(db)
-					})
-					.ok_or_message("Unable to open sqlite DB")?;
-				db::sqlite_adapter::SqliteDb::init(db)
 			}
-			#[cfg(not(feature = "sqlite"))]
+			db::Engine::Lmdb => {
-			"sqlite" | "sqlite3" | "rusqlite" => {
-				return Err(Error::Message(
-					"sqlite db not available in this build".into(),
-				))
-			}
-			// ---- LMDB DB ----
-			#[cfg(feature = "lmdb")]
-			"lmdb" | "heed" => {
 				db_path.push("db.lmdb");
-				info!("Opening LMDB database at: {}", db_path.display());
-				std::fs::create_dir_all(&db_path)
-					.ok_or_message("Unable to create LMDB data directory")?;
-				let map_size = match config.lmdb_map_size {
-					v if v == usize::default() => garage_db::lmdb_adapter::recommended_map_size(),
-					v => v - (v % 4096),
-				};
-
-				use db::lmdb_adapter::heed;
-				let mut env_builder = heed::EnvOpenOptions::new();
-				env_builder.max_dbs(100);
-				env_builder.max_readers(500);
-				env_builder.map_size(map_size);
-				unsafe {
-					env_builder.flag(heed::flags::Flags::MdbNoMetaSync);
-					if !config.metadata_fsync {
-						env_builder.flag(heed::flags::Flags::MdbNoSync);
-					}
-				}
-				let db = match env_builder.open(&db_path) {
-					Err(heed::Error::Io(e)) if e.kind() == std::io::ErrorKind::OutOfMemory => {
-						return Err(Error::Message(
-							"OutOfMemory error while trying to open LMDB database. This can happen \
-							if your operating system is not allowing you to use sufficient virtual \
-							memory address space. Please check that no limit is set (ulimit -v). \
-							You may also try to set a smaller `lmdb_map_size` configuration parameter. \
-							On 32-bit machines, you should probably switch to another database engine.".into()))
-					}
-					x => x.ok_or_message("Unable to open LMDB DB")?,
-				};
-				db::lmdb_adapter::LmdbDb::init(db)
-			}
-			#[cfg(not(feature = "lmdb"))]
-			"lmdb" | "heed" => return Err(Error::Message("lmdb db not available in this build".into())),
-			// ---- Unavailable DB engine ----
-			e => {
-				return Err(Error::Message(format!(
-					"Unsupported DB engine: {} (options: {})",
-					e,
-					vec![
-						#[cfg(feature = "sled")]
-						"sled",
-						#[cfg(feature = "sqlite")]
-						"sqlite",
-						#[cfg(feature = "lmdb")]
-						"lmdb",
-					]
-					.join(", ")
-				)));
 			}
+		}
+		let db_opt = db::OpenOpt {
+			fsync: config.metadata_fsync,
+			lmdb_map_size: match config.lmdb_map_size {
+				v if v == usize::default() => None,
+				v => Some(v),
+			},
+			sled_cache_capacity: config.sled_cache_capacity,
+			sled_flush_every_ms: config.sled_flush_every_ms,
 		};
+		let db = db::open_db(&db_path, db_engine, &db_opt)
+			.ok_or_message("Unable to open metadata db")?;
 
+		info!("Initializing RPC...");
 		let network_key = hex::decode(config.rpc_secret.as_ref().ok_or_message(
 			"rpc_secret value is missing, not present in config file or in environment",
 		)?)
@@ -251,14 +177,7 @@ impl Garage {
 		};
 
 		info!("Initialize block manager...");
-		let block_manager = BlockManager::new(
+		let block_manager = BlockManager::new(&db, &config, data_rep_param, system.clone())?;
-			&db,
-			config.data_dir.clone(),
-			config.data_fsync,
-			config.compression_level,
-			data_rep_param,
-			system.clone(),
-		)?;
 		block_manager.register_bg_vars(&mut bg_vars);
 
 		// ---- admin tables ----
@@ -359,7 +278,7 @@ impl Garage {
 		}))
 	}
 
-	pub fn spawn_workers(self: &Arc<Self>, bg: &BackgroundRunner) {
+	pub fn spawn_workers(self: &Arc<Self>, bg: &BackgroundRunner) -> Result<(), Error> {
 		self.block_manager.spawn_workers(bg);
 
 		self.bucket_table.spawn_workers(bg);
@@ -380,6 +299,23 @@ impl Garage {
 
 		#[cfg(feature = "k2v")]
 		self.k2v.spawn_workers(bg);
+
+		if let Some(itv) = self.config.metadata_auto_snapshot_interval.as_deref() {
+			let interval = parse_duration::parse(itv)
+				.ok_or_message("Invalid `metadata_auto_snapshot_interval`")?;
+			if interval < std::time::Duration::from_secs(600) {
+				return Err(Error::Message(
+					"metadata_auto_snapshot_interval too small or negative".into(),
+				));
+			}
+
+			bg.spawn_worker(crate::snapshot::AutoSnapshotWorker::new(
+				self.clone(),
+				interval,
+			));
+		}
+
+		Ok(())
 	}
 
 	pub fn bucket_helper(&self) -> helper::bucket::BucketHelper {

src/model/helper/bucket.rs

@@ -67,6 +67,49 @@ impl<'a> BucketHelper<'a> {
 		}
 	}
 
+	/// Find a bucket by its global alias or a prefix of its uuid
+	pub async fn admin_get_existing_matching_bucket(
+		&self,
+		pattern: &String,
+	) -> Result<Uuid, Error> {
+		if let Some(uuid) = self.resolve_global_bucket_name(pattern).await? {
+			return Ok(uuid);
+		} else if pattern.len() >= 2 {
+			let hexdec = pattern
+				.get(..pattern.len() & !1)
+				.and_then(|x| hex::decode(x).ok());
+			if let Some(hex) = hexdec {
+				let mut start = [0u8; 32];
+				start
+					.as_mut_slice()
+					.get_mut(..hex.len())
+					.ok_or_bad_request("invalid length")?
+					.copy_from_slice(&hex);
+				let mut candidates = self
+					.0
+					.bucket_table
+					.get_range(
+						&EmptyKey,
+						Some(start.into()),
+						Some(DeletedFilter::NotDeleted),
+						10,
+						EnumerationOrder::Forward,
+					)
+					.await?
+					.into_iter()
+					.collect::<Vec<_>>();
+				candidates.retain(|x| hex::encode(x.id).starts_with(pattern));
+				if candidates.len() == 1 {
+					return Ok(candidates.into_iter().next().unwrap().id);
+				}
+			}
+		}
+		Err(Error::BadRequest(format!(
+			"Bucket not found / several matching buckets: {}",
+			pattern
+		)))
+	}
+
 	/// Returns a Bucket if it is present in bucket table,
 	/// even if it is in deleted state. Querying a non-existing
 	/// bucket ID returns an internal error.

src/model/k2v/rpc.rs

@@ -300,7 +300,11 @@ impl K2VRpcHandler {
 		let rs = RequestStrategy::with_priority(PRIO_NORMAL).without_timeout();
 		let mut requests = nodes
 			.iter()
-			.map(|node| self.system.rpc.call(&self.endpoint, *node, msg.clone(), rs))
+			.map(|node| {
+				self.system
+					.rpc
+					.call(&self.endpoint, *node, msg.clone(), rs.clone())
+			})
 			.collect::<FuturesUnordered<_>>();
 
 		// Fetch responses. This procedure stops fetching responses when any of the following

src/model/lib.rs

@@ -19,3 +19,4 @@ pub mod s3;
 pub mod garage;
 pub mod helper;
 pub mod migrate;
+pub mod snapshot;

src/model/snapshot.rs

@@ -0,0 +1,136 @@
+use std::fs;
+use std::path::PathBuf;
+use std::sync::Arc;
+use std::sync::Mutex;
+use std::time::{Duration, Instant};
+
+use async_trait::async_trait;
+use rand::prelude::*;
+use tokio::sync::watch;
+
+use garage_util::background::*;
+use garage_util::error::*;
+
+use crate::garage::Garage;
+
+// The two most recent snapshots are kept
+const KEEP_SNAPSHOTS: usize = 2;
+
+static SNAPSHOT_MUTEX: Mutex<()> = Mutex::new(());
+
+// ================ snapshotting logic =====================
+
+/// Run snashot_metadata in a blocking thread and async await on it
+pub async fn async_snapshot_metadata(garage: &Arc<Garage>) -> Result<(), Error> {
+	let garage = garage.clone();
+	let worker = tokio::task::spawn_blocking(move || snapshot_metadata(&garage));
+	worker.await.unwrap()?;
+	Ok(())
+}
+
+/// Take a snapshot of the metadata database, and erase older
+/// snapshots if necessary.
+/// This is not an async function, it should be spawned on a thread pool
+pub fn snapshot_metadata(garage: &Garage) -> Result<(), Error> {
+	let lock = match SNAPSHOT_MUTEX.try_lock() {
+		Ok(lock) => lock,
+		Err(_) => {
+			return Err(Error::Message(
+				"Cannot acquire lock, another snapshot might be in progress".into(),
+			))
+		}
+	};
+
+	let mut snapshots_dir = garage.config.metadata_dir.clone();
+	snapshots_dir.push("snapshots");
+	fs::create_dir_all(&snapshots_dir)?;
+
+	let mut new_path = snapshots_dir.clone();
+	new_path.push(chrono::Utc::now().to_rfc3339_opts(chrono::SecondsFormat::Secs, true));
+
+	info!("Snapshotting metadata db to {}", new_path.display());
+	garage.db.snapshot(&new_path)?;
+	info!("Metadata db snapshot finished");
+
+	if let Err(e) = cleanup_snapshots(&snapshots_dir) {
+		error!("Failed to do cleanup in snapshots directory: {}", e);
+	}
+
+	drop(lock);
+
+	Ok(())
+}
+
+fn cleanup_snapshots(snapshots_dir: &PathBuf) -> Result<(), Error> {
+	let mut snapshots =
+		fs::read_dir(&snapshots_dir)?.collect::<Result<Vec<fs::DirEntry>, std::io::Error>>()?;
+
+	snapshots.retain(|x| x.file_name().len() > 8);
+	snapshots.sort_by_key(|x| x.file_name());
+
+	for to_delete in snapshots.iter().rev().skip(KEEP_SNAPSHOTS) {
+		let path = snapshots_dir.join(to_delete.path());
+		if to_delete.metadata()?.file_type().is_dir() {
+			for file in fs::read_dir(&path)? {
+				let file = file?;
+				if file.metadata()?.is_file() {
+					fs::remove_file(path.join(file.path()))?;
+				}
+			}
+			std::fs::remove_dir(&path)?;
+		} else {
+			std::fs::remove_file(&path)?;
+		}
+	}
+	Ok(())
+}
+
+// ================ auto snapshot worker =====================
+
+pub struct AutoSnapshotWorker {
+	garage: Arc<Garage>,
+	next_snapshot: Instant,
+	snapshot_interval: Duration,
+}
+
+impl AutoSnapshotWorker {
+	pub(crate) fn new(garage: Arc<Garage>, snapshot_interval: Duration) -> Self {
+		Self {
+			garage,
+			snapshot_interval,
+			next_snapshot: Instant::now() + (snapshot_interval / 2),
+		}
+	}
+}
+
+#[async_trait]
+impl Worker for AutoSnapshotWorker {
+	fn name(&self) -> String {
+		"Metadata snapshot worker".into()
+	}
+	fn status(&self) -> WorkerStatus {
+		WorkerStatus {
+			freeform: vec![format!(
+				"Next snapshot: {}",
+				(chrono::Utc::now() + (self.next_snapshot - Instant::now())).to_rfc3339()
+			)],
+			..Default::default()
+		}
+	}
+	async fn work(&mut self, _must_exit: &mut watch::Receiver<bool>) -> Result<WorkerState, Error> {
+		if Instant::now() < self.next_snapshot {
+			return Ok(WorkerState::Idle);
+		}
+
+		async_snapshot_metadata(&self.garage).await?;
+
+		let rand_factor = 1f32 + thread_rng().gen::<f32>() / 5f32;
+		self.next_snapshot = Instant::now() + self.snapshot_interval.mul_f32(rand_factor);
+
+		Ok(WorkerState::Idle)
+	}
+	async fn wait_for_work(&mut self) -> WorkerState {
+		tokio::time::sleep_until(self.next_snapshot.into()).await;
+		WorkerState::Busy
+	}
+}

src/net/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "garage_net"
-version = "0.9.2"
+version = "0.9.4"
 authors = ["Alex Auvolat <alex@adnab.me>"]
 edition = "2018"
 license = "AGPL-3.0"

src/net/message.rs

@@ -28,12 +28,30 @@ use crate::util::*;
 /// The same priority value is given to a request and to its associated response.
 pub type RequestPriority = u8;
 
+// Usage of priority levels in Garage:
+//
+// PRIO_HIGH
+//      for liveness check events such as pings and important
+//      reconfiguration events such as layout changes
+//
+// PRIO_NORMAL
+//      for standard interactive requests to exchange metadata
+//
+// PRIO_NORMAL | PRIO_SECONDARY
+//      for standard interactive requests to exchange block data
+//
+// PRIO_BACKGROUND
+//      for background resync requests to exchange metadata
+// PRIO_BACKGROUND | PRIO_SECONDARY
+//      for background resync requests to exchange block data
+
 /// Priority class: high
 pub const PRIO_HIGH: RequestPriority = 0x20;
 /// Priority class: normal
 pub const PRIO_NORMAL: RequestPriority = 0x40;
 /// Priority class: background
 pub const PRIO_BACKGROUND: RequestPriority = 0x80;
+
 /// Priority: primary among given class
 pub const PRIO_PRIMARY: RequestPriority = 0x00;
 /// Priority: secondary among given class (ex: `PRIO_HIGH | PRIO_SECONDARY`)

src/net/send.rs

@@ -109,7 +109,7 @@ impl SendQueuePriority {
 			let i = order_vec.iter().take_while(|o2| **o2 < order).count();
 			order_vec.insert(i, order);
 		}
-		self.items.push_front(item);
+		self.items.push_back(item);
 	}
 	fn remove(&mut self, id: RequestID) {
 		if let Some(i) = self.items.iter().position(|x| x.id == id) {
@@ -128,51 +128,56 @@ impl SendQueuePriority {
 		self.items.is_empty()
 	}
 	fn poll_next_ready(&mut self, ctx: &mut Context<'_>) -> Poll<(RequestID, DataFrame)> {
-		for (j, item) in self.items.iter_mut().enumerate() {
+		// in step 1: poll only streams that have sent 0 bytes, we want to send them in priority
-			if let Some(OrderTag(stream, order)) = item.order_tag {
+		//            as they most likely represent small requests to be sent first
-				if order > *self.order.get(&stream).unwrap().front().unwrap() {
+		// in step 2: poll all streams
+		for step in 0..2 {
+			for (j, item) in self.items.iter_mut().enumerate() {
+				if let Some(OrderTag(stream, order)) = item.order_tag {
+					if order > *self.order.get(&stream).unwrap().front().unwrap() {
+						continue;
+					}
+				}
+
+				if step == 0 && item.sent > 0 {
 					continue;
 				}
-			}
 
-			let mut item_reader = item.data.read_exact_or_eos(MAX_CHUNK_LENGTH as usize);
+				let mut item_reader = item.data.read_exact_or_eos(MAX_CHUNK_LENGTH as usize);
-			if let Poll::Ready(bytes_or_err) = Pin::new(&mut item_reader).poll(ctx) {
+				if let Poll::Ready(bytes_or_err) = Pin::new(&mut item_reader).poll(ctx) {
-				let id = item.id;
+					let id = item.id;
-				let eos = item.data.eos();
+					let eos = item.data.eos();
 
-				let packet = bytes_or_err.map_err(|e| match e {
+					let packet = bytes_or_err.map_err(|e| match e {
-					ReadExactError::Stream(err) => err,
+						ReadExactError::Stream(err) => err,
-					_ => unreachable!(),
+						_ => unreachable!(),
-				});
+					});
 
-				let is_err = packet.is_err();
+					let is_err = packet.is_err();
-				let data_frame = DataFrame::from_packet(packet, !eos);
+					let data_frame = DataFrame::from_packet(packet, !eos);
-				item.sent += data_frame.data().len();
+					item.sent += data_frame.data().len();
 
-				if eos || is_err {
+					if eos || is_err {
-					// If item had an order tag, remove it from the corresponding ordering list
+						// If item had an order tag, remove it from the corresponding ordering list
-					if let Some(OrderTag(stream, order)) = item.order_tag {
+						if let Some(OrderTag(stream, order)) = item.order_tag {
-						let order_stream = self.order.get_mut(&stream).unwrap();
+							let order_stream = self.order.get_mut(&stream).unwrap();
-						assert_eq!(order_stream.pop_front(), Some(order));
+							assert_eq!(order_stream.pop_front(), Some(order));
-						if order_stream.is_empty() {
+							if order_stream.is_empty() {
-							self.order.remove(&stream);
+								self.order.remove(&stream);
-						}
+							}
-					}
-					// Remove item from sending queue
-					self.items.remove(j);
-				} else {
-					// Move item later in send queue to implement LAS scheduling
-					// (LAS = Least Attained Service)
-					for k in j..self.items.len() - 1 {
-						if self.items[k].sent >= self.items[k + 1].sent {
-							self.items.swap(k, k + 1);
-						} else {
-							break;
 						}
+						// Remove item from sending queue
+						self.items.remove(j);
+					} else if step == 0 {
+						// Step 0 means that this stream had not sent any bytes yet.
+						// Now that it has, and it was not an EOS, we know that it is bigger
+						// than one chunk so move it at the end of the queue.
+						let item = self.items.remove(j).unwrap();
+						self.items.push_back(item);
 					}
+
+					return Poll::Ready((id, data_frame));
 				}
-
-				return Poll::Ready((id, data_frame));
 			}
 		}
 

src/net/server.rs

@@ -190,7 +190,7 @@ impl RecvLoop for ServerConn {
 
 			let (prio, resp_enc_result) = match ReqEnc::decode(stream).await {
 				Ok(req_enc) => (req_enc.prio, self2.recv_handler_aux(req_enc).await),
-				Err(e) => (PRIO_HIGH, Err(e)),
+				Err(e) => (PRIO_NORMAL, Err(e)),
 			};
 
 			debug!("server: sending response to {}", id);

src/rpc/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "garage_rpc"
-version = "0.9.2"
+version = "0.9.4"
 authors = ["Alex Auvolat <alex@adnab.me>"]
 edition = "2018"
 license = "AGPL-3.0"

src/rpc/rpc_helper.rs

@@ -33,8 +33,7 @@ use crate::ring::Ring;
 const DEFAULT_TIMEOUT: Duration = Duration::from_secs(300);
 
 /// Strategy to apply when making RPC
-#[derive(Copy, Clone)]
+pub struct RequestStrategy<T> {
-pub struct RequestStrategy {
 	/// Min number of response to consider the request successful
 	pub rs_quorum: Option<usize>,
 	/// Should requests be dropped after enough response are received
@@ -43,6 +42,8 @@ pub struct RequestStrategy {
 	pub rs_priority: RequestPriority,
 	/// Custom timeout for this request
 	rs_timeout: Timeout,
+	/// Data to drop when everything completes
+	rs_drop_on_complete: T,
 }
 
 #[derive(Copy, Clone)]
@@ -52,7 +53,19 @@ enum Timeout {
 	Custom(Duration),
 }
 
-impl RequestStrategy {
+impl Clone for RequestStrategy<()> {
+	fn clone(&self) -> Self {
+		RequestStrategy {
+			rs_quorum: self.rs_quorum,
+			rs_interrupt_after_quorum: self.rs_interrupt_after_quorum,
+			rs_priority: self.rs_priority,
+			rs_timeout: self.rs_timeout,
+			rs_drop_on_complete: (),
+		}
+	}
+}
+
+impl RequestStrategy<()> {
 	/// Create a RequestStrategy with default timeout and not interrupting when quorum reached
 	pub fn with_priority(prio: RequestPriority) -> Self {
 		RequestStrategy {
@@ -60,8 +73,22 @@ impl RequestStrategy {
 			rs_interrupt_after_quorum: false,
 			rs_priority: prio,
 			rs_timeout: Timeout::Default,
+			rs_drop_on_complete: (),
 		}
 	}
+	/// Add an item to be dropped on completion
+	pub fn with_drop_on_completion<T>(self, drop_on_complete: T) -> RequestStrategy<T> {
+		RequestStrategy {
+			rs_quorum: self.rs_quorum,
+			rs_interrupt_after_quorum: self.rs_interrupt_after_quorum,
+			rs_priority: self.rs_priority,
+			rs_timeout: self.rs_timeout,
+			rs_drop_on_complete: drop_on_complete,
+		}
+	}
+}
+
+impl<T> RequestStrategy<T> {
 	/// Set quorum to be reached for request
 	pub fn with_quorum(mut self, quorum: usize) -> Self {
 		self.rs_quorum = Some(quorum);
@@ -83,6 +110,19 @@ impl RequestStrategy {
 		self.rs_timeout = Timeout::Custom(timeout);
 		self
 	}
+	/// Extract drop_on_complete item
+	fn extract_drop_on_complete(self) -> (RequestStrategy<()>, T) {
+		(
+			RequestStrategy {
+				rs_quorum: self.rs_quorum,
+				rs_interrupt_after_quorum: self.rs_interrupt_after_quorum,
+				rs_priority: self.rs_priority,
+				rs_timeout: self.rs_timeout,
+				rs_drop_on_complete: (),
+			},
+			self.rs_drop_on_complete,
+		)
+	}
 }
 
 #[derive(Clone)]
@@ -123,7 +163,7 @@ impl RpcHelper {
 		endpoint: &Endpoint<M, H>,
 		to: Uuid,
 		msg: N,
-		strat: RequestStrategy,
+		strat: RequestStrategy<()>,
 	) -> Result<S, Error>
 	where
 		M: Rpc<Response = Result<S, Error>>,
@@ -176,7 +216,7 @@ impl RpcHelper {
 		endpoint: &Endpoint<M, H>,
 		to: &[Uuid],
 		msg: N,
-		strat: RequestStrategy,
+		strat: RequestStrategy<()>,
 	) -> Result<Vec<(Uuid, Result<S, Error>)>, Error>
 	where
 		M: Rpc<Response = Result<S, Error>>,
@@ -187,7 +227,7 @@ impl RpcHelper {
 
 		let resps = join_all(
 			to.iter()
-				.map(|to| self.call(endpoint, *to, msg.clone(), strat)),
+				.map(|to| self.call(endpoint, *to, msg.clone(), strat.clone())),
 		)
 		.await;
 		Ok(to
@@ -201,7 +241,7 @@ impl RpcHelper {
 		&self,
 		endpoint: &Endpoint<M, H>,
 		msg: N,
-		strat: RequestStrategy,
+		strat: RequestStrategy<()>,
 	) -> Result<Vec<(Uuid, Result<S, Error>)>, Error>
 	where
 		M: Rpc<Response = Result<S, Error>>,
@@ -220,18 +260,19 @@ impl RpcHelper {
 
 	/// Make a RPC call to multiple servers, returning either a Vec of responses,
 	/// or an error if quorum could not be reached due to too many errors
-	pub async fn try_call_many<M, N, H, S>(
+	pub async fn try_call_many<M, N, H, S, T>(
 		&self,
 		endpoint: &Arc<Endpoint<M, H>>,
 		to: &[Uuid],
 		msg: N,
-		strategy: RequestStrategy,
+		strategy: RequestStrategy<T>,
 	) -> Result<Vec<S>, Error>
 	where
 		M: Rpc<Response = Result<S, Error>> + 'static,
 		N: IntoReq<M>,
 		H: StreamingEndpointHandler<M> + 'static,
 		S: Send + 'static,
+		T: Send + 'static,
 	{
 		let quorum = strategy.rs_quorum.unwrap_or(to.len());
 
@@ -260,12 +301,12 @@ impl RpcHelper {
 			.await
 	}
 
-	async fn try_call_many_internal<M, N, H, S>(
+	async fn try_call_many_internal<M, N, H, S, T>(
 		&self,
 		endpoint: &Arc<Endpoint<M, H>>,
 		to: &[Uuid],
 		msg: N,
-		strategy: RequestStrategy,
+		strategy: RequestStrategy<T>,
 		quorum: usize,
 	) -> Result<Vec<S>, Error>
 	where
@@ -273,9 +314,12 @@ impl RpcHelper {
 		N: IntoReq<M>,
 		H: StreamingEndpointHandler<M> + 'static,
 		S: Send + 'static,
+		T: Send + 'static,
 	{
 		let msg = msg.into_req().map_err(garage_net::error::Error::from)?;
 
+		let (strategy, drop_on_complete) = strategy.extract_drop_on_complete();
+
 		// Build future for each request
 		// They are not started now: they are added below in a FuturesUnordered
 		// object that will take care of polling them (see below)
@@ -283,6 +327,7 @@ impl RpcHelper {
 			let self2 = self.clone();
 			let msg = msg.clone();
 			let endpoint2 = endpoint.clone();
+			let strategy = strategy.clone();
 			(to, async move {
 				self2.call(&endpoint2, to, msg, strategy).await
 			})
@@ -377,6 +422,7 @@ impl RpcHelper {
 				// they have to be put in a proper queue that is persisted to disk.
 				tokio::spawn(async move {
 					resp_stream.collect::<Vec<Result<_, _>>>().await;
+					drop(drop_on_complete);
 				});
 			}
 		}

src/table/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "garage_table"
-version = "0.9.2"
+version = "0.9.4"
 authors = ["Alex Auvolat <alex@adnab.me>"]
 edition = "2018"
 license = "AGPL-3.0"

src/util/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "garage_util"
-version = "0.9.2"
+version = "0.9.4"
 authors = ["Alex Auvolat <alex@adnab.me>"]
 edition = "2018"
 license = "AGPL-3.0"

src/util/config.rs

@@ -23,6 +23,14 @@ pub struct Config {
 	#[serde(default)]
 	pub data_fsync: bool,
 
+	/// Disable automatic scrubbing of the data directory
+	#[serde(default)]
+	pub disable_scrub: bool,
+
+	/// Automatic snapshot interval for metadata
+	#[serde(default)]
+	pub metadata_auto_snapshot_interval: Option<String>,
+
 	/// Size of data blocks to save to disk
 	#[serde(
 		deserialize_with = "deserialize_capacity",
@@ -44,6 +52,14 @@ pub struct Config {
 	)]
 	pub compression_level: Option<i32>,
 
+	/// Maximum amount of block data to buffer in RAM for sending to
+	/// remote nodes when these nodes are on slower links
+	#[serde(
+		deserialize_with = "deserialize_capacity",
+		default = "default_block_ram_buffer_max"
+	)]
+	pub block_ram_buffer_max: usize,
+
 	/// Skip the permission check of secret files. Useful when
 	/// POSIX ACLs (or more complex chmods) are used.
 	#[serde(default)]
@@ -247,6 +263,9 @@ fn default_sled_flush_every_ms() -> u64 {
 fn default_block_size() -> usize {
 	1048576
 }
+fn default_block_ram_buffer_max() -> usize {
+	256 * 1024 * 1024
+}
 
 fn default_compression() -> Option<i32> {
 	Some(1)

src/web/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "garage_web"
-version = "0.9.2"
+version = "0.9.4"
 authors = ["Alex Auvolat <alex@adnab.me>", "Quentin Dufour <quentin@dufour.io>"]
 edition = "2018"
 license = "AGPL-3.0"

src/web/web_server.rs

@@ -26,7 +26,7 @@ use garage_api::s3::cors::{add_cors_headers, find_matching_cors_rule, handle_opt
 use garage_api::s3::error::{
 	CommonErrorDerivative, Error as ApiError, OkOrBadRequest, OkOrInternalError,
 };
-use garage_api::s3::get::{handle_get, handle_head};
+use garage_api::s3::get::{handle_get_without_ctx, handle_head_without_ctx};
 
 use garage_model::garage::Garage;
 
@@ -219,14 +219,13 @@ impl WebServer {
 		// Check bucket isn't deleted and has website access enabled
 		let bucket = self
 			.garage
-			.bucket_table
+			.bucket_helper()
-			.get(&EmptyKey, &bucket_id)
+			.get_existing_bucket(bucket_id)
-			.await?
+			.await
-			.ok_or(Error::NotFound)?;
+			.map_err(|_| Error::NotFound)?;
+		let bucket_params = bucket.state.into_option().unwrap();
 
-		let website_config = bucket
+		let website_config = bucket_params
-			.params()
-			.ok_or(Error::NotFound)?
 			.website_config
 			.get()
 			.as_ref()
@@ -243,14 +242,16 @@ impl WebServer {
 		);
 
 		let ret_doc = match *req.method() {
-			Method::OPTIONS => handle_options_for_bucket(req, &bucket)
+			Method::OPTIONS => handle_options_for_bucket(req, &bucket_params)
 				.map_err(ApiError::from)
 				.map(|res| res.map(|_empty_body: EmptyBody| empty_body())),
-			Method::HEAD => handle_head(self.garage.clone(), &req, bucket_id, &key, None).await,
+			Method::HEAD => {
+				handle_head_without_ctx(self.garage.clone(), req, bucket_id, &key, None).await
+			}
 			Method::GET => {
-				handle_get(
+				handle_get_without_ctx(
 					self.garage.clone(),
-					&req,
+					req,
 					bucket_id,
 					&key,
 					None,
@@ -301,7 +302,7 @@ impl WebServer {
 					.body(empty_body::<Infallible>())
 					.unwrap();
 
-				match handle_get(
+				match handle_get_without_ctx(
 					self.garage.clone(),
 					&req2,
 					bucket_id,
@@ -344,7 +345,7 @@ impl WebServer {
 			}
 			Ok(mut resp) => {
 				// Maybe add CORS headers
-				if let Some(rule) = find_matching_cors_rule(&bucket, req)? {
+				if let Some(rule) = find_matching_cors_rule(&bucket_params, req)? {
 					add_cors_headers(&mut resp, rule)
 						.ok_or_internal_error("Invalid bucket CORS configuration")?;
 				}