history table refactoring

Update Rust Nix toolchain
K2V history and preparation for range watch
2026-08-14 08:03:11 +00:00 · 2023-01-09 15:00:00 +01:00 · 2023-01-09 13:21:37 +01:00 · 2023-01-05 13:11:48 +01:00 · 2023-01-04 14:43:45 +00:00 · 2023-01-04 13:47:42 +00:00
128 changed files with 4632 additions and 2195 deletions
@@ -1 +0,0 @@
-*.pdf filter=lfs diff=lfs merge=lfs -text
@@ -1048,7 +1048,7 @@ dependencies = [

 [[package]]
 name = "garage"
-version = "0.8.0"
+version = "0.8.1"
 dependencies = [
 "assert-json-diff",
 "async-trait",
@@ -1080,7 +1080,6 @@ dependencies = [
 "parse_duration",
 "prometheus",
 "rand 0.8.5",
- "rmp-serde",
 "serde",
 "serde_bytes",
 "serde_json",
@@ -1096,7 +1095,7 @@ dependencies = [

 [[package]]
 name = "garage_api"
-version = "0.8.0"
+version = "0.8.1"
 dependencies = [
 "async-trait",
 "base64",
@@ -1141,7 +1140,7 @@ dependencies = [

 [[package]]
 name = "garage_block"
-version = "0.8.0"
+version = "0.8.1"
 dependencies = [
 "arc-swap",
 "async-compression",
@@ -1156,7 +1155,6 @@ dependencies = [
 "hex",
 "opentelemetry",
 "rand 0.8.5",
- "rmp-serde",
 "serde",
 "serde_bytes",
 "tokio",
@@ -1167,7 +1165,7 @@ dependencies = [

 [[package]]
 name = "garage_db"
-version = "0.8.0"
+version = "0.8.1"
 dependencies = [
 "clap 3.1.18",
 "err-derive",
@@ -1182,7 +1180,7 @@ dependencies = [

 [[package]]
 name = "garage_model"
-version = "0.8.0"
+version = "0.8.1"
 dependencies = [
 "arc-swap",
 "async-trait",
@@ -1200,7 +1198,6 @@ dependencies = [
 "netapp",
 "opentelemetry",
 "rand 0.8.5",
- "rmp-serde",
 "serde",
 "serde_bytes",
 "tokio",
@@ -1210,7 +1207,7 @@ dependencies = [

 [[package]]
 name = "garage_rpc"
-version = "0.8.0"
+version = "0.8.1"
 dependencies = [
 "arc-swap",
 "async-trait",
@@ -1229,7 +1226,6 @@ dependencies = [
 "pnet_datalink",
 "rand 0.8.5",
 "reqwest",
- "rmp-serde",
 "schemars",
 "serde",
 "serde_bytes",
@@ -1241,8 +1237,9 @@ dependencies = [

 [[package]]
 name = "garage_table"
-version = "0.8.0"
+version = "0.8.1"
 dependencies = [
+ "arc-swap",
 "async-trait",
 "bytes",
 "futures",
@@ -1254,7 +1251,6 @@ dependencies = [
 "hexdump",
 "opentelemetry",
 "rand 0.8.5",
- "rmp-serde",
 "serde",
 "serde_bytes",
 "tokio",
@@ -1263,7 +1259,7 @@ dependencies = [

 [[package]]
 name = "garage_util"
-version = "0.8.0"
+version = "0.8.1"
 dependencies = [
 "arc-swap",
 "async-trait",
@@ -1276,6 +1272,7 @@ dependencies = [
 "garage_db",
 "git-version",
 "hex",
+ "hexdump",
 "http",
 "hyper",
 "lazy_static",
@@ -1294,7 +1291,7 @@ dependencies = [

 [[package]]
 name = "garage_web"
-version = "0.8.0"
+version = "0.8.1"
 dependencies = [
 "err-derive",
 "futures",
@@ -32,7 +32,7 @@ args@{
  ignoreLockHash,
 }:
 let
-  nixifiedLockHash = "90b29705f5037c7e1b33f4650841f1266f2e86fa03d5d0c87ad80be7619985c7";
+  nixifiedLockHash = "b6aeefc112eb232904b24398f4e5da776c8ee2c13d427a26dbdf1732205d4fc9";
  workspaceSrc = if args.workspaceSrc == null then ./. else args.workspaceSrc;
  currentLockHash = builtins.hashFile "sha256" (workspaceSrc + /Cargo.lock);
  lockHashIgnored = if ignoreLockHash
@@ -56,15 +56,15 @@ in
 {
  cargo2nixVersion = "0.11.0";
  workspace = {
-    garage_db = rustPackages.unknown.garage_db."0.8.0";
-    garage_util = rustPackages.unknown.garage_util."0.8.0";
-    garage_rpc = rustPackages.unknown.garage_rpc."0.8.0";
-    garage_table = rustPackages.unknown.garage_table."0.8.0";
-    garage_block = rustPackages.unknown.garage_block."0.8.0";
-    garage_model = rustPackages.unknown.garage_model."0.8.0";
-    garage_api = rustPackages.unknown.garage_api."0.8.0";
-    garage_web = rustPackages.unknown.garage_web."0.8.0";
-    garage = rustPackages.unknown.garage."0.8.0";
+    garage_db = rustPackages.unknown.garage_db."0.8.1";
+    garage_util = rustPackages.unknown.garage_util."0.8.1";
+    garage_rpc = rustPackages.unknown.garage_rpc."0.8.1";
+    garage_table = rustPackages.unknown.garage_table."0.8.1";
+    garage_block = rustPackages.unknown.garage_block."0.8.1";
+    garage_model = rustPackages.unknown.garage_model."0.8.1";
+    garage_api = rustPackages.unknown.garage_api."0.8.1";
+    garage_web = rustPackages.unknown.garage_web."0.8.1";
+    garage = rustPackages.unknown.garage."0.8.1";
    k2v-client = rustPackages.unknown.k2v-client."0.0.1";
  };
  "registry+https://github.com/rust-lang/crates.io-index".addr2line."0.17.0" = overridableMkRustCrate (profileName: rec {
@@ -1494,16 +1494,16 @@ in
    };
  });
  
-  "unknown".garage."0.8.0" = overridableMkRustCrate (profileName: rec {
+  "unknown".garage."0.8.1" = overridableMkRustCrate (profileName: rec {
    name = "garage";
-    version = "0.8.0";
+    version = "0.8.1";
    registry = "unknown";
    src = fetchCrateLocal (workspaceSrc + "/src/garage");
    features = builtins.concatLists [
      (lib.optional (rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default") "bundled-libs")
      (lib.optional (rootFeatures' ? "garage/consul-discovery") "consul-discovery")
      (lib.optional (rootFeatures' ? "garage/default") "default")
-      (lib.optional (rootFeatures' ? "garage/k2v") "k2v")
+      (lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/k2v") "k2v")
      (lib.optional (rootFeatures' ? "garage/kubernetes-discovery") "kubernetes-discovery")
      (lib.optional (rootFeatures' ? "garage/lmdb") "lmdb")
      (lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/metrics") "metrics")
@@ -1522,14 +1522,14 @@ in
      bytesize = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytesize."1.1.0" { inherit profileName; }).out;
      futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.21" { inherit profileName; }).out;
      futures_util = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.21" { inherit profileName; }).out;
-      garage_api = (rustPackages."unknown".garage_api."0.8.0" { inherit profileName; }).out;
-      garage_block = (rustPackages."unknown".garage_block."0.8.0" { inherit profileName; }).out;
-      garage_db = (rustPackages."unknown".garage_db."0.8.0" { inherit profileName; }).out;
-      garage_model = (rustPackages."unknown".garage_model."0.8.0" { inherit profileName; }).out;
-      garage_rpc = (rustPackages."unknown".garage_rpc."0.8.0" { inherit profileName; }).out;
-      garage_table = (rustPackages."unknown".garage_table."0.8.0" { inherit profileName; }).out;
-      garage_util = (rustPackages."unknown".garage_util."0.8.0" { inherit profileName; }).out;
-      garage_web = (rustPackages."unknown".garage_web."0.8.0" { inherit profileName; }).out;
+      garage_api = (rustPackages."unknown".garage_api."0.8.1" { inherit profileName; }).out;
+      garage_block = (rustPackages."unknown".garage_block."0.8.1" { inherit profileName; }).out;
+      garage_db = (rustPackages."unknown".garage_db."0.8.1" { inherit profileName; }).out;
+      garage_model = (rustPackages."unknown".garage_model."0.8.1" { inherit profileName; }).out;
+      garage_rpc = (rustPackages."unknown".garage_rpc."0.8.1" { inherit profileName; }).out;
+      garage_table = (rustPackages."unknown".garage_table."0.8.1" { inherit profileName; }).out;
+      garage_util = (rustPackages."unknown".garage_util."0.8.1" { inherit profileName; }).out;
+      garage_web = (rustPackages."unknown".garage_web."0.8.1" { inherit profileName; }).out;
      hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out;
      sodiumoxide = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".kuska-sodiumoxide."0.2.5-0" { inherit profileName; }).out;
      netapp = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".netapp."0.5.2" { inherit profileName; }).out;
@@ -1539,7 +1539,6 @@ in
      parse_duration = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".parse_duration."2.1.1" { inherit profileName; }).out;
      ${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/metrics" || rootFeatures' ? "garage/prometheus" then "prometheus" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".prometheus."0.13.0" { inherit profileName; }).out;
      rand = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand."0.8.5" { inherit profileName; }).out;
-      rmp_serde = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rmp-serde."0.15.5" { inherit profileName; }).out;
      serde = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde."1.0.137" { inherit profileName; }).out;
      serde_bytes = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde_bytes."0.11.5" { inherit profileName; }).out;
      structopt = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".structopt."0.3.26" { inherit profileName; }).out;
@@ -1563,13 +1562,13 @@ in
    };
  });
  
-  "unknown".garage_api."0.8.0" = overridableMkRustCrate (profileName: rec {
+  "unknown".garage_api."0.8.1" = overridableMkRustCrate (profileName: rec {
    name = "garage_api";
-    version = "0.8.0";
+    version = "0.8.1";
    registry = "unknown";
    src = fetchCrateLocal (workspaceSrc + "/src/api");
    features = builtins.concatLists [
-      (lib.optional (rootFeatures' ? "garage/k2v" || rootFeatures' ? "garage_api/k2v") "k2v")
+      (lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/k2v" || rootFeatures' ? "garage_api/k2v") "k2v")
      (lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/metrics" || rootFeatures' ? "garage_api/metrics") "metrics")
      (lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/metrics" || rootFeatures' ? "garage_api/metrics" || rootFeatures' ? "garage_api/opentelemetry-prometheus") "opentelemetry-prometheus")
      (lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/metrics" || rootFeatures' ? "garage_api/metrics" || rootFeatures' ? "garage_api/prometheus") "prometheus")
@@ -1584,11 +1583,11 @@ in
      form_urlencoded = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".form_urlencoded."1.0.1" { inherit profileName; }).out;
      futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.21" { inherit profileName; }).out;
      futures_util = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.21" { inherit profileName; }).out;
-      garage_block = (rustPackages."unknown".garage_block."0.8.0" { inherit profileName; }).out;
-      garage_model = (rustPackages."unknown".garage_model."0.8.0" { inherit profileName; }).out;
-      garage_rpc = (rustPackages."unknown".garage_rpc."0.8.0" { inherit profileName; }).out;
-      garage_table = (rustPackages."unknown".garage_table."0.8.0" { inherit profileName; }).out;
-      garage_util = (rustPackages."unknown".garage_util."0.8.0" { inherit profileName; }).out;
+      garage_block = (rustPackages."unknown".garage_block."0.8.1" { inherit profileName; }).out;
+      garage_model = (rustPackages."unknown".garage_model."0.8.1" { inherit profileName; }).out;
+      garage_rpc = (rustPackages."unknown".garage_rpc."0.8.1" { inherit profileName; }).out;
+      garage_table = (rustPackages."unknown".garage_table."0.8.1" { inherit profileName; }).out;
+      garage_util = (rustPackages."unknown".garage_util."0.8.1" { inherit profileName; }).out;
      hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out;
      hmac = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hmac."0.12.1" { inherit profileName; }).out;
      http = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.8" { inherit profileName; }).out;
@@ -1617,9 +1616,9 @@ in
    };
  });
  
-  "unknown".garage_block."0.8.0" = overridableMkRustCrate (profileName: rec {
+  "unknown".garage_block."0.8.1" = overridableMkRustCrate (profileName: rec {
    name = "garage_block";
-    version = "0.8.0";
+    version = "0.8.1";
    registry = "unknown";
    src = fetchCrateLocal (workspaceSrc + "/src/block");
    features = builtins.concatLists [
@@ -1632,14 +1631,13 @@ in
      bytes = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.2.0" { inherit profileName; }).out;
      futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.21" { inherit profileName; }).out;
      futures_util = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.21" { inherit profileName; }).out;
-      garage_db = (rustPackages."unknown".garage_db."0.8.0" { inherit profileName; }).out;
-      garage_rpc = (rustPackages."unknown".garage_rpc."0.8.0" { inherit profileName; }).out;
-      garage_table = (rustPackages."unknown".garage_table."0.8.0" { inherit profileName; }).out;
-      garage_util = (rustPackages."unknown".garage_util."0.8.0" { inherit profileName; }).out;
+      garage_db = (rustPackages."unknown".garage_db."0.8.1" { inherit profileName; }).out;
+      garage_rpc = (rustPackages."unknown".garage_rpc."0.8.1" { inherit profileName; }).out;
+      garage_table = (rustPackages."unknown".garage_table."0.8.1" { inherit profileName; }).out;
+      garage_util = (rustPackages."unknown".garage_util."0.8.1" { inherit profileName; }).out;
      hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out;
      opentelemetry = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry."0.17.0" { inherit profileName; }).out;
      rand = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand."0.8.5" { inherit profileName; }).out;
-      rmp_serde = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rmp-serde."0.15.5" { inherit profileName; }).out;
      serde = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde."1.0.137" { inherit profileName; }).out;
      serde_bytes = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde_bytes."0.11.5" { inherit profileName; }).out;
      tokio = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.17.0" { inherit profileName; }).out;
@@ -1649,9 +1647,9 @@ in
    };
  });
  
-  "unknown".garage_db."0.8.0" = overridableMkRustCrate (profileName: rec {
+  "unknown".garage_db."0.8.1" = overridableMkRustCrate (profileName: rec {
    name = "garage_db";
-    version = "0.8.0";
+    version = "0.8.1";
    registry = "unknown";
    src = fetchCrateLocal (workspaceSrc + "/src/db");
    features = builtins.concatLists [
@@ -1681,14 +1679,14 @@ in
    };
  });
  
-  "unknown".garage_model."0.8.0" = overridableMkRustCrate (profileName: rec {
+  "unknown".garage_model."0.8.1" = overridableMkRustCrate (profileName: rec {
    name = "garage_model";
-    version = "0.8.0";
+    version = "0.8.1";
    registry = "unknown";
    src = fetchCrateLocal (workspaceSrc + "/src/model");
    features = builtins.concatLists [
      [ "default" ]
-      (lib.optional (rootFeatures' ? "garage/k2v" || rootFeatures' ? "garage_api/k2v" || rootFeatures' ? "garage_model/k2v") "k2v")
+      (lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/k2v" || rootFeatures' ? "garage_api/k2v" || rootFeatures' ? "garage_model/k2v") "k2v")
      (lib.optional (rootFeatures' ? "garage/lmdb" || rootFeatures' ? "garage_model/lmdb") "lmdb")
      [ "sled" ]
      (lib.optional (rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_model/sqlite") "sqlite")
@@ -1701,16 +1699,15 @@ in
      err_derive = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".err-derive."0.3.1" { profileName = "__noProfile"; }).out;
      futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.21" { inherit profileName; }).out;
      futures_util = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.21" { inherit profileName; }).out;
-      garage_block = (rustPackages."unknown".garage_block."0.8.0" { inherit profileName; }).out;
-      garage_db = (rustPackages."unknown".garage_db."0.8.0" { inherit profileName; }).out;
-      garage_rpc = (rustPackages."unknown".garage_rpc."0.8.0" { inherit profileName; }).out;
-      garage_table = (rustPackages."unknown".garage_table."0.8.0" { inherit profileName; }).out;
-      garage_util = (rustPackages."unknown".garage_util."0.8.0" { inherit profileName; }).out;
+      garage_block = (rustPackages."unknown".garage_block."0.8.1" { inherit profileName; }).out;
+      garage_db = (rustPackages."unknown".garage_db."0.8.1" { inherit profileName; }).out;
+      garage_rpc = (rustPackages."unknown".garage_rpc."0.8.1" { inherit profileName; }).out;
+      garage_table = (rustPackages."unknown".garage_table."0.8.1" { inherit profileName; }).out;
+      garage_util = (rustPackages."unknown".garage_util."0.8.1" { inherit profileName; }).out;
      hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out;
      netapp = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".netapp."0.5.2" { inherit profileName; }).out;
      opentelemetry = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry."0.17.0" { inherit profileName; }).out;
      rand = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand."0.8.5" { inherit profileName; }).out;
-      rmp_serde = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rmp-serde."0.15.5" { inherit profileName; }).out;
      serde = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde."1.0.137" { inherit profileName; }).out;
      serde_bytes = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde_bytes."0.11.5" { inherit profileName; }).out;
      tokio = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.17.0" { inherit profileName; }).out;
@@ -1719,9 +1716,9 @@ in
    };
  });
  
-  "unknown".garage_rpc."0.8.0" = overridableMkRustCrate (profileName: rec {
+  "unknown".garage_rpc."0.8.1" = overridableMkRustCrate (profileName: rec {
    name = "garage_rpc";
-    version = "0.8.0";
+    version = "0.8.1";
    registry = "unknown";
    src = fetchCrateLocal (workspaceSrc + "/src/rpc");
    features = builtins.concatLists [
@@ -1741,7 +1738,7 @@ in
      ${ if rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/err-derive" then "err_derive" else null } = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".err-derive."0.3.1" { profileName = "__noProfile"; }).out;
      futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.21" { inherit profileName; }).out;
      futures_util = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.21" { inherit profileName; }).out;
-      garage_util = (rustPackages."unknown".garage_util."0.8.0" { inherit profileName; }).out;
+      garage_util = (rustPackages."unknown".garage_util."0.8.1" { inherit profileName; }).out;
      gethostname = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".gethostname."0.2.3" { inherit profileName; }).out;
      hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out;
      ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/k8s-openapi" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "k8s_openapi" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".k8s-openapi."0.16.0" { inherit profileName; }).out;
@@ -1752,7 +1749,6 @@ in
      pnet_datalink = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".pnet_datalink."0.28.0" { inherit profileName; }).out;
      rand = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand."0.8.5" { inherit profileName; }).out;
      ${ if rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest" then "reqwest" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".reqwest."0.11.12" { inherit profileName; }).out;
-      rmp_serde = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rmp-serde."0.15.5" { inherit profileName; }).out;
      ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kubernetes-discovery" || rootFeatures' ? "garage_rpc/schemars" then "schemars" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".schemars."0.8.8" { inherit profileName; }).out;
      serde = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde."1.0.137" { inherit profileName; }).out;
      serde_bytes = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde_bytes."0.11.5" { inherit profileName; }).out;
@@ -1763,24 +1759,24 @@ in
    };
  });
  
-  "unknown".garage_table."0.8.0" = overridableMkRustCrate (profileName: rec {
+  "unknown".garage_table."0.8.1" = overridableMkRustCrate (profileName: rec {
    name = "garage_table";
-    version = "0.8.0";
+    version = "0.8.1";
    registry = "unknown";
    src = fetchCrateLocal (workspaceSrc + "/src/table");
    dependencies = {
+      arc_swap = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".arc-swap."1.5.0" { inherit profileName; }).out;
      async_trait = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".async-trait."0.1.52" { profileName = "__noProfile"; }).out;
      bytes = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.2.0" { inherit profileName; }).out;
      futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.21" { inherit profileName; }).out;
      futures_util = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.21" { inherit profileName; }).out;
-      garage_db = (rustPackages."unknown".garage_db."0.8.0" { inherit profileName; }).out;
-      garage_rpc = (rustPackages."unknown".garage_rpc."0.8.0" { inherit profileName; }).out;
-      garage_util = (rustPackages."unknown".garage_util."0.8.0" { inherit profileName; }).out;
+      garage_db = (rustPackages."unknown".garage_db."0.8.1" { inherit profileName; }).out;
+      garage_rpc = (rustPackages."unknown".garage_rpc."0.8.1" { inherit profileName; }).out;
+      garage_util = (rustPackages."unknown".garage_util."0.8.1" { inherit profileName; }).out;
      hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out;
      hexdump = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hexdump."0.1.1" { inherit profileName; }).out;
      opentelemetry = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry."0.17.0" { inherit profileName; }).out;
      rand = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand."0.8.5" { inherit profileName; }).out;
-      rmp_serde = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rmp-serde."0.15.5" { inherit profileName; }).out;
      serde = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde."1.0.137" { inherit profileName; }).out;
      serde_bytes = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde_bytes."0.11.5" { inherit profileName; }).out;
      tokio = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.17.0" { inherit profileName; }).out;
@@ -1788,13 +1784,13 @@ in
    };
  });
  
-  "unknown".garage_util."0.8.0" = overridableMkRustCrate (profileName: rec {
+  "unknown".garage_util."0.8.1" = overridableMkRustCrate (profileName: rec {
    name = "garage_util";
-    version = "0.8.0";
+    version = "0.8.1";
    registry = "unknown";
    src = fetchCrateLocal (workspaceSrc + "/src/util");
    features = builtins.concatLists [
-      (lib.optional (rootFeatures' ? "garage/k2v" || rootFeatures' ? "garage_api/k2v" || rootFeatures' ? "garage_model/k2v" || rootFeatures' ? "garage_util/k2v") "k2v")
+      (lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/k2v" || rootFeatures' ? "garage_api/k2v" || rootFeatures' ? "garage_model/k2v" || rootFeatures' ? "garage_util/k2v") "k2v")
    ];
    dependencies = {
      arc_swap = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".arc-swap."1.5.0" { inherit profileName; }).out;
@@ -1805,9 +1801,10 @@ in
      digest = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".digest."0.10.3" { inherit profileName; }).out;
      err_derive = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".err-derive."0.3.1" { profileName = "__noProfile"; }).out;
      futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.21" { inherit profileName; }).out;
-      garage_db = (rustPackages."unknown".garage_db."0.8.0" { inherit profileName; }).out;
+      garage_db = (rustPackages."unknown".garage_db."0.8.1" { inherit profileName; }).out;
      git_version = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".git-version."0.3.5" { inherit profileName; }).out;
      hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out;
+      hexdump = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hexdump."0.1.1" { inherit profileName; }).out;
      http = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.8" { inherit profileName; }).out;
      hyper = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hyper."0.14.18" { inherit profileName; }).out;
      lazy_static = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".lazy_static."1.4.0" { inherit profileName; }).out;
@@ -1825,18 +1822,18 @@ in
    };
  });
  
-  "unknown".garage_web."0.8.0" = overridableMkRustCrate (profileName: rec {
+  "unknown".garage_web."0.8.1" = overridableMkRustCrate (profileName: rec {
    name = "garage_web";
-    version = "0.8.0";
+    version = "0.8.1";
    registry = "unknown";
    src = fetchCrateLocal (workspaceSrc + "/src/web");
    dependencies = {
      err_derive = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".err-derive."0.3.1" { profileName = "__noProfile"; }).out;
      futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.21" { inherit profileName; }).out;
-      garage_api = (rustPackages."unknown".garage_api."0.8.0" { inherit profileName; }).out;
-      garage_model = (rustPackages."unknown".garage_model."0.8.0" { inherit profileName; }).out;
-      garage_table = (rustPackages."unknown".garage_table."0.8.0" { inherit profileName; }).out;
-      garage_util = (rustPackages."unknown".garage_util."0.8.0" { inherit profileName; }).out;
+      garage_api = (rustPackages."unknown".garage_api."0.8.1" { inherit profileName; }).out;
+      garage_model = (rustPackages."unknown".garage_model."0.8.1" { inherit profileName; }).out;
+      garage_table = (rustPackages."unknown".garage_table."0.8.1" { inherit profileName; }).out;
+      garage_util = (rustPackages."unknown".garage_util."0.8.1" { inherit profileName; }).out;
      http = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.8" { inherit profileName; }).out;
      hyper = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hyper."0.14.18" { inherit profileName; }).out;
      opentelemetry = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry."0.17.0" { inherit profileName; }).out;
@@ -2450,7 +2447,7 @@ in
    dependencies = {
      base64 = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".base64."0.13.0" { inherit profileName; }).out;
      ${ if rootFeatures' ? "k2v-client/clap" || rootFeatures' ? "k2v-client/cli" then "clap" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".clap."3.1.18" { inherit profileName; }).out;
-      ${ if rootFeatures' ? "k2v-client/cli" || rootFeatures' ? "k2v-client/garage_util" then "garage_util" else null } = (rustPackages."unknown".garage_util."0.8.0" { inherit profileName; }).out;
+      ${ if rootFeatures' ? "k2v-client/cli" || rootFeatures' ? "k2v-client/garage_util" then "garage_util" else null } = (rustPackages."unknown".garage_util."0.8.1" { inherit profileName; }).out;
      http = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.8" { inherit profileName; }).out;
      log = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".log."0.4.16" { inherit profileName; }).out;
      rusoto_core = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rusoto_core."0.48.0" { inherit profileName; }).out;
@@ -5398,7 +5395,6 @@ in
      [ "attributes" ]
      [ "default" ]
      [ "log" ]
-      [ "log-always" ]
      [ "std" ]
      [ "tracing-attributes" ]
    ];
@@ -11,14 +11,14 @@ let

  build_debug_and_release = (target: {
    debug = (compile {
-      inherit target git_version;
+      inherit system target git_version pkgsSrc cargo2nixOverlay;
      release = false;
    }).workspace.garage {
      compileMode = "build";
    };

    release = (compile {
-      inherit target git_version;
+      inherit system target git_version pkgsSrc cargo2nixOverlay;
      release = true;
    }).workspace.garage {
      compileMode = "build";
@@ -39,7 +39,7 @@ in {
  };
  test = {
    amd64 = test (compile {
-      inherit git_version;
+      inherit system git_version pkgsSrc cargo2nixOverlay;
      target = "x86_64-unknown-linux-musl";
      features = [
        "garage/bundled-libs"
@@ -52,7 +52,7 @@ in {
  };
  clippy = {
    amd64 = (compile {
-      inherit git_version;
+      inherit system git_version pkgsSrc cargo2nixOverlay;
      target = "x86_64-unknown-linux-musl";
      compiler = "clippy";
    }).workspace.garage {
@@ -5,16 +5,59 @@ weight = 20

 ## S3

+### Using Minio SDK
+
+First install the SDK:
+
+```bash
+pip3 install minio
+```
+
+Then instantiate a client object using garage root domain, api key and secret:
+
+```python
+import minio
+
+client = minio.Minio(
+  "your.domain.tld",
+  "GKyourapikey",
+  "abcd[...]1234",
+  # Force the region, this is specific to garage
+  region="region",
+)
+```
+
+Then use all the standard S3 endpoints as implemented by the Minio SDK:
+
+```
+# List buckets
+print(client.list_buckets())
+
+# Put an object containing 'content' to /path in bucket named 'bucket':
+content = b"content"
+client.put_object(
+  "bucket",
+  "path",
+  io.BytesIO(content),
+  len(content),
+)
+
+# Read the object back and check contents
+data = client.get_object("bucket", "path").read()
+assert data == content
+```
+
+For further documentation, see the Minio SDK
+[Reference](https://docs.min.io/docs/python-client-api-reference.html)
+
+### Using Amazon boto3
+
 *Coming soon*

-Some refs:
-  - Minio SDK
-    - [Reference](https://docs.min.io/docs/python-client-api-reference.html)
-
-  - Amazon boto3
-    - [Installation](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/quickstart.html)
-    - [Reference](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/s3.html)
-    - [Example](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/s3-uploading-files.html)
+See the official documentation:
+  - [Installation](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/quickstart.html)
+  - [Reference](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/s3.html)
+  - [Example](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/s3-uploading-files.html)

 ## K2V

@@ -8,7 +8,7 @@ In this section, we cover the following web applications:
 | Name | Status | Note |
 |------|--------|------|
 | [Nextcloud](#nextcloud)     | ✅       |  Both Primary Storage and External Storage are supported    |
-| [Peertube](#peertube)     | ✅       | Must be configured with the website endpoint     |
+| [Peertube](#peertube)     | ✅       | Supported with the website endpoint, proxifying private videos unsupported     |
 | [Mastodon](#mastodon)     | ✅       | Natively supported    |
 | [Matrix](#matrix)     | ✅       |  Tested with `synapse-s3-storage-provider`    |
 | [Pixelfed](#pixelfed)     | ❓       |  Not yet tested    |
@@ -128,6 +128,10 @@ In other words, Peertube is only responsible of the "control plane" and offload
 In return, this system is a bit harder to configure.
 We show how it is still possible to configure Garage with Peertube, allowing you to spread the load and the bandwidth usage on the Garage cluster.

+Starting from version 5.0, Peertube also supports improving the security for private videos by not exposing them directly
+but relying on a single control point in the Peertube instance. This is based on S3 per-object and prefix ACL, which are not currently supported
+in Garage, so this feature is unsupported. While this technically impedes security for private videos, it is not a blocking issue and could be
+a reasonable trade-off for some instances.

 ### Create resources in Garage

@@ -195,6 +199,11 @@ object_storage:

  max_upload_part: 2GB

+  proxy:
+    # You may enable this feature, yet it will not provide any security benefit, so
+    # you should rather benefit from Garage public endpoint for all videos
+    proxify_private_files: false
+
  streaming_playlists:
    bucket_name: 'peertube-playlist'

@@ -109,7 +109,7 @@ especially you must consider the following folders/files:
  this folder will be your main data storage and must be on a large storage (e.g. large HDD)


-A valid `/etc/garage/garage.toml` for our cluster would look as follows:
+A valid `/etc/garage.toml` for our cluster would look as follows:

 ```toml
 metadata_dir = "/var/lib/garage/meta"
@@ -12,7 +12,7 @@ as pictures, video, images, documents, etc., in a redundant multi-node
 setting. S3 is versatile enough to also be used to publish a static
 website.

-Garage is an opinionated object storage solutoin, we focus on the following **desirable properties**:
+Garage is an opinionated object storage solution, we focus on the following **desirable properties**:

  - **Internet enabled**: made for multi-sites (eg. datacenters, offices, households, etc.) interconnected through regular Internet connections.
  - **Self-contained & lightweight**: works everywhere and integrates well in existing environments to target [hyperconverged infrastructures](https://en.wikipedia.org/wiki/Hyper-converged_infrastructure).
@@ -83,7 +83,7 @@ This feature is totally invisible to S3 clients and does not break compatibility
 ### Cluster administration API

 Garage provides a fully-fledged REST API to administer your cluster programatically.
-Functionnality included in the admin API include: setting up and monitoring
+Functionality included in the admin API include: setting up and monitoring
 cluster nodes, managing access credentials, and managing storage buckets and bucket aliases.
 A full reference of the administration API is available [here](@/documentation/reference-manual/admin-api.md).

@@ -0,0 +1,686 @@
+++
+title = "Administration API"
+weight = 60
+++
+
+The Garage administration API is accessible through a dedicated server whose
+listen address is specified in the `[admin]` section of the configuration
+file (see [configuration file
+reference](@/documentation/reference-manual/configuration.md))
+
+**WARNING.** At this point, there is no comittement to stability of the APIs described in this document.
+We will bump the version numbers prefixed to each API endpoint at each time the syntax
+or semantics change, meaning that code that relies on these endpoint will break
+when changes are introduced.
+
+The Garage administration API was introduced in version 0.7.2, this document
+does not apply to older versions of Garage.
+
+
+## Access control
+
+The admin API uses two different tokens for acces control, that are specified in the config file's `[admin]` section:
+
+- `metrics_token`: the token for accessing the Metrics endpoint (if this token
+  is not set in the config file, the Metrics endpoint can be accessed without
+  access control);
+
+- `admin_token`: the token for accessing all of the other administration
+  endpoints (if this token is not set in the config file, access to these
+  endpoints is disabled entirely).
+
+These tokens are used as simple HTTP bearer tokens. In other words, to
+authenticate access to an admin API endpoint, add the following HTTP header
+to your request:
+
+```
+Authorization: Bearer <token>
+```
+
+## Administration API endpoints
+
+### Metrics-related endpoints
+
+#### Metrics `GET /metrics`
+
+Returns internal Garage metrics in Prometheus format.
+
+#### Health `GET /health`
+
+Used for simple health checks in a cluster setting with an orchestrator.
+Returns an HTTP status 200 if the node is ready to answer user's requests,
+and an HTTP status 503 (Service Unavailable) if there are some partitions
+for which a quorum of nodes is not available.
+A simple textual message is also returned in a body with content-type `text/plain`.
+See `/v0/health` for an API that also returns JSON output.
+
+### Cluster operations
+
+#### GetClusterStatus `GET /v0/status`
+
+Returns the cluster's current status in JSON, including:
+
+- ID of the node being queried and its version of the Garage daemon
+- Live nodes
+- Currently configured cluster layout
+- Staged changes to the cluster layout
+
+Example response body:
+
+```json
+{
+  "node": "ec79480e0ce52ae26fd00c9da684e4fa56658d9c64cdcecb094e936de0bfe71f",
+  "garage_version": "git:v0.8.0",
+  "knownNodes": {
+    "ec79480e0ce52ae26fd00c9da684e4fa56658d9c64cdcecb094e936de0bfe71f": {
+      "addr": "10.0.0.11:3901",
+      "is_up": true,
+      "last_seen_secs_ago": 9,
+      "hostname": "node1"
+    },
+    "4a6ae5a1d0d33bf895f5bb4f0a418b7dc94c47c0dd2eb108d1158f3c8f60b0ff": {
+      "addr": "10.0.0.12:3901",
+      "is_up": true,
+      "last_seen_secs_ago": 1,
+      "hostname": "node2"
+    },
+    "23ffd0cdd375ebff573b20cc5cef38996b51c1a7d6dbcf2c6e619876e507cf27": {
+      "addr": "10.0.0.21:3901",
+      "is_up": true,
+      "last_seen_secs_ago": 7,
+      "hostname": "node3"
+    },
+    "e2ee7984ee65b260682086ec70026165903c86e601a4a5a501c1900afe28d84b": {
+      "addr": "10.0.0.22:3901",
+      "is_up": true,
+      "last_seen_secs_ago": 1,
+      "hostname": "node4"
+    }
+  },
+  "layout": {
+    "version": 12,
+    "roles": {
+      "ec79480e0ce52ae26fd00c9da684e4fa56658d9c64cdcecb094e936de0bfe71f": {
+        "zone": "dc1",
+        "capacity": 4,
+        "tags": [
+          "node1"
+        ]
+      },
+      "4a6ae5a1d0d33bf895f5bb4f0a418b7dc94c47c0dd2eb108d1158f3c8f60b0ff": {
+        "zone": "dc1",
+        "capacity": 6,
+        "tags": [
+          "node2"
+        ]
+      },
+      "23ffd0cdd375ebff573b20cc5cef38996b51c1a7d6dbcf2c6e619876e507cf27": {
+        "zone": "dc2",
+        "capacity": 10,
+        "tags": [
+          "node3"
+        ]
+      }
+    },
+    "stagedRoleChanges": {
+      "e2ee7984ee65b260682086ec70026165903c86e601a4a5a501c1900afe28d84b": {
+        "zone": "dc2",
+        "capacity": 5,
+        "tags": [
+          "node4"
+        ]
+      }
+    }
+  }
+}
+```
+
+#### GetClusterHealth `GET /v0/health`
+
+Returns the cluster's current health in JSON format, with the following variables:
+
+- `status`: one of `Healthy`, `Degraded` or `Unavailable`:
+  - Healthy: Garage node is connected to all storage nodes
+  - Degraded: Garage node is not connected to all storage nodes, but a quorum of write nodes is available for all partitions
+  - Unavailable: a quorum of write nodes is not available for some partitions
+- `known_nodes`: the number of nodes this Garage node has had a TCP connection to since the daemon started
+- `connected_nodes`: the nubmer of nodes this Garage node currently has an open connection to
+- `storage_nodes`: the number of storage nodes currently registered in the cluster layout
+- `storage_nodes_ok`: the number of storage nodes to which a connection is currently open
+- `partitions`: the total number of partitions of the data (currently always 256)
+- `partitions_quorum`: the number of partitions for which a quorum of write nodes is available
+- `partitions_all_ok`: the number of partitions for which we are connected to all storage nodes responsible of storing it
+
+Contrarily to `GET /health`, this endpoint always returns a 200 OK HTTP response code.
+
+Example response body:
+
+```json
+{
+    "status": "Degraded",
+    "known_nodes": 3,
+    "connected_nodes": 2,
+    "storage_nodes": 3,
+    "storage_nodes_ok": 2,
+    "partitions": 256,
+    "partitions_quorum": 256,
+    "partitions_all_ok": 0
+}
+```
+
+#### ConnectClusterNodes `POST /v0/connect`
+
+Instructs this Garage node to connect to other Garage nodes at specified addresses.
+
+Example request body:
+
+```json
+[
+    "ec79480e0ce52ae26fd00c9da684e4fa56658d9c64cdcecb094e936de0bfe71f@10.0.0.11:3901",
+    "4a6ae5a1d0d33bf895f5bb4f0a418b7dc94c47c0dd2eb108d1158f3c8f60b0ff@10.0.0.12:3901"
+]
+```
+
+The format of the string for a node to connect to is: `<node ID>@<ip address>:<port>`, same as in the `garage node connect` CLI call.
+
+Example response:
+
+```json
+[
+    {
+        "success": true,
+        "error": null
+    },
+    {
+        "success": false,
+        "error": "Handshake error"
+    }
+]
+```
+
+#### GetClusterLayout `GET /v0/layout`
+
+Returns the cluster's current layout in JSON, including:
+
+- Currently configured cluster layout
+- Staged changes to the cluster layout
+
+(the info returned by this endpoint is a subset of the info returned by GetClusterStatus)
+
+Example response body:
+
+```json
+{
+  "version": 12,
+  "roles": {
+    "ec79480e0ce52ae26fd00c9da684e4fa56658d9c64cdcecb094e936de0bfe71f": {
+      "zone": "dc1",
+      "capacity": 4,
+      "tags": [
+        "node1"
+      ]
+    },
+    "4a6ae5a1d0d33bf895f5bb4f0a418b7dc94c47c0dd2eb108d1158f3c8f60b0ff": {
+      "zone": "dc1",
+      "capacity": 6,
+      "tags": [
+        "node2"
+      ]
+    },
+    "23ffd0cdd375ebff573b20cc5cef38996b51c1a7d6dbcf2c6e619876e507cf27": {
+      "zone": "dc2",
+      "capacity": 10,
+      "tags": [
+        "node3"
+      ]
+    }
+  },
+  "stagedRoleChanges": {
+    "e2ee7984ee65b260682086ec70026165903c86e601a4a5a501c1900afe28d84b": {
+      "zone": "dc2",
+      "capacity": 5,
+      "tags": [
+        "node4"
+      ]
+    }
+  }
+}
+```
+
+#### UpdateClusterLayout `POST /v0/layout`
+
+Send modifications to the cluster layout. These modifications will
+be included in the staged role changes, visible in subsequent calls
+of `GetClusterLayout`. Once the set of staged changes is satisfactory,
+the user may call `ApplyClusterLayout` to apply the changed changes,
+or `Revert ClusterLayout` to clear all of the staged changes in
+the layout.
+
+Request body format:
+
+```json
+{
+  <node_id>: {
+    "capacity": <new_capacity>,
+    "zone": <new_zone>,
+    "tags": [
+      <new_tag>,
+      ...
+    ]
+  },
+  <node_id_to_remove>: null,
+  ...
+}
+```
+
+Contrary to the CLI that may update only a subset of the fields
+`capacity`, `zone` and `tags`, when calling this API all of these
+values must be specified.
+
+
+#### ApplyClusterLayout `POST /v0/layout/apply`
+
+Applies to the cluster the layout changes currently registered as
+staged layout changes.
+
+Request body format:
+
+```json
+{
+  "version": 13
+}
+```
+
+Similarly to the CLI, the body must include the version of the new layout
+that will be created, which MUST be 1 + the value of the currently
+existing layout in the cluster.
+
+#### RevertClusterLayout `POST /v0/layout/revert`
+
+Clears all of the staged layout changes.
+
+Request body format:
+
+```json
+{
+  "version": 13
+}
+```
+
+Reverting the staged changes is done by incrementing the version number
+and clearing the contents of the staged change list.
+Similarly to the CLI, the body must include the incremented
+version number, which MUST be 1 + the value of the currently
+existing layout in the cluster.
+
+
+### Access key operations
+
+#### ListKeys `GET /v0/key`
+
+Returns all API access keys in the cluster.
+
+Example response:
+
+```json
+[
+  {
+    "id": "GK31c2f218a2e44f485b94239e",
+    "name": "test"
+  },
+  {
+    "id": "GKe10061ac9c2921f09e4c5540",
+    "name": "test2"
+  }
+]
+```
+
+#### CreateKey `POST /v0/key`
+
+Creates a new API access key.
+
+Request body format:
+
+```json
+{
+    "name": "NameOfMyKey"
+}
+```
+
+#### ImportKey `POST /v0/key/import`
+
+Imports an existing API key.
+
+Request body format:
+
+```json
+{
+    "accessKeyId": "GK31c2f218a2e44f485b94239e",
+    "secretAccessKey": "b892c0665f0ada8a4755dae98baa3b133590e11dae3bcc1f9d769d67f16c3835",
+    "name": "NameOfMyKey"
+}
+```
+
+#### GetKeyInfo `GET /v0/key?id=<acces key id>`
+#### GetKeyInfo `GET /v0/key?search=<pattern>`
+
+Returns information about the requested API access key.
+
+If `id` is set, the key is looked up using its exact identifier (faster).
+If `search` is set, the key is looked up using its name or prefix
+of identifier (slower, all keys are enumerated to do this).
+
+Example response:
+
+```json
+{
+  "name": "test",
+  "accessKeyId": "GK31c2f218a2e44f485b94239e",
+  "secretAccessKey": "b892c0665f0ada8a4755dae98baa3b133590e11dae3bcc1f9d769d67f16c3835",
+  "permissions": {
+    "createBucket": false
+  },
+  "buckets": [
+    {
+      "id": "70dc3bed7fe83a75e46b66e7ddef7d56e65f3c02f9f80b6749fb97eccb5e1033",
+      "globalAliases": [
+        "test2"
+      ],
+      "localAliases": [],
+      "permissions": {
+        "read": true,
+        "write": true,
+        "owner": false
+      }
+    },
+    {
+      "id": "d7452a935e663fc1914f3a5515163a6d3724010ce8dfd9e4743ca8be5974f995",
+      "globalAliases": [
+        "test3"
+      ],
+      "localAliases": [],
+      "permissions": {
+        "read": true,
+        "write": true,
+        "owner": false
+      }
+    },
+    {
+      "id": "e6a14cd6a27f48684579ec6b381c078ab11697e6bc8513b72b2f5307e25fff9b",
+      "globalAliases": [],
+      "localAliases": [
+        "test"
+      ],
+      "permissions": {
+        "read": true,
+        "write": true,
+        "owner": true
+      }
+    },
+    {
+      "id": "96470e0df00ec28807138daf01915cfda2bee8eccc91dea9558c0b4855b5bf95",
+      "globalAliases": [
+        "alex"
+      ],
+      "localAliases": [],
+      "permissions": {
+        "read": true,
+        "write": true,
+        "owner": true
+      }
+    }
+  ]
+}
+```
+
+#### DeleteKey `DELETE /v0/key?id=<acces key id>`
+
+Deletes an API access key.
+
+#### UpdateKey `POST /v0/key?id=<acces key id>`
+
+Updates information about the specified API access key.
+
+Request body format:
+
+```json
+{
+    "name": "NameOfMyKey",
+    "allow": {
+        "createBucket": true,
+    },
+    "deny": {}
+}
+```
+
+All fields (`name`, `allow` and `deny`) are optionnal.
+If they are present, the corresponding modifications are applied to the key, otherwise nothing is changed.
+The possible flags in `allow` and `deny` are: `createBucket`.
+
+
+### Bucket operations
+
+#### ListBuckets `GET /v0/bucket`
+
+Returns all storage buckets in the cluster.
+
+Example response:
+
+```json
+[
+  {
+    "id": "70dc3bed7fe83a75e46b66e7ddef7d56e65f3c02f9f80b6749fb97eccb5e1033",
+    "globalAliases": [
+      "test2"
+    ],
+    "localAliases": []
+  },
+  {
+    "id": "96470e0df00ec28807138daf01915cfda2bee8eccc91dea9558c0b4855b5bf95",
+    "globalAliases": [
+      "alex"
+    ],
+    "localAliases": []
+  },
+  {
+    "id": "d7452a935e663fc1914f3a5515163a6d3724010ce8dfd9e4743ca8be5974f995",
+    "globalAliases": [
+      "test3"
+    ],
+    "localAliases": []
+  },
+  {
+    "id": "e6a14cd6a27f48684579ec6b381c078ab11697e6bc8513b72b2f5307e25fff9b",
+    "globalAliases": [],
+    "localAliases": [
+      {
+        "accessKeyId": "GK31c2f218a2e44f485b94239e",
+        "alias": "test"
+      }
+    ]
+  }
+]
+```
+
+#### GetBucketInfo `GET /v0/bucket?id=<bucket id>`
+#### GetBucketInfo `GET /v0/bucket?globalAlias=<alias>`
+
+Returns information about the requested storage bucket.
+
+If `id` is set, the bucket is looked up using its exact identifier.
+If `globalAlias` is set, the bucket is looked up using its global alias.
+(both are fast)
+
+Example response:
+
+```json
+{
+    "id": "afa8f0a22b40b1247ccd0affb869b0af5cff980924a20e4b5e0720a44deb8d39",
+        "globalAliases": [],
+        "websiteAccess": false,
+        "websiteConfig": null,
+        "keys": [
+        {
+            "accessKeyId": "GK31c2f218a2e44f485b94239e",
+            "name": "Imported key",
+            "permissions": {
+                "read": true,
+                "write": true,
+                "owner": true
+            },
+            "bucketLocalAliases": [
+                "debug"
+            ]
+        }
+        ],
+        "objects": 14827,
+        "bytes": 13189855625,
+        "unfinshedUploads": 0,
+        "quotas": {
+            "maxSize": null,
+            "maxObjects": null
+        }
+}
+```
+
+#### CreateBucket `POST /v0/bucket`
+
+Creates a new storage bucket.
+
+Request body format:
+
+```json
+{
+    "globalAlias": "NameOfMyBucket"
+}
+```
+
+OR
+
+```json
+{
+    "localAlias": {
+        "accessKeyId": "GK31c2f218a2e44f485b94239e",
+        "alias": "NameOfMyBucket",
+        "allow": {
+            "read": true,
+            "write": true,
+            "owner": false
+        }
+    }
+}
+```
+
+OR
+
+```json
+{}
+```
+
+Creates a new bucket, either with a global alias, a local one,
+or no alias at all.
+
+Technically, you can also specify both `globalAlias` and `localAlias` and that would create
+two aliases, but I don't see why you would want to do that.
+
+#### DeleteBucket `DELETE /v0/bucket?id=<bucket id>`
+
+Deletes a storage bucket. A bucket cannot be deleted if it is not empty.
+
+Warning: this will delete all aliases associated with the bucket!
+
+#### UpdateBucket `PUT /v0/bucket?id=<bucket id>`
+
+Updates configuration of the given bucket.
+
+Request body format:
+
+```json
+{
+    "websiteAccess": {
+        "enabled": true,
+        "indexDocument": "index.html",
+        "errorDocument": "404.html"
+    },
+    "quotas": {
+        "maxSize": 19029801,
+        "maxObjects": null,
+    }
+}
+```
+
+All fields (`websiteAccess` and `quotas`) are optionnal.
+If they are present, the corresponding modifications are applied to the bucket, otherwise nothing is changed.
+
+In `websiteAccess`: if `enabled` is `true`, `indexDocument` must be specified.
+The field `errorDocument` is optional, if no error document is set a generic
+error message is displayed when errors happen. Conversely, if `enabled` is
+`false`, neither `indexDocument` nor `errorDocument` must be specified.
+
+In `quotas`: new values of `maxSize` and `maxObjects` must both be specified, or set to `null`
+to remove the quotas. An absent value will be considered the same as a `null`. It is not possible
+to change only one of the two quotas.
+
+### Operations on permissions for keys on buckets
+
+#### BucketAllowKey `POST /v0/bucket/allow`
+
+Allows a key to do read/write/owner operations on a bucket.
+
+Request body format:
+
+```json
+{
+    "bucketId": "e6a14cd6a27f48684579ec6b381c078ab11697e6bc8513b72b2f5307e25fff9b",
+    "accessKeyId": "GK31c2f218a2e44f485b94239e",
+    "permissions": {
+        "read": true,
+        "write": true,
+        "owner": true
+    },
+}
+```
+
+Flags in `permissions` which have the value `true` will be activated.
+Other flags will remain unchanged.
+
+#### BucketDenyKey `POST /v0/bucket/deny`
+
+Denies a key from doing read/write/owner operations on a bucket.
+
+Request body format:
+
+```json
+{
+    "bucketId": "e6a14cd6a27f48684579ec6b381c078ab11697e6bc8513b72b2f5307e25fff9b",
+    "accessKeyId": "GK31c2f218a2e44f485b94239e",
+    "permissions": {
+        "read": false,
+        "write": false,
+        "owner": true
+    },
+}
+```
+
+Flags in `permissions` which have the value `true` will be deactivated.
+Other flags will remain unchanged.
+
+
+### Operations on bucket aliases
+
+#### GlobalAliasBucket `PUT /v0/bucket/alias/global?id=<bucket id>&alias=<global alias>`
+
+Empty body. Creates a global alias for a bucket.
+
+#### GlobalUnaliasBucket `DELETE /v0/bucket/alias/global?id=<bucket id>&alias=<global alias>`
+
+Removes a global alias for a bucket.
+
+#### LocalAliasBucket `PUT /v0/bucket/alias/local?id=<bucket id>&accessKeyId=<access key ID>&alias=<local alias>`
+
+Empty body. Creates a local alias for a bucket in the namespace of a specific access key.
+
+#### LocalUnaliasBucket `DELETE /v0/bucket/alias/local?id=<bucket id>&accessKeyId<access key ID>&alias=<local alias>`
+
+Removes a local alias for a bucket in the namespace of a specific access key.
+
@@ -0,0 +1,10 @@
+*.aux
+*.bbl
+*.blg
+*.log
+*.nav
+*.out
+*.snm
+*.synctex.gz
+*.toc
+*.dvi
@@ -0,0 +1,8 @@
+all:
+	pdflatex présentation.tex
+
+clean:
+	rm -f *.aux *.bbl *.blg *.log *.nav *.out *.snm *.synctex.gz *.toc *.dvi présentation.pdf
+
+clean_sauf_pdf:
+	rm -f *.aux *.bbl *.blg *.log *.nav *.out *.snm *.synctex.gz *.toc *.dvi
@@ -0,0 +1,340 @@
+\documentclass[11pt, aspectratio=1610]{beamer}
+\usetheme{Warsaw}
+\usepackage[utf8]{inputenc}
+\usepackage[french]{babel}
+\usepackage{amsmath}
+\usepackage{amsfonts}
+\usepackage{amssymb}
+\usepackage{tikz}
+\usepackage{graphicx}
+\usepackage{xcolor}
+\usepackage{setspace}
+\usepackage{todonotes}
+\presetkeys{todonotes}{inline}{}
+\renewcommand{\baselinestretch}{1.25}
+
+\definecolor{orange_garage}{RGB}{255,147,41}
+\definecolor{gris_garage}{RGB}{78,78,78}
+
+\author[Association Deuxfleurs]{~\linebreak Vincent Giraud}
+\title[De l'auto-hébergement à l'entre-hébergement avec Garage]{De l'auto-hébergement à l'entre-hébergement :\\Garage, pour conserver ses données ensemble}
+%\setbeamercovered{transparent} 
+%\setbeamertemplate{navigation symbols}{} 
+\date{Capitole du Libre 2022\linebreak
+
+\scriptsize Samedi 19 novembre 2022\linebreak
+} 
+
+\setbeamercolor{palette primary}{fg=gris_garage,bg=orange_garage}
+\setbeamercolor{palette secondary}{fg=gris_garage,bg=gris_garage}
+\setbeamercolor{palette tiertary}{fg=white,bg=gris_garage}
+\setbeamercolor{palette quaternary}{fg=white,bg=gris_garage}
+\setbeamercolor{navigation symbols}{fg=black, bg=white}
+\setbeamercolor{navigation symbols dimmed}{fg=darkgray, bg=white}
+\setbeamercolor{itemize item}{fg=gris_garage}
+\setbeamertemplate{itemize item}[circle]
+
+\addtobeamertemplate{navigation symbols}{}{%
+    \usebeamerfont{footline}%
+    \usebeamercolor[fg]{footline}%
+    \hspace{1em}%
+    \insertframenumber/\inserttotalframenumber
+}
+
+\setbeamertemplate{headline}
+{%
+  \leavevmode%
+  \begin{beamercolorbox}[wd=.5\paperwidth,ht=2.5ex,dp=1.125ex]{section in head/foot}%
+    \hbox to .5\paperwidth{\hfil\insertsectionhead\hfil}
+  \end{beamercolorbox}%
+  \begin{beamercolorbox}[wd=.5\paperwidth,ht=2.5ex,dp=1.125ex]{subsection in head/foot}%
+    \hbox to .5\paperwidth{\hfil\insertsubsectionhead\hfil}
+  \end{beamercolorbox}%
+}
+\addtobeamertemplate{footnote}{}{\vspace{2ex}}
+
+\begin{document}
+\begin{frame}
+\titlepage
+\end{frame}
+
+\section{Introduction}
+\subsection{Présentation}
+\begin{frame}
+\begin{columns}
+\column{0.5 \linewidth}
+\begin{center}
+\includegraphics[width=3.5cm]{deuxfleurs-logo.png}\linebreak
+
+\texttt{https://deuxfleurs.fr}
+\end{center}
+\column{0.4 \linewidth}
+\begin{center}
+Deuxfleurs est une association militant en faveur d'un internet plus convivial, avec une organisation et des rapports de force repensés.\linebreak
+
+Nous faisons partie du CHATONS\footnote[frame]{Collectif des Hébergeurs Alternatifs, Transparents, Ouverts, Neutres et Solidaires} depuis avril 2022.
+
+\includegraphics[width=2cm]{logo_chatons.png}
+\end{center}
+\end{columns}
+\end{frame}
+
+\subsection{Héberger à la maison}
+\begin{frame}
+\begin{columns}
+\begin{column}{0.5 \linewidth}
+\begin{center}
+Pour échapper au contrôle et au giron des opérateurs de clouds, héberger ses données à la maison présente de nombreux avantages...
+\end{center}
+
+\vspace{0.5cm}
+
+\begin{itemize}[<+(1)->]
+\item On récupère la souveraineté sur ses données
+\item On gagne en vie privée
+\item On gagne en libertés
+\item On est responsabilisé face à ses besoins
+\end{itemize}
+\end{column}
+\vrule{}
+\begin{column}{0.5 \linewidth}
+\begin{center}
+\onslide<6->{... mais aussi bien des contraintes...}
+\end{center}
+
+\vspace{0.5cm}
+
+\begin{itemize}[<+(2)->]
+\item On repose sur une connexion internet pour particulier
+\item Un certain savoir-faire et moultes compétences sont requis
+\item Assurer la résilience de ses services est difficile
+\item Bien sauvegarder ses données, et ceci au-delà de son site géographique, n'est pas évident
+\end{itemize}
+\end{column}
+\end{columns}
+\end{frame}
+
+\subsection{Sauvegarder pour se parer à tout imprévu}
+\begin{frame}
+\begin{center}
+Sauvegarder pour se parer contre les pannes matérielles est une chose...
+
+Sauvegarder pour se parer contre les cambriolages et les incendies en est une autre !\linebreak
+
+\vspace{1cm}
+\onslide<2->{Répartir géographiquement ses données devient alors nécessaire.}
+\end{center}
+\end{frame}
+
+\section{Les solutions à explorer}
+\subsection{L'entre-hébergement}
+\begin{frame}
+\begin{center}
+On a vu récemment se développer au sein du CHATONS la notion d'entre-hébergement : en plus de renforcer l'intégrité des sauvegardes, on va améliorer la disponibilité pendant les coupures de liaison internet, de courant, ou pendant les déménagements d'administrateurs par exemple.\linebreak
+
+\vspace{1cm}
+\onslide<2->
+{
+Dans le cadre du collectif, il s'agit de partager ses volumes de données entre hébergeurs.\linebreak
+
+Pour assurer la confidentialité, on peut chiffrer les données au niveau applicatif.
+}
+\end{center}
+\end{frame}
+
+\subsection{S3 contre les systèmes de fichiers}
+\begin{frame}
+\begin{center}
+Dans le cadre de l'administration de services en ligne, les systèmes de fichiers recèlent certaines difficultés.\linebreak
+
+\vspace{1cm}
+Le standard S3 apporte des facilités; on réduit le stockage à un paradigme de clé-valeur basé essentiellement sur deux opérations seulement: lire ou écrire une clé.
+\end{center}
+\end{frame}
+
+\section{Garage}
+\subsection{Présentation}
+\begin{frame}
+\begin{columns}
+\column{0.5 \linewidth}
+\begin{center}
+Garage essaye de répondre à l'ensemble de ces besoins.\linebreak
+
+\vspace{0.5cm}
+Il s'agit d'un logiciel libre permettant de distribuer un service S3 sur diverses machines éloignées.
+\end{center}
+\column{0.5 \linewidth}
+\begin{center}
+\includegraphics[width=4cm]{garage-logo.png}\linebreak
+
+\texttt{https://garagehq.deuxfleurs.fr/}
+\end{center}
+\end{columns}
+\end{frame}
+
+\subsection{Gestion des zones}
+\begin{frame}
+\begin{center}
+Garage va prendre en compte les zones géographiques au moment de répliquer les données.\linebreak
+
+\vspace{1cm}
+\includegraphics[width=13.25cm]{zones.png}
+\end{center}
+\end{frame}
+
+\subsection{Comment ça marche ?}
+\begin{frame}
+\begin{columns}
+\column{0.5 \linewidth}
+\input{schéma europe}
+\column{0.5 \linewidth}
+\begin{center}
+Chaque objet est dupliqué sur plusieurs zones différentes.\linebreak
+
+\onslide<5->{Lorsqu'un nouvel hébergeur rejoint le réseau, la charge se voit équilibrée.}\linebreak
+
+\onslide<12->{Si une zone devient indisponible, les autres continuent d'assurer le service.}\linebreak
+\end{center}
+\end{columns}
+\end{frame}
+
+\subsection{Financement}
+\begin{frame}
+\begin{center}
+Dans le cadre du programme \textit{Horizon 2021} de l'Union Européenne, nous avons reçu une subvention de la part de l'initiative NGI Pointer\footnote[frame]{Next Generation Internet Program for Open Internet Renovation}.\linebreak
+
+\includegraphics[width=3cm]{drapeau_européen.png}\hspace{1cm}
+\includegraphics[width=3cm]{NGI.png}\linebreak
+
+Nous avons ainsi pu financer le développement de Garage pendant 1 an.
+\end{center}
+\end{frame}
+
+\subsection{Licence}
+\begin{frame}
+\begin{center}
+De par nos valeurs, nous avons attribué la licence AGPL version 3 à Garage, notamment afin qu'il reste parmi les biens communs.\linebreak
+
+\vspace{0.5cm}
+\includegraphics[width=5cm]{agpl-v3-logo.png}\linebreak
+\end{center}
+\end{frame}
+
+\subsection{Langage utilisé}
+\begin{frame}
+\begin{center}
+Nous avons décidé d'écrire Garage à l'aide du langage Rust, afin d'obtenir une compilation vers des binaires natifs et efficaces.\linebreak
+
+\includegraphics[width=3.5cm]{rust-logo.png}\linebreak
+
+Ce choix permet également de bénéficier des avantages reconnus de Rust en termes de sécurité.
+\end{center}
+\end{frame}
+
+\subsection{Matériel requis}
+\begin{frame}
+\begin{center}
+Garage peut ainsi être performant sur des machines limitées. Les prérequis sont minimes : n'importe quelle machine avec un processeur qui a moins d'une décennie, 1~gigaoctet de mémoire vive, et 16~gigaoctets de stockage suffit.\linebreak
+
+\vspace{1cm}
+
+Cet aspect est déterminant : il permet en effet d'héberger sur du matériel acheté d'occasion, pour réduire l'impact écologique de nos infrastructures.
+\end{center}
+\end{frame}
+
+\subsection{Performances}
+\begin{frame}
+\begin{center}
+\includegraphics[width=13.25cm]{rpc-amplification.png}
+\end{center}
+\end{frame}
+
+\begin{frame}
+\begin{center}
+\includegraphics[width=11cm]{rpc-complexity.png}
+\end{center}
+\end{frame}
+
+\subsection{Services}
+\begin{frame}
+\begin{center}
+Puisqu'il suit le standard S3, beaucoup de services populaires sont par conséquence compatibles avec Garage :\linebreak
+
+\begin{columns}
+\column{0.2 \linewidth}
+\begin{center}
+\includegraphics[width=2.5cm]{nextcloud-logo.png}
+\end{center}
+\column{0.2 \linewidth}
+\begin{center}
+\includegraphics[width=2.5cm]{peertube-logo.png}
+\end{center}
+\column{0.2 \linewidth}
+\begin{center}
+\includegraphics[width=2.5cm]{matrix-logo.png}
+\end{center}
+\column{0.2 \linewidth}
+\begin{center}
+\includegraphics[width=2.5cm]{mastodon-logo.png}
+\end{center}
+\end{columns}
+~\linebreak
+
+Et comme souvent avec S3, on peut assimiler un bucket à un site, et utiliser le serveur pour héberger des sites web statiques.
+\end{center}
+\end{frame}
+
+\section{Intégration chez Deuxfleurs}
+\subsection{Matériel}
+\begin{frame}
+\begin{center}
+\includegraphics[width=13cm]{neptune.jpg}\linebreak
+
+En pratique, nos serveurs ne sont effectivement que des machines achetées d'occasion (très souvent des anciens ordinateurs destinés à la bureautique en entreprise).
+\end{center}
+\end{frame}
+
+\subsection{Environnement logiciel}
+\begin{frame}
+\begin{center}
+Pour faciliter la reproduction d'un environnement connu, NixOS est installé sur nos machines.\linebreak
+
+\vspace{1cm}
+Pour s’accommoder des réseaux qu'on trouve derrière des routeurs pour particuliers, on s'aide de notre logiciel Diplonat\footnote[frame]{\texttt{https://git.deuxfleurs.fr/Deuxfleurs/diplonat}}.
+\end{center}
+\end{frame}
+
+\section{Au-delà...}
+\subsection{... de Deuxfleurs}
+\begin{frame}
+\begin{center}
+\includegraphics[width=10cm]{tedomum.png}
+\end{center}
+\end{frame}
+
+\subsection{... de Garage}
+\begin{frame}
+\begin{center}
+Nous avons récemment lancé le développement d'Aérogramme\footnote[frame]{\texttt{https://git.deuxfleurs.fr/Deuxfleurs/aerogramme}}.\linebreak
+
+\vspace{1cm}
+Il s'agit d'un serveur de stockage de courriels chiffrés.\linebreak
+
+\vspace{1cm}
+Il est conçu pour pouvoir travailler avec Garage.
+\end{center}
+\end{frame}
+
+\section{Fin}
+\subsection{Contacts}
+\begin{frame}
+\begin{center}
+\begin{tikzpicture}
+\node (ronce) {\includegraphics[width=0.95\textwidth]{ronce.jpg}};
+\node[white] at (-5.1,3.6) {Intéressé(e) ?};
+\node[white, align=center] at (4.2,-2.6) {Contactez-nous !\\\texttt{coucou@deuxfleurs.fr}\\\texttt{\#forum:deuxfleurs.fr}};
+\end{tikzpicture}
+\end{center}
+\end{frame}
+\end{document}
+
@@ -0,0 +1,52 @@
+\begin{tikzpicture}
+\node (carte) {\includegraphics[width=\textwidth]{carte-Europe.pdf}};
+
+% \personnage{position X}{position Y}{facteur d'échelle}
+\newcommand{\personnage}[4]
+{
+\fill[#4] ({#1-(0.4 * #3)},{#2-(0.9 * #3)}) .. controls ({#1-(0.4 * #3)},#2) and ({#1+(0.4 * #3)},#2) .. ({#1+(0.4 * #3)},{#2-(0.9 * #3)}) -- ({#1-(0.4 * #3)},{#2-(0.9 * #3)});
+\fill[#4] (#1,#2) circle ({0.25 * #3});
+}
+
+\onslide<1-11>{\personnage{-2.25}{-0.75}{0.75}{green}}
+\onslide<1-11>{\draw (-1.9,-1.6) rectangle ++(1,1.2);}
+\onslide<2-11>{\draw[fill=green] (-1.8,-1.525) rectangle ++(0.35,0.3) node[pos=0.5] {\tiny 1};}
+\onslide<4-5>{\draw[fill=red] (-1.8,-1.15) rectangle ++(0.35,0.3) node[pos=0.5] {\tiny 3};}
+\onslide<7-11>{\draw[fill=yellow] (-1.8,-1.15) rectangle ++(0.35,0.3) node[pos=0.5] {\tiny 4};}
+\onslide<9-11>{\draw[fill=red] (-1.8,-0.775) rectangle ++(0.35,0.3) node[pos=0.5] {\tiny 6};}
+\onslide<3-11>{\draw[fill=blue] (-1.35,-1.525) rectangle ++(0.35,0.3) node[pos=0.5, white] {\tiny 2};}
+\onslide<8-11>{\draw[fill=blue] (-1.35,-1.15) rectangle ++(0.35,0.3) node[pos=0.5, white] {\tiny 5};}
+\onslide<11-11>{\draw[fill=yellow] (-1.35,-0.775) rectangle ++(0.35,0.3) node[pos=0.5] {\tiny 8};}
+
+\personnage{1.65}{1.5}{0.75}{blue}
+\draw (0.3,0.7) rectangle ++(1,1.2);
+\onslide<2->{\draw[fill=green] (0.4,0.775) rectangle ++(0.35,0.3) node[pos=0.5] {\tiny 1};}
+\onslide<4->{\draw[fill=red] (0.4,1.15) rectangle ++(0.35,0.3) node[pos=0.5] {\tiny 3};}
+\onslide<10->{\draw[fill=green] (0.4,1.525) rectangle ++(0.35,0.3) node[pos=0.5] {\tiny 7};}
+\onslide<3->{\draw[fill=blue] (0.85,0.775) rectangle ++(0.35,0.3) node[pos=0.5, white] {\tiny 2};}
+\onslide<9->{\draw[fill=red] (0.85,1.15) rectangle ++(0.35,0.3) node[pos=0.5] {\tiny 6};}
+\onslide<11->{\draw[fill=yellow] (0.85,1.525) rectangle ++(0.35,0.3) node[pos=0.5] {\tiny 8};}
+
+\personnage{1.85}{-2.3}{0.75}{red}
+\draw (0.5,-3.15) rectangle ++(1,1.2);
+\onslide<2->{\draw[fill=green] (0.6,-3.075) rectangle ++(0.35,0.3) node[pos=0.5] {\tiny 1};}
+\onslide<4-5>{\draw[fill=red] (0.6,-2.7) rectangle ++(0.35,0.3) node[pos=0.5] {\tiny 3};}
+\onslide<7->{\draw[fill=yellow] (0.6,-2.7) rectangle ++(0.35,0.3) node[pos=0.5] {\tiny 4};}
+\onslide<9->{\draw[fill=red] (0.6,-2.325) rectangle ++(0.35,0.3) node[pos=0.5] {\tiny 6};}
+\onslide<3-5>{\draw[fill=blue] (1.05,-3.075) rectangle ++(0.35,0.3) node[pos=0.5, white] {\tiny 2};}
+\onslide<6->{\draw[fill=red] (1.05,-3.075) rectangle ++(0.35,0.3) node[pos=0.5] {\tiny 3};}
+\onslide<8->{\draw[fill=blue] (1.05,-2.7) rectangle ++(0.35,0.3) node[pos=0.5, white] {\tiny 5};}
+\onslide<10->{\draw[fill=green] (1.05,-2.325) rectangle ++(0.35,0.3) node[pos=0.5] {\tiny 7};}
+
+\onslide<5->{\personnage{1.05}{-0.15}{0.75}{yellow}}
+\onslide<5->{\draw (-0.35,-1) rectangle ++(1,1.2);}
+\onslide<6->{\draw[fill=blue] (-0.25,-0.925) rectangle ++(0.35,0.3) node[pos=0.5, white] {\tiny 2};}
+\onslide<7->{\draw[fill=yellow] (-0.25,-0.55) rectangle ++(0.35,0.3) node[pos=0.5] {\tiny 4};}
+\onslide<10->{\draw[fill=green] (-0.25,-0.175) rectangle ++(0.35,0.3) node[pos=0.5] {\tiny 7};}
+\onslide<6->{\draw[fill=red] (0.2,-0.925) rectangle ++(0.35,0.3) node[pos=0.5] {\tiny 3};}
+\onslide<8->{\draw[fill=blue] (0.2,-0.55) rectangle ++(0.35,0.3) node[pos=0.5,white] {\tiny 5};}
+\onslide<11->{\draw[fill=yellow] (0.2,-0.175) rectangle ++(0.35,0.3) node[pos=0.5] {\tiny 8};}
+
+\onslide<12->{\draw[line width=0.25cm] (-2.15,-0.5) -- ++(1,-1);}
+\onslide<12->{\draw[line width=0.25cm] (-2.15,-1.5) -- ++(1,1);}
+\end{tikzpicture}
@@ -0,0 +1,108 @@
+{
+  "nodes": {
+    "cargo2nix": {
+      "inputs": {
+        "flake-compat": "flake-compat",
+        "flake-utils": "flake-utils",
+        "nixpkgs": [
+          "nixpkgs"
+        ],
+        "rust-overlay": "rust-overlay"
+      },
+      "locked": {
+        "lastModified": 1673262828,
+        "narHash": "sha256-pDqno5/2ghQDt4LjVt5ZUMV9pTSA5rGGdz6Skf2rBwc=",
+        "owner": "Alexis211",
+        "repo": "cargo2nix",
+        "rev": "505caa32110d42ee03bd68b47031142eff9c827b",
+        "type": "github"
+      },
+      "original": {
+        "owner": "Alexis211",
+        "repo": "cargo2nix",
+        "rev": "505caa32110d42ee03bd68b47031142eff9c827b",
+        "type": "github"
+      }
+    },
+    "flake-compat": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1650374568,
+        "narHash": "sha256-Z+s0J8/r907g149rllvwhb4pKi8Wam5ij0st8PwAh+E=",
+        "owner": "edolstra",
+        "repo": "flake-compat",
+        "rev": "b4a34015c698c7793d592d66adbab377907a2be8",
+        "type": "github"
+      },
+      "original": {
+        "owner": "edolstra",
+        "repo": "flake-compat",
+        "type": "github"
+      }
+    },
+    "flake-utils": {
+      "locked": {
+        "lastModified": 1659877975,
+        "narHash": "sha256-zllb8aq3YO3h8B/U0/J1WBgAL8EX5yWf5pMj3G0NAmc=",
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "rev": "c0e246b9b83f637f4681389ecabcb2681b4f3af0",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "type": "github"
+      }
+    },
+    "nixpkgs": {
+      "locked": {
+        "lastModified": 1673261889,
+        "narHash": "sha256-7trMsi0z7EfYNC/Nc5EtulvChBHQAo376XRICyWr89Q=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "baed728abe983508cabc99d05cccc164fe748744",
+        "type": "github"
+      },
+      "original": {
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "baed728abe983508cabc99d05cccc164fe748744",
+        "type": "github"
+      }
+    },
+    "root": {
+      "inputs": {
+        "cargo2nix": "cargo2nix",
+        "nixpkgs": "nixpkgs"
+      }
+    },
+    "rust-overlay": {
+      "inputs": {
+        "flake-utils": [
+          "cargo2nix",
+          "flake-utils"
+        ],
+        "nixpkgs": [
+          "cargo2nix",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1664247556,
+        "narHash": "sha256-J4vazHU3609ekn7dr+3wfqPo5WGlZVAgV7jfux352L0=",
+        "owner": "oxalica",
+        "repo": "rust-overlay",
+        "rev": "524db9c9ea7bc7743bb74cdd45b6d46ea3fcc2ab",
+        "type": "github"
+      },
+      "original": {
+        "owner": "oxalica",
+        "repo": "rust-overlay",
+        "type": "github"
+      }
+    }
+  },
+  "root": "root",
+  "version": 7
+}
@@ -0,0 +1,28 @@
+{
+  description = "Garage, an S3-compatible distributed object store for self-hosted deployments";
+
+  inputs.nixpkgs.url = "github:NixOS/nixpkgs/baed728abe983508cabc99d05cccc164fe748744";
+  inputs.cargo2nix = {
+    # As of 2022-10-18: two small patches over unstable branch, one for clippy and one to fix feature detection
+    url = "github:Alexis211/cargo2nix/505caa32110d42ee03bd68b47031142eff9c827b";
+    inputs.nixpkgs.follows = "nixpkgs";
+  };
+
+  outputs = { self, nixpkgs, cargo2nix }: let
+    git_version = self.lastModifiedDate;
+    compile = import ./nix/compile.nix;
+    forAllSystems = nixpkgs.lib.genAttrs nixpkgs.lib.systems.flakeExposed;
+  in
+  {
+    packages = forAllSystems (system: {
+      default = (compile {
+        inherit system git_version;
+        pkgsSrc = nixpkgs;
+        cargo2nixOverlay = cargo2nix.overlays.default;
+        release = true;
+      }).workspace.garage {
+        compileMode = "build";
+      };
+    });
+  };
+}
@@ -3,15 +3,15 @@ rec {
   * Fixed dependencies
   */
  pkgsSrc = fetchTarball {
-    # As of 2022-10-13
-    url = "https://github.com/NixOS/nixpkgs/archive/a3073c49bc0163fea6a121c276f526837672b555.zip";
-    sha256 = "1bz632psfbpmicyzjb8b4265y50shylccvfm6ry6mgnv5hvz324s";
+    # As of 2023-01-09
+    url = "https://github.com/NixOS/nixpkgs/archive/baed728abe983508cabc99d05cccc164fe748744.zip";
+    sha256 = "1m7kmcjhnj3lx7xqs0nh262c4nxs5n8p7k9g6kc4gv1k5nrcrnpf";
  };
  cargo2nixSrc = fetchGit {
    # As of 2022-10-18: two small patches over unstable branch, one for clippy and one to fix feature detection
    url = "https://github.com/Alexis211/cargo2nix";
    ref = "custom_unstable";
-    rev = "a7a61179b66054904ef6a195d8da736eaaa06c36";
+    rev = "505caa32110d42ee03bd68b47031142eff9c827b";
  };

  /*
@@ -1,25 +1,32 @@
 {
-  system ? builtins.currentSystem,
-  target,
+  system,
+  target ? null,
+  pkgsSrc,
+  cargo2nixOverlay,
  compiler ? "rustc",
  release ? false,
  git_version ? null,
  features ? null,
 }:

-with import ./common.nix;
-
 let
  log = v: builtins.trace v v;

-  pkgs = import pkgsSrc {
-    inherit system;
-    crossSystem = {
-      config = target;
-      isStatic = true;
-    };
-    overlays = [ cargo2nixOverlay ];
-  };
+  pkgs =
+    if target != null then
+      import pkgsSrc {
+        inherit system;
+        crossSystem = {
+          config = target;
+          isStatic = true;
+        };
+        overlays = [ cargo2nixOverlay ];
+      }
+    else
+      import pkgsSrc {
+        inherit system;
+        overlays = [ cargo2nixOverlay ];
+      };

  /*
   Cargo2nix is built for rustOverlay which installs Rust from Mozilla releases.
@@ -34,8 +41,8 @@ let
   NixOS ships them in separate ones. We reunite them with symlinkJoin.
  */
  toolchainOptions =
-    if target == "x86_64-unknown-linux-musl" || target == "aarch64-unknown-linux-musl" then {
-      rustVersion = "1.63.0";
+    if target == null || target == "x86_64-unknown-linux-musl" || target == "aarch64-unknown-linux-musl" then {
+      rustVersion = "1.65.0";
      extraRustComponents = [ "clippy" ];
    } else {
      rustToolchain = pkgs.symlinkJoin {
@@ -7,7 +7,7 @@ pkgs.buildGoModule rec {
    owner = "GoogleContainerTools";
    repo = "kaniko";
    rev = "v${version}";
-    sha256 = "1fnclr556avxay6pvgw5ya3xbxfnf2gv4njq2hr4fd6fcjyslq5h";
+    sha256 = "TXgzO/NfLXVo5a7yyO3XYSk+9H1CwMF+vwbRx3kchQ8=";
  };

  vendorSha256 = null;
@@ -15,7 +15,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.1.0
+version: 0.2.0

 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to
@@ -18,6 +18,9 @@ metadata:
  name: {{ $fullName }}-s3-api
  labels:
    {{- include "garage.labels" . | nindent 4 }}
+    {{- with .Values.ingress.s3.api.labels }}
+    {{- toYaml . | nindent 4 }}
+    {{- end }}
  {{- with .Values.ingress.s3.api.annotations }}
  annotations:
    {{- toYaml . | nindent 4 }}
@@ -80,6 +83,9 @@ metadata:
  name: {{ $fullName }}-s3-web
  labels:
    {{- include "garage.labels" . | nindent 4 }}
+    {{- with .Values.ingress.s3.web.labels }}
+    {{- toYaml . | nindent 4 }}
+    {{- end }}
  {{- with .Values.ingress.s3.web.annotations }}
  annotations:
    {{- toYaml . | nindent 4 }}
@@ -85,14 +85,15 @@ service:
 ingress:
  s3:
    api:
-      enabled: true
+      enabled: false
      # Rely either on the className or the annotation below but not both
      # replace "nginx" by an Ingress controller
      # you can find examples here https://kubernetes.io/docs/concepts/services-networking/ingress-controllers
-      className: "nginx"
-      annotations:
+      # className: "nginx"
+      annotations: {}
        # kubernetes.io/ingress.class: "nginx"
        # kubernetes.io/tls-acme: "true"
+      labels: {}
      hosts:
        - host: "s3.garage.tld" # garage S3 API endpoint
          paths:
@@ -107,11 +108,15 @@ ingress:
      #    hosts:
      #      - kubernetes.docker.internal
    web:
-      enabled: true
-      className: "nginx"
+      enabled: false
+      # Rely either on the className or the annotation below but not both
+      # replace "nginx" by an Ingress controller
+      # you can find examples here https://kubernetes.io/docs/concepts/services-networking/ingress-controllers
+      # className: "nginx"
      annotations: {}
        # kubernetes.io/ingress.class: nginx
        # kubernetes.io/tls-acme: "true"
+      labels: {}
      hosts:
       - host: "*.web.garage.tld" # wildcard website access with bucket name prefix
         paths:
@@ -1,5 +1,5 @@
 {
-  system ? builtins.currentSystem,
+    system ? builtins.currentSystem,
 }:

 with import ./nix/common.nix;
@@ -71,13 +71,25 @@ function refresh_cache {
  for attr in clippy.amd64 test.amd64 pkgs.{amd64,i386,arm,arm64}.{debug,release}; do
    echo "Updating cache for ''${attr}"
    derivation=$(nix-instantiate --attr ''${attr})
-    nix copy \
+    nix copy -j8 \
      --to 's3://nix?endpoint=garage.deuxfleurs.fr&region=garage&secret-key=/tmp/nix-signing-key.sec' \
      $(nix-store -qR ''${derivation%\!bin})
  done
  rm /tmp/nix-signing-key.sec
 }

+function refresh_flake_cache {
+  pass show deuxfleurs/nix_priv_key > /tmp/nix-signing-key.sec
+  for attr in packages.x86_64-linux.default; do
+    echo "Updating cache for ''${attr}"
+    derivation=$(nix path-info --derivation ".#''${attr}")
+    nix copy -j8 \
+      --to 's3://nix?endpoint=garage.deuxfleurs.fr&region=garage&secret-key=/tmp/nix-signing-key.sec' \
+      $(nix-store -qR ''${derivation})
+  done
+  rm /tmp/nix-signing-key.sec
+}
+
 function to_s3 {
  aws \
      --endpoint-url https://garage.deuxfleurs.fr \
@@ -1,6 +1,6 @@
 [package]
 name = "garage_api"
-version = "0.8.0"
+version = "0.8.1"
 authors = ["Alex Auvolat <alex@adnab.me>"]
 edition = "2018"
 license = "AGPL-3.0"
@@ -14,11 +14,11 @@ path = "lib.rs"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html

 [dependencies]
-garage_model = { version = "0.8.0", path = "../model" }
-garage_table = { version = "0.8.0", path = "../table" }
-garage_block = { version = "0.8.0", path = "../block" }
-garage_util = { version = "0.8.0", path = "../util" }
-garage_rpc = { version = "0.8.0", path = "../rpc" }
+garage_model = { version = "0.8.1", path = "../model" }
+garage_table = { version = "0.8.1", path = "../table" }
+garage_block = { version = "0.8.1", path = "../block" }
+garage_util = { version = "0.8.1", path = "../util" }
+garage_rpc = { version = "0.8.1", path = "../rpc" }

 async-trait = "0.1.7"
 base64 = "0.13"
@@ -15,6 +15,7 @@ use opentelemetry_prometheus::PrometheusExporter;
 use prometheus::{Encoder, TextEncoder};

 use garage_model::garage::Garage;
+use garage_rpc::system::ClusterHealthStatus;
 use garage_util::error::Error as GarageError;

 use crate::generic_server::*;
@@ -76,6 +77,31 @@ impl AdminApiServer {
 			.body(Body::empty())?)
 	}

+	fn handle_health(&self) -> Result<Response<Body>, Error> {
+		let health = self.garage.system.health();
+
+		let (status, status_str) = match health.status {
+			ClusterHealthStatus::Healthy => (StatusCode::OK, "Garage is fully operational"),
+			ClusterHealthStatus::Degraded => (
+				StatusCode::OK,
+				"Garage is operational but some storage nodes are unavailable",
+			),
+			ClusterHealthStatus::Unavailable => (
+				StatusCode::SERVICE_UNAVAILABLE,
+				"Quorum is not available for some/all partitions, reads and writes will fail",
+			),
+		};
+		let status_str = format!(
+			"{}\nConsult the full health check API endpoint at /v0/health for more details\n",
+			status_str
+		);
+
+		Ok(Response::builder()
+			.status(status)
+			.header(http::header::CONTENT_TYPE, "text/plain")
+			.body(Body::from(status_str))?)
+	}
+
 	fn handle_metrics(&self) -> Result<Response<Body>, Error> {
 		#[cfg(feature = "metrics")]
 		{
@@ -124,6 +150,7 @@ impl ApiHandler for AdminApiServer {
 	) -> Result<Response<Body>, Error> {
 		let expected_auth_header =
 			match endpoint.authorization_type() {
+				Authorization::None => None,
 				Authorization::MetricsToken => self.metrics_token.as_ref(),
 				Authorization::AdminToken => match &self.admin_token {
 					None => return Err(Error::forbidden(
@@ -147,8 +174,10 @@ impl ApiHandler for AdminApiServer {

 		match endpoint {
 			Endpoint::Options => self.handle_options(&req),
+			Endpoint::Health => self.handle_health(),
 			Endpoint::Metrics => self.handle_metrics(),
 			Endpoint::GetClusterStatus => handle_get_cluster_status(&self.garage).await,
+			Endpoint::GetClusterHealth => handle_get_cluster_health(&self.garage).await,
 			Endpoint::ConnectClusterNodes => handle_connect_cluster_nodes(&self.garage, req).await,
 			// Layout
 			Endpoint::GetClusterLayout => handle_get_cluster_layout(&self.garage).await,
@@ -43,6 +43,11 @@ pub async fn handle_get_cluster_status(garage: &Arc<Garage>) -> Result<Response<
 	Ok(json_ok_response(&res)?)
 }

+pub async fn handle_get_cluster_health(garage: &Arc<Garage>) -> Result<Response<Body>, Error> {
+	let health = garage.system.health();
+	Ok(json_ok_response(&health)?)
+}
+
 pub async fn handle_connect_cluster_nodes(
 	garage: &Arc<Garage>,
 	req: Request<Body>,
@@ -6,6 +6,7 @@ use crate::admin::error::*;
 use crate::router_macros::*;

 pub enum Authorization {
+	None,
 	MetricsToken,
 	AdminToken,
 }
@@ -16,8 +17,10 @@ router_match! {@func
 #[derive(Debug, Clone, PartialEq, Eq)]
 pub enum Endpoint {
 	Options,
+	Health,
 	Metrics,
 	GetClusterStatus,
+	GetClusterHealth,
 	ConnectClusterNodes,
 	// Layout
 	GetClusterLayout,
@@ -88,8 +91,10 @@ impl Endpoint {

 		let res = router_match!(@gen_path_parser (req.method(), path, query) [
 			OPTIONS _ => Options,
+			GET "/health" => Health,
 			GET "/metrics" => Metrics,
 			GET "/v0/status" => GetClusterStatus,
+			GET "/v0/health" => GetClusterHealth,
 			POST "/v0/connect" => ConnectClusterNodes,
 			// Layout endpoints
 			GET "/v0/layout" => GetClusterLayout,
@@ -130,6 +135,7 @@ impl Endpoint {
 	/// Get the kind of authorization which is required to perform the operation.
 	pub fn authorization_type(&self) -> Authorization {
 		match self {
+			Self::Health => Authorization::None,
 			Self::Metrics => Authorization::MetricsToken,
 			_ => Authorization::AdminToken,
 		}
@@ -137,9 +143,13 @@ impl Endpoint {
 }

 generateQueryParameters! {
-	"id" => id,
-	"search" => search,
-	"globalAlias" => global_alias,
-	"alias" => alias,
-	"accessKeyId" => access_key_id
+	keywords: [],
+	fields: [
+		"format" => format,
+		"id" => id,
+		"search" => search,
+		"globalAlias" => global_alias,
+		"alias" => alias,
+		"accessKeyId" => access_key_id
+	]
 }
@@ -211,7 +211,7 @@ pub async fn handle_poll_item(
 	let item = garage
 		.k2v
 		.rpc
-		.poll(
+		.poll_item(
 			bucket_id,
 			partition_key,
 			sort_key,
@@ -96,7 +96,7 @@ impl Endpoint {
 	fn from_get(partition_key: String, query: &mut QueryParameters<'_>) -> Result<Self, Error> {
 		router_match! {
 			@gen_parser
-			(query.keyword.take().unwrap_or_default().as_ref(), partition_key, query, None),
+			(query.keyword.take().unwrap_or_default(), partition_key, query, None),
 			key: [
 				EMPTY if causality_token => PollItem (query::sort_key, query::causality_token, opt_parse::timeout),
 				EMPTY => ReadItem (query::sort_key),
@@ -111,7 +111,7 @@ impl Endpoint {
 	fn from_search(partition_key: String, query: &mut QueryParameters<'_>) -> Result<Self, Error> {
 		router_match! {
 			@gen_parser
-			(query.keyword.take().unwrap_or_default().as_ref(), partition_key, query, None),
+			(query.keyword.take().unwrap_or_default(), partition_key, query, None),
 			key: [
 			],
 			no_key: [
@@ -125,7 +125,7 @@ impl Endpoint {
 	fn from_head(partition_key: String, query: &mut QueryParameters<'_>) -> Result<Self, Error> {
 		router_match! {
 			@gen_parser
-			(query.keyword.take().unwrap_or_default().as_ref(), partition_key, query, None),
+			(query.keyword.take().unwrap_or_default(), partition_key, query, None),
 			key: [
 				EMPTY => HeadObject(opt_parse::part_number, query_opt::version_id),
 			],
@@ -140,7 +140,7 @@ impl Endpoint {
 	fn from_post(partition_key: String, query: &mut QueryParameters<'_>) -> Result<Self, Error> {
 		router_match! {
 			@gen_parser
-			(query.keyword.take().unwrap_or_default().as_ref(), partition_key, query, None),
+			(query.keyword.take().unwrap_or_default(), partition_key, query, None),
 			key: [
 			],
 			no_key: [
@@ -155,7 +155,7 @@ impl Endpoint {
 	fn from_put(partition_key: String, query: &mut QueryParameters<'_>) -> Result<Self, Error> {
 		router_match! {
 			@gen_parser
-			(query.keyword.take().unwrap_or_default().as_ref(), partition_key, query, None),
+			(query.keyword.take().unwrap_or_default(), partition_key, query, None),
 			key: [
 				EMPTY => InsertItem (query::sort_key),

@@ -169,7 +169,7 @@ impl Endpoint {
 	fn from_delete(partition_key: String, query: &mut QueryParameters<'_>) -> Result<Self, Error> {
 		router_match! {
 			@gen_parser
-			(query.keyword.take().unwrap_or_default().as_ref(), partition_key, query, None),
+			(query.keyword.take().unwrap_or_default(), partition_key, query, None),
 			key: [
 				EMPTY => DeleteItem (query::sort_key),
 			],
@@ -232,21 +232,18 @@ impl Endpoint {

 // parameter name => struct field
 generateQueryParameters! {
-	"prefix" => prefix,
-	"start" => start,
-	"causality_token" => causality_token,
-	"end" => end,
-	"limit" => limit,
-	"reverse" => reverse,
-	"sort_key" => sort_key,
-	"timeout" => timeout
-}
-
-mod keywords {
-	//! This module contain all query parameters with no associated value
-	//! used to differentiate endpoints.
-	pub const EMPTY: &str = "";
-
-	pub const DELETE: &str = "delete";
-	pub const SEARCH: &str = "search";
+	keywords: [
+		"delete" => DELETE,
+		"search" => SEARCH
+	],
+	fields: [
+		"prefix" => prefix,
+		"start" => start,
+		"causality_token" => causality_token,
+		"end" => end,
+		"limit" => limit,
+		"reverse" => reverse,
+		"sort_key" => sort_key,
+		"timeout" => timeout
+	]
 }
@@ -4,10 +4,9 @@ macro_rules! router_match {
    (@match $enum:expr , [ $($endpoint:ident,)* ]) => {{
        // usage: router_match {@match my_enum, [ VariantWithField1, VariantWithField2 ..] }
        // returns true if the variant was one of the listed variants, false otherwise.
-        use Endpoint::*;
        match $enum {
            $(
-            $endpoint { .. } => true,
+            Endpoint::$endpoint { .. } => true,
            )*
            _ => false
        }
@@ -15,37 +14,35 @@ macro_rules! router_match {
    (@extract $enum:expr , $param:ident, [ $($endpoint:ident,)* ]) => {{
        // usage: router_match {@extract my_enum, field_name, [ VariantWithField1, VariantWithField2 ..] }
        // returns Some(field_value), or None if the variant was not one of the listed variants.
-        use Endpoint::*;
        match $enum {
            $(
-            $endpoint {$param, ..} => Some($param),
+            Endpoint::$endpoint {$param, ..} => Some($param),
            )*
            _ => None
        }
    }};
-	(@gen_path_parser ($method:expr, $reqpath:expr, $query:expr)
-	 [
-	 $($meth:ident $path:pat $(if $required:ident)? => $api:ident $(($($conv:ident :: $param:ident),*))?,)*
-	 ]) => {{
-		{
-			use Endpoint::*;
-			match ($method, $reqpath) {
-				$(
-					(&Method::$meth, $path) if true $(&& $query.$required.is_some())? => $api {
-						$($(
-							$param: router_match!(@@parse_param $query, $conv, $param),
-						)*)?
-					},
-				)*
-				(m, p) => {
-					return Err(Error::bad_request(format!(
-						"Unknown API endpoint: {} {}",
-						m, p
-					)))
-				}
-			}
-		}
-	}};
+    (@gen_path_parser ($method:expr, $reqpath:expr, $query:expr)
+     [
+     $($meth:ident $path:pat $(if $required:ident)? => $api:ident $(($($conv:ident :: $param:ident),*))?,)*
+     ]) => {{
+        {
+            match ($method, $reqpath) {
+                $(
+                    (&Method::$meth, $path) if true $(&& $query.$required.is_some())? => Endpoint::$api {
+                        $($(
+                            $param: router_match!(@@parse_param $query, $conv, $param),
+                        )*)?
+                    },
+                )*
+                (m, p) => {
+                    return Err(Error::bad_request(format!(
+                        "Unknown API endpoint: {} {}",
+                        m, p
+                    )))
+                }
+            }
+        }
+    }};
    (@gen_parser ($keyword:expr, $key:ident, $query:expr, $header:expr),
        key: [$($kw_k:ident $(if $required_k:ident)? $(header $header_k:expr)? => $api_k:ident $(($($conv_k:ident :: $param_k:ident),*))?,)*],
        no_key: [$($kw_nk:ident $(if $required_nk:ident)? $(if_header $header_nk:expr)? => $api_nk:ident $(($($conv_nk:ident :: $param_nk:ident),*))?,)*]) => {{
@@ -60,11 +57,9 @@ macro_rules! router_match {
        //   ]
        // }
        // See in from_{method} for more detailed usage.
-        use Endpoint::*;
-        use keywords::*;
        match ($keyword, !$key.is_empty()){
            $(
-            ($kw_k, true) if true $(&& $query.$required_k.is_some())? $(&& $header.contains_key($header_k))? => Ok($api_k {
+            (Keyword::$kw_k, true) if true $(&& $query.$required_k.is_some())? $(&& $header.contains_key($header_k))? => Ok(Endpoint::$api_k {
                $key,
                $($(
                    $param_k: router_match!(@@parse_param $query, $conv_k, $param_k),
@@ -72,7 +67,7 @@ macro_rules! router_match {
            }),
            )*
            $(
-            ($kw_nk, false) $(if $query.$required_nk.is_some())? $(if $header.contains($header_nk))? => Ok($api_nk {
+            (Keyword::$kw_nk, false) $(if $query.$required_nk.is_some())? $(if $header.contains($header_nk))? => Ok(Endpoint::$api_nk {
                $($(
                    $param_nk: router_match!(@@parse_param $query, $conv_nk, $param_nk),
                )*)?
@@ -84,7 +79,7 @@ macro_rules! router_match {

    (@@parse_param $query:expr, query_opt, $param:ident) => {{
        // extract optional query parameter
-		$query.$param.take().map(|param| param.into_owned())
+        $query.$param.take().map(|param| param.into_owned())
    }};
    (@@parse_param $query:expr, query, $param:ident) => {{
        // extract mendatory query parameter
@@ -93,7 +88,7 @@ macro_rules! router_match {
    (@@parse_param $query:expr, opt_parse, $param:ident) => {{
        // extract and parse optional query parameter
        // missing parameter is file, however parse error is reported as an error
-		$query.$param
+        $query.$param
            .take()
            .map(|param| param.parse())
            .transpose()
@@ -144,14 +139,40 @@ macro_rules! router_match {
 /// This macro is used to generate part of the code in this module. It must be called only one, and
 /// is useless outside of this module.
 macro_rules! generateQueryParameters {
-    ( $($rest:expr => $name:ident),* ) => {
+    (
+        keywords: [ $($kw_param:expr => $kw_name: ident),* ],
+        fields: [ $($f_param:expr => $f_name:ident),* ]
+    ) => {
+        #[derive(Debug)]
+        #[allow(non_camel_case_types)]
+        #[allow(clippy::upper_case_acronyms)]
+        enum Keyword {
+            EMPTY,
+            $( $kw_name, )*
+        }
+
+        impl std::fmt::Display for Keyword {
+            fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
+                match self {
+                    Keyword::EMPTY => write!(f, "``"),
+                    $( Keyword::$kw_name => write!(f, "`{}`", $kw_param), )*
+                }
+            }
+        }
+
+        impl Default for Keyword {
+            fn default() -> Self {
+                Keyword::EMPTY
+            }
+        }
+
        /// Struct containing all query parameters used in endpoints. Think of it as an HashMap,
        /// but with keys statically known.
        #[derive(Debug, Default)]
        struct QueryParameters<'a> {
-            keyword: Option<Cow<'a, str>>,
+            keyword: Option<Keyword>,
            $(
-            $name: Option<Cow<'a, str>>,
+            $f_name: Option<Cow<'a, str>>,
            )*
        }

@@ -160,34 +181,29 @@ macro_rules! generateQueryParameters {
            fn from_query(query: &'a str) -> Result<Self, Error> {
                let mut res: Self = Default::default();
                for (k, v) in url::form_urlencoded::parse(query.as_bytes()) {
-                    let repeated = match k.as_ref() {
+                    match k.as_ref() {
                        $(
-                            $rest => if !v.is_empty() {
-                                res.$name.replace(v).is_some()
-                            } else {
-                                false
+                            $kw_param => if let Some(prev_kw) = res.keyword.replace(Keyword::$kw_name) {
+                                return Err(Error::bad_request(format!(
+                                    "Multiple keywords: '{}' and '{}'", prev_kw, $kw_param
+                                )));
+                            },
+                        )*
+                        $(
+                            $f_param => if !v.is_empty() {
+                                if res.$f_name.replace(v).is_some() {
+                                    return Err(Error::bad_request(format!(
+                                        "Query parameter repeated: '{}'", k
+                                    )));
+                                }
                            },
                        )*
                        _ => {
-                            if k.starts_with("response-") || k.starts_with("X-Amz-") {
-                                false
-                            } else if v.as_ref().is_empty() {
-                                if res.keyword.replace(k).is_some() {
-                                    return Err(Error::bad_request("Multiple keywords"));
-                                }
-                                continue;
-                            } else {
+                            if !(k.starts_with("response-") || k.starts_with("X-Amz-")) {
                                debug!("Received an unknown query parameter: '{}'", k);
-                                false
                            }
                        }
                    };
-                    if repeated {
-                        return Err(Error::bad_request(format!(
-                            "Query parameter repeated: '{}'",
-                            k
-                        )));
-                    }
                }
                Ok(res)
            }
@@ -198,8 +214,8 @@ macro_rules! generateQueryParameters {
                if self.keyword.is_some() {
                    Some("Keyword not used")
                } $(
-                    else if self.$name.is_some() {
-                        Some(concat!("'", $rest, "'"))
+                    else if self.$f_name.is_some() {
+                        Some(concat!("'", $f_param, "'"))
                    }
                )* else {
                    None
@@ -161,6 +161,15 @@ pub async fn handle_create_bucket(
 			return Err(CommonError::BucketAlreadyExists.into());
 		}
 	} else {
+		// Check user is allowed to create bucket
+		if !key_params.allow_create_bucket.get() {
+			return Err(CommonError::Forbidden(format!(
+				"Access key {} is not allowed to create buckets",
+				api_key.key_id
+			))
+			.into());
+		}
+
 		// Create the bucket!
 		if !is_valid_bucket_name(&bucket_name) {
 			return Err(Error::bad_request(format!(
@@ -119,6 +119,17 @@ pub(crate) async fn save_stream<S: Stream<Item = Result<Bytes, Error>> + Unpin>(
 		return Ok((version_uuid, data_md5sum_hex));
 	}

+	// The following consists in many steps that can each fail.
+	// Keep track that some cleanup will be needed if things fail
+	// before everything is finished (cleanup is done using the Drop trait).
+	let mut interrupted_cleanup = InterruptedCleanup(Some((
+		garage.clone(),
+		bucket.id,
+		key.into(),
+		version_uuid,
+		version_timestamp,
+	)));
+
 	// Write version identifier in object table so that we have a trace
 	// that we are uploading something
 	let mut object_version = ObjectVersion {
@@ -139,44 +150,27 @@ pub(crate) async fn save_stream<S: Stream<Item = Result<Bytes, Error>> + Unpin>(
 	// Transfer data and verify checksum
 	let first_block_hash = async_blake2sum(first_block.clone()).await;

-	let tx_result = (|| async {
-		let (total_size, data_md5sum, data_sha256sum) = read_and_put_blocks(
-			&garage,
-			&version,
-			1,
-			first_block,
-			first_block_hash,
-			&mut chunker,
-		)
-		.await?;
+	let (total_size, data_md5sum, data_sha256sum) = read_and_put_blocks(
+		&garage,
+		&version,
+		1,
+		first_block,
+		first_block_hash,
+		&mut chunker,
+	)
+	.await?;

-		ensure_checksum_matches(
-			data_md5sum.as_slice(),
-			data_sha256sum,
-			content_md5.as_deref(),
-			content_sha256,
-		)?;
+	ensure_checksum_matches(
+		data_md5sum.as_slice(),
+		data_sha256sum,
+		content_md5.as_deref(),
+		content_sha256,
+	)?;

-		check_quotas(&garage, bucket, key, total_size).await?;
-
-		Ok((total_size, data_md5sum))
-	})()
-	.await;
-
-	// If something went wrong, clean up
-	let (total_size, md5sum_arr) = match tx_result {
-		Ok(rv) => rv,
-		Err(e) => {
-			// Mark object as aborted, this will free the blocks further down
-			object_version.state = ObjectVersionState::Aborted;
-			let object = Object::new(bucket.id, key.into(), vec![object_version.clone()]);
-			garage.object_table.insert(&object).await?;
-			return Err(e);
-		}
-	};
+	check_quotas(&garage, bucket, key, total_size).await?;

 	// Save final object state, marked as Complete
-	let md5sum_hex = hex::encode(md5sum_arr);
+	let md5sum_hex = hex::encode(data_md5sum);
 	object_version.state = ObjectVersionState::Complete(ObjectVersionData::FirstBlock(
 		ObjectVersionMeta {
 			headers,
@@ -188,6 +182,10 @@ pub(crate) async fn save_stream<S: Stream<Item = Result<Bytes, Error>> + Unpin>(
 	let object = Object::new(bucket.id, key.into(), vec![object_version]);
 	garage.object_table.insert(&object).await?;

+	// We were not interrupted, everything went fine.
+	// We won't have to clean up on drop.
+	interrupted_cleanup.cancel();
+
 	Ok((version_uuid, md5sum_hex))
 }

@@ -426,6 +424,33 @@ pub fn put_response(version_uuid: Uuid, md5sum_hex: String) -> Response<Body> {
 		.unwrap()
 }

+struct InterruptedCleanup(Option<(Arc<Garage>, Uuid, String, Uuid, u64)>);
+
+impl InterruptedCleanup {
+	fn cancel(&mut self) {
+		drop(self.0.take());
+	}
+}
+impl Drop for InterruptedCleanup {
+	fn drop(&mut self) {
+		if let Some((garage, bucket_id, key, version_uuid, version_ts)) = self.0.take() {
+			tokio::spawn(async move {
+				let object_version = ObjectVersion {
+					uuid: version_uuid,
+					timestamp: version_ts,
+					state: ObjectVersionState::Aborted,
+				};
+				let object = Object::new(bucket_id, key, vec![object_version]);
+				if let Err(e) = garage.object_table.insert(&object).await {
+					warn!("Cannot cleanup after aborted PutObject: {}", e);
+				}
+			});
+		}
+	}
+}
+
+// ----
+
 pub async fn handle_create_multipart_upload(
 	garage: Arc<Garage>,
 	req: &Request<Body>,
@@ -355,7 +355,7 @@ impl Endpoint {
 	fn from_get(key: String, query: &mut QueryParameters<'_>) -> Result<Self, Error> {
 		router_match! {
 			@gen_parser
-			(query.keyword.take().unwrap_or_default().as_ref(), key, query, None),
+			(query.keyword.take().unwrap_or_default(), key, query, None),
 			key: [
 				EMPTY if upload_id => ListParts (query::upload_id, opt_parse::max_parts, opt_parse::part_number_marker),
 				EMPTY => GetObject (query_opt::version_id, opt_parse::part_number),
@@ -412,7 +412,7 @@ impl Endpoint {
 	fn from_head(key: String, query: &mut QueryParameters<'_>) -> Result<Self, Error> {
 		router_match! {
 			@gen_parser
-			(query.keyword.take().unwrap_or_default().as_ref(), key, query, None),
+			(query.keyword.take().unwrap_or_default(), key, query, None),
 			key: [
 				EMPTY => HeadObject(opt_parse::part_number, query_opt::version_id),
 			],
@@ -426,7 +426,7 @@ impl Endpoint {
 	fn from_post(key: String, query: &mut QueryParameters<'_>) -> Result<Self, Error> {
 		router_match! {
 			@gen_parser
-			(query.keyword.take().unwrap_or_default().as_ref(), key, query, None),
+			(query.keyword.take().unwrap_or_default(), key, query, None),
 			key: [
 				EMPTY if upload_id  => CompleteMultipartUpload (query::upload_id),
 				RESTORE => RestoreObject (query_opt::version_id),
@@ -448,7 +448,7 @@ impl Endpoint {
 	) -> Result<Self, Error> {
 		router_match! {
 			@gen_parser
-			(query.keyword.take().unwrap_or_default().as_ref(), key, query, headers),
+			(query.keyword.take().unwrap_or_default(), key, query, headers),
 			key: [
 				EMPTY if part_number header "x-amz-copy-source" => UploadPartCopy (parse::part_number, query::upload_id),
 				EMPTY header "x-amz-copy-source" => CopyObject,
@@ -490,7 +490,7 @@ impl Endpoint {
 	fn from_delete(key: String, query: &mut QueryParameters<'_>) -> Result<Self, Error> {
 		router_match! {
 			@gen_parser
-			(query.keyword.take().unwrap_or_default().as_ref(), key, query, None),
+			(query.keyword.take().unwrap_or_default(), key, query, None),
 			key: [
 				EMPTY if upload_id => AbortMultipartUpload (query::upload_id),
 				EMPTY => DeleteObject (query_opt::version_id),
@@ -624,63 +624,60 @@ impl Endpoint {

 // parameter name => struct field
 generateQueryParameters! {
-	"continuation-token" => continuation_token,
-	"delimiter" => delimiter,
-	"encoding-type" => encoding_type,
-	"fetch-owner" => fetch_owner,
-	"id" => id,
-	"key-marker" => key_marker,
-	"list-type" => list_type,
-	"marker" => marker,
-	"max-keys" => max_keys,
-	"max-parts" => max_parts,
-	"max-uploads" => max_uploads,
-	"partNumber" => part_number,
-	"part-number-marker" => part_number_marker,
-	"prefix" => prefix,
-	"select-type" => select_type,
-	"start-after" => start_after,
-	"uploadId" => upload_id,
-	"upload-id-marker" => upload_id_marker,
-	"versionId" => version_id,
-	"version-id-marker" => version_id_marker
-}
-
-mod keywords {
-	//! This module contain all query parameters with no associated value S3 uses to differentiate
-	//! endpoints.
-	pub const EMPTY: &str = "";
-
-	pub const ACCELERATE: &str = "accelerate";
-	pub const ACL: &str = "acl";
-	pub const ANALYTICS: &str = "analytics";
-	pub const CORS: &str = "cors";
-	pub const DELETE: &str = "delete";
-	pub const ENCRYPTION: &str = "encryption";
-	pub const INTELLIGENT_TIERING: &str = "intelligent-tiering";
-	pub const INVENTORY: &str = "inventory";
-	pub const LEGAL_HOLD: &str = "legal-hold";
-	pub const LIFECYCLE: &str = "lifecycle";
-	pub const LOCATION: &str = "location";
-	pub const LOGGING: &str = "logging";
-	pub const METRICS: &str = "metrics";
-	pub const NOTIFICATION: &str = "notification";
-	pub const OBJECT_LOCK: &str = "object-lock";
-	pub const OWNERSHIP_CONTROLS: &str = "ownershipControls";
-	pub const POLICY: &str = "policy";
-	pub const POLICY_STATUS: &str = "policyStatus";
-	pub const PUBLIC_ACCESS_BLOCK: &str = "publicAccessBlock";
-	pub const REPLICATION: &str = "replication";
-	pub const REQUEST_PAYMENT: &str = "requestPayment";
-	pub const RESTORE: &str = "restore";
-	pub const RETENTION: &str = "retention";
-	pub const SELECT: &str = "select";
-	pub const TAGGING: &str = "tagging";
-	pub const TORRENT: &str = "torrent";
-	pub const UPLOADS: &str = "uploads";
-	pub const VERSIONING: &str = "versioning";
-	pub const VERSIONS: &str = "versions";
-	pub const WEBSITE: &str = "website";
+	keywords: [
+		"accelerate" => ACCELERATE,
+		"acl" => ACL,
+		"analytics" => ANALYTICS,
+		"cors" => CORS,
+		"delete" => DELETE,
+		"encryption" => ENCRYPTION,
+		"intelligent-tiering" => INTELLIGENT_TIERING,
+		"inventory" => INVENTORY,
+		"legal-hold" => LEGAL_HOLD,
+		"lifecycle" => LIFECYCLE,
+		"location" => LOCATION,
+		"logging" => LOGGING,
+		"metrics" => METRICS,
+		"notification" => NOTIFICATION,
+		"object-lock" => OBJECT_LOCK,
+		"ownershipControls" => OWNERSHIP_CONTROLS,
+		"policy" => POLICY,
+		"policyStatus" => POLICY_STATUS,
+		"publicAccessBlock" => PUBLIC_ACCESS_BLOCK,
+		"replication" => REPLICATION,
+		"requestPayment" => REQUEST_PAYMENT,
+		"restore" => RESTORE,
+		"retention" => RETENTION,
+		"select" => SELECT,
+		"tagging" => TAGGING,
+		"torrent" => TORRENT,
+		"uploads" => UPLOADS,
+		"versioning" => VERSIONING,
+		"versions" => VERSIONS,
+		"website" => WEBSITE
+	],
+	fields: [
+		"continuation-token" => continuation_token,
+		"delimiter" => delimiter,
+		"encoding-type" => encoding_type,
+		"fetch-owner" => fetch_owner,
+		"id" => id,
+		"key-marker" => key_marker,
+		"list-type" => list_type,
+		"marker" => marker,
+		"max-keys" => max_keys,
+		"max-parts" => max_parts,
+		"max-uploads" => max_uploads,
+		"partNumber" => part_number,
+		"part-number-marker" => part_number_marker,
+		"prefix" => prefix,
+		"select-type" => select_type,
+		"start-after" => start_after,
+		"uploadId" => upload_id,
+		"upload-id-marker" => upload_id_marker,
+		"versionId" => version_id,
+		"version-id-marker" => version_id_marker
+	]
 }

 #[cfg(test)]
@@ -1,6 +1,6 @@
 [package]
 name = "garage_block"
-version = "0.8.0"
+version = "0.8.1"
 authors = ["Alex Auvolat <alex@adnab.me>"]
 edition = "2018"
 license = "AGPL-3.0"
@@ -14,10 +14,10 @@ path = "lib.rs"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html

 [dependencies]
-garage_db = { version = "0.8.0", path = "../db" }
-garage_rpc = { version = "0.8.0", path = "../rpc" }
-garage_util = { version = "0.8.0", path = "../util" }
-garage_table = { version = "0.8.0", path = "../table" }
+garage_db = { version = "0.8.1", path = "../db" }
+garage_rpc = { version = "0.8.1", path = "../rpc" }
+garage_util = { version = "0.8.1", path = "../util" }
+garage_table = { version = "0.8.1", path = "../table" }

 opentelemetry = "0.17"

@@ -31,7 +31,6 @@ rand = "0.8"
 async-compression = { version = "0.3", features = ["tokio", "zstd"] }
 zstd = { version = "0.9", default-features = false }

-rmp-serde = "0.15"
 serde = { version = "1.0", default-features = false, features = ["derive", "rc"] }
 serde_bytes = "0.11"

@@ -3,8 +3,10 @@ use std::pin::Pin;
 use std::sync::Arc;
 use std::time::Duration;

+use arc_swap::ArcSwapOption;
 use async_trait::async_trait;
 use bytes::Bytes;
+use rand::prelude::*;
 use serde::{Deserialize, Serialize};

 use futures::Stream;
@@ -22,9 +24,12 @@ use garage_rpc::rpc_helper::netapp::stream::{stream_asyncread, ByteStream};

 use garage_db as db;

+use garage_util::background::{vars, BackgroundRunner};
 use garage_util::data::*;
 use garage_util::error::*;
 use garage_util::metrics::RecordDuration;
+use garage_util::persister::PersisterShared;
+use garage_util::time::msec_to_rfc3339;

 use garage_rpc::rpc_helper::OrderTag;
 use garage_rpc::system::System;
@@ -87,7 +92,17 @@ pub struct BlockManager {

 	pub(crate) metrics: BlockManagerMetrics,

-	tx_scrub_command: mpsc::Sender<ScrubWorkerCommand>,
+	pub scrub_persister: PersisterShared<ScrubWorkerPersisted>,
+	tx_scrub_command: ArcSwapOption<mpsc::Sender<ScrubWorkerCommand>>,
+}
+
+#[derive(Serialize, Deserialize, Clone, Debug)]
+pub struct BlockResyncErrorInfo {
+	pub hash: Hash,
+	pub refcount: u64,
+	pub error_count: u64,
+	pub last_try: u64,
+	pub next_try: u64,
 }

 // This custom struct contains functions that must only be ran
@@ -114,9 +129,10 @@ impl BlockManager {
 			.netapp
 			.endpoint("garage_block/manager.rs/Rpc".to_string());

-		let metrics = BlockManagerMetrics::new(resync.queue.clone(), resync.errors.clone());
+		let metrics =
+			BlockManagerMetrics::new(rc.rc.clone(), resync.queue.clone(), resync.errors.clone());

-		let (scrub_tx, scrub_rx) = mpsc::channel(1);
+		let scrub_persister = PersisterShared::new(&system.metadata_dir, "scrub_info");

 		let block_manager = Arc::new(Self {
 			replication,
@@ -128,21 +144,46 @@ impl BlockManager {
 			system,
 			endpoint,
 			metrics,
-			tx_scrub_command: scrub_tx,
+			scrub_persister,
+			tx_scrub_command: ArcSwapOption::new(None),
 		});
 		block_manager.endpoint.set_handler(block_manager.clone());

+		block_manager
+	}
+
+	pub fn spawn_workers(self: &Arc<Self>, bg: &BackgroundRunner) {
 		// Spawn a bunch of resync workers
 		for index in 0..MAX_RESYNC_WORKERS {
-			let worker = ResyncWorker::new(index, block_manager.clone());
-			block_manager.system.background.spawn_worker(worker);
+			let worker = ResyncWorker::new(index, self.clone());
+			bg.spawn_worker(worker);
 		}

 		// Spawn scrub worker
-		let scrub_worker = ScrubWorker::new(block_manager.clone(), scrub_rx);
-		block_manager.system.background.spawn_worker(scrub_worker);
+		let (scrub_tx, scrub_rx) = mpsc::channel(1);
+		self.tx_scrub_command.store(Some(Arc::new(scrub_tx)));
+		bg.spawn_worker(ScrubWorker::new(
+			self.clone(),
+			scrub_rx,
+			self.scrub_persister.clone(),
+		));
+	}

-		block_manager
+	pub fn register_bg_vars(&self, vars: &mut vars::BgVars) {
+		self.resync.register_bg_vars(vars);
+
+		vars.register_rw(
+			&self.scrub_persister,
+			"scrub-tranquility",
+			|p| p.get_with(|x| x.tranquility),
+			|p, tranquility| p.set_with(|x| x.tranquility = tranquility),
+		);
+		vars.register_ro(&self.scrub_persister, "scrub-last-completed", |p| {
+			p.get_with(|x| msec_to_rfc3339(x.time_last_complete_scrub))
+		});
+		vars.register_ro(&self.scrub_persister, "scrub-corruptions_detected", |p| {
+			p.get_with(|x| x.corruptions_detected)
+		});
 	}

 	/// Ask nodes that might have a (possibly compressed) block for it
@@ -309,9 +350,42 @@ impl BlockManager {
 		Ok(self.rc.rc.len()?)
 	}

+	/// Get number of items in the refcount table
+	pub fn rc_fast_len(&self) -> Result<Option<usize>, Error> {
+		Ok(self.rc.rc.fast_len()?)
+	}
+
 	/// Send command to start/stop/manager scrub worker
-	pub async fn send_scrub_command(&self, cmd: ScrubWorkerCommand) {
-		let _ = self.tx_scrub_command.send(cmd).await;
+	pub async fn send_scrub_command(&self, cmd: ScrubWorkerCommand) -> Result<(), Error> {
+		let tx = self.tx_scrub_command.load();
+		let tx = tx.as_ref().ok_or_message("scrub worker is not running")?;
+		tx.send(cmd).await.ok_or_message("send error")?;
+		Ok(())
+	}
+
+	/// Get the reference count of a block
+	pub fn get_block_rc(&self, hash: &Hash) -> Result<u64, Error> {
+		Ok(self.rc.get_block_rc(hash)?.as_u64())
+	}
+
+	/// List all resync errors
+	pub fn list_resync_errors(&self) -> Result<Vec<BlockResyncErrorInfo>, Error> {
+		let mut blocks = Vec::with_capacity(self.resync.errors.len());
+		for ent in self.resync.errors.iter()? {
+			let (hash, cnt) = ent?;
+			let cnt = ErrorCounter::decode(&cnt);
+			blocks.push(BlockResyncErrorInfo {
+				hash: Hash::try_from(&hash).unwrap(),
+				refcount: 0,
+				error_count: cnt.errors,
+				last_try: cnt.last_try,
+				next_try: cnt.next_try(),
+			});
+		}
+		for block in blocks.iter_mut() {
+			block.refcount = self.get_block_rc(&block.hash)?;
+		}
+		Ok(blocks)
 	}

 	//// ----- Managing the reference counter ----
@@ -603,14 +677,21 @@ impl BlockManagerLocked {
 			}
 		};

-		let mut path2 = path.clone();
-		path2.set_extension("tmp");
-		let mut f = fs::File::create(&path2).await?;
+		let mut path_tmp = path.clone();
+		let tmp_extension = format!("tmp{}", hex::encode(thread_rng().gen::<[u8; 4]>()));
+		path_tmp.set_extension(tmp_extension);
+
+		let mut delete_on_drop = DeleteOnDrop(Some(path_tmp.clone()));
+
+		let mut f = fs::File::create(&path_tmp).await?;
 		f.write_all(data).await?;
 		f.sync_all().await?;
 		drop(f);

-		fs::rename(path2, path).await?;
+		fs::rename(path_tmp, path).await?;
+
+		delete_on_drop.cancel();
+
 		if let Some(to_delete) = to_delete {
 			fs::remove_file(to_delete).await?;
 		}
@@ -676,3 +757,23 @@ async fn read_stream_to_end(mut stream: ByteStream) -> Result<Bytes, Error> {
 		.concat()
 		.into())
 }
+
+struct DeleteOnDrop(Option<PathBuf>);
+
+impl DeleteOnDrop {
+	fn cancel(&mut self) {
+		drop(self.0.take());
+	}
+}
+
+impl Drop for DeleteOnDrop {
+	fn drop(&mut self) {
+		if let Some(path) = self.0.take() {
+			tokio::spawn(async move {
+				if let Err(e) = fs::remove_file(&path).await {
+					debug!("DeleteOnDrop failed for {}: {}", path.display(), e);
+				}
+			});
+		}
+	}
+}
@@ -1,9 +1,11 @@
 use opentelemetry::{global, metrics::*};

+use garage_db as db;
 use garage_db::counted_tree_hack::CountedTree;

 /// TableMetrics reference all counter used for metrics
 pub struct BlockManagerMetrics {
+	pub(crate) _rc_size: ValueObserver<u64>,
 	pub(crate) _resync_queue_len: ValueObserver<u64>,
 	pub(crate) _resync_errored_blocks: ValueObserver<u64>,

@@ -23,9 +25,17 @@ pub struct BlockManagerMetrics {
 }

 impl BlockManagerMetrics {
-	pub fn new(resync_queue: CountedTree, resync_errors: CountedTree) -> Self {
+	pub fn new(rc_tree: db::Tree, resync_queue: CountedTree, resync_errors: CountedTree) -> Self {
 		let meter = global::meter("garage_model/block");
 		Self {
+			_rc_size: meter
+				.u64_value_observer("block.rc_size", move |observer| {
+					if let Ok(Some(v)) = rc_tree.fast_len() {
+						observer.observe(v as u64, &[])
+					}
+				})
+				.with_description("Number of blocks known to the reference counter")
+				.init(),
 			_resync_queue_len: meter
 				.u64_value_observer("block.resync_queue_length", move |observer| {
 					observer.observe(resync_queue.len() as u64, &[])
@@ -169,4 +169,11 @@ impl RcEntry {
 	pub(crate) fn is_needed(&self) -> bool {
 		!self.is_deletable()
 	}
+
+	pub(crate) fn as_u64(&self) -> u64 {
+		match self {
+			RcEntry::Present { count } => *count,
+			_ => 0,
+		}
+	}
 }
@@ -13,7 +13,7 @@ use tokio::sync::watch;
 use garage_util::background::*;
 use garage_util::data::*;
 use garage_util::error::*;
-use garage_util::persister::Persister;
+use garage_util::persister::PersisterShared;
 use garage_util::time::*;
 use garage_util::tranquilizer::Tranquilizer;

@@ -53,7 +53,7 @@ impl Worker for RepairWorker {
 		"Block repair worker".into()
 	}

-	fn info(&self) -> Option<String> {
+	fn status(&self) -> WorkerStatus {
 		match self.block_iter.as_ref() {
 			None => {
 				let idx_bytes = self
@@ -66,9 +66,20 @@ impl Worker for RepairWorker {
 				} else {
 					idx_bytes
 				};
-				Some(format!("Phase 1: {}", hex::encode(idx_bytes)))
+				WorkerStatus {
+					progress: Some("0.00%".into()),
+					freeform: vec![format!(
+						"Currently in phase 1, iterator position: {}",
+						hex::encode(idx_bytes)
+					)],
+					..Default::default()
+				}
 			}
-			Some(bi) => Some(format!("Phase 2: {:.2}% done", bi.progress() * 100.)),
+			Some(bi) => WorkerStatus {
+				progress: Some(format!("{:.2}%", bi.progress() * 100.)),
+				freeform: vec!["Currently in phase 2".into()],
+				..Default::default()
+			},
 		}
 	}

@@ -137,7 +148,7 @@ impl Worker for RepairWorker {
 		}
 	}

-	async fn wait_for_work(&mut self, _must_exit: &watch::Receiver<bool>) -> WorkerState {
+	async fn wait_for_work(&mut self) -> WorkerState {
 		unreachable!()
 	}
 }
@@ -157,15 +168,24 @@ pub struct ScrubWorker {
 	work: ScrubWorkerState,
 	tranquilizer: Tranquilizer,

-	persister: Persister<ScrubWorkerPersisted>,
-	persisted: ScrubWorkerPersisted,
+	persister: PersisterShared<ScrubWorkerPersisted>,
 }

 #[derive(Serialize, Deserialize)]
-struct ScrubWorkerPersisted {
-	tranquility: u32,
-	time_last_complete_scrub: u64,
-	corruptions_detected: u64,
+pub struct ScrubWorkerPersisted {
+	pub tranquility: u32,
+	pub(crate) time_last_complete_scrub: u64,
+	pub(crate) corruptions_detected: u64,
+}
+impl garage_util::migrate::InitialFormat for ScrubWorkerPersisted {}
+impl Default for ScrubWorkerPersisted {
+	fn default() -> Self {
+		ScrubWorkerPersisted {
+			time_last_complete_scrub: 0,
+			tranquility: INITIAL_SCRUB_TRANQUILITY,
+			corruptions_detected: 0,
+		}
+	}
 }

 enum ScrubWorkerState {
@@ -186,27 +206,20 @@ pub enum ScrubWorkerCommand {
 	Pause(Duration),
 	Resume,
 	Cancel,
-	SetTranquility(u32),
 }

 impl ScrubWorker {
-	pub fn new(manager: Arc<BlockManager>, rx_cmd: mpsc::Receiver<ScrubWorkerCommand>) -> Self {
-		let persister = Persister::new(&manager.system.metadata_dir, "scrub_info");
-		let persisted = match persister.load() {
-			Ok(v) => v,
-			Err(_) => ScrubWorkerPersisted {
-				time_last_complete_scrub: 0,
-				tranquility: INITIAL_SCRUB_TRANQUILITY,
-				corruptions_detected: 0,
-			},
-		};
+	pub(crate) fn new(
+		manager: Arc<BlockManager>,
+		rx_cmd: mpsc::Receiver<ScrubWorkerCommand>,
+		persister: PersisterShared<ScrubWorkerPersisted>,
+	) -> Self {
 		Self {
 			manager,
 			rx_cmd,
 			work: ScrubWorkerState::Finished,
 			tranquilizer: Tranquilizer::new(30),
 			persister,
-			persisted,
 		}
 	}

@@ -255,12 +268,6 @@ impl ScrubWorker {
 					}
 				}
 			}
-			ScrubWorkerCommand::SetTranquility(t) => {
-				self.persisted.tranquility = t;
-				if let Err(e) = self.persister.save_async(&self.persisted).await {
-					error!("Could not save new tranquilitiy value: {}", e);
-				}
-			}
 		}
 	}
 }
@@ -271,29 +278,37 @@ impl Worker for ScrubWorker {
 		"Block scrub worker".into()
 	}

-	fn info(&self) -> Option<String> {
-		let s = match &self.work {
-			ScrubWorkerState::Running(bsi) => format!(
-				"{:.2}% done (tranquility = {})",
-				bsi.progress() * 100.,
-				self.persisted.tranquility
-			),
-			ScrubWorkerState::Paused(bsi, rt) => {
-				format!(
-					"Paused, {:.2}% done, resumes at {}",
-					bsi.progress() * 100.,
-					msec_to_rfc3339(*rt)
+	fn status(&self) -> WorkerStatus {
+		let (corruptions_detected, tranquility, time_last_complete_scrub) =
+			self.persister.get_with(|p| {
+				(
+					p.corruptions_detected,
+					p.tranquility,
+					p.time_last_complete_scrub,
 				)
-			}
-			ScrubWorkerState::Finished => format!(
-				"Last completed scrub: {}",
-				msec_to_rfc3339(self.persisted.time_last_complete_scrub)
-			),
+			});
+
+		let mut s = WorkerStatus {
+			persistent_errors: Some(corruptions_detected),
+			tranquility: Some(tranquility),
+			..Default::default()
 		};
-		Some(format!(
-			"{} ; corruptions detected: {}",
-			s, self.persisted.corruptions_detected
-		))
+		match &self.work {
+			ScrubWorkerState::Running(bsi) => {
+				s.progress = Some(format!("{:.2}%", bsi.progress() * 100.));
+			}
+			ScrubWorkerState::Paused(bsi, rt) => {
+				s.progress = Some(format!("{:.2}%", bsi.progress() * 100.));
+				s.freeform = vec![format!("Scrub paused, resumes at {}", msec_to_rfc3339(*rt))];
+			}
+			ScrubWorkerState::Finished => {
+				s.freeform = vec![format!(
+					"Last scrub completed at {}",
+					msec_to_rfc3339(time_last_complete_scrub)
+				)];
+			}
+		}
+		s
 	}

 	async fn work(&mut self, _must_exit: &mut watch::Receiver<bool>) -> Result<WorkerState, Error> {
@@ -310,18 +325,17 @@ impl Worker for ScrubWorker {
 					match self.manager.read_block(&hash).await {
 						Err(Error::CorruptData(_)) => {
 							error!("Found corrupt data block during scrub: {:?}", hash);
-							self.persisted.corruptions_detected += 1;
-							self.persister.save_async(&self.persisted).await?;
+							self.persister.set_with(|p| p.corruptions_detected += 1)?;
 						}
 						Err(e) => return Err(e),
 						_ => (),
 					};
 					Ok(self
 						.tranquilizer
-						.tranquilize_worker(self.persisted.tranquility))
+						.tranquilize_worker(self.persister.get_with(|p| p.tranquility)))
 				} else {
-					self.persisted.time_last_complete_scrub = now_msec();
-					self.persister.save_async(&self.persisted).await?;
+					self.persister
+						.set_with(|p| p.time_last_complete_scrub = now_msec())?;
 					self.work = ScrubWorkerState::Finished;
 					self.tranquilizer.clear();
 					Ok(WorkerState::Idle)
@@ -331,12 +345,13 @@ impl Worker for ScrubWorker {
 		}
 	}

-	async fn wait_for_work(&mut self, _must_exit: &watch::Receiver<bool>) -> WorkerState {
+	async fn wait_for_work(&mut self) -> WorkerState {
 		let (wait_until, command) = match &self.work {
 			ScrubWorkerState::Running(_) => return WorkerState::Busy,
 			ScrubWorkerState::Paused(_, resume_time) => (*resume_time, ScrubWorkerCommand::Resume),
 			ScrubWorkerState::Finished => (
-				self.persisted.time_last_complete_scrub + SCRUB_INTERVAL.as_millis() as u64,
+				self.persister.get_with(|p| p.time_last_complete_scrub)
+					+ SCRUB_INTERVAL.as_millis() as u64,
 				ScrubWorkerCommand::Start,
 			),
 		};
@@ -3,7 +3,6 @@ use std::convert::TryInto;
 use std::sync::{Arc, Mutex};
 use std::time::Duration;

-use arc_swap::ArcSwap;
 use async_trait::async_trait;
 use serde::{Deserialize, Serialize};

@@ -22,7 +21,7 @@ use garage_util::background::*;
 use garage_util::data::*;
 use garage_util::error::*;
 use garage_util::metrics::RecordDuration;
-use garage_util::persister::Persister;
+use garage_util::persister::PersisterShared;
 use garage_util::time::*;
 use garage_util::tranquilizer::Tranquilizer;

@@ -49,13 +48,12 @@ const INITIAL_RESYNC_TRANQUILITY: u32 = 2;

 pub struct BlockResyncManager {
 	pub(crate) queue: CountedTree,
-	pub(crate) notify: Notify,
+	pub(crate) notify: Arc<Notify>,
 	pub(crate) errors: CountedTree,

 	busy_set: BusySet,

-	persister: Persister<ResyncPersistedConfig>,
-	persisted: ArcSwap<ResyncPersistedConfig>,
+	persister: PersisterShared<ResyncPersistedConfig>,
 }

 #[derive(Serialize, Deserialize, Clone, Copy)]
@@ -63,6 +61,15 @@ struct ResyncPersistedConfig {
 	n_workers: usize,
 	tranquility: u32,
 }
+impl garage_util::migrate::InitialFormat for ResyncPersistedConfig {}
+impl Default for ResyncPersistedConfig {
+	fn default() -> Self {
+		ResyncPersistedConfig {
+			n_workers: 1,
+			tranquility: INITIAL_RESYNC_TRANQUILITY,
+		}
+	}
+}

 enum ResyncIterResult {
 	BusyDidSomething,
@@ -90,22 +97,14 @@ impl BlockResyncManager {
 			.expect("Unable to open block_local_resync_errors tree");
 		let errors = CountedTree::new(errors).expect("Could not count block_local_resync_errors");

-		let persister = Persister::new(&system.metadata_dir, "resync_cfg");
-		let persisted = match persister.load() {
-			Ok(v) => v,
-			Err(_) => ResyncPersistedConfig {
-				n_workers: 1,
-				tranquility: INITIAL_RESYNC_TRANQUILITY,
-			},
-		};
+		let persister = PersisterShared::new(&system.metadata_dir, "resync_cfg");

 		Self {
 			queue,
-			notify: Notify::new(),
+			notify: Arc::new(Notify::new()),
 			errors,
 			busy_set: Arc::new(Mutex::new(HashSet::new())),
 			persister,
-			persisted: ArcSwap::new(Arc::new(persisted)),
 		}
 	}

@@ -123,6 +122,56 @@ impl BlockResyncManager {
 		Ok(self.errors.len())
 	}

+	/// Clear the error counter for a block and put it in queue immediately
+	pub fn clear_backoff(&self, hash: &Hash) -> Result<(), Error> {
+		let now = now_msec();
+		if let Some(ec) = self.errors.get(hash)? {
+			let mut ec = ErrorCounter::decode(&ec);
+			if ec.errors > 0 {
+				ec.last_try = now - ec.delay_msec();
+				self.errors.insert(hash, ec.encode())?;
+				self.put_to_resync_at(hash, now)?;
+				return Ok(());
+			}
+		}
+		Err(Error::Message(format!(
+			"Block {:?} was not in an errored state",
+			hash
+		)))
+	}
+
+	pub fn register_bg_vars(&self, vars: &mut vars::BgVars) {
+		let notify = self.notify.clone();
+		vars.register_rw(
+			&self.persister,
+			"resync-worker-count",
+			|p| p.get_with(|x| x.n_workers),
+			move |p, n_workers| {
+				if !(1..=MAX_RESYNC_WORKERS).contains(&n_workers) {
+					return Err(Error::Message(format!(
+						"Invalid number of resync workers, must be between 1 and {}",
+						MAX_RESYNC_WORKERS
+					)));
+				}
+				p.set_with(|x| x.n_workers = n_workers)?;
+				notify.notify_waiters();
+				Ok(())
+			},
+		);
+
+		let notify = self.notify.clone();
+		vars.register_rw(
+			&self.persister,
+			"resync-tranquility",
+			|p| p.get_with(|x| x.tranquility),
+			move |p, tranquility| {
+				p.set_with(|x| x.tranquility = tranquility)?;
+				notify.notify_waiters();
+				Ok(())
+			},
+		);
+	}
+
 	// ---- Resync loop ----

 	// This part manages a queue of blocks that need to be
@@ -257,7 +306,7 @@ impl BlockResyncManager {

 				if let Err(e) = &res {
 					manager.metrics.resync_error_counter.add(1);
-					warn!("Error when resyncing {:?}: {}", hash, e);
+					error!("Error when resyncing {:?}: {}", hash, e);

 					let err_counter = match self.errors.get(hash.as_slice())? {
 						Some(ec) => ErrorCounter::decode(&ec).add1(now + 1),
@@ -417,33 +466,6 @@ impl BlockResyncManager {

 		Ok(())
 	}
-
-	async fn update_persisted(
-		&self,
-		update: impl Fn(&mut ResyncPersistedConfig),
-	) -> Result<(), Error> {
-		let mut cfg: ResyncPersistedConfig = *self.persisted.load().as_ref();
-		update(&mut cfg);
-		self.persister.save_async(&cfg).await?;
-		self.persisted.store(Arc::new(cfg));
-		self.notify.notify_waiters();
-		Ok(())
-	}
-
-	pub async fn set_n_workers(&self, n_workers: usize) -> Result<(), Error> {
-		if !(1..=MAX_RESYNC_WORKERS).contains(&n_workers) {
-			return Err(Error::Message(format!(
-				"Invalid number of resync workers, must be between 1 and {}",
-				MAX_RESYNC_WORKERS
-			)));
-		}
-		self.update_persisted(|cfg| cfg.n_workers = n_workers).await
-	}
-
-	pub async fn set_tranquility(&self, tranquility: u32) -> Result<(), Error> {
-		self.update_persisted(|cfg| cfg.tranquility = tranquility)
-			.await
-	}
 }

 impl Drop for BusyBlock {
@@ -458,15 +480,18 @@ pub(crate) struct ResyncWorker {
 	manager: Arc<BlockManager>,
 	tranquilizer: Tranquilizer,
 	next_delay: Duration,
+	persister: PersisterShared<ResyncPersistedConfig>,
 }

 impl ResyncWorker {
 	pub(crate) fn new(index: usize, manager: Arc<BlockManager>) -> Self {
+		let persister = manager.resync.persister.clone();
 		Self {
 			index,
 			manager,
 			tranquilizer: Tranquilizer::new(30),
 			next_delay: Duration::from_secs(10),
+			persister,
 		}
 	}
 }
@@ -477,39 +502,36 @@ impl Worker for ResyncWorker {
 		format!("Block resync worker #{}", self.index + 1)
 	}

-	fn info(&self) -> Option<String> {
-		let persisted = self.manager.resync.persisted.load();
+	fn status(&self) -> WorkerStatus {
+		let (n_workers, tranquility) = self.persister.get_with(|x| (x.n_workers, x.tranquility));

-		if self.index >= persisted.n_workers {
-			return Some("(unused)".into());
+		if self.index >= n_workers {
+			return WorkerStatus {
+				freeform: vec!["This worker is currently disabled".into()],
+				..Default::default()
+			};
 		}

-		let mut ret = vec![];
-		ret.push(format!("tranquility = {}", persisted.tranquility));
-
-		let qlen = self.manager.resync.queue_len().unwrap_or(0);
-		if qlen > 0 {
-			ret.push(format!("{} blocks in queue", qlen));
+		WorkerStatus {
+			queue_length: Some(self.manager.resync.queue_len().unwrap_or(0) as u64),
+			tranquility: Some(tranquility),
+			persistent_errors: Some(self.manager.resync.errors_len().unwrap_or(0) as u64),
+			..Default::default()
 		}
-
-		let elen = self.manager.resync.errors_len().unwrap_or(0);
-		if elen > 0 {
-			ret.push(format!("{} blocks in error state", elen));
-		}
-
-		Some(ret.join(", "))
 	}

 	async fn work(&mut self, _must_exit: &mut watch::Receiver<bool>) -> Result<WorkerState, Error> {
-		if self.index >= self.manager.resync.persisted.load().n_workers {
+		let (n_workers, tranquility) = self.persister.get_with(|x| (x.n_workers, x.tranquility));
+
+		if self.index >= n_workers {
 			return Ok(WorkerState::Idle);
 		}

 		self.tranquilizer.reset();
 		match self.manager.resync.resync_iter(&self.manager).await {
-			Ok(ResyncIterResult::BusyDidSomething) => Ok(self
-				.tranquilizer
-				.tranquilize_worker(self.manager.resync.persisted.load().tranquility)),
+			Ok(ResyncIterResult::BusyDidSomething) => {
+				Ok(self.tranquilizer.tranquilize_worker(tranquility))
+			}
 			Ok(ResyncIterResult::BusyDidNothing) => Ok(WorkerState::Busy),
 			Ok(ResyncIterResult::IdleFor(delay)) => {
 				self.next_delay = delay;
@@ -527,8 +549,8 @@ impl Worker for ResyncWorker {
 		}
 	}

-	async fn wait_for_work(&mut self, _must_exit: &watch::Receiver<bool>) -> WorkerState {
-		while self.index >= self.manager.resync.persisted.load().n_workers {
+	async fn wait_for_work(&mut self) -> WorkerState {
+		while self.index >= self.persister.get_with(|x| x.n_workers) {
 			self.manager.resync.notify.notified().await
 		}

@@ -545,9 +567,9 @@ impl Worker for ResyncWorker {
 /// and the time of the last try.
 /// Used to implement exponential backoff.
 #[derive(Clone, Copy, Debug)]
-struct ErrorCounter {
-	errors: u64,
-	last_try: u64,
+pub(crate) struct ErrorCounter {
+	pub(crate) errors: u64,
+	pub(crate) last_try: u64,
 }

 impl ErrorCounter {
@@ -558,12 +580,13 @@ impl ErrorCounter {
 		}
 	}

-	fn decode(data: &[u8]) -> Self {
+	pub(crate) fn decode(data: &[u8]) -> Self {
 		Self {
 			errors: u64::from_be_bytes(data[0..8].try_into().unwrap()),
 			last_try: u64::from_be_bytes(data[8..16].try_into().unwrap()),
 		}
 	}
+
 	fn encode(&self) -> Vec<u8> {
 		[
 			u64::to_be_bytes(self.errors),
@@ -583,7 +606,8 @@ impl ErrorCounter {
 		(RESYNC_RETRY_DELAY.as_millis() as u64)
 			<< std::cmp::min(self.errors - 1, RESYNC_RETRY_DELAY_MAX_BACKOFF_POWER)
 	}
-	fn next_try(&self) -> u64 {
+
+	pub(crate) fn next_try(&self) -> u64 {
 		self.last_try + self.delay_msec()
 	}
 }
@@ -1,6 +1,6 @@
 [package]
 name = "garage_db"
-version = "0.8.0"
+version = "0.8.1"
 authors = ["Alex Auvolat <alex@adnab.me>"]
 edition = "2018"
 license = "AGPL-3.0"
@@ -181,6 +181,10 @@ impl Tree {
 	pub fn len(&self) -> Result<usize> {
 		self.0.len(self.1)
 	}
+	#[inline]
+	pub fn fast_len(&self) -> Result<Option<usize>> {
+		self.0.fast_len(self.1)
+	}

 	#[inline]
 	pub fn first(&self) -> Result<Option<(Value, Value)>> {
@@ -323,6 +327,9 @@ pub(crate) trait IDb: Send + Sync {

 	fn get(&self, tree: usize, key: &[u8]) -> Result<Option<Value>>;
 	fn len(&self, tree: usize) -> Result<usize>;
+	fn fast_len(&self, _tree: usize) -> Result<Option<usize>> {
+		Ok(None)
+	}

 	fn insert(&self, tree: usize, key: &[u8], value: &[u8]) -> Result<Option<Value>>;
 	fn remove(&self, tree: usize, key: &[u8]) -> Result<Option<Value>>;
@@ -121,6 +121,10 @@ impl IDb for LmdbDb {
 		Ok(tree.len(&tx)?.try_into().unwrap())
 	}

+	fn fast_len(&self, tree: usize) -> Result<Option<usize>> {
+		Ok(Some(self.len(tree)?))
+	}
+
 	fn insert(&self, tree: usize, key: &[u8], value: &[u8]) -> Result<Option<Value>> {
 		let tree = self.get_tree(tree)?;
 		let mut tx = self.db.write_txn()?;
@@ -144,6 +144,10 @@ impl IDb for SqliteDb {
 		}
 	}

+	fn fast_len(&self, tree: usize) -> Result<Option<usize>> {
+		Ok(Some(self.len(tree)?))
+	}
+
 	fn insert(&self, tree: usize, key: &[u8], value: &[u8]) -> Result<Option<Value>> {
 		trace!("insert {}: lock db", tree);
 		let this = self.0.lock().unwrap();
@@ -1,6 +1,6 @@
 [package]
 name = "garage"
-version = "0.8.0"
+version = "0.8.1"
 authors = ["Alex Auvolat <alex@adnab.me>"]
 edition = "2018"
 license = "AGPL-3.0"
@@ -21,14 +21,14 @@ path = "tests/lib.rs"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html

 [dependencies]
-garage_db = { version = "0.8.0", path = "../db" }
-garage_api = { version = "0.8.0", path = "../api" }
-garage_block = { version = "0.8.0", path = "../block" }
-garage_model = { version = "0.8.0", path = "../model" }
-garage_rpc = { version = "0.8.0", path = "../rpc" }
-garage_table = { version = "0.8.0", path = "../table" }
-garage_util = { version = "0.8.0", path = "../util" }
-garage_web = { version = "0.8.0", path = "../web" }
+garage_db = { version = "0.8.1", path = "../db" }
+garage_api = { version = "0.8.1", path = "../api" }
+garage_block = { version = "0.8.1", path = "../block" }
+garage_model = { version = "0.8.1", path = "../model" }
+garage_rpc = { version = "0.8.1", path = "../rpc" }
+garage_table = { version = "0.8.1", path = "../table" }
+garage_util = { version = "0.8.1", path = "../util" }
+garage_web = { version = "0.8.1", path = "../web" }

 backtrace = "0.3"
 bytes = "1.0"
@@ -36,13 +36,12 @@ bytesize = "1.1"
 timeago = "0.3"
 parse_duration = "2.1"
 hex = "0.4"
-tracing = { version = "0.1.30", features = ["log-always"] }
+tracing = { version = "0.1.30" }
 tracing-subscriber = { version = "0.3", features = ["env-filter"] }
 rand = "0.8"
 async-trait = "0.1.7"
 sodiumoxide = { version = "0.2.5-0", package = "kuska-sodiumoxide" }

-rmp-serde = "0.15"
 serde = { version = "1.0", default-features = false, features = ["derive", "rc"] }
 serde_bytes = "0.11"
 structopt = { version = "0.3", default-features = false }
@@ -74,7 +73,7 @@ base64 = "0.13"


 [features]
-default = [ "bundled-libs", "metrics", "sled" ]
+default = [ "bundled-libs", "metrics", "sled", "k2v" ]

 k2v = [ "garage_util/k2v", "garage_api/k2v" ]

@@ -5,9 +5,11 @@ use std::sync::Arc;
 use async_trait::async_trait;
 use serde::{Deserialize, Serialize};

+use garage_util::background::BackgroundRunner;
 use garage_util::crdt::*;
 use garage_util::data::*;
 use garage_util::error::Error as GarageError;
+use garage_util::formater::format_table_to_string;
 use garage_util::time::*;

 use garage_table::replication::*;
@@ -15,7 +17,7 @@ use garage_table::*;

 use garage_rpc::*;

-use garage_block::repair::ScrubWorkerCommand;
+use garage_block::manager::BlockResyncErrorInfo;

 use garage_model::bucket_alias_table::*;
 use garage_model::bucket_table::*;
@@ -24,6 +26,8 @@ use garage_model::helper::error::{Error, OkOrBadRequest};
 use garage_model::key_table::*;
 use garage_model::migrate::Migrate;
 use garage_model::permission::*;
+use garage_model::s3::object_table::*;
+use garage_model::s3::version_table::Version;

 use crate::cli::*;
 use crate::repair::online::launch_online_repair;
@@ -38,7 +42,8 @@ pub enum AdminRpc {
 	LaunchRepair(RepairOpt),
 	Migrate(MigrateOpt),
 	Stats(StatsOpt),
-	Worker(WorkerOpt),
+	Worker(WorkerOperation),
+	BlockOperation(BlockOperation),

 	// Replies
 	Ok(String),
@@ -54,6 +59,14 @@ pub enum AdminRpc {
 		HashMap<usize, garage_util::background::WorkerInfo>,
 		WorkerListOpt,
 	),
+	WorkerVars(Vec<(Uuid, String, String)>),
+	WorkerInfo(usize, garage_util::background::WorkerInfo),
+	BlockErrorList(Vec<BlockResyncErrorInfo>),
+	BlockInfo {
+		hash: Hash,
+		refcount: u64,
+		versions: Vec<Result<Version, Uuid>>,
+	},
 }

 impl Rpc for AdminRpc {
@@ -62,17 +75,24 @@ impl Rpc for AdminRpc {

 pub struct AdminRpcHandler {
 	garage: Arc<Garage>,
+	background: Arc<BackgroundRunner>,
 	endpoint: Arc<Endpoint<AdminRpc, Self>>,
 }

 impl AdminRpcHandler {
-	pub fn new(garage: Arc<Garage>) -> Arc<Self> {
+	pub fn new(garage: Arc<Garage>, background: Arc<BackgroundRunner>) -> Arc<Self> {
 		let endpoint = garage.system.netapp.endpoint(ADMIN_RPC_PATH.into());
-		let admin = Arc::new(Self { garage, endpoint });
+		let admin = Arc::new(Self {
+			garage,
+			background,
+			endpoint,
+		});
 		admin.endpoint.set_handler(admin.clone());
 		admin
 	}

+	// ================ BUCKET COMMANDS ====================
+
 	async fn handle_bucket_cmd(&self, cmd: &BucketOperation) -> Result<AdminRpc, Error> {
 		match cmd {
 			BucketOperation::List => self.handle_list_buckets().await,
@@ -551,6 +571,8 @@ impl AdminRpcHandler {
 		Ok(AdminRpc::Ok(ret))
 	}

+	// ================ KEY COMMANDS ====================
+
 	async fn handle_key_cmd(&self, cmd: &KeyOperation) -> Result<AdminRpc, Error> {
 		match cmd {
 			KeyOperation::List => self.handle_list_keys().await,
@@ -688,6 +710,8 @@ impl AdminRpcHandler {
 		Ok(AdminRpc::KeyInfo(key, relevant_buckets))
 	}

+	// ================ MIGRATION COMMANDS ====================
+
 	async fn handle_migrate(self: &Arc<Self>, opt: MigrateOpt) -> Result<AdminRpc, Error> {
 		if !opt.yes {
 			return Err(Error::BadRequest(
@@ -704,6 +728,8 @@ impl AdminRpcHandler {
 		Ok(AdminRpc::Ok("Migration successfull.".into()))
 	}

+	// ================ REPAIR COMMANDS ====================
+
 	async fn handle_launch_repair(self: &Arc<Self>, opt: RepairOpt) -> Result<AdminRpc, Error> {
 		if !opt.yes {
 			return Err(Error::BadRequest(
@@ -739,7 +765,7 @@ impl AdminRpcHandler {
 				)))
 			}
 		} else {
-			launch_online_repair(self.garage.clone(), opt).await;
+			launch_online_repair(&self.garage, &self.background, opt).await?;
 			Ok(AdminRpc::Ok(format!(
 				"Repair launched on {:?}",
 				self.garage.system.id
@@ -747,6 +773,8 @@ impl AdminRpcHandler {
 		}
 	}

+	// ================ STATS COMMANDS ====================
+
 	async fn handle_stats(&self, opt: StatsOpt) -> Result<AdminRpc, Error> {
 		if opt.all_nodes {
 			let mut ret = String::new();
@@ -763,11 +791,12 @@ impl AdminRpcHandler {
 				match self
 					.endpoint
 					.call(&node_id, AdminRpc::Stats(opt), PRIO_NORMAL)
-					.await?
+					.await
 				{
-					Ok(AdminRpc::Ok(s)) => writeln!(&mut ret, "{}", s).unwrap(),
-					Ok(x) => writeln!(&mut ret, "Bad answer: {:?}", x).unwrap(),
-					Err(e) => writeln!(&mut ret, "Error: {}", e).unwrap(),
+					Ok(Ok(AdminRpc::Ok(s))) => writeln!(&mut ret, "{}", s).unwrap(),
+					Ok(Ok(x)) => writeln!(&mut ret, "Bad answer: {:?}", x).unwrap(),
+					Ok(Err(e)) => writeln!(&mut ret, "Remote error: {}", e).unwrap(),
+					Err(e) => writeln!(&mut ret, "Network error: {}", e).unwrap(),
 				}
 			}
 			Ok(AdminRpc::Ok(ret))
@@ -787,6 +816,7 @@ impl AdminRpcHandler {
 				.unwrap_or_else(|| "(unknown)".into()),
 		)
 		.unwrap();
+
 		writeln!(&mut ret, "\nDatabase engine: {}", self.garage.db.engine()).unwrap();

 		// Gather ring statistics
@@ -805,21 +835,38 @@ impl AdminRpcHandler {
 			writeln!(&mut ret, "  {:?} {}", n, c).unwrap();
 		}

-		self.gather_table_stats(&mut ret, &self.garage.bucket_table, &opt)?;
-		self.gather_table_stats(&mut ret, &self.garage.key_table, &opt)?;
-		self.gather_table_stats(&mut ret, &self.garage.object_table, &opt)?;
-		self.gather_table_stats(&mut ret, &self.garage.version_table, &opt)?;
-		self.gather_table_stats(&mut ret, &self.garage.block_ref_table, &opt)?;
+		// Gather table statistics
+		let mut table = vec!["  Table\tItems\tMklItems\tMklTodo\tGcTodo".into()];
+		table.push(self.gather_table_stats(&self.garage.bucket_table, opt.detailed)?);
+		table.push(self.gather_table_stats(&self.garage.key_table, opt.detailed)?);
+		table.push(self.gather_table_stats(&self.garage.object_table, opt.detailed)?);
+		table.push(self.gather_table_stats(&self.garage.version_table, opt.detailed)?);
+		table.push(self.gather_table_stats(&self.garage.block_ref_table, opt.detailed)?);
+		write!(
+			&mut ret,
+			"\nTable stats:\n{}",
+			format_table_to_string(table)
+		)
+		.unwrap();

+		// Gather block manager statistics
 		writeln!(&mut ret, "\nBlock manager stats:").unwrap();
-		if opt.detailed {
-			writeln!(
-				&mut ret,
-				"  number of RC entries (~= number of blocks): {}",
-				self.garage.block_manager.rc_len()?
-			)
-			.unwrap();
-		}
+		let rc_len = if opt.detailed {
+			self.garage.block_manager.rc_len()?.to_string()
+		} else {
+			self.garage
+				.block_manager
+				.rc_fast_len()?
+				.map(|x| x.to_string())
+				.unwrap_or_else(|| "NC".into())
+		};
+
+		writeln!(
+			&mut ret,
+			"  number of RC entries (~= number of blocks): {}",
+			rc_len
+		)
+		.unwrap();
 		writeln!(
 			&mut ret,
 			"  resync queue length: {}",
@@ -833,79 +880,305 @@ impl AdminRpcHandler {
 		)
 		.unwrap();

+		if !opt.detailed {
+			writeln!(&mut ret, "\nIf values are missing (marked as NC), consider adding the --detailed flag - this will be slow.").unwrap();
+		}
+
 		Ok(ret)
 	}

 	fn gather_table_stats<F, R>(
 		&self,
-		to: &mut String,
 		t: &Arc<Table<F, R>>,
-		opt: &StatsOpt,
-	) -> Result<(), Error>
+		detailed: bool,
+	) -> Result<String, Error>
 	where
 		F: TableSchema + 'static,
 		R: TableReplication + 'static,
 	{
-		writeln!(to, "\nTable stats for {}", F::TABLE_NAME).unwrap();
-		if opt.detailed {
-			writeln!(
-				to,
-				"  number of items: {}",
-				t.data.store.len().map_err(GarageError::from)?
+		let (data_len, mkl_len) = if detailed {
+			(
+				t.data.store.len().map_err(GarageError::from)?.to_string(),
+				t.merkle_updater.merkle_tree_len()?.to_string(),
 			)
-			.unwrap();
-			writeln!(
-				to,
-				"  Merkle tree size: {}",
-				t.merkle_updater.merkle_tree_len()?
+		} else {
+			(
+				t.data
+					.store
+					.fast_len()
+					.map_err(GarageError::from)?
+					.map(|x| x.to_string())
+					.unwrap_or_else(|| "NC".into()),
+				t.merkle_updater
+					.merkle_tree_fast_len()?
+					.map(|x| x.to_string())
+					.unwrap_or_else(|| "NC".into()),
 			)
-			.unwrap();
-		}
-		writeln!(
-			to,
-			"  Merkle updater todo queue length: {}",
-			t.merkle_updater.todo_len()?
-		)
-		.unwrap();
-		writeln!(to, "  GC todo queue length: {}", t.data.gc_todo_len()?).unwrap();
+		};

-		Ok(())
+		Ok(format!(
+			"  {}\t{}\t{}\t{}\t{}",
+			F::TABLE_NAME,
+			data_len,
+			mkl_len,
+			t.merkle_updater.todo_len()?,
+			t.data.gc_todo_len()?
+		))
 	}

-	// ----
+	// ================ WORKER COMMANDS ====================

-	async fn handle_worker_cmd(&self, opt: WorkerOpt) -> Result<AdminRpc, Error> {
-		match opt.cmd {
-			WorkerCmd::List { opt } => {
-				let workers = self.garage.background.get_worker_info();
-				Ok(AdminRpc::WorkerList(workers, opt))
+	async fn handle_worker_cmd(&self, cmd: &WorkerOperation) -> Result<AdminRpc, Error> {
+		match cmd {
+			WorkerOperation::List { opt } => {
+				let workers = self.background.get_worker_info();
+				Ok(AdminRpc::WorkerList(workers, *opt))
 			}
-			WorkerCmd::Set { opt } => match opt {
-				WorkerSetCmd::ScrubTranquility { tranquility } => {
-					let scrub_command = ScrubWorkerCommand::SetTranquility(tranquility);
-					self.garage
-						.block_manager
-						.send_scrub_command(scrub_command)
-						.await;
-					Ok(AdminRpc::Ok("Scrub tranquility updated".into()))
+			WorkerOperation::Info { tid } => {
+				let info = self
+					.background
+					.get_worker_info()
+					.get(tid)
+					.ok_or_bad_request(format!("No worker with TID {}", tid))?
+					.clone();
+				Ok(AdminRpc::WorkerInfo(*tid, info))
+			}
+			WorkerOperation::Get {
+				all_nodes,
+				variable,
+			} => self.handle_get_var(*all_nodes, variable).await,
+			WorkerOperation::Set {
+				all_nodes,
+				variable,
+				value,
+			} => self.handle_set_var(*all_nodes, variable, value).await,
+		}
+	}
+
+	async fn handle_get_var(
+		&self,
+		all_nodes: bool,
+		variable: &Option<String>,
+	) -> Result<AdminRpc, Error> {
+		if all_nodes {
+			let mut ret = vec![];
+			let ring = self.garage.system.ring.borrow().clone();
+			for node in ring.layout.node_ids().iter() {
+				let node = (*node).into();
+				match self
+					.endpoint
+					.call(
+						&node,
+						AdminRpc::Worker(WorkerOperation::Get {
+							all_nodes: false,
+							variable: variable.clone(),
+						}),
+						PRIO_NORMAL,
+					)
+					.await??
+				{
+					AdminRpc::WorkerVars(v) => ret.extend(v),
+					m => return Err(GarageError::unexpected_rpc_message(m).into()),
 				}
-				WorkerSetCmd::ResyncNWorkers { n_workers } => {
-					self.garage
-						.block_manager
-						.resync
-						.set_n_workers(n_workers)
+			}
+			Ok(AdminRpc::WorkerVars(ret))
+		} else {
+			#[allow(clippy::collapsible_else_if)]
+			if let Some(v) = variable {
+				Ok(AdminRpc::WorkerVars(vec![(
+					self.garage.system.id,
+					v.clone(),
+					self.garage.bg_vars.get(v)?,
+				)]))
+			} else {
+				let mut vars = self.garage.bg_vars.get_all();
+				vars.sort();
+				Ok(AdminRpc::WorkerVars(
+					vars.into_iter()
+						.map(|(k, v)| (self.garage.system.id, k.to_string(), v))
+						.collect(),
+				))
+			}
+		}
+	}
+
+	async fn handle_set_var(
+		&self,
+		all_nodes: bool,
+		variable: &str,
+		value: &str,
+	) -> Result<AdminRpc, Error> {
+		if all_nodes {
+			let mut ret = vec![];
+			let ring = self.garage.system.ring.borrow().clone();
+			for node in ring.layout.node_ids().iter() {
+				let node = (*node).into();
+				match self
+					.endpoint
+					.call(
+						&node,
+						AdminRpc::Worker(WorkerOperation::Set {
+							all_nodes: false,
+							variable: variable.to_string(),
+							value: value.to_string(),
+						}),
+						PRIO_NORMAL,
+					)
+					.await??
+				{
+					AdminRpc::WorkerVars(v) => ret.extend(v),
+					m => return Err(GarageError::unexpected_rpc_message(m).into()),
+				}
+			}
+			Ok(AdminRpc::WorkerVars(ret))
+		} else {
+			self.garage.bg_vars.set(variable, value)?;
+			Ok(AdminRpc::WorkerVars(vec![(
+				self.garage.system.id,
+				variable.to_string(),
+				value.to_string(),
+			)]))
+		}
+	}
+
+	// ================ BLOCK COMMANDS ====================
+
+	async fn handle_block_cmd(&self, cmd: &BlockOperation) -> Result<AdminRpc, Error> {
+		match cmd {
+			BlockOperation::ListErrors => Ok(AdminRpc::BlockErrorList(
+				self.garage.block_manager.list_resync_errors()?,
+			)),
+			BlockOperation::Info { hash } => {
+				let hash = hex::decode(hash).ok_or_bad_request("invalid hash")?;
+				let hash = Hash::try_from(&hash).ok_or_bad_request("invalid hash")?;
+				let refcount = self.garage.block_manager.get_block_rc(&hash)?;
+				let block_refs = self
+					.garage
+					.block_ref_table
+					.get_range(&hash, None, None, 10000, Default::default())
+					.await?;
+				let mut versions = vec![];
+				for br in block_refs {
+					if let Some(v) = self
+						.garage
+						.version_table
+						.get(&br.version, &EmptyKey)
+						.await?
+					{
+						versions.push(Ok(v));
+					} else {
+						versions.push(Err(br.version));
+					}
+				}
+				Ok(AdminRpc::BlockInfo {
+					hash,
+					refcount,
+					versions,
+				})
+			}
+			BlockOperation::RetryNow { all, blocks } => {
+				if *all {
+					if !blocks.is_empty() {
+						return Err(Error::BadRequest(
+							"--all was specified, cannot also specify blocks".into(),
+						));
+					}
+					let blocks = self.garage.block_manager.list_resync_errors()?;
+					for b in blocks.iter() {
+						self.garage.block_manager.resync.clear_backoff(&b.hash)?;
+					}
+					Ok(AdminRpc::Ok(format!(
+						"{} blocks returned in queue for a retry now (check logs to see results)",
+						blocks.len()
+					)))
+				} else {
+					for hash in blocks {
+						let hash = hex::decode(hash).ok_or_bad_request("invalid hash")?;
+						let hash = Hash::try_from(&hash).ok_or_bad_request("invalid hash")?;
+						self.garage.block_manager.resync.clear_backoff(&hash)?;
+					}
+					Ok(AdminRpc::Ok(format!(
+						"{} blocks returned in queue for a retry now (check logs to see results)",
+						blocks.len()
+					)))
+				}
+			}
+			BlockOperation::Purge { yes, blocks } => {
+				if !yes {
+					return Err(Error::BadRequest(
+						"Pass the --yes flag to confirm block purge operation.".into(),
+					));
+				}
+
+				let mut obj_dels = 0;
+				let mut ver_dels = 0;
+
+				for hash in blocks {
+					let hash = hex::decode(hash).ok_or_bad_request("invalid hash")?;
+					let hash = Hash::try_from(&hash).ok_or_bad_request("invalid hash")?;
+					let block_refs = self
+						.garage
+						.block_ref_table
+						.get_range(&hash, None, None, 10000, Default::default())
 						.await?;
-					Ok(AdminRpc::Ok("Number of resync workers updated".into()))
+
+					for br in block_refs {
+						let version = match self
+							.garage
+							.version_table
+							.get(&br.version, &EmptyKey)
+							.await?
+						{
+							Some(v) => v,
+							None => continue,
+						};
+
+						if let Some(object) = self
+							.garage
+							.object_table
+							.get(&version.bucket_id, &version.key)
+							.await?
+						{
+							let ov = object.versions().iter().rev().find(|v| v.is_complete());
+							if let Some(ov) = ov {
+								if ov.uuid == br.version {
+									let del_uuid = gen_uuid();
+									let deleted_object = Object::new(
+										version.bucket_id,
+										version.key.clone(),
+										vec![ObjectVersion {
+											uuid: del_uuid,
+											timestamp: ov.timestamp + 1,
+											state: ObjectVersionState::Complete(
+												ObjectVersionData::DeleteMarker,
+											),
+										}],
+									);
+									self.garage.object_table.insert(&deleted_object).await?;
+									obj_dels += 1;
+								}
+							}
+						}
+
+						if !version.deleted.get() {
+							let deleted_version = Version::new(
+								version.uuid,
+								version.bucket_id,
+								version.key.clone(),
+								true,
+							);
+							self.garage.version_table.insert(&deleted_version).await?;
+							ver_dels += 1;
+						}
+					}
 				}
-				WorkerSetCmd::ResyncTranquility { tranquility } => {
-					self.garage
-						.block_manager
-						.resync
-						.set_tranquility(tranquility)
-						.await?;
-					Ok(AdminRpc::Ok("Resync tranquility updated".into()))
-				}
-			},
+				Ok(AdminRpc::Ok(format!(
+					"{} blocks were purged: {} object deletion markers added, {} versions marked deleted",
+					blocks.len(),
+					obj_dels,
+					ver_dels
+				)))
+			}
 		}
 	}
 }
@@ -923,7 +1196,8 @@ impl EndpointHandler<AdminRpc> for AdminRpcHandler {
 			AdminRpc::Migrate(opt) => self.handle_migrate(opt.clone()).await,
 			AdminRpc::LaunchRepair(opt) => self.handle_launch_repair(opt.clone()).await,
 			AdminRpc::Stats(opt) => self.handle_stats(opt.clone()).await,
-			AdminRpc::Worker(opt) => self.handle_worker_cmd(opt.clone()).await,
+			AdminRpc::Worker(wo) => self.handle_worker_cmd(wo).await,
+			AdminRpc::BlockOperation(bo) => self.handle_block_cmd(bo).await,
 			m => Err(GarageError::unexpected_rpc_message(m).into()),
 		}
 	}
@@ -41,6 +41,9 @@ pub async fn cli_command_dispatch(
 		}
 		Command::Stats(so) => cmd_admin(admin_rpc_endpoint, rpc_host, AdminRpc::Stats(so)).await,
 		Command::Worker(wo) => cmd_admin(admin_rpc_endpoint, rpc_host, AdminRpc::Worker(wo)).await,
+		Command::Block(bo) => {
+			cmd_admin(admin_rpc_endpoint, rpc_host, AdminRpc::BlockOperation(bo)).await
+		}
 		_ => unreachable!(),
 	}
 }
@@ -186,7 +189,23 @@ pub async fn cmd_admin(
 			print_key_info(&key, &rb);
 		}
 		AdminRpc::WorkerList(wi, wlo) => {
-			print_worker_info(wi, wlo);
+			print_worker_list(wi, wlo);
+		}
+		AdminRpc::WorkerVars(wv) => {
+			print_worker_vars(wv);
+		}
+		AdminRpc::WorkerInfo(tid, wi) => {
+			print_worker_info(tid, wi);
+		}
+		AdminRpc::BlockErrorList(el) => {
+			print_block_error_list(el);
+		}
+		AdminRpc::BlockInfo {
+			hash,
+			refcount,
+			versions,
+		} => {
+			print_block_info(hash, refcount, versions);
 		}
 		r => {
 			error!("Unexpected response: {:?}", r);
@@ -49,7 +49,11 @@ pub enum Command {

 	/// Manage background workers
 	#[structopt(name = "worker", version = garage_version())]
-	Worker(WorkerOpt),
+	Worker(WorkerOperation),
+
+	/// Low-level debug operations on data blocks
+	#[structopt(name = "block", version = garage_version())]
+	Block(BlockOperation),
 }

 #[derive(StructOpt, Debug)]
@@ -502,25 +506,36 @@ pub struct StatsOpt {
 	pub detailed: bool,
 }

-#[derive(Serialize, Deserialize, StructOpt, Debug, Clone)]
-pub struct WorkerOpt {
-	#[structopt(subcommand)]
-	pub cmd: WorkerCmd,
-}
-
 #[derive(Serialize, Deserialize, StructOpt, Debug, Eq, PartialEq, Clone)]
-pub enum WorkerCmd {
+pub enum WorkerOperation {
 	/// List all workers on Garage node
 	#[structopt(name = "list", version = garage_version())]
 	List {
 		#[structopt(flatten)]
 		opt: WorkerListOpt,
 	},
+	/// Get detailed information about a worker
+	#[structopt(name = "info", version = garage_version())]
+	Info { tid: usize },
+	/// Get worker parameter
+	#[structopt(name = "get", version = garage_version())]
+	Get {
+		/// Gather variable values from all nodes
+		#[structopt(short = "a", long = "all-nodes")]
+		all_nodes: bool,
+		/// Variable name to get, or none to get all variables
+		variable: Option<String>,
+	},
 	/// Set worker parameter
 	#[structopt(name = "set", version = garage_version())]
 	Set {
-		#[structopt(subcommand)]
-		opt: WorkerSetCmd,
+		/// Set variable values on all nodes
+		#[structopt(short = "a", long = "all-nodes")]
+		all_nodes: bool,
+		/// Variable node to set
+		variable: String,
+		/// Value to set the variable to
+		value: String,
 	},
 }

@@ -535,14 +550,33 @@ pub struct WorkerListOpt {
 }

 #[derive(Serialize, Deserialize, StructOpt, Debug, Eq, PartialEq, Clone)]
-pub enum WorkerSetCmd {
-	/// Set tranquility of scrub operations
-	#[structopt(name = "scrub-tranquility", version = garage_version())]
-	ScrubTranquility { tranquility: u32 },
-	/// Set number of concurrent block resync workers
-	#[structopt(name = "resync-n-workers", version = garage_version())]
-	ResyncNWorkers { n_workers: usize },
-	/// Set tranquility of block resync operations
-	#[structopt(name = "resync-tranquility", version = garage_version())]
-	ResyncTranquility { tranquility: u32 },
+pub enum BlockOperation {
+	/// List all blocks that currently have a resync error
+	#[structopt(name = "list-errors", version = garage_version())]
+	ListErrors,
+	/// Get detailed information about a single block
+	#[structopt(name = "info", version = garage_version())]
+	Info {
+		/// Hash of the block for which to retrieve information
+		hash: String,
+	},
+	/// Retry now the resync of one or many blocks
+	#[structopt(name = "retry-now", version = garage_version())]
+	RetryNow {
+		/// Retry all blocks that have a resync error
+		#[structopt(long = "all")]
+		all: bool,
+		/// Hashes of the block to retry to resync now
+		blocks: Vec<String>,
+	},
+	/// Delete all objects referencing a missing block
+	#[structopt(name = "purge", version = garage_version())]
+	Purge {
+		/// Mandatory to confirm this operation
+		#[structopt(long = "yes")]
+		yes: bool,
+		/// Hashes of the block to purge
+		#[structopt(required = true)]
+		blocks: Vec<String>,
+	},
 }
@@ -3,14 +3,17 @@ use std::time::Duration;

 use garage_util::background::*;
 use garage_util::crdt::*;
-use garage_util::data::Uuid;
+use garage_util::data::*;
 use garage_util::error::*;
 use garage_util::formater::format_table;
 use garage_util::time::*;

+use garage_block::manager::BlockResyncErrorInfo;
+
 use garage_model::bucket_table::*;
 use garage_model::key_table::*;
 use garage_model::s3::object_table::{BYTES, OBJECTS, UNFINISHED_UPLOADS};
+use garage_model::s3::version_table::Version;

 use crate::cli::structs::WorkerListOpt;

@@ -241,7 +244,7 @@ pub fn find_matching_node(
 	}
 }

-pub fn print_worker_info(wi: HashMap<usize, WorkerInfo>, wlo: WorkerListOpt) {
+pub fn print_worker_list(wi: HashMap<usize, WorkerInfo>, wlo: WorkerListOpt) {
 	let mut wi = wi.into_iter().collect::<Vec<_>>();
 	wi.sort_by_key(|(tid, info)| {
 		(
@@ -254,7 +257,7 @@ pub fn print_worker_info(wi: HashMap<usize, WorkerInfo>, wlo: WorkerListOpt) {
 		)
 	});

-	let mut table = vec![];
+	let mut table = vec!["TID\tState\tName\tTranq\tDone\tQueue\tErrors\tConsec\tLast".to_string()];
 	for (tid, info) in wi.iter() {
 		if wlo.busy && !matches!(info.state, WorkerState::Busy | WorkerState::Throttled(_)) {
 			continue;
@@ -263,33 +266,155 @@ pub fn print_worker_info(wi: HashMap<usize, WorkerInfo>, wlo: WorkerListOpt) {
 			continue;
 		}

-		table.push(format!("{}\t{}\t{}", tid, info.state, info.name));
-		if let Some(i) = &info.info {
-			table.push(format!("\t\t  {}", i));
-		}
 		let tf = timeago::Formatter::new();
-		let (err_ago, err_msg) = info
+		let err_ago = info
 			.last_error
 			.as_ref()
-			.map(|(m, t)| {
-				(
-					tf.convert(Duration::from_millis(now_msec() - t)),
-					m.as_str(),
-				)
-			})
-			.unwrap_or(("(?) ago".into(), "(?)"));
-		if info.consecutive_errors > 0 {
+			.map(|(_, t)| tf.convert(Duration::from_millis(now_msec() - t)))
+			.unwrap_or_default();
+		let (total_err, consec_err) = if info.errors > 0 {
+			(info.errors.to_string(), info.consecutive_errors.to_string())
+		} else {
+			("-".into(), "-".into())
+		};
+
+		table.push(format!(
+			"{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}",
+			tid,
+			info.state,
+			info.name,
+			info.status
+				.tranquility
+				.as_ref()
+				.map(ToString::to_string)
+				.unwrap_or_else(|| "-".into()),
+			info.status.progress.as_deref().unwrap_or("-"),
+			info.status
+				.queue_length
+				.as_ref()
+				.map(ToString::to_string)
+				.unwrap_or_else(|| "-".into()),
+			total_err,
+			consec_err,
+			err_ago,
+		));
+	}
+	format_table(table);
+}
+
+pub fn print_worker_info(tid: usize, info: WorkerInfo) {
+	let mut table = vec![];
+	table.push(format!("Task id:\t{}", tid));
+	table.push(format!("Worker name:\t{}", info.name));
+	match info.state {
+		WorkerState::Throttled(t) => {
 			table.push(format!(
-				"\t\t  {} consecutive errors ({} total), last {}",
-				info.consecutive_errors, info.errors, err_ago,
+				"Worker state:\tBusy (throttled, paused for {:.3}s)",
+				t
 			));
-			table.push(format!("\t\t  {}", err_msg));
-		} else if info.errors > 0 {
-			table.push(format!("\t\t  ({} errors, last {})", info.errors, err_ago,));
-			if wlo.errors {
-				table.push(format!("\t\t  {}", err_msg));
+		}
+		s => {
+			table.push(format!("Worker state:\t{}", s));
+		}
+	};
+	if let Some(tql) = info.status.tranquility {
+		table.push(format!("Tranquility:\t{}", tql));
+	}
+
+	table.push("".into());
+	table.push(format!("Total errors:\t{}", info.errors));
+	table.push(format!("Consecutive errs:\t{}", info.consecutive_errors));
+	if let Some((s, t)) = info.last_error {
+		table.push(format!("Last error:\t{}", s));
+		let tf = timeago::Formatter::new();
+		table.push(format!(
+			"Last error time:\t{}",
+			tf.convert(Duration::from_millis(now_msec() - t))
+		));
+	}
+
+	table.push("".into());
+	if let Some(p) = info.status.progress {
+		table.push(format!("Progress:\t{}", p));
+	}
+	if let Some(ql) = info.status.queue_length {
+		table.push(format!("Queue length:\t{}", ql));
+	}
+	if let Some(pe) = info.status.persistent_errors {
+		table.push(format!("Persistent errors:\t{}", pe));
+	}
+
+	for (i, s) in info.status.freeform.iter().enumerate() {
+		if i == 0 {
+			if table.last() != Some(&"".into()) {
+				table.push("".into());
 			}
+			table.push(format!("Message:\t{}", s));
+		} else {
+			table.push(format!("\t{}", s));
 		}
 	}
 	format_table(table);
 }
+
+pub fn print_worker_vars(wv: Vec<(Uuid, String, String)>) {
+	let table = wv
+		.into_iter()
+		.map(|(n, k, v)| format!("{:?}\t{}\t{}", n, k, v))
+		.collect::<Vec<_>>();
+	format_table(table);
+}
+
+pub fn print_block_error_list(el: Vec<BlockResyncErrorInfo>) {
+	let now = now_msec();
+	let tf = timeago::Formatter::new();
+	let mut tf2 = timeago::Formatter::new();
+	tf2.ago("");
+
+	let mut table = vec!["Hash\tRC\tErrors\tLast error\tNext try".into()];
+	for e in el {
+		table.push(format!(
+			"{}\t{}\t{}\t{}\tin {}",
+			hex::encode(e.hash.as_slice()),
+			e.refcount,
+			e.error_count,
+			tf.convert(Duration::from_millis(now - e.last_try)),
+			tf2.convert(Duration::from_millis(e.next_try - now))
+		));
+	}
+	format_table(table);
+}
+
+pub fn print_block_info(hash: Hash, refcount: u64, versions: Vec<Result<Version, Uuid>>) {
+	println!("Block hash: {}", hex::encode(hash.as_slice()));
+	println!("Refcount: {}", refcount);
+	println!();
+
+	let mut table = vec!["Version\tBucket\tKey\tDeleted".into()];
+	let mut nondeleted_count = 0;
+	for v in versions.iter() {
+		match v {
+			Ok(ver) => {
+				table.push(format!(
+					"{:?}\t{:?}\t{}\t{:?}",
+					ver.uuid,
+					ver.bucket_id,
+					ver.key,
+					ver.deleted.get()
+				));
+				if !ver.deleted.get() {
+					nondeleted_count += 1;
+				}
+			}
+			Err(vh) => {
+				table.push(format!("{:?}\t\t\tyes", vh));
+			}
+		}
+	}
+	format_table(table);
+
+	if refcount != nondeleted_count {
+		println!();
+		println!("Warning: refcount does not match number of non-deleted versions");
+	}
+}
@@ -127,9 +127,16 @@ async fn main() {
 		std::process::abort();
 	}));

+	// Parse arguments and dispatch command line
+	let opt = Opt::from_clap(&Opt::clap().version(version.as_str()).get_matches());
+
 	// Initialize logging as well as other libraries used in Garage
 	if std::env::var("RUST_LOG").is_err() {
-		std::env::set_var("RUST_LOG", "netapp=info,garage=info")
+		let default_log = match &opt.cmd {
+			Command::Server => "netapp=info,garage=info",
+			_ => "netapp=warn,garage=warn",
+		};
+		std::env::set_var("RUST_LOG", default_log)
 	}
 	tracing_subscriber::fmt()
 		.with_writer(std::io::stderr)
@@ -137,9 +144,6 @@ async fn main() {
 		.init();
 	sodiumoxide::init().expect("Unable to init sodiumoxide");

-	// Parse arguments and dispatch command line
-	let opt = Opt::from_clap(&Opt::clap().version(version.as_str()).get_matches());
-
 	let res = match opt.cmd {
 		Command::Server => server::run_server(opt.config_file).await,
 		Command::OfflineRepair(repair_opt) => {
@@ -182,9 +186,9 @@ async fn cli_command(opt: Opt) -> Result<(), Error> {
 	let netapp = NetApp::new(GARAGE_VERSION_TAG, network_key, sk);

 	// Find and parse the address of the target host
-	let (id, addr) = if let Some(h) = opt.rpc_host {
+	let (id, addr, is_default_addr) = if let Some(h) = opt.rpc_host {
 		let (id, addrs) = parse_and_resolve_peer_addr(&h).ok_or_else(|| format!("Invalid RPC remote node identifier: {}. Expected format is <pubkey>@<IP or hostname>:<port>.", h))?;
-		(id, addrs[0])
+		(id, addrs[0], false)
 	} else {
 		let node_id = garage_rpc::system::read_node_id(&config.as_ref().unwrap().metadata_dir)
 			.err_context(READ_KEY_ERROR)?;
@@ -195,24 +199,26 @@ async fn cli_command(opt: Opt) -> Result<(), Error> {
 				.ok_or_message("unable to resolve rpc_public_addr specified in config file")?
 				.next()
 				.ok_or_message("unable to resolve rpc_public_addr specified in config file")?;
-			(node_id, a)
+			(node_id, a, false)
 		} else {
 			let default_addr = SocketAddr::new(
 				"127.0.0.1".parse().unwrap(),
 				config.as_ref().unwrap().rpc_bind_addr.port(),
 			);
-			warn!(
-				"Trying to contact Garage node at default address {}",
-				default_addr
-			);
-			warn!("If this doesn't work, consider adding rpc_public_addr in your config file or specifying the -h command line parameter.");
-			(node_id, default_addr)
+			(node_id, default_addr, true)
 		}
 	};

 	// Connect to target host
-	netapp.clone().try_connect(addr, id).await
-		.err_context("Unable to connect to destination RPC host. Check that you are using the same value of rpc_secret as them, and that you have their correct public key.")?;
+	if let Err(e) = netapp.clone().try_connect(addr, id).await {
+		if is_default_addr {
+			warn!(
+				"Tried to contact Garage node at default address {}, which didn't work. If that address is wrong, consider setting rpc_public_addr in your config file.",
+				addr
+			);
+		}
+		Err(e).err_context("Unable to connect to destination RPC host. Check that you are using the same value of rpc_secret as them, and that you have their correct public key.")?;
+	}

 	let system_rpc_endpoint = netapp.endpoint::<SystemRpc, ()>(SYSTEM_RPC_PATH.into());
 	let admin_rpc_endpoint = netapp.endpoint::<AdminRpc, ()>(ADMIN_RPC_PATH.into());
@@ -1,8 +1,5 @@
 use std::path::PathBuf;

-use tokio::sync::watch;
-
-use garage_util::background::*;
 use garage_util::config::*;
 use garage_util::error::*;

@@ -20,12 +17,8 @@ pub async fn offline_repair(config_file: PathBuf, opt: OfflineRepairOpt) -> Resu
 	info!("Loading configuration...");
 	let config = read_config(config_file)?;

-	info!("Initializing background runner...");
-	let (done_tx, done_rx) = watch::channel(false);
-	let (background, await_background_done) = BackgroundRunner::new(16, done_rx);
-
 	info!("Initializing Garage main data store...");
-	let garage = Garage::new(config.clone(), background)?;
+	let garage = Garage::new(config)?;

 	info!("Launching repair operation...");
 	match opt.what {
@@ -43,13 +36,7 @@ pub async fn offline_repair(config_file: PathBuf, opt: OfflineRepairOpt) -> Resu
 		}
 	}

-	info!("Repair operation finished, shutting down Garage internals...");
-	done_tx.send(true).unwrap();
-	drop(garage);
-
-	await_background_done.await?;
-
-	info!("Cleaning up...");
+	info!("Repair operation finished, shutting down...");

 	Ok(())
 }
@@ -12,38 +12,37 @@ use garage_model::s3::version_table::*;
 use garage_table::*;
 use garage_util::background::*;
 use garage_util::error::Error;
+use garage_util::migrate::Migrate;

 use crate::*;

-pub async fn launch_online_repair(garage: Arc<Garage>, opt: RepairOpt) {
+pub async fn launch_online_repair(
+	garage: &Arc<Garage>,
+	bg: &BackgroundRunner,
+	opt: RepairOpt,
+) -> Result<(), Error> {
 	match opt.what {
 		RepairWhat::Tables => {
 			info!("Launching a full sync of tables");
-			garage.bucket_table.syncer.add_full_sync();
-			garage.object_table.syncer.add_full_sync();
-			garage.version_table.syncer.add_full_sync();
-			garage.block_ref_table.syncer.add_full_sync();
-			garage.key_table.syncer.add_full_sync();
+			garage.bucket_table.syncer.add_full_sync()?;
+			garage.object_table.syncer.add_full_sync()?;
+			garage.version_table.syncer.add_full_sync()?;
+			garage.block_ref_table.syncer.add_full_sync()?;
+			garage.key_table.syncer.add_full_sync()?;
 		}
 		RepairWhat::Versions => {
 			info!("Repairing the versions table");
-			garage
-				.background
-				.spawn_worker(RepairVersionsWorker::new(garage.clone()));
+			bg.spawn_worker(RepairVersionsWorker::new(garage.clone()));
 		}
 		RepairWhat::BlockRefs => {
 			info!("Repairing the block refs table");
-			garage
-				.background
-				.spawn_worker(RepairBlockrefsWorker::new(garage.clone()));
+			bg.spawn_worker(RepairBlockrefsWorker::new(garage.clone()));
 		}
 		RepairWhat::Blocks => {
 			info!("Repairing the stored blocks");
-			garage
-				.background
-				.spawn_worker(garage_block::repair::RepairWorker::new(
-					garage.block_manager.clone(),
-				));
+			bg.spawn_worker(garage_block::repair::RepairWorker::new(
+				garage.block_manager.clone(),
+			));
 		}
 		RepairWhat::Scrub { cmd } => {
 			let cmd = match cmd {
@@ -52,13 +51,18 @@ pub async fn launch_online_repair(garage: Arc<Garage>, opt: RepairOpt) {
 				ScrubCmd::Resume => ScrubWorkerCommand::Resume,
 				ScrubCmd::Cancel => ScrubWorkerCommand::Cancel,
 				ScrubCmd::SetTranquility { tranquility } => {
-					ScrubWorkerCommand::SetTranquility(tranquility)
+					garage
+						.block_manager
+						.scrub_persister
+						.set_with(|x| x.tranquility = tranquility)?;
+					return Ok(());
 				}
 			};
 			info!("Sending command to scrub worker: {:?}", cmd);
-			garage.block_manager.send_scrub_command(cmd).await;
+			garage.block_manager.send_scrub_command(cmd).await?;
 		}
 	}
+	Ok(())
 }

 // ----
@@ -85,25 +89,23 @@ impl Worker for RepairVersionsWorker {
 		"Version repair worker".into()
 	}

-	fn info(&self) -> Option<String> {
-		Some(format!("{} items done", self.counter))
+	fn status(&self) -> WorkerStatus {
+		WorkerStatus {
+			progress: Some(self.counter.to_string()),
+			..Default::default()
+		}
 	}

 	async fn work(&mut self, _must_exit: &mut watch::Receiver<bool>) -> Result<WorkerState, Error> {
-		let item_bytes = match self.garage.version_table.data.store.get_gt(&self.pos)? {
-			Some((k, v)) => {
-				self.pos = k;
-				v
-			}
+		let (item_bytes, next_pos) = match self.garage.version_table.data.store.get_gt(&self.pos)? {
+			Some((k, v)) => (v, k),
 			None => {
 				info!("repair_versions: finished, done {}", self.counter);
 				return Ok(WorkerState::Done);
 			}
 		};

-		self.counter += 1;
-
-		let version = rmp_serde::decode::from_read_ref::<_, Version>(&item_bytes)?;
+		let version = Version::decode(&item_bytes).ok_or_message("Cannot decode Version")?;
 		if !version.deleted.get() {
 			let object = self
 				.garage
@@ -131,10 +133,13 @@ impl Worker for RepairVersionsWorker {
 			}
 		}

+		self.counter += 1;
+		self.pos = next_pos;
+
 		Ok(WorkerState::Busy)
 	}

-	async fn wait_for_work(&mut self, _must_exit: &watch::Receiver<bool>) -> WorkerState {
+	async fn wait_for_work(&mut self) -> WorkerState {
 		unreachable!()
 	}
 }
@@ -163,25 +168,24 @@ impl Worker for RepairBlockrefsWorker {
 		"Block refs repair worker".into()
 	}

-	fn info(&self) -> Option<String> {
-		Some(format!("{} items done", self.counter))
+	fn status(&self) -> WorkerStatus {
+		WorkerStatus {
+			progress: Some(self.counter.to_string()),
+			..Default::default()
+		}
 	}

 	async fn work(&mut self, _must_exit: &mut watch::Receiver<bool>) -> Result<WorkerState, Error> {
-		let item_bytes = match self.garage.block_ref_table.data.store.get_gt(&self.pos)? {
-			Some((k, v)) => {
-				self.pos = k;
-				v
-			}
-			None => {
-				info!("repair_block_ref: finished, done {}", self.counter);
-				return Ok(WorkerState::Done);
-			}
-		};
+		let (item_bytes, next_pos) =
+			match self.garage.block_ref_table.data.store.get_gt(&self.pos)? {
+				Some((k, v)) => (v, k),
+				None => {
+					info!("repair_block_ref: finished, done {}", self.counter);
+					return Ok(WorkerState::Done);
+				}
+			};

-		self.counter += 1;
-
-		let block_ref = rmp_serde::decode::from_read_ref::<_, BlockRef>(&item_bytes)?;
+		let block_ref = BlockRef::decode(&item_bytes).ok_or_message("Cannot decode BlockRef")?;
 		if !block_ref.deleted.get() {
 			let version = self
 				.garage
@@ -206,10 +210,13 @@ impl Worker for RepairBlockrefsWorker {
 			}
 		}

+		self.counter += 1;
+		self.pos = next_pos;
+
 		Ok(WorkerState::Busy)
 	}

-	async fn wait_for_work(&mut self, _must_exit: &watch::Receiver<bool>) -> WorkerState {
+	async fn wait_for_work(&mut self) -> WorkerState {
 		unreachable!()
 	}
 }
@@ -35,12 +35,15 @@ pub async fn run_server(config_file: PathBuf) -> Result<(), Error> {
 	#[cfg(feature = "metrics")]
 	let metrics_exporter = opentelemetry_prometheus::exporter().init();

+	info!("Initializing Garage main data store...");
+	let garage = Garage::new(config.clone())?;
+
 	info!("Initializing background runner...");
 	let watch_cancel = watch_shutdown_signal();
-	let (background, await_background_done) = BackgroundRunner::new(16, watch_cancel.clone());
+	let (background, await_background_done) = BackgroundRunner::new(watch_cancel.clone());

-	info!("Initializing Garage main data store...");
-	let garage = Garage::new(config.clone(), background)?;
+	info!("Spawning Garage workers...");
+	garage.spawn_workers(&background);

 	if config.admin.trace_sink.is_some() {
 		info!("Initialize tracing...");
@@ -63,7 +66,7 @@ pub async fn run_server(config_file: PathBuf) -> Result<(), Error> {
 	let run_system = tokio::spawn(garage.system.clone().run(watch_cancel.clone()));

 	info!("Create admin RPC handler...");
-	AdminRpcHandler::new(garage.clone());
+	AdminRpcHandler::new(garage.clone(), background.clone());

 	// ---- Launch public-facing API servers ----

@@ -1,4 +1,5 @@
 use crate::common;
+use crate::common::ext::CommandExt;
 use aws_sdk_s3::model::BucketLocationConstraint;
 use aws_sdk_s3::output::DeleteBucketOutput;

@@ -8,6 +9,27 @@ async fn test_bucket_all() {
 	let bucket_name = "hello";

 	{
+		// Check bucket cannot be created if not authorized
+		ctx.garage
+			.command()
+			.args(["key", "deny"])
+			.args(["--create-bucket", &ctx.garage.key.id])
+			.quiet()
+			.expect_success_output("Could not deny key to create buckets");
+
+		// Try create bucket, should fail
+		let r = ctx.client.create_bucket().bucket(bucket_name).send().await;
+		assert!(r.is_err());
+	}
+	{
+		// Now allow key to create bucket
+		ctx.garage
+			.command()
+			.args(["key", "allow"])
+			.args(["--create-bucket", &ctx.garage.key.id])
+			.quiet()
+			.expect_success_output("Could not deny key to create buckets");
+
 		// Create bucket
 		//@TODO check with an invalid bucket name + with an already existing bucket
 		let r = ctx
@@ -1,6 +1,7 @@
 use std::collections::HashMap;

 use crate::common;
+use crate::common::ext::CommandExt;
 use common::custom_requester::BodySignature;
 use hyper::Method;

@@ -105,6 +106,13 @@ async fn test_create_bucket_streaming() {
 	let ctx = common::context();
 	let bucket = "createbucket-streaming";

+	ctx.garage
+		.command()
+		.args(["key", "allow"])
+		.args(["--create-bucket", &ctx.garage.key.id])
+		.quiet()
+		.expect_success_output("Could not allow key to create buckets");
+
 	{
 		// create bucket
 		let _ = ctx
@@ -22,7 +22,7 @@ tokio = "1.17.0"

 # cli deps
 clap = { version = "3.1.18", optional = true, features = ["derive", "env"] }
-garage_util = { version = "0.8.0", path = "../util", optional = true }
+garage_util = { version = "0.8.1", path = "../util", optional = true }


 [features]
@@ -1,6 +1,6 @@
 [package]
 name = "garage_model"
-version = "0.8.0"
+version = "0.8.1"
 authors = ["Alex Auvolat <alex@adnab.me>"]
 edition = "2018"
 license = "AGPL-3.0"
@@ -14,11 +14,11 @@ path = "lib.rs"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html

 [dependencies]
-garage_db = { version = "0.8.0", default-features = false, path = "../db" }
-garage_rpc = { version = "0.8.0", path = "../rpc" }
-garage_table = { version = "0.8.0", path = "../table" }
-garage_block = { version = "0.8.0", path = "../block" }
-garage_util = { version = "0.8.0", path = "../util" }
+garage_db = { version = "0.8.1", default-features = false, path = "../db" }
+garage_rpc = { version = "0.8.1", path = "../rpc" }
+garage_table = { version = "0.8.1", path = "../table" }
+garage_block = { version = "0.8.1", path = "../block" }
+garage_util = { version = "0.8.1", path = "../util" }

 async-trait = "0.1.7"
 arc-swap = "1.0"
@@ -30,7 +30,6 @@ tracing = "0.1.30"
 rand = "0.8"
 zstd = { version = "0.9", default-features = false }

-rmp-serde = "0.15"
 serde = { version = "1.0", default-features = false, features = ["derive", "rc"] }
 serde_bytes = "0.11"

@@ -1,18 +1,26 @@
-use serde::{Deserialize, Serialize};
-
 use garage_util::data::*;

 use garage_table::crdt::*;
 use garage_table::*;

-/// The bucket alias table holds the names given to buckets
-/// in the global namespace.
-#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
-pub struct BucketAlias {
-	name: String,
-	pub state: crdt::Lww<Option<Uuid>>,
+mod v08 {
+	use garage_util::crdt;
+	use garage_util::data::Uuid;
+	use serde::{Deserialize, Serialize};
+
+	/// The bucket alias table holds the names given to buckets
+	/// in the global namespace.
+	#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
+	pub struct BucketAlias {
+		pub(super) name: String,
+		pub state: crdt::Lww<Option<Uuid>>,
+	}
+
+	impl garage_util::migrate::InitialFormat for BucketAlias {}
 }

+pub use v08::*;
+
 impl BucketAlias {
 	pub fn new(name: String, ts: u64, bucket_id: Option<Uuid>) -> Option<Self> {
 		if !is_valid_bucket_name(&name) {
@@ -1,5 +1,3 @@
-use serde::{Deserialize, Serialize};
-
 use garage_table::crdt::*;
 use garage_table::*;
 use garage_util::data::*;
@@ -7,71 +5,82 @@ use garage_util::time::*;

 use crate::permission::BucketKeyPerm;

-/// A bucket is a collection of objects
-///
-/// Its parameters are not directly accessible as:
-///  - It must be possible to merge paramaters, hence the use of a LWW CRDT.
-///  - A bucket has 2 states, Present or Deleted and parameters make sense only if present.
-#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
-pub struct Bucket {
-	/// ID of the bucket
-	pub id: Uuid,
-	/// State, and configuration if not deleted, of the bucket
-	pub state: crdt::Deletable<BucketParams>,
+mod v08 {
+	use crate::permission::BucketKeyPerm;
+	use garage_util::crdt;
+	use garage_util::data::Uuid;
+	use serde::{Deserialize, Serialize};
+
+	/// A bucket is a collection of objects
+	///
+	/// Its parameters are not directly accessible as:
+	///  - It must be possible to merge paramaters, hence the use of a LWW CRDT.
+	///  - A bucket has 2 states, Present or Deleted and parameters make sense only if present.
+	#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
+	pub struct Bucket {
+		/// ID of the bucket
+		pub id: Uuid,
+		/// State, and configuration if not deleted, of the bucket
+		pub state: crdt::Deletable<BucketParams>,
+	}
+
+	/// Configuration for a bucket
+	#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
+	pub struct BucketParams {
+		/// Bucket's creation date
+		pub creation_date: u64,
+		/// Map of key with access to the bucket, and what kind of access they give
+		pub authorized_keys: crdt::Map<String, BucketKeyPerm>,
+
+		/// Map of aliases that are or have been given to this bucket
+		/// in the global namespace
+		/// (not authoritative: this is just used as an indication to
+		/// map back to aliases when doing ListBuckets)
+		pub aliases: crdt::LwwMap<String, bool>,
+		/// Map of aliases that are or have been given to this bucket
+		/// in namespaces local to keys
+		/// key = (access key id, alias name)
+		pub local_aliases: crdt::LwwMap<(String, String), bool>,
+
+		/// Whether this bucket is allowed for website access
+		/// (under all of its global alias names),
+		/// and if so, the website configuration XML document
+		pub website_config: crdt::Lww<Option<WebsiteConfig>>,
+		/// CORS rules
+		pub cors_config: crdt::Lww<Option<Vec<CorsRule>>>,
+		/// Bucket quotas
+		#[serde(default)]
+		pub quotas: crdt::Lww<BucketQuotas>,
+	}
+
+	#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
+	pub struct WebsiteConfig {
+		pub index_document: String,
+		pub error_document: Option<String>,
+	}
+
+	#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
+	pub struct CorsRule {
+		pub id: Option<String>,
+		pub max_age_seconds: Option<u64>,
+		pub allow_origins: Vec<String>,
+		pub allow_methods: Vec<String>,
+		pub allow_headers: Vec<String>,
+		pub expose_headers: Vec<String>,
+	}
+
+	#[derive(Default, PartialEq, Eq, PartialOrd, Ord, Clone, Debug, Serialize, Deserialize)]
+	pub struct BucketQuotas {
+		/// Maximum size in bytes (bucket size = sum of sizes of objects in the bucket)
+		pub max_size: Option<u64>,
+		/// Maximum number of non-deleted objects in the bucket
+		pub max_objects: Option<u64>,
+	}
+
+	impl garage_util::migrate::InitialFormat for Bucket {}
 }

-/// Configuration for a bucket
-#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
-pub struct BucketParams {
-	/// Bucket's creation date
-	pub creation_date: u64,
-	/// Map of key with access to the bucket, and what kind of access they give
-	pub authorized_keys: crdt::Map<String, BucketKeyPerm>,
-
-	/// Map of aliases that are or have been given to this bucket
-	/// in the global namespace
-	/// (not authoritative: this is just used as an indication to
-	/// map back to aliases when doing ListBuckets)
-	pub aliases: crdt::LwwMap<String, bool>,
-	/// Map of aliases that are or have been given to this bucket
-	/// in namespaces local to keys
-	/// key = (access key id, alias name)
-	pub local_aliases: crdt::LwwMap<(String, String), bool>,
-
-	/// Whether this bucket is allowed for website access
-	/// (under all of its global alias names),
-	/// and if so, the website configuration XML document
-	pub website_config: crdt::Lww<Option<WebsiteConfig>>,
-	/// CORS rules
-	pub cors_config: crdt::Lww<Option<Vec<CorsRule>>>,
-	/// Bucket quotas
-	#[serde(default)]
-	pub quotas: crdt::Lww<BucketQuotas>,
-}
-
-#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
-pub struct WebsiteConfig {
-	pub index_document: String,
-	pub error_document: Option<String>,
-}
-
-#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
-pub struct CorsRule {
-	pub id: Option<String>,
-	pub max_age_seconds: Option<u64>,
-	pub allow_origins: Vec<String>,
-	pub allow_methods: Vec<String>,
-	pub allow_headers: Vec<String>,
-	pub expose_headers: Vec<String>,
-}
-
-#[derive(Default, PartialEq, Eq, PartialOrd, Ord, Clone, Debug, Serialize, Deserialize)]
-pub struct BucketQuotas {
-	/// Maximum size in bytes (bucket size = sum of sizes of objects in the bucket)
-	pub max_size: Option<u64>,
-	/// Maximum number of non-deleted objects in the bucket
-	pub max_objects: Option<u64>,
-}
+pub use v08::*;

 impl AutoCrdt for BucketQuotas {
 	const WARN_IF_DIFFERENT: bool = true;
@@ -8,10 +8,10 @@ use garage_util::background::*;
 use garage_util::config::*;
 use garage_util::error::*;

+use garage_rpc::replication_mode::ReplicationMode;
 use garage_rpc::system::System;

 use garage_block::manager::*;
-use garage_table::replication::ReplicationMode;
 use garage_table::replication::TableFullReplication;
 use garage_table::replication::TableShardedReplication;
 use garage_table::*;
@@ -27,17 +27,20 @@ use crate::index_counter::*;
 use crate::key_table::*;

 #[cfg(feature = "k2v")]
-use crate::k2v::{item_table::*, poll::*, rpc::*};
+use crate::k2v::{history_table::*, item_table::*, sub::*, rpc::*};

 /// An entire Garage full of data
 pub struct Garage {
 	/// The parsed configuration Garage is running
 	pub config: Config,
+	/// The set of background variables that can be viewed/modified at runtime
+	pub bg_vars: vars::BgVars,
+
+	/// The replication mode of this cluster
+	pub replication_mode: ReplicationMode,

 	/// The local database
 	pub db: db::Db,
-	/// A background job runner
-	pub background: Arc<BackgroundRunner>,
 	/// The membership manager
 	pub system: Arc<System>,
 	/// The block manager
@@ -67,6 +70,8 @@ pub struct Garage {
 pub struct GarageK2V {
 	/// Table containing K2V items
 	pub item_table: Arc<Table<K2VItemTable, TableShardedReplication>>,
+	/// Table containing K2V modification history
+	pub history_table: Arc<Table<K2VHistoryTable, TableShardedReplication>>,
 	/// Indexing table containing K2V item counters
 	pub counter_table: Arc<IndexCounter<K2VItem>>,
 	/// K2V RPC handler
@@ -75,7 +80,7 @@ pub struct GarageK2V {

 impl Garage {
 	/// Create and run garage
-	pub fn new(config: Config, background: Arc<BackgroundRunner>) -> Result<Arc<Self>, Error> {
+	pub fn new(config: Config) -> Result<Arc<Self>, Error> {
 		// Create meta dir and data dir if they don't exist already
 		std::fs::create_dir_all(&config.metadata_dir)
 			.ok_or_message("Unable to create Garage metadata directory")?;
@@ -164,12 +169,7 @@ impl Garage {
 			.expect("Invalid replication_mode in config file.");

 		info!("Initialize membership management system...");
-		let system = System::new(
-			network_key,
-			background.clone(),
-			replication_mode.replication_factor(),
-			&config,
-		)?;
+		let system = System::new(network_key, replication_mode, &config)?;

 		let data_rep_param = TableShardedReplication {
 			system: system.clone(),
@@ -227,7 +227,6 @@ impl Garage {
 		info!("Initialize version_table...");
 		let version_table = Table::new(
 			VersionTable {
-				background: background.clone(),
 				block_ref_table: block_ref_table.clone(),
 			},
 			meta_rep_param.clone(),
@@ -242,7 +241,6 @@ impl Garage {
 		#[allow(clippy::redundant_clone)]
 		let object_table = Table::new(
 			ObjectTable {
-				background: background.clone(),
 				version_table: version_table.clone(),
 				object_counter_table: object_counter_table.clone(),
 			},
@@ -255,11 +253,16 @@ impl Garage {
 		#[cfg(feature = "k2v")]
 		let k2v = GarageK2V::new(system.clone(), &db, meta_rep_param);

+		// Initialize bg vars
+		let mut bg_vars = vars::BgVars::new();
+		block_manager.register_bg_vars(&mut bg_vars);
+
 		// -- done --
 		Ok(Arc::new(Self {
 			config,
+			bg_vars,
+			replication_mode,
 			db,
-			background,
 			system,
 			block_manager,
 			bucket_table,
@@ -274,6 +277,22 @@ impl Garage {
 		}))
 	}

+	pub fn spawn_workers(&self, bg: &BackgroundRunner) {
+		self.block_manager.spawn_workers(bg);
+
+		self.bucket_table.spawn_workers(bg);
+		self.bucket_alias_table.spawn_workers(bg);
+		self.key_table.spawn_workers(bg);
+
+		self.object_table.spawn_workers(bg);
+		self.object_counter_table.spawn_workers(bg);
+		self.version_table.spawn_workers(bg);
+		self.block_ref_table.spawn_workers(bg);
+
+		#[cfg(feature = "k2v")]
+		self.k2v.spawn_workers(bg);
+	}
+
 	pub fn bucket_helper(&self) -> helper::bucket::BucketHelper {
 		helper::bucket::BucketHelper(self)
 	}
@@ -288,24 +307,49 @@ impl GarageK2V {
 	fn new(system: Arc<System>, db: &db::Db, meta_rep_param: TableShardedReplication) -> Self {
 		info!("Initialize K2V counter table...");
 		let counter_table = IndexCounter::new(system.clone(), meta_rep_param.clone(), db);
+
 		info!("Initialize K2V subscription manager...");
 		let subscriptions = Arc::new(SubscriptionManager::new());
+
 		info!("Initialize K2V item table...");
 		let item_table = Table::new(
 			K2VItemTable {
 				counter_table: counter_table.clone(),
 				subscriptions: subscriptions.clone(),
 			},
+			meta_rep_param.clone(),
+			system.clone(),
+			db,
+		);
+		info!("Initialize K2V history table...");
+		let history_table = Table::new(
+			K2VHistoryTable {
+				subscriptions: subscriptions.clone(),
+			},
 			meta_rep_param,
 			system.clone(),
 			db,
 		);
-		let rpc = K2VRpcHandler::new(system, item_table.clone(), subscriptions);
+
+		info!("Initialize K2V RPC handler...");
+		let rpc = K2VRpcHandler::new(
+			system,
+			db,
+			item_table.clone(),
+			history_table.clone(),
+			subscriptions,
+		);

 		Self {
 			item_table,
+			history_table,
 			counter_table,
 			rpc,
 		}
 	}
+
+	pub fn spawn_workers(&self, bg: &BackgroundRunner) {
+		self.item_table.spawn_workers(bg);
+		self.counter_table.spawn_workers(bg);
+	}
 }
@@ -1,19 +1,18 @@
 use core::ops::Bound;
-use std::collections::{hash_map, BTreeMap, HashMap};
+use std::collections::{BTreeMap, HashMap};
 use std::marker::PhantomData;
 use std::sync::Arc;

-use async_trait::async_trait;
 use serde::{Deserialize, Serialize};
-use tokio::sync::{mpsc, watch};

 use garage_db as db;

 use garage_rpc::ring::Ring;
 use garage_rpc::system::System;
-use garage_util::background::*;
+use garage_util::background::BackgroundRunner;
 use garage_util::data::*;
 use garage_util::error::*;
+use garage_util::migrate::Migrate;
 use garage_util::time::*;

 use garage_table::crdt::*;
@@ -31,14 +30,44 @@ pub trait CountedItem: Clone + PartialEq + Send + Sync + 'static {
 	fn counts(&self) -> Vec<(&'static str, i64)>;
 }

-/// A counter entry in the global table
-#[derive(Clone, PartialEq, Debug, Serialize, Deserialize)]
-pub struct CounterEntry<T: CountedItem> {
-	pub pk: T::CP,
-	pub sk: T::CS,
-	pub values: BTreeMap<String, CounterValue>,
+mod v08 {
+	use super::CountedItem;
+	use garage_util::data::Uuid;
+	use serde::{Deserialize, Serialize};
+	use std::collections::BTreeMap;
+
+	// ---- Global part (the table everyone queries) ----
+
+	/// A counter entry in the global table
+	#[derive(Clone, PartialEq, Debug, Serialize, Deserialize)]
+	pub struct CounterEntry<T: CountedItem> {
+		pub pk: T::CP,
+		pub sk: T::CS,
+		pub values: BTreeMap<String, CounterValue>,
+	}
+
+	/// A counter entry in the global table
+	#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
+	pub struct CounterValue {
+		pub node_values: BTreeMap<Uuid, (u64, i64)>,
+	}
+
+	impl<T: CountedItem> garage_util::migrate::InitialFormat for CounterEntry<T> {}
+
+	// ---- Local part (the counter we maintain transactionnaly on each node) ----
+
+	#[derive(PartialEq, Clone, Debug, Serialize, Deserialize)]
+	pub(super) struct LocalCounterEntry<T: CountedItem> {
+		pub(super) pk: T::CP,
+		pub(super) sk: T::CS,
+		pub(super) values: BTreeMap<String, (u64, i64)>,
+	}
+
+	impl<T: CountedItem> garage_util::migrate::InitialFormat for LocalCounterEntry<T> {}
 }

+pub use v08::*;
+
 impl<T: CountedItem> Entry<T::CP, T::CS> for CounterEntry<T> {
 	fn partition_key(&self) -> &T::CP {
 		&self.pk
@@ -80,12 +109,6 @@ impl<T: CountedItem> CounterEntry<T> {
 	}
 }

-/// A counter entry in the global table
-#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
-pub struct CounterValue {
-	pub node_values: BTreeMap<Uuid, (u64, i64)>,
-}
-
 impl<T: CountedItem> Crdt for CounterEntry<T> {
 	fn merge(&mut self, other: &Self) {
 		for (name, e2) in other.values.iter() {
@@ -142,7 +165,6 @@ impl<T: CountedItem> TableSchema for CounterTable<T> {
 pub struct IndexCounter<T: CountedItem> {
 	this_node: Uuid,
 	local_counter: db::Tree,
-	propagate_tx: mpsc::UnboundedSender<(T::CP, T::CS, LocalCounterEntry<T>)>,
 	pub table: Arc<Table<CounterTable<T>, TableShardedReplication>>,
 }

@@ -152,16 +174,11 @@ impl<T: CountedItem> IndexCounter<T> {
 		replication: TableShardedReplication,
 		db: &db::Db,
 	) -> Arc<Self> {
-		let background = system.background.clone();
-
-		let (propagate_tx, propagate_rx) = mpsc::unbounded_channel();
-
-		let this = Arc::new(Self {
+		Arc::new(Self {
 			this_node: system.id,
 			local_counter: db
 				.open_tree(format!("local_counter_v2:{}", T::COUNTER_TABLE_NAME))
 				.expect("Unable to open local counter tree"),
-			propagate_tx,
 			table: Table::new(
 				CounterTable {
 					_phantom_t: Default::default(),
@@ -170,16 +187,11 @@ impl<T: CountedItem> IndexCounter<T> {
 				system,
 				db,
 			),
-		});
+		})
+	}

-		background.spawn_worker(IndexPropagatorWorker {
-			index_counter: this.clone(),
-			propagate_rx,
-			buf: HashMap::new(),
-			errors: 0,
-		});
-
-		this
+	pub fn spawn_workers(&self, bg: &BackgroundRunner) {
+		self.table.spawn_workers(bg);
 	}

 	pub fn count(
@@ -208,11 +220,9 @@ impl<T: CountedItem> IndexCounter<T> {
 		let tree_key = self.table.data.tree_key(pk, sk);

 		let mut entry = match tx.get(&self.local_counter, &tree_key[..])? {
-			Some(old_bytes) => {
-				rmp_serde::decode::from_read_ref::<_, LocalCounterEntry<T>>(&old_bytes)
-					.map_err(Error::RmpDecode)
-					.map_err(db::TxError::Abort)?
-			}
+			Some(old_bytes) => LocalCounterEntry::<T>::decode(&old_bytes)
+				.ok_or_message("Cannot decode local counter entry")
+				.map_err(db::TxError::Abort)?,
 			None => LocalCounterEntry {
 				pk: pk.clone(),
 				sk: sk.clone(),
@@ -227,17 +237,14 @@ impl<T: CountedItem> IndexCounter<T> {
 			ent.1 += *inc;
 		}

-		let new_entry_bytes = rmp_to_vec_all_named(&entry)
+		let new_entry_bytes = entry
+			.encode()
 			.map_err(Error::RmpEncode)
 			.map_err(db::TxError::Abort)?;
 		tx.insert(&self.local_counter, &tree_key[..], new_entry_bytes)?;

-		if let Err(e) = self.propagate_tx.send((pk.clone(), sk.clone(), entry)) {
-			error!(
-				"Could not propagate updated counter values, failed to send to channel: {}",
-				e
-			);
-		}
+		let dist_entry = entry.into_counter_entry(self.this_node);
+		self.table.queue_insert(tx, &dist_entry)?;

 		Ok(())
 	}
@@ -250,23 +257,6 @@ impl<T: CountedItem> IndexCounter<T> {
 		TS: TableSchema<E = T>,
 		TR: TableReplication,
 	{
-		let save_counter_entry = |entry: CounterEntry<T>| -> Result<(), Error> {
-			let entry_k = self
-				.table
-				.data
-				.tree_key(entry.partition_key(), entry.sort_key());
-			self.table
-				.data
-				.update_entry_with(&entry_k, |ent| match ent {
-					Some(mut ent) => {
-						ent.merge(&entry);
-						ent
-					}
-					None => entry.clone(),
-				})?;
-			Ok(())
-		};
-
 		// 1. Set all old local counters to zero
 		let now = now_msec();
 		let mut next_start: Option<Vec<u8>> = None;
@@ -289,20 +279,22 @@ impl<T: CountedItem> IndexCounter<T> {

 			info!("zeroing old counters... ({})", hex::encode(&batch[0].0));
 			for (local_counter_k, local_counter) in batch {
-				let mut local_counter =
-					rmp_serde::decode::from_read_ref::<_, LocalCounterEntry<T>>(&local_counter)?;
+				let mut local_counter = LocalCounterEntry::<T>::decode(&local_counter)
+					.ok_or_message("Cannot decode local counter entry")?;

 				for (_, tv) in local_counter.values.iter_mut() {
 					tv.0 = std::cmp::max(tv.0 + 1, now);
 					tv.1 = 0;
 				}

-				let local_counter_bytes = rmp_to_vec_all_named(&local_counter)?;
+				let local_counter_bytes = local_counter.encode()?;
 				self.local_counter
 					.insert(&local_counter_k, &local_counter_bytes)?;

 				let counter_entry = local_counter.into_counter_entry(self.this_node);
-				save_counter_entry(counter_entry)?;
+				self.local_counter
+					.db()
+					.transaction(|mut tx| self.table.queue_insert(&mut tx, &counter_entry))?;

 				next_start = Some(local_counter_k);
 			}
@@ -343,9 +335,8 @@ impl<T: CountedItem> IndexCounter<T> {
 				let local_counter_key = self.table.data.tree_key(pk, sk);
 				let mut local_counter = match self.local_counter.get(&local_counter_key)? {
 					Some(old_bytes) => {
-						let ent = rmp_serde::decode::from_read_ref::<_, LocalCounterEntry<T>>(
-							&old_bytes,
-						)?;
+						let ent = LocalCounterEntry::<T>::decode(&old_bytes)
+							.ok_or_message("Cannot decode local counter entry")?;
 						assert!(ent.pk == *pk);
 						assert!(ent.sk == *sk);
 						ent
@@ -362,12 +353,14 @@ impl<T: CountedItem> IndexCounter<T> {
 					tv.1 += v;
 				}

-				let local_counter_bytes = rmp_to_vec_all_named(&local_counter)?;
+				let local_counter_bytes = local_counter.encode()?;
 				self.local_counter
 					.insert(&local_counter_key, local_counter_bytes)?;

 				let counter_entry = local_counter.into_counter_entry(self.this_node);
-				save_counter_entry(counter_entry)?;
+				self.local_counter
+					.db()
+					.transaction(|mut tx| self.table.queue_insert(&mut tx, &counter_entry))?;

 				next_start = Some(counted_entry_k);
 			}
@@ -378,104 +371,7 @@ impl<T: CountedItem> IndexCounter<T> {
 	}
 }

-struct IndexPropagatorWorker<T: CountedItem> {
-	index_counter: Arc<IndexCounter<T>>,
-	propagate_rx: mpsc::UnboundedReceiver<(T::CP, T::CS, LocalCounterEntry<T>)>,
-
-	buf: HashMap<Vec<u8>, CounterEntry<T>>,
-	errors: usize,
-}
-
-impl<T: CountedItem> IndexPropagatorWorker<T> {
-	fn add_ent(&mut self, pk: T::CP, sk: T::CS, counters: LocalCounterEntry<T>) {
-		let tree_key = self.index_counter.table.data.tree_key(&pk, &sk);
-		let dist_entry = counters.into_counter_entry(self.index_counter.this_node);
-		match self.buf.entry(tree_key) {
-			hash_map::Entry::Vacant(e) => {
-				e.insert(dist_entry);
-			}
-			hash_map::Entry::Occupied(mut e) => {
-				e.get_mut().merge(&dist_entry);
-			}
-		}
-	}
-}
-
-#[async_trait]
-impl<T: CountedItem> Worker for IndexPropagatorWorker<T> {
-	fn name(&self) -> String {
-		format!("{} index counter propagator", T::COUNTER_TABLE_NAME)
-	}
-
-	fn info(&self) -> Option<String> {
-		if !self.buf.is_empty() {
-			Some(format!("{} items in queue", self.buf.len()))
-		} else {
-			None
-		}
-	}
-
-	async fn work(&mut self, must_exit: &mut watch::Receiver<bool>) -> Result<WorkerState, Error> {
-		// This loop batches updates to counters to be sent all at once.
-		// They are sent once the propagate_rx channel has been emptied (or is closed).
-		let closed = loop {
-			match self.propagate_rx.try_recv() {
-				Ok((pk, sk, counters)) => {
-					self.add_ent(pk, sk, counters);
-				}
-				Err(mpsc::error::TryRecvError::Empty) => break false,
-				Err(mpsc::error::TryRecvError::Disconnected) => break true,
-			}
-		};
-
-		if !self.buf.is_empty() {
-			let entries_k = self.buf.keys().take(100).cloned().collect::<Vec<_>>();
-			let entries = entries_k.iter().map(|k| self.buf.get(k).unwrap());
-			if let Err(e) = self.index_counter.table.insert_many(entries).await {
-				self.errors += 1;
-				if self.errors >= 2 && *must_exit.borrow() {
-					error!("({}) Could not propagate {} counter values: {}, these counters will not be updated correctly.", T::COUNTER_TABLE_NAME, self.buf.len(), e);
-					return Ok(WorkerState::Done);
-				}
-				// Propagate error up to worker manager, it will log it, increment a counter,
-				// and sleep for a certain delay (with exponential backoff), waiting for
-				// things to go back to normal
-				return Err(e);
-			} else {
-				for k in entries_k {
-					self.buf.remove(&k);
-				}
-				self.errors = 0;
-			}
-
-			return Ok(WorkerState::Busy);
-		} else if closed {
-			return Ok(WorkerState::Done);
-		} else {
-			return Ok(WorkerState::Idle);
-		}
-	}
-
-	async fn wait_for_work(&mut self, _must_exit: &watch::Receiver<bool>) -> WorkerState {
-		match self.propagate_rx.recv().await {
-			Some((pk, sk, counters)) => {
-				self.add_ent(pk, sk, counters);
-				WorkerState::Busy
-			}
-			None => match self.buf.is_empty() {
-				false => WorkerState::Busy,
-				true => WorkerState::Done,
-			},
-		}
-	}
-}
-
-#[derive(PartialEq, Clone, Debug, Serialize, Deserialize)]
-struct LocalCounterEntry<T: CountedItem> {
-	pk: T::CP,
-	sk: T::CS,
-	values: BTreeMap<String, (u64, i64)>,
-}
+// ----

 impl<T: CountedItem> LocalCounterEntry<T> {
 	fn into_counter_entry(self, this_node: Uuid) -> CounterEntry<T> {
@@ -0,0 +1,107 @@
+use std::sync::Arc;
+
+use garage_db as db;
+
+use garage_table::crdt::*;
+use garage_table::*;
+
+use crate::k2v::sub::*;
+
+mod v08 {
+	use crate::k2v::causality::K2VNodeId;
+	pub use crate::k2v::item_table::v08::{DvvsValue, K2VItem, K2VItemPartition};
+	use garage_util::crdt;
+	use serde::{Deserialize, Serialize};
+
+	#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
+	pub struct K2VHistoryEntry {
+		// Partition key: the partition key of ins_item
+
+		/// The inserted item
+		pub ins_item: K2VItem,
+
+		/// Sort key: the node ID and its local counter
+		pub node_counter: K2VHistorySortKey,
+
+		/// The value of the node's local counter before this entry was updated
+		pub prev_counter: u64,
+		/// The timesamp of the update (!= counter, counters are incremented
+		/// by one, timestamps are real clock timestamps)
+		pub timestamp: u64,
+
+		/// Mark this history entry for deletion
+		pub deleted: crdt::Bool,
+	}
+
+	#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
+	pub struct K2VHistorySortKey {
+		pub node: K2VNodeId,
+		pub counter: u64,
+	}
+
+	impl garage_util::migrate::InitialFormat for K2VHistoryEntry {
+		const VERSION_MARKER: &'static [u8] = b"Gk2vhe08";
+	}
+}
+
+pub use v08::*;
+
+impl Crdt for K2VHistoryEntry {
+	fn merge(&mut self, other: &Self) {
+		self.ins_item.merge(&other.ins_item);
+		self.deleted.merge(&other.deleted);
+	}
+}
+
+impl SortKey for K2VHistorySortKey {
+	type B<'a> = [u8; 16];
+
+	fn sort_key(&self) -> [u8; 16] {
+		let mut ret = [0u8; 16];
+		ret[0..8].copy_from_slice(&u64::to_be_bytes(self.node));
+		ret[8..16].copy_from_slice(&u64::to_be_bytes(self.counter));
+		ret
+	}
+}
+
+impl Entry<K2VItemPartition, K2VHistorySortKey> for K2VHistoryEntry {
+	fn partition_key(&self) -> &K2VItemPartition {
+		&self.ins_item.partition
+	}
+	fn sort_key(&self) -> &K2VHistorySortKey {
+		&self.node_counter
+	}
+	fn is_tombstone(&self) -> bool {
+		self.deleted.get()
+	}
+}
+
+pub struct K2VHistoryTable {
+	pub(crate) subscriptions: Arc<SubscriptionManager>,
+}
+
+impl TableSchema for K2VHistoryTable {
+	const TABLE_NAME: &'static str = "k2v_history";
+
+	type P = K2VItemPartition;
+	type S = K2VHistorySortKey;
+	type E = K2VHistoryEntry;
+	type Filter = DeletedFilter;
+
+	fn updated(
+		&self,
+		_tx: &mut db::Transaction,
+		_old: Option<&Self::E>,
+		new: Option<&Self::E>,
+	) -> db::TxOpResult<()> {
+		if let Some(new_ent) = new {
+			self.subscriptions.notify_range(new_ent);
+		}
+
+		Ok(())
+	}
+
+	fn matches_filter(entry: &Self::E, filter: &Self::Filter) -> bool {
+		filter.apply(entry.deleted.get())
+	}
+}
@@ -1,7 +1,8 @@
-use serde::{Deserialize, Serialize};
 use std::collections::BTreeMap;
 use std::sync::Arc;

+use serde::{Deserialize, Serialize};
+
 use garage_db as db;
 use garage_util::data::*;

@@ -10,38 +11,49 @@ use garage_table::*;

 use crate::index_counter::*;
 use crate::k2v::causality::*;
-use crate::k2v::poll::*;
+use crate::k2v::sub::*;

 pub const ENTRIES: &str = "entries";
 pub const CONFLICTS: &str = "conflicts";
 pub const VALUES: &str = "values";
 pub const BYTES: &str = "bytes";

-#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
-pub struct K2VItem {
-	pub partition: K2VItemPartition,
-	pub sort_key: String,
+pub(super) mod v08 {
+	use crate::k2v::causality::K2VNodeId;
+	use garage_util::data::Uuid;
+	use serde::{Deserialize, Serialize};
+	use std::collections::BTreeMap;

-	items: BTreeMap<K2VNodeId, DvvsEntry>,
+	#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
+	pub struct K2VItem {
+		pub partition: K2VItemPartition,
+		pub sort_key: String,
+
+		pub(super) items: BTreeMap<K2VNodeId, DvvsEntry>,
+	}
+
+	#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize, Hash)]
+	pub struct K2VItemPartition {
+		pub bucket_id: Uuid,
+		pub partition_key: String,
+	}
+
+	#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
+	pub struct DvvsEntry {
+		pub(super) t_discard: u64,
+		pub(super) values: Vec<(u64, DvvsValue)>,
+	}
+
+	#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
+	pub enum DvvsValue {
+		Value(#[serde(with = "serde_bytes")] Vec<u8>),
+		Deleted,
+	}
+
+	impl garage_util::migrate::InitialFormat for K2VItem {}
 }

-#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize, Hash)]
-pub struct K2VItemPartition {
-	pub bucket_id: Uuid,
-	pub partition_key: String,
-}
-
-#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
-struct DvvsEntry {
-	t_discard: u64,
-	values: Vec<(u64, DvvsValue)>,
-}
-
-#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
-pub enum DvvsValue {
-	Value(#[serde(with = "serde_bytes")] Vec<u8>),
-	Deleted,
-}
+pub use v08::*;

 impl K2VItem {
 	/// Creates a new K2VItem when no previous entry existed in the db
@@ -61,7 +73,8 @@ impl K2VItem {
 		this_node: Uuid,
 		context: &Option<CausalContext>,
 		new_value: DvvsValue,
-	) {
+		node_counter: u64,
+	) -> u64 {
 		if let Some(context) = context {
 			for (node, t_discard) in context.vector_clock.iter() {
 				if let Some(e) = self.items.get_mut(node) {
@@ -86,7 +99,9 @@ impl K2VItem {
 			values: vec![],
 		});
 		let t_prev = e.max_time();
-		e.values.push((t_prev + 1, new_value));
+		let t_new = std::cmp::max(node_counter + 1, t_prev + 1);
+		e.values.push((t_new, new_value));
+		t_new
 	}

 	/// Extract the causality context of a K2V Item
@@ -225,7 +240,7 @@ impl TableSchema for K2VItemTable {

 		// 2. Notify
 		if let Some(new_ent) = new {
-			self.subscriptions.notify(new_ent);
+			self.subscriptions.notify_item(new_ent);
 		}

 		Ok(())
@@ -1,6 +1,8 @@
 pub mod causality;

+pub mod history_table;
 pub mod item_table;

-pub mod poll;
+pub(crate) mod sub;
+
 pub mod rpc;
@@ -1,50 +0,0 @@
-use std::collections::HashMap;
-use std::sync::Mutex;
-
-use serde::{Deserialize, Serialize};
-use tokio::sync::broadcast;
-
-use crate::k2v::item_table::*;
-
-#[derive(Debug, Hash, Clone, PartialEq, Eq, Serialize, Deserialize)]
-pub struct PollKey {
-	pub partition: K2VItemPartition,
-	pub sort_key: String,
-}
-
-#[derive(Default)]
-pub struct SubscriptionManager {
-	subscriptions: Mutex<HashMap<PollKey, broadcast::Sender<K2VItem>>>,
-}
-
-impl SubscriptionManager {
-	pub fn new() -> Self {
-		Self::default()
-	}
-
-	pub fn subscribe(&self, key: &PollKey) -> broadcast::Receiver<K2VItem> {
-		let mut subs = self.subscriptions.lock().unwrap();
-		if let Some(s) = subs.get(key) {
-			s.subscribe()
-		} else {
-			let (tx, rx) = broadcast::channel(8);
-			subs.insert(key.clone(), tx);
-			rx
-		}
-	}
-
-	pub fn notify(&self, item: &K2VItem) {
-		let key = PollKey {
-			partition: item.partition.clone(),
-			sort_key: item.sort_key.clone(),
-		};
-		let mut subs = self.subscriptions.lock().unwrap();
-		if let Some(s) = subs.get(&key) {
-			if s.send(item.clone()).is_err() {
-				// no more subscribers, remove channel from here
-				// (we will re-create it later if we need to subscribe again)
-				subs.remove(&key);
-			}
-		}
-	}
-}
@@ -6,6 +6,7 @@
 //! mean the vector clock gets much larger than needed).

 use std::collections::HashMap;
+use std::convert::TryInto;
 use std::sync::Arc;
 use std::time::Duration;

@@ -15,9 +16,12 @@ use futures::StreamExt;
 use serde::{Deserialize, Serialize};
 use tokio::select;

+use garage_db as db;
+
 use garage_util::crdt::*;
 use garage_util::data::*;
 use garage_util::error::*;
+use garage_util::time::*;

 use garage_rpc::system::System;
 use garage_rpc::*;
@@ -26,8 +30,9 @@ use garage_table::replication::{TableReplication, TableShardedReplication};
 use garage_table::{PartitionKey, Table};

 use crate::k2v::causality::*;
+use crate::k2v::history_table::*;
 use crate::k2v::item_table::*;
-use crate::k2v::poll::*;
+use crate::k2v::sub::*;

 /// RPC messages for K2V
 #[derive(Debug, Serialize, Deserialize)]
@@ -59,6 +64,8 @@ impl Rpc for K2VRpc {
 pub struct K2VRpcHandler {
 	system: Arc<System>,
 	item_table: Arc<Table<K2VItemTable, TableShardedReplication>>,
+	history_table: Arc<Table<K2VHistoryTable, TableShardedReplication>>,
+	local_counter_tree: db::Tree,
 	endpoint: Arc<Endpoint<K2VRpc, Self>>,
 	subscriptions: Arc<SubscriptionManager>,
 }
@@ -66,14 +73,21 @@ pub struct K2VRpcHandler {
 impl K2VRpcHandler {
 	pub fn new(
 		system: Arc<System>,
+		db: &db::Db,
 		item_table: Arc<Table<K2VItemTable, TableShardedReplication>>,
+		history_table: Arc<Table<K2VHistoryTable, TableShardedReplication>>,
 		subscriptions: Arc<SubscriptionManager>,
 	) -> Arc<Self> {
+		let local_counter_tree = db
+			.open_tree("k2v_local_counter")
+			.expect("Unable to open DB tree for k2v local counter");
 		let endpoint = system.netapp.endpoint("garage_model/k2v/Rpc".to_string());

 		let rpc_handler = Arc::new(Self {
 			system,
 			item_table,
+			history_table,
+			local_counter_tree,
 			endpoint,
 			subscriptions,
 		});
@@ -181,7 +195,7 @@ impl K2VRpcHandler {
 		Ok(())
 	}

-	pub async fn poll(
+	pub async fn poll_item(
 		&self,
 		bucket_id: Uuid,
 		partition_key: String,
@@ -273,14 +287,18 @@ impl K2VRpcHandler {
 	}

 	fn local_insert(&self, item: &InsertedItem) -> Result<Option<K2VItem>, Error> {
-		let tree_key = self
-			.item_table
-			.data
-			.tree_key(&item.partition, &item.sort_key);
+		let now = now_msec();

 		self.item_table
 			.data
-			.update_entry_with(&tree_key[..], |ent| {
+			.update_entry_with(&item.partition, &item.sort_key, |tx, ent| {
+				let local_counter_key = item.partition.hash();
+				let old_local_counter = tx
+					.get(&self.local_counter_tree, &local_counter_key)?
+					.and_then(|x| x.try_into().ok())
+					.map(u64::from_be_bytes)
+					.unwrap_or_default();
+
 				let mut ent = ent.unwrap_or_else(|| {
 					K2VItem::new(
 						item.partition.bucket_id,
@@ -288,13 +306,37 @@ impl K2VRpcHandler {
 						item.sort_key.clone(),
 					)
 				});
-				ent.update(self.system.id, &item.causal_context, item.value.clone());
-				ent
+				let new_local_counter = ent.update(
+					self.system.id,
+					&item.causal_context,
+					item.value.clone(),
+					old_local_counter,
+				);
+
+				tx.insert(
+					&self.local_counter_tree,
+					&local_counter_key,
+					u64::to_be_bytes(new_local_counter),
+				)?;
+
+				let hist_entry = K2VHistoryEntry {
+					ins_item: ent.clone(),
+					node_counter: K2VHistorySortKey {
+						node: make_node_id(self.system.id),
+						counter: new_local_counter,
+					},
+					prev_counter: old_local_counter,
+					timestamp: now,
+					deleted: false.into(),
+				};
+				self.history_table.queue_insert(tx, &hist_entry)?;
+
+				Ok(ent)
 			})
 	}

-	async fn handle_poll(&self, key: &PollKey, ct: &CausalContext) -> Result<K2VItem, Error> {
-		let mut chan = self.subscriptions.subscribe(key);
+	async fn handle_poll_item(&self, key: &PollKey, ct: &CausalContext) -> Result<K2VItem, Error> {
+		let mut chan = self.subscriptions.subscribe_item(key);

 		let mut value = self
 			.item_table
@@ -331,7 +373,7 @@ impl EndpointHandler<K2VRpc> for K2VRpcHandler {
 			} => {
 				let delay = tokio::time::sleep(Duration::from_millis(*timeout_msec));
 				select! {
-					ret = self.handle_poll(key, causal_context) => ret.map(Some).map(K2VRpc::PollItemResponse),
+					ret = self.handle_poll_item(key, causal_context) => ret.map(Some).map(K2VRpc::PollItemResponse),
 					_ = delay => Ok(K2VRpc::PollItemResponse(None)),
 				}
 			}
@@ -0,0 +1,115 @@
+use std::collections::HashMap;
+use std::sync::Mutex;
+
+use serde::{Deserialize, Serialize};
+use tokio::sync::broadcast;
+
+use crate::k2v::history_table::*;
+use crate::k2v::item_table::*;
+
+#[derive(Debug, Hash, Clone, PartialEq, Eq, Serialize, Deserialize)]
+pub struct PollKey {
+	pub partition: K2VItemPartition,
+	pub sort_key: String,
+}
+
+#[derive(Debug, Hash, Clone, PartialEq, Eq, Serialize, Deserialize)]
+pub struct PollRange {
+	pub partition: K2VItemPartition,
+	pub prefix: Option<String>,
+	pub start: Option<String>,
+	pub end: Option<String>,
+}
+
+#[derive(Default)]
+pub struct SubscriptionManager {
+	item_subscriptions: Mutex<HashMap<PollKey, broadcast::Sender<K2VItem>>>,
+	range_subscriptions: Mutex<HashMap<PollRange, broadcast::Sender<K2VHistoryEntry>>>,
+}
+
+impl SubscriptionManager {
+	pub fn new() -> Self {
+		Self::default()
+	}
+
+	// ---- simple item polling ----
+
+	pub fn subscribe_item(&self, key: &PollKey) -> broadcast::Receiver<K2VItem> {
+		let mut subs = self.item_subscriptions.lock().unwrap();
+		if let Some(s) = subs.get(key) {
+			s.subscribe()
+		} else {
+			let (tx, rx) = broadcast::channel(8);
+			subs.insert(key.clone(), tx);
+			rx
+		}
+	}
+
+	pub fn notify_item(&self, item: &K2VItem) {
+		let key = PollKey {
+			partition: item.partition.clone(),
+			sort_key: item.sort_key.clone(),
+		};
+		let mut subs = self.item_subscriptions.lock().unwrap();
+		if let Some(s) = subs.get(&key) {
+			if s.send(item.clone()).is_err() {
+				// no more subscribers, remove channel from here
+				// (we will re-create it later if we need to subscribe again)
+				subs.remove(&key);
+			}
+		}
+	}
+
+	// ---- range polling ----
+
+	pub fn subscribe_range(&self, key: &PollRange) -> broadcast::Receiver<K2VHistoryEntry> {
+		let mut subs = self.range_subscriptions.lock().unwrap();
+		if let Some(s) = subs.get(key) {
+			s.subscribe()
+		} else {
+			let (tx, rx) = broadcast::channel(8);
+			subs.insert(key.clone(), tx);
+			rx
+		}
+	}
+
+	pub fn notify_range(&self, entry: &K2VHistoryEntry) {
+		let mut subs = self.range_subscriptions.lock().unwrap();
+		let mut dead_subs = vec![];
+
+		for (sub, chan) in subs.iter() {
+			if sub.matches(&entry) {
+				if chan.send(entry.clone()).is_err() {
+					dead_subs.push(sub.clone());
+				}
+			} else if chan.receiver_count() == 0 {
+				dead_subs.push(sub.clone());
+			}
+		}
+
+		for sub in dead_subs.iter() {
+			subs.remove(sub);
+		}
+	}
+}
+
+impl PollRange {
+	fn matches(&self, entry: &K2VHistoryEntry) -> bool {
+		entry.ins_item.partition == self.partition
+			&& self
+				.prefix
+				.as_ref()
+				.map(|x| entry.ins_item.sort_key.starts_with(x))
+				.unwrap_or(true)
+			&& self
+				.start
+				.as_ref()
+				.map(|x| entry.ins_item.sort_key >= *x)
+				.unwrap_or(true)
+			&& self
+				.end
+				.as_ref()
+				.map(|x| entry.ins_item.sort_key < *x)
+				.unwrap_or(true)
+	}
+}
@@ -1,45 +1,121 @@
 use serde::{Deserialize, Serialize};

-use garage_table::crdt::*;
-use garage_table::*;
+use garage_util::crdt::{self, Crdt};
 use garage_util::data::*;

+use garage_table::{DeletedFilter, EmptyKey, Entry, TableSchema};
+
 use crate::permission::BucketKeyPerm;

-use crate::prev::v051::key_table as old;
+pub(crate) mod v05 {
+	use garage_util::crdt;
+	use serde::{Deserialize, Serialize};

-/// An api key
-#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
-pub struct Key {
-	/// The id of the key (immutable), used as partition key
-	pub key_id: String,
+	/// An api key
+	#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
+	pub struct Key {
+		/// The id of the key (immutable), used as partition key
+		pub key_id: String,

-	/// Internal state of the key
-	pub state: crdt::Deletable<KeyParams>,
+		/// The secret_key associated
+		pub secret_key: String,
+
+		/// Name for the key
+		pub name: crdt::Lww<String>,
+
+		/// Is the key deleted
+		pub deleted: crdt::Bool,
+
+		/// Buckets in which the key is authorized. Empty if `Key` is deleted
+		// CRDT interaction: deleted implies authorized_buckets is empty
+		pub authorized_buckets: crdt::LwwMap<String, PermissionSet>,
+	}
+
+	/// Permission given to a key in a bucket
+	#[derive(PartialOrd, Ord, PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
+	pub struct PermissionSet {
+		/// The key can be used to read the bucket
+		pub allow_read: bool,
+		/// The key can be used to write in the bucket
+		pub allow_write: bool,
+	}
+
+	impl crdt::AutoCrdt for PermissionSet {
+		const WARN_IF_DIFFERENT: bool = true;
+	}
+
+	impl garage_util::migrate::InitialFormat for Key {}
 }

-/// Configuration for a key
-#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
-pub struct KeyParams {
-	/// The secret_key associated (immutable)
-	pub secret_key: String,
+mod v08 {
+	use super::v05;
+	use crate::permission::BucketKeyPerm;
+	use garage_util::crdt;
+	use garage_util::data::Uuid;
+	use serde::{Deserialize, Serialize};

-	/// Name for the key
-	pub name: crdt::Lww<String>,
+	/// An api key
+	#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
+	pub struct Key {
+		/// The id of the key (immutable), used as partition key
+		pub key_id: String,

-	/// Flag to allow users having this key to create buckets
-	pub allow_create_bucket: crdt::Lww<bool>,
+		/// Internal state of the key
+		pub state: crdt::Deletable<KeyParams>,
+	}

-	/// If the key is present: it gives some permissions,
-	/// a map of bucket IDs (uuids) to permissions.
-	/// Otherwise no permissions are granted to key
-	pub authorized_buckets: crdt::Map<Uuid, BucketKeyPerm>,
+	/// Configuration for a key
+	#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
+	pub struct KeyParams {
+		/// The secret_key associated (immutable)
+		pub secret_key: String,

-	/// A key can have a local view of buckets names it is
-	/// the only one to see, this is the namespace for these aliases
-	pub local_aliases: crdt::LwwMap<String, Option<Uuid>>,
+		/// Name for the key
+		pub name: crdt::Lww<String>,
+
+		/// Flag to allow users having this key to create buckets
+		pub allow_create_bucket: crdt::Lww<bool>,
+
+		/// If the key is present: it gives some permissions,
+		/// a map of bucket IDs (uuids) to permissions.
+		/// Otherwise no permissions are granted to key
+		pub authorized_buckets: crdt::Map<Uuid, BucketKeyPerm>,
+
+		/// A key can have a local view of buckets names it is
+		/// the only one to see, this is the namespace for these aliases
+		pub local_aliases: crdt::LwwMap<String, Option<Uuid>>,
+	}
+
+	impl garage_util::migrate::Migrate for Key {
+		type Previous = v05::Key;
+
+		fn migrate(old_k: v05::Key) -> Key {
+			let name = crdt::Lww::raw(old_k.name.timestamp(), old_k.name.get().clone());
+
+			let state = if old_k.deleted.get() {
+				crdt::Deletable::Deleted
+			} else {
+				// Authorized buckets is ignored here,
+				// migration is performed in specific migration code in
+				// garage/migrate.rs
+				crdt::Deletable::Present(KeyParams {
+					secret_key: old_k.secret_key,
+					name,
+					allow_create_bucket: crdt::Lww::new(false),
+					authorized_buckets: crdt::Map::new(),
+					local_aliases: crdt::LwwMap::new(),
+				})
+			};
+			Key {
+				key_id: old_k.key_id,
+				state,
+			}
+		}
+	}
 }

+pub use v08::*;
+
 impl KeyParams {
 	fn new(secret_key: &str, name: &str) -> Self {
 		KeyParams {
@@ -173,28 +249,4 @@ impl TableSchema for KeyTable {
 			}
 		}
 	}
-
-	fn try_migrate(bytes: &[u8]) -> Option<Self::E> {
-		let old_k = rmp_serde::decode::from_read_ref::<_, old::Key>(bytes).ok()?;
-		let name = crdt::Lww::raw(old_k.name.timestamp(), old_k.name.get().clone());
-
-		let state = if old_k.deleted.get() {
-			crdt::Deletable::Deleted
-		} else {
-			// Authorized buckets is ignored here,
-			// migration is performed in specific migration code in
-			// garage/migrate.rs
-			crdt::Deletable::Present(KeyParams {
-				secret_key: old_k.secret_key,
-				name,
-				allow_create_bucket: crdt::Lww::new(false),
-				authorized_buckets: crdt::Map::new(),
-				local_aliases: crdt::LwwMap::new(),
-			})
-		};
-		Some(Key {
-			key_id: old_k.key_id,
-			state,
-		})
-	}
 }
@@ -2,6 +2,7 @@ use std::sync::Arc;

 use garage_util::crdt::*;
 use garage_util::data::*;
+use garage_util::encode::nonversioned_decode;
 use garage_util::error::Error as GarageError;
 use garage_util::time::*;

@@ -28,8 +29,8 @@ impl Migrate {
 		let mut old_buckets = vec![];
 		for res in tree.iter().map_err(GarageError::from)? {
 			let (_k, v) = res.map_err(GarageError::from)?;
-			let bucket = rmp_serde::decode::from_read_ref::<_, old_bucket::Bucket>(&v[..])
-				.map_err(GarageError::from)?;
+			let bucket =
+				nonversioned_decode::<old_bucket::Bucket>(&v[..]).map_err(GarageError::from)?;
 			old_buckets.push(bucket);
 		}

@@ -3,7 +3,7 @@ use serde::{Deserialize, Serialize};
 use garage_table::crdt::Crdt;
 use garage_table::*;

-use super::key_table::PermissionSet;
+use crate::key_table::v05::PermissionSet;

 /// A bucket is a collection of objects
 ///
@@ -1,50 +0,0 @@
-use serde::{Deserialize, Serialize};
-
-use garage_table::crdt::*;
-use garage_table::*;
-
-/// An api key
-#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
-pub struct Key {
-	/// The id of the key (immutable), used as partition key
-	pub key_id: String,
-
-	/// The secret_key associated
-	pub secret_key: String,
-
-	/// Name for the key
-	pub name: crdt::Lww<String>,
-
-	/// Is the key deleted
-	pub deleted: crdt::Bool,
-
-	/// Buckets in which the key is authorized. Empty if `Key` is deleted
-	// CRDT interaction: deleted implies authorized_buckets is empty
-	pub authorized_buckets: crdt::LwwMap<String, PermissionSet>,
-}
-
-/// Permission given to a key in a bucket
-#[derive(PartialOrd, Ord, PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
-pub struct PermissionSet {
-	/// The key can be used to read the bucket
-	pub allow_read: bool,
-	/// The key can be used to write in the bucket
-	pub allow_write: bool,
-}
-
-impl AutoCrdt for PermissionSet {
-	const WARN_IF_DIFFERENT: bool = true;
-}
-
-impl Crdt for Key {
-	fn merge(&mut self, other: &Self) {
-		self.name.merge(&other.name);
-		self.deleted.merge(&other.deleted);
-
-		if self.deleted.get() {
-			self.authorized_buckets.clear();
-		} else {
-			self.authorized_buckets.merge(&other.authorized_buckets);
-		}
-	}
-}
@@ -1,4 +1 @@
 pub(crate) mod bucket_table;
-pub(crate) mod key_table;
-pub(crate) mod object_table;
-pub(crate) mod version_table;
@@ -1,149 +0,0 @@
-use serde::{Deserialize, Serialize};
-use std::collections::BTreeMap;
-
-use garage_util::data::*;
-
-use garage_table::crdt::*;
-
-/// An object
-#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
-pub struct Object {
-	/// The bucket in which the object is stored, used as partition key
-	pub bucket: String,
-
-	/// The key at which the object is stored in its bucket, used as sorting key
-	pub key: String,
-
-	/// The list of currenty stored versions of the object
-	versions: Vec<ObjectVersion>,
-}
-
-impl Object {
-	/// Get a list of currently stored versions of `Object`
-	pub fn versions(&self) -> &[ObjectVersion] {
-		&self.versions[..]
-	}
-}
-
-/// Informations about a version of an object
-#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
-pub struct ObjectVersion {
-	/// Id of the version
-	pub uuid: Uuid,
-	/// Timestamp of when the object was created
-	pub timestamp: u64,
-	/// State of the version
-	pub state: ObjectVersionState,
-}
-
-/// State of an object version
-#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
-pub enum ObjectVersionState {
-	/// The version is being received
-	Uploading(ObjectVersionHeaders),
-	/// The version is fully received
-	Complete(ObjectVersionData),
-	/// The version uploaded containded errors or the upload was explicitly aborted
-	Aborted,
-}
-
-impl Crdt for ObjectVersionState {
-	fn merge(&mut self, other: &Self) {
-		use ObjectVersionState::*;
-		match other {
-			Aborted => {
-				*self = Aborted;
-			}
-			Complete(b) => match self {
-				Aborted => {}
-				Complete(a) => {
-					a.merge(b);
-				}
-				Uploading(_) => {
-					*self = Complete(b.clone());
-				}
-			},
-			Uploading(_) => {}
-		}
-	}
-}
-
-/// Data stored in object version
-#[derive(PartialEq, Eq, PartialOrd, Ord, Clone, Debug, Serialize, Deserialize)]
-pub enum ObjectVersionData {
-	/// The object was deleted, this Version is a tombstone to mark it as such
-	DeleteMarker,
-	/// The object is short, it's stored inlined
-	Inline(ObjectVersionMeta, #[serde(with = "serde_bytes")] Vec<u8>),
-	/// The object is not short, Hash of first block is stored here, next segments hashes are
-	/// stored in the version table
-	FirstBlock(ObjectVersionMeta, Hash),
-}
-
-impl AutoCrdt for ObjectVersionData {
-	const WARN_IF_DIFFERENT: bool = true;
-}
-
-/// Metadata about the object version
-#[derive(PartialEq, Eq, PartialOrd, Ord, Clone, Debug, Serialize, Deserialize)]
-pub struct ObjectVersionMeta {
-	/// Headers to send to the client
-	pub headers: ObjectVersionHeaders,
-	/// Size of the object
-	pub size: u64,
-	/// etag of the object
-	pub etag: String,
-}
-
-/// Additional headers for an object
-#[derive(PartialEq, Eq, PartialOrd, Ord, Clone, Debug, Serialize, Deserialize)]
-pub struct ObjectVersionHeaders {
-	/// Content type of the object
-	pub content_type: String,
-	/// Any other http headers to send
-	pub other: BTreeMap<String, String>,
-}
-
-impl ObjectVersion {
-	fn cmp_key(&self) -> (u64, Uuid) {
-		(self.timestamp, self.uuid)
-	}
-
-	/// Is the object version completely received
-	pub fn is_complete(&self) -> bool {
-		matches!(self.state, ObjectVersionState::Complete(_))
-	}
-}
-
-impl Crdt for Object {
-	fn merge(&mut self, other: &Self) {
-		// Merge versions from other into here
-		for other_v in other.versions.iter() {
-			match self
-				.versions
-				.binary_search_by(|v| v.cmp_key().cmp(&other_v.cmp_key()))
-			{
-				Ok(i) => {
-					self.versions[i].state.merge(&other_v.state);
-				}
-				Err(i) => {
-					self.versions.insert(i, other_v.clone());
-				}
-			}
-		}
-
-		// Remove versions which are obsolete, i.e. those that come
-		// before the last version which .is_complete().
-		let last_complete = self
-			.versions
-			.iter()
-			.enumerate()
-			.rev()
-			.find(|(_, v)| v.is_complete())
-			.map(|(vi, _)| vi);
-
-		if let Some(last_vi) = last_complete {
-			self.versions = self.versions.drain(last_vi..).collect::<Vec<_>>();
-		}
-	}
-}
@@ -1,79 +0,0 @@
-use serde::{Deserialize, Serialize};
-
-use garage_util::data::*;
-
-use garage_table::crdt::*;
-use garage_table::*;
-
-/// A version of an object
-#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
-pub struct Version {
-	/// UUID of the version, used as partition key
-	pub uuid: Uuid,
-
-	// Actual data: the blocks for this version
-	// In the case of a multipart upload, also store the etags
-	// of individual parts and check them when doing CompleteMultipartUpload
-	/// Is this version deleted
-	pub deleted: crdt::Bool,
-	/// list of blocks of data composing the version
-	pub blocks: crdt::Map<VersionBlockKey, VersionBlock>,
-	/// Etag of each part in case of a multipart upload, empty otherwise
-	pub parts_etags: crdt::Map<u64, String>,
-
-	// Back link to bucket+key so that we can figure if
-	// this was deleted later on
-	/// Bucket in which the related object is stored
-	pub bucket: String,
-	/// Key in which the related object is stored
-	pub key: String,
-}
-
-#[derive(PartialEq, Eq, Clone, Copy, Debug, Serialize, Deserialize)]
-pub struct VersionBlockKey {
-	/// Number of the part
-	pub part_number: u64,
-	/// Offset of this sub-segment in its part
-	pub offset: u64,
-}
-
-impl Ord for VersionBlockKey {
-	fn cmp(&self, other: &Self) -> std::cmp::Ordering {
-		self.part_number
-			.cmp(&other.part_number)
-			.then(self.offset.cmp(&other.offset))
-	}
-}
-
-impl PartialOrd for VersionBlockKey {
-	fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
-		Some(self.cmp(other))
-	}
-}
-
-/// Informations about a single block
-#[derive(PartialEq, Eq, Ord, PartialOrd, Clone, Copy, Debug, Serialize, Deserialize)]
-pub struct VersionBlock {
-	/// Blake2 sum of the block
-	pub hash: Hash,
-	/// Size of the block
-	pub size: u64,
-}
-
-impl AutoCrdt for VersionBlock {
-	const WARN_IF_DIFFERENT: bool = true;
-}
-
-impl Crdt for Version {
-	fn merge(&mut self, other: &Self) {
-		self.deleted.merge(&other.deleted);
-
-		if self.deleted.get() {
-			self.blocks.clear();
-			self.parts_etags.clear();
-		} else {
-			self.blocks.merge(&other.blocks);
-			self.parts_etags.merge(&other.parts_etags);
-		}
-	}
-}
@@ -1,4 +1,3 @@
-use serde::{Deserialize, Serialize};
 use std::sync::Arc;

 use garage_db as db;
@@ -10,19 +9,29 @@ use garage_table::*;

 use garage_block::manager::*;

-#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
-pub struct BlockRef {
-	/// Hash (blake2 sum) of the block, used as partition key
-	pub block: Hash,
+mod v08 {
+	use garage_util::crdt;
+	use garage_util::data::{Hash, Uuid};
+	use serde::{Deserialize, Serialize};

-	/// Id of the Version for the object containing this block, used as sorting key
-	pub version: Uuid,
+	#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
+	pub struct BlockRef {
+		/// Hash (blake2 sum) of the block, used as partition key
+		pub block: Hash,

-	// Keep track of deleted status
-	/// Is the Version that contains this block deleted
-	pub deleted: crdt::Bool,
+		/// Id of the Version for the object containing this block, used as sorting key
+		pub version: Uuid,
+
+		// Keep track of deleted status
+		/// Is the Version that contains this block deleted
+		pub deleted: crdt::Bool,
+	}
+
+	impl garage_util::migrate::InitialFormat for BlockRef {}
 }

+pub use v08::*;
+
 impl Entry<Hash, Uuid> for BlockRef {
 	fn partition_key(&self) -> &Hash {
 		&self.block
@@ -1,10 +1,8 @@
 use serde::{Deserialize, Serialize};
-use std::collections::BTreeMap;
 use std::sync::Arc;

 use garage_db as db;

-use garage_util::background::BackgroundRunner;
 use garage_util::data::*;

 use garage_table::crdt::*;
@@ -14,25 +12,126 @@ use garage_table::*;
 use crate::index_counter::*;
 use crate::s3::version_table::*;

-use crate::prev::v051::object_table as old;
-
 pub const OBJECTS: &str = "objects";
 pub const UNFINISHED_UPLOADS: &str = "unfinished_uploads";
 pub const BYTES: &str = "bytes";

-/// An object
-#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
-pub struct Object {
-	/// The bucket in which the object is stored, used as partition key
-	pub bucket_id: Uuid,
+mod v05 {
+	use garage_util::data::{Hash, Uuid};
+	use serde::{Deserialize, Serialize};
+	use std::collections::BTreeMap;

-	/// The key at which the object is stored in its bucket, used as sorting key
-	pub key: String,
+	/// An object
+	#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
+	pub struct Object {
+		/// The bucket in which the object is stored, used as partition key
+		pub bucket: String,

-	/// The list of currenty stored versions of the object
-	versions: Vec<ObjectVersion>,
+		/// The key at which the object is stored in its bucket, used as sorting key
+		pub key: String,
+
+		/// The list of currenty stored versions of the object
+		pub(super) versions: Vec<ObjectVersion>,
+	}
+
+	/// Informations about a version of an object
+	#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
+	pub struct ObjectVersion {
+		/// Id of the version
+		pub uuid: Uuid,
+		/// Timestamp of when the object was created
+		pub timestamp: u64,
+		/// State of the version
+		pub state: ObjectVersionState,
+	}
+
+	/// State of an object version
+	#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
+	pub enum ObjectVersionState {
+		/// The version is being received
+		Uploading(ObjectVersionHeaders),
+		/// The version is fully received
+		Complete(ObjectVersionData),
+		/// The version uploaded containded errors or the upload was explicitly aborted
+		Aborted,
+	}
+
+	/// Data stored in object version
+	#[derive(PartialEq, Eq, PartialOrd, Ord, Clone, Debug, Serialize, Deserialize)]
+	pub enum ObjectVersionData {
+		/// The object was deleted, this Version is a tombstone to mark it as such
+		DeleteMarker,
+		/// The object is short, it's stored inlined
+		Inline(ObjectVersionMeta, #[serde(with = "serde_bytes")] Vec<u8>),
+		/// The object is not short, Hash of first block is stored here, next segments hashes are
+		/// stored in the version table
+		FirstBlock(ObjectVersionMeta, Hash),
+	}
+
+	/// Metadata about the object version
+	#[derive(PartialEq, Eq, PartialOrd, Ord, Clone, Debug, Serialize, Deserialize)]
+	pub struct ObjectVersionMeta {
+		/// Headers to send to the client
+		pub headers: ObjectVersionHeaders,
+		/// Size of the object
+		pub size: u64,
+		/// etag of the object
+		pub etag: String,
+	}
+
+	/// Additional headers for an object
+	#[derive(PartialEq, Eq, PartialOrd, Ord, Clone, Debug, Serialize, Deserialize)]
+	pub struct ObjectVersionHeaders {
+		/// Content type of the object
+		pub content_type: String,
+		/// Any other http headers to send
+		pub other: BTreeMap<String, String>,
+	}
+
+	impl garage_util::migrate::InitialFormat for Object {}
 }

+mod v08 {
+	use garage_util::data::Uuid;
+	use serde::{Deserialize, Serialize};
+
+	use super::v05;
+
+	pub use v05::{
+		ObjectVersion, ObjectVersionData, ObjectVersionHeaders, ObjectVersionMeta,
+		ObjectVersionState,
+	};
+
+	/// An object
+	#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
+	pub struct Object {
+		/// The bucket in which the object is stored, used as partition key
+		pub bucket_id: Uuid,
+
+		/// The key at which the object is stored in its bucket, used as sorting key
+		pub key: String,
+
+		/// The list of currenty stored versions of the object
+		pub(super) versions: Vec<ObjectVersion>,
+	}
+
+	impl garage_util::migrate::Migrate for Object {
+		type Previous = v05::Object;
+
+		fn migrate(old: v05::Object) -> Object {
+			use garage_util::data::blake2sum;
+
+			Object {
+				bucket_id: blake2sum(old.bucket.as_bytes()),
+				key: old.key,
+				versions: old.versions,
+			}
+		}
+	}
+}
+
+pub use v08::*;
+
 impl Object {
 	/// Initialize an Object struct from parts
 	pub fn new(bucket_id: Uuid, key: String, versions: Vec<ObjectVersion>) -> Self {
@@ -69,28 +168,6 @@ impl Object {
 	}
 }

-/// Informations about a version of an object
-#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
-pub struct ObjectVersion {
-	/// Id of the version
-	pub uuid: Uuid,
-	/// Timestamp of when the object was created
-	pub timestamp: u64,
-	/// State of the version
-	pub state: ObjectVersionState,
-}
-
-/// State of an object version
-#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
-pub enum ObjectVersionState {
-	/// The version is being received
-	Uploading(ObjectVersionHeaders),
-	/// The version is fully received
-	Complete(ObjectVersionData),
-	/// The version uploaded containded errors or the upload was explicitly aborted
-	Aborted,
-}
-
 impl Crdt for ObjectVersionState {
 	fn merge(&mut self, other: &Self) {
 		use ObjectVersionState::*;
@@ -112,42 +189,10 @@ impl Crdt for ObjectVersionState {
 	}
 }

-/// Data stored in object version
-#[derive(PartialEq, Eq, PartialOrd, Ord, Clone, Debug, Serialize, Deserialize)]
-pub enum ObjectVersionData {
-	/// The object was deleted, this Version is a tombstone to mark it as such
-	DeleteMarker,
-	/// The object is short, it's stored inlined
-	Inline(ObjectVersionMeta, #[serde(with = "serde_bytes")] Vec<u8>),
-	/// The object is not short, Hash of first block is stored here, next segments hashes are
-	/// stored in the version table
-	FirstBlock(ObjectVersionMeta, Hash),
-}
-
 impl AutoCrdt for ObjectVersionData {
 	const WARN_IF_DIFFERENT: bool = true;
 }

-/// Metadata about the object version
-#[derive(PartialEq, Eq, PartialOrd, Ord, Clone, Debug, Serialize, Deserialize)]
-pub struct ObjectVersionMeta {
-	/// Headers to send to the client
-	pub headers: ObjectVersionHeaders,
-	/// Size of the object
-	pub size: u64,
-	/// etag of the object
-	pub etag: String,
-}
-
-/// Additional headers for an object
-#[derive(PartialEq, Eq, PartialOrd, Ord, Clone, Debug, Serialize, Deserialize)]
-pub struct ObjectVersionHeaders {
-	/// Content type of the object
-	pub content_type: String,
-	/// Any other http headers to send
-	pub other: BTreeMap<String, String>,
-}
-
 impl ObjectVersion {
 	fn cmp_key(&self) -> (u64, Uuid) {
 		(self.timestamp, self.uuid)
@@ -221,7 +266,6 @@ impl Crdt for Object {
 }

 pub struct ObjectTable {
-	pub background: Arc<BackgroundRunner>,
 	pub version_table: Arc<Table<VersionTable, TableShardedReplication>>,
 	pub object_counter_table: Arc<IndexCounter<Object>>,
 }
@@ -255,34 +299,34 @@ impl TableSchema for ObjectTable {
 			);
 		}

-		// 2. Spawn threads that propagates deletions to version table
-		let version_table = self.version_table.clone();
-		let old = old.cloned();
-		let new = new.cloned();
-
-		self.background.spawn(async move {
-			if let (Some(old_v), Some(new_v)) = (old, new) {
-				// Propagate deletion of old versions
-				for v in old_v.versions.iter() {
-					let newly_deleted = match new_v
-						.versions
-						.binary_search_by(|nv| nv.cmp_key().cmp(&v.cmp_key()))
-					{
-						Err(_) => true,
-						Ok(i) => {
-							new_v.versions[i].state == ObjectVersionState::Aborted
-								&& v.state != ObjectVersionState::Aborted
-						}
-					};
-					if newly_deleted {
-						let deleted_version =
-							Version::new(v.uuid, old_v.bucket_id, old_v.key.clone(), true);
-						version_table.insert(&deleted_version).await?;
+		// 2. Enqueue propagation deletions to version table
+		if let (Some(old_v), Some(new_v)) = (old, new) {
+			// Propagate deletion of old versions
+			for v in old_v.versions.iter() {
+				let newly_deleted = match new_v
+					.versions
+					.binary_search_by(|nv| nv.cmp_key().cmp(&v.cmp_key()))
+				{
+					Err(_) => true,
+					Ok(i) => {
+						new_v.versions[i].state == ObjectVersionState::Aborted
+							&& v.state != ObjectVersionState::Aborted
+					}
+				};
+				if newly_deleted {
+					let deleted_version =
+						Version::new(v.uuid, old_v.bucket_id, old_v.key.clone(), true);
+					let res = self.version_table.queue_insert(tx, &deleted_version);
+					if let Err(e) = db::unabort(res)? {
+						error!(
+							"Unable to enqueue version deletion propagation: {}. A repair will be needed.",
+							e
+						);
 					}
 				}
 			}
-			Ok(())
-		});
+		}
+
 		Ok(())
 	}

@@ -292,11 +336,6 @@ impl TableSchema for ObjectTable {
 			ObjectFilter::IsUploading => entry.versions.iter().any(|v| v.is_uploading()),
 		}
 	}
-
-	fn try_migrate(bytes: &[u8]) -> Option<Self::E> {
-		let old_obj = rmp_serde::decode::from_read_ref::<_, old::Object>(bytes).ok()?;
-		Some(migrate_object(old_obj))
-	}
 }

 impl CountedItem for Object {
@@ -341,64 +380,3 @@ impl CountedItem for Object {
 		]
 	}
 }
-
-// vvvvvvvv migration code, stupid stuff vvvvvvvvvvvv
-// (we just want to change bucket into bucket_id by hashing it)
-
-fn migrate_object(o: old::Object) -> Object {
-	let versions = o
-		.versions()
-		.iter()
-		.cloned()
-		.map(migrate_object_version)
-		.collect();
-	Object {
-		bucket_id: blake2sum(o.bucket.as_bytes()),
-		key: o.key,
-		versions,
-	}
-}
-
-fn migrate_object_version(v: old::ObjectVersion) -> ObjectVersion {
-	ObjectVersion {
-		uuid: Uuid::try_from(v.uuid.as_slice()).unwrap(),
-		timestamp: v.timestamp,
-		state: match v.state {
-			old::ObjectVersionState::Uploading(h) => {
-				ObjectVersionState::Uploading(migrate_object_version_headers(h))
-			}
-			old::ObjectVersionState::Complete(d) => {
-				ObjectVersionState::Complete(migrate_object_version_data(d))
-			}
-			old::ObjectVersionState::Aborted => ObjectVersionState::Aborted,
-		},
-	}
-}
-
-fn migrate_object_version_headers(h: old::ObjectVersionHeaders) -> ObjectVersionHeaders {
-	ObjectVersionHeaders {
-		content_type: h.content_type,
-		other: h.other,
-	}
-}
-
-fn migrate_object_version_data(d: old::ObjectVersionData) -> ObjectVersionData {
-	match d {
-		old::ObjectVersionData::DeleteMarker => ObjectVersionData::DeleteMarker,
-		old::ObjectVersionData::Inline(m, b) => {
-			ObjectVersionData::Inline(migrate_object_version_meta(m), b)
-		}
-		old::ObjectVersionData::FirstBlock(m, h) => ObjectVersionData::FirstBlock(
-			migrate_object_version_meta(m),
-			Hash::try_from(h.as_slice()).unwrap(),
-		),
-	}
-}
-
-fn migrate_object_version_meta(m: old::ObjectVersionMeta) -> ObjectVersionMeta {
-	ObjectVersionMeta {
-		headers: migrate_object_version_headers(m.headers),
-		size: m.size,
-		etag: m.etag,
-	}
-}
@@ -1,9 +1,7 @@
-use serde::{Deserialize, Serialize};
 use std::sync::Arc;

 use garage_db as db;

-use garage_util::background::BackgroundRunner;
 use garage_util::data::*;

 use garage_table::crdt::*;
@@ -12,32 +10,108 @@ use garage_table::*;

 use crate::s3::block_ref_table::*;

-use crate::prev::v051::version_table as old;
+mod v05 {
+	use garage_util::crdt;
+	use garage_util::data::{Hash, Uuid};
+	use serde::{Deserialize, Serialize};

-/// A version of an object
-#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
-pub struct Version {
-	/// UUID of the version, used as partition key
-	pub uuid: Uuid,
+	/// A version of an object
+	#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
+	pub struct Version {
+		/// UUID of the version, used as partition key
+		pub uuid: Uuid,

-	// Actual data: the blocks for this version
-	// In the case of a multipart upload, also store the etags
-	// of individual parts and check them when doing CompleteMultipartUpload
-	/// Is this version deleted
-	pub deleted: crdt::Bool,
-	/// list of blocks of data composing the version
-	pub blocks: crdt::Map<VersionBlockKey, VersionBlock>,
-	/// Etag of each part in case of a multipart upload, empty otherwise
-	pub parts_etags: crdt::Map<u64, String>,
+		// Actual data: the blocks for this version
+		// In the case of a multipart upload, also store the etags
+		// of individual parts and check them when doing CompleteMultipartUpload
+		/// Is this version deleted
+		pub deleted: crdt::Bool,
+		/// list of blocks of data composing the version
+		pub blocks: crdt::Map<VersionBlockKey, VersionBlock>,
+		/// Etag of each part in case of a multipart upload, empty otherwise
+		pub parts_etags: crdt::Map<u64, String>,

-	// Back link to bucket+key so that we can figure if
-	// this was deleted later on
-	/// Bucket in which the related object is stored
-	pub bucket_id: Uuid,
-	/// Key in which the related object is stored
-	pub key: String,
+		// Back link to bucket+key so that we can figure if
+		// this was deleted later on
+		/// Bucket in which the related object is stored
+		pub bucket: String,
+		/// Key in which the related object is stored
+		pub key: String,
+	}
+
+	#[derive(PartialEq, Eq, Clone, Copy, Debug, Serialize, Deserialize)]
+	pub struct VersionBlockKey {
+		/// Number of the part
+		pub part_number: u64,
+		/// Offset of this sub-segment in its part
+		pub offset: u64,
+	}
+
+	/// Informations about a single block
+	#[derive(PartialEq, Eq, Ord, PartialOrd, Clone, Copy, Debug, Serialize, Deserialize)]
+	pub struct VersionBlock {
+		/// Blake2 sum of the block
+		pub hash: Hash,
+		/// Size of the block
+		pub size: u64,
+	}
+
+	impl garage_util::migrate::InitialFormat for Version {}
 }

+mod v08 {
+	use garage_util::crdt;
+	use garage_util::data::Uuid;
+	use serde::{Deserialize, Serialize};
+
+	use super::v05;
+
+	/// A version of an object
+	#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
+	pub struct Version {
+		/// UUID of the version, used as partition key
+		pub uuid: Uuid,
+
+		// Actual data: the blocks for this version
+		// In the case of a multipart upload, also store the etags
+		// of individual parts and check them when doing CompleteMultipartUpload
+		/// Is this version deleted
+		pub deleted: crdt::Bool,
+		/// list of blocks of data composing the version
+		pub blocks: crdt::Map<VersionBlockKey, VersionBlock>,
+		/// Etag of each part in case of a multipart upload, empty otherwise
+		pub parts_etags: crdt::Map<u64, String>,
+
+		// Back link to bucket+key so that we can figure if
+		// this was deleted later on
+		/// Bucket in which the related object is stored
+		pub bucket_id: Uuid,
+		/// Key in which the related object is stored
+		pub key: String,
+	}
+
+	pub use v05::{VersionBlock, VersionBlockKey};
+
+	impl garage_util::migrate::Migrate for Version {
+		type Previous = v05::Version;
+
+		fn migrate(old: v05::Version) -> Version {
+			use garage_util::data::blake2sum;
+
+			Version {
+				uuid: old.uuid,
+				deleted: old.deleted,
+				blocks: old.blocks,
+				parts_etags: old.parts_etags,
+				bucket_id: blake2sum(old.bucket.as_bytes()),
+				key: old.key,
+			}
+		}
+	}
+}
+
+pub use v08::*;
+
 impl Version {
 	pub fn new(uuid: Uuid, bucket_id: Uuid, key: String, deleted: bool) -> Self {
 		Self {
@@ -65,14 +139,6 @@ impl Version {
 	}
 }

-#[derive(PartialEq, Eq, Clone, Copy, Debug, Serialize, Deserialize)]
-pub struct VersionBlockKey {
-	/// Number of the part
-	pub part_number: u64,
-	/// Offset of this sub-segment in its part
-	pub offset: u64,
-}
-
 impl Ord for VersionBlockKey {
 	fn cmp(&self, other: &Self) -> std::cmp::Ordering {
 		self.part_number
@@ -87,15 +153,6 @@ impl PartialOrd for VersionBlockKey {
 	}
 }

-/// Informations about a single block
-#[derive(PartialEq, Eq, Ord, PartialOrd, Clone, Copy, Debug, Serialize, Deserialize)]
-pub struct VersionBlock {
-	/// Blake2 sum of the block
-	pub hash: Hash,
-	/// Size of the block
-	pub size: u64,
-}
-
 impl AutoCrdt for VersionBlock {
 	const WARN_IF_DIFFERENT: bool = true;
 }
@@ -127,7 +184,6 @@ impl Crdt for Version {
 }

 pub struct VersionTable {
-	pub background: Arc<BackgroundRunner>,
 	pub block_ref_table: Arc<Table<BlockRefTable, TableShardedReplication>>,
 }

@@ -141,33 +197,26 @@ impl TableSchema for VersionTable {

 	fn updated(
 		&self,
-		_tx: &mut db::Transaction,
+		tx: &mut db::Transaction,
 		old: Option<&Self::E>,
 		new: Option<&Self::E>,
 	) -> db::TxOpResult<()> {
-		let block_ref_table = self.block_ref_table.clone();
-		let old = old.cloned();
-		let new = new.cloned();
-
-		self.background.spawn(async move {
-			if let (Some(old_v), Some(new_v)) = (old, new) {
-				// Propagate deletion of version blocks
-				if new_v.deleted.get() && !old_v.deleted.get() {
-					let deleted_block_refs = old_v
-						.blocks
-						.items()
-						.iter()
-						.map(|(_k, vb)| BlockRef {
-							block: vb.hash,
-							version: old_v.uuid,
-							deleted: true.into(),
-						})
-						.collect::<Vec<_>>();
-					block_ref_table.insert_many(&deleted_block_refs[..]).await?;
+		if let (Some(old_v), Some(new_v)) = (old, new) {
+			// Propagate deletion of version blocks
+			if new_v.deleted.get() && !old_v.deleted.get() {
+				let deleted_block_refs = old_v.blocks.items().iter().map(|(_k, vb)| BlockRef {
+					block: vb.hash,
+					version: old_v.uuid,
+					deleted: true.into(),
+				});
+				for block_ref in deleted_block_refs {
+					let res = self.block_ref_table.queue_insert(tx, &block_ref);
+					if let Err(e) = db::unabort(res)? {
+						error!("Unable to enqueue block ref deletion propagation: {}. A repair will be needed.", e);
+					}
 				}
 			}
-			Ok(())
-		});
+		}

 		Ok(())
 	}
@@ -175,42 +224,4 @@ impl TableSchema for VersionTable {
 	fn matches_filter(entry: &Self::E, filter: &Self::Filter) -> bool {
 		filter.apply(entry.deleted.get())
 	}
-
-	fn try_migrate(bytes: &[u8]) -> Option<Self::E> {
-		let old = rmp_serde::decode::from_read_ref::<_, old::Version>(bytes).ok()?;
-
-		let blocks = old
-			.blocks
-			.items()
-			.iter()
-			.map(|(k, v)| {
-				(
-					VersionBlockKey {
-						part_number: k.part_number,
-						offset: k.offset,
-					},
-					VersionBlock {
-						hash: Hash::try_from(v.hash.as_slice()).unwrap(),
-						size: v.size,
-					},
-				)
-			})
-			.collect::<crdt::Map<_, _>>();
-
-		let parts_etags = old
-			.parts_etags
-			.items()
-			.iter()
-			.map(|(k, v)| (*k, v.clone()))
-			.collect::<crdt::Map<_, _>>();
-
-		Some(Version {
-			uuid: Hash::try_from(old.uuid.as_slice()).unwrap(),
-			deleted: crdt::Bool::new(old.deleted.get()),
-			blocks,
-			parts_etags,
-			bucket_id: blake2sum(old.bucket.as_bytes()),
-			key: old.key,
-		})
-	}
 }
@@ -1,6 +1,6 @@
 [package]
 name = "garage_rpc"
-version = "0.8.0"
+version = "0.8.1"
 authors = ["Alex Auvolat <alex@adnab.me>"]
 edition = "2018"
 license = "AGPL-3.0"
@@ -14,7 +14,7 @@ path = "lib.rs"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html

 [dependencies]
-garage_util = { version = "0.8.0", path = "../util" }
+garage_util = { version = "0.8.1", path = "../util" }

 arc-swap = "1.0"
 bytes = "1.0"
@@ -25,7 +25,6 @@ rand = "0.8"
 sodiumoxide = { version = "0.2.5-0", package = "kuska-sodiumoxide" }

 async-trait = "0.1.7"
-rmp-serde = "0.15"
 serde = { version = "1.0", default-features = false, features = ["derive", "rc"] }
 serde_bytes = "0.11"
 serde_json = "1.0"
@@ -5,6 +5,7 @@ use serde::{Deserialize, Serialize};

 use garage_util::crdt::{AutoCrdt, Crdt, LwwMap};
 use garage_util::data::*;
+use garage_util::encode::nonversioned_encode;
 use garage_util::error::*;

 use crate::ring::*;
@@ -35,6 +36,8 @@ pub struct ClusterLayout {
 	pub staging_hash: Hash,
 }

+impl garage_util::migrate::InitialFormat for ClusterLayout {}
+
 #[derive(PartialEq, Eq, PartialOrd, Ord, Clone, Debug, Serialize, Deserialize)]
 pub struct NodeRoleV(pub Option<NodeRole>);

@@ -68,7 +71,7 @@ impl NodeRole {
 impl ClusterLayout {
 	pub fn new(replication_factor: usize) -> Self {
 		let empty_lwwmap = LwwMap::new();
-		let empty_lwwmap_hash = blake2sum(&rmp_to_vec_all_named(&empty_lwwmap).unwrap()[..]);
+		let empty_lwwmap_hash = blake2sum(&nonversioned_encode(&empty_lwwmap).unwrap()[..]);

 		ClusterLayout {
 			version: 0,
@@ -90,7 +93,7 @@ impl ClusterLayout {
 			Ordering::Equal => {
 				self.staging.merge(&other.staging);

-				let new_staging_hash = blake2sum(&rmp_to_vec_all_named(&self.staging).unwrap()[..]);
+				let new_staging_hash = blake2sum(&nonversioned_encode(&self.staging).unwrap()[..]);
 				let changed = new_staging_hash != self.staging_hash;

 				self.staging_hash = new_staging_hash;
@@ -125,7 +128,7 @@ To know the correct value of the new layout version, invoke `garage layout show`
 		}

 		self.staging.clear();
-		self.staging_hash = blake2sum(&rmp_to_vec_all_named(&self.staging).unwrap()[..]);
+		self.staging_hash = blake2sum(&nonversioned_encode(&self.staging).unwrap()[..]);

 		self.version += 1;

@@ -149,7 +152,7 @@ To know the correct value of the new layout version, invoke `garage layout show`
 		}

 		self.staging.clear();
-		self.staging_hash = blake2sum(&rmp_to_vec_all_named(&self.staging).unwrap()[..]);
+		self.staging_hash = blake2sum(&nonversioned_encode(&self.staging).unwrap()[..]);

 		self.version += 1;

@@ -178,7 +181,7 @@ To know the correct value of the new layout version, invoke `garage layout show`
 	/// returns true if consistent, false if error
 	pub fn check(&self) -> bool {
 		// Check that the hash of the staging data is correct
-		let staging_hash = blake2sum(&rmp_to_vec_all_named(&self.staging).unwrap()[..]);
+		let staging_hash = blake2sum(&nonversioned_encode(&self.staging).unwrap()[..]);
 		if staging_hash != self.staging_hash {
 			return false;
 		}
@@ -254,9 +257,11 @@ To know the correct value of the new layout version, invoke `garage layout show`
 		match self.initial_partition_assignation() {
 			Some(initial_partitions) => {
 				for (part, ipart) in partitions.iter_mut().zip(initial_partitions.iter()) {
-					for (id, info) in ipart.nodes.iter() {
-						if part.nodes.len() < self.replication_factor {
-							part.add(None, n_zones, id, info.unwrap());
+					for _ in 0..2 {
+						for (id, info) in ipart.nodes.iter() {
+							if part.nodes.len() < self.replication_factor {
+								part.add(None, n_zones, id, info.unwrap());
+							}
 						}
 					}
 					assert!(part.nodes.len() == self.replication_factor);
@@ -9,6 +9,7 @@ mod consul;
 mod kubernetes;

 pub mod layout;
+pub mod replication_mode;
 pub mod ring;
 pub mod system;

@@ -1,3 +1,4 @@
+#[derive(Clone, Copy)]
 pub enum ReplicationMode {
 	None,
 	TwoWay,
@@ -5,7 +5,6 @@ use std::time::Duration;
 use futures::future::join_all;
 use futures::stream::futures_unordered::FuturesUnordered;
 use futures::stream::StreamExt;
-use futures_util::future::FutureExt;
 use tokio::select;
 use tokio::sync::watch;

@@ -24,7 +23,6 @@ pub use netapp::message::{
 use netapp::peering::fullmesh::FullMeshPeeringStrategy;
 pub use netapp::{self, NetApp, NodeID};

-use garage_util::background::BackgroundRunner;
 use garage_util::data::*;
 use garage_util::error::Error;
 use garage_util::metrics::RecordDuration;
@@ -94,7 +92,6 @@ pub struct RpcHelper(Arc<RpcHelperInner>);
 struct RpcHelperInner {
 	our_node_id: Uuid,
 	fullmesh: Arc<FullMeshPeeringStrategy>,
-	background: Arc<BackgroundRunner>,
 	ring: watch::Receiver<Arc<Ring>>,
 	metrics: RpcMetrics,
 	rpc_timeout: Duration,
@@ -104,7 +101,6 @@ impl RpcHelper {
 	pub(crate) fn new(
 		our_node_id: Uuid,
 		fullmesh: Arc<FullMeshPeeringStrategy>,
-		background: Arc<BackgroundRunner>,
 		ring: watch::Receiver<Arc<Ring>>,
 		rpc_timeout: Option<Duration>,
 	) -> Self {
@@ -113,7 +109,6 @@ impl RpcHelper {
 		Self(Arc::new(RpcHelperInner {
 			our_node_id,
 			fullmesh,
-			background,
 			ring,
 			metrics,
 			rpc_timeout: rpc_timeout.unwrap_or(DEFAULT_TIMEOUT),
@@ -377,16 +372,13 @@ impl RpcHelper {

 			if !resp_stream.is_empty() {
 				// Continue remaining requests in background.
-				// Continue the remaining requests immediately using tokio::spawn
-				// but enqueue a task in the background runner
-				// to ensure that the process won't exit until the requests are done
-				// (if we had just enqueued the resp_stream.collect directly in the background runner,
-				// the requests might have been put on hold in the background runner's queue,
-				// in which case they might timeout or otherwise fail)
-				let wait_finished_fut = tokio::spawn(async move {
+				// Note: these requests can get interrupted on process shutdown,
+				// we must not count on them being executed for certain.
+				// For all background things that have to happen with certainty,
+				// they have to be put in a proper queue that is persisted to disk.
+				tokio::spawn(async move {
 					resp_stream.collect::<Vec<Result<_, _>>>().await;
 				});
-				self.0.background.spawn(wait_finished_fut.map(|_| Ok(())));
 			}
 		}

--- a/Show More
+++ b/Show More
Author	SHA1	Message	Date
Alex Auvolat	32715d462e	history table refactoring	2023-01-09 15:00:00 +01:00
Alex Auvolat	b337895fce	Update Rust Nix toolchain	2023-01-09 13:21:37 +01:00
Alex Auvolat	49b5d18554	K2V history and preparation for range watch	2023-01-05 13:11:48 +01:00
Alex	02e8eb167e	Merge pull request 'PutObject: better cleanup when request is interrupted in the middle' (#462 ) from interrupted-cleanup into main Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/462	2023-01-04 14:43:45 +00:00
Alex	329c0e64f9	Merge pull request 'Improve `garage worker set` and add `garage worker get`' (#464 ) from worker-get into main Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/464	2023-01-04 13:47:42 +00:00
Alex Auvolat	29dbcb8278	bg var operation on all nodes at once	2023-01-04 13:25:57 +01:00
Alex Auvolat	f3f27293df	Uniform framework for bg variable management	2023-01-04 13:07:13 +01:00
Alex Auvolat	13c5549886	Remove token_bucket.rs	2023-01-04 11:47:56 +01:00
Alex Auvolat	936b6cb563	When saving block, delete .tmp file if we could not complete	2023-01-03 17:34:26 +01:00
Alex Auvolat	0650a43cf1	PutObject: better cleanup on Drop (incl. when request is interrupted in the middle)	2023-01-03 17:05:17 +01:00
Alex	4eb8ca3a52	Merge pull request 'Fix Consul & Kubernetes discovery with new way of doing background things' (#463 ) from fix-background into main Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/463	2023-01-03 16:04:40 +00:00
Alex Auvolat	1fc220886a	Fix Consul & Kubernetes discovery with new way of doing background things	2023-01-03 16:55:59 +01:00
Alex	73ed9c7403	Merge pull request 'Refactor how things are migrated' (#461 ) from format-migration into main Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/461	2023-01-03 15:28:24 +00:00
Alex Auvolat	1d5bdc17a4	use impossible enum type	2023-01-03 16:04:06 +01:00
Alex Auvolat	c106304b9c	more idiomatic and shorter	2023-01-03 16:00:19 +01:00
Alex Auvolat	33f25d26c7	fix doc and add tests for migrate.rs	2023-01-03 15:53:13 +01:00
Alex Auvolat	d6d571d512	cargo fmt	2023-01-03 15:30:21 +01:00
Alex Auvolat	a54b67740d	move debug_serialize to garage_util::encode	2023-01-03 15:29:29 +01:00
Alex Auvolat	8d5505514f	Make it explicit when using nonversioned encoding	2023-01-03 15:27:36 +01:00
Alex Auvolat	426d8784da	cleanup	2023-01-03 15:08:37 +01:00
Alex Auvolat	a81200d345	Update cargo.nix	2023-01-03 14:45:47 +01:00
Alex Auvolat	cdb2a591e9	Refactor how things are migrated	2023-01-03 14:44:47 +01:00
Alex	582b076179	Merge pull request 'Some improvements to Garage internals' (#451 ) from internals-rework into main Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/451	2023-01-03 11:37:31 +00:00
Alex Auvolat	939a6d67e8	Merge branch 'main' into internals-rework	2023-01-02 15:07:44 +01:00
Alex	76230f2028	Merge pull request 'Bump everything to v0.8.1' (#458 ) from up-v0.8.1 into main Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/458	2023-01-02 13:32:45 +00:00
Alex Auvolat	6775569525	Bump everything to v0.8.1	2023-01-02 14:15:33 +01:00
Alex Auvolat	6b857a9b8c	cargo fmt	2023-01-02 13:50:42 +01:00
Alex	1649002e2b	Merge pull request 'Add a note about Peertube 5.0 private videos' (#456 ) from kaiyou/garage:docs-apps into main Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/456	2023-01-02 12:49:14 +00:00
Alex	822e344845	Merge pull request 'Add some docs about using Python Minio SDK' (#455 ) from kaiyou/garage:docs-s3-libs into main Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/455	2023-01-02 12:48:52 +00:00
Alex	7f7d53cfa9	Merge pull request 'improvements to CLI and new debug features' (#448 ) from cli-improvements into main Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/448	2023-01-02 12:42:24 +00:00
kaiyou	fd10200bec	Add a note about Peertube 5.0 private videos	2022-12-25 14:20:01 +01:00
kaiyou	0c7ed0b0af	Add some docs about using Python Minio SDK	2022-12-25 13:55:12 +01:00
Alex	1af4a5ed56	Merge pull request 'Fix router keyword handling (fix #442 )' (#446 ) from router-keywords-fix into main Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/446	2022-12-15 08:40:26 +00:00
Alex Auvolat	1fcd0b371b	online repair workers: retry on error	2022-12-14 16:31:31 +01:00
Alex Auvolat	13c8662126	factorize	2022-12-14 16:16:55 +01:00
Alex Auvolat	e6f14ab5cf	better error message handling	2022-12-14 16:11:19 +01:00
Alex Auvolat	510b620108	Get rid of background::spawn	2022-12-14 16:08:05 +01:00
Alex Auvolat	dfc131850a	Simplified and more aggressive worker exit logic	2022-12-14 15:25:29 +01:00
Alex Auvolat	d4af27f920	Add missing notify	2022-12-14 13:54:21 +01:00
Alex Auvolat	0d6b05bb6c	Update cargo.nix	2022-12-14 12:58:24 +01:00
Alex Auvolat	a19bfef508	Improve error message on rpc connection failure	2022-12-14 12:57:33 +01:00
Alex Auvolat	d56c472712	Refactor background runner and get rid of job worker	2022-12-14 12:51:42 +01:00
Alex Auvolat	2183518edc	Spawn all background workers in a separate step	2022-12-14 12:28:07 +01:00
Alex Auvolat	83c8467e23	Proper queueing for delayed inserts, now backed to disk	2022-12-14 11:58:06 +01:00
Alex Auvolat	f8e528c15d	Small refactor of tables internals	2022-12-14 10:48:49 +01:00
Alex Auvolat	d1279e04f3	Fix error messages	2022-12-13 16:18:01 +01:00
Alex Auvolat	041b60ed1d	Add block.rc_size, table.size and table.merkle_tree_size metrics	2022-12-13 15:54:03 +01:00
Alex Auvolat	f8d5409894	cli: more info displayed on error in garage stats	2022-12-13 15:46:04 +01:00
Alex Auvolat	d6040e32a6	cli: prettier table in garage stats	2022-12-13 15:43:22 +01:00
Alex Auvolat	d7f90cabb0	Implement `block retry-now` and `block purge`	2022-12-13 15:02:42 +01:00
Alex Auvolat	687660b27f	Implement `block list-errors` and `block info`	2022-12-13 14:23:45 +01:00
Alex Auvolat	9d82196945	cli: new worker info command	2022-12-13 12:24:30 +01:00
Alex Auvolat	a51e8d94c6	cli: rename resync-n-workers into resync-worker-count	2022-12-13 11:44:11 +01:00
Alex Auvolat	de9d6cddf7	Prettier worker list table; remove useless CLI log messages	2022-12-12 17:17:05 +01:00
Alex	f7c65e830e	Merge pull request 'Properly enforce allow_create_bucket' (#447 ) from fix-allow-create-bucket into main Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/447	2022-12-12 14:55:12 +00:00
Alex Auvolat	0e61e3b6fb	Fix bucket creation tests to take permissions into account	2022-12-12 15:47:55 +01:00
Alex Auvolat	a0abf41762	Fix router keyword handling (fix #442 )	2022-12-12 12:05:37 +01:00
Alex Auvolat	2ac75018a1	Properly enforce allow_create_bucket	2022-12-12 12:03:54 +01:00
Maximilien R	980572a887	Merge pull request 'helm: ingress improvements' (#422 ) from patrickjahns:helm-refactor-ingress into main As discussed in the chat yesterday, I want to propose to disable the ingress per default. The motivation behind this change is, that per default the ingress is "misconfigured" meaning it can not work with the default values and requires a user of the chart to add additional configuration. When installing the chart per default, I would not expect to already expose garage publicly without my explicit configuration to do so Commenting the ingressClass resource also allows for relying only on annotations - otherwise the ingressClass would be always set to nginx or require a user to override it with ingressClass: null A small change on top, I've added the ability to specify user defined labels per ingress	2022-12-12 00:53:57 +01:00
Patrick Jahns	7a0014b6f7	chore(helm): bump chart number	2022-12-11 23:11:56 +00:00
Patrick Jahns	edb0b9c1ee	feat(helm): allow to add custom labels to created ingress resources	2022-12-11 23:11:56 +00:00
Patrick Jahns	f58a813a36	refactor(helm): disable the ingress per default The default values forces people to create an ingress resources, where per default an ingress is not necessary to start garage. If someone wants to utilize an ingress, he would need to define the values for the ingress either way, so enabling the ingress explicitly makes more sense, then requiring it to be disabled per default	2022-12-11 23:11:56 +00:00
Alex	defd7d9e63	Merge pull request 'Implement /health admin API endpoint to check node health' (#440 ) from admin-health-api into main Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/440	2022-12-11 17:25:28 +00:00
Alex Auvolat	533afcf4e1	simplify	2022-12-11 18:17:08 +01:00
Alex Auvolat	5ea5fd2130	Always return 200 OK on /v0/health, reinstate admin api doc as draft and complete it	2022-12-11 18:11:28 +01:00
Alex	35f8e8e2fb	Merge pull request 'Fix typo in documentation' (#441 ) from felix.scheinost/garage:documentation-typo into main Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/441	2022-12-07 20:42:24 +00:00
felix.scheinost	d5a2502b09	Fix typo in documentation	2022-12-07 12:43:49 +00:00
Alex Auvolat	d7868c48a4	Separate /health (simple text answer) and /v0/health (full json answer, authenticated)	2022-12-05 15:38:32 +01:00
Alex Auvolat	280d1be7b1	Refactor health check and add ability to return it in json	2022-12-05 15:28:57 +01:00
Alex Auvolat	2065f011ca	Implement /health admin API endpoint to check node health	2022-12-05 14:59:15 +01:00
Alex	243b7c9a1c	Merge pull request 'Fix spelling mistake in docs' (#438 ) from tompearson/garage:fix-typo into main Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/438	2022-12-05 12:27:14 +00:00
tompearson	a3afc761b6	Update 'doc/book/design/goals.md'	2022-12-04 16:27:46 +00:00
Alex	19bdd1c799	Merge pull request 'Fix logs appearing twice' (#435 ) from fix-logs into main Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/435	2022-11-29 21:30:39 +00:00
Alex	448dcc5cf4	Merge pull request 'Make repository into a Nix flake' (#424 ) from nix-remove-system into main Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/424	2022-11-29 21:26:41 +00:00
Alex Auvolat	26121bb619	Fix logs appearing twice	2022-11-29 22:23:27 +01:00
Alex	280330ac72	Merge pull request 'Add talk to the Capitole du Libre 2022' (#434 ) from CdL_talk into main Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/434	2022-11-27 13:38:13 +00:00
vincent	4d7b4d9d20	Add talk to the Capitole du Libre 2022	2022-11-27 11:36:01 +01:00
Alex	fc450ec13a	Merge pull request 'Fix #432 : documentation issue' (#433 ) from fix-432 into main Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/433	2022-11-24 14:36:53 +00:00
Alex Auvolat	379b2049f5	Fix #432 : documentation issue	2022-11-24 15:33:33 +01:00
Alex	293139a94a	Merge pull request 'Tentative fix #414 ' (#429 ) from try-fix-414 into main Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/429	2022-11-21 21:45:17 +00:00
Alex Auvolat	54e800ef8d	Tentative fix for issue #414	2022-11-21 17:13:41 +01:00
Alex Auvolat	60c26fbc62	Inject last modified date as git_version; flake cache uploading	2022-11-16 23:47:10 +01:00
Alex Auvolat	e76dba9561	Make repository into a Nix flake	2022-11-16 23:25:34 +01:00