Enable k2v feature flag in CI

Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/374

.drone.yml

@@ -2,68 +2,26 @@
 kind: pipeline
 name: default
 
-workspace:
-  base: /drone/garage
-
-volumes:
-- name: nix_store
-  host:
-    path: /var/lib/drone/nix
-- name: nix_config
-  temp: {}
-
-environment:
-  HOME: /drone/garage
+node:
+  nix-daemon: 1
 
 steps:
-  - name: setup nix
-    image: nixpkgs/nix:nixos-21.05
-    volumes:
-    - name: nix_store
-      path: /nix
-    - name: nix_config
-      path: /etc/nix
+  - name: check formatting
+    image: nixpkgs/nix:nixos-22.05
     commands:
-      - cp nix/nix.conf /etc/nix/nix.conf
-      - nix-build --no-build-output --no-out-link shell.nix --arg release false  -A inputDerivation
-
-  - name: code quality
-    image: nixpkgs/nix:nixos-21.05
-    volumes:
-    - name: nix_store
-      path: /nix
-    - name: nix_config
-      path: /etc/nix
-    commands:
-      - nix-shell --arg release false --run "cargo fmt -- --check"
-      - nix-shell --arg release false --run "cargo clippy -- --deny warnings"
+      - nix-shell --attr rust --run "cargo fmt -- --check"
 
   - name: build
-    image: nixpkgs/nix:nixos-21.05
-    volumes:
-    - name: nix_store
-      path: /nix
-    - name: nix_config
-      path: /etc/nix
+    image: nixpkgs/nix:nixos-22.05
     commands:
-      - nix-build --no-build-output --option log-lines 100 --argstr target x86_64-unknown-linux-musl --arg release false --argstr git_version $DRONE_COMMIT
+      - nix-build --no-build-output --attr clippy.amd64 --argstr git_version ${DRONE_TAG:-$DRONE_COMMIT}
 
   - name: unit + func tests
-    image: nixpkgs/nix:nixos-21.05
+    image: nixpkgs/nix:nixos-22.05
     environment:
       GARAGE_TEST_INTEGRATION_EXE: result/bin/garage
-    volumes:
-    - name: nix_store
-      path: /nix
-    - name: nix_config
-      path: /etc/nix
     commands:
-      - |
-        nix-build \
-          --no-build-output \
-          --option log-lines 100 \
-          --argstr target x86_64-unknown-linux-musl \
-          --argstr compileMode test
+      - nix-build --no-build-output --attr test.amd64
       - ./result/bin/garage_api-*
       - ./result/bin/garage_model-*
       - ./result/bin/garage_rpc-*
@@ -73,16 +31,11 @@ steps:
       - ./result/bin/garage-*
       - ./result/bin/integration-*
 
-  - name: smoke-test
-    image: nixpkgs/nix:nixos-21.05
-    volumes:
-    - name: nix_store
-      path: /nix
-    - name: nix_config
-      path: /etc/nix
+  - name: integration tests
+    image: nixpkgs/nix:nixos-22.05
     commands:
-      - nix-build --no-build-output --argstr target x86_64-unknown-linux-musl --arg release false --argstr git_version $DRONE_COMMIT
-      - nix-shell --arg release false --run ./script/test-smoke.sh || (cat /tmp/garage.log; false)
+      - nix-build --no-build-output --attr clippy.amd64 --argstr git_version ${DRONE_TAG:-$DRONE_COMMIT}
+      - nix-shell --attr integration --run ./script/test-smoke.sh || (cat /tmp/garage.log; false)
 
 trigger:
   event:
@@ -92,78 +45,39 @@ trigger:
   - tag
   - cron
 
-node:
-  nix: 1
-
 ---
 kind: pipeline
 type: docker
-name: release-linux-x86_64
+name: release-linux-amd64
 
-volumes:
-- name: nix_store
-  host:
-    path: /var/lib/drone/nix
-- name: nix_config
-  temp: {}
-
-environment:
-  TARGET: x86_64-unknown-linux-musl
+node:
+  nix-daemon: 1
 
 steps:
-  - name: setup nix
-    image: nixpkgs/nix:nixos-21.05
-    volumes:
-    - name: nix_store
-      path: /nix
-    - name: nix_config
-      path: /etc/nix
-    commands:
-      - cp nix/nix.conf /etc/nix/nix.conf
-      - nix-build --no-build-output --no-out-link shell.nix -A inputDerivation
-
   - name: build
-    image: nixpkgs/nix:nixos-21.05
-    volumes:
-    - name: nix_store
-      path: /nix
-    - name: nix_config
-      path: /etc/nix
+    image: nixpkgs/nix:nixos-22.05
     commands:
-      - nix-build --no-build-output --argstr target $TARGET --arg release true --argstr git_version $DRONE_COMMIT
+      - nix-build --no-build-output --attr pkgs.amd64.release --argstr git_version ${DRONE_TAG:-$DRONE_COMMIT}
+      - nix-shell --attr rust --run "./script/not-dynamic.sh result/bin/garage"
 
   - name: integration
-    image: nixpkgs/nix:nixos-21.05
-    volumes:
-    - name: nix_store
-      path: /nix
-    - name: nix_config
-      path: /etc/nix
+    image: nixpkgs/nix:nixos-22.05
     commands:
-      - nix-shell --run ./script/test-smoke.sh || (cat /tmp/garage.log; false)
+      - nix-shell --attr integration --run ./script/test-smoke.sh || (cat /tmp/garage.log; false)
 
   - name: push static binary
-    image: nixpkgs/nix:nixos-21.05
-    volumes:
-    - name: nix_store
-      path: /nix
-    - name: nix_config
-      path: /etc/nix
+    image: nixpkgs/nix:nixos-22.05
     environment:
       AWS_ACCESS_KEY_ID:
         from_secret: garagehq_aws_access_key_id
       AWS_SECRET_ACCESS_KEY:
         from_secret: garagehq_aws_secret_access_key
+      TARGET: "x86_64-unknown-linux-musl"
     commands:
-      - nix-shell --arg rust false --arg integration false --run "to_s3"
+      - nix-shell --attr release --run "to_s3"
 
   - name: docker build and publish
-    image: nixpkgs/nix:nixos-21.05
-    volumes:
-    - name: nix_store
-      path: /nix
-    - name: nix_config
-      path: /etc/nix
+    image: nixpkgs/nix:nixos-22.05
     environment:
       DOCKER_AUTH:
         from_secret: docker_auth
@@ -174,7 +88,7 @@ steps:
       - mkdir -p /kaniko/.docker
       - echo $DOCKER_AUTH > /kaniko/.docker/config.json
       - export CONTAINER_TAG=${DRONE_TAG:-$DRONE_COMMIT}
-      - nix-shell --arg rust false --arg integration false --run "to_docker"
+      - nix-shell --attr release --run "to_docker"
 
 
 trigger:
@@ -182,78 +96,39 @@ trigger:
   - promote
   - cron
 
-node:
-  nix: 1
-
 ---
 kind: pipeline
 type: docker
-name: release-linux-i686
+name: release-linux-i386
 
-volumes:
-- name: nix_store
-  host:
-    path: /var/lib/drone/nix
-- name: nix_config
-  temp: {}
-
-environment:
-  TARGET: i686-unknown-linux-musl
+node:
+  nix-daemon: 1
 
 steps:
-  - name: setup nix
-    image: nixpkgs/nix:nixos-21.05
-    volumes:
-    - name: nix_store
-      path: /nix
-    - name: nix_config
-      path: /etc/nix
-    commands:
-      - cp nix/nix.conf /etc/nix/nix.conf
-      - nix-build --no-build-output --no-out-link shell.nix -A inputDerivation
-
   - name: build
-    image: nixpkgs/nix:nixos-21.05
-    volumes:
-    - name: nix_store
-      path: /nix
-    - name: nix_config
-      path: /etc/nix
+    image: nixpkgs/nix:nixos-22.05
     commands:
-      - nix-build --no-build-output --argstr target $TARGET --arg release true --argstr git_version $DRONE_COMMIT
+      - nix-build --no-build-output --attr pkgs.i386.release --argstr git_version ${DRONE_TAG:-$DRONE_COMMIT}
+      - nix-shell --attr rust --run "./script/not-dynamic.sh result/bin/garage"
 
   - name: integration
-    image: nixpkgs/nix:nixos-21.05
-    volumes:
-    - name: nix_store
-      path: /nix
-    - name: nix_config
-      path: /etc/nix
+    image: nixpkgs/nix:nixos-22.05
     commands:
-      - nix-shell --run ./script/test-smoke.sh || (cat /tmp/garage.log; false)
+      - nix-shell --attr integration --run ./script/test-smoke.sh || (cat /tmp/garage.log; false)
 
   - name: push static binary
-    image: nixpkgs/nix:nixos-21.05
-    volumes:
-    - name: nix_store
-      path: /nix
-    - name: nix_config
-      path: /etc/nix
+    image: nixpkgs/nix:nixos-22.05
     environment:
       AWS_ACCESS_KEY_ID:
         from_secret: garagehq_aws_access_key_id
       AWS_SECRET_ACCESS_KEY:
         from_secret: garagehq_aws_secret_access_key
+      TARGET: "i686-unknown-linux-musl"
     commands:
-      - nix-shell --arg rust false --arg integration false --run "to_s3"
+      - nix-shell --attr release --run "to_s3"
 
   - name: docker build and publish
-    image: nixpkgs/nix:nixos-21.05
-    volumes:
-    - name: nix_store
-      path: /nix
-    - name: nix_config
-      path: /etc/nix
+    image: nixpkgs/nix:nixos-22.05
     environment:
       DOCKER_AUTH:
         from_secret: docker_auth
@@ -264,75 +139,41 @@ steps:
       - mkdir -p /kaniko/.docker
       - echo $DOCKER_AUTH > /kaniko/.docker/config.json
       - export CONTAINER_TAG=${DRONE_TAG:-$DRONE_COMMIT}
-      - nix-shell --arg rust false --arg integration false --run "to_docker"
+      - nix-shell --attr release --run "to_docker"
 
 trigger:
   event:
   - promote
   - cron
 
-node:
-  nix: 1
-
 ---
 kind: pipeline
 type: docker
-name: release-linux-aarch64
+name: release-linux-arm64
 
-volumes:
-- name: nix_store
-  host:
-    path: /var/lib/drone/nix
-- name: nix_config
-  temp: {}
-
-environment:
-  TARGET: aarch64-unknown-linux-musl
+node:
+  nix-daemon: 1
 
 steps:
-  - name: setup nix
-    image: nixpkgs/nix:nixos-21.05
-    volumes:
-    - name: nix_store
-      path: /nix
-    - name: nix_config
-      path: /etc/nix
-    commands:
-      - cp nix/nix.conf /etc/nix/nix.conf
-      - nix-build --no-build-output --no-out-link ./shell.nix --arg rust false --arg integration false -A inputDerivation
-
   - name: build
-    image: nixpkgs/nix:nixos-21.05
-    volumes:
-    - name: nix_store
-      path: /nix
-    - name: nix_config
-      path: /etc/nix
+    image: nixpkgs/nix:nixos-22.05
     commands:
-      - nix-build --no-build-output --argstr target $TARGET --arg release true --argstr git_version $DRONE_COMMIT
+      - nix-build --no-build-output --attr pkgs.arm64.release --argstr git_version ${DRONE_TAG:-$DRONE_COMMIT}
+      - nix-shell --attr rust --run "./script/not-dynamic.sh result/bin/garage"
 
   - name: push static binary
-    image: nixpkgs/nix:nixos-21.05
-    volumes:
-    - name: nix_store
-      path: /nix
-    - name: nix_config
-      path: /etc/nix
+    image: nixpkgs/nix:nixos-22.05
     environment:
       AWS_ACCESS_KEY_ID:
         from_secret: garagehq_aws_access_key_id
       AWS_SECRET_ACCESS_KEY:
         from_secret: garagehq_aws_secret_access_key
+      TARGET: "aarch64-unknown-linux-musl"
     commands:
-      - nix-shell --arg rust false --arg integration false --run "to_s3"
+      - nix-shell --attr release --run "to_s3"
 
   - name: docker build and publish
-    image: nixpkgs/nix:nixos-21.05
-    volumes:
-    - name: nix_store
-      path: /nix
-    - name: nix_config
-      path: /etc/nix
+    image: nixpkgs/nix:nixos-22.05
     environment:
       DOCKER_AUTH:
         from_secret: docker_auth
@@ -343,75 +184,41 @@ steps:
       - mkdir -p /kaniko/.docker
       - echo $DOCKER_AUTH > /kaniko/.docker/config.json
       - export CONTAINER_TAG=${DRONE_TAG:-$DRONE_COMMIT}
-      - nix-shell --arg rust false --arg integration false --run "to_docker"
+      - nix-shell --attr release --run "to_docker"
 
 trigger:
   event:
   - promote
   - cron
 
-node:
-  nix: 1
-
 ---
 kind: pipeline
 type: docker
-name: release-linux-armv6l
+name: release-linux-arm
 
-volumes:
-- name: nix_store
-  host:
-    path: /var/lib/drone/nix
-- name: nix_config
-  temp: {}
-
-environment:
-  TARGET: armv6l-unknown-linux-musleabihf
+node:
+  nix-daemon: 1
 
 steps:
-  - name: setup nix
-    image: nixpkgs/nix:nixos-21.05
-    volumes:
-    - name: nix_store
-      path: /nix
-    - name: nix_config
-      path: /etc/nix
-    commands:
-      - cp nix/nix.conf /etc/nix/nix.conf
-      - nix-build --no-build-output --no-out-link --arg rust false --arg integration false -A inputDerivation
-
   - name: build
-    image: nixpkgs/nix:nixos-21.05
-    volumes:
-    - name: nix_store
-      path: /nix
-    - name: nix_config
-      path: /etc/nix
+    image: nixpkgs/nix:nixos-22.05
     commands:
-      - nix-build --no-build-output --argstr target $TARGET --arg release true --argstr git_version $DRONE_COMMIT
+      - nix-build --no-build-output --attr pkgs.arm.release --argstr git_version ${DRONE_TAG:-$DRONE_COMMIT}
+      - nix-shell --attr rust --run "./script/not-dynamic.sh result/bin/garage"
 
   - name: push static binary
-    image: nixpkgs/nix:nixos-21.05
-    volumes:
-    - name: nix_store
-      path: /nix
-    - name: nix_config
-      path: /etc/nix
+    image: nixpkgs/nix:nixos-22.05
     environment:
       AWS_ACCESS_KEY_ID:
         from_secret: garagehq_aws_access_key_id
       AWS_SECRET_ACCESS_KEY:
         from_secret: garagehq_aws_secret_access_key
+      TARGET: "armv6l-unknown-linux-musleabihf"
     commands:
-      - nix-shell --arg integration false --arg rust false --run "to_s3"
+      - nix-shell --attr release --run "to_s3"
 
   - name: docker build and publish
-    image: nixpkgs/nix:nixos-21.05
-    volumes:
-    - name: nix_store
-      path: /nix
-    - name: nix_config
-      path: /etc/nix
+    image: nixpkgs/nix:nixos-22.05
     environment:
       DOCKER_AUTH:
         from_secret: docker_auth
@@ -422,32 +229,24 @@ steps:
       - mkdir -p /kaniko/.docker
       - echo $DOCKER_AUTH > /kaniko/.docker/config.json
       - export CONTAINER_TAG=${DRONE_TAG:-$DRONE_COMMIT}
-      - nix-shell --arg rust false --arg integration false --run "to_docker"
+      - nix-shell --attr release --run "to_docker"
 
 trigger:
   event:
   - promote
   - cron
 
-node:
-  nix: 1
-
 ---
 kind: pipeline
 type: docker
 name: refresh-release-page
 
-volumes:
-- name: nix_store
-  host:
-    path: /var/lib/drone/nix
+node:
+  nix-daemon: 1
 
 steps:
   - name: refresh-index
-    image: nixpkgs/nix:nixos-21.05
-    volumes:
-    - name: nix_store
-      path: /nix
+    image: nixpkgs/nix:nixos-22.05
     environment:
       AWS_ACCESS_KEY_ID:
         from_secret: garagehq_aws_access_key_id
@@ -455,24 +254,21 @@ steps:
         from_secret: garagehq_aws_secret_access_key
     commands:
       - mkdir -p /etc/nix && cp nix/nix.conf /etc/nix/nix.conf
-      - nix-shell --arg integration false --arg rust false --run "refresh_index"
+      - nix-shell --attr release --run "refresh_index"
 
 depends_on:
-  - release-linux-x86_64
-  - release-linux-i686
-  - release-linux-aarch64
-  - release-linux-armv6l
+  - release-linux-amd64
+  - release-linux-i386
+  - release-linux-arm64
+  - release-linux-arm
 
 trigger:
   event:
   - promote
   - cron
 
-node:
-  nix: 1
-
 ---
 kind: signature
-hmac: 3fc19d6f9a3555519c8405e3281b2e74289bb802f644740d5481d53df3a01fa4
+hmac: 362639b4c9541ad9bd06ff7f72b5235b2b0216bcb16eececd25285b6fe94ba6f
 
 ...

Cargo.lock

@@ -93,6 +93,15 @@ dependencies = [
  "winapi",
 ]
 
+[[package]]
+name = "autocfg"
+version = "0.1.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0dde43e75fd43e8a1bf86103336bc699aa8d17ad1be60c76c0bdfd4828e19b78"
+dependencies = [
+ "autocfg 1.1.0",
+]
+
 [[package]]
 name = "autocfg"
 version = "1.1.0"
@@ -543,7 +552,7 @@ version = "0.9.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1145cf131a2c6ba0615079ab6a638f7e1973ac9c2634fcbeaaad6114246efe8c"
 dependencies = [
- "autocfg",
+ "autocfg 1.1.0",
  "cfg-if 1.0.0",
  "crossbeam-utils 0.8.8",
  "lazy_static",
@@ -959,6 +968,7 @@ dependencies = [
  "futures",
  "futures-util",
  "garage_api",
+ "garage_block",
  "garage_db",
  "garage_model 0.7.0",
  "garage_rpc 0.7.0",
@@ -984,6 +994,7 @@ dependencies = [
  "sha2",
  "static_init",
  "structopt",
+ "timeago",
  "tokio",
  "toml",
  "tracing",
@@ -1038,6 +1049,7 @@ dependencies = [
 name = "garage_block"
 version = "0.7.0"
 dependencies = [
+ "arc-swap",
  "async-trait",
  "bytes 1.1.0",
  "futures",
@@ -1065,11 +1077,11 @@ dependencies = [
  "err-derive 0.3.1",
  "heed",
  "hexdump",
- "log",
  "mktemp",
  "pretty_env_logger",
  "rusqlite",
  "sled",
+ "tracing",
 ]
 
 [[package]]
@@ -1164,7 +1176,6 @@ dependencies = [
  "futures-util",
  "garage_util 0.7.0",
  "gethostname",
- "git-version",
  "hex",
  "hyper",
  "k8s-openapi",
@@ -1258,11 +1269,13 @@ dependencies = [
 name = "garage_util"
 version = "0.7.0"
 dependencies = [
+ "async-trait",
  "blake2",
  "chrono",
  "err-derive 0.3.1",
  "futures",
  "garage_db",
+ "git-version",
  "hex",
  "http",
  "hyper",
@@ -1629,7 +1642,7 @@ version = "1.8.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "282a6247722caba404c065016bbfa522806e51714c34f5dfc3e4a3a46fcb4223"
 dependencies = [
- "autocfg",
+ "autocfg 1.1.0",
  "hashbrown",
 ]
 
@@ -1651,6 +1664,16 @@ dependencies = [
  "serde",
 ]
 
+[[package]]
+name = "isolang"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "265ef164908329e47e753c769b14cbb27434abf0c41984dca201484022f09ce5"
+dependencies = [
+ "phf",
+ "phf_codegen",
+]
+
 [[package]]
 name = "itertools"
 version = "0.4.19"
@@ -1714,7 +1737,7 @@ dependencies = [
 
 [[package]]
 name = "k2v-client"
-version = "0.1.0"
+version = "0.0.1"
 dependencies = [
  "base64",
  "clap 3.1.18",
@@ -1975,7 +1998,7 @@ version = "0.6.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "5aa361d4faea93603064a027415f07bd8e1d5c88c9fbf68bf56a285428fd79ce"
 dependencies = [
- "autocfg",
+ "autocfg 1.1.0",
 ]
 
 [[package]]
@@ -2137,7 +2160,7 @@ version = "0.1.44"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d2cc698a63b549a70bc047073d2949cce27cd1c7b0a4a862d08a8031bc2801db"
 dependencies = [
- "autocfg",
+ "autocfg 1.1.0",
  "num-traits",
 ]
 
@@ -2147,7 +2170,7 @@ version = "0.2.14"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9a64b1ec5cda2586e284722486d802acf1f7dbdc623e2bfc57e65ca1cd099290"
 dependencies = [
- "autocfg",
+ "autocfg 1.1.0",
 ]
 
 [[package]]
@@ -2216,7 +2239,7 @@ version = "0.9.72"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7e46109c383602735fa0a2e48dd2b7c892b048e1bf69e5c3b1d804b7d9c203cb"
 dependencies = [
- "autocfg",
+ "autocfg 1.1.0",
  "cc",
  "libc",
  "openssl-src",
@@ -2386,6 +2409,44 @@ dependencies = [
  "indexmap",
 ]
 
+[[package]]
+name = "phf"
+version = "0.7.24"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b3da44b85f8e8dfaec21adae67f95d93244b2ecf6ad2a692320598dcc8e6dd18"
+dependencies = [
+ "phf_shared",
+]
+
+[[package]]
+name = "phf_codegen"
+version = "0.7.24"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b03e85129e324ad4166b06b2c7491ae27fe3ec353af72e72cd1654c7225d517e"
+dependencies = [
+ "phf_generator",
+ "phf_shared",
+]
+
+[[package]]
+name = "phf_generator"
+version = "0.7.24"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "09364cc93c159b8b06b1f4dd8a4398984503483891b0c26b867cf431fb132662"
+dependencies = [
+ "phf_shared",
+ "rand 0.6.5",
+]
+
+[[package]]
+name = "phf_shared"
+version = "0.7.24"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "234f71a15de2288bcb7e3b6515828d22af7ec8598ee6d24c3b526fa0a80b67a0"
+dependencies = [
+ "siphasher",
+]
+
 [[package]]
 name = "pin-project"
 version = "0.4.29"
@@ -2640,6 +2701,25 @@ dependencies = [
  "winapi",
 ]
 
+[[package]]
+name = "rand"
+version = "0.6.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6d71dacdc3c88c1fde3885a3be3fbab9f35724e6ce99467f7d9c5026132184ca"
+dependencies = [
+ "autocfg 0.1.8",
+ "libc",
+ "rand_chacha 0.1.1",
+ "rand_core 0.4.2",
+ "rand_hc",
+ "rand_isaac",
+ "rand_jitter",
+ "rand_os",
+ "rand_pcg",
+ "rand_xorshift",
+ "winapi",
+]
+
 [[package]]
 name = "rand"
 version = "0.8.5"
@@ -2647,10 +2727,20 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404"
 dependencies = [
  "libc",
- "rand_chacha",
+ "rand_chacha 0.3.1",
  "rand_core 0.6.3",
 ]
 
+[[package]]
+name = "rand_chacha"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "556d3a1ca6600bfcbab7c7c91ccb085ac7fbbcd70e008a98742e7847f4f7bcef"
+dependencies = [
+ "autocfg 0.1.8",
+ "rand_core 0.3.1",
+]
+
 [[package]]
 name = "rand_chacha"
 version = "0.3.1"
@@ -2685,6 +2775,77 @@ dependencies = [
  "getrandom",
 ]
 
+[[package]]
+name = "rand_hc"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7b40677c7be09ae76218dc623efbf7b18e34bced3f38883af07bb75630a21bc4"
+dependencies = [
+ "rand_core 0.3.1",
+]
+
+[[package]]
+name = "rand_isaac"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ded997c9d5f13925be2a6fd7e66bf1872597f759fd9dd93513dd7e92e5a5ee08"
+dependencies = [
+ "rand_core 0.3.1",
+]
+
+[[package]]
+name = "rand_jitter"
+version = "0.1.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1166d5c91dc97b88d1decc3285bb0a99ed84b05cfd0bc2341bdf2d43fc41e39b"
+dependencies = [
+ "libc",
+ "rand_core 0.4.2",
+ "winapi",
+]
+
+[[package]]
+name = "rand_os"
+version = "0.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7b75f676a1e053fc562eafbb47838d67c84801e38fc1ba459e8f180deabd5071"
+dependencies = [
+ "cloudabi",
+ "fuchsia-cprng",
+ "libc",
+ "rand_core 0.4.2",
+ "rdrand",
+ "winapi",
+]
+
+[[package]]
+name = "rand_pcg"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "abf9b09b01790cfe0364f52bf32995ea3c39f4d2dd011eac241d2914146d0b44"
+dependencies = [
+ "autocfg 0.1.8",
+ "rand_core 0.4.2",
+]
+
+[[package]]
+name = "rand_xorshift"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cbf7e9e623549b0e21f6e97cf8ecf247c1a8fd2e8a992ae265314300b2455d5c"
+dependencies = [
+ "rand_core 0.3.1",
+]
+
+[[package]]
+name = "rdrand"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "678054eb77286b51581ba43620cc911abf02758c91f93f479767aed0f90458b2"
+dependencies = [
+ "rand_core 0.3.1",
+]
+
 [[package]]
 name = "redox_syscall"
 version = "0.2.11"
@@ -3107,6 +3268,12 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "siphasher"
+version = "0.2.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0b8de496cf83d4ed58b6be86c3a275b8602f6ffe98d3024a869e124147a9a3ac"
+
 [[package]]
 name = "slab"
 version = "0.4.5"
@@ -3355,6 +3522,16 @@ dependencies = [
  "num_threads",
 ]
 
+[[package]]
+name = "timeago"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6ec32dde57efb15c035ac074118d7f32820451395f28cb0524a01d4e94983b26"
+dependencies = [
+ "chrono",
+ "isolang",
+]
+
 [[package]]
 name = "tinyvec"
 version = "1.5.1"

Cargo.nix

@@ -55,7 +55,7 @@ in
     garage_api = rustPackages.unknown.garage_api."0.7.0";
     garage_web = rustPackages.unknown.garage_web."0.7.0";
     garage = rustPackages.unknown.garage."0.7.0";
-    k2v-client = rustPackages.unknown.k2v-client."0.1.0";
+    k2v-client = rustPackages.unknown.k2v-client."0.0.1";
   };
   "registry+https://github.com/rust-lang/crates.io-index".ahash."0.7.6" = overridableMkRustCrate (profileName: rec {
     name = "ahash";
@@ -172,6 +172,16 @@ in
     };
   });
   
+  "registry+https://github.com/rust-lang/crates.io-index".autocfg."0.1.8" = overridableMkRustCrate (profileName: rec {
+    name = "autocfg";
+    version = "0.1.8";
+    registry = "registry+https://github.com/rust-lang/crates.io-index";
+    src = fetchCratesIo { inherit name version; sha256 = "0dde43e75fd43e8a1bf86103336bc699aa8d17ad1be60c76c0bdfd4828e19b78"; };
+    dependencies = {
+      autocfg = rustPackages."registry+https://github.com/rust-lang/crates.io-index".autocfg."1.1.0" { inherit profileName; };
+    };
+  });
+  
   "registry+https://github.com/rust-lang/crates.io-index".autocfg."1.1.0" = overridableMkRustCrate (profileName: rec {
     name = "autocfg";
     version = "1.1.0";
@@ -739,7 +749,7 @@ in
     registry = "registry+https://github.com/rust-lang/crates.io-index";
     src = fetchCratesIo { inherit name version; sha256 = "59a6001667ab124aebae2a495118e11d30984c3a653e99d86d58971708cf5e4b"; };
     dependencies = {
-      ${ if hostPlatform.config == "aarch64-linux-android" || hostPlatform.parsed.cpu.name == "aarch64" && hostPlatform.parsed.kernel.name == "linux" || hostPlatform.config == "aarch64-apple-darwin" then "libc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.121" { inherit profileName; };
+      ${ if hostPlatform.config == "aarch64-linux-android" || hostPlatform.config == "aarch64-apple-darwin" || hostPlatform.parsed.cpu.name == "aarch64" && hostPlatform.parsed.kernel.name == "linux" then "libc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.121" { inherit profileName; };
     };
   });
   
@@ -1364,6 +1374,7 @@ in
       futures = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.21" { inherit profileName; };
       futures_util = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.21" { inherit profileName; };
       garage_api = rustPackages."unknown".garage_api."0.7.0" { inherit profileName; };
+      garage_block = rustPackages."unknown".garage_block."0.7.0" { inherit profileName; };
       garage_db = rustPackages."unknown".garage_db."0.8.0" { inherit profileName; };
       garage_model = rustPackages."unknown".garage_model."0.7.0" { inherit profileName; };
       garage_rpc = rustPackages."unknown".garage_rpc."0.7.0" { inherit profileName; };
@@ -1383,6 +1394,7 @@ in
       serde = rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde."1.0.137" { inherit profileName; };
       serde_bytes = rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde_bytes."0.11.5" { inherit profileName; };
       structopt = rustPackages."registry+https://github.com/rust-lang/crates.io-index".structopt."0.3.26" { inherit profileName; };
+      timeago = rustPackages."registry+https://github.com/rust-lang/crates.io-index".timeago."0.3.1" { inherit profileName; };
       tokio = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.17.0" { inherit profileName; };
       toml = rustPackages."registry+https://github.com/rust-lang/crates.io-index".toml."0.5.8" { inherit profileName; };
       tracing = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing."0.1.32" { inherit profileName; };
@@ -1458,6 +1470,7 @@ in
     registry = "unknown";
     src = fetchCrateLocal (workspaceSrc + "/src/block");
     dependencies = {
+      arc_swap = rustPackages."registry+https://github.com/rust-lang/crates.io-index".arc-swap."1.5.0" { inherit profileName; };
       async_trait = buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".async-trait."0.1.52" { profileName = "__noProfile"; };
       bytes = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.1.0" { inherit profileName; };
       futures = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.21" { inherit profileName; };
@@ -1493,10 +1506,10 @@ in
       err_derive = buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".err-derive."0.3.1" { profileName = "__noProfile"; };
       heed = rustPackages."registry+https://github.com/rust-lang/crates.io-index".heed."0.11.0" { inherit profileName; };
       hexdump = rustPackages."registry+https://github.com/rust-lang/crates.io-index".hexdump."0.1.1" { inherit profileName; };
-      log = rustPackages."registry+https://github.com/rust-lang/crates.io-index".log."0.4.16" { inherit profileName; };
       ${ if rootFeatures' ? "garage_db" then "pretty_env_logger" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".pretty_env_logger."0.4.0" { inherit profileName; };
       rusqlite = rustPackages."registry+https://github.com/rust-lang/crates.io-index".rusqlite."0.27.0" { inherit profileName; };
       sled = rustPackages."registry+https://github.com/rust-lang/crates.io-index".sled."0.34.7" { inherit profileName; };
+      tracing = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing."0.1.32" { inherit profileName; };
     };
     devDependencies = {
       mktemp = rustPackages."registry+https://github.com/rust-lang/crates.io-index".mktemp."0.4.1" { inherit profileName; };
@@ -1612,7 +1625,6 @@ in
       ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web" then "futures_util" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.21" { inherit profileName; };
       ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web" then "garage_util" else null } = rustPackages."unknown".garage_util."0.7.0" { inherit profileName; };
       ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web" then "gethostname" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".gethostname."0.2.3" { inherit profileName; };
-      ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web" then "git_version" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".git-version."0.3.5" { inherit profileName; };
       ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web" then "hex" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; };
       ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web" then "hyper" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".hyper."0.14.18" { inherit profileName; };
       ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_rpc" then "k8s_openapi" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".k8s-openapi."0.13.1" { inherit profileName; };
@@ -1717,11 +1729,13 @@ in
       (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_util") "k2v")
     ];
     dependencies = {
+      ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "async_trait" else null } = buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".async-trait."0.1.52" { profileName = "__noProfile"; };
       ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "blake2" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".blake2."0.9.2" { inherit profileName; };
       ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "chrono" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".chrono."0.4.19" { inherit profileName; };
       ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "err_derive" else null } = buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".err-derive."0.3.1" { profileName = "__noProfile"; };
       ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "futures" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.21" { inherit profileName; };
       ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "garage_db" else null } = rustPackages."unknown".garage_db."0.8.0" { inherit profileName; };
+      ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "git_version" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".git-version."0.3.5" { inherit profileName; };
       ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "hex" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; };
       ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "http" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.6" { inherit profileName; };
       ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "hyper" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".hyper."0.14.18" { inherit profileName; };
@@ -2219,6 +2233,22 @@ in
     };
   });
   
+  "registry+https://github.com/rust-lang/crates.io-index".isolang."1.0.0" = overridableMkRustCrate (profileName: rec {
+    name = "isolang";
+    version = "1.0.0";
+    registry = "registry+https://github.com/rust-lang/crates.io-index";
+    src = fetchCratesIo { inherit name version; sha256 = "265ef164908329e47e753c769b14cbb27434abf0c41984dca201484022f09ce5"; };
+    features = builtins.concatLists [
+      [ "default" ]
+    ];
+    dependencies = {
+      phf = rustPackages."registry+https://github.com/rust-lang/crates.io-index".phf."0.7.24" { inherit profileName; };
+    };
+    buildDependencies = {
+      phf_codegen = buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".phf_codegen."0.7.24" { profileName = "__noProfile"; };
+    };
+  });
+  
   "registry+https://github.com/rust-lang/crates.io-index".itertools."0.4.19" = overridableMkRustCrate (profileName: rec {
     name = "itertools";
     version = "0.4.19";
@@ -2297,9 +2327,9 @@ in
     };
   });
   
-  "unknown".k2v-client."0.1.0" = overridableMkRustCrate (profileName: rec {
+  "unknown".k2v-client."0.0.1" = overridableMkRustCrate (profileName: rec {
     name = "k2v-client";
-    version = "0.1.0";
+    version = "0.0.1";
     registry = "unknown";
     src = fetchCrateLocal (workspaceSrc + "/src/k2v-client");
     features = builtins.concatLists [
@@ -2726,7 +2756,7 @@ in
       [ "os-poll" ]
     ];
     dependencies = {
-      ${ if hostPlatform.isUnix || hostPlatform.parsed.kernel.name == "wasi" then "libc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.121" { inherit profileName; };
+      ${ if hostPlatform.parsed.kernel.name == "wasi" || hostPlatform.isUnix then "libc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.121" { inherit profileName; };
       log = rustPackages."registry+https://github.com/rust-lang/crates.io-index".log."0.4.16" { inherit profileName; };
       ${ if hostPlatform.isWindows then "miow" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".miow."0.3.7" { inherit profileName; };
       ${ if hostPlatform.isWindows then "ntapi" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".ntapi."0.3.7" { inherit profileName; };
@@ -3242,6 +3272,48 @@ in
     };
   });
   
+  "registry+https://github.com/rust-lang/crates.io-index".phf."0.7.24" = overridableMkRustCrate (profileName: rec {
+    name = "phf";
+    version = "0.7.24";
+    registry = "registry+https://github.com/rust-lang/crates.io-index";
+    src = fetchCratesIo { inherit name version; sha256 = "b3da44b85f8e8dfaec21adae67f95d93244b2ecf6ad2a692320598dcc8e6dd18"; };
+    dependencies = {
+      phf_shared = rustPackages."registry+https://github.com/rust-lang/crates.io-index".phf_shared."0.7.24" { inherit profileName; };
+    };
+  });
+  
+  "registry+https://github.com/rust-lang/crates.io-index".phf_codegen."0.7.24" = overridableMkRustCrate (profileName: rec {
+    name = "phf_codegen";
+    version = "0.7.24";
+    registry = "registry+https://github.com/rust-lang/crates.io-index";
+    src = fetchCratesIo { inherit name version; sha256 = "b03e85129e324ad4166b06b2c7491ae27fe3ec353af72e72cd1654c7225d517e"; };
+    dependencies = {
+      phf_generator = rustPackages."registry+https://github.com/rust-lang/crates.io-index".phf_generator."0.7.24" { inherit profileName; };
+      phf_shared = rustPackages."registry+https://github.com/rust-lang/crates.io-index".phf_shared."0.7.24" { inherit profileName; };
+    };
+  });
+  
+  "registry+https://github.com/rust-lang/crates.io-index".phf_generator."0.7.24" = overridableMkRustCrate (profileName: rec {
+    name = "phf_generator";
+    version = "0.7.24";
+    registry = "registry+https://github.com/rust-lang/crates.io-index";
+    src = fetchCratesIo { inherit name version; sha256 = "09364cc93c159b8b06b1f4dd8a4398984503483891b0c26b867cf431fb132662"; };
+    dependencies = {
+      phf_shared = rustPackages."registry+https://github.com/rust-lang/crates.io-index".phf_shared."0.7.24" { inherit profileName; };
+      rand = rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand."0.6.5" { inherit profileName; };
+    };
+  });
+  
+  "registry+https://github.com/rust-lang/crates.io-index".phf_shared."0.7.24" = overridableMkRustCrate (profileName: rec {
+    name = "phf_shared";
+    version = "0.7.24";
+    registry = "registry+https://github.com/rust-lang/crates.io-index";
+    src = fetchCratesIo { inherit name version; sha256 = "234f71a15de2288bcb7e3b6515828d22af7ec8598ee6d24c3b526fa0a80b67a0"; };
+    dependencies = {
+      siphasher = rustPackages."registry+https://github.com/rust-lang/crates.io-index".siphasher."0.2.3" { inherit profileName; };
+    };
+  });
+  
   "registry+https://github.com/rust-lang/crates.io-index".pin-project."0.4.29" = overridableMkRustCrate (profileName: rec {
     name = "pin-project";
     version = "0.4.29";
@@ -3568,6 +3640,34 @@ in
     };
   });
   
+  "registry+https://github.com/rust-lang/crates.io-index".rand."0.6.5" = overridableMkRustCrate (profileName: rec {
+    name = "rand";
+    version = "0.6.5";
+    registry = "registry+https://github.com/rust-lang/crates.io-index";
+    src = fetchCratesIo { inherit name version; sha256 = "6d71dacdc3c88c1fde3885a3be3fbab9f35724e6ce99467f7d9c5026132184ca"; };
+    features = builtins.concatLists [
+      [ "alloc" ]
+      [ "default" ]
+      [ "rand_os" ]
+      [ "std" ]
+    ];
+    dependencies = {
+      ${ if hostPlatform.isUnix then "libc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.121" { inherit profileName; };
+      rand_chacha = rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand_chacha."0.1.1" { inherit profileName; };
+      rand_core = rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand_core."0.4.2" { inherit profileName; };
+      rand_hc = rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand_hc."0.1.0" { inherit profileName; };
+      rand_isaac = rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand_isaac."0.1.1" { inherit profileName; };
+      rand_jitter = rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand_jitter."0.1.4" { inherit profileName; };
+      rand_os = rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand_os."0.1.3" { inherit profileName; };
+      rand_pcg = rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand_pcg."0.1.2" { inherit profileName; };
+      rand_xorshift = rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand_xorshift."0.1.1" { inherit profileName; };
+      ${ if hostPlatform.isWindows then "winapi" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".winapi."0.3.9" { inherit profileName; };
+    };
+    buildDependencies = {
+      autocfg = buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".autocfg."0.1.8" { profileName = "__noProfile"; };
+    };
+  });
+  
   "registry+https://github.com/rust-lang/crates.io-index".rand."0.8.5" = overridableMkRustCrate (profileName: rec {
     name = "rand";
     version = "0.8.5";
@@ -3590,6 +3690,19 @@ in
     };
   });
   
+  "registry+https://github.com/rust-lang/crates.io-index".rand_chacha."0.1.1" = overridableMkRustCrate (profileName: rec {
+    name = "rand_chacha";
+    version = "0.1.1";
+    registry = "registry+https://github.com/rust-lang/crates.io-index";
+    src = fetchCratesIo { inherit name version; sha256 = "556d3a1ca6600bfcbab7c7c91ccb085ac7fbbcd70e008a98742e7847f4f7bcef"; };
+    dependencies = {
+      rand_core = rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand_core."0.3.1" { inherit profileName; };
+    };
+    buildDependencies = {
+      autocfg = buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".autocfg."0.1.8" { profileName = "__noProfile"; };
+    };
+  });
+  
   "registry+https://github.com/rust-lang/crates.io-index".rand_chacha."0.3.1" = overridableMkRustCrate (profileName: rec {
     name = "rand_chacha";
     version = "0.3.1";
@@ -3644,6 +3757,93 @@ in
     };
   });
   
+  "registry+https://github.com/rust-lang/crates.io-index".rand_hc."0.1.0" = overridableMkRustCrate (profileName: rec {
+    name = "rand_hc";
+    version = "0.1.0";
+    registry = "registry+https://github.com/rust-lang/crates.io-index";
+    src = fetchCratesIo { inherit name version; sha256 = "7b40677c7be09ae76218dc623efbf7b18e34bced3f38883af07bb75630a21bc4"; };
+    dependencies = {
+      rand_core = rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand_core."0.3.1" { inherit profileName; };
+    };
+  });
+  
+  "registry+https://github.com/rust-lang/crates.io-index".rand_isaac."0.1.1" = overridableMkRustCrate (profileName: rec {
+    name = "rand_isaac";
+    version = "0.1.1";
+    registry = "registry+https://github.com/rust-lang/crates.io-index";
+    src = fetchCratesIo { inherit name version; sha256 = "ded997c9d5f13925be2a6fd7e66bf1872597f759fd9dd93513dd7e92e5a5ee08"; };
+    dependencies = {
+      rand_core = rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand_core."0.3.1" { inherit profileName; };
+    };
+  });
+  
+  "registry+https://github.com/rust-lang/crates.io-index".rand_jitter."0.1.4" = overridableMkRustCrate (profileName: rec {
+    name = "rand_jitter";
+    version = "0.1.4";
+    registry = "registry+https://github.com/rust-lang/crates.io-index";
+    src = fetchCratesIo { inherit name version; sha256 = "1166d5c91dc97b88d1decc3285bb0a99ed84b05cfd0bc2341bdf2d43fc41e39b"; };
+    features = builtins.concatLists [
+      [ "std" ]
+    ];
+    dependencies = {
+      ${ if hostPlatform.parsed.kernel.name == "darwin" || hostPlatform.parsed.kernel.name == "ios" then "libc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.121" { inherit profileName; };
+      rand_core = rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand_core."0.4.2" { inherit profileName; };
+      ${ if hostPlatform.parsed.kernel.name == "windows" then "winapi" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".winapi."0.3.9" { inherit profileName; };
+    };
+  });
+  
+  "registry+https://github.com/rust-lang/crates.io-index".rand_os."0.1.3" = overridableMkRustCrate (profileName: rec {
+    name = "rand_os";
+    version = "0.1.3";
+    registry = "registry+https://github.com/rust-lang/crates.io-index";
+    src = fetchCratesIo { inherit name version; sha256 = "7b75f676a1e053fc562eafbb47838d67c84801e38fc1ba459e8f180deabd5071"; };
+    dependencies = {
+      ${ if hostPlatform.parsed.kernel.name == "cloudabi" then "cloudabi" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".cloudabi."0.0.3" { inherit profileName; };
+      ${ if hostPlatform.parsed.kernel.name == "fuchsia" then "fuchsia_cprng" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".fuchsia-cprng."0.1.1" { inherit profileName; };
+      ${ if hostPlatform.isUnix then "libc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.121" { inherit profileName; };
+      rand_core = rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand_core."0.4.2" { inherit profileName; };
+      ${ if hostPlatform.parsed.abi.name == "sgx" then "rdrand" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".rdrand."0.4.0" { inherit profileName; };
+      ${ if hostPlatform.isWindows then "winapi" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".winapi."0.3.9" { inherit profileName; };
+    };
+  });
+  
+  "registry+https://github.com/rust-lang/crates.io-index".rand_pcg."0.1.2" = overridableMkRustCrate (profileName: rec {
+    name = "rand_pcg";
+    version = "0.1.2";
+    registry = "registry+https://github.com/rust-lang/crates.io-index";
+    src = fetchCratesIo { inherit name version; sha256 = "abf9b09b01790cfe0364f52bf32995ea3c39f4d2dd011eac241d2914146d0b44"; };
+    dependencies = {
+      rand_core = rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand_core."0.4.2" { inherit profileName; };
+    };
+    buildDependencies = {
+      autocfg = buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".autocfg."0.1.8" { profileName = "__noProfile"; };
+    };
+  });
+  
+  "registry+https://github.com/rust-lang/crates.io-index".rand_xorshift."0.1.1" = overridableMkRustCrate (profileName: rec {
+    name = "rand_xorshift";
+    version = "0.1.1";
+    registry = "registry+https://github.com/rust-lang/crates.io-index";
+    src = fetchCratesIo { inherit name version; sha256 = "cbf7e9e623549b0e21f6e97cf8ecf247c1a8fd2e8a992ae265314300b2455d5c"; };
+    dependencies = {
+      rand_core = rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand_core."0.3.1" { inherit profileName; };
+    };
+  });
+  
+  "registry+https://github.com/rust-lang/crates.io-index".rdrand."0.4.0" = overridableMkRustCrate (profileName: rec {
+    name = "rdrand";
+    version = "0.4.0";
+    registry = "registry+https://github.com/rust-lang/crates.io-index";
+    src = fetchCratesIo { inherit name version; sha256 = "678054eb77286b51581ba43620cc911abf02758c91f93f479767aed0f90458b2"; };
+    features = builtins.concatLists [
+      [ "default" ]
+      [ "std" ]
+    ];
+    dependencies = {
+      rand_core = rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand_core."0.3.1" { inherit profileName; };
+    };
+  });
+  
   "registry+https://github.com/rust-lang/crates.io-index".redox_syscall."0.2.11" = overridableMkRustCrate (profileName: rec {
     name = "redox_syscall";
     version = "0.2.11";
@@ -3738,7 +3938,7 @@ in
     ];
     dependencies = {
       ${ if hostPlatform.parsed.kernel.name == "android" || hostPlatform.parsed.kernel.name == "linux" then "libc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.121" { inherit profileName; };
-      ${ if hostPlatform.parsed.kernel.name == "dragonfly" || hostPlatform.parsed.kernel.name == "freebsd" || hostPlatform.parsed.kernel.name == "illumos" || hostPlatform.parsed.kernel.name == "netbsd" || hostPlatform.parsed.kernel.name == "openbsd" || hostPlatform.parsed.kernel.name == "solaris" || hostPlatform.parsed.kernel.name == "android" || hostPlatform.parsed.kernel.name == "linux" then "once_cell" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".once_cell."1.10.0" { inherit profileName; };
+      ${ if hostPlatform.parsed.kernel.name == "android" || hostPlatform.parsed.kernel.name == "linux" || hostPlatform.parsed.kernel.name == "dragonfly" || hostPlatform.parsed.kernel.name == "freebsd" || hostPlatform.parsed.kernel.name == "illumos" || hostPlatform.parsed.kernel.name == "netbsd" || hostPlatform.parsed.kernel.name == "openbsd" || hostPlatform.parsed.kernel.name == "solaris" then "once_cell" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".once_cell."1.10.0" { inherit profileName; };
       ${ if hostPlatform.parsed.cpu.name == "i686" || hostPlatform.parsed.cpu.name == "x86_64" || (hostPlatform.parsed.cpu.name == "aarch64" || hostPlatform.parsed.cpu.name == "armv6l" || hostPlatform.parsed.cpu.name == "armv7l") && (hostPlatform.parsed.kernel.name == "android" || hostPlatform.parsed.kernel.name == "fuchsia" || hostPlatform.parsed.kernel.name == "linux") then "spin" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".spin."0.5.2" { inherit profileName; };
       untrusted = rustPackages."registry+https://github.com/rust-lang/crates.io-index".untrusted."0.7.1" { inherit profileName; };
       ${ if hostPlatform.parsed.cpu.name == "wasm32" && hostPlatform.parsed.vendor.name == "unknown" && hostPlatform.parsed.kernel.name == "unknown" && hostPlatform.parsed.abi.name == "" then "web_sys" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".web-sys."0.3.56" { inherit profileName; };
@@ -4213,6 +4413,13 @@ in
     };
   });
   
+  "registry+https://github.com/rust-lang/crates.io-index".siphasher."0.2.3" = overridableMkRustCrate (profileName: rec {
+    name = "siphasher";
+    version = "0.2.3";
+    registry = "registry+https://github.com/rust-lang/crates.io-index";
+    src = fetchCratesIo { inherit name version; sha256 = "0b8de496cf83d4ed58b6be86c3a275b8602f6ffe98d3024a869e124147a9a3ac"; };
+  });
+  
   "registry+https://github.com/rust-lang/crates.io-index".slab."0.4.5" = overridableMkRustCrate (profileName: rec {
     name = "slab";
     version = "0.4.5";
@@ -4410,7 +4617,7 @@ in
       [ "proc-macro" ]
       [ "quote" ]
       [ "visit" ]
-      (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client") "visit-mut")
+      [ "visit-mut" ]
     ];
     dependencies = {
       proc_macro2 = rustPackages."registry+https://github.com/rust-lang/crates.io-index".proc-macro2."1.0.36" { inherit profileName; };
@@ -4539,6 +4746,23 @@ in
     };
   });
   
+  "registry+https://github.com/rust-lang/crates.io-index".timeago."0.3.1" = overridableMkRustCrate (profileName: rec {
+    name = "timeago";
+    version = "0.3.1";
+    registry = "registry+https://github.com/rust-lang/crates.io-index";
+    src = fetchCratesIo { inherit name version; sha256 = "6ec32dde57efb15c035ac074118d7f32820451395f28cb0524a01d4e94983b26"; };
+    features = builtins.concatLists [
+      [ "chrono" ]
+      [ "default" ]
+      [ "isolang" ]
+      [ "translations" ]
+    ];
+    dependencies = {
+      chrono = rustPackages."registry+https://github.com/rust-lang/crates.io-index".chrono."0.4.19" { inherit profileName; };
+      isolang = rustPackages."registry+https://github.com/rust-lang/crates.io-index".isolang."1.0.0" { inherit profileName; };
+    };
+  });
+  
   "registry+https://github.com/rust-lang/crates.io-index".tinyvec."1.5.1" = overridableMkRustCrate (profileName: rec {
     name = "tinyvec";
     version = "1.5.1";
@@ -4883,19 +5107,19 @@ in
     registry = "registry+https://github.com/rust-lang/crates.io-index";
     src = fetchCratesIo { inherit name version; sha256 = "4a1bdf54a7c28a2bbf701e1d2233f6c77f473486b94bee4f9678da5a148dca7f"; };
     features = builtins.concatLists [
-      (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client") "attributes")
-      (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client") "default")
+      [ "attributes" ]
+      [ "default" ]
       (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_web") "log")
       (lib.optional (rootFeatures' ? "garage") "log-always")
-      (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client") "std")
-      (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client") "tracing-attributes")
+      [ "std" ]
+      [ "tracing-attributes" ]
     ];
     dependencies = {
-      ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "cfg_if" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".cfg-if."1.0.0" { inherit profileName; };
+      cfg_if = rustPackages."registry+https://github.com/rust-lang/crates.io-index".cfg-if."1.0.0" { inherit profileName; };
       ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_web" then "log" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".log."0.4.16" { inherit profileName; };
-      ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "pin_project_lite" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".pin-project-lite."0.2.8" { inherit profileName; };
-      ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "tracing_attributes" else null } = buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing-attributes."0.1.20" { profileName = "__noProfile"; };
-      ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "tracing_core" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing-core."0.1.23" { inherit profileName; };
+      pin_project_lite = rustPackages."registry+https://github.com/rust-lang/crates.io-index".pin-project-lite."0.2.8" { inherit profileName; };
+      tracing_attributes = buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing-attributes."0.1.20" { profileName = "__noProfile"; };
+      tracing_core = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing-core."0.1.23" { inherit profileName; };
     };
   });
   
@@ -5331,10 +5555,10 @@ in
     ];
     dependencies = {
       ${ if hostPlatform.config == "aarch64-uwp-windows-msvc" || hostPlatform.config == "aarch64-pc-windows-msvc" then "windows_aarch64_msvc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_aarch64_msvc."0.32.0" { inherit profileName; };
-      ${ if hostPlatform.config == "i686-pc-windows-gnu" || hostPlatform.config == "i686-uwp-windows-gnu" then "windows_i686_gnu" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_i686_gnu."0.32.0" { inherit profileName; };
-      ${ if hostPlatform.config == "i686-uwp-windows-msvc" || hostPlatform.config == "i686-pc-windows-msvc" then "windows_i686_msvc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_i686_msvc."0.32.0" { inherit profileName; };
-      ${ if hostPlatform.config == "x86_64-uwp-windows-gnu" || hostPlatform.config == "x86_64-pc-windows-gnu" then "windows_x86_64_gnu" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_x86_64_gnu."0.32.0" { inherit profileName; };
-      ${ if hostPlatform.config == "x86_64-uwp-windows-msvc" || hostPlatform.config == "x86_64-pc-windows-msvc" then "windows_x86_64_msvc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_x86_64_msvc."0.32.0" { inherit profileName; };
+      ${ if hostPlatform.config == "i686-uwp-windows-gnu" || hostPlatform.config == "i686-pc-windows-gnu" then "windows_i686_gnu" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_i686_gnu."0.32.0" { inherit profileName; };
+      ${ if hostPlatform.config == "i686-pc-windows-msvc" || hostPlatform.config == "i686-uwp-windows-msvc" then "windows_i686_msvc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_i686_msvc."0.32.0" { inherit profileName; };
+      ${ if hostPlatform.config == "x86_64-pc-windows-gnu" || hostPlatform.config == "x86_64-uwp-windows-gnu" then "windows_x86_64_gnu" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_x86_64_gnu."0.32.0" { inherit profileName; };
+      ${ if hostPlatform.config == "x86_64-pc-windows-msvc" || hostPlatform.config == "x86_64-uwp-windows-msvc" then "windows_x86_64_msvc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_x86_64_msvc."0.32.0" { inherit profileName; };
     };
   });
   

Makefile

@@ -1,13 +1,27 @@
-.PHONY: doc all release shell
+.PHONY: doc all release shell run1 run2 run3
 
 all:
 	clear; cargo build --all-features
 
-doc:
-	cd doc/book; mdbook build
-
 release:
 	nix-build --arg release true
 
 shell:
 	nix-shell
+
+# ----
+
+run1:
+	RUST_LOG=garage=debug ./target/debug/garage -c tmp/config1.toml server
+run1rel:
+	RUST_LOG=garage=debug ./target/release/garage -c tmp/config1.toml server
+
+run2:
+	RUST_LOG=garage=debug ./target/debug/garage -c tmp/config2.toml server
+run2rel:
+	RUST_LOG=garage=debug ./target/release/garage -c tmp/config2.toml server
+
+run3:
+	RUST_LOG=garage=debug ./target/debug/garage -c tmp/config3.toml server
+run3rel:
+	RUST_LOG=garage=debug ./target/release/garage -c tmp/config3.toml server

default.nix

@@ -1,147 +1,33 @@
 {
   system ? builtins.currentSystem,
-  release ? false,
-  target ? "x86_64-unknown-linux-musl",
-  compileMode ? null,
   git_version ? null,
 }:
 
 with import ./nix/common.nix;
 
-let
-  crossSystem = { config = target; };
-in let
-  log = v: builtins.trace v v;
+let 
+  pkgs = import pkgsSrc { };
+  compile = import ./nix/compile.nix;
+  build_debug_and_release = (target: {
+    debug = (compile { inherit target git_version; release = false; }).workspace.garage { compileMode = "build"; }; 
+    release = (compile { inherit target git_version; release = true; }).workspace.garage { compileMode = "build"; }; 
+  });
+  test = (rustPkgs: pkgs.symlinkJoin {
+    name ="garage-tests";
+    paths = builtins.map (key: rustPkgs.workspace.${key} { compileMode = "test"; }) (builtins.attrNames rustPkgs.workspace);
+  });
 
-  pkgs = import pkgsSrc {
-    inherit system crossSystem;
-    overlays = [ cargo2nixOverlay ];
+in {
+  pkgs = {
+    amd64 = build_debug_and_release "x86_64-unknown-linux-musl";
+    i386 = build_debug_and_release "i686-unknown-linux-musl";
+    arm64 = build_debug_and_release "aarch64-unknown-linux-musl";
+    arm = build_debug_and_release "armv6l-unknown-linux-musleabihf";
   };
-
-
-  /*
-   Rust and Nix triples are not the same. Cargo2nix has a dedicated library
-   to convert Nix triples to Rust ones. We need this conversion as we want to
-   set later options linked to our (rust) target in a generic way. Not only
-   the triple terminology is different, but also the "roles" are named differently.
-   Nix uses a build/host/target terminology where Nix's "host" maps to Cargo's "target".
-  */
-  rustTarget = log (pkgs.rustBuilder.rustLib.rustTriple pkgs.stdenv.hostPlatform);
-
-  /*
-   Cargo2nix is built for rustOverlay which installs Rust from Mozilla releases.
-   We want our own Rust to avoid incompatibilities, like we had with musl 1.2.0.
-   rustc was built with musl < 1.2.0 and nix shipped musl >= 1.2.0 which lead to compilation breakage.
-   So we want a Rust release that is bound to our Nix repository to avoid these problems.
-   See here for more info: https://musl.libc.org/time64.html
-   Because Cargo2nix does not support the Rust environment shipped by NixOS,
-   we emulate the structure of the Rust object created by rustOverlay.
-   In practise, rustOverlay ships rustc+cargo in a single derivation while
-   NixOS ships them in separate ones. We reunite them with symlinkJoin.
-   */
-  rustChannel = pkgs.symlinkJoin {
-    name ="rust-channel";
-    paths = [ 
-      pkgs.rustPlatform.rust.rustc
-      pkgs.rustPlatform.rust.cargo
-    ];
+  test = {
+    amd64 = test (compile { inherit git_version; target = "x86_64-unknown-linux-musl"; });
   };
-
-  /*
-   Cargo2nix provides many overrides by default, you can take inspiration from them:
-   https://github.com/cargo2nix/cargo2nix/blob/master/overlay/overrides.nix
-
-   You can have a complete list of the available options by looking at the overriden object, mkcrate:
-   https://github.com/cargo2nix/cargo2nix/blob/master/overlay/mkcrate.nix
-  */
-  overrides = pkgs.rustBuilder.overrides.all ++ [
-    /*
-     [1] We need to alter Nix hardening to be able to statically compile: PIE,
-     Position Independent Executables seems to be supported only on amd64. Having
-     this flags set either make our executables crash or compile as dynamic on many platforms.
-     In the following section codegenOpts, we reactive it for the supported targets
-     (only amd64 curently) through the `-static-pie` flag. PIE is a feature used
-     by ASLR, which helps mitigate security issues.
-     Learn more about Nix Hardening: https://github.com/NixOS/nixpkgs/blob/master/pkgs/build-support/cc-wrapper/add-hardening.sh
-
-     [2] We want to inject the git version while keeping the build deterministic.
-     As we do not want to consider the .git folder as part of the input source,
-     we ask the user (the CI often) to pass the value to Nix.
-    */
-    (pkgs.rustBuilder.rustLib.makeOverride {
-      name = "garage_rpc";
-      overrideAttrs = drv:
-        /* [1] */ { hardeningDisable = [ "pie" ]; }
-        //
-        /* [2] */ (if git_version != null then {
-          preConfigure = ''
-            ${drv.preConfigure or ""}
-            export GIT_VERSION="${git_version}"
-          '';
-        } else {});
-    })
-
-    /*
-     We ship some parts of the code disabled by default by putting them behind a flag.
-     It speeds up the compilation (when the feature is not required) and released crates have less dependency by default (less attack surface, disk space, etc.).
-     But we want to ship these additional features when we release Garage.
-     In the end, we chose to exclude all features from debug builds while putting (all of) them in the release builds.
-     Currently, the only feature of Garage is kubernetes-discovery from the garage_rpc crate.
-    */
-    (pkgs.rustBuilder.rustLib.makeOverride {
-      name = "garage_rpc";
-      overrideArgs = old:
-        {
-          features = if release then [ "kubernetes-discovery" ] else [];
-        };
-    })
-  ];
-
-  packageFun = import ./Cargo.nix;
-
-  /*
-    We compile fully static binaries with musl to simplify deployment on most systems.
-    When possible, we reactivate PIE hardening (see above).
-
-    Also, if you set the RUSTFLAGS environment variable, the following parameters will
-    be ignored.
-
-    For more information on static builds, please refer to Rust's RFC 1721.
-    https://rust-lang.github.io/rfcs/1721-crt-static.html#specifying-dynamicstatic-c-runtime-linkage
-  */
-
-  codegenOpts = {
-   "armv6l-unknown-linux-musleabihf" = [ "target-feature=+crt-static" "link-arg=-static" ]; /* compile as dynamic with static-pie */
-   "aarch64-unknown-linux-musl" = [ "target-feature=+crt-static" "link-arg=-static" ]; /* segfault with static-pie */
-   "i686-unknown-linux-musl" = [ "target-feature=+crt-static" "link-arg=-static" ]; /* segfault with static-pie */
-   "x86_64-unknown-linux-musl" = [ "target-feature=+crt-static" "link-arg=-static-pie" ];
+  clippy = {
+    amd64 = (compile { inherit git_version; compiler = "clippy"; }).workspace.garage { compileMode = "build"; } ;
   };
-
-  /*
-   The following definition is not elegant as we use a low level function of Cargo2nix
-   that enables us to pass our custom rustChannel object. We need this low level definition
-   to pass Nix's Rust toolchains instead of Mozilla's one.
-
-   target is mandatory but must be kept to null to allow cargo2nix to set it to the appropriate value
-   for each crate.
-  */ 
-  rustPkgs = pkgs.rustBuilder.makePackageSet {
-    inherit packageFun rustChannel release codegenOpts;
-    packageOverrides = overrides;
-    target = null;
-
-    buildRustPackages = pkgs.buildPackages.rustBuilder.makePackageSet {
-      inherit rustChannel packageFun codegenOpts;
-      packageOverrides = overrides;
-      target = null;
-    };
-  };
-
-
-in
-  if compileMode == "test"
-    then pkgs.symlinkJoin {
-      name ="garage-tests";
-      paths = builtins.map (key: rustPkgs.workspace.${key} { inherit compileMode; }) (builtins.attrNames rustPkgs.workspace);
-    }
-    else rustPkgs.workspace.garage { inherit compileMode; }
+}

doc/book/cookbook/reverse-proxy.md

@@ -280,3 +280,25 @@ Traefik's caching middleware is only available on [entreprise version](https://d
 [http.middlewares]
   [http.middlewares.gzip_compress.compress]
 ```
+
+## Caddy
+
+Your Caddy configuration can be as simple as:
+
+```caddy
+s3.garage.tld, *.s3.garage.tld {
+	reverse_proxy localhost:3900 192.168.1.2:3900 example.tld:3900
+}
+
+*.web.garage.tld {
+	reverse_proxy localhost:3902 192.168.1.2:3900 example.tld:3900
+}
+
+admin.garage.tld {
+	reverse_proxy localhost:3903
+}
+```
+
+But at the same time, the `reverse_proxy` is very flexible.
+For a production deployment, you should [read its documentation](https://caddyserver.com/docs/caddyfile/directives/reverse_proxy) as it supports features like DNS discovery of upstreams, load balancing with checks, streaming parameters, etc.
+

doc/book/reference-manual/s3-compatibility.md

@@ -148,7 +148,7 @@ Please open an issue if you have a use case for replication.
 | [PutBucketReplication](https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutBucketReplication.html) | ❌ Missing | ❌| ⚠ | ❌| ❌|
 
 *Note: Ceph documentation briefly says that Ceph supports
-[replication though the S3 API](https://docs.ceph.com/en/latest/radosgw/multisite-sync-policy/#s3-replication-api)
+[replication through the S3 API](https://docs.ceph.com/en/latest/radosgw/multisite-sync-policy/#s3-replication-api)
 but with some limitations.
 Additionaly, replication endpoints are not documented in the S3 compatibility page so I don't know what kind of support we can expect.*
 

doc/talks/2022-06-23-stack/.gitignore

@@ -0,0 +1,14 @@
+*
+
+!assets
+
+!.gitignore
+!*.svg
+!*.png
+!*.jpg
+!*.tex
+!Makefile
+!.gitignore
+!assets/*.drawio.pdf
+
+!talk.pdf

doc/talks/2022-06-23-stack/Makefile

@@ -0,0 +1,5 @@
+talk.pdf: talk.tex assets/consistent_hashing_1.pdf assets/consistent_hashing_2.pdf assets/consistent_hashing_3.pdf assets/consistent_hashing_4.pdf assets/garage_tables.pdf assets/deuxfleurs.pdf
+	pdflatex talk.tex
+
+assets/%.pdf: assets/%.svg
+	inkscape -D -z --file=$^ --export-pdf=$@

doc/talks/2022-06-23-stack/assets/aerogramme_components1.drawio.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9198d0cfc0e04a56f84a353dd660b294ecf0dc68fe429892c8e9e604de016748
+size 31966

doc/talks/2022-06-23-stack/assets/aerogramme_components2.drawio.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:14dbf4a45545889babb2fdc6580399811366f6cc786cb28e3467cbcedbfe9482
+size 31688

doc/talks/2022-06-23-stack/assets/aerogramme_datatype.drawio.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:21a3bb004ad35c8b47f14195c720048f8db31f47bd6346c1d747000570149c67
+size 31073

doc/talks/2022-06-23-stack/assets/aerogramme_keys.drawio.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c4e109078a784c3999ca2ccd1adda8ff5793d1b230a9e27b4ad2e419afa0d37d
+size 25145

doc/talks/2022-06-23-stack/assets/deuxfleurs.svg

@@ -0,0 +1,91 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<svg
+   viewBox="0 0 70.424515 70.300102"
+   version="1.1"
+   id="svg8"
+   sodipodi:docname="logo.svg"
+   inkscape:version="1.1 (c68e22c387, 2021-05-23)"
+   inkscape:export-filename="/home/quentin/Documents/dev/deuxfleurs/site/src/img/logo.png"
+   inkscape:export-xdpi="699.30194"
+   inkscape:export-ydpi="699.30194"
+   width="70.424515"
+   height="70.300102"
+   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
+   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
+   xmlns="http://www.w3.org/2000/svg"
+   xmlns:svg="http://www.w3.org/2000/svg">
+  <defs
+     id="defs12" />
+  <sodipodi:namedview
+     id="namedview10"
+     pagecolor="#ffffff"
+     bordercolor="#666666"
+     borderopacity="1.0"
+     inkscape:pageshadow="2"
+     inkscape:pageopacity="0.0"
+     inkscape:pagecheckerboard="0"
+     showgrid="false"
+     inkscape:zoom="12.125"
+     inkscape:cx="43.092783"
+     inkscape:cy="48.082474"
+     inkscape:window-width="3072"
+     inkscape:window-height="1659"
+     inkscape:window-x="0"
+     inkscape:window-y="0"
+     inkscape:window-maximized="1"
+     inkscape:current-layer="svg8" />
+  <g
+     id="g79969"
+     transform="translate(-0.827,34.992103)">
+    <path
+       fill="#ffffff"
+       d="m 15.632,34.661 c -0.799,-0.597 -1.498,-1.484 -2.035,-2.592 l -0.228,-0.47 -0.46,0.249 c -0.975,0.528 -1.913,0.858 -2.744,0.969 L 9.963,29.061 6.327,30.029 C 6.17,29.175 6.202,28.142 6.423,27.007 L 6.526,26.482 5.994,26.416 C 4.752,26.262 3.688,25.891 2.89,25.336 L 4.411,22.419 1.423,20.896 C 1.742,19.952 2.371,19.014 3.257,18.161 L 3.634,17.798 3.255,17.438 C 2.452,16.674 1.847,15.884 1.485,15.127 L 4.995,13.774 2.95,10.615 C 3.69,10.213 4.643,9.929 5.739,9.783 L 6.258,9.715 6.167,9.201 C 5.952,7.99 5.995,6.863 6.291,5.913 l 3.308,0.523 0.524,-3.308 c 0.988,0.013 2.08,0.326 3.164,0.907 L 13.749,4.283 13.975,3.81 C 14.454,2.807 15.019,1.986 15.628,1.406 L 18,4.326 20.372,1.406 c 0.609,0.58 1.175,1.401 1.653,2.404 l 0.226,0.473 0.462,-0.247 C 23.798,3.455 24.891,3.142 25.877,3.13 L 26.4,6.438 29.71,5.913 c 0.296,0.951 0.34,2.078 0.124,3.288 l -0.092,0.515 0.518,0.069 c 1.095,0.145 2.048,0.43 2.788,0.832 l -2.046,3.156 3.511,1.355 c -0.361,0.757 -0.966,1.547 -1.77,2.311 l -0.379,0.36 0.377,0.363 c 0.888,0.854 1.516,1.793 1.835,2.736 l -2.984,1.52 1.521,2.984 c -0.812,0.574 -1.871,0.964 -3.094,1.134 l -0.518,0.072 0.096,0.514 c 0.201,1.089 0.226,2.083 0.073,2.909 l -3.634,-0.97 -0.204,3.757 c -0.83,-0.11 -1.768,-0.44 -2.742,-0.968 l -0.459,-0.249 -0.228,0.47 c -0.539,1.107 -1.237,1.994 -2.036,2.591 L 18,32.293 Z"
+       id="path2" />
+    <path
+       d="M 7.092,10.678 C 6.562,9.189 6.394,7.708 6.66,6.478 l 2.368,0.375 0.987,0.156 0.157,-0.988 0.375,-2.368 C 11.808,3.78 13.16,4.396 14.409,5.359 14.527,5.022 14.653,4.696 14.791,4.392 13.24,3.257 11.568,2.629 10.061,2.629 9.938,2.629 9.816,2.633 9.695,2.642 L 9.184,5.865 5.96,5.354 C 5.36,6.841 5.395,8.769 6.045,10.747 6.38,10.71 6.729,10.686 7.092,10.678 Z M 21.593,5.359 c 1.248,-0.962 2.6,-1.578 3.86,-1.705 l 0.376,2.368 0.156,0.988 0.987,-0.157 2.369,-0.376 c 0.266,1.23 0.098,2.71 -0.432,4.2 0.361,0.009 0.711,0.032 1.046,0.07 C 30.606,8.769 30.64,6.841 30.04,5.353 L 26.815,5.865 26.304,2.641 c -0.12,-0.008 -0.242,-0.012 -0.365,-0.012 -1.507,0 -3.179,0.628 -4.73,1.762 0.14,0.306 0.266,0.631 0.384,0.968 z M 7.368,27 h 0.035 c 0.067,0 0.157,-0.604 0.26,-0.947 -0.098,0.004 -0.197,0.046 -0.294,0.046 -1.496,0 -2.826,-0.303 -3.83,-0.89 L 4.628,23.081 5.082,22.194 4.191,21.742 2.055,20.654 C 2.563,19.503 3.57,18.404 4.873,17.511 4.586,17.292 4.312,17.07 4.063,16.842 2.376,18.059 1.217,19.597 0.828,21.152 l 2.908,1.483 -1.482,2.843 C 3.475,26.501 5.303,27 7.368,27 Z m 27.806,-5.846 c -0.39,-1.555 -1.548,-3.093 -3.234,-4.311 -0.25,0.228 -0.523,0.451 -0.81,0.669 1.304,0.893 2.31,1.992 2.817,3.145 l -2.136,1.088 -0.891,0.453 0.454,0.892 1.089,2.137 c -1.004,0.587 -2.332,0.904 -3.828,0.904 -0.099,0 -0.199,-0.01 -0.299,-0.013 0.103,0.344 0.192,0.683 0.26,1.011 l 0.039,0.002 c 2.066,0 3.892,-0.563 5.112,-1.587 l -1.482,-2.908 z m -12.653,9.182 c -0.447,1.517 -1.181,2.812 -2.119,3.651 L 18.707,32.293 18,31.586 l -0.707,0.707 -1.695,1.694 c -0.938,-0.839 -1.673,-2.136 -2.12,-3.652 -0.296,0.206 -0.593,0.397 -0.886,0.563 0.636,1.98 1.741,3.559 3.1,4.409 L 18,33 l 2.308,2.308 c 1.358,-0.851 2.464,-2.428 3.101,-4.408 -0.295,-0.168 -0.591,-0.359 -0.888,-0.564 z"
+       fill="#ea596e"
+       id="path4" />
+    <path
+       fill="#ea596e"
+       d="m 20.118,5.683 c 0.426,1.146 0.748,2.596 0.841,4.284 l 0.2,3.683 3.564,-0.946 c 1.32,-0.351 2.655,-0.536 3.86,-0.536 0.16,0 0.318,0.003 0.474,0.01 l -1.827,2.819 3.139,1.211 c -0.958,0.759 -2.237,1.514 -3.814,2.123 l -3.441,1.328 2.001,3.099 c 0.918,1.42 1.509,2.782 1.838,3.96 L 23.709,25.853 23.527,29.21 C 22.508,28.533 21.395,27.55 20.329,26.237 L 18,23.374 15.672,26.236 c -1.066,1.312 -2.179,2.295 -3.198,2.972 l -0.18,-3.354 -3.248,0.864 c 0.329,-1.178 0.921,-2.54 1.839,-3.961 L 12.889,19.658 9.447,18.33 C 7.87,17.721 6.591,16.967 5.633,16.208 L 8.768,15 6.941,12.177 c 0.155,-0.006 0.313,-0.01 0.473,-0.01 1.206,0 2.541,0.185 3.861,0.536 l 3.564,0.947 0.202,-3.683 c 0.092,-1.688 0.415,-3.138 0.84,-4.284 L 18,8.292 20.118,5.683 M 20.308,0.692 18,3.533 15.692,0.692 C 13.703,2.224 12.271,5.684 12.046,9.804 10.429,9.374 8.854,9.167 7.414,9.167 c -2.11,0 -3.929,0.445 -5.161,1.289 l 1.989,3.073 -3.415,1.316 c 0.842,2.366 3.69,4.797 7.54,6.283 -2.241,3.465 -3.116,7.106 -2.407,9.516 l 3.537,-0.941 0.196,3.654 c 2.512,-0.07 5.703,-2.027 8.307,-5.228 2.603,3.201 5.796,5.158 8.306,5.228 l 0.198,-3.655 3.535,0.943 c 0.71,-2.411 -0.165,-6.05 -2.404,-9.517 3.849,-1.485 6.696,-3.918 7.538,-6.283 l -3.415,-1.318 1.99,-3.07 c -1.233,-0.844 -3.053,-1.29 -5.164,-1.29 -1.438,0 -3.013,0.207 -4.63,0.636 C 23.729,5.684 22.297,2.224 20.308,0.692 Z"
+       id="path6" />
+  </g>
+  <g
+     id="g79964"
+     transform="translate(-1.043816,35.993714)">
+    <path
+       fill="#ffffff"
+       d="m 51.92633,-2.0247139 c -0.799,-0.597 -1.498,-1.484 -2.035,-2.592 l -0.228,-0.47 -0.46,0.249 c -0.975,0.528 -1.913,0.858 -2.744,0.969 l -0.202,-3.7560001 -3.636,0.968 c -0.157,-0.854 -0.125,-1.887 0.096,-3.022 l 0.103,-0.525 -0.532,-0.066 c -1.242,-0.154 -2.306,-0.525 -3.104,-1.08 l 1.521,-2.917 -2.988,-1.523 c 0.319,-0.944 0.948,-1.882 1.834,-2.735 l 0.377,-0.363 -0.379,-0.36 c -0.803,-0.764 -1.408,-1.554 -1.77,-2.311 l 3.51,-1.353 -2.045,-3.159 c 0.74,-0.402 1.693,-0.686 2.789,-0.832 l 0.519,-0.068 -0.091,-0.514 c -0.215,-1.211 -0.172,-2.338 0.124,-3.288 l 3.308,0.523 0.524,-3.308 c 0.988,0.013 2.08,0.326 3.164,0.907 l 0.462,0.248 0.226,-0.473 c 0.479,-1.003 1.044,-1.824 1.653,-2.404 l 2.372,2.92 2.372,-2.92 c 0.609,0.58 1.175,1.401 1.653,2.404 l 0.226,0.473 0.462,-0.247 c 1.085,-0.581 2.178,-0.894 3.164,-0.906 l 0.523,3.308 3.31,-0.525 c 0.296,0.951 0.34,2.078 0.124,3.288 l -0.092,0.515 0.518,0.069 c 1.095,0.145 2.048,0.43 2.788,0.832 l -2.046,3.156 3.511,1.355 c -0.361,0.757 -0.966,1.547 -1.77,2.311 l -0.379,0.36 0.377,0.363 c 0.888,0.854 1.516,1.793 1.835,2.736 l -2.984,1.52 1.521,2.984 c -0.812,0.574 -1.871,0.964 -3.094,1.134 l -0.518,0.072 0.096,0.514 c 0.201,1.089 0.226,2.083 0.073,2.909 l -3.634,-0.97 -0.204,3.7570001 c -0.83,-0.11 -1.768,-0.44 -2.742,-0.968 l -0.459,-0.249 -0.228,0.47 c -0.539,1.107 -1.237,1.994 -2.036,2.591 l -2.367,-2.369 z"
+       id="path2-9" />
+    <path
+       d="m 43.38633,-26.007714 c -0.53,-1.489 -0.698,-2.97 -0.432,-4.2 l 2.368,0.375 0.987,0.156 0.157,-0.988 0.375,-2.368 c 1.261,0.127 2.613,0.743 3.862,1.706 0.118,-0.337 0.244,-0.663 0.382,-0.967 -1.551,-1.135 -3.223,-1.763 -4.73,-1.763 -0.123,0 -0.245,0.004 -0.366,0.013 l -0.511,3.223 -3.224,-0.511 c -0.6,1.487 -0.565,3.415 0.085,5.393 0.335,-0.037 0.684,-0.061 1.047,-0.069 z m 14.501,-5.319 c 1.248,-0.962 2.6,-1.578 3.86,-1.705 l 0.376,2.368 0.156,0.988 0.987,-0.157 2.369,-0.376 c 0.266,1.23 0.098,2.71 -0.432,4.2 0.361,0.009 0.711,0.032 1.046,0.07 0.651,-1.978 0.685,-3.906 0.085,-5.394 l -3.225,0.512 -0.511,-3.224 c -0.12,-0.008 -0.242,-0.012 -0.365,-0.012 -1.507,0 -3.179,0.628 -4.73,1.762 0.14,0.306 0.266,0.631 0.384,0.968 z m -14.225,21.641 h 0.035 c 0.067,0 0.157,-0.604 0.26,-0.947 -0.098,0.004 -0.197,0.046 -0.294,0.046 -1.496,0 -2.826,-0.303 -3.83,-0.89 l 1.089,-2.128 0.454,-0.887 -0.891,-0.452 -2.136,-1.088 c 0.508,-1.151 1.515,-2.25 2.818,-3.143 -0.287,-0.219 -0.561,-0.441 -0.81,-0.669 -1.687,1.217 -2.846,2.755 -3.235,4.31 l 2.908,1.483 -1.482,2.843 c 1.221,1.023 3.049,1.522 5.114,1.522 z m 27.806,-5.846 c -0.39,-1.555 -1.548,-3.093 -3.234,-4.311 -0.25,0.228 -0.523,0.451 -0.81,0.669 1.304,0.893 2.31,1.992 2.817,3.145 l -2.136,1.088 -0.891,0.453 0.454,0.892 1.089,2.137 c -1.004,0.587 -2.332,0.904 -3.828,0.904 -0.099,0 -0.199,-0.01 -0.299,-0.013 0.103,0.344 0.192,0.683 0.26,1.011 l 0.039,0.002 c 2.066,0 3.892,-0.563 5.112,-1.587 l -1.482,-2.908 z m -12.653,9.182 c -0.447,1.5170001 -1.181,2.8120001 -2.119,3.6510001 l -1.695,-1.694 -0.707,-0.707 -0.707,0.707 -1.695,1.694 c -0.938,-0.839 -1.673,-2.136 -2.12,-3.6520001 -0.296,0.2060001 -0.593,0.3970001 -0.886,0.5630001 0.636,1.98 1.741,3.559 3.1,4.409 l 2.308,-2.307 2.308,2.308 c 1.358,-0.851 2.464,-2.428 3.101,-4.408 -0.295,-0.168 -0.591,-0.359 -0.888,-0.5640001 z"
+       fill="#ea596e"
+       id="path4-3" />
+    <path
+       fill="#ea596e"
+       d="m 56.41233,-31.002714 c 0.426,1.146 0.748,2.596 0.841,4.284 l 0.2,3.683 3.564,-0.946 c 1.32,-0.351 2.655,-0.536 3.86,-0.536 0.16,0 0.318,0.003 0.474,0.01 l -1.827,2.819 3.139,1.211 c -0.958,0.759 -2.237,1.514 -3.814,2.123 l -3.441,1.328 2.001,3.099 c 0.918,1.42 1.509,2.782 1.838,3.96 l -3.244,-0.865 -0.182,3.357 c -1.019,-0.677 -2.132,-1.66 -3.198,-2.973 l -2.329,-2.863 -2.328,2.862 c -1.066,1.312 -2.179,2.295 -3.198,2.972 l -0.18,-3.354 -3.248,0.864 c 0.329,-1.178 0.921,-2.54 1.839,-3.961 l 2.004,-3.099 -3.442,-1.328 c -1.577,-0.609 -2.856,-1.363 -3.814,-2.122 l 3.135,-1.208 -1.827,-2.823 c 0.155,-0.006 0.313,-0.01 0.473,-0.01 1.206,0 2.541,0.185 3.861,0.536 l 3.564,0.947 0.202,-3.683 c 0.092,-1.688 0.415,-3.138 0.84,-4.284 l 2.119,2.609 2.118,-2.609 m 0.19,-4.991 -2.308,2.841 -2.308,-2.841 c -1.989,1.532 -3.421,4.992 -3.646,9.112 -1.617,-0.43 -3.192,-0.637 -4.632,-0.637 -2.11,0 -3.929,0.445 -5.161,1.289 l 1.989,3.073 -3.415,1.316 c 0.842,2.366 3.69,4.797 7.54,6.283 -2.241,3.465 -3.116,7.106 -2.407,9.5160001 l 3.537,-0.9410001 0.196,3.6540001 c 2.512,-0.07 5.703,-2.027 8.307,-5.2280001 2.603,3.2010001 5.796,5.1580001 8.306,5.2280001 l 0.198,-3.6550001 3.535,0.9430001 c 0.71,-2.4110001 -0.165,-6.0500001 -2.404,-9.5170001 3.849,-1.485 6.696,-3.918 7.538,-6.283 l -3.415,-1.318 1.99,-3.07 c -1.233,-0.844 -3.053,-1.29 -5.164,-1.29 -1.438,0 -3.013,0.207 -4.63,0.636 -0.225,-4.119 -1.657,-7.579 -3.646,-9.111 z"
+       id="path6-6" />
+  </g>
+  <text
+     xml:space="preserve"
+     style="font-style:normal;font-weight:normal;font-size:42.6667px;line-height:1.25;font-family:sans-serif;fill:#ea596e;fill-opacity:1;stroke:none"
+     x="2.2188232"
+     y="31.430677"
+     id="text46212"><tspan
+       sodipodi:role="line"
+       id="tspan46210"
+       x="2.2188232"
+       y="31.430677"
+       style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:42.6667px;font-family:'TeX Gyre Termes';-inkscape-font-specification:'TeX Gyre Termes'">D</tspan></text>
+  <text
+     xml:space="preserve"
+     style="font-style:normal;font-weight:normal;font-size:42.6667px;line-height:1.25;font-family:sans-serif;fill:#ea596e;fill-opacity:1;stroke:none"
+     x="41.347008"
+     y="67.114784"
+     id="text46212-1"><tspan
+       sodipodi:role="line"
+       id="tspan46210-5"
+       x="41.347008"
+       y="67.114784"
+       style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:42.6667px;font-family:'TeX Gyre Termes';-inkscape-font-specification:'TeX Gyre Termes'">F</tspan></text>
+</svg>

doc/talks/2022-06-23-stack/assets/garage.drawio.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0d9ca1a04f943664e14d0e1edbde596789e4798f8ad5e1cff68bc9dd0cc1334f
+size 26098

doc/talks/2022-06-23-stack/assets/garage2a.drawio.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bffa5d4a72ad25e0b18b43022258a2e4471ce94ff5397aab892fe43ec9d4d7d5
+size 33911

doc/talks/2022-06-23-stack/assets/garage2b.drawio.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9c4913b52e84b5af3f8effbcb7e2060845a8c6f43d5b38072b9ee33c0300d49f
+size 31051

doc/talks/2022-06-23-stack/assets/garage_tables.svg

@@ -0,0 +1,537 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<svg
+   width="850"
+   height="480"
+   viewBox="0 0 224.89584 127"
+   version="1.1"
+   id="svg8"
+   inkscape:version="1.2 (dc2aedaf03, 2022-05-15)"
+   sodipodi:docname="garage_tables.svg"
+   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
+   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
+   xmlns="http://www.w3.org/2000/svg"
+   xmlns:svg="http://www.w3.org/2000/svg"
+   xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+   xmlns:cc="http://creativecommons.org/ns#"
+   xmlns:dc="http://purl.org/dc/elements/1.1/">
+  <defs
+     id="defs2">
+    <marker
+       style="overflow:visible"
+       id="marker1262"
+       refX="0"
+       refY="0"
+       orient="auto"
+       inkscape:stockid="Arrow1Mend"
+       inkscape:isstock="true">
+      <path
+         transform="matrix(-0.4,0,0,-0.4,-4,0)"
+         style="fill:#000000;fill-opacity:1;fill-rule:evenodd;stroke:#000000;stroke-width:1pt;stroke-opacity:1"
+         d="M 0,0 5,-5 -12.5,0 5,5 Z"
+         id="path1260" />
+    </marker>
+    <marker
+       style="overflow:visible"
+       id="Arrow1Mend"
+       refX="0"
+       refY="0"
+       orient="auto"
+       inkscape:stockid="Arrow1Mend"
+       inkscape:isstock="true"
+       inkscape:collect="always">
+      <path
+         transform="matrix(-0.4,0,0,-0.4,-4,0)"
+         style="fill:#000000;fill-opacity:1;fill-rule:evenodd;stroke:#000000;stroke-width:1pt;stroke-opacity:1"
+         d="M 0,0 5,-5 -12.5,0 5,5 Z"
+         id="path965" />
+    </marker>
+    <marker
+       style="overflow:visible"
+       id="Arrow1Lend"
+       refX="0"
+       refY="0"
+       orient="auto"
+       inkscape:stockid="Arrow1Lend"
+       inkscape:isstock="true">
+      <path
+         transform="matrix(-0.8,0,0,-0.8,-10,0)"
+         style="fill:#000000;fill-opacity:1;fill-rule:evenodd;stroke:#000000;stroke-width:1pt;stroke-opacity:1"
+         d="M 0,0 5,-5 -12.5,0 5,5 Z"
+         id="path959" />
+    </marker>
+  </defs>
+  <sodipodi:namedview
+     id="base"
+     pagecolor="#ffffff"
+     bordercolor="#666666"
+     borderopacity="1.0"
+     inkscape:pageopacity="0.0"
+     inkscape:pageshadow="2"
+     inkscape:zoom="0.98994949"
+     inkscape:cx="429.31483"
+     inkscape:cy="289.40871"
+     inkscape:document-units="mm"
+     inkscape:current-layer="layer1"
+     inkscape:document-rotation="0"
+     showgrid="false"
+     units="px"
+     inkscape:window-width="1678"
+     inkscape:window-height="993"
+     inkscape:window-x="0"
+     inkscape:window-y="0"
+     inkscape:window-maximized="1"
+     inkscape:showpageshadow="2"
+     inkscape:pagecheckerboard="0"
+     inkscape:deskcolor="#d1d1d1" />
+  <metadata
+     id="metadata5">
+    <rdf:RDF>
+      <cc:Work
+         rdf:about="">
+        <dc:format>image/svg+xml</dc:format>
+        <dc:type
+           rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
+        <dc:title />
+      </cc:Work>
+    </rdf:RDF>
+  </metadata>
+  <g
+     inkscape:label="Layer 1"
+     inkscape:groupmode="layer"
+     id="layer1">
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:5.64444px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.264583"
+       x="39.570904"
+       y="38.452755"
+       id="text2025"><tspan
+         sodipodi:role="line"
+         id="tspan2023"
+         x="39.570904"
+         y="38.452755"
+         style="font-size:5.64444px;stroke-width:0.264583" /></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:10.5833px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.264583"
+       x="101.95796"
+       y="92.835831"
+       id="text2139"><tspan
+         sodipodi:role="line"
+         id="tspan2137"
+         x="101.95796"
+         y="92.835831"
+         style="stroke-width:0.264583"> </tspan></text>
+    <g
+       id="g2316"
+       transform="translate(-11.455511,1.5722486)">
+      <g
+         id="g2277">
+        <rect
+           style="fill:none;stroke:#000000;stroke-width:0.8;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1"
+           id="rect833"
+           width="47.419891"
+           height="95.353409"
+           x="18.534418"
+           y="24.42766" />
+        <rect
+           style="fill:none;stroke:#000000;stroke-width:0.799999;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1"
+           id="rect833-3"
+           width="47.419891"
+           height="86.973076"
+           x="18.534418"
+           y="32.807987" />
+        <text
+           xml:space="preserve"
+           style="font-style:normal;font-weight:normal;font-size:5.64444px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.264583"
+           x="32.250839"
+           y="29.894743"
+           id="text852"><tspan
+             sodipodi:role="line"
+             id="tspan850"
+             x="32.250839"
+             y="29.894743"
+             style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:5.64444px;font-family:'Liberation Mono';-inkscape-font-specification:'Liberation Mono Bold';stroke-width:0.264583">Object</tspan></text>
+      </g>
+      <g
+         id="g2066"
+         transform="translate(-2.1807817,-3.0621439)">
+        <g
+           id="g1969"
+           transform="matrix(0.12763631,0,0,0.12763631,0.7215051,24.717273)"
+           style="fill:#ff6600;fill-opacity:1;stroke:none;stroke-opacity:1">
+          <path
+             style="fill:#ff6600;fill-opacity:1;stroke:none;stroke-width:0.264583;stroke-opacity:1"
+             d="m 203.71837,154.80038 c -1.11451,3.75057 -2.45288,5.84095 -5.11132,7.98327 -2.2735,1.83211 -4.66721,2.65982 -8.09339,2.79857 -2.59227,0.10498 -2.92868,0.0577 -5.02863,-0.70611 -3.99215,-1.45212 -7.1627,-4.65496 -8.48408,-8.57046 -1.28374,-3.80398 -0.61478,-8.68216 1.64793,-12.01698 0.87317,-1.28689 3.15089,-3.48326 4.18771,-4.03815 l 0.53332,-28.51234 5.78454,-5.09197 6.95158,6.16704 -3.21112,3.49026 3.17616,3.45499 -3.17616,3.40822 2.98973,3.28645 -3.24843,3.3829 4.49203,4.58395 0.0516,5.69106 c 1.06874,0.64848 3.81974,3.24046 4.69548,4.56257 0.452,0.68241 1.06834,2.0197 1.36962,2.97176 0.62932,1.98864 0.88051,5.785 0.47342,7.15497 z m -10.0406,2.32604 c -0.88184,-3.17515 -4.92402,-3.78864 -6.75297,-1.02492 -0.58328,0.8814 -0.6898,1.28852 -0.58362,2.23056 0.26492,2.35041 2.45434,3.95262 4.60856,3.37255 1.19644,-0.32217 2.39435,-1.44872 2.72875,-2.56621 0.30682,-1.02529 0.30686,-0.9045 -7.9e-4,-2.01198 z"
+             id="path1971"
+             sodipodi:nodetypes="ssscsscccccccccccssscsssscc" />
+        </g>
+        <text
+           xml:space="preserve"
+           style="font-style:normal;font-weight:normal;font-size:5.64444px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.264583"
+           x="28.809687"
+           y="44.070885"
+           id="text852-9"><tspan
+             sodipodi:role="line"
+             id="tspan850-4"
+             x="28.809687"
+             y="44.070885"
+             style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:5.64444px;font-family:'Liberation Mono';-inkscape-font-specification:'Liberation Mono Bold';stroke-width:0.264583">bucket </tspan></text>
+      </g>
+      <g
+         id="g2066-7"
+         transform="translate(-2.1807817,6.2627616)">
+        <g
+           id="g1969-8"
+           transform="matrix(0.12763631,0,0,0.12763631,0.7215051,24.717273)"
+           style="fill:#ff6600;fill-opacity:1;stroke:none;stroke-opacity:1">
+          <path
+             style="fill:#4040ff;fill-opacity:1;stroke:none;stroke-width:0.264583;stroke-opacity:1"
+             d="m 203.71837,154.80038 c -1.11451,3.75057 -2.45288,5.84095 -5.11132,7.98327 -2.2735,1.83211 -4.66721,2.65982 -8.09339,2.79857 -2.59227,0.10498 -2.92868,0.0577 -5.02863,-0.70611 -3.99215,-1.45212 -7.1627,-4.65496 -8.48408,-8.57046 -1.28374,-3.80398 -0.61478,-8.68216 1.64793,-12.01698 0.87317,-1.28689 3.15089,-3.48326 4.18771,-4.03815 l 0.53332,-28.51234 5.78454,-5.09197 6.95158,6.16704 -3.21112,3.49026 3.17616,3.45499 -3.17616,3.40822 2.98973,3.28645 -3.24843,3.3829 4.49203,4.58395 0.0516,5.69106 c 1.06874,0.64848 3.81974,3.24046 4.69548,4.56257 0.452,0.68241 1.06834,2.0197 1.36962,2.97176 0.62932,1.98864 0.88051,5.785 0.47342,7.15497 z m -10.0406,2.32604 c -0.88184,-3.17515 -4.92402,-3.78864 -6.75297,-1.02492 -0.58328,0.8814 -0.6898,1.28852 -0.58362,2.23056 0.26492,2.35041 2.45434,3.95262 4.60856,3.37255 1.19644,-0.32217 2.39435,-1.44872 2.72875,-2.56621 0.30682,-1.02529 0.30686,-0.9045 -7.9e-4,-2.01198 z"
+             id="path1971-4"
+             sodipodi:nodetypes="ssscsscccccccccccssscsssscc" />
+        </g>
+        <text
+           xml:space="preserve"
+           style="font-style:normal;font-weight:normal;font-size:5.64444px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.264583"
+           x="28.809687"
+           y="44.070885"
+           id="text852-9-5"><tspan
+             sodipodi:role="line"
+             id="tspan850-4-0"
+             x="28.809687"
+             y="44.070885"
+             style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:5.64444px;font-family:'Liberation Mono';-inkscape-font-specification:'Liberation Mono Bold';stroke-width:0.264583">file path </tspan></text>
+        <path
+           style="fill:#ff6600;fill-opacity:1;stroke:none;stroke-width:0.0337704;stroke-opacity:1"
+           d="m 174.20027,104.45585 c -0.14225,0.47871 -0.31308,0.74552 -0.65239,1.01896 -0.29018,0.23384 -0.5957,0.33949 -1.03301,0.3572 -0.33087,0.0134 -0.37381,0.007 -0.64184,-0.0901 -0.50954,-0.18534 -0.91422,-0.59414 -1.08287,-1.0939 -0.16385,-0.48552 -0.0785,-1.10816 0.21033,-1.5338 0.11145,-0.16426 0.40217,-0.44459 0.53451,-0.51542 l 0.0681,-3.639207 0.73832,-0.64992 0.88727,0.787138 -0.40986,0.445484 0.4054,0.440982 -0.4054,0.435013 0.3816,0.41947 -0.41461,0.43178 0.57334,0.58508 0.007,0.72639 c 0.13641,0.0828 0.48753,0.4136 0.59931,0.58235 0.0577,0.0871 0.13636,0.25778 0.17481,0.3793 0.0803,0.25382 0.11239,0.73838 0.0604,0.91323 z m -1.28154,0.29689 c -0.11256,-0.40526 -0.62849,-0.48357 -0.86193,-0.13082 -0.0745,0.1125 -0.088,0.16447 -0.0745,0.2847 0.0338,0.3 0.31326,0.5045 0.58822,0.43046 0.15271,-0.0411 0.30561,-0.1849 0.34829,-0.32754 0.0392,-0.13086 0.0392,-0.11544 -1e-4,-0.2568 z"
+           id="path1971-3"
+           sodipodi:nodetypes="ssscsscccccccccccssscsssscc" />
+        <text
+           xml:space="preserve"
+           style="font-style:normal;font-weight:normal;font-size:5.64444px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.264583"
+           x="177.8474"
+           y="104.05132"
+           id="text852-9-6"><tspan
+             sodipodi:role="line"
+             id="tspan850-4-7"
+             x="177.8474"
+             y="104.05132"
+             style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:5.64444px;font-family:'Liberation Mono';-inkscape-font-specification:'Liberation Mono Bold';stroke-width:0.264583">= partition key </tspan></text>
+        <path
+           style="fill:#4040ff;fill-opacity:1;stroke:none;stroke-width:0.0337704;stroke-opacity:1"
+           d="m 174.20027,113.78076 c -0.14225,0.47871 -0.31308,0.74552 -0.65239,1.01895 -0.29018,0.23385 -0.5957,0.33949 -1.03301,0.3572 -0.33087,0.0134 -0.37381,0.007 -0.64184,-0.0901 -0.50954,-0.18534 -0.91422,-0.59414 -1.08287,-1.0939 -0.16385,-0.48553 -0.0785,-1.10816 0.21033,-1.53381 0.11145,-0.16425 0.40217,-0.44459 0.53451,-0.51541 l 0.0681,-3.63921 0.73832,-0.64992 0.88727,0.78714 -0.40986,0.44548 0.4054,0.44098 -0.4054,0.43502 0.3816,0.41947 -0.41461,0.43178 0.57334,0.58508 0.007,0.72638 c 0.13641,0.0828 0.48753,0.4136 0.59931,0.58235 0.0577,0.0871 0.13636,0.25779 0.17481,0.37931 0.0803,0.25382 0.11239,0.73837 0.0604,0.91323 z m -1.28154,0.29689 c -0.11256,-0.40527 -0.62849,-0.48357 -0.86193,-0.13082 -0.0745,0.1125 -0.088,0.16446 -0.0745,0.2847 0.0338,0.3 0.31326,0.5045 0.58822,0.43046 0.15271,-0.0411 0.30561,-0.18491 0.34829,-0.32754 0.0392,-0.13087 0.0392,-0.11545 -1e-4,-0.2568 z"
+           id="path1971-4-5"
+           sodipodi:nodetypes="ssscsscccccccccccssscsssscc" />
+        <text
+           xml:space="preserve"
+           style="font-style:normal;font-weight:normal;font-size:5.64444px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.264583"
+           x="177.8474"
+           y="113.37622"
+           id="text852-9-5-3"><tspan
+             sodipodi:role="line"
+             id="tspan850-4-0-5"
+             x="177.8474"
+             y="113.37622"
+             style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:5.64444px;font-family:'Liberation Mono';-inkscape-font-specification:'Liberation Mono Bold';stroke-width:0.264583">= sort key </tspan></text>
+      </g>
+      <g
+         id="g2161"
+         transform="translate(-62.264403,-59.333115)">
+        <g
+           id="g2271"
+           transform="translate(0,67.042823)">
+          <rect
+             style="fill:none;stroke:#000000;stroke-width:0.799999;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1"
+             id="rect833-6"
+             width="39.008453"
+             height="16.775949"
+             x="84.896881"
+             y="90.266838" />
+          <rect
+             style="fill:none;stroke:#000000;stroke-width:0.799999;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1"
+             id="rect833-3-1"
+             width="39.008453"
+             height="8.673645"
+             x="84.896881"
+             y="98.369141" />
+          <text
+             xml:space="preserve"
+             style="font-style:normal;font-weight:normal;font-size:5.64444px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.264583"
+             x="89.826942"
+             y="96.212921"
+             id="text852-0"><tspan
+               sodipodi:role="line"
+               id="tspan850-6"
+               x="89.826942"
+               y="96.212921"
+               style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:5.64444px;font-family:'Liberation Mono';-inkscape-font-specification:'Liberation Mono Bold';stroke-width:0.264583">Version 1</tspan></text>
+          <text
+             xml:space="preserve"
+             style="font-style:normal;font-weight:normal;font-size:5.64444px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.264583"
+             x="89.826942"
+             y="104.71013"
+             id="text852-0-3"><tspan
+               sodipodi:role="line"
+               id="tspan850-6-2"
+               x="89.826942"
+               y="104.71013"
+               style="font-style:italic;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:5.64444px;font-family:'Liberation Mono';-inkscape-font-specification:'Liberation Mono Bold';fill:#4d4d4d;stroke-width:0.264583">deleted</tspan></text>
+        </g>
+      </g>
+      <g
+         id="g2263"
+         transform="translate(0,-22.791204)">
+        <g
+           id="g2161-1"
+           transform="translate(-62.264403,-10.910843)">
+          <rect
+             style="fill:none;stroke:#000000;stroke-width:0.799999;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1"
+             id="rect833-6-5"
+             width="39.008453"
+             height="36.749603"
+             x="84.896881"
+             y="90.266838" />
+          <rect
+             style="fill:none;stroke:#000000;stroke-width:0.799999;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1"
+             id="rect833-3-1-5"
+             width="39.008453"
+             height="28.647301"
+             x="84.896881"
+             y="98.369141" />
+          <text
+             xml:space="preserve"
+             style="font-style:normal;font-weight:normal;font-size:5.64444px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.264583"
+             x="89.826942"
+             y="96.212921"
+             id="text852-0-4"><tspan
+               sodipodi:role="line"
+               id="tspan850-6-7"
+               x="89.826942"
+               y="96.212921"
+               style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:5.64444px;font-family:'Liberation Mono';-inkscape-font-specification:'Liberation Mono Bold';stroke-width:0.264583">Version 2</tspan></text>
+          <text
+             xml:space="preserve"
+             style="font-style:normal;font-weight:normal;font-size:5.64444px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.264583"
+             x="89.826942"
+             y="104.71013"
+             id="text852-0-3-6"><tspan
+               sodipodi:role="line"
+               id="tspan850-6-2-5"
+               x="89.826942"
+               y="104.71013"
+               style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:5.64444px;font-family:'Liberation Mono';-inkscape-font-specification:'Liberation Mono Bold';fill:#000000;stroke-width:0.264583">id</tspan></text>
+        </g>
+        <text
+           xml:space="preserve"
+           style="font-style:normal;font-weight:normal;font-size:5.64444px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.264583"
+           x="27.56254"
+           y="100.34132"
+           id="text852-0-3-6-6"><tspan
+             sodipodi:role="line"
+             id="tspan850-6-2-5-9"
+             x="27.56254"
+             y="100.34132"
+             style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:5.64444px;font-family:'Liberation Mono';-inkscape-font-specification:'Liberation Mono Bold';fill:#000000;stroke-width:0.264583">size</tspan></text>
+        <text
+           xml:space="preserve"
+           style="font-style:normal;font-weight:normal;font-size:5.64444px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.264583"
+           x="27.56254"
+           y="106.90263"
+           id="text852-0-3-6-6-3"><tspan
+             sodipodi:role="line"
+             id="tspan850-6-2-5-9-7"
+             x="27.56254"
+             y="106.90263"
+             style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:5.64444px;font-family:'Liberation Mono';-inkscape-font-specification:'Liberation Mono Bold';fill:#000000;stroke-width:0.264583">MIME type</tspan></text>
+        <text
+           xml:space="preserve"
+           style="font-style:normal;font-weight:normal;font-size:5.64444px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.264583"
+           x="27.56254"
+           y="111.92816"
+           id="text852-0-3-6-6-3-4"><tspan
+             sodipodi:role="line"
+             id="tspan850-6-2-5-9-7-5"
+             x="27.56254"
+             y="111.92816"
+             style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:5.64444px;font-family:'Liberation Mono';-inkscape-font-specification:'Liberation Mono Bold';fill:#000000;stroke-width:0.264583">...</tspan></text>
+      </g>
+    </g>
+    <g
+       id="g898"
+       transform="translate(-6.2484318,29.95006)">
+      <rect
+         style="fill:none;stroke:#000000;stroke-width:0.799999;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1"
+         id="rect833-7"
+         width="47.419891"
+         height="44.007515"
+         x="95.443573"
+         y="24.42766" />
+      <rect
+         style="fill:none;stroke:#000000;stroke-width:0.799999;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1"
+         id="rect833-3-4"
+         width="47.419891"
+         height="35.627186"
+         x="95.443573"
+         y="32.807987" />
+      <text
+         xml:space="preserve"
+         style="font-style:normal;font-weight:normal;font-size:5.64444px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.264583"
+         x="107.46638"
+         y="29.894743"
+         id="text852-4"><tspan
+           sodipodi:role="line"
+           id="tspan850-3"
+           x="107.46638"
+           y="29.894743"
+           style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:5.64444px;font-family:'Liberation Mono';-inkscape-font-specification:'Liberation Mono Bold';stroke-width:0.264583">Version</tspan></text>
+      <path
+         style="fill:#ff6600;fill-opacity:1;stroke:none;stroke-width:0.0337704;stroke-opacity:1"
+         d="m 102.90563,41.413279 c -0.14226,0.478709 -0.31308,0.745518 -0.65239,1.018956 -0.29019,0.233843 -0.59571,0.339489 -1.03301,0.357199 -0.33087,0.0134 -0.37381,0.0074 -0.64184,-0.09013 -0.50954,-0.185343 -0.914221,-0.594142 -1.082877,-1.093901 -0.163852,-0.485526 -0.07847,-1.108159 0.210335,-1.533803 0.111448,-0.164254 0.402172,-0.444591 0.534502,-0.515415 l 0.0681,-3.63921 0.73832,-0.64992 0.88727,0.787138 -0.40985,0.445484 0.40539,0.440982 -0.40539,0.435013 0.3816,0.41947 -0.41462,0.431781 0.57335,0.585078 0.007,0.726386 c 0.13641,0.08277 0.48753,0.413601 0.59931,0.58235 0.0577,0.0871 0.13636,0.257787 0.17481,0.379304 0.0803,0.253823 0.11239,0.738377 0.0604,0.913234 z m -1.28155,0.296888 c -0.11255,-0.405265 -0.62848,-0.483569 -0.86192,-0.130817 -0.0744,0.112498 -0.088,0.164461 -0.0745,0.2847 0.0338,0.299998 0.31326,0.504498 0.58822,0.43046 0.15271,-0.04112 0.3056,-0.184909 0.34828,-0.327542 0.0392,-0.130864 0.0392,-0.115447 -1e-4,-0.256801 z"
+         id="path1971-0"
+         sodipodi:nodetypes="ssscsscccccccccccssscsssscc" />
+      <text
+         xml:space="preserve"
+         style="font-style:normal;font-weight:normal;font-size:5.64444px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.264583"
+         x="104.99195"
+         y="41.008743"
+         id="text852-9-7"><tspan
+           sodipodi:role="line"
+           id="tspan850-4-8"
+           x="104.99195"
+           y="41.008743"
+           style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:5.64444px;font-family:'Liberation Mono';-inkscape-font-specification:'Liberation Mono Bold';stroke-width:0.264583">id </tspan></text>
+      <text
+         xml:space="preserve"
+         style="font-style:normal;font-weight:normal;font-size:5.64444px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.264583"
+         x="104.99195"
+         y="49.168018"
+         id="text852-9-7-6"><tspan
+           sodipodi:role="line"
+           id="tspan850-4-8-8"
+           x="104.99195"
+           y="49.168018"
+           style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:5.64444px;font-family:'Liberation Mono';-inkscape-font-specification:'Liberation Mono Bold';stroke-width:0.264583">h(block 1)</tspan></text>
+      <text
+         xml:space="preserve"
+         style="font-style:normal;font-weight:normal;font-size:5.64444px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.264583"
+         x="104.99195"
+         y="56.583336"
+         id="text852-9-7-6-8"><tspan
+           sodipodi:role="line"
+           id="tspan850-4-8-8-4"
+           x="104.99195"
+           y="56.583336"
+           style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:5.64444px;font-family:'Liberation Mono';-inkscape-font-specification:'Liberation Mono Bold';stroke-width:0.264583">h(block 2)</tspan></text>
+      <text
+         xml:space="preserve"
+         style="font-style:normal;font-weight:normal;font-size:5.64444px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.264583"
+         x="104.99195"
+         y="64.265732"
+         id="text852-9-7-6-3"><tspan
+           sodipodi:role="line"
+           id="tspan850-4-8-8-1"
+           x="104.99195"
+           y="64.265732"
+           style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:5.64444px;font-family:'Liberation Mono';-inkscape-font-specification:'Liberation Mono Bold';stroke-width:0.264583">...</tspan></text>
+    </g>
+    <g
+       id="g898-3"
+       transform="translate(75.777779,38.888663)">
+      <rect
+         style="fill:none;stroke:#000000;stroke-width:0.799999;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1"
+         id="rect833-7-6"
+         width="47.419891"
+         height="29.989157"
+         x="95.443573"
+         y="24.42766" />
+      <rect
+         style="fill:none;stroke:#000000;stroke-width:0.799999;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1"
+         id="rect833-3-4-7"
+         width="47.419891"
+         height="21.608831"
+         x="95.443573"
+         y="32.807987" />
+      <text
+         xml:space="preserve"
+         style="font-style:normal;font-weight:normal;font-size:5.64444px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.264583"
+         x="102.11134"
+         y="29.894743"
+         id="text852-4-5"><tspan
+           sodipodi:role="line"
+           id="tspan850-3-3"
+           x="102.11134"
+           y="29.894743"
+           style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:5.64444px;font-family:'Liberation Mono';-inkscape-font-specification:'Liberation Mono Bold';stroke-width:0.264583">Data block</tspan></text>
+      <path
+         style="fill:#ff6600;fill-opacity:1;stroke:none;stroke-width:0.0337704;stroke-opacity:1"
+         d="m 102.90563,41.413279 c -0.14226,0.478709 -0.31308,0.745518 -0.65239,1.018956 -0.29019,0.233843 -0.59571,0.339489 -1.03301,0.357199 -0.33087,0.0134 -0.37381,0.0074 -0.64184,-0.09013 -0.50954,-0.185343 -0.914221,-0.594142 -1.082877,-1.093901 -0.163852,-0.485526 -0.07847,-1.108159 0.210335,-1.533803 0.111448,-0.164254 0.402172,-0.444591 0.534502,-0.515415 l 0.0681,-3.63921 0.73832,-0.64992 0.88727,0.787138 -0.40985,0.445484 0.40539,0.440982 -0.40539,0.435013 0.3816,0.41947 -0.41462,0.431781 0.57335,0.585078 0.007,0.726386 c 0.13641,0.08277 0.48753,0.413601 0.59931,0.58235 0.0577,0.0871 0.13636,0.257787 0.17481,0.379304 0.0803,0.253823 0.11239,0.738377 0.0604,0.913234 z m -1.28155,0.296888 c -0.11255,-0.405265 -0.62848,-0.483569 -0.86192,-0.130817 -0.0744,0.112498 -0.088,0.164461 -0.0745,0.2847 0.0338,0.299998 0.31326,0.504498 0.58822,0.43046 0.15271,-0.04112 0.3056,-0.184909 0.34828,-0.327542 0.0392,-0.130864 0.0392,-0.115447 -1e-4,-0.256801 z"
+         id="path1971-0-5"
+         sodipodi:nodetypes="ssscsscccccccccccssscsssscc" />
+      <text
+         xml:space="preserve"
+         style="font-style:normal;font-weight:normal;font-size:5.64444px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.264583"
+         x="104.99195"
+         y="41.008743"
+         id="text852-9-7-62"><tspan
+           sodipodi:role="line"
+           id="tspan850-4-8-9"
+           x="104.99195"
+           y="41.008743"
+           style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:5.64444px;font-family:'Liberation Mono';-inkscape-font-specification:'Liberation Mono Bold';stroke-width:0.264583">hash </tspan></text>
+      <text
+         xml:space="preserve"
+         style="font-style:normal;font-weight:normal;font-size:5.64444px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.264583"
+         x="104.99195"
+         y="49.168018"
+         id="text852-9-7-6-1"><tspan
+           sodipodi:role="line"
+           id="tspan850-4-8-8-2"
+           x="104.99195"
+           y="49.168018"
+           style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:5.64444px;font-family:'Liberation Mono';-inkscape-font-specification:'Liberation Mono Bold';stroke-width:0.264583">data</tspan></text>
+    </g>
+    <path
+       style="fill:none;stroke:#000000;stroke-width:1;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1;marker-end:url(#Arrow1Mend)"
+       d="M 42.105292,69.455903 89.563703,69.317144"
+       id="path954"
+       sodipodi:nodetypes="cc" />
+    <path
+       style="fill:none;stroke:#000000;stroke-width:1;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1;marker-end:url(#marker1262)"
+       d="m 134.32612,77.363197 38.12618,0.260865"
+       id="path1258"
+       sodipodi:nodetypes="cc" />
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:5.64444px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.264583"
+       x="8.6727352"
+       y="16.687063"
+       id="text852-3"><tspan
+         sodipodi:role="line"
+         id="tspan850-67"
+         x="8.6727352"
+         y="16.687063"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:5.64444px;font-family:'Liberation Mono';-inkscape-font-specification:'Liberation Mono Bold';stroke-width:0.264583">Objects table </tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:5.64444px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.264583"
+       x="89.190445"
+       y="16.687063"
+       id="text852-3-5"><tspan
+         sodipodi:role="line"
+         id="tspan850-67-3"
+         x="89.190445"
+         y="16.687063"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:5.64444px;font-family:'Liberation Mono';-inkscape-font-specification:'Liberation Mono Bold';stroke-width:0.264583">Versions table </tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:5.64444px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.264583"
+       x="174.55702"
+       y="16.687063"
+       id="text852-3-56"><tspan
+         sodipodi:role="line"
+         id="tspan850-67-2"
+         x="174.55702"
+         y="16.687063"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:5.64444px;font-family:'Liberation Mono';-inkscape-font-specification:'Liberation Mono Bold';stroke-width:0.264583">Blocks table</tspan></text>
+  </g>
+</svg>

doc/talks/2022-06-23-stack/talk.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f245bb017eb85a227d96e576ae821f165ef478d46de02535c196638aa5fc84b0
+size 2575953

doc/talks/2022-06-23-stack/talk.tex

@@ -0,0 +1,480 @@
+%\nonstopmode
+\documentclass[aspectratio=169]{beamer}
+\usepackage[utf8]{inputenc}
+% \usepackage[frenchb]{babel}
+\usepackage{amsmath}
+\usepackage{mathtools}
+\usepackage{breqn}
+\usepackage{multirow}
+\usetheme{boxes}
+\usepackage{graphicx}
+\usepackage{adjustbox}
+%\useoutertheme[footline=authortitle,subsection=false]{miniframes}
+%\useoutertheme[footline=authorinstitute,subsection=false]{miniframes}
+\useoutertheme{infolines}
+\setbeamertemplate{headline}{}
+
+\beamertemplatenavigationsymbolsempty
+
+\definecolor{TitleOrange}{RGB}{255,137,0}
+\setbeamercolor{title}{fg=TitleOrange}
+\setbeamercolor{frametitle}{fg=TitleOrange}
+
+\definecolor{ListOrange}{RGB}{255,145,5}
+\setbeamertemplate{itemize item}{\color{ListOrange}$\blacktriangleright$}
+
+\definecolor{verygrey}{RGB}{70,70,70}
+\setbeamercolor{normal text}{fg=verygrey}
+
+
+\usepackage{tabu}
+\usepackage{multicol}
+\usepackage{vwcol}
+\usepackage{stmaryrd}
+\usepackage{graphicx}
+
+\usepackage[normalem]{ulem}
+
+\title{Introducing Garage}
+\subtitle{a new storage platform for self-hosted geo-distributed clusters}
+\author{Deuxfleurs Association}
+\date{IMT Atlantique, 2022-06-23}
+
+\begin{document}
+
+\begin{frame}
+	\centering
+	\includegraphics[width=.3\linewidth]{../../sticker/Garage.pdf}
+	\vspace{1em}
+
+	{\large\bf Deuxfleurs Association}
+	\vspace{1em}
+
+	\url{https://garagehq.deuxfleurs.fr/}
+
+	Matrix channel: \texttt{\#garage:deuxfleurs.fr}
+\end{frame}
+
+\begin{frame}
+	\frametitle{Who we are}
+	\begin{columns}[t]
+		\begin{column}{.2\textwidth}
+			\centering
+			\adjincludegraphics[width=.4\linewidth, valign=t]{assets/alex.jpg}
+		\end{column}
+		\begin{column}{.6\textwidth}
+			\textbf{Alex Auvolat}\\
+			PhD at Inria, team WIDE; co-founder of Deuxfleurs
+		\end{column}
+		\begin{column}{.2\textwidth}
+			~
+		\end{column}
+	\end{columns}
+	\vspace{1em}
+
+	\begin{columns}[t]
+		\begin{column}{.2\textwidth}
+			~
+		\end{column}
+		\begin{column}{.6\textwidth}
+			\textbf{Quentin Dufour}\\
+			PhD at Inria, team WIDE; co-founder of Deuxfleurs
+		\end{column}
+		\begin{column}{.2\textwidth}
+			\centering
+			\adjincludegraphics[width=.5\linewidth, valign=t]{assets/quentin.jpg}
+		\end{column}
+	\end{columns}
+	\vspace{2em}
+
+	\begin{columns}[t]
+		\begin{column}{.2\textwidth}
+			\centering
+			\adjincludegraphics[width=.5\linewidth, valign=t]{assets/deuxfleurs.pdf}
+		\end{column}
+		\begin{column}{.6\textwidth}
+			\textbf{Deuxfleurs}\\
+			A non-profit self-hosting collective,\\
+			member of the CHATONS network
+		\end{column}
+		\begin{column}{.2\textwidth}
+			\centering
+			\adjincludegraphics[width=.7\linewidth, valign=t]{assets/logo_chatons.png}
+		\end{column}
+	\end{columns}
+
+\end{frame}
+
+\begin{frame}
+	\frametitle{Our objective at Deuxfleurs}
+	
+	\begin{center}
+		\textbf{Promote self-hosting and small-scale hosting\\
+			as an alternative to large cloud providers}
+	\end{center}
+	\vspace{2em}
+	\visible<2->{
+		Why is it hard?
+	}
+	\visible<3->{
+		\vspace{2em}
+		\begin{center}
+			\textbf{\underline{Resilience}}\\
+			{\footnotesize (we want good uptime/availability with low supervision)}
+		\end{center}
+	}
+\end{frame}
+
+\begin{frame}
+	\frametitle{How to make a \underline{stable} system}
+
+	Enterprise-grade systems typically employ:
+	\vspace{1em}
+	\begin{itemize}
+		\item RAID
+		\item Redundant power grid + UPS
+		\item Redundant Internet connections
+		\item Low-latency links
+		\item ... 
+	\end{itemize}
+	\vspace{1em}
+	$\to$ it's costly and only worth it at DC scale
+\end{frame}
+
+\begin{frame}
+	\frametitle{How to make a \underline{resilient} system}
+
+	\only<1,4-5>{
+		Instead, we use:
+		\vspace{1em}
+		\begin{itemize}
+			\item \textcolor<2->{gray}{Commodity hardware (e.g. old desktop PCs)}
+				\vspace{.5em}
+			\item<4-> \textcolor<5->{gray}{Commodity Internet (e.g. FTTB, FTTH) and power grid}
+				\vspace{.5em}
+			\item<5-> \textcolor<6->{gray}{\textbf{Geographical redundancy} (multi-site replication)}
+		\end{itemize}
+	}
+	\only<2>{
+		\begin{center}
+			\includegraphics[width=.8\linewidth]{assets/atuin.jpg}
+		\end{center}
+	}
+	\only<3>{
+		\begin{center}
+			\includegraphics[width=.8\linewidth]{assets/neptune.jpg}
+		\end{center}
+	}
+	\only<6>{
+		\begin{center}
+			\includegraphics[width=.5\linewidth]{assets/inframap.jpg}
+		\end{center}
+	}
+\end{frame}
+
+\begin{frame}
+	\frametitle{How to make this happen}
+	\begin{center}
+		\only<1>{\includegraphics[width=.8\linewidth]{assets/slide1.png}}%
+		\only<2>{\includegraphics[width=.8\linewidth]{assets/slide2.png}}%
+		\only<3>{\includegraphics[width=.8\linewidth]{assets/slide3.png}}%
+	\end{center}
+\end{frame}
+
+\begin{frame}
+	\frametitle{Distributed file systems are slow}
+	File systems are complex, for example:
+	\vspace{1em}
+	\begin{itemize}
+		\item Concurrent modification by several processes
+			\vspace{1em}
+		\item Folder hierarchies
+			\vspace{1em}
+		\item Other requirements of the POSIX spec
+	\end{itemize}
+	\vspace{1em}
+	Coordination in a distributed system is costly
+
+	\vspace{1em}
+	Costs explode with commodity hardware / Internet connections\\
+	{\small (we experienced this!)}
+\end{frame}
+
+\begin{frame}
+	\frametitle{A simpler solution: object storage}
+	Only two operations:
+	\vspace{1em}
+	\begin{itemize}
+		\item Put an object at a key
+			\vspace{1em}
+		\item Retrieve an object from its key
+	\end{itemize}
+	\vspace{1em}
+	{\footnotesize (and a few others)}
+
+	\vspace{1em}
+	Sufficient for many applications!
+\end{frame}
+
+\begin{frame}
+	\frametitle{A simpler solution: object storage}
+		\begin{center}
+			\includegraphics[width=.2\linewidth]{../2020-12-02_wide-team/img/Amazon-S3.jpg}
+			\hspace{5em}
+			\includegraphics[width=.2\linewidth]{assets/minio.png}
+		\end{center}
+		\vspace{1em}
+	S3: a de-facto standard, many compatible applications
+
+	\vspace{1em}
+
+	MinIO is self-hostable but not suited for geo-distributed deployments
+\end{frame}
+
+
+\begin{frame}
+	\frametitle{But what is Garage, exactly?}
+	\textbf{Garage is a self-hosted drop-in replacement for the Amazon S3 object store}\\
+	\vspace{.5em}
+	that implements resilience through geographical redundancy on commodity hardware
+		\begin{center}
+			\includegraphics[width=.8\linewidth]{assets/garageuses.png}
+		\end{center}
+\end{frame}
+
+\begin{frame}
+	\frametitle{Overview}
+	\begin{center}
+		\only<1>{\includegraphics[width=.45\linewidth]{assets/garage2a.drawio.pdf}}%
+		\only<2>{\includegraphics[width=.45\linewidth]{assets/garage2b.drawio.pdf}}%
+	\end{center}
+\end{frame}
+
+\begin{frame}
+	\frametitle{Garage is \emph{location-aware}}
+	\begin{center}
+		\includegraphics[width=\linewidth]{assets/location-aware.png}
+	\end{center}
+	\vspace{2em}
+	Garage replicates data on different zones when possible
+\end{frame}
+
+\begin{frame}
+	\frametitle{Garage is \emph{location-aware}}
+		\begin{center}
+			\includegraphics[width=.8\linewidth]{assets/map.png}
+		\end{center}
+\end{frame}
+
+\begin{frame}
+	\frametitle{How to spread files over different cluster nodes?}
+	\textbf{Consistent hashing (DynamoDB):}
+	\vspace{1em}
+
+	\begin{center}
+	\only<1>{\includegraphics[width=.45\columnwidth]{assets/consistent_hashing_1.pdf}}%
+	\only<2>{\includegraphics[width=.45\columnwidth]{assets/consistent_hashing_2.pdf}}%
+	\only<3>{\includegraphics[width=.45\columnwidth]{assets/consistent_hashing_3.pdf}}%
+	\only<4>{\includegraphics[width=.45\columnwidth]{assets/consistent_hashing_4.pdf}}%
+	\end{center}
+\end{frame}
+
+\begin{frame}
+	\frametitle{How to spread files over different cluster nodes?}
+	\textbf{Issues with consistent hashing:}
+	\vspace{1em}
+	\begin{itemize}
+		\item Doesn't dispatch data based on geographical location of nodes
+			\vspace{1em}
+		\item<2-> Geographically aware adaptation, try 1:\\
+			data quantities not well balanced between nodes
+			\vspace{1em}
+		\item<3-> Geographically aware adaptation, try 2:\\
+			too many reshuffles when adding/removing nodes
+	\end{itemize}
+\end{frame}
+
+\begin{frame}
+	\frametitle{How to spread files over different cluster nodes?}
+	\textbf{Garage's method: build an index table}
+	\vspace{1em}
+
+	Realization: we can actually precompute an optimal solution
+	\vspace{1em}
+
+	\visible<2->{
+		\begin{center}
+		\begin{tabular}{|l|l|l|l|}
+			\hline
+			\textbf{Partition} & \textbf{Node 1} & \textbf{Node 2} & \textbf{Node 3} \\
+			\hline
+			\hline
+			Partition 0 & Io (jupiter) & Drosera (atuin) & Courgette (neptune) \\
+			\hline
+			Partition 1 & Datura (atuin) & Courgette (neptune) & Io (jupiter) \\
+			\hline
+			Partition 2 & Io(jupiter) & Celeri (neptune) & Drosera (atuin) \\
+			\hline
+			\hspace{1em}$\vdots$ & \hspace{1em}$\vdots$ & \hspace{1em}$\vdots$ & \hspace{1em}$\vdots$ \\
+			\hline
+			Partition 255 & Concombre (neptune) & Io (jupiter) & Drosera (atuin) \\
+			\hline
+		\end{tabular}
+		\end{center}
+	}
+	\vspace{1em}
+	\visible<3->{
+		The index table is built centrally using an optimal* algorithm,\\
+		then propagated to all nodes\\
+		\hfill\footnotesize *not yet optimal but will be soon
+	}
+\end{frame}
+
+\begin{frame}
+	\frametitle{Garage's internal data structures}
+	\centering
+	\includegraphics[width=.75\columnwidth]{assets/garage_tables.pdf}
+\end{frame}
+
+%\begin{frame}
+%	\frametitle{Garage's architecture}
+%	\begin{center}
+%		\includegraphics[width=.35\linewidth]{assets/garage.drawio.pdf}
+%	\end{center}
+%\end{frame}
+
+\begin{frame}
+	\frametitle{Garage is \emph{coordination-free}:}
+	\begin{itemize}
+		\item No Raft or Paxos
+			\vspace{1em}
+		\item Internal data types are CRDTs
+			\vspace{1em}
+		\item All nodes are equivalent (no master/leader/index node)
+	\end{itemize}
+			\vspace{2em}
+	$\to$ less sensitive to higher latencies between nodes
+\end{frame}
+
+\begin{frame}
+	\frametitle{Consistency model}
+	\begin{itemize}
+		\item Not ACID (not required by S3 spec) / not linearizable
+			\vspace{1em}
+		\item \textbf{Read-after-write consistency}\\
+			{\footnotesize (stronger than eventual consistency)}
+	\end{itemize}
+\end{frame}
+
+\begin{frame}
+	\frametitle{Impact on performances}
+		\begin{center}
+			\includegraphics[width=.8\linewidth]{assets/endpoint-latency-dc.png}
+		\end{center}
+\end{frame}
+
+
+\begin{frame}
+	\frametitle{An ever-increasing compatibility list}
+		\begin{center}
+			\includegraphics[width=.7\linewidth]{assets/compatibility.png}
+		\end{center}
+\end{frame}
+
+\begin{frame}
+	\frametitle{Further plans for Garage}
+	\begin{center}
+		\only<1>{\includegraphics[width=.8\linewidth]{assets/slideB1.png}}%
+		\only<2>{\includegraphics[width=.8\linewidth]{assets/slideB2.png}}%
+		\only<3>{\includegraphics[width=.8\linewidth]{assets/slideB3.png}}%
+	\end{center}
+\end{frame}
+
+\begin{frame}
+	\frametitle{K2V Design}
+	\begin{itemize}
+		\item A new, custom, minimal API
+			\vspace{1em}
+		\item<2-> Exposes the partitoning mechanism of Garage\\
+			K2V = partition key / sort key / value (like Dynamo)
+			\vspace{1em}
+		\item<3-> Coordination-free, CRDT-friendly (inspired by Riak)\\
+			\vspace{1em}
+		\item<4-> Cryptography-friendly: values are binary blobs
+	\end{itemize}
+\end{frame}
+
+\begin{frame}
+	\frametitle{Application: an e-mail storage server}
+	\begin{center}
+		\only<1>{\includegraphics[width=.9\linewidth]{assets/aerogramme.png}}%
+	\end{center}
+\end{frame}
+
+\begin{frame}
+	\frametitle{Aerogramme data model}
+	\begin{center}
+		\only<1>{\includegraphics[width=.4\linewidth]{assets/aerogramme_datatype.drawio.pdf}}%
+		\only<2->{\includegraphics[width=.9\linewidth]{assets/aerogramme_keys.drawio.pdf}\vspace{1em}}%
+	\end{center}
+	\visible<3->{Aerogramme encrypts all stored values for privacy\\
+	(Garage server administrators can't read your mail)}
+\end{frame}
+
+\begin{frame}
+	\frametitle{Different deployment scenarios}
+	\begin{center}
+		\only<1>{\includegraphics[width=.9\linewidth]{assets/aerogramme_components1.drawio.pdf}}%
+		\only<2>{\includegraphics[width=.9\linewidth]{assets/aerogramme_components2.drawio.pdf}}%
+	\end{center}
+\end{frame}
+
+\begin{frame}
+	\frametitle{A new model for building resilient software}
+	\begin{itemize}
+		\item Design a data model suited to K2V\\
+			{\footnotesize (see Cassandra docs on porting SQL data models to Cassandra)}
+			\vspace{1em}
+			\begin{itemize}
+				\item Use CRDTs or other eventually consistent data types (see e.g. Bayou)
+					\vspace{1em}
+				\item Store opaque binary blobs to provide End-to-End Encryption\\
+			\end{itemize}
+			\vspace{1em}
+		\item Store big blobs (files) in S3
+			\vspace{1em}
+		\item Let Garage manage sharding, replication, failover, etc.
+	\end{itemize}
+\end{frame}
+
+\begin{frame}
+	\frametitle{Research perspectives}
+	\begin{itemize}
+		\item Write about Garage's global architecture \emph{(paper in progress)}
+			\vspace{1em}
+		\item Measure and improve Garage's performances
+			\vspace{1em}
+		\item Discuss the optimal layout algorithm, provide proofs
+			\vspace{1em}
+		\item Write about our proposed architecture for (E2EE) apps over K2V+S3
+	\end{itemize}
+\end{frame}
+
+\begin{frame}
+	\frametitle{Where to find us}
+	\begin{center}
+			\includegraphics[width=.25\linewidth]{../../logo/garage_hires.png}\\
+			\vspace{-1em}
+		\url{https://garagehq.deuxfleurs.fr/}\\
+		\url{mailto:garagehq@deuxfleurs.fr}\\
+		\texttt{\#garage:deuxfleurs.fr} on Matrix
+
+		\vspace{1.5em}
+			\includegraphics[width=.06\linewidth]{assets/rust_logo.png}
+			\includegraphics[width=.13\linewidth]{assets/AGPLv3_Logo.png}
+	\end{center}
+\end{frame}
+
+\end{document}
+
+%% vim: set ts=4 sw=4 tw=0 noet spelllang=en :

nix/common.nix

@@ -4,18 +4,16 @@ rec {
    */
   pkgsSrc = fetchTarball {
     # As of 2021-10-04
-    url ="https://github.com/NixOS/nixpkgs/archive/b27d18a412b071f5d7991d1648cfe78ee7afe68a.tar.gz";
+    url = "https://github.com/NixOS/nixpkgs/archive/b27d18a412b071f5d7991d1648cfe78ee7afe68a.tar.gz";
     sha256 = "1xy9zpypqfxs5gcq5dcla4bfkhxmh5nzn9dyqkr03lqycm9wg5cr";
   };
   cargo2nixSrc = fetchGit {
     # As of 2022-03-17
     url = "https://github.com/superboum/cargo2nix";
-    ref = "main";
-    rev = "bcbf3ba99e9e01a61eb83a24624419c2dd9dec64";
+    ref = "dedup_propagate";
+    rev = "486675c67249e735dd7eb68e1b9feac9db102be7";
   };
 
-
-
   /*
    * Shared objects
    */

nix/compile.nix

@@ -0,0 +1,227 @@
+{
+  system ? builtins.currentSystem,
+  target ? null,
+  compiler ? "rustc",
+  release ? false,
+  git_version ? null,
+}:
+
+with import ./common.nix;
+
+let
+  log = v: builtins.trace v v;
+
+  pkgs = import pkgsSrc {
+    inherit system; 
+    ${ if target == null then null else "crossSystem" } = { config = target; };
+    overlays = [ cargo2nixOverlay ];
+  };
+
+  /*
+   Rust and Nix triples are not the same. Cargo2nix has a dedicated library
+   to convert Nix triples to Rust ones. We need this conversion as we want to
+   set later options linked to our (rust) target in a generic way. Not only
+   the triple terminology is different, but also the "roles" are named differently.
+   Nix uses a build/host/target terminology where Nix's "host" maps to Cargo's "target".
+  */
+  rustTarget = log (pkgs.rustBuilder.rustLib.rustTriple pkgs.stdenv.hostPlatform);
+
+  /*
+   Cargo2nix is built for rustOverlay which installs Rust from Mozilla releases.
+   We want our own Rust to avoid incompatibilities, like we had with musl 1.2.0.
+   rustc was built with musl < 1.2.0 and nix shipped musl >= 1.2.0 which lead to compilation breakage.
+   So we want a Rust release that is bound to our Nix repository to avoid these problems.
+   See here for more info: https://musl.libc.org/time64.html
+   Because Cargo2nix does not support the Rust environment shipped by NixOS,
+   we emulate the structure of the Rust object created by rustOverlay.
+   In practise, rustOverlay ships rustc+cargo in a single derivation while
+   NixOS ships them in separate ones. We reunite them with symlinkJoin.
+   */
+  rustChannel = {
+    rustc = pkgs.symlinkJoin {
+      name = "rust-channel";
+      paths = [ 
+        pkgs.rustPlatform.rust.cargo
+        pkgs.rustPlatform.rust.rustc
+      ];
+    };
+    clippy = pkgs.symlinkJoin {
+      name = "clippy-channel";
+      paths = [ 
+        pkgs.rustPlatform.rust.cargo
+        pkgs.rustPlatform.rust.rustc
+        pkgs.clippy
+      ];
+    };
+  }.${compiler};
+
+  clippyBuilder = pkgs.writeScriptBin "clippy" ''
+    #!${pkgs.stdenv.shell}
+    . ${cargo2nixSrc + "/overlay/utils.sh"}
+    isBuildScript=
+    args=("$@")
+    for i in "''${!args[@]}"; do
+      if [ "xmetadata=" = "x''${args[$i]::9}" ]; then
+        args[$i]=metadata=$NIX_RUST_METADATA
+      elif [ "x--crate-name" = "x''${args[$i]}" ] && [ "xbuild_script_" = "x''${args[$i+1]::13}" ]; then
+        isBuildScript=1
+      fi
+    done
+    if [ "$isBuildScript" ]; then
+      args+=($NIX_RUST_BUILD_LINK_FLAGS)
+    else
+      args+=($NIX_RUST_LINK_FLAGS)
+    fi
+    touch invoke.log
+    echo "''${args[@]}" >>invoke.log
+
+    exec ${rustChannel}/bin/clippy-driver --deny warnings "''${args[@]}"
+  '';
+
+  buildEnv = (drv: {
+    rustc = drv.setBuildEnv;
+    clippy = ''
+      ${drv.setBuildEnv or "" }
+      echo
+      echo --- BUILDING WITH CLIPPY ---
+      echo 
+
+      export RUSTC=${clippyBuilder}/bin/clippy
+    '';
+  }.${compiler});
+
+  /*
+   Cargo2nix provides many overrides by default, you can take inspiration from them:
+   https://github.com/cargo2nix/cargo2nix/blob/master/overlay/overrides.nix
+
+   You can have a complete list of the available options by looking at the overriden object, mkcrate:
+   https://github.com/cargo2nix/cargo2nix/blob/master/overlay/mkcrate.nix
+  */
+  overrides = pkgs.rustBuilder.overrides.all ++ [
+    /*
+     [1] We add some logic to compile our crates with clippy, it provides us many additional lints
+
+     [2] We need to alter Nix hardening to make static binaries: PIE,
+     Position Independent Executables seems to be supported only on amd64. Having
+     this flag set either 1. make our executables crash or 2. compile as dynamic on some platforms.
+     Here, we deactivate it. Later (find `codegenOpts`), we reactivate it for supported targets
+     (only amd64 curently) through the `-static-pie` flag.
+     PIE is a feature used by ASLR, which helps mitigate security issues.
+     Learn more about Nix Hardening at: https://github.com/NixOS/nixpkgs/blob/master/pkgs/build-support/cc-wrapper/add-hardening.sh
+
+     [3] We want to inject the git version while keeping the build deterministic.
+     As we do not want to consider the .git folder as part of the input source,
+     we ask the user (the CI often) to pass the value to Nix.
+
+     [4] We ship some parts of the code disabled by default by putting them behind a flag.
+     It speeds up the compilation (when the feature is not required) and released crates have less dependency by default (less attack surface, disk space, etc.).
+     But we want to ship these additional features when we release Garage.
+     In the end, we chose to exclude all features from debug builds while putting (all of) them in the release builds.
+     Currently, the only feature of Garage is kubernetes-discovery from the garage_rpc crate.
+     The experimental feature k2v is also enabled here in all builds.
+    */
+    (pkgs.rustBuilder.rustLib.makeOverride {
+      name = "garage";
+      overrideAttrs = drv: { 
+        /* [1] */ setBuildEnv = (buildEnv drv);
+        /* [2] */ hardeningDisable = [ "pie" ];
+      };
+    })
+
+    (pkgs.rustBuilder.rustLib.makeOverride {
+      name = "garage_rpc";
+      overrideAttrs = drv: { /* [1] */ setBuildEnv = (buildEnv drv); };
+      overrideArgs = old: {
+        /* [4] */ features = if release then [ "kubernetes-discovery" "k2v" ] else [ "k2v" ];
+      };
+    })
+
+    (pkgs.rustBuilder.rustLib.makeOverride {
+      name = "garage_db";
+      overrideAttrs = drv: { /* [1] */ setBuildEnv = (buildEnv drv); };
+    })
+
+    (pkgs.rustBuilder.rustLib.makeOverride {
+      name = "garage_util";
+      overrideAttrs = drv:
+        (if git_version != null then {
+          /* [3] */ preConfigure = ''
+            ${drv.preConfigure or ""}
+            export GIT_VERSION="${git_version}"
+          '';
+        } else {})
+        // 
+        { /* [1] */ setBuildEnv = (buildEnv drv); };
+    })
+
+    (pkgs.rustBuilder.rustLib.makeOverride {
+      name = "garage_table";
+      overrideAttrs = drv: { /* [1] */ setBuildEnv = (buildEnv drv); };
+    })
+
+    (pkgs.rustBuilder.rustLib.makeOverride {
+      name = "garage_block";
+      overrideAttrs = drv: { /* [1] */ setBuildEnv = (buildEnv drv); };
+    })
+
+    (pkgs.rustBuilder.rustLib.makeOverride {
+      name = "garage_model";
+      overrideAttrs = drv: { /* [1] */ setBuildEnv = (buildEnv drv); };
+    })
+
+    (pkgs.rustBuilder.rustLib.makeOverride {
+      name = "garage_api";
+      overrideAttrs = drv: { /* [1] */ setBuildEnv = (buildEnv drv); };
+    })
+
+    (pkgs.rustBuilder.rustLib.makeOverride {
+      name = "garage_web";
+      overrideAttrs = drv: { /* [1] */ setBuildEnv = (buildEnv drv); };
+    })
+
+    (pkgs.rustBuilder.rustLib.makeOverride {
+      name = "k2v-client";
+      overrideAttrs = drv: { /* [1] */ setBuildEnv = (buildEnv drv); };
+    })
+  ];
+
+  packageFun = import ../Cargo.nix;
+
+  /*
+    We compile fully static binaries with musl to simplify deployment on most systems.
+    When possible, we reactivate PIE hardening (see above).
+
+    Also, if you set the RUSTFLAGS environment variable, the following parameters will
+    be ignored.
+
+    For more information on static builds, please refer to Rust's RFC 1721.
+    https://rust-lang.github.io/rfcs/1721-crt-static.html#specifying-dynamicstatic-c-runtime-linkage
+  */
+
+  codegenOpts = {
+   "armv6l-unknown-linux-musleabihf" = [ "target-feature=+crt-static" "link-arg=-static" ]; /* compile as dynamic with static-pie */
+   "aarch64-unknown-linux-musl" = [ "target-feature=+crt-static" "link-arg=-static" ]; /* segfault with static-pie */
+   "i686-unknown-linux-musl" = [ "target-feature=+crt-static" "link-arg=-static" ]; /* segfault with static-pie */
+   "x86_64-unknown-linux-musl" = [ "target-feature=+crt-static" "link-arg=-static-pie" ];
+  };
+
+in
+  /*
+   The following definition is not elegant as we use a low level function of Cargo2nix
+   that enables us to pass our custom rustChannel object. We need this low level definition
+   to pass Nix's Rust toolchains instead of Mozilla's one.
+
+   target is mandatory but must be kept to null to allow cargo2nix to set it to the appropriate value
+   for each crate.
+  */ 
+  pkgs.rustBuilder.makePackageSet {
+    inherit packageFun rustChannel release codegenOpts;
+    packageOverrides = overrides;
+    target = null;
+
+    buildRustPackages = pkgs.buildPackages.rustBuilder.makePackageSet {
+      inherit rustChannel packageFun codegenOpts;
+      packageOverrides = overrides;
+      target = null;
+    };
+  }

nix/nix.conf

@@ -1,4 +1,9 @@
 substituters = https://cache.nixos.org https://nix.web.deuxfleurs.fr
 trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= nix.web.deuxfleurs.fr:eTGL6kvaQn6cDR/F9lDYUIP9nCVR/kkshYfLDJf1yKs=
 max-jobs = auto
-cores = 4
+cores = 0
+log-lines = 200
+filter-syscalls = false
+sandbox = false
+keep-outputs = true
+keep-derivations = true

script/not-dynamic.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+
+set -e
+
+if [ "$#" -ne 1 ]; then
+    echo "[fail] usage: $0 binary"
+    exit 2
+fi
+
+if file $1 | grep 'dynamically linked' 2>&1; then
+    echo "[fail] $1 is dynamic"
+    exit 1
+fi
+echo "[ok] $1 is probably static"

shell.nix

@@ -1,8 +1,5 @@
 {
   system ? builtins.currentSystem,
-  rust ? true,
-  integration ? true,
-  release ? true,
 }:
 
 with import ./nix/common.nix;
@@ -16,9 +13,59 @@ let
   winscp = (import ./nix/winscp.nix) pkgs;
 
 in
+  {
 
-pkgs.mkShell {
-  shellHook = ''
+    /* --- Rust Shell ---
+     * Use it to compile Garage
+     */
+    rust = pkgs.mkShell {
+      shellHook = ''
+function refresh_toolchain {
+  nix copy \
+    --to 's3://nix?endpoint=garage.deuxfleurs.fr&region=garage&secret-key=/etc/nix/signing-key.sec' \
+    $(nix-store -qR \
+      $(nix-build --quiet --no-build-output --no-out-link nix/toolchain.nix))
+}
+      '';
+
+      nativeBuildInputs = [
+        #pkgs.rustPlatform.rust.rustc
+        pkgs.rustPlatform.rust.cargo
+        #pkgs.clippy
+        pkgs.rustfmt
+        #pkgs.perl
+        #pkgs.protobuf
+        #pkgs.pkg-config
+        #pkgs.openssl
+        pkgs.file
+        #cargo2nix.packages.x86_64-linux.cargo2nix
+      ];
+    };
+
+    /* --- Integration shell ---
+     * Use it to test Garage with common S3 clients
+     */
+    integration = pkgs.mkShell {
+      nativeBuildInputs = [
+        winscp
+        pkgs.s3cmd
+        pkgs.awscli2
+        pkgs.minio-client
+        pkgs.rclone
+        pkgs.socat
+        pkgs.psmisc
+        pkgs.which
+        pkgs.openssl
+        pkgs.curl
+        pkgs.jq
+      ];
+    };
+
+    /* --- Release shell ---
+     * A shell built to make releasing easier
+     */
+    release = pkgs.mkShell {
+      shellHook = ''
 function to_s3 {
   aws \
       --endpoint-url https://garage.deuxfleurs.fr \
@@ -62,45 +109,12 @@ function refresh_index {
       result/share/_releases.html \
       s3://garagehq.deuxfleurs.fr/
 }
+        '';
+      nativeBuildInputs = [
+        pkgs.awscli2
+        kaniko 
+      ];
+    };
+  }
 
-function refresh_toolchain {
-  nix copy \
-    --to 's3://nix?endpoint=garage.deuxfleurs.fr&region=garage&secret-key=/etc/nix/signing-key.sec' \
-    $(nix-store -qR \
-      $(nix-build --quiet --no-build-output --no-out-link nix/toolchain.nix))
-}
-  '';
 
-  nativeBuildInputs = 
-   (if rust then [
-     pkgs.rustPlatform.rust.rustc
-     pkgs.rustPlatform.rust.cargo
-     pkgs.clippy
-     pkgs.rustfmt
-     pkgs.perl
-     pkgs.protobuf
-     pkgs.pkg-config
-     pkgs.openssl
-     cargo2nix.packages.x86_64-linux.cargo2nix
-    ] else [])
-   ++
-   (if integration then [
-     winscp
-     pkgs.s3cmd
-     pkgs.awscli2
-     pkgs.minio-client
-     pkgs.rclone
-     pkgs.socat
-     pkgs.psmisc
-     pkgs.which
-     pkgs.openssl
-     pkgs.curl
-     pkgs.jq
-    ] else [])
-   ++
-   (if release then [ 
-     pkgs.awscli2
-     kaniko 
-    ] else [])
-   ;
-}

src/block/Cargo.toml

@@ -21,6 +21,7 @@ garage_table = { version = "0.7.0", path = "../table" }
 
 opentelemetry = "0.17"
 
+arc-swap = "1.5"
 async-trait = "0.1.7"
 bytes = "1.0"
 hex = "0.4"

src/block/lib.rs

@@ -2,6 +2,8 @@
 extern crate tracing;
 
 pub mod manager;
+pub mod repair;
+pub mod resync;
 
 mod block;
 mod metrics;

src/block/manager.rs

@@ -1,32 +1,19 @@
-use core::ops::Bound;
-
-use std::convert::TryInto;
-use std::path::{Path, PathBuf};
+use std::path::PathBuf;
 use std::sync::Arc;
 use std::time::Duration;
 
 use async_trait::async_trait;
 use serde::{Deserialize, Serialize};
 
-use futures::future::*;
-use futures::select;
 use tokio::fs;
 use tokio::io::{AsyncReadExt, AsyncWriteExt};
-use tokio::sync::{watch, Mutex, Notify};
-
-use opentelemetry::{
-	trace::{FutureExt as OtelFutureExt, TraceContextExt, Tracer},
-	Context, KeyValue,
-};
+use tokio::sync::{mpsc, Mutex};
 
 use garage_db as db;
-use garage_db::counted_tree_hack::CountedTree;
 
 use garage_util::data::*;
 use garage_util::error::*;
 use garage_util::metrics::RecordDuration;
-use garage_util::time::*;
-use garage_util::tranquilizer::Tranquilizer;
 
 use garage_rpc::system::System;
 use garage_rpc::*;
@@ -36,23 +23,14 @@ use garage_table::replication::{TableReplication, TableShardedReplication};
 use crate::block::*;
 use crate::metrics::*;
 use crate::rc::*;
+use crate::repair::*;
+use crate::resync::*;
 
 /// Size under which data will be stored inlined in database instead of as files
 pub const INLINE_THRESHOLD: usize = 3072;
 
 // Timeout for RPCs that read and write blocks to remote nodes
-const BLOCK_RW_TIMEOUT: Duration = Duration::from_secs(30);
-// Timeout for RPCs that ask other nodes whether they need a copy
-// of a given block before we delete it locally
-const NEED_BLOCK_QUERY_TIMEOUT: Duration = Duration::from_secs(5);
-
-// The delay between the time where a resync operation fails
-// and the time when it is retried, with exponential backoff
-// (multiplied by 2, 4, 8, 16, etc. for every consecutive failure).
-const RESYNC_RETRY_DELAY: Duration = Duration::from_secs(60);
-// The minimum retry delay is 60 seconds = 1 minute
-// The maximum retry delay is 60 seconds * 2^6 = 60 seconds << 6 = 64 minutes (~1 hour)
-const RESYNC_RETRY_DELAY_MAX_BACKOFF_POWER: u64 = 6;
+pub(crate) const BLOCK_RW_TIMEOUT: Duration = Duration::from_secs(30);
 
 // The delay between the moment when the reference counter
 // drops to zero, and the moment where we allow ourselves
@@ -89,20 +67,18 @@ pub struct BlockManager {
 	pub data_dir: PathBuf,
 
 	compression_level: Option<i32>,
-	background_tranquility: u32,
 
 	mutation_lock: Mutex<BlockManagerLocked>,
 
-	rc: BlockRc,
+	pub(crate) rc: BlockRc,
+	pub resync: BlockResyncManager,
 
-	resync_queue: CountedTree,
-	resync_notify: Notify,
-	resync_errors: CountedTree,
+	pub(crate) system: Arc<System>,
+	pub(crate) endpoint: Arc<Endpoint<BlockRpc, Self>>,
 
-	system: Arc<System>,
-	endpoint: Arc<Endpoint<BlockRpc, Self>>,
+	pub(crate) metrics: BlockManagerMetrics,
 
-	metrics: BlockManagerMetrics,
+	tx_scrub_command: mpsc::Sender<ScrubWorkerCommand>,
 }
 
 // This custom struct contains functions that must only be ran
@@ -115,7 +91,6 @@ impl BlockManager {
 		db: &db::Db,
 		data_dir: PathBuf,
 		compression_level: Option<i32>,
-		background_tranquility: u32,
 		replication: TableShardedReplication,
 		system: Arc<System>,
 	) -> Arc<Self> {
@@ -124,17 +99,7 @@ impl BlockManager {
 			.expect("Unable to open block_local_rc tree");
 		let rc = BlockRc::new(rc);
 
-		let resync_queue = db
-			.open_tree("block_local_resync_queue")
-			.expect("Unable to open block_local_resync_queue tree");
-		let resync_queue =
-			CountedTree::new(resync_queue).expect("Could not count block_local_resync_queue");
-
-		let resync_errors = db
-			.open_tree("block_local_resync_errors")
-			.expect("Unable to open block_local_resync_errors tree");
-		let resync_errors =
-			CountedTree::new(resync_errors).expect("Could not count block_local_resync_errors");
+		let resync = BlockResyncManager::new(db, &system);
 
 		let endpoint = system
 			.netapp
@@ -142,31 +107,39 @@ impl BlockManager {
 
 		let manager_locked = BlockManagerLocked();
 
-		let metrics = BlockManagerMetrics::new(resync_queue.clone(), resync_errors.clone());
+		let metrics = BlockManagerMetrics::new(resync.queue.clone(), resync.errors.clone());
+
+		let (scrub_tx, scrub_rx) = mpsc::channel(1);
 
 		let block_manager = Arc::new(Self {
 			replication,
 			data_dir,
 			compression_level,
-			background_tranquility,
 			mutation_lock: Mutex::new(manager_locked),
 			rc,
-			resync_queue,
-			resync_notify: Notify::new(),
-			resync_errors,
+			resync,
 			system,
 			endpoint,
 			metrics,
+			tx_scrub_command: scrub_tx,
 		});
 		block_manager.endpoint.set_handler(block_manager.clone());
 
-		block_manager.clone().spawn_background_worker();
+		// Spawn a bunch of resync workers
+		for index in 0..MAX_RESYNC_WORKERS {
+			let worker = ResyncWorker::new(index, block_manager.clone());
+			block_manager.system.background.spawn_worker(worker);
+		}
+
+		// Spawn scrub worker
+		let scrub_worker = ScrubWorker::new(block_manager.clone(), scrub_rx);
+		block_manager.system.background.spawn_worker(scrub_worker);
 
 		block_manager
 	}
 
 	/// Ask nodes that might have a (possibly compressed) block for it
-	async fn rpc_get_raw_block(&self, hash: &Hash) -> Result<DataBlock, Error> {
+	pub(crate) async fn rpc_get_raw_block(&self, hash: &Hash) -> Result<DataBlock, Error> {
 		let who = self.replication.read_nodes(hash);
 		let resps = self
 			.system
@@ -218,109 +191,16 @@ impl BlockManager {
 		Ok(())
 	}
 
-	/// Launch the repair procedure on the data store
-	///
-	/// This will list all blocks locally present, as well as those
-	/// that are required because of refcount > 0, and will try
-	/// to fix any mismatch between the two.
-	pub async fn repair_data_store(&self, must_exit: &watch::Receiver<bool>) -> Result<(), Error> {
-		// 1. Repair blocks from RC table.
-		let mut next_start: Option<Hash> = None;
-		loop {
-			// We have to do this complicated two-step process where we first read a bunch
-			// of hashes from the RC table, and then insert them in the to-resync queue,
-			// because of SQLite. Basically, as long as we have an iterator on a DB table,
-			// we can't do anything else on the DB. The naive approach (which we had previously)
-			// of just iterating on the RC table and inserting items one to one in the resync
-			// queue can't work here, it would just provoke a deadlock in the SQLite adapter code.
-			// This is mostly because the Rust bindings for SQLite assume a worst-case scenario
-			// where SQLite is not compiled in thread-safe mode, so we have to wrap everything
-			// in a mutex (see db/sqlite_adapter.rs and discussion in PR #322).
-			let mut batch_of_hashes = vec![];
-			let start_bound = match next_start.as_ref() {
-				None => Bound::Unbounded,
-				Some(x) => Bound::Excluded(x.as_slice()),
-			};
-			for entry in self
-				.rc
-				.rc
-				.range::<&[u8], _>((start_bound, Bound::Unbounded))?
-			{
-				let (hash, _) = entry?;
-				let hash = Hash::try_from(&hash[..]).unwrap();
-				batch_of_hashes.push(hash);
-				if batch_of_hashes.len() >= 1000 {
-					break;
-				}
-			}
-			if batch_of_hashes.is_empty() {
-				break;
-			}
-
-			for hash in batch_of_hashes.into_iter() {
-				self.put_to_resync(&hash, Duration::from_secs(0))?;
-				next_start = Some(hash)
-			}
-
-			if *must_exit.borrow() {
-				return Ok(());
-			}
-		}
-
-		// 2. Repair blocks actually on disk
-		// Lists all blocks on disk and adds them to the resync queue.
-		// This allows us to find blocks we are storing but don't actually need,
-		// so that we can offload them if necessary and then delete them locally.
-		self.for_each_file(
-			(),
-			move |_, hash| async move {
-				self.put_to_resync(&hash, Duration::from_secs(0))
-					.map_err(Into::into)
-			},
-			must_exit,
-		)
-		.await
-	}
-
-	/// Verify integrity of each block on disk. Use `speed_limit` to limit the load generated by
-	/// this function.
-	pub async fn scrub_data_store(
-		&self,
-		must_exit: &watch::Receiver<bool>,
-		tranquility: u32,
-	) -> Result<(), Error> {
-		let tranquilizer = Tranquilizer::new(30);
-		self.for_each_file(
-			tranquilizer,
-			move |mut tranquilizer, hash| async move {
-				let _ = self.read_block(&hash).await;
-				tranquilizer.tranquilize(tranquility).await;
-				Ok(tranquilizer)
-			},
-			must_exit,
-		)
-		.await
-	}
-
-	/// Get lenght of resync queue
-	pub fn resync_queue_len(&self) -> Result<usize, Error> {
-		// This currently can't return an error because the CountedTree hack
-		// doesn't error on .len(), but this will change when we remove the hack
-		// (hopefully someday!)
-		Ok(self.resync_queue.len())
-	}
-
-	/// Get number of blocks that have an error
-	pub fn resync_errors_len(&self) -> Result<usize, Error> {
-		// (see resync_queue_len comment)
-		Ok(self.resync_errors.len())
-	}
-
 	/// Get number of items in the refcount table
 	pub fn rc_len(&self) -> Result<usize, Error> {
 		Ok(self.rc.rc.len()?)
 	}
 
+	/// Send command to start/stop/manager scrub worker
+	pub async fn send_scrub_command(&self, cmd: ScrubWorkerCommand) {
+		let _ = self.tx_scrub_command.send(cmd).await;
+	}
+
 	//// ----- Managing the reference counter ----
 
 	/// Increment the number of time a block is used, putting it to resynchronization if it is
@@ -339,7 +219,7 @@ impl BlockManager {
 			// we will fecth it from someone.
 			let this = self.clone();
 			tokio::spawn(async move {
-				if let Err(e) = this.put_to_resync(&hash, 2 * BLOCK_RW_TIMEOUT) {
+				if let Err(e) = this.resync.put_to_resync(&hash, 2 * BLOCK_RW_TIMEOUT) {
 					error!("Block {:?} could not be put in resync queue: {}.", hash, e);
 				}
 			});
@@ -361,7 +241,9 @@ impl BlockManager {
 			// after that delay has passed.
 			let this = self.clone();
 			tokio::spawn(async move {
-				if let Err(e) = this.put_to_resync(&hash, BLOCK_GC_DELAY + Duration::from_secs(10))
+				if let Err(e) = this
+					.resync
+					.put_to_resync(&hash, BLOCK_GC_DELAY + Duration::from_secs(10))
 				{
 					error!("Block {:?} could not be put in resync queue: {}.", hash, e);
 				}
@@ -373,7 +255,11 @@ impl BlockManager {
 	// ---- Reading and writing blocks locally ----
 
 	/// Write a block to disk
-	async fn write_block(&self, hash: &Hash, data: &DataBlock) -> Result<BlockRpc, Error> {
+	pub(crate) async fn write_block(
+		&self,
+		hash: &Hash,
+		data: &DataBlock,
+	) -> Result<BlockRpc, Error> {
 		let write_size = data.inner_buffer().len() as u64;
 
 		let res = self
@@ -390,7 +276,7 @@ impl BlockManager {
 	}
 
 	/// Read block from disk, verifying it's integrity
-	async fn read_block(&self, hash: &Hash) -> Result<BlockRpc, Error> {
+	pub(crate) async fn read_block(&self, hash: &Hash) -> Result<BlockRpc, Error> {
 		let data = self
 			.read_block_internal(hash)
 			.bound_record_duration(&self.metrics.block_read_duration)
@@ -409,7 +295,7 @@ impl BlockManager {
 			Ok(c) => c,
 			Err(e) => {
 				// Not found but maybe we should have had it ??
-				self.put_to_resync(hash, 2 * BLOCK_RW_TIMEOUT)?;
+				self.resync.put_to_resync(hash, 2 * BLOCK_RW_TIMEOUT)?;
 				return Err(Into::into(e));
 			}
 		};
@@ -436,24 +322,37 @@ impl BlockManager {
 				.await
 				.move_block_to_corrupted(hash, self)
 				.await?;
-			self.put_to_resync(hash, Duration::from_millis(0))?;
+			self.resync.put_to_resync(hash, Duration::from_millis(0))?;
 			return Err(Error::CorruptData(*hash));
 		}
 
 		Ok(data)
 	}
 
-	/// Check if this node should have a block, but don't actually have it
-	async fn need_block(&self, hash: &Hash) -> Result<bool, Error> {
-		let BlockStatus { exists, needed } = self
-			.mutation_lock
+	/// Check if this node has a block and whether it needs it
+	pub(crate) async fn check_block_status(&self, hash: &Hash) -> Result<BlockStatus, Error> {
+		self.mutation_lock
 			.lock()
 			.await
 			.check_block_status(hash, self)
-			.await?;
+			.await
+	}
+
+	/// Check if this node should have a block, but don't actually have it
+	async fn need_block(&self, hash: &Hash) -> Result<bool, Error> {
+		let BlockStatus { exists, needed } = self.check_block_status(hash).await?;
 		Ok(needed.is_nonzero() && !exists)
 	}
 
+	/// Delete block if it is not needed anymore
+	pub(crate) async fn delete_if_unneeded(&self, hash: &Hash) -> Result<(), Error> {
+		self.mutation_lock
+			.lock()
+			.await
+			.delete_if_unneeded(hash, self)
+			.await
+	}
+
 	/// Utility: gives the path of the directory in which a block should be found
 	fn block_dir(&self, hash: &Hash) -> PathBuf {
 		let mut path = self.data_dir.clone();
@@ -480,406 +379,6 @@ impl BlockManager {
 		path.set_extension("");
 		fs::metadata(&path).await.map(|_| false).map_err(Into::into)
 	}
-
-	// ---- Resync loop ----
-
-	// This part manages a queue of blocks that need to be
-	// "resynchronized", i.e. that need to have a check that
-	// they are at present if we need them, or that they are
-	// deleted once the garbage collection delay has passed.
-	//
-	// Here are some explanations on how the resync queue works.
-	// There are two Sled trees that are used to have information
-	// about the status of blocks that need to be resynchronized:
-	//
-	// - resync_queue: a tree that is ordered first by a timestamp
-	//   (in milliseconds since Unix epoch) that is the time at which
-	//   the resync must be done, and second by block hash.
-	//   The key in this tree is just:
-	//       concat(timestamp (8 bytes), hash (32 bytes))
-	//   The value is the same 32-byte hash.
-	//
-	// - resync_errors: a tree that indicates for each block
-	//   if the last resync resulted in an error, and if so,
-	//   the following two informations (see the ErrorCounter struct):
-	//   - how many consecutive resync errors for this block?
-	//   - when was the last try?
-	//   These two informations are used to implement an
-	//   exponential backoff retry strategy.
-	//   The key in this tree is the 32-byte hash of the block,
-	//   and the value is the encoded ErrorCounter value.
-	//
-	// We need to have these two trees, because the resync queue
-	// is not just a queue of items to process, but a set of items
-	// that are waiting a specific delay until we can process them
-	// (the delay being necessary both internally for the exponential
-	// backoff strategy, and exposed as a parameter when adding items
-	// to the queue, e.g. to wait until the GC delay has passed).
-	// This is why we need one tree ordered by time, and one
-	// ordered by identifier of item to be processed (block hash).
-	//
-	// When the worker wants to process an item it takes from
-	// resync_queue, it checks in resync_errors that if there is an
-	// exponential back-off delay to await, it has passed before we
-	// process the item. If not, the item in the queue is skipped
-	// (but added back for later processing after the time of the
-	// delay).
-	//
-	// An alternative that would have seemed natural is to
-	// only add items to resync_queue with a processing time that is
-	// after the delay, but there are several issues with this:
-	// - This requires to synchronize updates to resync_queue and
-	//   resync_errors (with the current model, there is only one thread,
-	//   the worker thread, that accesses resync_errors,
-	//   so no need to synchronize) by putting them both in a lock.
-	//   This would mean that block_incref might need to take a lock
-	//   before doing its thing, meaning it has much more chances of
-	//   not completing successfully if something bad happens to Garage.
-	//   Currently Garage is not able to recover from block_incref that
-	//   doesn't complete successfully, because it is necessary to ensure
-	//   the consistency between the state of the block manager and
-	//   information in the BlockRef table.
-	// - If a resync fails, we put that block in the resync_errors table,
-	//   and also add it back to resync_queue to be processed after
-	//   the exponential back-off delay,
-	//   but maybe the block is already scheduled to be resynced again
-	//   at another time that is before the exponential back-off delay,
-	//   and we have no way to check that easily. This means that
-	//   in all cases, we need to check the resync_errors table
-	//   in the resync loop at the time when a block is popped from
-	//   the resync_queue.
-	// Overall, the current design is therefore simpler and more robust
-	// because it tolerates inconsistencies between the resync_queue
-	// and resync_errors table (items being scheduled in resync_queue
-	// for times that are earlier than the exponential back-off delay
-	// is a natural condition that is handled properly).
-
-	fn spawn_background_worker(self: Arc<Self>) {
-		// Launch a background workers for background resync loop processing
-		let background = self.system.background.clone();
-		tokio::spawn(async move {
-			tokio::time::sleep(Duration::from_secs(10)).await;
-			background.spawn_worker("block resync worker".into(), move |must_exit| {
-				self.resync_loop(must_exit)
-			});
-		});
-	}
-
-	fn put_to_resync(&self, hash: &Hash, delay: Duration) -> db::Result<()> {
-		let when = now_msec() + delay.as_millis() as u64;
-		self.put_to_resync_at(hash, when)
-	}
-
-	fn put_to_resync_at(&self, hash: &Hash, when: u64) -> db::Result<()> {
-		trace!("Put resync_queue: {} {:?}", when, hash);
-		let mut key = u64::to_be_bytes(when).to_vec();
-		key.extend(hash.as_ref());
-		self.resync_queue.insert(key, hash.as_ref())?;
-		self.resync_notify.notify_waiters();
-		Ok(())
-	}
-
-	async fn resync_loop(self: Arc<Self>, mut must_exit: watch::Receiver<bool>) {
-		let mut tranquilizer = Tranquilizer::new(30);
-
-		while !*must_exit.borrow() {
-			match self.resync_iter(&mut must_exit).await {
-				Ok(true) => {
-					tranquilizer.tranquilize(self.background_tranquility).await;
-				}
-				Ok(false) => {
-					tranquilizer.reset();
-				}
-				Err(e) => {
-					// The errors that we have here are only Sled errors
-					// We don't really know how to handle them so just ¯\_(ツ)_/¯
-					// (there is kind of an assumption that Sled won't error on us,
-					// if it does there is not much we can do -- TODO should we just panic?)
-					error!(
-						"Could not do a resync iteration: {} (this is a very bad error)",
-						e
-					);
-					tranquilizer.reset();
-				}
-			}
-		}
-	}
-
-	// The result of resync_iter is:
-	// - Ok(true) -> a block was processed (successfully or not)
-	// - Ok(false) -> no block was processed, but we are ready for the next iteration
-	// - Err(_) -> a Sled error occurred when reading/writing from resync_queue/resync_errors
-	async fn resync_iter(&self, must_exit: &mut watch::Receiver<bool>) -> Result<bool, db::Error> {
-		if let Some((time_bytes, hash_bytes)) = self.resync_queue.first()? {
-			let time_msec = u64::from_be_bytes(time_bytes[0..8].try_into().unwrap());
-			let now = now_msec();
-
-			if now >= time_msec {
-				let hash = Hash::try_from(&hash_bytes[..]).unwrap();
-
-				if let Some(ec) = self.resync_errors.get(hash.as_slice())? {
-					let ec = ErrorCounter::decode(&ec);
-					if now < ec.next_try() {
-						// if next retry after an error is not yet,
-						// don't do resync and return early, but still
-						// make sure the item is still in queue at expected time
-						self.put_to_resync_at(&hash, ec.next_try())?;
-						// ec.next_try() > now >= time_msec, so this remove
-						// is not removing the one we added just above
-						// (we want to do the remove after the insert to ensure
-						// that the item is not lost if we crash in-between)
-						self.resync_queue.remove(time_bytes)?;
-						return Ok(false);
-					}
-				}
-
-				let tracer = opentelemetry::global::tracer("garage");
-				let trace_id = gen_uuid();
-				let span = tracer
-					.span_builder("Resync block")
-					.with_trace_id(
-						opentelemetry::trace::TraceId::from_hex(&hex::encode(
-							&trace_id.as_slice()[..16],
-						))
-						.unwrap(),
-					)
-					.with_attributes(vec![KeyValue::new("block", format!("{:?}", hash))])
-					.start(&tracer);
-
-				let res = self
-					.resync_block(&hash)
-					.with_context(Context::current_with_span(span))
-					.bound_record_duration(&self.metrics.resync_duration)
-					.await;
-
-				self.metrics.resync_counter.add(1);
-
-				if let Err(e) = &res {
-					self.metrics.resync_error_counter.add(1);
-					warn!("Error when resyncing {:?}: {}", hash, e);
-
-					let err_counter = match self.resync_errors.get(hash.as_slice())? {
-						Some(ec) => ErrorCounter::decode(&ec).add1(now + 1),
-						None => ErrorCounter::new(now + 1),
-					};
-
-					self.resync_errors
-						.insert(hash.as_slice(), err_counter.encode())?;
-
-					self.put_to_resync_at(&hash, err_counter.next_try())?;
-					// err_counter.next_try() >= now + 1 > now,
-					// the entry we remove from the queue is not
-					// the entry we inserted with put_to_resync_at
-					self.resync_queue.remove(time_bytes)?;
-				} else {
-					self.resync_errors.remove(hash.as_slice())?;
-					self.resync_queue.remove(time_bytes)?;
-				}
-
-				Ok(true)
-			} else {
-				let delay = tokio::time::sleep(Duration::from_millis(time_msec - now));
-				select! {
-					_ = delay.fuse() => {},
-					_ = self.resync_notify.notified().fuse() => {},
-					_ = must_exit.changed().fuse() => {},
-				}
-				Ok(false)
-			}
-		} else {
-			// Here we wait either for a notification that an item has been
-			// added to the queue, or for a constant delay of 10 secs to expire.
-			// The delay avoids a race condition where the notification happens
-			// between the time we checked the queue and the first poll
-			// to resync_notify.notified(): if that happens, we'll just loop
-			// back 10 seconds later, which is fine.
-			let delay = tokio::time::sleep(Duration::from_secs(10));
-			select! {
-				_ = delay.fuse() => {},
-				_ = self.resync_notify.notified().fuse() => {},
-				_ = must_exit.changed().fuse() => {},
-			}
-			Ok(false)
-		}
-	}
-
-	async fn resync_block(&self, hash: &Hash) -> Result<(), Error> {
-		let BlockStatus { exists, needed } = self
-			.mutation_lock
-			.lock()
-			.await
-			.check_block_status(hash, self)
-			.await?;
-
-		if exists != needed.is_needed() || exists != needed.is_nonzero() {
-			debug!(
-				"Resync block {:?}: exists {}, nonzero rc {}, deletable {}",
-				hash,
-				exists,
-				needed.is_nonzero(),
-				needed.is_deletable(),
-			);
-		}
-
-		if exists && needed.is_deletable() {
-			info!("Resync block {:?}: offloading and deleting", hash);
-
-			let mut who = self.replication.write_nodes(hash);
-			if who.len() < self.replication.write_quorum() {
-				return Err(Error::Message("Not trying to offload block because we don't have a quorum of nodes to write to".to_string()));
-			}
-			who.retain(|id| *id != self.system.id);
-
-			let msg = Arc::new(BlockRpc::NeedBlockQuery(*hash));
-			let who_needs_fut = who.iter().map(|to| {
-				self.system.rpc.call_arc(
-					&self.endpoint,
-					*to,
-					msg.clone(),
-					RequestStrategy::with_priority(PRIO_BACKGROUND)
-						.with_timeout(NEED_BLOCK_QUERY_TIMEOUT),
-				)
-			});
-			let who_needs_resps = join_all(who_needs_fut).await;
-
-			let mut need_nodes = vec![];
-			for (node, needed) in who.iter().zip(who_needs_resps.into_iter()) {
-				match needed.err_context("NeedBlockQuery RPC")? {
-					BlockRpc::NeedBlockReply(needed) => {
-						if needed {
-							need_nodes.push(*node);
-						}
-					}
-					m => {
-						return Err(Error::unexpected_rpc_message(m));
-					}
-				}
-			}
-
-			if !need_nodes.is_empty() {
-				trace!(
-					"Block {:?} needed by {} nodes, sending",
-					hash,
-					need_nodes.len()
-				);
-
-				for node in need_nodes.iter() {
-					self.metrics
-						.resync_send_counter
-						.add(1, &[KeyValue::new("to", format!("{:?}", node))]);
-				}
-
-				let put_block_message = self.read_block(hash).await?;
-				self.system
-					.rpc
-					.try_call_many(
-						&self.endpoint,
-						&need_nodes[..],
-						put_block_message,
-						RequestStrategy::with_priority(PRIO_BACKGROUND)
-							.with_quorum(need_nodes.len())
-							.with_timeout(BLOCK_RW_TIMEOUT),
-					)
-					.await
-					.err_context("PutBlock RPC")?;
-			}
-			info!(
-				"Deleting unneeded block {:?}, offload finished ({} / {})",
-				hash,
-				need_nodes.len(),
-				who.len()
-			);
-
-			self.mutation_lock
-				.lock()
-				.await
-				.delete_if_unneeded(hash, self)
-				.await?;
-
-			self.rc.clear_deleted_block_rc(hash)?;
-		}
-
-		if needed.is_nonzero() && !exists {
-			info!(
-				"Resync block {:?}: fetching absent but needed block (refcount > 0)",
-				hash
-			);
-
-			let block_data = self.rpc_get_raw_block(hash).await?;
-
-			self.metrics.resync_recv_counter.add(1);
-
-			self.write_block(hash, &block_data).await?;
-		}
-
-		Ok(())
-	}
-
-	// ---- Utility: iteration on files in the data directory ----
-
-	async fn for_each_file<F, Fut, State>(
-		&self,
-		state: State,
-		mut f: F,
-		must_exit: &watch::Receiver<bool>,
-	) -> Result<(), Error>
-	where
-		F: FnMut(State, Hash) -> Fut + Send,
-		Fut: Future<Output = Result<State, Error>> + Send,
-		State: Send,
-	{
-		self.for_each_file_rec(&self.data_dir, state, &mut f, must_exit)
-			.await
-			.map(|_| ())
-	}
-
-	fn for_each_file_rec<'a, F, Fut, State>(
-		&'a self,
-		path: &'a Path,
-		mut state: State,
-		f: &'a mut F,
-		must_exit: &'a watch::Receiver<bool>,
-	) -> BoxFuture<'a, Result<State, Error>>
-	where
-		F: FnMut(State, Hash) -> Fut + Send,
-		Fut: Future<Output = Result<State, Error>> + Send,
-		State: Send + 'a,
-	{
-		async move {
-			let mut ls_data_dir = fs::read_dir(path).await?;
-			while let Some(data_dir_ent) = ls_data_dir.next_entry().await? {
-				if *must_exit.borrow() {
-					break;
-				}
-
-				let name = data_dir_ent.file_name();
-				let name = if let Ok(n) = name.into_string() {
-					n
-				} else {
-					continue;
-				};
-				let ent_type = data_dir_ent.file_type().await?;
-
-				let name = name.strip_suffix(".zst").unwrap_or(&name);
-				if name.len() == 2 && hex::decode(&name).is_ok() && ent_type.is_dir() {
-					state = self
-						.for_each_file_rec(&data_dir_ent.path(), state, f, must_exit)
-						.await?;
-				} else if name.len() == 64 {
-					let hash_bytes = if let Ok(h) = hex::decode(&name) {
-						h
-					} else {
-						continue;
-					};
-					let mut hash = [0u8; 32];
-					hash.copy_from_slice(&hash_bytes[..]);
-					state = f(state, hash.into()).await?;
-				}
-			}
-			Ok(state)
-		}
-		.boxed()
-	}
 }
 
 #[async_trait]
@@ -898,9 +397,9 @@ impl EndpointHandler<BlockRpc> for BlockManager {
 	}
 }
 
-struct BlockStatus {
-	exists: bool,
-	needed: RcEntry,
+pub(crate) struct BlockStatus {
+	pub(crate) exists: bool,
+	pub(crate) needed: RcEntry,
 }
 
 impl BlockManagerLocked {
@@ -1005,50 +504,3 @@ impl BlockManagerLocked {
 		Ok(())
 	}
 }
-
-/// Counts the number of errors when resyncing a block,
-/// and the time of the last try.
-/// Used to implement exponential backoff.
-#[derive(Clone, Copy, Debug)]
-struct ErrorCounter {
-	errors: u64,
-	last_try: u64,
-}
-
-impl ErrorCounter {
-	fn new(now: u64) -> Self {
-		Self {
-			errors: 1,
-			last_try: now,
-		}
-	}
-
-	fn decode(data: &[u8]) -> Self {
-		Self {
-			errors: u64::from_be_bytes(data[0..8].try_into().unwrap()),
-			last_try: u64::from_be_bytes(data[8..16].try_into().unwrap()),
-		}
-	}
-	fn encode(&self) -> Vec<u8> {
-		[
-			u64::to_be_bytes(self.errors),
-			u64::to_be_bytes(self.last_try),
-		]
-		.concat()
-	}
-
-	fn add1(self, now: u64) -> Self {
-		Self {
-			errors: self.errors + 1,
-			last_try: now,
-		}
-	}
-
-	fn delay_msec(&self) -> u64 {
-		(RESYNC_RETRY_DELAY.as_millis() as u64)
-			<< std::cmp::min(self.errors - 1, RESYNC_RETRY_DELAY_MAX_BACKOFF_POWER)
-	}
-	fn next_try(&self) -> u64 {
-		self.last_try + self.delay_msec()
-	}
-}

src/block/repair.rs

@@ -0,0 +1,466 @@
+use core::ops::Bound;
+use std::path::PathBuf;
+use std::sync::Arc;
+use std::time::Duration;
+
+use async_trait::async_trait;
+use serde::{Deserialize, Serialize};
+use tokio::fs;
+use tokio::select;
+use tokio::sync::mpsc;
+use tokio::sync::watch;
+
+use garage_util::background::*;
+use garage_util::data::*;
+use garage_util::error::*;
+use garage_util::persister::Persister;
+use garage_util::time::*;
+use garage_util::tranquilizer::Tranquilizer;
+
+use crate::manager::*;
+
+// Full scrub every 30 days
+const SCRUB_INTERVAL: Duration = Duration::from_secs(3600 * 24 * 30);
+// Scrub tranquility is initially set to 4, but can be changed in the CLI
+// and the updated version is persisted over Garage restarts
+const INITIAL_SCRUB_TRANQUILITY: u32 = 4;
+
+// ---- ---- ----
+// FIRST KIND OF REPAIR: FINDING MISSING BLOCKS/USELESS BLOCKS
+// This is a one-shot repair operation that can be launched,
+// checks everything, and then exits.
+// ---- ---- ----
+
+pub struct RepairWorker {
+	manager: Arc<BlockManager>,
+	next_start: Option<Hash>,
+	block_iter: Option<BlockStoreIterator>,
+}
+
+impl RepairWorker {
+	pub fn new(manager: Arc<BlockManager>) -> Self {
+		Self {
+			manager,
+			next_start: None,
+			block_iter: None,
+		}
+	}
+}
+
+#[async_trait]
+impl Worker for RepairWorker {
+	fn name(&self) -> String {
+		"Block repair worker".into()
+	}
+
+	fn info(&self) -> Option<String> {
+		match self.block_iter.as_ref() {
+			None => {
+				let idx_bytes = self
+					.next_start
+					.as_ref()
+					.map(|x| x.as_slice())
+					.unwrap_or(&[]);
+				let idx_bytes = if idx_bytes.len() > 4 {
+					&idx_bytes[..4]
+				} else {
+					idx_bytes
+				};
+				Some(format!("Phase 1: {}", hex::encode(idx_bytes)))
+			}
+			Some(bi) => Some(format!("Phase 2: {:.2}% done", bi.progress() * 100.)),
+		}
+	}
+
+	async fn work(&mut self, _must_exit: &mut watch::Receiver<bool>) -> Result<WorkerState, Error> {
+		match self.block_iter.as_mut() {
+			None => {
+				// Phase 1: Repair blocks from RC table.
+
+				// We have to do this complicated two-step process where we first read a bunch
+				// of hashes from the RC table, and then insert them in the to-resync queue,
+				// because of SQLite. Basically, as long as we have an iterator on a DB table,
+				// we can't do anything else on the DB. The naive approach (which we had previously)
+				// of just iterating on the RC table and inserting items one to one in the resync
+				// queue can't work here, it would just provoke a deadlock in the SQLite adapter code.
+				// This is mostly because the Rust bindings for SQLite assume a worst-case scenario
+				// where SQLite is not compiled in thread-safe mode, so we have to wrap everything
+				// in a mutex (see db/sqlite_adapter.rs and discussion in PR #322).
+				// TODO: maybe do this with tokio::task::spawn_blocking ?
+				let mut batch_of_hashes = vec![];
+				let start_bound = match self.next_start.as_ref() {
+					None => Bound::Unbounded,
+					Some(x) => Bound::Excluded(x.as_slice()),
+				};
+				for entry in self
+					.manager
+					.rc
+					.rc
+					.range::<&[u8], _>((start_bound, Bound::Unbounded))?
+				{
+					let (hash, _) = entry?;
+					let hash = Hash::try_from(&hash[..]).unwrap();
+					batch_of_hashes.push(hash);
+					if batch_of_hashes.len() >= 1000 {
+						break;
+					}
+				}
+				if batch_of_hashes.is_empty() {
+					// move on to phase 2
+					self.block_iter = Some(BlockStoreIterator::new(&self.manager));
+					return Ok(WorkerState::Busy);
+				}
+
+				for hash in batch_of_hashes.into_iter() {
+					self.manager
+						.resync
+						.put_to_resync(&hash, Duration::from_secs(0))?;
+					self.next_start = Some(hash)
+				}
+
+				Ok(WorkerState::Busy)
+			}
+			Some(bi) => {
+				// Phase 2: Repair blocks actually on disk
+				// Lists all blocks on disk and adds them to the resync queue.
+				// This allows us to find blocks we are storing but don't actually need,
+				// so that we can offload them if necessary and then delete them locally.
+				if let Some(hash) = bi.next().await? {
+					self.manager
+						.resync
+						.put_to_resync(&hash, Duration::from_secs(0))?;
+					Ok(WorkerState::Busy)
+				} else {
+					Ok(WorkerState::Done)
+				}
+			}
+		}
+	}
+
+	async fn wait_for_work(&mut self, _must_exit: &watch::Receiver<bool>) -> WorkerState {
+		unreachable!()
+	}
+}
+
+// ---- ---- ----
+// SECOND KIND OF REPAIR: SCRUBBING THE DATASTORE
+// This is significantly more complex than the process above,
+// as it is a continuously-running task that triggers automatically
+// every SCRUB_INTERVAL, but can also be triggered manually
+// and whose parameter (esp. speed) can be controlled at runtime.
+// ---- ---- ----
+
+pub struct ScrubWorker {
+	manager: Arc<BlockManager>,
+	rx_cmd: mpsc::Receiver<ScrubWorkerCommand>,
+
+	work: ScrubWorkerState,
+	tranquilizer: Tranquilizer,
+
+	persister: Persister<ScrubWorkerPersisted>,
+	persisted: ScrubWorkerPersisted,
+}
+
+#[derive(Serialize, Deserialize)]
+struct ScrubWorkerPersisted {
+	tranquility: u32,
+	time_last_complete_scrub: u64,
+	corruptions_detected: u64,
+}
+
+enum ScrubWorkerState {
+	Running(BlockStoreIterator),
+	Paused(BlockStoreIterator, u64), // u64 = time when to resume scrub
+	Finished,
+}
+
+impl Default for ScrubWorkerState {
+	fn default() -> Self {
+		ScrubWorkerState::Finished
+	}
+}
+
+#[derive(Debug)]
+pub enum ScrubWorkerCommand {
+	Start,
+	Pause(Duration),
+	Resume,
+	Cancel,
+	SetTranquility(u32),
+}
+
+impl ScrubWorker {
+	pub fn new(manager: Arc<BlockManager>, rx_cmd: mpsc::Receiver<ScrubWorkerCommand>) -> Self {
+		let persister = Persister::new(&manager.system.metadata_dir, "scrub_info");
+		let persisted = match persister.load() {
+			Ok(v) => v,
+			Err(_) => ScrubWorkerPersisted {
+				time_last_complete_scrub: 0,
+				tranquility: INITIAL_SCRUB_TRANQUILITY,
+				corruptions_detected: 0,
+			},
+		};
+		Self {
+			manager,
+			rx_cmd,
+			work: ScrubWorkerState::Finished,
+			tranquilizer: Tranquilizer::new(30),
+			persister,
+			persisted,
+		}
+	}
+
+	async fn handle_cmd(&mut self, cmd: ScrubWorkerCommand) {
+		match cmd {
+			ScrubWorkerCommand::Start => {
+				self.work = match std::mem::take(&mut self.work) {
+					ScrubWorkerState::Finished => {
+						let iterator = BlockStoreIterator::new(&self.manager);
+						ScrubWorkerState::Running(iterator)
+					}
+					work => {
+						error!("Cannot start scrub worker: already running!");
+						work
+					}
+				};
+			}
+			ScrubWorkerCommand::Pause(dur) => {
+				self.work = match std::mem::take(&mut self.work) {
+					ScrubWorkerState::Running(it) | ScrubWorkerState::Paused(it, _) => {
+						ScrubWorkerState::Paused(it, now_msec() + dur.as_millis() as u64)
+					}
+					work => {
+						error!("Cannot pause scrub worker: not running!");
+						work
+					}
+				};
+			}
+			ScrubWorkerCommand::Resume => {
+				self.work = match std::mem::take(&mut self.work) {
+					ScrubWorkerState::Paused(it, _) => ScrubWorkerState::Running(it),
+					work => {
+						error!("Cannot resume scrub worker: not paused!");
+						work
+					}
+				};
+			}
+			ScrubWorkerCommand::Cancel => {
+				self.work = match std::mem::take(&mut self.work) {
+					ScrubWorkerState::Running(_) | ScrubWorkerState::Paused(_, _) => {
+						ScrubWorkerState::Finished
+					}
+					work => {
+						error!("Cannot cancel scrub worker: not running!");
+						work
+					}
+				}
+			}
+			ScrubWorkerCommand::SetTranquility(t) => {
+				self.persisted.tranquility = t;
+				if let Err(e) = self.persister.save_async(&self.persisted).await {
+					error!("Could not save new tranquilitiy value: {}", e);
+				}
+			}
+		}
+	}
+}
+
+#[async_trait]
+impl Worker for ScrubWorker {
+	fn name(&self) -> String {
+		"Block scrub worker".into()
+	}
+
+	fn info(&self) -> Option<String> {
+		let s = match &self.work {
+			ScrubWorkerState::Running(bsi) => format!(
+				"{:.2}% done (tranquility = {})",
+				bsi.progress() * 100.,
+				self.persisted.tranquility
+			),
+			ScrubWorkerState::Paused(bsi, rt) => {
+				format!(
+					"Paused, {:.2}% done, resumes at {}",
+					bsi.progress() * 100.,
+					msec_to_rfc3339(*rt)
+				)
+			}
+			ScrubWorkerState::Finished => format!(
+				"Last completed scrub: {}",
+				msec_to_rfc3339(self.persisted.time_last_complete_scrub)
+			),
+		};
+		Some(format!(
+			"{} ; corruptions detected: {}",
+			s, self.persisted.corruptions_detected
+		))
+	}
+
+	async fn work(&mut self, _must_exit: &mut watch::Receiver<bool>) -> Result<WorkerState, Error> {
+		match self.rx_cmd.try_recv() {
+			Ok(cmd) => self.handle_cmd(cmd).await,
+			Err(mpsc::error::TryRecvError::Disconnected) => return Ok(WorkerState::Done),
+			Err(mpsc::error::TryRecvError::Empty) => (),
+		};
+
+		match &mut self.work {
+			ScrubWorkerState::Running(bsi) => {
+				self.tranquilizer.reset();
+				if let Some(hash) = bsi.next().await? {
+					match self.manager.read_block(&hash).await {
+						Err(Error::CorruptData(_)) => {
+							error!("Found corrupt data block during scrub: {:?}", hash);
+							self.persisted.corruptions_detected += 1;
+							self.persister.save_async(&self.persisted).await?;
+						}
+						Err(e) => return Err(e),
+						_ => (),
+					};
+					Ok(self
+						.tranquilizer
+						.tranquilize_worker(self.persisted.tranquility))
+				} else {
+					self.persisted.time_last_complete_scrub = now_msec();
+					self.persister.save_async(&self.persisted).await?;
+					self.work = ScrubWorkerState::Finished;
+					self.tranquilizer.clear();
+					Ok(WorkerState::Idle)
+				}
+			}
+			_ => Ok(WorkerState::Idle),
+		}
+	}
+
+	async fn wait_for_work(&mut self, _must_exit: &watch::Receiver<bool>) -> WorkerState {
+		let (wait_until, command) = match &self.work {
+			ScrubWorkerState::Running(_) => return WorkerState::Busy,
+			ScrubWorkerState::Paused(_, resume_time) => (*resume_time, ScrubWorkerCommand::Resume),
+			ScrubWorkerState::Finished => (
+				self.persisted.time_last_complete_scrub + SCRUB_INTERVAL.as_millis() as u64,
+				ScrubWorkerCommand::Start,
+			),
+		};
+
+		let now = now_msec();
+		if now >= wait_until {
+			self.handle_cmd(command).await;
+			return WorkerState::Busy;
+		}
+		let delay = Duration::from_millis(wait_until - now);
+		select! {
+			_ = tokio::time::sleep(delay) => self.handle_cmd(command).await,
+			cmd = self.rx_cmd.recv() => if let Some(cmd) = cmd {
+				self.handle_cmd(cmd).await;
+			} else {
+				return WorkerState::Done;
+			}
+		}
+
+		match &self.work {
+			ScrubWorkerState::Running(_) => WorkerState::Busy,
+			_ => WorkerState::Idle,
+		}
+	}
+}
+
+// ---- ---- ----
+// UTILITY FOR ENUMERATING THE BLOCK STORE
+// ---- ---- ----
+
+struct BlockStoreIterator {
+	path: Vec<ReadingDir>,
+}
+
+enum ReadingDir {
+	Pending(PathBuf),
+	Read {
+		subpaths: Vec<fs::DirEntry>,
+		pos: usize,
+	},
+}
+
+impl BlockStoreIterator {
+	fn new(manager: &BlockManager) -> Self {
+		let root_dir = manager.data_dir.clone();
+		Self {
+			path: vec![ReadingDir::Pending(root_dir)],
+		}
+	}
+
+	/// Returns progress done, between 0 and 1
+	fn progress(&self) -> f32 {
+		if self.path.is_empty() {
+			1.0
+		} else {
+			let mut ret = 0.0;
+			let mut next_div = 1;
+			for p in self.path.iter() {
+				match p {
+					ReadingDir::Pending(_) => break,
+					ReadingDir::Read { subpaths, pos } => {
+						next_div *= subpaths.len();
+						ret += ((*pos - 1) as f32) / (next_div as f32);
+					}
+				}
+			}
+			ret
+		}
+	}
+
+	async fn next(&mut self) -> Result<Option<Hash>, Error> {
+		loop {
+			let last_path = match self.path.last_mut() {
+				None => return Ok(None),
+				Some(lp) => lp,
+			};
+
+			if let ReadingDir::Pending(path) = last_path {
+				let mut reader = fs::read_dir(&path).await?;
+				let mut subpaths = vec![];
+				while let Some(ent) = reader.next_entry().await? {
+					subpaths.push(ent);
+				}
+				*last_path = ReadingDir::Read { subpaths, pos: 0 };
+			}
+
+			let (subpaths, pos) = match *last_path {
+				ReadingDir::Read {
+					ref subpaths,
+					ref mut pos,
+				} => (subpaths, pos),
+				ReadingDir::Pending(_) => unreachable!(),
+			};
+
+			let data_dir_ent = match subpaths.get(*pos) {
+				None => {
+					self.path.pop();
+					continue;
+				}
+				Some(ent) => {
+					*pos += 1;
+					ent
+				}
+			};
+
+			let name = data_dir_ent.file_name();
+			let name = if let Ok(n) = name.into_string() {
+				n
+			} else {
+				continue;
+			};
+			let ent_type = data_dir_ent.file_type().await?;
+
+			let name = name.strip_suffix(".zst").unwrap_or(&name);
+			if name.len() == 2 && hex::decode(&name).is_ok() && ent_type.is_dir() {
+				let path = data_dir_ent.path();
+				self.path.push(ReadingDir::Pending(path));
+			} else if name.len() == 64 {
+				if let Ok(h) = hex::decode(&name) {
+					let mut hash = [0u8; 32];
+					hash.copy_from_slice(&h);
+					return Ok(Some(hash.into()));
+				}
+			}
+		}
+	}
+}

src/block/resync.rs

@@ -0,0 +1,590 @@
+use std::collections::HashSet;
+use std::convert::TryInto;
+use std::sync::{Arc, Mutex};
+use std::time::Duration;
+
+use arc_swap::ArcSwap;
+use async_trait::async_trait;
+use serde::{Deserialize, Serialize};
+
+use futures::future::*;
+use tokio::select;
+use tokio::sync::{watch, Notify};
+
+use opentelemetry::{
+	trace::{FutureExt as OtelFutureExt, TraceContextExt, Tracer},
+	Context, KeyValue,
+};
+
+use garage_db as db;
+use garage_db::counted_tree_hack::CountedTree;
+
+use garage_util::background::*;
+use garage_util::data::*;
+use garage_util::error::*;
+use garage_util::metrics::RecordDuration;
+use garage_util::persister::Persister;
+use garage_util::time::*;
+use garage_util::tranquilizer::Tranquilizer;
+
+use garage_rpc::system::System;
+use garage_rpc::*;
+
+use garage_table::replication::TableReplication;
+
+use crate::manager::*;
+
+// Timeout for RPCs that ask other nodes whether they need a copy
+// of a given block before we delete it locally
+pub(crate) const NEED_BLOCK_QUERY_TIMEOUT: Duration = Duration::from_secs(5);
+
+// The delay between the time where a resync operation fails
+// and the time when it is retried, with exponential backoff
+// (multiplied by 2, 4, 8, 16, etc. for every consecutive failure).
+pub(crate) const RESYNC_RETRY_DELAY: Duration = Duration::from_secs(60);
+// The minimum retry delay is 60 seconds = 1 minute
+// The maximum retry delay is 60 seconds * 2^6 = 60 seconds << 6 = 64 minutes (~1 hour)
+pub(crate) const RESYNC_RETRY_DELAY_MAX_BACKOFF_POWER: u64 = 6;
+
+// No more than 4 resync workers can be running in the system
+pub(crate) const MAX_RESYNC_WORKERS: usize = 4;
+// Resync tranquility is initially set to 2, but can be changed in the CLI
+// and the updated version is persisted over Garage restarts
+const INITIAL_RESYNC_TRANQUILITY: u32 = 2;
+
+pub struct BlockResyncManager {
+	pub(crate) queue: CountedTree,
+	pub(crate) notify: Notify,
+	pub(crate) errors: CountedTree,
+
+	busy_set: BusySet,
+
+	persister: Persister<ResyncPersistedConfig>,
+	persisted: ArcSwap<ResyncPersistedConfig>,
+}
+
+#[derive(Serialize, Deserialize, Clone, Copy)]
+struct ResyncPersistedConfig {
+	n_workers: usize,
+	tranquility: u32,
+}
+
+enum ResyncIterResult {
+	BusyDidSomething,
+	BusyDidNothing,
+	IdleFor(Duration),
+}
+
+type BusySet = Arc<Mutex<HashSet<Vec<u8>>>>;
+
+struct BusyBlock {
+	time_bytes: Vec<u8>,
+	hash_bytes: Vec<u8>,
+	busy_set: BusySet,
+}
+
+impl BlockResyncManager {
+	pub(crate) fn new(db: &db::Db, system: &System) -> Self {
+		let queue = db
+			.open_tree("block_local_resync_queue")
+			.expect("Unable to open block_local_resync_queue tree");
+		let queue = CountedTree::new(queue).expect("Could not count block_local_resync_queue");
+
+		let errors = db
+			.open_tree("block_local_resync_errors")
+			.expect("Unable to open block_local_resync_errors tree");
+		let errors = CountedTree::new(errors).expect("Could not count block_local_resync_errors");
+
+		let persister = Persister::new(&system.metadata_dir, "resync_cfg");
+		let persisted = match persister.load() {
+			Ok(v) => v,
+			Err(_) => ResyncPersistedConfig {
+				n_workers: 1,
+				tranquility: INITIAL_RESYNC_TRANQUILITY,
+			},
+		};
+
+		Self {
+			queue,
+			notify: Notify::new(),
+			errors,
+			busy_set: Arc::new(Mutex::new(HashSet::new())),
+			persister,
+			persisted: ArcSwap::new(Arc::new(persisted)),
+		}
+	}
+
+	/// Get lenght of resync queue
+	pub fn queue_len(&self) -> Result<usize, Error> {
+		// This currently can't return an error because the CountedTree hack
+		// doesn't error on .len(), but this will change when we remove the hack
+		// (hopefully someday!)
+		Ok(self.queue.len())
+	}
+
+	/// Get number of blocks that have an error
+	pub fn errors_len(&self) -> Result<usize, Error> {
+		// (see queue_len comment)
+		Ok(self.errors.len())
+	}
+
+	// ---- Resync loop ----
+
+	// This part manages a queue of blocks that need to be
+	// "resynchronized", i.e. that need to have a check that
+	// they are at present if we need them, or that they are
+	// deleted once the garbage collection delay has passed.
+	//
+	// Here are some explanations on how the resync queue works.
+	// There are two Sled trees that are used to have information
+	// about the status of blocks that need to be resynchronized:
+	//
+	// - resync.queue: a tree that is ordered first by a timestamp
+	//   (in milliseconds since Unix epoch) that is the time at which
+	//   the resync must be done, and second by block hash.
+	//   The key in this tree is just:
+	//       concat(timestamp (8 bytes), hash (32 bytes))
+	//   The value is the same 32-byte hash.
+	//
+	// - resync.errors: a tree that indicates for each block
+	//   if the last resync resulted in an error, and if so,
+	//   the following two informations (see the ErrorCounter struct):
+	//   - how many consecutive resync errors for this block?
+	//   - when was the last try?
+	//   These two informations are used to implement an
+	//   exponential backoff retry strategy.
+	//   The key in this tree is the 32-byte hash of the block,
+	//   and the value is the encoded ErrorCounter value.
+	//
+	// We need to have these two trees, because the resync queue
+	// is not just a queue of items to process, but a set of items
+	// that are waiting a specific delay until we can process them
+	// (the delay being necessary both internally for the exponential
+	// backoff strategy, and exposed as a parameter when adding items
+	// to the queue, e.g. to wait until the GC delay has passed).
+	// This is why we need one tree ordered by time, and one
+	// ordered by identifier of item to be processed (block hash).
+	//
+	// When the worker wants to process an item it takes from
+	// resync.queue, it checks in resync.errors that if there is an
+	// exponential back-off delay to await, it has passed before we
+	// process the item. If not, the item in the queue is skipped
+	// (but added back for later processing after the time of the
+	// delay).
+	//
+	// An alternative that would have seemed natural is to
+	// only add items to resync.queue with a processing time that is
+	// after the delay, but there are several issues with this:
+	// - This requires to synchronize updates to resync.queue and
+	//   resync.errors (with the current model, there is only one thread,
+	//   the worker thread, that accesses resync.errors,
+	//   so no need to synchronize) by putting them both in a lock.
+	//   This would mean that block_incref might need to take a lock
+	//   before doing its thing, meaning it has much more chances of
+	//   not completing successfully if something bad happens to Garage.
+	//   Currently Garage is not able to recover from block_incref that
+	//   doesn't complete successfully, because it is necessary to ensure
+	//   the consistency between the state of the block manager and
+	//   information in the BlockRef table.
+	// - If a resync fails, we put that block in the resync.errors table,
+	//   and also add it back to resync.queue to be processed after
+	//   the exponential back-off delay,
+	//   but maybe the block is already scheduled to be resynced again
+	//   at another time that is before the exponential back-off delay,
+	//   and we have no way to check that easily. This means that
+	//   in all cases, we need to check the resync.errors table
+	//   in the resync loop at the time when a block is popped from
+	//   the resync.queue.
+	// Overall, the current design is therefore simpler and more robust
+	// because it tolerates inconsistencies between the resync.queue
+	// and resync.errors table (items being scheduled in resync.queue
+	// for times that are earlier than the exponential back-off delay
+	// is a natural condition that is handled properly).
+
+	pub(crate) fn put_to_resync(&self, hash: &Hash, delay: Duration) -> db::Result<()> {
+		let when = now_msec() + delay.as_millis() as u64;
+		self.put_to_resync_at(hash, when)
+	}
+
+	pub(crate) fn put_to_resync_at(&self, hash: &Hash, when: u64) -> db::Result<()> {
+		trace!("Put resync_queue: {} {:?}", when, hash);
+		let mut key = u64::to_be_bytes(when).to_vec();
+		key.extend(hash.as_ref());
+		self.queue.insert(key, hash.as_ref())?;
+		self.notify.notify_waiters();
+		Ok(())
+	}
+
+	async fn resync_iter(&self, manager: &BlockManager) -> Result<ResyncIterResult, db::Error> {
+		if let Some(block) = self.get_block_to_resync()? {
+			let time_msec = u64::from_be_bytes(block.time_bytes[0..8].try_into().unwrap());
+			let now = now_msec();
+
+			if now >= time_msec {
+				let hash = Hash::try_from(&block.hash_bytes[..]).unwrap();
+
+				if let Some(ec) = self.errors.get(hash.as_slice())? {
+					let ec = ErrorCounter::decode(&ec);
+					if now < ec.next_try() {
+						// if next retry after an error is not yet,
+						// don't do resync and return early, but still
+						// make sure the item is still in queue at expected time
+						self.put_to_resync_at(&hash, ec.next_try())?;
+						// ec.next_try() > now >= time_msec, so this remove
+						// is not removing the one we added just above
+						// (we want to do the remove after the insert to ensure
+						// that the item is not lost if we crash in-between)
+						self.queue.remove(&block.time_bytes)?;
+						return Ok(ResyncIterResult::BusyDidNothing);
+					}
+				}
+
+				let tracer = opentelemetry::global::tracer("garage");
+				let trace_id = gen_uuid();
+				let span = tracer
+					.span_builder("Resync block")
+					.with_trace_id(
+						opentelemetry::trace::TraceId::from_hex(&hex::encode(
+							&trace_id.as_slice()[..16],
+						))
+						.unwrap(),
+					)
+					.with_attributes(vec![KeyValue::new("block", format!("{:?}", hash))])
+					.start(&tracer);
+
+				let res = self
+					.resync_block(manager, &hash)
+					.with_context(Context::current_with_span(span))
+					.bound_record_duration(&manager.metrics.resync_duration)
+					.await;
+
+				manager.metrics.resync_counter.add(1);
+
+				if let Err(e) = &res {
+					manager.metrics.resync_error_counter.add(1);
+					warn!("Error when resyncing {:?}: {}", hash, e);
+
+					let err_counter = match self.errors.get(hash.as_slice())? {
+						Some(ec) => ErrorCounter::decode(&ec).add1(now + 1),
+						None => ErrorCounter::new(now + 1),
+					};
+
+					self.errors.insert(hash.as_slice(), err_counter.encode())?;
+
+					self.put_to_resync_at(&hash, err_counter.next_try())?;
+					// err_counter.next_try() >= now + 1 > now,
+					// the entry we remove from the queue is not
+					// the entry we inserted with put_to_resync_at
+					self.queue.remove(&block.time_bytes)?;
+				} else {
+					self.errors.remove(hash.as_slice())?;
+					self.queue.remove(&block.time_bytes)?;
+				}
+
+				Ok(ResyncIterResult::BusyDidSomething)
+			} else {
+				Ok(ResyncIterResult::IdleFor(Duration::from_millis(
+					time_msec - now,
+				)))
+			}
+		} else {
+			// Here we wait either for a notification that an item has been
+			// added to the queue, or for a constant delay of 10 secs to expire.
+			// The delay avoids a race condition where the notification happens
+			// between the time we checked the queue and the first poll
+			// to resync_notify.notified(): if that happens, we'll just loop
+			// back 10 seconds later, which is fine.
+			Ok(ResyncIterResult::IdleFor(Duration::from_secs(10)))
+		}
+	}
+
+	fn get_block_to_resync(&self) -> Result<Option<BusyBlock>, db::Error> {
+		let mut busy = self.busy_set.lock().unwrap();
+		for it in self.queue.iter()? {
+			let (time_bytes, hash_bytes) = it?;
+			if !busy.contains(&time_bytes) {
+				busy.insert(time_bytes.clone());
+				return Ok(Some(BusyBlock {
+					time_bytes,
+					hash_bytes,
+					busy_set: self.busy_set.clone(),
+				}));
+			}
+		}
+		Ok(None)
+	}
+
+	async fn resync_block(&self, manager: &BlockManager, hash: &Hash) -> Result<(), Error> {
+		let BlockStatus { exists, needed } = manager.check_block_status(hash).await?;
+
+		if exists != needed.is_needed() || exists != needed.is_nonzero() {
+			debug!(
+				"Resync block {:?}: exists {}, nonzero rc {}, deletable {}",
+				hash,
+				exists,
+				needed.is_nonzero(),
+				needed.is_deletable(),
+			);
+		}
+
+		if exists && needed.is_deletable() {
+			info!("Resync block {:?}: offloading and deleting", hash);
+
+			let mut who = manager.replication.write_nodes(hash);
+			if who.len() < manager.replication.write_quorum() {
+				return Err(Error::Message("Not trying to offload block because we don't have a quorum of nodes to write to".to_string()));
+			}
+			who.retain(|id| *id != manager.system.id);
+
+			let msg = Arc::new(BlockRpc::NeedBlockQuery(*hash));
+			let who_needs_fut = who.iter().map(|to| {
+				manager.system.rpc.call_arc(
+					&manager.endpoint,
+					*to,
+					msg.clone(),
+					RequestStrategy::with_priority(PRIO_BACKGROUND)
+						.with_timeout(NEED_BLOCK_QUERY_TIMEOUT),
+				)
+			});
+			let who_needs_resps = join_all(who_needs_fut).await;
+
+			let mut need_nodes = vec![];
+			for (node, needed) in who.iter().zip(who_needs_resps.into_iter()) {
+				match needed.err_context("NeedBlockQuery RPC")? {
+					BlockRpc::NeedBlockReply(needed) => {
+						if needed {
+							need_nodes.push(*node);
+						}
+					}
+					m => {
+						return Err(Error::unexpected_rpc_message(m));
+					}
+				}
+			}
+
+			if !need_nodes.is_empty() {
+				trace!(
+					"Block {:?} needed by {} nodes, sending",
+					hash,
+					need_nodes.len()
+				);
+
+				for node in need_nodes.iter() {
+					manager
+						.metrics
+						.resync_send_counter
+						.add(1, &[KeyValue::new("to", format!("{:?}", node))]);
+				}
+
+				let put_block_message = manager.read_block(hash).await?;
+				manager
+					.system
+					.rpc
+					.try_call_many(
+						&manager.endpoint,
+						&need_nodes[..],
+						put_block_message,
+						RequestStrategy::with_priority(PRIO_BACKGROUND)
+							.with_quorum(need_nodes.len())
+							.with_timeout(BLOCK_RW_TIMEOUT),
+					)
+					.await
+					.err_context("PutBlock RPC")?;
+			}
+			info!(
+				"Deleting unneeded block {:?}, offload finished ({} / {})",
+				hash,
+				need_nodes.len(),
+				who.len()
+			);
+
+			manager.delete_if_unneeded(hash).await?;
+
+			manager.rc.clear_deleted_block_rc(hash)?;
+		}
+
+		if needed.is_nonzero() && !exists {
+			info!(
+				"Resync block {:?}: fetching absent but needed block (refcount > 0)",
+				hash
+			);
+
+			let block_data = manager.rpc_get_raw_block(hash).await?;
+
+			manager.metrics.resync_recv_counter.add(1);
+
+			manager.write_block(hash, &block_data).await?;
+		}
+
+		Ok(())
+	}
+
+	async fn update_persisted(
+		&self,
+		update: impl Fn(&mut ResyncPersistedConfig),
+	) -> Result<(), Error> {
+		let mut cfg: ResyncPersistedConfig = *self.persisted.load().as_ref();
+		update(&mut cfg);
+		self.persister.save_async(&cfg).await?;
+		self.persisted.store(Arc::new(cfg));
+		self.notify.notify_waiters();
+		Ok(())
+	}
+
+	pub async fn set_n_workers(&self, n_workers: usize) -> Result<(), Error> {
+		if !(1..=MAX_RESYNC_WORKERS).contains(&n_workers) {
+			return Err(Error::Message(format!(
+				"Invalid number of resync workers, must be between 1 and {}",
+				MAX_RESYNC_WORKERS
+			)));
+		}
+		self.update_persisted(|cfg| cfg.n_workers = n_workers).await
+	}
+
+	pub async fn set_tranquility(&self, tranquility: u32) -> Result<(), Error> {
+		self.update_persisted(|cfg| cfg.tranquility = tranquility)
+			.await
+	}
+}
+
+impl Drop for BusyBlock {
+	fn drop(&mut self) {
+		let mut busy = self.busy_set.lock().unwrap();
+		busy.remove(&self.time_bytes);
+	}
+}
+
+pub(crate) struct ResyncWorker {
+	index: usize,
+	manager: Arc<BlockManager>,
+	tranquilizer: Tranquilizer,
+	next_delay: Duration,
+}
+
+impl ResyncWorker {
+	pub(crate) fn new(index: usize, manager: Arc<BlockManager>) -> Self {
+		Self {
+			index,
+			manager,
+			tranquilizer: Tranquilizer::new(30),
+			next_delay: Duration::from_secs(10),
+		}
+	}
+}
+
+#[async_trait]
+impl Worker for ResyncWorker {
+	fn name(&self) -> String {
+		format!("Block resync worker #{}", self.index + 1)
+	}
+
+	fn info(&self) -> Option<String> {
+		let persisted = self.manager.resync.persisted.load();
+
+		if self.index >= persisted.n_workers {
+			return Some("(unused)".into());
+		}
+
+		let mut ret = vec![];
+		ret.push(format!("tranquility = {}", persisted.tranquility));
+
+		let qlen = self.manager.resync.queue_len().unwrap_or(0);
+		if qlen > 0 {
+			ret.push(format!("{} blocks in queue", qlen));
+		}
+
+		let elen = self.manager.resync.errors_len().unwrap_or(0);
+		if elen > 0 {
+			ret.push(format!("{} blocks in error state", elen));
+		}
+
+		Some(ret.join(", "))
+	}
+
+	async fn work(&mut self, _must_exit: &mut watch::Receiver<bool>) -> Result<WorkerState, Error> {
+		if self.index >= self.manager.resync.persisted.load().n_workers {
+			return Ok(WorkerState::Idle);
+		}
+
+		self.tranquilizer.reset();
+		match self.manager.resync.resync_iter(&self.manager).await {
+			Ok(ResyncIterResult::BusyDidSomething) => Ok(self
+				.tranquilizer
+				.tranquilize_worker(self.manager.resync.persisted.load().tranquility)),
+			Ok(ResyncIterResult::BusyDidNothing) => Ok(WorkerState::Busy),
+			Ok(ResyncIterResult::IdleFor(delay)) => {
+				self.next_delay = delay;
+				Ok(WorkerState::Idle)
+			}
+			Err(e) => {
+				// The errors that we have here are only Sled errors
+				// We don't really know how to handle them so just ¯\_(ツ)_/¯
+				// (there is kind of an assumption that Sled won't error on us,
+				// if it does there is not much we can do -- TODO should we just panic?)
+				// Here we just give the error to the worker manager,
+				// it will print it to the logs and increment a counter
+				Err(e.into())
+			}
+		}
+	}
+
+	async fn wait_for_work(&mut self, _must_exit: &watch::Receiver<bool>) -> WorkerState {
+		while self.index >= self.manager.resync.persisted.load().n_workers {
+			self.manager.resync.notify.notified().await
+		}
+
+		select! {
+			_ = tokio::time::sleep(self.next_delay) => (),
+			_ = self.manager.resync.notify.notified() => (),
+		};
+
+		WorkerState::Busy
+	}
+}
+
+/// Counts the number of errors when resyncing a block,
+/// and the time of the last try.
+/// Used to implement exponential backoff.
+#[derive(Clone, Copy, Debug)]
+struct ErrorCounter {
+	errors: u64,
+	last_try: u64,
+}
+
+impl ErrorCounter {
+	fn new(now: u64) -> Self {
+		Self {
+			errors: 1,
+			last_try: now,
+		}
+	}
+
+	fn decode(data: &[u8]) -> Self {
+		Self {
+			errors: u64::from_be_bytes(data[0..8].try_into().unwrap()),
+			last_try: u64::from_be_bytes(data[8..16].try_into().unwrap()),
+		}
+	}
+	fn encode(&self) -> Vec<u8> {
+		[
+			u64::to_be_bytes(self.errors),
+			u64::to_be_bytes(self.last_try),
+		]
+		.concat()
+	}
+
+	fn add1(self, now: u64) -> Self {
+		Self {
+			errors: self.errors + 1,
+			last_try: now,
+		}
+	}
+
+	fn delay_msec(&self) -> u64 {
+		(RESYNC_RETRY_DELAY.as_millis() as u64)
+			<< std::cmp::min(self.errors - 1, RESYNC_RETRY_DELAY_MAX_BACKOFF_POWER)
+	}
+	fn next_try(&self) -> u64 {
+		self.last_try + self.delay_msec()
+	}
+}

src/db/Cargo.toml

@@ -19,7 +19,7 @@ required-features = ["cli"]
 [dependencies]
 err-derive = "0.3"
 hexdump = "0.1"
-log = "0.4"
+tracing = "0.1.30"
 
 heed = "0.11"
 rusqlite = { version = "0.27", features = ["bundled"] }

src/db/lib.rs

@@ -1,3 +1,6 @@
+#[macro_use]
+extern crate tracing;
+
 pub mod lmdb_adapter;
 pub mod sled_adapter;
 pub mod sqlite_adapter;

src/db/lmdb_adapter.rs

@@ -345,7 +345,6 @@ pub fn recommended_map_size() -> usize {
 
 #[cfg(target_pointer_width = "32")]
 pub fn recommended_map_size() -> usize {
-	use log::warn;
 	warn!("LMDB is not recommended on 32-bit systems, database size will be limited");
 	1usize << 30
 }

src/db/sqlite_adapter.rs

@@ -6,8 +6,6 @@ use std::pin::Pin;
 use std::ptr::NonNull;
 use std::sync::{Arc, Mutex, MutexGuard};
 
-use log::trace;
-
 use rusqlite::{params, Connection, Rows, Statement, Transaction};
 
 use crate::{

src/garage/Cargo.toml

@@ -23,6 +23,7 @@ path = "tests/lib.rs"
 [dependencies]
 garage_db = { version = "0.8.0", path = "../db" }
 garage_api = { version = "0.7.0", path = "../api" }
+garage_block = { version = "0.7.0", path = "../block" }
 garage_model = { version = "0.7.0", path = "../model" }
 garage_rpc = { version = "0.7.0", path = "../rpc" }
 garage_table = { version = "0.7.0", path = "../table" }
@@ -31,6 +32,7 @@ garage_web = { version = "0.7.0", path = "../web" }
 
 bytes = "1.0"
 bytesize = "1.1"
+timeago = "0.3"
 hex = "0.4"
 tracing = { version = "0.1.30", features = ["log-always"] }
 pretty_env_logger = "0.4"

src/garage/admin.rs

@@ -15,6 +15,8 @@ use garage_table::*;
 
 use garage_rpc::*;
 
+use garage_block::repair::ScrubWorkerCommand;
+
 use garage_model::bucket_alias_table::*;
 use garage_model::bucket_table::*;
 use garage_model::garage::Garage;
@@ -24,7 +26,7 @@ use garage_model::migrate::Migrate;
 use garage_model::permission::*;
 
 use crate::cli::*;
-use crate::repair::online::OnlineRepair;
+use crate::repair::online::launch_online_repair;
 
 pub const ADMIN_RPC_PATH: &str = "garage/admin_rpc.rs/Rpc";
 
@@ -36,6 +38,7 @@ pub enum AdminRpc {
 	LaunchRepair(RepairOpt),
 	Migrate(MigrateOpt),
 	Stats(StatsOpt),
+	Worker(WorkerOpt),
 
 	// Replies
 	Ok(String),
@@ -47,6 +50,10 @@ pub enum AdminRpc {
 	},
 	KeyList(Vec<(String, String)>),
 	KeyInfo(Key, HashMap<Uuid, Bucket>),
+	WorkerList(
+		HashMap<usize, garage_util::background::WorkerInfo>,
+		WorkerListOpt,
+	),
 }
 
 impl Rpc for AdminRpc {
@@ -693,15 +700,7 @@ impl AdminRpcHandler {
 				)))
 			}
 		} else {
-			let repair = OnlineRepair {
-				garage: self.garage.clone(),
-			};
-			self.garage
-				.system
-				.background
-				.spawn_worker("Repair worker".into(), move |must_exit| async move {
-					repair.repair_worker(opt, must_exit).await
-				});
+			launch_online_repair(self.garage.clone(), opt).await;
 			Ok(AdminRpc::Ok(format!(
 				"Repair launched on {:?}",
 				self.garage.system.id
@@ -782,13 +781,13 @@ impl AdminRpcHandler {
 		writeln!(
 			&mut ret,
 			"  resync queue length: {}",
-			self.garage.block_manager.resync_queue_len()?
+			self.garage.block_manager.resync.queue_len()?
 		)
 		.unwrap();
 		writeln!(
 			&mut ret,
 			"  blocks with resync errors: {}",
-			self.garage.block_manager.resync_errors_len()?
+			self.garage.block_manager.resync.errors_len()?
 		)
 		.unwrap();
 
@@ -830,6 +829,43 @@ impl AdminRpcHandler {
 
 		Ok(())
 	}
+
+	// ----
+
+	async fn handle_worker_cmd(&self, opt: WorkerOpt) -> Result<AdminRpc, Error> {
+		match opt.cmd {
+			WorkerCmd::List { opt } => {
+				let workers = self.garage.background.get_worker_info();
+				Ok(AdminRpc::WorkerList(workers, opt))
+			}
+			WorkerCmd::Set { opt } => match opt {
+				WorkerSetCmd::ScrubTranquility { tranquility } => {
+					let scrub_command = ScrubWorkerCommand::SetTranquility(tranquility);
+					self.garage
+						.block_manager
+						.send_scrub_command(scrub_command)
+						.await;
+					Ok(AdminRpc::Ok("Scrub tranquility updated".into()))
+				}
+				WorkerSetCmd::ResyncNWorkers { n_workers } => {
+					self.garage
+						.block_manager
+						.resync
+						.set_n_workers(n_workers)
+						.await?;
+					Ok(AdminRpc::Ok("Number of resync workers updated".into()))
+				}
+				WorkerSetCmd::ResyncTranquility { tranquility } => {
+					self.garage
+						.block_manager
+						.resync
+						.set_tranquility(tranquility)
+						.await?;
+					Ok(AdminRpc::Ok("Resync tranquility updated".into()))
+				}
+			},
+		}
+	}
 }
 
 #[async_trait]
@@ -845,6 +881,7 @@ impl EndpointHandler<AdminRpc> for AdminRpcHandler {
 			AdminRpc::Migrate(opt) => self.handle_migrate(opt.clone()).await,
 			AdminRpc::LaunchRepair(opt) => self.handle_launch_repair(opt.clone()).await,
 			AdminRpc::Stats(opt) => self.handle_stats(opt.clone()).await,
+			AdminRpc::Worker(opt) => self.handle_worker_cmd(opt.clone()).await,
 			m => Err(GarageError::unexpected_rpc_message(m).into()),
 		}
 	}

src/garage/cli/cmd.rs

@@ -1,4 +1,5 @@
 use std::collections::HashSet;
+use std::time::Duration;
 
 use garage_util::error::*;
 use garage_util::formater::format_table;
@@ -39,6 +40,7 @@ pub async fn cli_command_dispatch(
 			cmd_admin(admin_rpc_endpoint, rpc_host, AdminRpc::LaunchRepair(ro)).await
 		}
 		Command::Stats(so) => cmd_admin(admin_rpc_endpoint, rpc_host, AdminRpc::Stats(so)).await,
+		Command::Worker(wo) => cmd_admin(admin_rpc_endpoint, rpc_host, AdminRpc::Worker(wo)).await,
 		_ => unreachable!(),
 	}
 }
@@ -100,6 +102,7 @@ pub async fn cmd_status(rpc_cli: &Endpoint<SystemRpc, ()>, rpc_host: NodeID) ->
 			vec!["ID\tHostname\tAddress\tTags\tZone\tCapacity\tLast seen".to_string()];
 		for adv in status.iter().filter(|adv| !adv.is_up) {
 			if let Some(NodeRoleV(Some(cfg))) = layout.roles.get(&adv.id) {
+				let tf = timeago::Formatter::new();
 				failed_nodes.push(format!(
 					"{id:?}\t{host}\t{addr}\t[{tags}]\t{zone}\t{capacity}\t{last_seen}",
 					id = adv.id,
@@ -110,7 +113,7 @@ pub async fn cmd_status(rpc_cli: &Endpoint<SystemRpc, ()>, rpc_host: NodeID) ->
 					capacity = cfg.capacity_string(),
 					last_seen = adv
 						.last_seen_secs_ago
-						.map(|s| format!("{}s ago", s))
+						.map(|s| tf.convert(Duration::from_secs(s)))
 						.unwrap_or_else(|| "never seen".into()),
 				));
 			}
@@ -182,6 +185,9 @@ pub async fn cmd_admin(
 		AdminRpc::KeyInfo(key, rb) => {
 			print_key_info(&key, &rb);
 		}
+		AdminRpc::WorkerList(wi, wlo) => {
+			print_worker_info(wi, wlo);
+		}
 		r => {
 			error!("Unexpected response: {:?}", r);
 		}

src/garage/cli/structs.rs

@@ -1,60 +1,65 @@
 use serde::{Deserialize, Serialize};
 
+use garage_util::version;
 use structopt::StructOpt;
 
 #[derive(StructOpt, Debug)]
 pub enum Command {
 	/// Run Garage server
-	#[structopt(name = "server")]
+	#[structopt(name = "server", version = version::garage())]
 	Server,
 
 	/// Get network status
-	#[structopt(name = "status")]
+	#[structopt(name = "status", version = version::garage())]
 	Status,
 
 	/// Operations on individual Garage nodes
-	#[structopt(name = "node")]
+	#[structopt(name = "node", version = version::garage())]
 	Node(NodeOperation),
 
 	/// Operations on the assignation of node roles in the cluster layout
-	#[structopt(name = "layout")]
+	#[structopt(name = "layout", version = version::garage())]
 	Layout(LayoutOperation),
 
 	/// Operations on buckets
-	#[structopt(name = "bucket")]
+	#[structopt(name = "bucket", version = version::garage())]
 	Bucket(BucketOperation),
 
 	/// Operations on S3 access keys
-	#[structopt(name = "key")]
+	#[structopt(name = "key", version = version::garage())]
 	Key(KeyOperation),
 
 	/// Run migrations from previous Garage version
 	/// (DO NOT USE WITHOUT READING FULL DOCUMENTATION)
-	#[structopt(name = "migrate")]
+	#[structopt(name = "migrate", version = version::garage())]
 	Migrate(MigrateOpt),
 
 	/// Start repair of node data on remote node
-	#[structopt(name = "repair")]
+	#[structopt(name = "repair", version = version::garage())]
 	Repair(RepairOpt),
 
 	/// Offline reparation of node data (these repairs must be run offline
 	/// directly on the server node)
-	#[structopt(name = "offline-repair")]
+	#[structopt(name = "offline-repair", version = version::garage())]
 	OfflineRepair(OfflineRepairOpt),
 
 	/// Gather node statistics
-	#[structopt(name = "stats")]
+	#[structopt(name = "stats", version = version::garage())]
 	Stats(StatsOpt),
+
+	/// Manage background workers
+	#[structopt(name = "worker", version = version::garage())]
+	Worker(WorkerOpt),
 }
 
 #[derive(StructOpt, Debug)]
 pub enum NodeOperation {
 	/// Print identifier (public key) of this Garage node
-	#[structopt(name = "id")]
+	#[structopt(name = "id", version = version::garage())]
 	NodeId(NodeIdOpt),
 
 	/// Connect to Garage node that is currently isolated from the system
-	#[structopt(name = "connect")]
+	#[structopt(name = "connect", version = version::garage())]
 	Connect(ConnectNodeOpt),
 }
 
@@ -75,23 +80,23 @@ pub struct ConnectNodeOpt {
 #[derive(StructOpt, Debug)]
 pub enum LayoutOperation {
 	/// Assign role to Garage node
-	#[structopt(name = "assign")]
+	#[structopt(name = "assign", version = version::garage())]
 	Assign(AssignRoleOpt),
 
 	/// Remove role from Garage cluster node
-	#[structopt(name = "remove")]
+	#[structopt(name = "remove", version = version::garage())]
 	Remove(RemoveRoleOpt),
 
 	/// Show roles currently assigned to nodes and changes staged for commit
-	#[structopt(name = "show")]
+	#[structopt(name = "show", version = version::garage())]
 	Show,
 
 	/// Apply staged changes to cluster layout
-	#[structopt(name = "apply")]
+	#[structopt(name = "apply", version = version::garage())]
 	Apply(ApplyLayoutOpt),
 
 	/// Revert staged changes to cluster layout
-	#[structopt(name = "revert")]
+	#[structopt(name = "revert", version = version::garage())]
 	Revert(RevertLayoutOpt),
 }
 
@@ -146,43 +151,43 @@ pub struct RevertLayoutOpt {
 #[derive(Serialize, Deserialize, StructOpt, Debug)]
 pub enum BucketOperation {
 	/// List buckets
-	#[structopt(name = "list")]
+	#[structopt(name = "list", version = version::garage())]
 	List,
 
 	/// Get bucket info
-	#[structopt(name = "info")]
+	#[structopt(name = "info", version = version::garage())]
 	Info(BucketOpt),
 
 	/// Create bucket
-	#[structopt(name = "create")]
+	#[structopt(name = "create", version = version::garage())]
 	Create(BucketOpt),
 
 	/// Delete bucket
-	#[structopt(name = "delete")]
+	#[structopt(name = "delete", version = version::garage())]
 	Delete(DeleteBucketOpt),
 
 	/// Alias bucket under new name
-	#[structopt(name = "alias")]
+	#[structopt(name = "alias", version = version::garage())]
 	Alias(AliasBucketOpt),
 
 	/// Remove bucket alias
-	#[structopt(name = "unalias")]
+	#[structopt(name = "unalias", version = version::garage())]
 	Unalias(UnaliasBucketOpt),
 
 	/// Allow key to read or write to bucket
-	#[structopt(name = "allow")]
+	#[structopt(name = "allow", version = version::garage())]
 	Allow(PermBucketOpt),
 
 	/// Deny key from reading or writing to bucket
-	#[structopt(name = "deny")]
+	#[structopt(name = "deny", version = version::garage())]
 	Deny(PermBucketOpt),
 
 	/// Expose as website or not
-	#[structopt(name = "website")]
+	#[structopt(name = "website", version = version::garage())]
 	Website(WebsiteOpt),
 
 	/// Set the quotas for this bucket
-	#[structopt(name = "set-quotas")]
+	#[structopt(name = "set-quotas", version = version::garage())]
 	SetQuotas(SetQuotasOpt),
 }
 
@@ -288,35 +293,35 @@ pub struct SetQuotasOpt {
 #[derive(Serialize, Deserialize, StructOpt, Debug)]
 pub enum KeyOperation {
 	/// List keys
-	#[structopt(name = "list")]
+	#[structopt(name = "list", version = version::garage())]
 	List,
 
 	/// Get key info
-	#[structopt(name = "info")]
+	#[structopt(name = "info", version = version::garage())]
 	Info(KeyOpt),
 
 	/// Create new key
-	#[structopt(name = "new")]
+	#[structopt(name = "new", version = version::garage())]
 	New(KeyNewOpt),
 
 	/// Rename key
-	#[structopt(name = "rename")]
+	#[structopt(name = "rename", version = version::garage())]
 	Rename(KeyRenameOpt),
 
 	/// Delete key
-	#[structopt(name = "delete")]
+	#[structopt(name = "delete", version = version::garage())]
 	Delete(KeyDeleteOpt),
 
 	/// Set permission flags for key
-	#[structopt(name = "allow")]
+	#[structopt(name = "allow", version = version::garage())]
 	Allow(KeyPermOpt),
 
 	/// Unset permission flags for key
-	#[structopt(name = "deny")]
+	#[structopt(name = "deny", version = version::garage())]
 	Deny(KeyPermOpt),
 
 	/// Import key
-	#[structopt(name = "import")]
+	#[structopt(name = "import", version = version::garage())]
 	Import(KeyImportOpt),
 }
 
@@ -388,7 +393,7 @@ pub struct MigrateOpt {
 #[derive(Serialize, Deserialize, StructOpt, Debug, Eq, PartialEq, Clone)]
 pub enum MigrateWhat {
 	/// Migrate buckets and permissions from v0.5.0
-	#[structopt(name = "buckets050")]
+	#[structopt(name = "buckets050", version = version::garage())]
 	Buckets050,
 }
 
@@ -409,22 +414,43 @@ pub struct RepairOpt {
 #[derive(Serialize, Deserialize, StructOpt, Debug, Eq, PartialEq, Clone)]
 pub enum RepairWhat {
 	/// Only do a full sync of metadata tables
-	#[structopt(name = "tables")]
+	#[structopt(name = "tables", version = version::garage())]
 	Tables,
 	/// Only repair (resync/rebalance) the set of stored blocks
-	#[structopt(name = "blocks")]
+	#[structopt(name = "blocks", version = version::garage())]
 	Blocks,
 	/// Only redo the propagation of object deletions to the version table (slow)
-	#[structopt(name = "versions")]
+	#[structopt(name = "versions", version = version::garage())]
 	Versions,
 	/// Only redo the propagation of version deletions to the block ref table (extremely slow)
-	#[structopt(name = "block_refs")]
+	#[structopt(name = "block_refs", version = version::garage())]
 	BlockRefs,
 	/// Verify integrity of all blocks on disc (extremely slow, i/o intensive)
-	#[structopt(name = "scrub")]
+	#[structopt(name = "scrub", version = version::garage())]
 	Scrub {
-		/// Tranquility factor (see tranquilizer documentation)
-		#[structopt(name = "tranquility", default_value = "2")]
+		#[structopt(subcommand)]
+		cmd: ScrubCmd,
+	},
+}
+
+#[derive(Serialize, Deserialize, StructOpt, Debug, Eq, PartialEq, Clone)]
+pub enum ScrubCmd {
+	/// Start scrub
+	#[structopt(name = "start", version = version::garage())]
+	Start,
+	/// Pause scrub (it will resume automatically after 24 hours)
+	#[structopt(name = "pause", version = version::garage())]
+	Pause,
+	/// Resume paused scrub
+	#[structopt(name = "resume", version = version::garage())]
+	Resume,
+	/// Cancel scrub in progress
+	#[structopt(name = "cancel", version = version::garage())]
+	Cancel,
+	/// Set tranquility level for in-progress and future scrubs
+	#[structopt(name = "set-tranquility", version = version::garage())]
+	SetTranquility {
+		#[structopt()]
 		tranquility: u32,
 	},
 }
@@ -443,10 +469,10 @@ pub struct OfflineRepairOpt {
 pub enum OfflineRepairWhat {
 	/// Repair K2V item counters
 	#[cfg(feature = "k2v")]
-	#[structopt(name = "k2v_item_counters")]
+	#[structopt(name = "k2v_item_counters", version = version::garage())]
 	K2VItemCounters,
 	/// Repair object counters
-	#[structopt(name = "object_counters")]
+	#[structopt(name = "object_counters", version = version::garage())]
 	ObjectCounters,
 }
 
@@ -460,3 +486,48 @@ pub struct StatsOpt {
 	#[structopt(short = "d", long = "detailed")]
 	pub detailed: bool,
 }
+
+#[derive(Serialize, Deserialize, StructOpt, Debug, Clone)]
+pub struct WorkerOpt {
+	#[structopt(subcommand)]
+	pub cmd: WorkerCmd,
+}
+
+#[derive(Serialize, Deserialize, StructOpt, Debug, Eq, PartialEq, Clone)]
+pub enum WorkerCmd {
+	/// List all workers on Garage node
+	#[structopt(name = "list", version = version::garage())]
+	List {
+		#[structopt(flatten)]
+		opt: WorkerListOpt,
+	},
+	/// Set worker parameter
+	#[structopt(name = "set", version = version::garage())]
+	Set {
+		#[structopt(subcommand)]
+		opt: WorkerSetCmd,
+	},
+}
+
+#[derive(Serialize, Deserialize, StructOpt, Debug, Eq, PartialEq, Clone, Copy)]
+pub struct WorkerListOpt {
+	/// Show only busy workers
+	#[structopt(short = "b", long = "busy")]
+	pub busy: bool,
+	/// Show only workers with errors
+	#[structopt(short = "e", long = "errors")]
+	pub errors: bool,
+}
+
+#[derive(Serialize, Deserialize, StructOpt, Debug, Eq, PartialEq, Clone)]
+pub enum WorkerSetCmd {
+	/// Set tranquility of scrub operations
+	#[structopt(name = "scrub-tranquility", version = version::garage())]
+	ScrubTranquility { tranquility: u32 },
+	/// Set number of concurrent block resync workers
+	#[structopt(name = "resync-n-workers", version = version::garage())]
+	ResyncNWorkers { n_workers: usize },
+	/// Set tranquility of block resync operations
+	#[structopt(name = "resync-tranquility", version = version::garage())]
+	ResyncTranquility { tranquility: u32 },
+}

src/garage/cli/util.rs

@@ -1,14 +1,19 @@
 use std::collections::HashMap;
+use std::time::Duration;
 
+use garage_util::background::*;
 use garage_util::crdt::*;
 use garage_util::data::Uuid;
 use garage_util::error::*;
 use garage_util::formater::format_table;
+use garage_util::time::*;
 
 use garage_model::bucket_table::*;
 use garage_model::key_table::*;
 use garage_model::s3::object_table::{BYTES, OBJECTS, UNFINISHED_UPLOADS};
 
+use crate::cli::structs::WorkerListOpt;
+
 pub fn print_bucket_list(bl: Vec<Bucket>) {
 	println!("List of buckets:");
 
@@ -235,3 +240,56 @@ pub fn find_matching_node(
 		Ok(candidates[0])
 	}
 }
+
+pub fn print_worker_info(wi: HashMap<usize, WorkerInfo>, wlo: WorkerListOpt) {
+	let mut wi = wi.into_iter().collect::<Vec<_>>();
+	wi.sort_by_key(|(tid, info)| {
+		(
+			match info.state {
+				WorkerState::Busy | WorkerState::Throttled(_) => 0,
+				WorkerState::Idle => 1,
+				WorkerState::Done => 2,
+			},
+			*tid,
+		)
+	});
+
+	let mut table = vec![];
+	for (tid, info) in wi.iter() {
+		if wlo.busy && !matches!(info.state, WorkerState::Busy | WorkerState::Throttled(_)) {
+			continue;
+		}
+		if wlo.errors && info.errors == 0 {
+			continue;
+		}
+
+		table.push(format!("{}\t{}\t{}", tid, info.state, info.name));
+		if let Some(i) = &info.info {
+			table.push(format!("\t\t  {}", i));
+		}
+		let tf = timeago::Formatter::new();
+		let (err_ago, err_msg) = info
+			.last_error
+			.as_ref()
+			.map(|(m, t)| {
+				(
+					tf.convert(Duration::from_millis(now_msec() - t)),
+					m.as_str(),
+				)
+			})
+			.unwrap_or(("(?) ago".into(), "(?)"));
+		if info.consecutive_errors > 0 {
+			table.push(format!(
+				"\t\t  {} consecutive errors ({} total), last {}",
+				info.consecutive_errors, info.errors, err_ago,
+			));
+			table.push(format!("\t\t  {}", err_msg));
+		} else if info.errors > 0 {
+			table.push(format!("\t\t  ({} errors, last {})", info.errors, err_ago,));
+			if wlo.errors {
+				table.push(format!("\t\t  {}", err_msg));
+			}
+		}
+	}
+	format_table(table);
+}

src/garage/main.rs

@@ -22,6 +22,7 @@ use garage_util::error::*;
 
 use garage_rpc::system::*;
 use garage_rpc::*;
+use garage_util::version;
 
 use garage_model::helper::error::Error as HelperError;
 
@@ -29,7 +30,7 @@ use admin::*;
 use cli::*;
 
 #[derive(StructOpt, Debug)]
-#[structopt(name = "garage")]
+#[structopt(name = "garage", version = version::garage(), about = "S3-compatible object store for self-hosted geo-distributed deployments")]
 struct Opt {
 	/// Host to connect to for admin operations, in the format:
 	/// <public-key>@<ip>:<port>

src/garage/repair/online.rs

@@ -1,89 +1,110 @@
 use std::sync::Arc;
+use std::time::Duration;
 
+use async_trait::async_trait;
 use tokio::sync::watch;
 
+use garage_block::repair::ScrubWorkerCommand;
 use garage_model::garage::Garage;
 use garage_model::s3::block_ref_table::*;
 use garage_model::s3::object_table::*;
 use garage_model::s3::version_table::*;
 use garage_table::*;
+use garage_util::background::*;
 use garage_util::error::Error;
 
 use crate::*;
 
-pub struct OnlineRepair {
-	pub garage: Arc<Garage>,
+pub async fn launch_online_repair(garage: Arc<Garage>, opt: RepairOpt) {
+	match opt.what {
+		RepairWhat::Tables => {
+			info!("Launching a full sync of tables");
+			garage.bucket_table.syncer.add_full_sync();
+			garage.object_table.syncer.add_full_sync();
+			garage.version_table.syncer.add_full_sync();
+			garage.block_ref_table.syncer.add_full_sync();
+			garage.key_table.syncer.add_full_sync();
+		}
+		RepairWhat::Versions => {
+			info!("Repairing the versions table");
+			garage
+				.background
+				.spawn_worker(RepairVersionsWorker::new(garage.clone()));
+		}
+		RepairWhat::BlockRefs => {
+			info!("Repairing the block refs table");
+			garage
+				.background
+				.spawn_worker(RepairBlockrefsWorker::new(garage.clone()));
+		}
+		RepairWhat::Blocks => {
+			info!("Repairing the stored blocks");
+			garage
+				.background
+				.spawn_worker(garage_block::repair::RepairWorker::new(
+					garage.block_manager.clone(),
+				));
+		}
+		RepairWhat::Scrub { cmd } => {
+			let cmd = match cmd {
+				ScrubCmd::Start => ScrubWorkerCommand::Start,
+				ScrubCmd::Pause => ScrubWorkerCommand::Pause(Duration::from_secs(3600 * 24)),
+				ScrubCmd::Resume => ScrubWorkerCommand::Resume,
+				ScrubCmd::Cancel => ScrubWorkerCommand::Cancel,
+				ScrubCmd::SetTranquility { tranquility } => {
+					ScrubWorkerCommand::SetTranquility(tranquility)
+				}
+			};
+			info!("Sending command to scrub worker: {:?}", cmd);
+			garage.block_manager.send_scrub_command(cmd).await;
+		}
+	}
 }
 
-impl OnlineRepair {
-	pub async fn repair_worker(&self, opt: RepairOpt, must_exit: watch::Receiver<bool>) {
-		if let Err(e) = self.repair_worker_aux(opt, must_exit).await {
-			warn!("Repair worker failed with error: {}", e);
+// ----
+
+struct RepairVersionsWorker {
+	garage: Arc<Garage>,
+	pos: Vec<u8>,
+	counter: usize,
+}
+
+impl RepairVersionsWorker {
+	fn new(garage: Arc<Garage>) -> Self {
+		Self {
+			garage,
+			pos: vec![],
+			counter: 0,
 		}
 	}
+}
 
-	async fn repair_worker_aux(
-		&self,
-		opt: RepairOpt,
-		must_exit: watch::Receiver<bool>,
-	) -> Result<(), Error> {
-		match opt.what {
-			RepairWhat::Tables => {
-				info!("Launching a full sync of tables");
-				self.garage.bucket_table.syncer.add_full_sync();
-				self.garage.object_table.syncer.add_full_sync();
-				self.garage.version_table.syncer.add_full_sync();
-				self.garage.block_ref_table.syncer.add_full_sync();
-				self.garage.key_table.syncer.add_full_sync();
-			}
-			RepairWhat::Versions => {
-				info!("Repairing the versions table");
-				self.repair_versions(&must_exit).await?;
-			}
-			RepairWhat::BlockRefs => {
-				info!("Repairing the block refs table");
-				self.repair_block_ref(&must_exit).await?;
-			}
-			RepairWhat::Blocks => {
-				info!("Repairing the stored blocks");
-				self.garage
-					.block_manager
-					.repair_data_store(&must_exit)
-					.await?;
-			}
-			RepairWhat::Scrub { tranquility } => {
-				info!("Verifying integrity of stored blocks");
-				self.garage
-					.block_manager
-					.scrub_data_store(&must_exit, tranquility)
-					.await?;
-			}
-		}
-		Ok(())
+#[async_trait]
+impl Worker for RepairVersionsWorker {
+	fn name(&self) -> String {
+		"Version repair worker".into()
 	}
 
-	async fn repair_versions(&self, must_exit: &watch::Receiver<bool>) -> Result<(), Error> {
-		let mut pos = vec![];
-		let mut i = 0;
+	fn info(&self) -> Option<String> {
+		Some(format!("{} items done", self.counter))
+	}
 
-		while !*must_exit.borrow() {
-			let item_bytes = match self.garage.version_table.data.store.get_gt(pos)? {
-				Some((k, v)) => {
-					pos = k;
-					v
-				}
-				None => break,
-			};
-
-			i += 1;
-			if i % 1000 == 0 {
-				info!("repair_versions: {}", i);
+	async fn work(&mut self, _must_exit: &mut watch::Receiver<bool>) -> Result<WorkerState, Error> {
+		let item_bytes = match self.garage.version_table.data.store.get_gt(&self.pos)? {
+			Some((k, v)) => {
+				self.pos = k;
+				v
 			}
-
-			let version = rmp_serde::decode::from_read_ref::<_, Version>(&item_bytes)?;
-			if version.deleted.get() {
-				continue;
+			None => {
+				info!("repair_versions: finished, done {}", self.counter);
+				return Ok(WorkerState::Done);
 			}
+		};
+
+		self.counter += 1;
+
+		let version = rmp_serde::decode::from_read_ref::<_, Version>(&item_bytes)?;
+		if !version.deleted.get() {
 			let object = self
 				.garage
 				.object_table
@@ -109,32 +130,59 @@ impl OnlineRepair {
 					.await?;
 			}
 		}
-		info!("repair_versions: finished, done {}", i);
-		Ok(())
+
+		Ok(WorkerState::Busy)
 	}
 
-	async fn repair_block_ref(&self, must_exit: &watch::Receiver<bool>) -> Result<(), Error> {
-		let mut pos = vec![];
-		let mut i = 0;
+	async fn wait_for_work(&mut self, _must_exit: &watch::Receiver<bool>) -> WorkerState {
+		unreachable!()
+	}
+}
 
-		while !*must_exit.borrow() {
-			let item_bytes = match self.garage.block_ref_table.data.store.get_gt(pos)? {
-				Some((k, v)) => {
-					pos = k;
-					v
-				}
-				None => break,
-			};
+// ----
 
-			i += 1;
-			if i % 1000 == 0 {
-				info!("repair_block_ref: {}", i);
+struct RepairBlockrefsWorker {
+	garage: Arc<Garage>,
+	pos: Vec<u8>,
+	counter: usize,
+}
+
+impl RepairBlockrefsWorker {
+	fn new(garage: Arc<Garage>) -> Self {
+		Self {
+			garage,
+			pos: vec![],
+			counter: 0,
+		}
+	}
+}
+
+#[async_trait]
+impl Worker for RepairBlockrefsWorker {
+	fn name(&self) -> String {
+		"Block refs repair worker".into()
+	}
+
+	fn info(&self) -> Option<String> {
+		Some(format!("{} items done", self.counter))
+	}
+
+	async fn work(&mut self, _must_exit: &mut watch::Receiver<bool>) -> Result<WorkerState, Error> {
+		let item_bytes = match self.garage.block_ref_table.data.store.get_gt(&self.pos)? {
+			Some((k, v)) => {
+				self.pos = k;
+				v
 			}
-
-			let block_ref = rmp_serde::decode::from_read_ref::<_, BlockRef>(&item_bytes)?;
-			if block_ref.deleted.get() {
-				continue;
+			None => {
+				info!("repair_block_ref: finished, done {}", self.counter);
+				return Ok(WorkerState::Done);
 			}
+		};
+
+		self.counter += 1;
+
+		let block_ref = rmp_serde::decode::from_read_ref::<_, BlockRef>(&item_bytes)?;
+		if !block_ref.deleted.get() {
 			let version = self
 				.garage
 				.version_table
@@ -157,7 +205,11 @@ impl OnlineRepair {
 					.await?;
 			}
 		}
-		info!("repair_block_ref: finished, done {}", i);
-		Ok(())
+
+		Ok(WorkerState::Busy)
+	}
+
+	async fn wait_for_work(&mut self, _must_exit: &watch::Receiver<bool>) -> WorkerState {
+		unreachable!()
 	}
 }

src/garage/tests/k2v/item.rs

@@ -1,3 +1,5 @@
+use std::time::Duration;
+
 use crate::common;
 
 use assert_json_diff::assert_json_eq;
@@ -86,6 +88,7 @@ async fn test_items_and_indices() {
 		assert_eq!(res_body, content);
 
 		// ReadIndex -- now there should be some stuff
+		tokio::time::sleep(Duration::from_secs(1)).await;
 		let res = ctx
 			.k2v
 			.request
@@ -154,6 +157,7 @@ async fn test_items_and_indices() {
 		assert_eq!(res_body, content2);
 
 		// ReadIndex -- now there should be some stuff
+		tokio::time::sleep(Duration::from_secs(1)).await;
 		let res = ctx
 			.k2v
 			.request
@@ -222,6 +226,7 @@ async fn test_items_and_indices() {
 		);
 
 		// ReadIndex -- now there should be some stuff
+		tokio::time::sleep(Duration::from_secs(1)).await;
 		let res = ctx
 			.k2v
 			.request
@@ -290,6 +295,7 @@ async fn test_items_and_indices() {
 		assert_eq!(res.status(), 204);
 
 		// ReadIndex -- now there should be some stuff
+		tokio::time::sleep(Duration::from_secs(1)).await;
 		let res = ctx
 			.k2v
 			.request

src/k2v-client/Cargo.toml

@@ -1,7 +1,12 @@
 [package]
 name = "k2v-client"
-version = "0.1.0"
+version = "0.0.1"
+authors = ["Trinity Pointard <trinity.pointard@gmail.com>", "Alex Auvolat <alex@adnab.me>"]
 edition = "2018"
+license = "AGPL-3.0"
+description = "Client library for the Garage K2V protocol"
+repository = "https://git.deuxfleurs.fr/Deuxfleurs/garage"
+readme = "../../README.md"
 
 [dependencies]
 base64 = "0.13.0"
@@ -17,7 +22,7 @@ tokio = "1.17.0"
 
 # cli deps
 clap = { version = "3.1.18", optional = true, features = ["derive", "env"] }
-garage_util = { path = "../util", optional = true }
+garage_util = { version = "0.7.0", path = "../util", optional = true }
 
 
 [features]

src/k2v-client/bin/k2v-cli.rs

@@ -1,3 +1,5 @@
+use std::time::Duration;
+
 use k2v_client::*;
 
 use garage_util::formater::format_table;
@@ -55,6 +57,22 @@ enum Command {
 		#[clap(flatten)]
 		output_kind: ReadOutputKind,
 	},
+	/// Watch changes on  a single value
+	Poll {
+		/// Partition key to delete from
+		partition_key: String,
+		/// Sort key to delete from
+		sort_key: String,
+		/// Causality information
+		#[clap(short, long)]
+		causality: String,
+		/// Timeout, in seconds
+		#[clap(short, long)]
+		timeout: Option<u64>,
+		/// Output formating
+		#[clap(flatten)]
+		output_kind: ReadOutputKind,
+	},
 	/// Delete a single value
 	Delete {
 		/// Partition key to delete from
@@ -324,6 +342,23 @@ async fn main() -> Result<(), Error> {
 			let res = client.read_item(&partition_key, &sort_key).await?;
 			output_kind.display_output(res);
 		}
+		Command::Poll {
+			partition_key,
+			sort_key,
+			causality,
+			timeout,
+			output_kind,
+		} => {
+			let timeout = timeout.map(Duration::from_secs);
+			let res_opt = client
+				.poll_item(&partition_key, &sort_key, causality.into(), timeout)
+				.await?;
+			if let Some(res) = res_opt {
+				output_kind.display_output(res);
+			} else {
+				println!("Delay expired and value didn't change.");
+			}
+		}
 		Command::ReadIndex {
 			output_kind,
 			filter,

src/k2v-client/lib.rs

@@ -122,14 +122,14 @@ impl K2vClient {
 
 		let res = self.dispatch(req, Some(timeout + DEFAULT_TIMEOUT)).await?;
 
-		let causality = res
-			.causality_token
-			.ok_or_else(|| Error::InvalidResponse("missing causality token".into()))?;
-
 		if res.status == StatusCode::NOT_MODIFIED {
 			return Ok(None);
 		}
 
+		let causality = res
+			.causality_token
+			.ok_or_else(|| Error::InvalidResponse("missing causality token".into()))?;
+
 		if res.status == StatusCode::NO_CONTENT {
 			return Ok(Some(CausalValue {
 				causality,

src/model/garage.rs

@@ -159,7 +159,6 @@ impl Garage {
 			&db,
 			config.data_dir.clone(),
 			config.compression_level,
-			config.block_manager_background_tranquility,
 			data_rep_param,
 			system.clone(),
 		);

src/model/index_counter.rs

@@ -2,8 +2,8 @@ use core::ops::Bound;
 use std::collections::{hash_map, BTreeMap, HashMap};
 use std::marker::PhantomData;
 use std::sync::Arc;
-use std::time::Duration;
 
+use async_trait::async_trait;
 use serde::{Deserialize, Serialize};
 use tokio::sync::{mpsc, watch};
 
@@ -11,6 +11,7 @@ use garage_db as db;
 
 use garage_rpc::ring::Ring;
 use garage_rpc::system::System;
+use garage_util::background::*;
 use garage_util::data::*;
 use garage_util::error::*;
 use garage_util::time::*;
@@ -171,11 +172,13 @@ impl<T: CountedItem> IndexCounter<T> {
 			),
 		});
 
-		let this2 = this.clone();
-		background.spawn_worker(
-			format!("{} index counter propagator", T::COUNTER_TABLE_NAME),
-			move |must_exit| this2.clone().propagate_loop(propagate_rx, must_exit),
-		);
+		background.spawn_worker(IndexPropagatorWorker {
+			index_counter: this.clone(),
+			propagate_rx,
+			buf: HashMap::new(),
+			errors: 0,
+		});
+
 		this
 	}
 
@@ -239,68 +242,6 @@ impl<T: CountedItem> IndexCounter<T> {
 		Ok(())
 	}
 
-	async fn propagate_loop(
-		self: Arc<Self>,
-		mut propagate_rx: mpsc::UnboundedReceiver<(T::CP, T::CS, LocalCounterEntry<T>)>,
-		must_exit: watch::Receiver<bool>,
-	) {
-		// This loop batches updates to counters to be sent all at once.
-		// They are sent once the propagate_rx channel has been emptied (or is closed).
-		let mut buf = HashMap::new();
-		let mut errors = 0;
-
-		loop {
-			let (ent, closed) = match propagate_rx.try_recv() {
-				Ok(ent) => (Some(ent), false),
-				Err(mpsc::error::TryRecvError::Empty) if buf.is_empty() => {
-					match propagate_rx.recv().await {
-						Some(ent) => (Some(ent), false),
-						None => (None, true),
-					}
-				}
-				Err(mpsc::error::TryRecvError::Empty) => (None, false),
-				Err(mpsc::error::TryRecvError::Disconnected) => (None, true),
-			};
-
-			if let Some((pk, sk, counters)) = ent {
-				let tree_key = self.table.data.tree_key(&pk, &sk);
-				let dist_entry = counters.into_counter_entry(self.this_node);
-				match buf.entry(tree_key) {
-					hash_map::Entry::Vacant(e) => {
-						e.insert(dist_entry);
-					}
-					hash_map::Entry::Occupied(mut e) => {
-						e.get_mut().merge(&dist_entry);
-					}
-				}
-				// As long as we can add entries, loop back and add them to batch
-				// before sending batch to other nodes
-				continue;
-			}
-
-			if !buf.is_empty() {
-				let entries = buf.iter().map(|(_k, v)| v);
-				if let Err(e) = self.table.insert_many(entries).await {
-					errors += 1;
-					if errors >= 2 && *must_exit.borrow() {
-						error!("({}) Could not propagate {} counter values: {}, these counters will not be updated correctly.", T::COUNTER_TABLE_NAME, buf.len(), e);
-						break;
-					}
-					warn!("({}) Could not propagate {} counter values: {}, retrying in 5 seconds (retry #{})", T::COUNTER_TABLE_NAME, buf.len(), e, errors);
-					tokio::time::sleep(Duration::from_secs(5)).await;
-					continue;
-				}
-
-				buf.clear();
-				errors = 0;
-			}
-
-			if closed || *must_exit.borrow() {
-				break;
-			}
-		}
-	}
-
 	pub fn offline_recount_all<TS, TR>(
 		&self,
 		counted_table: &Arc<Table<TS, TR>>,
@@ -437,6 +378,98 @@ impl<T: CountedItem> IndexCounter<T> {
 	}
 }
 
+struct IndexPropagatorWorker<T: CountedItem> {
+	index_counter: Arc<IndexCounter<T>>,
+	propagate_rx: mpsc::UnboundedReceiver<(T::CP, T::CS, LocalCounterEntry<T>)>,
+
+	buf: HashMap<Vec<u8>, CounterEntry<T>>,
+	errors: usize,
+}
+
+impl<T: CountedItem> IndexPropagatorWorker<T> {
+	fn add_ent(&mut self, pk: T::CP, sk: T::CS, counters: LocalCounterEntry<T>) {
+		let tree_key = self.index_counter.table.data.tree_key(&pk, &sk);
+		let dist_entry = counters.into_counter_entry(self.index_counter.this_node);
+		match self.buf.entry(tree_key) {
+			hash_map::Entry::Vacant(e) => {
+				e.insert(dist_entry);
+			}
+			hash_map::Entry::Occupied(mut e) => {
+				e.get_mut().merge(&dist_entry);
+			}
+		}
+	}
+}
+
+#[async_trait]
+impl<T: CountedItem> Worker for IndexPropagatorWorker<T> {
+	fn name(&self) -> String {
+		format!("{} index counter propagator", T::COUNTER_TABLE_NAME)
+	}
+
+	fn info(&self) -> Option<String> {
+		if !self.buf.is_empty() {
+			Some(format!("{} items in queue", self.buf.len()))
+		} else {
+			None
+		}
+	}
+
+	async fn work(&mut self, must_exit: &mut watch::Receiver<bool>) -> Result<WorkerState, Error> {
+		// This loop batches updates to counters to be sent all at once.
+		// They are sent once the propagate_rx channel has been emptied (or is closed).
+		let closed = loop {
+			match self.propagate_rx.try_recv() {
+				Ok((pk, sk, counters)) => {
+					self.add_ent(pk, sk, counters);
+				}
+				Err(mpsc::error::TryRecvError::Empty) => break false,
+				Err(mpsc::error::TryRecvError::Disconnected) => break true,
+			}
+		};
+
+		if !self.buf.is_empty() {
+			let entries_k = self.buf.keys().take(100).cloned().collect::<Vec<_>>();
+			let entries = entries_k.iter().map(|k| self.buf.get(k).unwrap());
+			if let Err(e) = self.index_counter.table.insert_many(entries).await {
+				self.errors += 1;
+				if self.errors >= 2 && *must_exit.borrow() {
+					error!("({}) Could not propagate {} counter values: {}, these counters will not be updated correctly.", T::COUNTER_TABLE_NAME, self.buf.len(), e);
+					return Ok(WorkerState::Done);
+				}
+				// Propagate error up to worker manager, it will log it, increment a counter,
+				// and sleep for a certain delay (with exponential backoff), waiting for
+				// things to go back to normal
+				return Err(e);
+			} else {
+				for k in entries_k {
+					self.buf.remove(&k);
+				}
+				self.errors = 0;
+			}
+
+			return Ok(WorkerState::Busy);
+		} else if closed {
+			return Ok(WorkerState::Done);
+		} else {
+			return Ok(WorkerState::Idle);
+		}
+	}
+
+	async fn wait_for_work(&mut self, _must_exit: &watch::Receiver<bool>) -> WorkerState {
+		match self.propagate_rx.recv().await {
+			Some((pk, sk, counters)) => {
+				self.add_ent(pk, sk, counters);
+				WorkerState::Busy
+			}
+			None => match self.buf.is_empty() {
+				false => WorkerState::Busy,
+				true => WorkerState::Done,
+			},
+		}
+	}
+}
+
 #[derive(PartialEq, Clone, Debug, Serialize, Deserialize)]
 struct LocalCounterEntry<T: CountedItem> {
 	pk: T::CP,

src/rpc/Cargo.toml

@@ -19,7 +19,6 @@ garage_util = { version = "0.7.0", path = "../util" }
 arc-swap = "1.0"
 bytes = "1.0"
 gethostname = "0.2"
-git-version = "0.3.4"
 hex = "0.4"
 tracing = "0.1.30"
 rand = "0.8"

src/rpc/system.rs

@@ -2,7 +2,7 @@
 use std::collections::HashMap;
 use std::io::{Read, Write};
 use std::net::{IpAddr, SocketAddr};
-use std::path::Path;
+use std::path::{Path, PathBuf};
 use std::sync::{Arc, RwLock};
 use std::time::{Duration, Instant};
 
@@ -27,6 +27,7 @@ use garage_util::data::*;
 use garage_util::error::*;
 use garage_util::persister::Persister;
 use garage_util::time::*;
+use garage_util::version;
 
 use crate::consul::*;
 #[cfg(feature = "kubernetes-discovery")]
@@ -104,6 +105,9 @@ pub struct System {
 
 	/// The job runner of this node
 	pub background: Arc<BackgroundRunner>,
+
+	/// Path to metadata directory
+	pub metadata_dir: PathBuf,
 }
 
 #[derive(Debug, Clone, Serialize, Deserialize)]
@@ -295,6 +299,7 @@ impl System {
 			ring,
 			update_ring: Mutex::new(update_ring),
 			background,
+			metadata_dir: config.metadata_dir.clone(),
 		});
 		sys.system_endpoint.set_handler(sys.clone());
 		sys
@@ -316,11 +321,7 @@ impl System {
 	//      also available through RPC) ----
 
 	pub fn garage_version(&self) -> &'static str {
-		option_env!("GIT_VERSION").unwrap_or(git_version::git_version!(
-			prefix = "git:",
-			cargo_prefix = "cargo:",
-			fallback = "unknown"
-		))
+		version::garage()
 	}
 
 	pub fn get_known_nodes(&self) -> Vec<KnownNodeInfo> {

src/table/gc.rs

@@ -8,12 +8,11 @@ use serde::{Deserialize, Serialize};
 use serde_bytes::ByteBuf;
 
 use futures::future::join_all;
-use futures::select;
-use futures_util::future::*;
 use tokio::sync::watch;
 
 use garage_db::counted_tree_hack::CountedTree;
 
+use garage_util::background::*;
 use garage_util::data::*;
 use garage_util::error::*;
 use garage_util::time::*;
@@ -69,35 +68,11 @@ where
 
 		gc.endpoint.set_handler(gc.clone());
 
-		let gc1 = gc.clone();
-		system.background.spawn_worker(
-			format!("GC loop for {}", F::TABLE_NAME),
-			move |must_exit: watch::Receiver<bool>| gc1.gc_loop(must_exit),
-		);
+		system.background.spawn_worker(GcWorker::new(gc.clone()));
 
 		gc
 	}
 
-	async fn gc_loop(self: Arc<Self>, mut must_exit: watch::Receiver<bool>) {
-		while !*must_exit.borrow() {
-			match self.gc_loop_iter().await {
-				Ok(None) => {
-					// Stuff was done, loop immediately
-				}
-				Ok(Some(wait_delay)) => {
-					// Nothing was done, wait specified delay.
-					select! {
-						_ = tokio::time::sleep(wait_delay).fuse() => {},
-						_ = must_exit.changed().fuse() => {},
-					}
-				}
-				Err(e) => {
-					warn!("({}) Error doing GC: {}", F::TABLE_NAME, e);
-				}
-			}
-		}
-	}
-
 	async fn gc_loop_iter(&self) -> Result<Option<Duration>, Error> {
 		let now = now_msec();
 
@@ -328,6 +303,66 @@ where
 	}
 }
 
+struct GcWorker<F, R>
+where
+	F: TableSchema + 'static,
+	R: TableReplication + 'static,
+{
+	gc: Arc<TableGc<F, R>>,
+	wait_delay: Duration,
+}
+
+impl<F, R> GcWorker<F, R>
+where
+	F: TableSchema + 'static,
+	R: TableReplication + 'static,
+{
+	fn new(gc: Arc<TableGc<F, R>>) -> Self {
+		Self {
+			gc,
+			wait_delay: Duration::from_secs(0),
+		}
+	}
+}
+
+#[async_trait]
+impl<F, R> Worker for GcWorker<F, R>
+where
+	F: TableSchema + 'static,
+	R: TableReplication + 'static,
+{
+	fn name(&self) -> String {
+		format!("{} GC", F::TABLE_NAME)
+	}
+
+	fn info(&self) -> Option<String> {
+		let l = self.gc.data.gc_todo_len().unwrap_or(0);
+		if l > 0 {
+			Some(format!("{} items in queue", l))
+		} else {
+			None
+		}
+	}
+
+	async fn work(&mut self, _must_exit: &mut watch::Receiver<bool>) -> Result<WorkerState, Error> {
+		match self.gc.gc_loop_iter().await? {
+			None => Ok(WorkerState::Busy),
+			Some(delay) => {
+				self.wait_delay = delay;
+				Ok(WorkerState::Idle)
+			}
+		}
+	}
+
+	async fn wait_for_work(&mut self, must_exit: &watch::Receiver<bool>) -> WorkerState {
+		if *must_exit.borrow() {
+			return WorkerState::Done;
+		}
+		tokio::time::sleep(self.wait_delay).await;
+		WorkerState::Busy
+	}
+}
+
 /// An entry stored in the gc_todo Sled tree associated with the table
 /// Contains helper function for parsing, saving, and removing
 /// such entry in Sled

src/table/merkle.rs

@@ -1,14 +1,13 @@
 use std::sync::Arc;
 use std::time::Duration;
 
-use futures::select;
-use futures_util::future::*;
+use async_trait::async_trait;
 use serde::{Deserialize, Serialize};
 use tokio::sync::watch;
 
 use garage_db as db;
 
-use garage_util::background::BackgroundRunner;
+use garage_util::background::*;
 use garage_util::data::*;
 use garage_util::error::Error;
 
@@ -78,43 +77,17 @@ where
 			empty_node_hash,
 		});
 
-		let ret2 = ret.clone();
-		background.spawn_worker(
-			format!("Merkle tree updater for {}", F::TABLE_NAME),
-			|must_exit: watch::Receiver<bool>| ret2.updater_loop(must_exit),
-		);
+		background.spawn_worker(MerkleWorker(ret.clone()));
 
 		ret
 	}
 
-	async fn updater_loop(self: Arc<Self>, mut must_exit: watch::Receiver<bool>) {
-		while !*must_exit.borrow() {
-			match self.updater_loop_iter() {
-				Ok(true) => (),
-				Ok(false) => {
-					select! {
-						_ = self.data.merkle_todo_notify.notified().fuse() => {},
-						_ = must_exit.changed().fuse() => {},
-					}
-				}
-				Err(e) => {
-					warn!(
-						"({}) Error while updating Merkle tree item: {}",
-						F::TABLE_NAME,
-						e
-					);
-					tokio::time::sleep(Duration::from_secs(10)).await;
-				}
-			}
-		}
-	}
-
-	fn updater_loop_iter(&self) -> Result<bool, Error> {
+	fn updater_loop_iter(&self) -> Result<WorkerState, Error> {
 		if let Some((key, valhash)) = self.data.merkle_todo.first()? {
 			self.update_item(&key, &valhash)?;
-			Ok(true)
+			Ok(WorkerState::Busy)
 		} else {
-			Ok(false)
+			Ok(WorkerState::Idle)
 		}
 	}
 
@@ -325,6 +298,54 @@ where
 	}
 }
 
+struct MerkleWorker<F, R>(Arc<MerkleUpdater<F, R>>)
+where
+	F: TableSchema + 'static,
+	R: TableReplication + 'static;
+
+#[async_trait]
+impl<F, R> Worker for MerkleWorker<F, R>
+where
+	F: TableSchema + 'static,
+	R: TableReplication + 'static,
+{
+	fn name(&self) -> String {
+		format!("{} Merkle tree updater", F::TABLE_NAME)
+	}
+
+	fn info(&self) -> Option<String> {
+		let l = self.0.todo_len().unwrap_or(0);
+		if l > 0 {
+			Some(format!("{} items in queue", l))
+		} else {
+			None
+		}
+	}
+
+	async fn work(&mut self, _must_exit: &mut watch::Receiver<bool>) -> Result<WorkerState, Error> {
+		let updater = self.0.clone();
+		tokio::task::spawn_blocking(move || {
+			for _i in 0..100 {
+				let s = updater.updater_loop_iter();
+				if !matches!(s, Ok(WorkerState::Busy)) {
+					return s;
+				}
+			}
+			Ok(WorkerState::Busy)
+		})
+		.await
+		.unwrap()
+	}
+
+	async fn wait_for_work(&mut self, must_exit: &watch::Receiver<bool>) -> WorkerState {
+		if *must_exit.borrow() {
+			return WorkerState::Done;
+		}
+		tokio::time::sleep(Duration::from_secs(10)).await;
+		WorkerState::Busy
+	}
+}
+
 impl MerkleNodeKey {
 	fn encode(&self) -> Vec<u8> {
 		let mut ret = Vec::with_capacity(2 + self.prefix.len());

src/table/sync.rs

@@ -1,17 +1,17 @@
 use std::collections::VecDeque;
-use std::sync::{Arc, Mutex};
+use std::sync::Arc;
 use std::time::{Duration, Instant};
 
 use async_trait::async_trait;
-use futures::select;
-use futures_util::future::*;
 use futures_util::stream::*;
 use opentelemetry::KeyValue;
 use rand::Rng;
 use serde::{Deserialize, Serialize};
 use serde_bytes::ByteBuf;
+use tokio::select;
 use tokio::sync::{mpsc, watch};
 
+use garage_util::background::*;
 use garage_util::data::*;
 use garage_util::error::Error;
 
@@ -34,7 +34,7 @@ pub struct TableSyncer<F: TableSchema + 'static, R: TableReplication + 'static>
 	data: Arc<TableData<F, R>>,
 	merkle: Arc<MerkleUpdater<F, R>>,
 
-	todo: Mutex<SyncTodo>,
+	add_full_sync_tx: mpsc::UnboundedSender<()>,
 	endpoint: Arc<Endpoint<SyncRpc, Self>>,
 }
 
@@ -52,10 +52,6 @@ impl Rpc for SyncRpc {
 	type Response = Result<SyncRpc, Error>;
 }
 
-struct SyncTodo {
-	todo: Vec<TodoPartition>,
-}
-
 #[derive(Debug, Clone)]
 struct TodoPartition {
 	partition: Partition,
@@ -80,118 +76,40 @@ where
 			.netapp
 			.endpoint(format!("garage_table/sync.rs/Rpc:{}", F::TABLE_NAME));
 
-		let todo = SyncTodo { todo: vec![] };
+		let (add_full_sync_tx, add_full_sync_rx) = mpsc::unbounded_channel();
 
 		let syncer = Arc::new(Self {
 			system: system.clone(),
 			data,
 			merkle,
-			todo: Mutex::new(todo),
+			add_full_sync_tx,
 			endpoint,
 		});
 
 		syncer.endpoint.set_handler(syncer.clone());
 
-		let (busy_tx, busy_rx) = mpsc::unbounded_channel();
-
-		let s1 = syncer.clone();
-		system.background.spawn_worker(
-			format!("table sync watcher for {}", F::TABLE_NAME),
-			move |must_exit: watch::Receiver<bool>| s1.watcher_task(must_exit, busy_rx),
-		);
-
-		let s2 = syncer.clone();
-		system.background.spawn_worker(
-			format!("table syncer for {}", F::TABLE_NAME),
-			move |must_exit: watch::Receiver<bool>| s2.syncer_task(must_exit, busy_tx),
-		);
-
-		let s3 = syncer.clone();
-		tokio::spawn(async move {
-			tokio::time::sleep(Duration::from_secs(20)).await;
-			s3.add_full_sync();
+		system.background.spawn_worker(SyncWorker {
+			syncer: syncer.clone(),
+			ring_recv: system.ring.clone(),
+			ring: system.ring.borrow().clone(),
+			add_full_sync_rx,
+			todo: vec![],
+			next_full_sync: Instant::now() + Duration::from_secs(20),
 		});
 
 		syncer
 	}
 
-	async fn watcher_task(
-		self: Arc<Self>,
-		mut must_exit: watch::Receiver<bool>,
-		mut busy_rx: mpsc::UnboundedReceiver<bool>,
-	) {
-		let mut prev_ring: Arc<Ring> = self.system.ring.borrow().clone();
-		let mut ring_recv: watch::Receiver<Arc<Ring>> = self.system.ring.clone();
-		let mut nothing_to_do_since = Some(Instant::now());
-
-		while !*must_exit.borrow() {
-			select! {
-				_ = ring_recv.changed().fuse() => {
-					let new_ring = ring_recv.borrow();
-					if !Arc::ptr_eq(&new_ring, &prev_ring) {
-						debug!("({}) Ring changed, adding full sync to syncer todo list", F::TABLE_NAME);
-						self.add_full_sync();
-						prev_ring = new_ring.clone();
-					}
-				}
-				busy_opt = busy_rx.recv().fuse() => {
-					if let Some(busy) = busy_opt {
-						if busy {
-							nothing_to_do_since = None;
-						} else if nothing_to_do_since.is_none() {
-							nothing_to_do_since = Some(Instant::now());
-						}
-					}
-				}
-				_ = must_exit.changed().fuse() => {},
-				_ = tokio::time::sleep(Duration::from_secs(1)).fuse() => {
-					if nothing_to_do_since.map(|t| Instant::now() - t >= ANTI_ENTROPY_INTERVAL).unwrap_or(false) {
-						nothing_to_do_since = None;
-						debug!("({}) Interval passed, adding full sync to syncer todo list", F::TABLE_NAME);
-						self.add_full_sync();
-					}
-				}
-			}
-		}
-	}
-
 	pub fn add_full_sync(&self) {
-		self.todo
-			.lock()
-			.unwrap()
-			.add_full_sync(&self.data, &self.system);
-	}
-
-	async fn syncer_task(
-		self: Arc<Self>,
-		mut must_exit: watch::Receiver<bool>,
-		busy_tx: mpsc::UnboundedSender<bool>,
-	) {
-		while !*must_exit.borrow() {
-			let task = self.todo.lock().unwrap().pop_task();
-			if let Some(partition) = task {
-				busy_tx.send(true).unwrap();
-				let res = self
-					.clone()
-					.sync_partition(&partition, &mut must_exit)
-					.await;
-				if let Err(e) = res {
-					warn!(
-						"({}) Error while syncing {:?}: {}",
-						F::TABLE_NAME,
-						partition,
-						e
-					);
-				}
-			} else {
-				busy_tx.send(false).unwrap();
-				tokio::time::sleep(Duration::from_secs(1)).await;
-			}
+		if self.add_full_sync_tx.send(()).is_err() {
+			error!("({}) Could not add full sync", F::TABLE_NAME);
 		}
 	}
 
+	// ----
+
 	async fn sync_partition(
-		self: Arc<Self>,
+		self: &Arc<Self>,
 		partition: &TodoPartition,
 		must_exit: &mut watch::Receiver<bool>,
 	) -> Result<(), Error> {
@@ -577,12 +495,22 @@ where
 	}
 }
 
-impl SyncTodo {
-	fn add_full_sync<F: TableSchema, R: TableReplication>(
-		&mut self,
-		data: &TableData<F, R>,
-		system: &System,
-	) {
+// -------- Sync Worker ---------
+
+struct SyncWorker<F: TableSchema + 'static, R: TableReplication + 'static> {
+	syncer: Arc<TableSyncer<F, R>>,
+	ring_recv: watch::Receiver<Arc<Ring>>,
+	ring: Arc<Ring>,
+	add_full_sync_rx: mpsc::UnboundedReceiver<()>,
+	todo: Vec<TodoPartition>,
+	next_full_sync: Instant,
+}
+
+impl<F: TableSchema + 'static, R: TableReplication + 'static> SyncWorker<F, R> {
+	fn add_full_sync(&mut self) {
+		let system = &self.syncer.system;
+		let data = &self.syncer.data;
+
 		let my_id = system.id;
 
 		self.todo.clear();
@@ -623,6 +551,8 @@ impl SyncTodo {
 				retain,
 			});
 		}
+
+		self.next_full_sync = Instant::now() + ANTI_ENTROPY_INTERVAL;
 	}
 
 	fn pop_task(&mut self) -> Option<TodoPartition> {
@@ -641,6 +571,62 @@ impl SyncTodo {
 	}
 }
 
+#[async_trait]
+impl<F: TableSchema + 'static, R: TableReplication + 'static> Worker for SyncWorker<F, R> {
+	fn name(&self) -> String {
+		format!("{} sync", F::TABLE_NAME)
+	}
+
+	fn info(&self) -> Option<String> {
+		let l = self.todo.len();
+		if l > 0 {
+			Some(format!("{} partitions remaining", l))
+		} else {
+			None
+		}
+	}
+
+	async fn work(&mut self, must_exit: &mut watch::Receiver<bool>) -> Result<WorkerState, Error> {
+		if let Some(partition) = self.pop_task() {
+			self.syncer.sync_partition(&partition, must_exit).await?;
+			Ok(WorkerState::Busy)
+		} else {
+			Ok(WorkerState::Idle)
+		}
+	}
+
+	async fn wait_for_work(&mut self, must_exit: &watch::Receiver<bool>) -> WorkerState {
+		if *must_exit.borrow() {
+			return WorkerState::Done;
+		}
+		select! {
+			s = self.add_full_sync_rx.recv() => {
+				if let Some(()) = s {
+					self.add_full_sync();
+				}
+			},
+			_ = self.ring_recv.changed() => {
+				let new_ring = self.ring_recv.borrow();
+				if !Arc::ptr_eq(&new_ring, &self.ring) {
+					self.ring = new_ring.clone();
+					drop(new_ring);
+					debug!("({}) Ring changed, adding full sync to syncer todo list", F::TABLE_NAME);
+					self.add_full_sync();
+				}
+			},
+			_ = tokio::time::sleep(self.next_full_sync - Instant::now()) => {
+				self.add_full_sync();
+			}
+		}
+		match self.todo.is_empty() {
+			false => WorkerState::Busy,
+			true => WorkerState::Idle,
+		}
+	}
+}
+
+// ---- UTIL ----
+
 fn hash_of<T: Serialize>(x: &T) -> Result<Hash, Error> {
 	Ok(blake2sum(&rmp_to_vec_all_named(x)?[..]))
 }

src/util/Cargo.toml

@@ -16,6 +16,7 @@ path = "lib.rs"
 [dependencies]
 garage_db = { version = "0.8.0", path = "../db" }
 
+async-trait = "0.1"
 blake2 = "0.9"
 err-derive = "0.3"
 xxhash-rust = { version = "0.8", default-features = false, features = ["xxh3"] }
@@ -23,6 +24,7 @@ hex = "0.4"
 tracing = "0.1.30"
 rand = "0.8"
 sha2 = "0.9"
+git-version = "0.3.4"
 
 chrono = "0.4"
 rmp-serde = "0.15"
@@ -42,5 +44,6 @@ hyper = "0.14"
 
 opentelemetry = { version = "0.17", features = [ "rt-tokio", "metrics", "trace" ] }
 
+
 [features]
 k2v = []

src/util/background.rs

@@ -1,160 +0,0 @@
-//! Job runner for futures and async functions
-use core::future::Future;
-use std::pin::Pin;
-use std::sync::Arc;
-use std::time::Duration;
-
-use futures::future::*;
-use futures::select;
-use futures::stream::FuturesUnordered;
-use futures::StreamExt;
-use tokio::sync::{mpsc, mpsc::error::TryRecvError, watch, Mutex};
-
-use crate::error::Error;
-
-type JobOutput = Result<(), Error>;
-type Job = Pin<Box<dyn Future<Output = JobOutput> + Send>>;
-
-/// Job runner for futures and async functions
-pub struct BackgroundRunner {
-	stop_signal: watch::Receiver<bool>,
-	queue_in: mpsc::UnboundedSender<(Job, bool)>,
-	worker_in: mpsc::UnboundedSender<tokio::task::JoinHandle<()>>,
-}
-
-impl BackgroundRunner {
-	/// Create a new BackgroundRunner
-	pub fn new(
-		n_runners: usize,
-		stop_signal: watch::Receiver<bool>,
-	) -> (Arc<Self>, tokio::task::JoinHandle<()>) {
-		let (worker_in, mut worker_out) = mpsc::unbounded_channel();
-
-		let stop_signal_2 = stop_signal.clone();
-		let await_all_done = tokio::spawn(async move {
-			let mut workers = FuturesUnordered::new();
-			let mut shutdown_timer = 0;
-			loop {
-				let closed = match worker_out.try_recv() {
-					Ok(wkr) => {
-						workers.push(wkr);
-						false
-					}
-					Err(TryRecvError::Empty) => false,
-					Err(TryRecvError::Disconnected) => true,
-				};
-				select! {
-					res = workers.next() => {
-						if let Some(Err(e)) = res {
-							error!("Worker exited with error: {}", e);
-						}
-					}
-					_ = tokio::time::sleep(Duration::from_secs(1)).fuse() => {
-						if closed || *stop_signal_2.borrow() {
-							shutdown_timer += 1;
-							if shutdown_timer >= 10 {
-								break;
-							}
-						}
-					}
-				}
-			}
-		});
-
-		let (queue_in, queue_out) = mpsc::unbounded_channel();
-		let queue_out = Arc::new(Mutex::new(queue_out));
-
-		for i in 0..n_runners {
-			let queue_out = queue_out.clone();
-			let stop_signal = stop_signal.clone();
-
-			worker_in
-				.send(tokio::spawn(async move {
-					loop {
-						let (job, cancellable) = {
-							select! {
-								item = wait_job(&queue_out).fuse() => match item {
-									// We received a task, process it
-									Some(x) => x,
-									// We received a signal that no more tasks will ever be sent
-									// because the sending side was dropped. Exit now.
-									None => break,
-								},
-								_ = tokio::time::sleep(Duration::from_secs(5)).fuse() => {
-									if *stop_signal.borrow() {
-										// Nothing has been going on for 5 secs, and we are shutting
-										// down. Exit now.
-										break;
-									} else {
-										// Nothing is going on but we don't want to exit.
-										continue;
-									}
-								}
-							}
-						};
-						if cancellable && *stop_signal.borrow() {
-							continue;
-						}
-						if let Err(e) = job.await {
-							error!("Job failed: {}", e)
-						}
-					}
-					info!("Background worker {} exiting", i);
-				}))
-				.unwrap();
-		}
-
-		let bgrunner = Arc::new(Self {
-			stop_signal,
-			queue_in,
-			worker_in,
-		});
-		(bgrunner, await_all_done)
-	}
-
-	/// Spawn a task to be run in background
-	pub fn spawn<T>(&self, job: T)
-	where
-		T: Future<Output = JobOutput> + Send + 'static,
-	{
-		let boxed: Job = Box::pin(job);
-		self.queue_in
-			.send((boxed, false))
-			.map_err(|_| "could not put job in queue")
-			.unwrap();
-	}
-
-	/// Spawn a task to be run in background. It may get discarded before running if spawned while
-	/// the runner is stopping
-	pub fn spawn_cancellable<T>(&self, job: T)
-	where
-		T: Future<Output = JobOutput> + Send + 'static,
-	{
-		let boxed: Job = Box::pin(job);
-		self.queue_in
-			.send((boxed, true))
-			.map_err(|_| "could not put job in queue")
-			.unwrap();
-	}
-
-	pub fn spawn_worker<F, T>(&self, name: String, worker: F)
-	where
-		F: FnOnce(watch::Receiver<bool>) -> T + Send + 'static,
-		T: Future<Output = ()> + Send + 'static,
-	{
-		let stop_signal = self.stop_signal.clone();
-		let task = tokio::spawn(async move {
-			info!("Worker started: {}", name);
-			worker(stop_signal).await;
-			info!("Worker exited: {}", name);
-		});
-		self.worker_in
-			.send(task)
-			.map_err(|_| "could not put job in queue")
-			.unwrap();
-	}
-}
-
-async fn wait_job(q: &Mutex<mpsc::UnboundedReceiver<(Job, bool)>>) -> Option<(Job, bool)> {
-	q.lock().await.recv().await
-}

src/util/background/job_worker.rs

@@ -0,0 +1,48 @@
+//! Job worker: a generic worker that just processes incoming
+//! jobs one by one
+
+use std::sync::Arc;
+
+use async_trait::async_trait;
+use tokio::sync::{mpsc, Mutex};
+
+use crate::background::worker::*;
+use crate::background::*;
+
+pub(crate) struct JobWorker {
+	pub(crate) index: usize,
+	pub(crate) job_chan: Arc<Mutex<mpsc::UnboundedReceiver<(Job, bool)>>>,
+	pub(crate) next_job: Option<Job>,
+}
+
+#[async_trait]
+impl Worker for JobWorker {
+	fn name(&self) -> String {
+		format!("Job worker #{}", self.index)
+	}
+
+	async fn work(&mut self, _must_exit: &mut watch::Receiver<bool>) -> Result<WorkerState, Error> {
+		match self.next_job.take() {
+			None => return Ok(WorkerState::Idle),
+			Some(job) => {
+				job.await?;
+				Ok(WorkerState::Busy)
+			}
+		}
+	}
+
+	async fn wait_for_work(&mut self, must_exit: &watch::Receiver<bool>) -> WorkerState {
+		loop {
+			match self.job_chan.lock().await.recv().await {
+				Some((job, cancellable)) => {
+					if cancellable && *must_exit.borrow() {
+						continue;
+					}
+					self.next_job = Some(job);
+					return WorkerState::Busy;
+				}
+				None => return WorkerState::Done,
+			}
+		}
+	}
+}

src/util/background/mod.rs

@@ -0,0 +1,117 @@
+//! Job runner for futures and async functions
+
+pub mod job_worker;
+pub mod worker;
+
+use core::future::Future;
+
+use std::collections::HashMap;
+use std::pin::Pin;
+use std::sync::Arc;
+
+use serde::{Deserialize, Serialize};
+use tokio::sync::{mpsc, watch, Mutex};
+
+use crate::error::Error;
+use worker::WorkerProcessor;
+pub use worker::{Worker, WorkerState};
+
+pub(crate) type JobOutput = Result<(), Error>;
+pub(crate) type Job = Pin<Box<dyn Future<Output = JobOutput> + Send>>;
+
+/// Job runner for futures and async functions
+pub struct BackgroundRunner {
+	send_job: mpsc::UnboundedSender<(Job, bool)>,
+	send_worker: mpsc::UnboundedSender<Box<dyn Worker>>,
+	worker_info: Arc<std::sync::Mutex<HashMap<usize, WorkerInfo>>>,
+}
+
+#[derive(Clone, Serialize, Deserialize, Debug)]
+pub struct WorkerInfo {
+	pub name: String,
+	pub info: Option<String>,
+	pub state: WorkerState,
+	pub errors: usize,
+	pub consecutive_errors: usize,
+	pub last_error: Option<(String, u64)>,
+}
+
+impl BackgroundRunner {
+	/// Create a new BackgroundRunner
+	pub fn new(
+		n_runners: usize,
+		stop_signal: watch::Receiver<bool>,
+	) -> (Arc<Self>, tokio::task::JoinHandle<()>) {
+		let (send_worker, worker_out) = mpsc::unbounded_channel::<Box<dyn Worker>>();
+
+		let worker_info = Arc::new(std::sync::Mutex::new(HashMap::new()));
+		let mut worker_processor =
+			WorkerProcessor::new(worker_out, stop_signal, worker_info.clone());
+
+		let await_all_done = tokio::spawn(async move {
+			worker_processor.run().await;
+		});
+
+		let (send_job, queue_out) = mpsc::unbounded_channel();
+		let queue_out = Arc::new(Mutex::new(queue_out));
+
+		for i in 0..n_runners {
+			let queue_out = queue_out.clone();
+
+			send_worker
+				.send(Box::new(job_worker::JobWorker {
+					index: i,
+					job_chan: queue_out.clone(),
+					next_job: None,
+				}))
+				.ok()
+				.unwrap();
+		}
+
+		let bgrunner = Arc::new(Self {
+			send_job,
+			send_worker,
+			worker_info,
+		});
+		(bgrunner, await_all_done)
+	}
+
+	pub fn get_worker_info(&self) -> HashMap<usize, WorkerInfo> {
+		self.worker_info.lock().unwrap().clone()
+	}
+
+	/// Spawn a task to be run in background
+	pub fn spawn<T>(&self, job: T)
+	where
+		T: Future<Output = JobOutput> + Send + 'static,
+	{
+		let boxed: Job = Box::pin(job);
+		self.send_job
+			.send((boxed, false))
+			.ok()
+			.expect("Could not put job in queue");
+	}
+
+	/// Spawn a task to be run in background. It may get discarded before running if spawned while
+	/// the runner is stopping
+	pub fn spawn_cancellable<T>(&self, job: T)
+	where
+		T: Future<Output = JobOutput> + Send + 'static,
+	{
+		let boxed: Job = Box::pin(job);
+		self.send_job
+			.send((boxed, true))
+			.ok()
+			.expect("Could not put job in queue");
+	}
+
+	pub fn spawn_worker<W>(&self, worker: W)
+	where
+		W: Worker + 'static,
+	{
+		self.send_worker
+			.send(Box::new(worker))
+			.ok()
+			.expect("Could not put worker in queue");
+	}
+}

src/util/background/worker.rs

@@ -0,0 +1,260 @@
+use std::collections::HashMap;
+use std::sync::Arc;
+use std::time::{Duration, Instant};
+
+use async_trait::async_trait;
+use futures::future::*;
+use futures::stream::FuturesUnordered;
+use futures::StreamExt;
+use serde::{Deserialize, Serialize};
+use tokio::select;
+use tokio::sync::{mpsc, watch};
+
+use crate::background::WorkerInfo;
+use crate::error::Error;
+use crate::time::now_msec;
+
+#[derive(PartialEq, Copy, Clone, Serialize, Deserialize, Debug)]
+pub enum WorkerState {
+	Busy,
+	Throttled(f32),
+	Idle,
+	Done,
+}
+
+impl std::fmt::Display for WorkerState {
+	fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+		match self {
+			WorkerState::Busy => write!(f, "Busy"),
+			WorkerState::Throttled(t) => write!(f, "Thr:{:.3}", t),
+			WorkerState::Idle => write!(f, "Idle"),
+			WorkerState::Done => write!(f, "Done"),
+		}
+	}
+}
+
+#[async_trait]
+pub trait Worker: Send {
+	fn name(&self) -> String;
+
+	fn info(&self) -> Option<String> {
+		None
+	}
+
+	/// Work: do a basic unit of work, if one is available (otherwise, should return
+	/// WorkerState::Idle immediately).  We will do our best to not interrupt this future in the
+	/// middle of processing, it will only be interrupted at the last minute when Garage is trying
+	/// to exit and this hasn't returned yet. This function may return an error to indicate that
+	/// its unit of work could not be processed due to an error: the error will be logged and
+	/// .work() will be called again after a short delay.
+	async fn work(&mut self, must_exit: &mut watch::Receiver<bool>) -> Result<WorkerState, Error>;
+
+	/// Wait for work: await for some task to become available.  This future can be interrupted in
+	/// the middle for any reason.  This future doesn't have to await on must_exit.changed(), we
+	/// are doing it for you.  Therefore it only receives a read refernce to must_exit which allows
+	/// it to check if we are exiting.
+	async fn wait_for_work(&mut self, must_exit: &watch::Receiver<bool>) -> WorkerState;
+}
+
+pub(crate) struct WorkerProcessor {
+	stop_signal: watch::Receiver<bool>,
+	worker_chan: mpsc::UnboundedReceiver<Box<dyn Worker>>,
+	worker_info: Arc<std::sync::Mutex<HashMap<usize, WorkerInfo>>>,
+}
+
+impl WorkerProcessor {
+	pub(crate) fn new(
+		worker_chan: mpsc::UnboundedReceiver<Box<dyn Worker>>,
+		stop_signal: watch::Receiver<bool>,
+		worker_info: Arc<std::sync::Mutex<HashMap<usize, WorkerInfo>>>,
+	) -> Self {
+		Self {
+			stop_signal,
+			worker_chan,
+			worker_info,
+		}
+	}
+
+	pub(crate) async fn run(&mut self) {
+		let mut workers = FuturesUnordered::new();
+		let mut next_task_id = 1;
+
+		while !*self.stop_signal.borrow() {
+			let await_next_worker = async {
+				if workers.is_empty() {
+					futures::future::pending().await
+				} else {
+					workers.next().await
+				}
+			};
+			select! {
+				new_worker_opt = self.worker_chan.recv() => {
+					if let Some(new_worker) = new_worker_opt {
+						let task_id = next_task_id;
+						next_task_id += 1;
+						let stop_signal = self.stop_signal.clone();
+						let stop_signal_worker = self.stop_signal.clone();
+						let mut worker = WorkerHandler {
+								task_id,
+								stop_signal,
+								stop_signal_worker,
+								worker: new_worker,
+								state: WorkerState::Busy,
+								errors: 0,
+								consecutive_errors: 0,
+								last_error: None,
+							};
+						workers.push(async move {
+							worker.step().await;
+							worker
+						}.boxed());
+					}
+				}
+				worker = await_next_worker => {
+					if let Some(mut worker) = worker {
+						trace!("{} (TID {}): {:?}", worker.worker.name(), worker.task_id, worker.state);
+
+						// Save worker info
+						let mut wi = self.worker_info.lock().unwrap();
+						match wi.get_mut(&worker.task_id) {
+							Some(i) => {
+								i.state = worker.state;
+								i.info = worker.worker.info();
+								i.errors = worker.errors;
+								i.consecutive_errors = worker.consecutive_errors;
+								if worker.last_error.is_some() {
+									i.last_error = worker.last_error.take();
+								}
+							}
+							None => {
+								wi.insert(worker.task_id, WorkerInfo {
+									name: worker.worker.name(),
+									state: worker.state,
+									info: worker.worker.info(),
+									errors: worker.errors,
+									consecutive_errors: worker.consecutive_errors,
+									last_error: worker.last_error.take(),
+								});
+							}
+						}
+
+						if worker.state == WorkerState::Done {
+							info!("Worker {} (TID {}) exited", worker.worker.name(), worker.task_id);
+						} else {
+							workers.push(async move {
+								worker.step().await;
+								worker
+							}.boxed());
+						}
+					}
+				}
+				_ = self.stop_signal.changed() => (),
+			}
+		}
+
+		// We are exiting, drain everything
+		let drain_half_time = Instant::now() + Duration::from_secs(5);
+		let drain_everything = async move {
+			while let Some(mut worker) = workers.next().await {
+				if worker.state == WorkerState::Done {
+					info!(
+						"Worker {} (TID {}) exited",
+						worker.worker.name(),
+						worker.task_id
+					);
+				} else if Instant::now() > drain_half_time {
+					warn!("Worker {} (TID {}) interrupted between two iterations in state {:?} (this should be fine)", worker.worker.name(), worker.task_id, worker.state);
+				} else {
+					workers.push(
+						async move {
+							worker.step().await;
+							worker
+						}
+						.boxed(),
+					);
+				}
+			}
+		};
+
+		select! {
+			_ = drain_everything => {
+				info!("All workers exited peacefully \\o/");
+			}
+			_ = tokio::time::sleep(Duration::from_secs(9)) => {
+				error!("Some workers could not exit in time, we are cancelling some things in the middle");
+			}
+		}
+	}
+}
+
+struct WorkerHandler {
+	task_id: usize,
+	stop_signal: watch::Receiver<bool>,
+	stop_signal_worker: watch::Receiver<bool>,
+	worker: Box<dyn Worker>,
+	state: WorkerState,
+	errors: usize,
+	consecutive_errors: usize,
+	last_error: Option<(String, u64)>,
+}
+
+impl WorkerHandler {
+	async fn step(&mut self) {
+		match self.state {
+			WorkerState::Busy => match self.worker.work(&mut self.stop_signal).await {
+				Ok(s) => {
+					self.state = s;
+					self.consecutive_errors = 0;
+				}
+				Err(e) => {
+					error!(
+						"Error in worker {} (TID {}): {}",
+						self.worker.name(),
+						self.task_id,
+						e
+					);
+					self.errors += 1;
+					self.consecutive_errors += 1;
+					self.last_error = Some((format!("{}", e), now_msec()));
+					// Sleep a bit so that error won't repeat immediately, exponential backoff
+					// strategy (min 1sec, max ~60sec)
+					self.state = WorkerState::Throttled(
+						(1.5f32).powf(std::cmp::min(10, self.consecutive_errors - 1) as f32),
+					);
+				}
+			},
+			WorkerState::Throttled(delay) => {
+				// Sleep for given delay and go back to busy state
+				if !*self.stop_signal.borrow() {
+					select! {
+						_ = tokio::time::sleep(Duration::from_secs_f32(delay)) => (),
+						_ = self.stop_signal.changed() => (),
+					}
+				}
+				self.state = WorkerState::Busy;
+			}
+			WorkerState::Idle => {
+				if *self.stop_signal.borrow() {
+					select! {
+						new_st = self.worker.wait_for_work(&self.stop_signal_worker) => {
+							self.state = new_st;
+						}
+						_ = tokio::time::sleep(Duration::from_secs(1)) => {
+							// stay in Idle state
+						}
+					}
+				} else {
+					select! {
+						new_st = self.worker.wait_for_work(&self.stop_signal_worker) => {
+							self.state = new_st;
+						}
+						_ = self.stop_signal.changed() => {
+							// stay in Idle state
+						}
+					}
+				}
+			}
+			WorkerState::Done => unreachable!(),
+		}
+	}
+}

src/util/config.rs

@@ -23,10 +23,6 @@ pub struct Config {
 	#[serde(default = "default_block_size")]
 	pub block_size: usize,
 
-	/// Size of data blocks to save to disk
-	#[serde(default = "default_block_manager_background_tranquility")]
-	pub block_manager_background_tranquility: u32,
-
 	/// Replication mode. Supported values:
 	/// - none, 1 -> no replication
 	/// - 2 -> 2-way replication
@@ -147,9 +143,6 @@ fn default_sled_flush_every_ms() -> u64 {
 fn default_block_size() -> usize {
 	1048576
 }
-fn default_block_manager_background_tranquility() -> u32 {
-	2
-}
 
 /// Read and parse configuration
 pub fn read_config(config_file: PathBuf) -> Result<Config, Error> {

src/util/lib.rs

@@ -11,7 +11,7 @@ pub mod error;
 pub mod formater;
 pub mod metrics;
 pub mod persister;
-//pub mod sled_counter;
 pub mod time;
 pub mod token_bucket;
 pub mod tranquilizer;
+pub mod version;

src/util/tranquilizer.rs

@@ -3,6 +3,8 @@ use std::time::{Duration, Instant};
 
 use tokio::time::sleep;
 
+use crate::background::WorkerState;
+
 /// A tranquilizer is a helper object that is used to make
 /// background operations not take up too much time.
 ///
@@ -33,7 +35,7 @@ impl Tranquilizer {
 		}
 	}
 
-	pub async fn tranquilize(&mut self, tranquility: u32) {
+	fn tranquilize_internal(&mut self, tranquility: u32) -> Option<Duration> {
 		let observation = Instant::now() - self.last_step_begin;
 
 		self.observations.push_back(observation);
@@ -45,13 +47,32 @@ impl Tranquilizer {
 
 		if !self.observations.is_empty() {
 			let delay = (tranquility * self.sum_observations) / (self.observations.len() as u32);
-			sleep(delay).await;
+			Some(delay)
+		} else {
+			None
 		}
+	}
 
-		self.reset();
+	pub async fn tranquilize(&mut self, tranquility: u32) {
+		if let Some(delay) = self.tranquilize_internal(tranquility) {
+			sleep(delay).await;
+			self.reset();
+		}
+	}
+
+	#[must_use]
+	pub fn tranquilize_worker(&mut self, tranquility: u32) -> WorkerState {
+		match self.tranquilize_internal(tranquility) {
+			Some(delay) => WorkerState::Throttled(delay.as_secs_f32()),
+			None => WorkerState::Busy,
+		}
 	}
 
 	pub fn reset(&mut self) {
 		self.last_step_begin = Instant::now();
 	}
+
+	pub fn clear(&mut self) {
+		self.observations.clear();
+	}
 }

src/util/version.rs

@@ -0,0 +1,7 @@
+pub fn garage() -> &'static str {
+	option_env!("GIT_VERSION").unwrap_or(git_version::git_version!(
+		prefix = "git:",
+		cargo_prefix = "cargo:",
+		fallback = "unknown"
+	))
+}