Merge pull request 'New layout: fixes and UX improvements' (#634 ) from new-layout-ux into next

Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/634
layout doc: reformulate
2026-08-14 16:13:11 +00:00 · 2023-09-27 09:04:32 +00:00 · 2023-09-22 16:14:47 +02:00 · 2023-09-22 16:07:46 +02:00 · 2023-09-22 10:06:31 +02:00 · 2023-09-22 09:49:07 +02:00
56 changed files with 2819 additions and 793 deletions
@@ -1198,7 +1198,7 @@ dependencies = [

 [[package]]
 name = "garage"
-version = "0.8.3"
+version = "0.8.4"
 dependencies = [
 "assert-json-diff",
 "async-trait",
@@ -1249,7 +1249,7 @@ dependencies = [

 [[package]]
 name = "garage_api"
-version = "0.8.3"
+version = "0.8.4"
 dependencies = [
 "async-trait",
 "base64 0.21.3",
@@ -1294,12 +1294,13 @@ dependencies = [

 [[package]]
 name = "garage_block"
-version = "0.8.3"
+version = "0.8.4"
 dependencies = [
 "arc-swap",
 "async-compression",
 "async-trait",
 "bytes",
+ "bytesize",
 "futures",
 "futures-util",
 "garage_db",
@@ -1319,7 +1320,7 @@ dependencies = [

 [[package]]
 name = "garage_db"
-version = "0.8.3"
+version = "0.8.4"
 dependencies = [
 "clap 4.4.0",
 "err-derive",
@@ -1334,12 +1335,13 @@ dependencies = [

 [[package]]
 name = "garage_model"
-version = "0.8.3"
+version = "0.8.4"
 dependencies = [
 "arc-swap",
 "async-trait",
 "base64 0.21.3",
 "blake2",
+ "chrono",
 "err-derive",
 "futures",
 "futures-util",
@@ -1361,13 +1363,14 @@ dependencies = [

 [[package]]
 name = "garage_rpc"
-version = "0.8.3"
+version = "0.8.4"
 dependencies = [
 "arc-swap",
 "async-trait",
 "bytes",
 "bytesize",
 "err-derive",
+ "format_table",
 "futures",
 "futures-util",
 "garage_db",
@@ -1379,6 +1382,7 @@ dependencies = [
 "kube",
 "kuska-sodiumoxide",
 "netapp",
+ "nix",
 "opentelemetry",
 "pnet_datalink",
 "rand",
@@ -1387,7 +1391,6 @@ dependencies = [
 "serde",
 "serde_bytes",
 "serde_json",
- "systemstat",
 "tokio",
 "tokio-stream",
 "tracing",
@@ -1395,7 +1398,7 @@ dependencies = [

 [[package]]
 name = "garage_table"
-version = "0.8.3"
+version = "0.8.4"
 dependencies = [
 "arc-swap",
 "async-trait",
@@ -1417,12 +1420,13 @@ dependencies = [

 [[package]]
 name = "garage_util"
-version = "0.8.3"
+version = "0.8.4"
 dependencies = [
 "arc-swap",
 "async-trait",
 "blake2",
 "bytes",
+ "bytesize",
 "chrono",
 "digest",
 "err-derive",
@@ -1450,7 +1454,7 @@ dependencies = [

 [[package]]
 name = "garage_web"
-version = "0.8.3"
+version = "0.8.4"
 dependencies = [
 "err-derive",
 "futures",
@@ -2268,9 +2272,9 @@ checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a"

 [[package]]
 name = "netapp"
-version = "0.5.3"
+version = "0.5.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0eb28e89e7b21d3fdf2b46e2a580ccaab68f469f2e5c27db74f61994be90b558"
+checksum = "4ffe47ac46d3b2ce2f736a70865492df082e042eb2bfdddfca3b8dd66bd9469d"
 dependencies = [
 "arc-swap",
 "async-trait",
@@ -2293,6 +2297,17 @@ dependencies = [
 "tokio-util 0.7.8",
 ]

+[[package]]
+name = "nix"
+version = "0.27.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2eb04e9c688eff1c89d72b407f168cf79bb9e867a9d3323ed6c01519eb9cc053"
+dependencies = [
+ "bitflags 2.4.0",
+ "cfg-if",
+ "libc",
+]
+
 [[package]]
 name = "no-std-net"
 version = "0.6.0"
@@ -3006,9 +3021,9 @@ dependencies = [

 [[package]]
 name = "rmp-serde"
-version = "1.1.2"
+version = "0.15.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bffea85eea980d8a74453e5d02a8d93028f3c34725de143085a844ebe953258a"
+checksum = "723ecff9ad04f4ad92fe1c8ca6c20d2196d9286e9c60727c4cb5511629260e9d"
 dependencies = [
 "byteorder",
 "rmp",
@@ -3535,20 +3550,6 @@ dependencies = [
 "unicode-xid",
 ]

-[[package]]
-name = "systemstat"
-version = "0.2.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a24aec24a9312c83999a28e3ef9db7e2afd5c64bf47725b758cdc1cafd5b0bd2"
-dependencies = [
- "bytesize",
- "lazy_static",
- "libc",
- "nom",
- "time 0.3.28",
- "winapi",
-]
-
 [[package]]
 name = "tempfile"
 version = "3.8.0"
@@ -7,13 +7,13 @@ args@{
    "garage_db/default"
    "garage_util/default"
    "garage_rpc/default"
+    "format_table/default"
    "garage_table/default"
    "garage_block/default"
    "garage_model/default"
    "garage_api/default"
    "garage_web/default"
    "garage/default"
-    "format_table/default"
    "k2v-client/default"
  ],
  rustPackages,
@@ -33,7 +33,7 @@ args@{
  ignoreLockHash,
 }:
 let
-  nixifiedLockHash = "d4392b23d407f7ebc20d7f5db7583847e362665c1abb09f1c1d3305205e5996d";
+  nixifiedLockHash = "5df33eefe787762bf831e92c723c153faf8d5910332dcdf2fd941fe03be59936";
  workspaceSrc = if args.workspaceSrc == null then ./. else args.workspaceSrc;
  currentLockHash = builtins.hashFile "sha256" (workspaceSrc + /Cargo.lock);
  lockHashIgnored = if ignoreLockHash
@@ -57,16 +57,16 @@ in
 {
  cargo2nixVersion = "0.11.0";
  workspace = {
-    garage_db = rustPackages.unknown.garage_db."0.8.3";
-    garage_util = rustPackages.unknown.garage_util."0.8.3";
-    garage_rpc = rustPackages.unknown.garage_rpc."0.8.3";
-    garage_table = rustPackages.unknown.garage_table."0.8.3";
-    garage_block = rustPackages.unknown.garage_block."0.8.3";
-    garage_model = rustPackages.unknown.garage_model."0.8.3";
-    garage_api = rustPackages.unknown.garage_api."0.8.3";
-    garage_web = rustPackages.unknown.garage_web."0.8.3";
-    garage = rustPackages.unknown.garage."0.8.3";
+    garage_db = rustPackages.unknown.garage_db."0.8.4";
+    garage_util = rustPackages.unknown.garage_util."0.8.4";
+    garage_rpc = rustPackages.unknown.garage_rpc."0.8.4";
    format_table = rustPackages.unknown.format_table."0.1.1";
+    garage_table = rustPackages.unknown.garage_table."0.8.4";
+    garage_block = rustPackages.unknown.garage_block."0.8.4";
+    garage_model = rustPackages.unknown.garage_model."0.8.4";
+    garage_api = rustPackages.unknown.garage_api."0.8.4";
+    garage_web = rustPackages.unknown.garage_web."0.8.4";
+    garage = rustPackages.unknown.garage."0.8.4";
    k2v-client = rustPackages.unknown.k2v-client."0.0.4";
  };
  "registry+https://github.com/rust-lang/crates.io-index".addr2line."0.21.0" = overridableMkRustCrate (profileName: rec {
@@ -1705,9 +1705,9 @@ in
    };
  });
  
-  "unknown".garage."0.8.3" = overridableMkRustCrate (profileName: rec {
+  "unknown".garage."0.8.4" = overridableMkRustCrate (profileName: rec {
    name = "garage";
-    version = "0.8.3";
+    version = "0.8.4";
    registry = "unknown";
    src = fetchCrateLocal (workspaceSrc + "/src/garage");
    features = builtins.concatLists [
@@ -1734,18 +1734,18 @@ in
      format_table = (rustPackages."unknown".format_table."0.1.1" { inherit profileName; }).out;
      futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.28" { inherit profileName; }).out;
      futures_util = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.28" { inherit profileName; }).out;
-      garage_api = (rustPackages."unknown".garage_api."0.8.3" { inherit profileName; }).out;
-      garage_block = (rustPackages."unknown".garage_block."0.8.3" { inherit profileName; }).out;
-      garage_db = (rustPackages."unknown".garage_db."0.8.3" { inherit profileName; }).out;
-      garage_model = (rustPackages."unknown".garage_model."0.8.3" { inherit profileName; }).out;
-      garage_rpc = (rustPackages."unknown".garage_rpc."0.8.3" { inherit profileName; }).out;
-      garage_table = (rustPackages."unknown".garage_table."0.8.3" { inherit profileName; }).out;
-      garage_util = (rustPackages."unknown".garage_util."0.8.3" { inherit profileName; }).out;
-      garage_web = (rustPackages."unknown".garage_web."0.8.3" { inherit profileName; }).out;
+      garage_api = (rustPackages."unknown".garage_api."0.8.4" { inherit profileName; }).out;
+      garage_block = (rustPackages."unknown".garage_block."0.8.4" { inherit profileName; }).out;
+      garage_db = (rustPackages."unknown".garage_db."0.8.4" { inherit profileName; }).out;
+      garage_model = (rustPackages."unknown".garage_model."0.8.4" { inherit profileName; }).out;
+      garage_rpc = (rustPackages."unknown".garage_rpc."0.8.4" { inherit profileName; }).out;
+      garage_table = (rustPackages."unknown".garage_table."0.8.4" { inherit profileName; }).out;
+      garage_util = (rustPackages."unknown".garage_util."0.8.4" { inherit profileName; }).out;
+      garage_web = (rustPackages."unknown".garage_web."0.8.4" { inherit profileName; }).out;
      git_version = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".git-version."0.3.5" { inherit profileName; }).out;
      hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out;
      sodiumoxide = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".kuska-sodiumoxide."0.2.5-0" { inherit profileName; }).out;
-      netapp = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".netapp."0.5.3" { inherit profileName; }).out;
+      netapp = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".netapp."0.5.2" { inherit profileName; }).out;
      opentelemetry = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry."0.17.0" { inherit profileName; }).out;
      ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "opentelemetry_otlp" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry-otlp."0.10.0" { inherit profileName; }).out;
      ${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/metrics" || rootFeatures' ? "garage/opentelemetry-prometheus" then "opentelemetry_prometheus" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry-prometheus."0.10.0" { inherit profileName; }).out;
@@ -1777,9 +1777,9 @@ in
    };
  });
  
-  "unknown".garage_api."0.8.3" = overridableMkRustCrate (profileName: rec {
+  "unknown".garage_api."0.8.4" = overridableMkRustCrate (profileName: rec {
    name = "garage_api";
-    version = "0.8.3";
+    version = "0.8.4";
    registry = "unknown";
    src = fetchCrateLocal (workspaceSrc + "/src/api");
    features = builtins.concatLists [
@@ -1798,11 +1798,11 @@ in
      form_urlencoded = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".form_urlencoded."1.2.0" { inherit profileName; }).out;
      futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.28" { inherit profileName; }).out;
      futures_util = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.28" { inherit profileName; }).out;
-      garage_block = (rustPackages."unknown".garage_block."0.8.3" { inherit profileName; }).out;
-      garage_model = (rustPackages."unknown".garage_model."0.8.3" { inherit profileName; }).out;
-      garage_rpc = (rustPackages."unknown".garage_rpc."0.8.3" { inherit profileName; }).out;
-      garage_table = (rustPackages."unknown".garage_table."0.8.3" { inherit profileName; }).out;
-      garage_util = (rustPackages."unknown".garage_util."0.8.3" { inherit profileName; }).out;
+      garage_block = (rustPackages."unknown".garage_block."0.8.4" { inherit profileName; }).out;
+      garage_model = (rustPackages."unknown".garage_model."0.8.4" { inherit profileName; }).out;
+      garage_rpc = (rustPackages."unknown".garage_rpc."0.8.4" { inherit profileName; }).out;
+      garage_table = (rustPackages."unknown".garage_table."0.8.4" { inherit profileName; }).out;
+      garage_util = (rustPackages."unknown".garage_util."0.8.4" { inherit profileName; }).out;
      hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out;
      hmac = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hmac."0.12.1" { inherit profileName; }).out;
      http = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.9" { inherit profileName; }).out;
@@ -1831,9 +1831,9 @@ in
    };
  });
  
-  "unknown".garage_block."0.8.3" = overridableMkRustCrate (profileName: rec {
+  "unknown".garage_block."0.8.4" = overridableMkRustCrate (profileName: rec {
    name = "garage_block";
-    version = "0.8.3";
+    version = "0.8.4";
    registry = "unknown";
    src = fetchCrateLocal (workspaceSrc + "/src/block");
    features = builtins.concatLists [
@@ -1844,12 +1844,13 @@ in
      async_compression = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".async-compression."0.4.1" { inherit profileName; }).out;
      async_trait = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".async-trait."0.1.73" { profileName = "__noProfile"; }).out;
      bytes = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.4.0" { inherit profileName; }).out;
+      bytesize = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytesize."1.3.0" { inherit profileName; }).out;
      futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.28" { inherit profileName; }).out;
      futures_util = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.28" { inherit profileName; }).out;
-      garage_db = (rustPackages."unknown".garage_db."0.8.3" { inherit profileName; }).out;
-      garage_rpc = (rustPackages."unknown".garage_rpc."0.8.3" { inherit profileName; }).out;
-      garage_table = (rustPackages."unknown".garage_table."0.8.3" { inherit profileName; }).out;
-      garage_util = (rustPackages."unknown".garage_util."0.8.3" { inherit profileName; }).out;
+      garage_db = (rustPackages."unknown".garage_db."0.8.4" { inherit profileName; }).out;
+      garage_rpc = (rustPackages."unknown".garage_rpc."0.8.4" { inherit profileName; }).out;
+      garage_table = (rustPackages."unknown".garage_table."0.8.4" { inherit profileName; }).out;
+      garage_util = (rustPackages."unknown".garage_util."0.8.4" { inherit profileName; }).out;
      hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out;
      opentelemetry = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry."0.17.0" { inherit profileName; }).out;
      rand = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand."0.8.5" { inherit profileName; }).out;
@@ -1862,9 +1863,9 @@ in
    };
  });
  
-  "unknown".garage_db."0.8.3" = overridableMkRustCrate (profileName: rec {
+  "unknown".garage_db."0.8.4" = overridableMkRustCrate (profileName: rec {
    name = "garage_db";
-    version = "0.8.3";
+    version = "0.8.4";
    registry = "unknown";
    src = fetchCrateLocal (workspaceSrc + "/src/db");
    features = builtins.concatLists [
@@ -1894,9 +1895,9 @@ in
    };
  });
  
-  "unknown".garage_model."0.8.3" = overridableMkRustCrate (profileName: rec {
+  "unknown".garage_model."0.8.4" = overridableMkRustCrate (profileName: rec {
    name = "garage_model";
-    version = "0.8.3";
+    version = "0.8.4";
    registry = "unknown";
    src = fetchCrateLocal (workspaceSrc + "/src/model");
    features = builtins.concatLists [
@@ -1911,16 +1912,17 @@ in
      async_trait = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".async-trait."0.1.73" { profileName = "__noProfile"; }).out;
      base64 = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".base64."0.21.3" { inherit profileName; }).out;
      blake2 = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".blake2."0.10.6" { inherit profileName; }).out;
+      chrono = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".chrono."0.4.26" { inherit profileName; }).out;
      err_derive = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".err-derive."0.3.1" { profileName = "__noProfile"; }).out;
      futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.28" { inherit profileName; }).out;
      futures_util = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.28" { inherit profileName; }).out;
-      garage_block = (rustPackages."unknown".garage_block."0.8.3" { inherit profileName; }).out;
-      garage_db = (rustPackages."unknown".garage_db."0.8.3" { inherit profileName; }).out;
-      garage_rpc = (rustPackages."unknown".garage_rpc."0.8.3" { inherit profileName; }).out;
-      garage_table = (rustPackages."unknown".garage_table."0.8.3" { inherit profileName; }).out;
-      garage_util = (rustPackages."unknown".garage_util."0.8.3" { inherit profileName; }).out;
+      garage_block = (rustPackages."unknown".garage_block."0.8.4" { inherit profileName; }).out;
+      garage_db = (rustPackages."unknown".garage_db."0.8.4" { inherit profileName; }).out;
+      garage_rpc = (rustPackages."unknown".garage_rpc."0.8.4" { inherit profileName; }).out;
+      garage_table = (rustPackages."unknown".garage_table."0.8.4" { inherit profileName; }).out;
+      garage_util = (rustPackages."unknown".garage_util."0.8.4" { inherit profileName; }).out;
      hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out;
-      netapp = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".netapp."0.5.3" { inherit profileName; }).out;
+      netapp = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".netapp."0.5.2" { inherit profileName; }).out;
      opentelemetry = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry."0.17.0" { inherit profileName; }).out;
      rand = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand."0.8.5" { inherit profileName; }).out;
      serde = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde."1.0.188" { inherit profileName; }).out;
@@ -1931,9 +1933,9 @@ in
    };
  });
  
-  "unknown".garage_rpc."0.8.3" = overridableMkRustCrate (profileName: rec {
+  "unknown".garage_rpc."0.8.4" = overridableMkRustCrate (profileName: rec {
    name = "garage_rpc";
-    version = "0.8.3";
+    version = "0.8.4";
    registry = "unknown";
    src = fetchCrateLocal (workspaceSrc + "/src/rpc");
    features = builtins.concatLists [
@@ -1952,17 +1954,19 @@ in
      bytes = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.4.0" { inherit profileName; }).out;
      bytesize = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytesize."1.3.0" { inherit profileName; }).out;
      ${ if rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/err-derive" then "err_derive" else null } = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".err-derive."0.3.1" { profileName = "__noProfile"; }).out;
+      format_table = (rustPackages."unknown".format_table."0.1.1" { inherit profileName; }).out;
      futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.28" { inherit profileName; }).out;
      futures_util = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.28" { inherit profileName; }).out;
-      garage_db = (rustPackages."unknown".garage_db."0.8.3" { inherit profileName; }).out;
-      garage_util = (rustPackages."unknown".garage_util."0.8.3" { inherit profileName; }).out;
+      garage_db = (rustPackages."unknown".garage_db."0.8.4" { inherit profileName; }).out;
+      garage_util = (rustPackages."unknown".garage_util."0.8.4" { inherit profileName; }).out;
      gethostname = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".gethostname."0.4.3" { inherit profileName; }).out;
      hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out;
      itertools = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".itertools."0.10.5" { inherit profileName; }).out;
      ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/k8s-openapi" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "k8s_openapi" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".k8s-openapi."0.16.0" { inherit profileName; }).out;
      ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "kube" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".kube."0.75.0" { inherit profileName; }).out;
      sodiumoxide = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".kuska-sodiumoxide."0.2.5-0" { inherit profileName; }).out;
-      netapp = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".netapp."0.5.3" { inherit profileName; }).out;
+      netapp = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".netapp."0.5.2" { inherit profileName; }).out;
+      nix = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".nix."0.27.1" { inherit profileName; }).out;
      opentelemetry = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry."0.17.0" { inherit profileName; }).out;
      pnet_datalink = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".pnet_datalink."0.33.0" { inherit profileName; }).out;
      rand = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand."0.8.5" { inherit profileName; }).out;
@@ -1971,16 +1975,15 @@ in
      serde = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde."1.0.188" { inherit profileName; }).out;
      serde_bytes = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde_bytes."0.11.12" { inherit profileName; }).out;
      serde_json = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde_json."1.0.105" { inherit profileName; }).out;
-      systemstat = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".systemstat."0.2.3" { inherit profileName; }).out;
      tokio = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.32.0" { inherit profileName; }).out;
      tokio_stream = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio-stream."0.1.14" { inherit profileName; }).out;
      tracing = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing."0.1.37" { inherit profileName; }).out;
    };
  });
  
-  "unknown".garage_table."0.8.3" = overridableMkRustCrate (profileName: rec {
+  "unknown".garage_table."0.8.4" = overridableMkRustCrate (profileName: rec {
    name = "garage_table";
-    version = "0.8.3";
+    version = "0.8.4";
    registry = "unknown";
    src = fetchCrateLocal (workspaceSrc + "/src/table");
    dependencies = {
@@ -1989,9 +1992,9 @@ in
      bytes = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.4.0" { inherit profileName; }).out;
      futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.28" { inherit profileName; }).out;
      futures_util = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.28" { inherit profileName; }).out;
-      garage_db = (rustPackages."unknown".garage_db."0.8.3" { inherit profileName; }).out;
-      garage_rpc = (rustPackages."unknown".garage_rpc."0.8.3" { inherit profileName; }).out;
-      garage_util = (rustPackages."unknown".garage_util."0.8.3" { inherit profileName; }).out;
+      garage_db = (rustPackages."unknown".garage_db."0.8.4" { inherit profileName; }).out;
+      garage_rpc = (rustPackages."unknown".garage_rpc."0.8.4" { inherit profileName; }).out;
+      garage_util = (rustPackages."unknown".garage_util."0.8.4" { inherit profileName; }).out;
      hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out;
      hexdump = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hexdump."0.1.1" { inherit profileName; }).out;
      opentelemetry = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry."0.17.0" { inherit profileName; }).out;
@@ -2003,9 +2006,9 @@ in
    };
  });
  
-  "unknown".garage_util."0.8.3" = overridableMkRustCrate (profileName: rec {
+  "unknown".garage_util."0.8.4" = overridableMkRustCrate (profileName: rec {
    name = "garage_util";
-    version = "0.8.3";
+    version = "0.8.4";
    registry = "unknown";
    src = fetchCrateLocal (workspaceSrc + "/src/util");
    features = builtins.concatLists [
@@ -2016,20 +2019,21 @@ in
      async_trait = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".async-trait."0.1.73" { profileName = "__noProfile"; }).out;
      blake2 = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".blake2."0.10.6" { inherit profileName; }).out;
      bytes = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.4.0" { inherit profileName; }).out;
+      bytesize = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytesize."1.3.0" { inherit profileName; }).out;
      chrono = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".chrono."0.4.26" { inherit profileName; }).out;
      digest = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".digest."0.10.7" { inherit profileName; }).out;
      err_derive = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".err-derive."0.3.1" { profileName = "__noProfile"; }).out;
      futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.28" { inherit profileName; }).out;
-      garage_db = (rustPackages."unknown".garage_db."0.8.3" { inherit profileName; }).out;
+      garage_db = (rustPackages."unknown".garage_db."0.8.4" { inherit profileName; }).out;
      hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out;
      hexdump = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hexdump."0.1.1" { inherit profileName; }).out;
      http = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.9" { inherit profileName; }).out;
      hyper = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hyper."0.14.27" { inherit profileName; }).out;
      lazy_static = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".lazy_static."1.4.0" { inherit profileName; }).out;
-      netapp = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".netapp."0.5.3" { inherit profileName; }).out;
+      netapp = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".netapp."0.5.2" { inherit profileName; }).out;
      opentelemetry = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry."0.17.0" { inherit profileName; }).out;
      rand = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand."0.8.5" { inherit profileName; }).out;
-      rmp_serde = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rmp-serde."1.1.2" { inherit profileName; }).out;
+      rmp_serde = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rmp-serde."0.15.5" { inherit profileName; }).out;
      serde = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde."1.0.188" { inherit profileName; }).out;
      serde_json = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde_json."1.0.105" { inherit profileName; }).out;
      sha2 = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".sha2."0.10.7" { inherit profileName; }).out;
@@ -2046,18 +2050,18 @@ in
    };
  });
  
-  "unknown".garage_web."0.8.3" = overridableMkRustCrate (profileName: rec {
+  "unknown".garage_web."0.8.4" = overridableMkRustCrate (profileName: rec {
    name = "garage_web";
-    version = "0.8.3";
+    version = "0.8.4";
    registry = "unknown";
    src = fetchCrateLocal (workspaceSrc + "/src/web");
    dependencies = {
      err_derive = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".err-derive."0.3.1" { profileName = "__noProfile"; }).out;
      futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.28" { inherit profileName; }).out;
-      garage_api = (rustPackages."unknown".garage_api."0.8.3" { inherit profileName; }).out;
-      garage_model = (rustPackages."unknown".garage_model."0.8.3" { inherit profileName; }).out;
-      garage_table = (rustPackages."unknown".garage_table."0.8.3" { inherit profileName; }).out;
-      garage_util = (rustPackages."unknown".garage_util."0.8.3" { inherit profileName; }).out;
+      garage_api = (rustPackages."unknown".garage_api."0.8.4" { inherit profileName; }).out;
+      garage_model = (rustPackages."unknown".garage_model."0.8.4" { inherit profileName; }).out;
+      garage_table = (rustPackages."unknown".garage_table."0.8.4" { inherit profileName; }).out;
+      garage_util = (rustPackages."unknown".garage_util."0.8.4" { inherit profileName; }).out;
      http = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.9" { inherit profileName; }).out;
      hyper = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hyper."0.14.27" { inherit profileName; }).out;
      opentelemetry = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry."0.17.0" { inherit profileName; }).out;
@@ -2957,7 +2961,7 @@ in
    src = fetchCratesIo { inherit name version; sha256 = "b4668fb0ea861c1df094127ac5f1da3409a82116a4ba74fca2e58ef927159bb3"; };
    features = builtins.concatLists [
      [ "default" ]
-      (lib.optional (rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" || rootFeatures' ? "garage_db/cli" || rootFeatures' ? "garage_db/pretty_env_logger") "extra_traits")
+      [ "extra_traits" ]
      [ "std" ]
    ];
  });
@@ -3199,11 +3203,11 @@ in
    src = fetchCratesIo { inherit name version; sha256 = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a"; };
  });
  
-  "registry+https://github.com/rust-lang/crates.io-index".netapp."0.5.3" = overridableMkRustCrate (profileName: rec {
+  "registry+https://github.com/rust-lang/crates.io-index".netapp."0.5.2" = overridableMkRustCrate (profileName: rec {
    name = "netapp";
-    version = "0.5.3";
+    version = "0.5.2";
    registry = "registry+https://github.com/rust-lang/crates.io-index";
-    src = fetchCratesIo { inherit name version; sha256 = "0eb28e89e7b21d3fdf2b46e2a580ccaab68f469f2e5c27db74f61994be90b558"; };
+    src = fetchCratesIo { inherit name version; sha256 = "4ffe47ac46d3b2ce2f736a70865492df082e042eb2bfdddfca3b8dd66bd9469d"; };
    features = builtins.concatLists [
      [ "default" ]
      [ "opentelemetry" ]
@@ -3225,7 +3229,7 @@ in
      opentelemetry_contrib = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry-contrib."0.9.0" { inherit profileName; }).out;
      pin_project = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".pin-project."1.1.3" { inherit profileName; }).out;
      rand = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand."0.8.5" { inherit profileName; }).out;
-      rmp_serde = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rmp-serde."1.1.2" { inherit profileName; }).out;
+      rmp_serde = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rmp-serde."0.15.5" { inherit profileName; }).out;
      serde = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde."1.0.188" { inherit profileName; }).out;
      tokio = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.32.0" { inherit profileName; }).out;
      tokio_stream = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio-stream."0.1.14" { inherit profileName; }).out;
@@ -3233,6 +3237,21 @@ in
    };
  });
  
+  "registry+https://github.com/rust-lang/crates.io-index".nix."0.27.1" = overridableMkRustCrate (profileName: rec {
+    name = "nix";
+    version = "0.27.1";
+    registry = "registry+https://github.com/rust-lang/crates.io-index";
+    src = fetchCratesIo { inherit name version; sha256 = "2eb04e9c688eff1c89d72b407f168cf79bb9e867a9d3323ed6c01519eb9cc053"; };
+    features = builtins.concatLists [
+      [ "fs" ]
+    ];
+    dependencies = {
+      bitflags = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bitflags."2.4.0" { inherit profileName; }).out;
+      cfg_if = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".cfg-if."1.0.0" { inherit profileName; }).out;
+      libc = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.147" { inherit profileName; }).out;
+    };
+  });
+  
  "registry+https://github.com/rust-lang/crates.io-index".no-std-net."0.6.0" = overridableMkRustCrate (profileName: rec {
    name = "no-std-net";
    version = "0.6.0";
@@ -4275,11 +4294,11 @@ in
    };
  });
  
-  "registry+https://github.com/rust-lang/crates.io-index".rmp-serde."1.1.2" = overridableMkRustCrate (profileName: rec {
+  "registry+https://github.com/rust-lang/crates.io-index".rmp-serde."0.15.5" = overridableMkRustCrate (profileName: rec {
    name = "rmp-serde";
-    version = "1.1.2";
+    version = "0.15.5";
    registry = "registry+https://github.com/rust-lang/crates.io-index";
-    src = fetchCratesIo { inherit name version; sha256 = "bffea85eea980d8a74453e5d02a8d93028f3c34725de143085a844ebe953258a"; };
+    src = fetchCratesIo { inherit name version; sha256 = "723ecff9ad04f4ad92fe1c8ca6c20d2196d9286e9c60727c4cb5511629260e9d"; };
    dependencies = {
      byteorder = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".byteorder."1.4.3" { inherit profileName; }).out;
      rmp = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rmp."0.8.12" { inherit profileName; }).out;
@@ -5002,21 +5021,6 @@ in
    };
  });
  
-  "registry+https://github.com/rust-lang/crates.io-index".systemstat."0.2.3" = overridableMkRustCrate (profileName: rec {
-    name = "systemstat";
-    version = "0.2.3";
-    registry = "registry+https://github.com/rust-lang/crates.io-index";
-    src = fetchCratesIo { inherit name version; sha256 = "a24aec24a9312c83999a28e3ef9db7e2afd5c64bf47725b758cdc1cafd5b0bd2"; };
-    dependencies = {
-      bytesize = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytesize."1.3.0" { inherit profileName; }).out;
-      lazy_static = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".lazy_static."1.4.0" { inherit profileName; }).out;
-      libc = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.147" { inherit profileName; }).out;
-      ${ if hostPlatform.parsed.kernel.name == "linux" || hostPlatform.parsed.kernel.name == "android" then "nom" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".nom."7.1.3" { inherit profileName; }).out;
-      time = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".time."0.3.28" { inherit profileName; }).out;
-      ${ if hostPlatform.isWindows then "winapi" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".winapi."0.3.9" { inherit profileName; }).out;
-    };
-  });
-  
  "registry+https://github.com/rust-lang/crates.io-index".tempfile."3.8.0" = overridableMkRustCrate (profileName: rec {
    name = "tempfile";
    version = "3.8.0";
@@ -6036,7 +6040,6 @@ in
      [ "ntsecapi" ]
      [ "ntstatus" ]
      (lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "objbase")
-      [ "pdh" ]
      [ "processenv" ]
      (lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sled" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/sled" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sled") "processthreadsapi")
      [ "profileapi" ]
@@ -6050,7 +6053,6 @@ in
      [ "winerror" ]
      [ "winnt" ]
      [ "winsock2" ]
-      [ "ws2def" ]
      [ "ws2ipdef" ]
      [ "ws2tcpip" ]
      [ "wtypesbase" ]
@@ -18,14 +18,14 @@ default-members = ["src/garage"]

 [workspace.dependencies]
 format_table = { version = "0.1.1", path = "src/format-table" }
-garage_api = { version = "0.8.3", path = "src/api" }
-garage_block = { version = "0.8.3", path = "src/block" }
-garage_db = { version = "0.8.3", path = "src/db", default-features = false }
-garage_model = { version = "0.8.3", path = "src/model", default-features = false }
-garage_rpc = { version = "0.8.3", path = "src/rpc" }
-garage_table = { version = "0.8.3", path = "src/table" }
-garage_util = { version = "0.8.3", path = "src/util" }
-garage_web = { version = "0.8.3", path = "src/web" }
+garage_api = { version = "0.8.4", path = "src/api" }
+garage_block = { version = "0.8.4", path = "src/block" }
+garage_db = { version = "0.8.4", path = "src/db", default-features = false }
+garage_model = { version = "0.8.4", path = "src/model", default-features = false }
+garage_rpc = { version = "0.8.4", path = "src/rpc" }
+garage_table = { version = "0.8.4", path = "src/table" }
+garage_util = { version = "0.8.4", path = "src/util" }
+garage_web = { version = "0.8.4", path = "src/web" }
 k2v-client = { version = "0.0.4", path = "src/k2v-client" }

 [profile.dev]
@@ -75,16 +75,11 @@ to store 2 TB of data in total.

 - For the metadata storage, Garage does not do checksumming and integrity
  verification on its own. If you are afraid of bitrot/data corruption,
-  put your metadata directory on a BTRFS partition. Otherwise, just use regular
+  put your metadata directory on a ZFS or BTRFS partition. Otherwise, just use regular
  EXT4 or XFS.

- Having a single server with several storage drives is currently not very well
-  supported in Garage ([#218](https://git.deuxfleurs.fr/Deuxfleurs/garage/issues/218)).
-  For an easy setup, just put all your drives in a RAID0 or a ZFS RAIDZ array.
-  If you're adventurous, you can try to format each of your disk as
-  a separate XFS partition, and then run one `garage` daemon per disk drive,
-  or use something like [`mergerfs`](https://github.com/trapexit/mergerfs) to merge
-  all your disks in a single union filesystem that spreads load over them.
+- Servers with multiple HDDs are supported natively by Garage without resorting
+  to RAID, see [our dedicated documentation page](@/documentation/operations/multi-hdd.md).

 ## Get a Docker image

@@ -91,6 +91,16 @@ is definitely lost, then there is no other choice than to declare your S3 object
 as unrecoverable, and to delete them properly from the data store. This can be done
 using the `garage block purge` command.

+## Rebalancing data directories
+
+In [multi-HDD setups](@/documentation/operations/multi-hdd.md), to ensure that
+data blocks are well balanced between storage locations, you may run a
+rebalance operation using `garage repair rebalance`. This is usefull when
+adding storage locations or when capacities of the storage locations have been
+changed.  Once this is finished, Garage will know for each block of a single
+possible location where it can be, which can increase access speed.  This
+operation will also move out all data from locations marked as read-only.
+

 # Metadata operations

@@ -114,4 +124,3 @@ in your cluster, you can run one of the following repair procedures:

 - `garage repair versions`: checks that all versions belong to a non-deleted object, and purges any orphan version
 - `garage repair block_refs`: checks that all block references belong to a non-deleted object version, and purges any orphan block reference (this will then allow the blocks to be garbage-collected)
-
@@ -9,18 +9,30 @@ a certain capacity, or a gateway node that does not store data and is only
 used as an API entry point for faster cluster access.
 An introduction to building cluster layouts can be found in the [production deployment](@/documentation/cookbook/real-world.md) page.

+In Garage, all of the data that can be stored in a given cluster is divided
+into slices which we call *partitions*. Each partition is stored by
+one or several nodes in the cluster
+(see [`replication_mode`](@/documentation/reference-manual/configuration.md#replication-mode)).
+The layout determines the correspondence between these partition,
+which exist on a logical level, and actual storage nodes.
+
 ## How cluster layouts work in Garage

-In Garage, a cluster layout is composed of the following components:
+A cluster layout is composed of the following components:

- a table of roles assigned to nodes
+- a table of roles assigned to nodes, defined by the user
+- an optimal assignation of partitions to nodes, computed by an algorithm that is ran once when calling `garage layout apply` or the ApplyClusterLayout API endpoint
 - a version number

 Garage nodes will always use the cluster layout with the highest version number.

 Garage nodes also maintain and synchronize between them a set of proposed role
 changes that haven't yet been applied. These changes will be applied (or
-canceled) in the next version of the layout
+canceled) in the next version of the layout.
+
+All operations on the layout can be realized using the `garage` CLI or using the
+[administration API endpoint](@/documentation/reference-manual/admin-api.md).
+We give here a description of CLI commands, the admin API semantics are very similar.

 The following commands insert modifications to the set of proposed role changes
 for the next layout version (but they do not create the new layout immediately):
@@ -51,7 +63,7 @@ commands will fail otherwise.

 ## Warnings about Garage cluster layout management

-**Warning: never make several calls to `garage layout apply` or `garage layout
+**⚠️ Never make several calls to `garage layout apply` or `garage layout
 revert` with the same value of the `--version` flag. Doing so can lead to the
 creation of several different layouts with the same version number, in which
 case your Garage cluster will become inconsistent until fixed.** If a call to
@@ -65,13 +77,198 @@ shell, you shouldn't have much issues as long as you run commands one after
 the other and take care of checking the output of `garage layout show`
 before applying any changes.

-If you are using the `garage` CLI to script layout changes, follow the following recommendations:
+If you are using the `garage` CLI or the admin API to script layout changes,
+follow the following recommendations:

- Make all of your `garage` CLI calls to the same RPC host. Do not use the
-  `garage` CLI to connect to individual nodes to send them each a piece of the
-  layout changes you are making, as the changes propagate asynchronously
-  between nodes and might not all be taken into account at the time when the
-  new layout is applied.
+- If using the CLI, make all of your `garage` CLI calls to the same RPC host.
+  If using the admin API, make all of your API calls to the same Garage node. Do
+  not connect to individual nodes to send them each a piece of the layout changes
+  you are making, as the changes propagate asynchronously between nodes and might
+  not all be taken into account at the time when the new layout is applied.

- **Only call `garage layout apply` once**, and call it **strictly after** all
-  of the `layout assign` and `layout remove` commands have returned.
+- **Only call `garage layout apply`/ApplyClusterLayout once**, and call it
+  **strictly after** all of the `layout assign` and `layout remove`
+  commands/UpdateClusterLayout API calls have returned.
+
+
+## Understanding unexpected layout calculations
+
+When adding, removing or modifying nodes in a cluster layout, sometimes
+unexpected assigntations of partitions to node can occur. These assignations
+are in fact normal and logical, given the objectives of the algorihtm.  Indeed,
+**the layout algorithm prioritizes moving less data between nodes over the fact
+of achieving equal distribution of load. It also tries to use all links between
+pairs of nodes in equal proportions when moving data.**  This section presents
+two examples and illustrates how one can control Garage's behavior to obtain
+the desired results.
+
+### Example 1
+
+In this example, a cluster is originally composed of 3 nodes in 3 different
+zones (data centers).  The three nodes are of equal capacity, therefore they
+are all fully exploited and all store a copy of all of the data in the cluster.
+
+Then, a fourth node of the same size is added in the datacenter `dc1`.
+As illustrated by the following, **Garage will by default not store any data on the new node**:
+
+```
+$ garage layout show
+==== CURRENT CLUSTER LAYOUT ====
+ID                Tags   Zone  Capacity   Usable capacity
+b10c110e4e854e5a  node1  dc1   1000.0 MB  1000.0 MB (100.0%)
+a235ac7695e0c54d  node2  dc2   1000.0 MB  1000.0 MB (100.0%)
+62b218d848e86a64  node3  dc3   1000.0 MB  1000.0 MB (100.0%)
+
+Zone redundancy: maximum
+
+Current cluster layout version: 6
+
+==== STAGED ROLE CHANGES ====
+ID                Tags   Zone  Capacity
+a11c7cf18af29737  node4  dc1   1000.0 MB
+
+
+==== NEW CLUSTER LAYOUT AFTER APPLYING CHANGES ====
+ID                Tags   Zone  Capacity   Usable capacity
+b10c110e4e854e5a  node1  dc1   1000.0 MB  1000.0 MB (100.0%)
+a11c7cf18af29737  node4  dc1   1000.0 MB  0 B (0.0%)
+a235ac7695e0c54d  node2  dc2   1000.0 MB  1000.0 MB (100.0%)
+62b218d848e86a64  node3  dc3   1000.0 MB  1000.0 MB (100.0%)
+
+Zone redundancy: maximum
+
+==== COMPUTATION OF A NEW PARTITION ASSIGNATION ====
+
+Partitions are replicated 3 times on at least 3 distinct zones.
+
+Optimal partition size:                     3.9 MB (3.9 MB in previous layout)
+Usable capacity / total cluster capacity:   3.0 GB / 4.0 GB (75.0 %)
+Effective capacity (replication factor 3):  1000.0 MB
+
+A total of 0 new copies of partitions need to be transferred.
+
+dc1                 Tags   Partitions        Capacity   Usable capacity
+  b10c110e4e854e5a  node1  256 (0 new)       1000.0 MB  1000.0 MB (100.0%)
+  a11c7cf18af29737  node4  0 (0 new)         1000.0 MB  0 B (0.0%)
+  TOTAL                    256 (256 unique)  2.0 GB     1000.0 MB (50.0%)
+
+dc2                 Tags   Partitions        Capacity   Usable capacity
+  a235ac7695e0c54d  node2  256 (0 new)       1000.0 MB  1000.0 MB (100.0%)
+  TOTAL                    256 (256 unique)  1000.0 MB  1000.0 MB (100.0%)
+
+dc3                 Tags   Partitions        Capacity   Usable capacity
+  62b218d848e86a64  node3  256 (0 new)       1000.0 MB  1000.0 MB (100.0%)
+  TOTAL                    256 (256 unique)  1000.0 MB  1000.0 MB (100.0%)
+```
+
+While unexpected, this is logical because of the following facts:
+
+- storing some data on the new node does not help increase the total quantity
+  of data that can be stored on the cluster, as the two other zones (`dc2` and
+  `dc3`) still need to store a full copy of everything, and their capacity is
+  still the same;
+
+- there is therefore no need to move any data on the new node as this would be pointless;
+
+- moving data to the new node has a cost which the algorithm decides to not pay if not necessary.
+
+This distribution of data can however not be what the administrator wanted: if
+they added a new node to `dc1`, it might be because the existing node is too
+slow, and they wish to divide its load by half. In that case, what they need to
+do to force Garage to distribute the data between the two nodes is to attribute
+only half of the capacity to each node in `dc1` (in our example, 500M instead of 1G).
+In that case, Garage would determine that to be able to store 1G in total, it
+would need to store 500M on the old node and 500M on the added one.
+
+
+### Example 2
+
+The following example is a slightly different scenario, where `dc1` had two
+nodes that were used at 50%, and `dc2` and `dc3` each have one node that is
+100% used. All node capacities are the same.
+
+Then, a node from `dc1` is moved into `dc3`. One could expect that the roles of
+`dc1` and `dc3` would simply be swapped: the remaining node in `dc1` would be
+used at 100%, and the two nodes now in `dc3` would be used at 50%. Instead,
+this happens:
+
+```
+==== CURRENT CLUSTER LAYOUT ====
+ID                Tags   Zone  Capacity   Usable capacity
+b10c110e4e854e5a  node1  dc1   1000.0 MB  500.0 MB (50.0%)
+a11c7cf18af29737  node4  dc1   1000.0 MB  500.0 MB (50.0%)
+a235ac7695e0c54d  node2  dc2   1000.0 MB  1000.0 MB (100.0%)
+62b218d848e86a64  node3  dc3   1000.0 MB  1000.0 MB (100.0%)
+
+Zone redundancy: maximum
+
+Current cluster layout version: 8
+
+==== STAGED ROLE CHANGES ====
+ID                Tags   Zone  Capacity
+a11c7cf18af29737  node4  dc3   1000.0 MB
+
+
+==== NEW CLUSTER LAYOUT AFTER APPLYING CHANGES ====
+ID                Tags   Zone  Capacity   Usable capacity
+b10c110e4e854e5a  node1  dc1   1000.0 MB  1000.0 MB (100.0%)
+a235ac7695e0c54d  node2  dc2   1000.0 MB  1000.0 MB (100.0%)
+62b218d848e86a64  node3  dc3   1000.0 MB  753.9 MB (75.4%)
+a11c7cf18af29737  node4  dc3   1000.0 MB  246.1 MB (24.6%)
+
+Zone redundancy: maximum
+
+==== COMPUTATION OF A NEW PARTITION ASSIGNATION ====
+
+Partitions are replicated 3 times on at least 3 distinct zones.
+
+Optimal partition size:                     3.9 MB (3.9 MB in previous layout)
+Usable capacity / total cluster capacity:   3.0 GB / 4.0 GB (75.0 %)
+Effective capacity (replication factor 3):  1000.0 MB
+
+A total of 128 new copies of partitions need to be transferred.
+
+dc1                 Tags   Partitions        Capacity   Usable capacity
+  b10c110e4e854e5a  node1  256 (128 new)     1000.0 MB  1000.0 MB (100.0%)
+  TOTAL                    256 (256 unique)  1000.0 MB  1000.0 MB (100.0%)
+
+dc2                 Tags   Partitions        Capacity   Usable capacity
+  a235ac7695e0c54d  node2  256 (0 new)       1000.0 MB  1000.0 MB (100.0%)
+  TOTAL                    256 (256 unique)  1000.0 MB  1000.0 MB (100.0%)
+
+dc3                 Tags   Partitions        Capacity   Usable capacity
+  62b218d848e86a64  node3  193 (0 new)       1000.0 MB  753.9 MB (75.4%)
+  a11c7cf18af29737  node4  63 (0 new)        1000.0 MB  246.1 MB (24.6%)
+  TOTAL                    256 (256 unique)  2.0 GB     1000.0 MB (50.0%)
+```
+
+As we can see, the node that was moved to `dc3` (node4) is only used at 25% (approximatively),
+whereas the node that was already in `dc3` (node3) is used at 75%.
+
+This can be explained by the following:
+
+- node1 will now be the only node remaining in `dc1`, thus it has to store all
+  of the data in the cluster. Since it was storing only half of it before, it has
+  to retrieve the other half from other nodes in the cluster.
+
+- The data which it does not have is entirely stored by the other node that was
+  in `dc1` and that is now in `dc3` (node4). There is also a copy of it on node2
+  and node3 since both these nodes have a copy of everything.
+
+- node3 and node4 are the two nodes that will now be in a datacenter that is
+ under-utilized (`dc3`), this means that those are the two candidates from which
+ data can be removed to be moved to node1.
+
+- Garage will move data in equal proportions from all possible sources, in this
+  case it means that it will tranfer 25% of the entire data set from node3 to
+  node1 and another 25% from node4 to node1.
+
+This explains why node3 ends with 75% utilization (100% from before minus 25%
+that is moved to node1), and node4 ends with 25% (50% from before minus 25%
+that is moved to node1).
+
+This illustrates the second principle of the layout computation: **if there is
+a choice in moving data out of some nodes, then all links between pairs of
+nodes are used in equal proportions** (this is approximately true, there is
+randomness in the algorihtm to achieve this so there might be some small
+fluctuations, as we see above).
@@ -0,0 +1,101 @@
+++
+title = "Multi-HDD support"
+weight = 15
+++
+
+
+Since v0.9, Garage natively supports nodes that have several storage drives
+for storing data blocks (not for metadata storage).
+
+## Initial setup
+
+To set up a new Garage storage node with multiple HDDs,
+format and mount all your drives in different directories,
+and use a Garage configuration as follows:
+
+```toml
+data_dir = [
+    { path = "/path/to/hdd1", capacity = "2T" },
+    { path = "/path/to/hdd2", capacity = "4T" },
+]
+```
+
+Garage will automatically balance all blocks stored by the node
+among the different specified directories, proportionnally to the
+specified capacities.
+
+## Updating the list of storage locations
+
+If you add new storage locations to your `data_dir`,
+Garage will not rebalance existing data between storage locations.
+Newly written blocks will be balanced proportionnally to the specified capacities,
+and existing data may be moved between drives to improve balancing,
+but only opportunistically when a data block is re-written (e.g. an object
+is re-uploaded, or an object with a duplicate block is uploaded).
+
+To understand precisely what is happening, we need to dive in to how Garage
+splits data among the different storage locations.
+
+First of all, Garage divides the set of all possible block hashes
+in a fixed number of slices (currently 1024), and assigns
+to each slice a primary storage location among the specified data directories.
+The number of slices having their primary location in each data directory
+is proportionnal to the capacity specified in the config file.
+
+When Garage receives a block to write, it will always write it in the primary
+directory of the slice that contains its hash.
+
+Now, to be able to not lose existing data blocks when storage locations
+are added, Garage also keeps a list of secondary data directories
+for all of the hash slices. Secondary data directories for a slice indicates
+storage locations that once were primary directories for that slice, i.e. where
+Garage knows that data blocks of that slice might be stored.
+When Garage is requested to read a certain data block,
+it will first look in the primary storage directory of its slice,
+and if it doesn't find it there it goes through all of the secondary storage
+locations until it finds it. This allows Garage to continue operating
+normally when storage locations are added, without having to shuffle
+files between drives to place them in the correct location.
+
+This relatively simple strategy works well but does not ensure that data
+is correctly balanced among drives according to their capacity.
+To rebalance data, two strategies can be used:
+
+- Lazy rebalancing: when a block is re-written (e.g. the object is re-uploaded),
+  Garage checks whether the existing copy is in the primary directory of the slice
+  or in a secondary directory. If the current copy is in a secondary directory,
+  Garage re-writes a copy in the primary directory and deletes the one from the
+  secondary directory. This might never end up rebalancing everything if there
+  are data blocks that are only read and never written.
+
+- Active rebalancing: an operator of a Garage node can explicitly launch a repair
+  procedure that rebalances the data directories, moving all blocks to their
+  primary location. Once done, all secondary locations for all hash slices are
+  removed so that they won't be checked anymore when looking for a data block.
+
+## Read-only storage locations
+
+If you would like to move all data blocks from an existing data directory to one
+or several new data directories, mark the old directory as read-only:
+
+```toml
+data_dir = [
+    { path = "/path/to/old_data", read_only = true },
+    { path = "/path/to/new_hdd1", capacity = "2T" },
+    { path = "/path/to/new_hdd2", capacity = "4T" },
+]
+```
+
+Garage will be able to read requested blocks from the read-only directory.
+Garage will also move data out of the read-only directory either progressively
+(lazy rebalancing) or if requested explicitly (active rebalancing).
+
+Once an active rebalancing has finished, your read-only directory should be empty:
+it might still contain subdirectories, but no data files. You can check that
+it contains no files using:
+
+```bash
+find -type f /path/to/old_data      # should not print anything
+```
+
+at which point it can be removed from the `data_dir` list in your config file.
@@ -80,6 +80,6 @@ The entire procedure would look something like this:
 5. If any specific migration procedure is required, it is usually in one of the two cases:

  - It can be run on online nodes after the new version has started, during regular cluster operation.
-  - it has to be run offline
+  - it has to be run offline, in which case you will have to again take all nodes offline one after the other to run the repair

   For this last step, please refer to the specific documentation pertaining to the version upgrade you are doing.
@@ -17,8 +17,9 @@ db_engine = "lmdb"

 block_size = 1048576

-sled_cache_capacity = 134217728
+sled_cache_capacity = "128MiB"
 sled_flush_every_ms = 2000
+lmdb_map_size = "1T"

 replication_mode = "3"

@@ -91,6 +92,19 @@ This folder can be placed on an HDD. The space available for `data_dir`
 should be counted to determine a node's capacity
 when [adding it to the cluster layout](@/documentation/cookbook/real-world.md).

+Since `v0.9.0`, Garage supports multiple data directories with the following syntax:
+
+```toml
+data_dir = [
+    { path = "/path/to/old_data", read_only = true },
+    { path = "/path/to/new_hdd1", capacity = "2T" },
+    { path = "/path/to/new_hdd2", capacity = "4T" },
+]
+```
+
+See [the dedicated documentation page](@/documentation/operations/multi-hdd.md)
+on how to operate Garage in such a setup.
+
 ### `db_engine` (since `v0.8.0`)

 By default, Garage uses the Sled embedded database library
@@ -178,8 +192,8 @@ if geographical replication is used.
 ### `block_size`

 Garage splits stored objects in consecutive chunks of size `block_size`
-(except the last one which might be smaller). The default size is 1MB and
-should work in most cases. We recommend increasing it to e.g. 10MB if
+(except the last one which might be smaller). The default size is 1MiB and
+should work in most cases. We recommend increasing it to e.g. 10MiB if
 you are using Garage to store large files and have fast network connections
 between all nodes (e.g. 1gbps).

@@ -205,6 +219,14 @@ Increase this if sled is thrashing your SSD, at the risk of losing more data in
 of a power outage (though this should not matter much as data is replicated on other
 nodes). The default value, 2000ms, should be appropriate for most use cases.

+### `lmdb_map_size`
+
+This parameters can be used to set the map size used by LMDB,
+which is the size of the virtual memory region used for mapping the database file.
+The value of this parameter is the maximum size the metadata database can take.
+This value is not bound by the physical RAM size of the machine running Garage.
+If not specified, it defaults to 1GiB on 32-bit machines and 1TiB on 64-bit machines.
+
 ### `replication_mode`

 Garage supports the following replication modes:
@@ -75,16 +75,13 @@ but these endpoints are documented in [Red Hat Ceph Storage - Chapter 2. Ceph Ob

 | Endpoint                     | Garage                           | [Openstack Swift](https://docs.openstack.org/swift/latest/s3_compat.html) | [Ceph Object Gateway](https://docs.ceph.com/en/latest/radosgw/s3/) | [Riak CS](https://docs.riak.com/riak/cs/2.1.1/references/apis/storage/s3/index.html) | [OpenIO](https://docs.openio.io/latest/source/arch-design/s3_compliancy.html) |
 |------------------------------|----------------------------------|-----------------|---------------|---------|-----|
-| [AbortMultipartUpload](https://docs.aws.amazon.com/AmazonS3/latest/API/API_AbortMultipartUpload.html)         | ✅ Implemented                      |  ✅ | ✅ | ✅ | ✅ |
-| [CompleteMultipartUpload](https://docs.aws.amazon.com/AmazonS3/latest/API/API_CompleteMultipartUpload.html)   | ✅ Implemented (see details below)                      | ✅ | ✅ | ✅ | ✅ |
-| [CreateMultipartUpload](https://docs.aws.amazon.com/AmazonS3/latest/API/API_CreateMultipartUpload.html)        | ✅ Implemented                      | ✅| ✅ | ✅ | ✅ |
-| [ListMultipartUpload](https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListMultipartUpload.html)          | ✅ Implemented                      | ✅ | ✅ | ✅ | ✅ |
-| [ListParts](https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListParts.html)                    | ✅ Implemented                      | ✅ | ✅ | ✅ | ✅ |
-| [UploadPart](https://docs.aws.amazon.com/AmazonS3/latest/API/API_UploadPart.html)                  | ✅ Implemented (see details below)                      | ✅ | ✅| ✅ | ✅ |
-| [UploadPartCopy](https://docs.aws.amazon.com/AmazonS3/latest/API/API_UploadPartCopy.html)               | ✅ Implemented                      | ✅ | ✅ | ✅ | ✅ |
-
-Our implementation of Multipart Upload is currently a bit more restrictive than Amazon's one in some edge cases.
-For more information, please refer to our [issue tracker](https://git.deuxfleurs.fr/Deuxfleurs/garage/issues/204).
+| [AbortMultipartUpload](https://docs.aws.amazon.com/AmazonS3/latest/API/API_AbortMultipartUpload.html)         | ✅ Implemented  |  ✅ | ✅ | ✅ | ✅ |
+| [CompleteMultipartUpload](https://docs.aws.amazon.com/AmazonS3/latest/API/API_CompleteMultipartUpload.html)   | ✅ Implemented  | ✅ | ✅ | ✅ | ✅ |
+| [CreateMultipartUpload](https://docs.aws.amazon.com/AmazonS3/latest/API/API_CreateMultipartUpload.html)        | ✅ Implemented | ✅| ✅ | ✅ | ✅ |
+| [ListMultipartUpload](https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListMultipartUpload.html)          | ✅ Implemented   | ✅ | ✅ | ✅ | ✅ |
+| [ListParts](https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListParts.html)                    | ✅ Implemented             | ✅ | ✅ | ✅ | ✅ |
+| [UploadPart](https://docs.aws.amazon.com/AmazonS3/latest/API/API_UploadPart.html)                  | ✅ Implemented             | ✅ | ✅| ✅ | ✅ |
+| [UploadPartCopy](https://docs.aws.amazon.com/AmazonS3/latest/API/API_UploadPartCopy.html)               | ✅ Implemented        | ✅ | ✅ | ✅ | ✅ |

 ### Website endpoints

@@ -127,15 +124,22 @@ If you need this feature, please [share your use case in our dedicated issue](ht

 | Endpoint                     | Garage                           | [Openstack Swift](https://docs.openstack.org/swift/latest/s3_compat.html) | [Ceph Object Gateway](https://docs.ceph.com/en/latest/radosgw/s3/) | [Riak CS](https://docs.riak.com/riak/cs/2.1.1/references/apis/storage/s3/index.html) | [OpenIO](https://docs.openio.io/latest/source/arch-design/s3_compliancy.html) |
 |------------------------------|----------------------------------|-----------------|---------------|---------|-----|
-| [DeleteBucketLifecycle](https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteBucketLifecycle.html) | ❌ Missing | ❌| ✅| ❌| ✅|
-| [GetBucketLifecycleConfiguration](https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetBucketLifecycleConfiguration.html) | ❌ Missing | ❌| ✅ | ❌| ✅|
-| [PutBucketLifecycleConfiguration](https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutBucketLifecycleConfiguration.html) | ❌ Missing | ❌| ✅ | ❌| ✅|
+| [DeleteBucketLifecycle](https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteBucketLifecycle.html) | ✅ Implemented | ❌| ✅| ❌| ✅|
+| [GetBucketLifecycleConfiguration](https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetBucketLifecycleConfiguration.html) | ✅ Implemented | ❌| ✅ | ❌| ✅|
+| [PutBucketLifecycleConfiguration](https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutBucketLifecycleConfiguration.html) | ⚠ Partially implemented (see below) | ❌| ✅ | ❌| ✅|
 | [GetBucketVersioning](https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetBucketVersioning.html)          | ❌ Stub (see below)       | ✅| ✅ | ❌| ✅|
 | [ListObjectVersions](https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListObjectVersions.html) | ❌ Missing | ❌| ✅ | ❌| ✅|
 | [PutBucketVersioning](https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutBucketVersioning.html) | ❌ Missing | ❌| ✅| ❌| ✅|

+**PutBucketLifecycleConfiguration:** The only actions supported are
+`AbortIncompleteMultipartUpload` and `Expiration` (without the
+`ExpiredObjectDeleteMarker` field).  All other operations are dependent on
+either bucket versionning or storage classes which Garage currently does not
+implement. The deprecated `Prefix` member directly in the the `Rule`
+structure/XML tag is not supported, specified prefixes must be inside the
+`Filter` structure/XML tag.

-**GetBucketVersioning:** Stub implementation (Garage does not yet support versionning so this always returns "versionning not enabled").
+**GetBucketVersioning:** Stub implementation which always returns "versionning not enabled", since Garage does not yet support bucket versionning.

 ### Replication endpoints

@@ -21,4 +21,4 @@ version: 0.4.1
 # incremented each time you make changes to the application. Versions are not expected to
 # follow Semantic Versioning. They should reflect the version the application is using.
 # It is recommended to use it with quotes.
-appVersion: "v0.8.3"
+appVersion: "v0.8.4"
@@ -1,6 +1,6 @@
 [package]
 name = "garage_api"
-version = "0.8.3"
+version = "0.8.4"
 authors = ["Alex Auvolat <alex@adnab.me>"]
 edition = "2018"
 license = "AGPL-3.0"
@@ -26,6 +26,7 @@ use crate::s3::copy::*;
 use crate::s3::cors::*;
 use crate::s3::delete::*;
 use crate::s3::get::*;
+use crate::s3::lifecycle::*;
 use crate::s3::list::*;
 use crate::s3::multipart::*;
 use crate::s3::post_object::handle_post_object;
@@ -354,14 +355,21 @@ impl ApiHandler for S3ApiServer {
 			}
 			Endpoint::GetBucketWebsite {} => handle_get_website(&bucket).await,
 			Endpoint::PutBucketWebsite {} => {
-				handle_put_website(garage, bucket_id, req, content_sha256).await
+				handle_put_website(garage, bucket.clone(), req, content_sha256).await
 			}
-			Endpoint::DeleteBucketWebsite {} => handle_delete_website(garage, bucket_id).await,
+			Endpoint::DeleteBucketWebsite {} => handle_delete_website(garage, bucket.clone()).await,
 			Endpoint::GetBucketCors {} => handle_get_cors(&bucket).await,
 			Endpoint::PutBucketCors {} => {
-				handle_put_cors(garage, bucket_id, req, content_sha256).await
+				handle_put_cors(garage, bucket.clone(), req, content_sha256).await
+			}
+			Endpoint::DeleteBucketCors {} => handle_delete_cors(garage, bucket.clone()).await,
+			Endpoint::GetBucketLifecycleConfiguration {} => handle_get_lifecycle(&bucket).await,
+			Endpoint::PutBucketLifecycleConfiguration {} => {
+				handle_put_lifecycle(garage, bucket.clone(), req, content_sha256).await
+			}
+			Endpoint::DeleteBucketLifecycle {} => {
+				handle_delete_lifecycle(garage, bucket.clone()).await
 			}
-			Endpoint::DeleteBucketCors {} => handle_delete_cors(garage, bucket_id).await,
 			endpoint => Err(Error::NotImplemented(endpoint.name().to_owned())),
 		};

@@ -44,14 +44,11 @@ pub async fn handle_get_cors(bucket: &Bucket) -> Result<Response<Body>, Error> {

 pub async fn handle_delete_cors(
 	garage: Arc<Garage>,
-	bucket_id: Uuid,
+	mut bucket: Bucket,
 ) -> Result<Response<Body>, Error> {
-	let mut bucket = garage
-		.bucket_helper()
-		.get_existing_bucket(bucket_id)
-		.await?;
-
-	let param = bucket.params_mut().unwrap();
+	let param = bucket
+		.params_mut()
+		.ok_or_internal_error("Bucket should not be deleted at this point")?;

 	param.cors_config.update(None);
 	garage.bucket_table.insert(&bucket).await?;
@@ -63,7 +60,7 @@ pub async fn handle_delete_cors(

 pub async fn handle_put_cors(
 	garage: Arc<Garage>,
-	bucket_id: Uuid,
+	mut bucket: Bucket,
 	req: Request<Body>,
 	content_sha256: Option<Hash>,
 ) -> Result<Response<Body>, Error> {
@@ -73,12 +70,9 @@ pub async fn handle_put_cors(
 		verify_signed_content(content_sha256, &body[..])?;
 	}

-	let mut bucket = garage
-		.bucket_helper()
-		.get_existing_bucket(bucket_id)
-		.await?;
-
-	let param = bucket.params_mut().unwrap();
+	let param = bucket
+		.params_mut()
+		.ok_or_internal_error("Bucket should not be deleted at this point")?;

 	let conf: CorsConfiguration = from_reader(&body as &[u8])?;
 	conf.validate()?;
@@ -0,0 +1,401 @@
+use quick_xml::de::from_reader;
+use std::sync::Arc;
+
+use hyper::{Body, Request, Response, StatusCode};
+
+use serde::{Deserialize, Serialize};
+
+use crate::s3::error::*;
+use crate::s3::xml::{to_xml_with_header, xmlns_tag, IntValue, Value};
+use crate::signature::verify_signed_content;
+
+use garage_model::bucket_table::{
+	parse_lifecycle_date, Bucket, LifecycleExpiration as GarageLifecycleExpiration,
+	LifecycleFilter as GarageLifecycleFilter, LifecycleRule as GarageLifecycleRule,
+};
+use garage_model::garage::Garage;
+use garage_util::data::*;
+
+pub async fn handle_get_lifecycle(bucket: &Bucket) -> Result<Response<Body>, Error> {
+	let param = bucket
+		.params()
+		.ok_or_internal_error("Bucket should not be deleted at this point")?;
+
+	if let Some(lifecycle) = param.lifecycle_config.get() {
+		let wc = LifecycleConfiguration::from_garage_lifecycle_config(lifecycle);
+		let xml = to_xml_with_header(&wc)?;
+		Ok(Response::builder()
+			.status(StatusCode::OK)
+			.header(http::header::CONTENT_TYPE, "application/xml")
+			.body(Body::from(xml))?)
+	} else {
+		Ok(Response::builder()
+			.status(StatusCode::NO_CONTENT)
+			.body(Body::empty())?)
+	}
+}
+
+pub async fn handle_delete_lifecycle(
+	garage: Arc<Garage>,
+	mut bucket: Bucket,
+) -> Result<Response<Body>, Error> {
+	let param = bucket
+		.params_mut()
+		.ok_or_internal_error("Bucket should not be deleted at this point")?;
+
+	param.lifecycle_config.update(None);
+	garage.bucket_table.insert(&bucket).await?;
+
+	Ok(Response::builder()
+		.status(StatusCode::NO_CONTENT)
+		.body(Body::empty())?)
+}
+
+pub async fn handle_put_lifecycle(
+	garage: Arc<Garage>,
+	mut bucket: Bucket,
+	req: Request<Body>,
+	content_sha256: Option<Hash>,
+) -> Result<Response<Body>, Error> {
+	let body = hyper::body::to_bytes(req.into_body()).await?;
+
+	if let Some(content_sha256) = content_sha256 {
+		verify_signed_content(content_sha256, &body[..])?;
+	}
+
+	let param = bucket
+		.params_mut()
+		.ok_or_internal_error("Bucket should not be deleted at this point")?;
+
+	let conf: LifecycleConfiguration = from_reader(&body as &[u8])?;
+	let config = conf
+		.validate_into_garage_lifecycle_config()
+		.ok_or_bad_request("Invalid lifecycle configuration")?;
+
+	param.lifecycle_config.update(Some(config));
+	garage.bucket_table.insert(&bucket).await?;
+
+	Ok(Response::builder()
+		.status(StatusCode::OK)
+		.body(Body::empty())?)
+}
+
+// ---- SERIALIZATION AND DESERIALIZATION TO/FROM S3 XML ----
+
+#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)]
+pub struct LifecycleConfiguration {
+	#[serde(serialize_with = "xmlns_tag", skip_deserializing)]
+	pub xmlns: (),
+	#[serde(rename = "Rule")]
+	pub lifecycle_rules: Vec<LifecycleRule>,
+}
+
+#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)]
+pub struct LifecycleRule {
+	#[serde(rename = "ID")]
+	pub id: Option<Value>,
+	#[serde(rename = "Status")]
+	pub status: Value,
+	#[serde(rename = "Filter", default)]
+	pub filter: Option<Filter>,
+	#[serde(rename = "Expiration", default)]
+	pub expiration: Option<Expiration>,
+	#[serde(rename = "AbortIncompleteMultipartUpload", default)]
+	pub abort_incomplete_mpu: Option<AbortIncompleteMpu>,
+}
+
+#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord, Default)]
+pub struct Filter {
+	#[serde(rename = "And")]
+	pub and: Option<Box<Filter>>,
+	#[serde(rename = "Prefix")]
+	pub prefix: Option<Value>,
+	#[serde(rename = "ObjectSizeGreaterThan")]
+	pub size_gt: Option<IntValue>,
+	#[serde(rename = "ObjectSizeLessThan")]
+	pub size_lt: Option<IntValue>,
+}
+
+#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)]
+pub struct Expiration {
+	#[serde(rename = "Days")]
+	pub days: Option<IntValue>,
+	#[serde(rename = "Date")]
+	pub at_date: Option<Value>,
+}
+
+#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)]
+pub struct AbortIncompleteMpu {
+	#[serde(rename = "DaysAfterInitiation")]
+	pub days: IntValue,
+}
+
+impl LifecycleConfiguration {
+	pub fn validate_into_garage_lifecycle_config(
+		self,
+	) -> Result<Vec<GarageLifecycleRule>, &'static str> {
+		let mut ret = vec![];
+		for rule in self.lifecycle_rules {
+			ret.push(rule.validate_into_garage_lifecycle_rule()?);
+		}
+		Ok(ret)
+	}
+
+	pub fn from_garage_lifecycle_config(config: &[GarageLifecycleRule]) -> Self {
+		Self {
+			xmlns: (),
+			lifecycle_rules: config
+				.iter()
+				.map(LifecycleRule::from_garage_lifecycle_rule)
+				.collect(),
+		}
+	}
+}
+
+impl LifecycleRule {
+	pub fn validate_into_garage_lifecycle_rule(self) -> Result<GarageLifecycleRule, &'static str> {
+		let enabled = match self.status.0.as_str() {
+			"Enabled" => true,
+			"Disabled" => false,
+			_ => return Err("invalid value for <Status>"),
+		};
+
+		let filter = self
+			.filter
+			.map(Filter::validate_into_garage_lifecycle_filter)
+			.transpose()?
+			.unwrap_or_default();
+
+		let abort_incomplete_mpu_days = self.abort_incomplete_mpu.map(|x| x.days.0 as usize);
+
+		let expiration = self
+			.expiration
+			.map(Expiration::validate_into_garage_lifecycle_expiration)
+			.transpose()?;
+
+		Ok(GarageLifecycleRule {
+			id: self.id.map(|x| x.0),
+			enabled,
+			filter,
+			abort_incomplete_mpu_days,
+			expiration,
+		})
+	}
+
+	pub fn from_garage_lifecycle_rule(rule: &GarageLifecycleRule) -> Self {
+		Self {
+			id: rule.id.as_deref().map(Value::from),
+			status: if rule.enabled {
+				Value::from("Enabled")
+			} else {
+				Value::from("Disabled")
+			},
+			filter: Filter::from_garage_lifecycle_filter(&rule.filter),
+			abort_incomplete_mpu: rule
+				.abort_incomplete_mpu_days
+				.map(|days| AbortIncompleteMpu {
+					days: IntValue(days as i64),
+				}),
+			expiration: rule
+				.expiration
+				.as_ref()
+				.map(Expiration::from_garage_lifecycle_expiration),
+		}
+	}
+}
+
+impl Filter {
+	pub fn count(&self) -> i32 {
+		fn count<T>(x: &Option<T>) -> i32 {
+			x.as_ref().map(|_| 1).unwrap_or(0)
+		}
+		count(&self.prefix) + count(&self.size_gt) + count(&self.size_lt)
+	}
+
+	pub fn validate_into_garage_lifecycle_filter(
+		self,
+	) -> Result<GarageLifecycleFilter, &'static str> {
+		if self.count() > 0 && self.and.is_some() {
+			Err("Filter tag cannot contain both <And> and another condition")
+		} else if let Some(and) = self.and {
+			if and.and.is_some() {
+				return Err("Nested <And> tags");
+			}
+			Ok(and.internal_into_garage_lifecycle_filter())
+		} else if self.count() > 1 {
+			Err("Multiple Filter conditions must be wrapped in an <And> tag")
+		} else {
+			Ok(self.internal_into_garage_lifecycle_filter())
+		}
+	}
+
+	fn internal_into_garage_lifecycle_filter(self) -> GarageLifecycleFilter {
+		GarageLifecycleFilter {
+			prefix: self.prefix.map(|x| x.0),
+			size_gt: self.size_gt.map(|x| x.0 as u64),
+			size_lt: self.size_lt.map(|x| x.0 as u64),
+		}
+	}
+
+	pub fn from_garage_lifecycle_filter(rule: &GarageLifecycleFilter) -> Option<Self> {
+		let filter = Filter {
+			and: None,
+			prefix: rule.prefix.as_deref().map(Value::from),
+			size_gt: rule.size_gt.map(|x| IntValue(x as i64)),
+			size_lt: rule.size_lt.map(|x| IntValue(x as i64)),
+		};
+		match filter.count() {
+			0 => None,
+			1 => Some(filter),
+			_ => Some(Filter {
+				and: Some(Box::new(filter)),
+				..Default::default()
+			}),
+		}
+	}
+}
+
+impl Expiration {
+	pub fn validate_into_garage_lifecycle_expiration(
+		self,
+	) -> Result<GarageLifecycleExpiration, &'static str> {
+		match (self.days, self.at_date) {
+			(Some(_), Some(_)) => Err("cannot have both <Days> and <Date> in <Expiration>"),
+			(None, None) => Err("<Expiration> must contain either <Days> or <Date>"),
+			(Some(days), None) => Ok(GarageLifecycleExpiration::AfterDays(days.0 as usize)),
+			(None, Some(date)) => {
+				parse_lifecycle_date(&date.0)?;
+				Ok(GarageLifecycleExpiration::AtDate(date.0))
+			}
+		}
+	}
+
+	pub fn from_garage_lifecycle_expiration(exp: &GarageLifecycleExpiration) -> Self {
+		match exp {
+			GarageLifecycleExpiration::AfterDays(days) => Expiration {
+				days: Some(IntValue(*days as i64)),
+				at_date: None,
+			},
+			GarageLifecycleExpiration::AtDate(date) => Expiration {
+				days: None,
+				at_date: Some(Value(date.to_string())),
+			},
+		}
+	}
+}
+
+#[cfg(test)]
+mod tests {
+	use super::*;
+
+	use quick_xml::de::from_str;
+
+	#[test]
+	fn test_deserialize_lifecycle_config() -> Result<(), Error> {
+		let message = r#"<?xml version="1.0" encoding="UTF-8"?>
+<LifecycleConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
+  <Rule>
+    <ID>id1</ID>
+    <Status>Enabled</Status>
+    <Filter>
+       <Prefix>documents/</Prefix>
+    </Filter>
+    <AbortIncompleteMultipartUpload>
+       <DaysAfterInitiation>7</DaysAfterInitiation>
+    </AbortIncompleteMultipartUpload>
+  </Rule>
+  <Rule>
+    <ID>id2</ID>
+    <Status>Enabled</Status>
+    <Filter>
+       <And>
+          <Prefix>logs/</Prefix>
+          <ObjectSizeGreaterThan>1000000</ObjectSizeGreaterThan>
+       </And>
+    </Filter>
+    <Expiration>
+      <Days>365</Days>
+    </Expiration>
+  </Rule>
+</LifecycleConfiguration>"#;
+		let conf: LifecycleConfiguration = from_str(message).unwrap();
+		let ref_value = LifecycleConfiguration {
+			xmlns: (),
+			lifecycle_rules: vec![
+				LifecycleRule {
+					id: Some("id1".into()),
+					status: "Enabled".into(),
+					filter: Some(Filter {
+						prefix: Some("documents/".into()),
+						..Default::default()
+					}),
+					expiration: None,
+					abort_incomplete_mpu: Some(AbortIncompleteMpu { days: IntValue(7) }),
+				},
+				LifecycleRule {
+					id: Some("id2".into()),
+					status: "Enabled".into(),
+					filter: Some(Filter {
+						and: Some(Box::new(Filter {
+							prefix: Some("logs/".into()),
+							size_gt: Some(IntValue(1000000)),
+							..Default::default()
+						})),
+						..Default::default()
+					}),
+					expiration: Some(Expiration {
+						days: Some(IntValue(365)),
+						at_date: None,
+					}),
+					abort_incomplete_mpu: None,
+				},
+			],
+		};
+		assert_eq! {
+			ref_value,
+			conf
+		};
+
+		let message2 = to_xml_with_header(&ref_value)?;
+
+		let cleanup = |c: &str| c.replace(char::is_whitespace, "");
+		assert_eq!(cleanup(message), cleanup(&message2));
+
+		// Check validation
+		let validated = ref_value
+			.validate_into_garage_lifecycle_config()
+			.ok_or_bad_request("invalid xml config")?;
+
+		let ref_config = vec![
+			GarageLifecycleRule {
+				id: Some("id1".into()),
+				enabled: true,
+				filter: GarageLifecycleFilter {
+					prefix: Some("documents/".into()),
+					..Default::default()
+				},
+				expiration: None,
+				abort_incomplete_mpu_days: Some(7),
+			},
+			GarageLifecycleRule {
+				id: Some("id2".into()),
+				enabled: true,
+				filter: GarageLifecycleFilter {
+					prefix: Some("logs/".into()),
+					size_gt: Some(1000000),
+					..Default::default()
+				},
+				expiration: Some(GarageLifecycleExpiration::AfterDays(365)),
+				abort_incomplete_mpu_days: None,
+			},
+		];
+		assert_eq!(validated, ref_config);
+
+		let message3 = to_xml_with_header(&LifecycleConfiguration::from_garage_lifecycle_config(
+			&validated,
+		))?;
+		assert_eq!(cleanup(message), cleanup(&message3));
+
+		Ok(())
+	}
+}
@@ -6,6 +6,7 @@ mod copy;
 pub mod cors;
 mod delete;
 pub mod get;
+mod lifecycle;
 mod list;
 mod multipart;
 mod post_object;
@@ -43,14 +43,11 @@ pub async fn handle_get_website(bucket: &Bucket) -> Result<Response<Body>, Error

 pub async fn handle_delete_website(
 	garage: Arc<Garage>,
-	bucket_id: Uuid,
+	mut bucket: Bucket,
 ) -> Result<Response<Body>, Error> {
-	let mut bucket = garage
-		.bucket_helper()
-		.get_existing_bucket(bucket_id)
-		.await?;
-
-	let param = bucket.params_mut().unwrap();
+	let param = bucket
+		.params_mut()
+		.ok_or_internal_error("Bucket should not be deleted at this point")?;

 	param.website_config.update(None);
 	garage.bucket_table.insert(&bucket).await?;
@@ -62,7 +59,7 @@ pub async fn handle_delete_website(

 pub async fn handle_put_website(
 	garage: Arc<Garage>,
-	bucket_id: Uuid,
+	mut bucket: Bucket,
 	req: Request<Body>,
 	content_sha256: Option<Hash>,
 ) -> Result<Response<Body>, Error> {
@@ -72,12 +69,9 @@ pub async fn handle_put_website(
 		verify_signed_content(content_sha256, &body[..])?;
 	}

-	let mut bucket = garage
-		.bucket_helper()
-		.get_existing_bucket(bucket_id)
-		.await?;
-
-	let param = bucket.params_mut().unwrap();
+	let param = bucket
+		.params_mut()
+		.ok_or_internal_error("Bucket should not be deleted at this point")?;

 	let conf: WebsiteConfiguration = from_reader(&body as &[u8])?;
 	conf.validate()?;
@@ -1,6 +1,6 @@
 [package]
 name = "garage_block"
-version = "0.8.3"
+version = "0.8.4"
 authors = ["Alex Auvolat <alex@adnab.me>"]
 edition = "2018"
 license = "AGPL-3.0"
@@ -24,6 +24,7 @@ opentelemetry = "0.17"
 arc-swap = "1.5"
 async-trait = "0.1.7"
 bytes = "1.0"
+bytesize = "1.2"
 hex = "0.4"
 tracing = "0.1"
 rand = "0.8"
@@ -1,3 +1,5 @@
+use std::path::PathBuf;
+
 use bytes::Bytes;
 use serde::{Deserialize, Serialize};
 use zstd::stream::{decode_all as zstd_decode, Encoder};
@@ -19,6 +21,14 @@ pub enum DataBlock {
 	Compressed(Bytes),
 }

+#[derive(Debug)]
+pub enum DataBlockPath {
+	/// Uncompressed data fail
+	Plain(PathBuf),
+	/// Compressed data fail
+	Compressed(PathBuf),
+}
+
 impl DataBlock {
 	/// Query whether this block is compressed
 	pub fn is_compressed(&self) -> bool {
@@ -0,0 +1,337 @@
+use std::path::PathBuf;
+
+use serde::{Deserialize, Serialize};
+
+use garage_util::config::DataDirEnum;
+use garage_util::data::Hash;
+use garage_util::error::{Error, OkOrMessage};
+use garage_util::migrate::*;
+
+type Idx = u16;
+
+const DRIVE_NPART: usize = 1024;
+
+const HASH_DRIVE_BYTES: (usize, usize) = (2, 3);
+
+#[derive(Serialize, Deserialize, Debug, Clone)]
+pub(crate) struct DataLayout {
+	pub(crate) data_dirs: Vec<DataDir>,
+
+	/// Primary storage location (index in data_dirs) for each partition
+	/// = the location where the data is supposed to be, blocks are always
+	/// written there (copies in other dirs may be deleted if they exist)
+	pub(crate) part_prim: Vec<Idx>,
+	/// Secondary storage locations for each partition = locations
+	/// where data blocks might be, we check from these dirs when reading
+	pub(crate) part_sec: Vec<Vec<Idx>>,
+}
+
+#[derive(Serialize, Deserialize, Debug, Clone, Eq, PartialEq)]
+pub(crate) struct DataDir {
+	pub(crate) path: PathBuf,
+	pub(crate) state: DataDirState,
+}
+
+#[derive(Serialize, Deserialize, Debug, Clone, Copy, Eq, PartialEq)]
+pub(crate) enum DataDirState {
+	Active { capacity: u64 },
+	ReadOnly,
+}
+
+impl DataLayout {
+	pub(crate) fn initialize(dirs: &DataDirEnum) -> Result<Self, Error> {
+		let data_dirs = make_data_dirs(dirs)?;
+
+		// Split partitions proportionnally to capacity for all drives
+		// to affect primary storage location
+		let total_cap = data_dirs.iter().filter_map(|x| x.capacity()).sum::<u64>();
+		assert!(total_cap > 0);
+
+		let mut part_prim = Vec::with_capacity(DRIVE_NPART);
+		let mut cum_cap = 0;
+		for (i, dd) in data_dirs.iter().enumerate() {
+			if let DataDirState::Active { capacity } = dd.state {
+				cum_cap += capacity;
+				let n_total = (cum_cap * DRIVE_NPART as u64) / total_cap;
+				part_prim.resize(n_total as usize, i as Idx);
+			}
+		}
+		assert_eq!(cum_cap, total_cap);
+		assert_eq!(part_prim.len(), DRIVE_NPART);
+
+		// If any of the storage locations is non-empty, it probably existed before
+		// this algorithm was added, so add it as a secondary storage location for all partitions
+		// to make sure existing files are not lost
+		let mut part_sec = vec![vec![]; DRIVE_NPART];
+		for (i, dd) in data_dirs.iter().enumerate() {
+			if dir_not_empty(&dd.path)? {
+				for (sec, prim) in part_sec.iter_mut().zip(part_prim.iter()) {
+					if *prim != i as Idx {
+						sec.push(i as Idx);
+					}
+				}
+			}
+		}
+
+		Ok(Self {
+			data_dirs,
+			part_prim,
+			part_sec,
+		})
+	}
+
+	pub(crate) fn update(&mut self, dirs: &DataDirEnum) -> Result<(), Error> {
+		// Make list of new data directories, exit if nothing changed
+		let data_dirs = make_data_dirs(dirs)?;
+		if data_dirs == self.data_dirs {
+			return Ok(());
+		}
+
+		let total_cap = data_dirs.iter().filter_map(|x| x.capacity()).sum::<u64>();
+		assert!(total_cap > 0);
+
+		// Compute mapping of old indices to new indices
+		let old2new = self
+			.data_dirs
+			.iter()
+			.map(|x| {
+				data_dirs
+					.iter()
+					.position(|y| y.path == x.path)
+					.map(|x| x as Idx)
+			})
+			.collect::<Vec<_>>();
+
+		// Compute secondary location list for partitions based on existing
+		// folders, translating indices from old to new
+		let mut part_sec = self
+			.part_sec
+			.iter()
+			.map(|dl| {
+				dl.iter()
+					.filter_map(|old| old2new.get(*old as usize).copied().flatten())
+					.collect::<Vec<_>>()
+			})
+			.collect::<Vec<_>>();
+
+		// Compute a vector that, for each data dir,
+		// contains the list of partitions primarily stored on that drive
+		let mut dir_prim = vec![vec![]; data_dirs.len()];
+		for (ipart, prim) in self.part_prim.iter().enumerate() {
+			if let Some(new) = old2new.get(*prim as usize).copied().flatten() {
+				dir_prim[new as usize].push(ipart);
+			}
+		}
+
+		// Compute the target number of partitions per data directory
+		let mut cum_cap = 0;
+		let mut npart_per_dir = vec![0; data_dirs.len()];
+		for (idir, dd) in data_dirs.iter().enumerate() {
+			if let DataDirState::Active { capacity } = dd.state {
+				let begin = (cum_cap * DRIVE_NPART as u64) / total_cap;
+				cum_cap += capacity;
+				let end = (cum_cap * DRIVE_NPART as u64) / total_cap;
+				npart_per_dir[idir] = (end - begin) as usize;
+			}
+		}
+		assert_eq!(cum_cap, total_cap);
+		assert_eq!(npart_per_dir.iter().sum::<usize>(), DRIVE_NPART);
+
+		// For all directories that have too many primary partitions,
+		// move that partition to secondary
+		for (idir, (parts, tgt_npart)) in dir_prim.iter_mut().zip(npart_per_dir.iter()).enumerate()
+		{
+			while parts.len() > *tgt_npart {
+				let part = parts.pop().unwrap();
+				if !part_sec[part].contains(&(idir as Idx)) {
+					part_sec[part].push(idir as Idx);
+				}
+			}
+		}
+
+		// Calculate the vector of primary partition dir index
+		let mut part_prim = vec![None; DRIVE_NPART];
+		for (idir, parts) in dir_prim.iter().enumerate() {
+			for part in parts.iter() {
+				assert!(part_prim[*part].is_none());
+				part_prim[*part] = Some(idir as Idx)
+			}
+		}
+
+		// Calculate a vector of unassigned partitions
+		let mut unassigned = part_prim
+			.iter()
+			.enumerate()
+			.filter(|(_, dir)| dir.is_none())
+			.map(|(ipart, _)| ipart)
+			.collect::<Vec<_>>();
+
+		// For all directories that don't have enough primary partitions,
+		// add partitions from unassigned
+		for (idir, (parts, tgt_npart)) in dir_prim.iter_mut().zip(npart_per_dir.iter()).enumerate()
+		{
+			if parts.len() < *tgt_npart {
+				let required = *tgt_npart - parts.len();
+				assert!(unassigned.len() >= required);
+				for _ in 0..required {
+					let new_part = unassigned.pop().unwrap();
+					part_prim[new_part] = Some(idir as Idx);
+					part_sec[new_part].retain(|x| *x != idir as Idx);
+				}
+			}
+		}
+
+		// Sanity checks
+		assert!(part_prim.iter().all(|x| x.is_some()));
+		assert!(unassigned.is_empty());
+
+		// Transform part_prim from vec of Option<Idx> to vec of Idx
+		let part_prim = part_prim
+			.into_iter()
+			.map(|x| x.unwrap())
+			.collect::<Vec<_>>();
+		assert!(part_prim.iter().all(|p| data_dirs
+			.get(*p as usize)
+			.and_then(|x| x.capacity())
+			.unwrap_or(0)
+			> 0));
+
+		// If any of the newly added storage locations is non-empty,
+		// it might have been removed and added again and might contain data,
+		// so add it as a secondary storage location for all partitions
+		// to make sure existing files are not lost
+		for (i, dd) in data_dirs.iter().enumerate() {
+			if self.data_dirs.iter().any(|ed| ed.path == dd.path) {
+				continue;
+			}
+			if dir_not_empty(&dd.path)? {
+				for (sec, prim) in part_sec.iter_mut().zip(part_prim.iter()) {
+					if *prim != i as Idx && !sec.contains(&(i as Idx)) {
+						sec.push(i as Idx);
+					}
+				}
+			}
+		}
+
+		// Apply newly generated config
+		*self = Self {
+			data_dirs,
+			part_prim,
+			part_sec,
+		};
+		Ok(())
+	}
+
+	pub(crate) fn primary_block_dir(&self, hash: &Hash) -> PathBuf {
+		let ipart = self.partition_from(hash);
+		let idir = self.part_prim[ipart] as usize;
+		self.block_dir_from(hash, &self.data_dirs[idir].path)
+	}
+
+	pub(crate) fn secondary_block_dirs<'a>(
+		&'a self,
+		hash: &'a Hash,
+	) -> impl Iterator<Item = PathBuf> + 'a {
+		let ipart = self.partition_from(hash);
+		self.part_sec[ipart]
+			.iter()
+			.map(move |idir| self.block_dir_from(hash, &self.data_dirs[*idir as usize].path))
+	}
+
+	fn partition_from(&self, hash: &Hash) -> usize {
+		u16::from_be_bytes([
+			hash.as_slice()[HASH_DRIVE_BYTES.0],
+			hash.as_slice()[HASH_DRIVE_BYTES.1],
+		]) as usize % DRIVE_NPART
+	}
+
+	fn block_dir_from(&self, hash: &Hash, dir: &PathBuf) -> PathBuf {
+		let mut path = dir.clone();
+		path.push(hex::encode(&hash.as_slice()[0..1]));
+		path.push(hex::encode(&hash.as_slice()[1..2]));
+		path
+	}
+
+	pub(crate) fn without_secondary_locations(&self) -> Self {
+		Self {
+			data_dirs: self.data_dirs.clone(),
+			part_prim: self.part_prim.clone(),
+			part_sec: self.part_sec.iter().map(|_| vec![]).collect::<Vec<_>>(),
+		}
+	}
+}
+
+impl InitialFormat for DataLayout {
+	const VERSION_MARKER: &'static [u8] = b"G09bmdl";
+}
+
+impl DataDir {
+	pub fn capacity(&self) -> Option<u64> {
+		match self.state {
+			DataDirState::Active { capacity } => Some(capacity),
+			_ => None,
+		}
+	}
+}
+
+fn make_data_dirs(dirs: &DataDirEnum) -> Result<Vec<DataDir>, Error> {
+	let mut data_dirs = vec![];
+	match dirs {
+		DataDirEnum::Single(path) => data_dirs.push(DataDir {
+			path: path.clone(),
+			state: DataDirState::Active {
+				capacity: 1_000_000_000, // whatever, doesn't matter
+			},
+		}),
+		DataDirEnum::Multiple(dirs) => {
+			let mut ok = false;
+			for dir in dirs.iter() {
+				let state = match &dir.capacity {
+					Some(cap) if dir.read_only == false => {
+						let capacity = cap.parse::<bytesize::ByteSize>()
+							.ok_or_message("invalid capacity value")?.as_u64();
+						if capacity == 0 {
+							return Err(Error::Message(format!("data directory {} should have non-zero capacity", dir.path.to_string_lossy())));
+						}
+						ok = true;
+						DataDirState::Active {
+							capacity,
+						}
+					}
+					None if dir.read_only == true => {
+						DataDirState::ReadOnly
+					}
+					_ => return Err(Error::Message(format!("data directories in data_dir should have a capacity value or be marked read_only, not the case for {}", dir.path.to_string_lossy()))),
+				};
+				data_dirs.push(DataDir {
+					path: dir.path.clone(),
+					state,
+				});
+			}
+			if !ok {
+				return Err(Error::Message(
+					"incorrect data_dir configuration, no primary writable directory specified"
+						.into(),
+				));
+			}
+		}
+	}
+	Ok(data_dirs)
+}
+
+fn dir_not_empty(path: &PathBuf) -> Result<bool, Error> {
+	for entry in std::fs::read_dir(&path)? {
+		let dir = entry?;
+		if dir.file_type()?.is_dir()
+			&& dir
+				.file_name()
+				.into_string()
+				.ok()
+				.and_then(|hex| hex::decode(&hex).ok())
+				.is_some()
+		{
+			return Ok(true);
+		}
+	}
+	Ok(false)
+}
@@ -6,5 +6,6 @@ pub mod repair;
 pub mod resync;

 mod block;
+mod layout;
 mod metrics;
 mod rc;
@@ -3,7 +3,7 @@ use std::pin::Pin;
 use std::sync::Arc;
 use std::time::Duration;

-use arc_swap::ArcSwapOption;
+use arc_swap::{ArcSwap, ArcSwapOption};
 use async_trait::async_trait;
 use bytes::Bytes;
 use rand::prelude::*;
@@ -25,10 +25,11 @@ use garage_rpc::rpc_helper::netapp::stream::{stream_asyncread, ByteStream};
 use garage_db as db;

 use garage_util::background::{vars, BackgroundRunner};
+use garage_util::config::DataDirEnum;
 use garage_util::data::*;
 use garage_util::error::*;
 use garage_util::metrics::RecordDuration;
-use garage_util::persister::PersisterShared;
+use garage_util::persister::{Persister, PersisterShared};
 use garage_util::time::msec_to_rfc3339;

 use garage_rpc::rpc_helper::OrderTag;
@@ -38,6 +39,7 @@ use garage_rpc::*;
 use garage_table::replication::{TableReplication, TableShardedReplication};

 use crate::block::*;
+use crate::layout::*;
 use crate::metrics::*;
 use crate::rc::*;
 use crate::repair::*;
@@ -77,13 +79,16 @@ impl Rpc for BlockRpc {
 pub struct BlockManager {
 	/// Replication strategy, allowing to find on which node blocks should be located
 	pub replication: TableShardedReplication,
-	/// Directory in which block are stored
-	pub data_dir: PathBuf,
+
+	/// Data layout
+	pub(crate) data_layout: ArcSwap<DataLayout>,
+	/// Data layout persister
+	pub(crate) data_layout_persister: Persister<DataLayout>,

 	data_fsync: bool,
 	compression_level: Option<i32>,

-	mutation_lock: [Mutex<BlockManagerLocked>; 256],
+	mutation_lock: Vec<Mutex<BlockManagerLocked>>,

 	pub(crate) rc: BlockRc,
 	pub resync: BlockResyncManager,
@@ -106,6 +111,9 @@ pub struct BlockResyncErrorInfo {
 	pub next_try: u64,
 }

+// The number of different mutexes used to parallelize write access to data blocks
+const MUTEX_COUNT: usize = 256;
+
 // This custom struct contains functions that must only be ran
 // when the lock is held. We ensure that it is the case by storing
 // it INSIDE a Mutex.
@@ -114,12 +122,29 @@ struct BlockManagerLocked();
 impl BlockManager {
 	pub fn new(
 		db: &db::Db,
-		data_dir: PathBuf,
+		data_dir: DataDirEnum,
 		data_fsync: bool,
 		compression_level: Option<i32>,
 		replication: TableShardedReplication,
 		system: Arc<System>,
-	) -> Arc<Self> {
+	) -> Result<Arc<Self>, Error> {
+		// Load or compute layout, i.e. assignment of data blocks to the different data directories
+		let data_layout_persister: Persister<DataLayout> =
+			Persister::new(&system.metadata_dir, "data_layout");
+		let data_layout = match data_layout_persister.load() {
+			Ok(mut layout) => {
+				layout
+					.update(&data_dir)
+					.ok_or_message("invalid data_dir config")?;
+				layout
+			}
+			Err(_) => DataLayout::initialize(&data_dir).ok_or_message("invalid data_dir config")?,
+		};
+		data_layout_persister
+			.save(&data_layout)
+			.expect("cannot save data_layout");
+
+		// Open metadata tables
 		let rc = db
 			.open_tree("block_local_rc")
 			.expect("Unable to open block_local_rc tree");
@@ -142,10 +167,14 @@ impl BlockManager {

 		let block_manager = Arc::new(Self {
 			replication,
-			data_dir,
+			data_layout: ArcSwap::new(Arc::new(data_layout)),
+			data_layout_persister,
 			data_fsync,
 			compression_level,
-			mutation_lock: [(); 256].map(|_| Mutex::new(BlockManagerLocked())),
+			mutation_lock: vec![(); MUTEX_COUNT]
+				.iter()
+				.map(|_| Mutex::new(BlockManagerLocked()))
+				.collect::<Vec<_>>(),
 			rc,
 			resync,
 			system,
@@ -157,7 +186,7 @@ impl BlockManager {
 		block_manager.endpoint.set_handler(block_manager.clone());
 		block_manager.scrub_persister.set_with(|_| ()).unwrap();

-		block_manager
+		Ok(block_manager)
 	}

 	pub fn spawn_workers(self: &Arc<Self>, bg: &BackgroundRunner) {
@@ -204,44 +233,10 @@ impl BlockManager {
 		hash: &Hash,
 		order_tag: Option<OrderTag>,
 	) -> Result<(DataBlockHeader, ByteStream), Error> {
-		let who = self.replication.read_nodes(hash);
-		let who = self.system.rpc.request_order(&who);
-
-		for node in who.iter() {
-			let node_id = NodeID::from(*node);
-			let rpc = self.endpoint.call_streaming(
-				&node_id,
-				BlockRpc::GetBlock(*hash, order_tag),
-				PRIO_NORMAL | PRIO_SECONDARY,
-			);
-			tokio::select! {
-				res = rpc => {
-					let res = match res {
-						Ok(res) => res,
-						Err(e) => {
-							debug!("Node {:?} returned error: {}", node, e);
-							continue;
-						}
-					};
-					let (header, stream) = match res.into_parts() {
-						(Ok(BlockRpc::PutBlock { hash: _, header }), Some(stream)) => (header, stream),
-						_ => {
-							debug!("Node {:?} returned a malformed response", node);
-							continue;
-						}
-					};
-					return Ok((header, stream));
-				}
-				_ = tokio::time::sleep(self.system.rpc.rpc_timeout()) => {
-					debug!("Node {:?} didn't return block in time, trying next.", node);
-				}
-			};
-		}
-
-		Err(Error::Message(format!(
-			"Unable to read block {:?}: no node returned a valid block",
-			hash
-		)))
+		self.rpc_get_raw_block_internal(hash, order_tag, |header, stream| async move {
+			Ok((header, stream))
+		})
+		.await
 	}

 	/// Ask nodes that might have a (possibly compressed) block for it
@@ -251,6 +246,24 @@ impl BlockManager {
 		hash: &Hash,
 		order_tag: Option<OrderTag>,
 	) -> Result<DataBlock, Error> {
+		self.rpc_get_raw_block_internal(hash, order_tag, |header, stream| async move {
+			read_stream_to_end(stream)
+				.await
+				.map(|data| DataBlock::from_parts(header, data))
+		})
+		.await
+	}
+
+	async fn rpc_get_raw_block_internal<F, Fut, T>(
+		&self,
+		hash: &Hash,
+		order_tag: Option<OrderTag>,
+		f: F,
+	) -> Result<T, Error>
+	where
+		F: Fn(DataBlockHeader, ByteStream) -> Fut,
+		Fut: futures::Future<Output = Result<T, Error>>,
+	{
 		let who = self.replication.read_nodes(hash);
 		let who = self.system.rpc.request_order(&who);

@@ -266,34 +279,41 @@ impl BlockManager {
 					let res = match res {
 						Ok(res) => res,
 						Err(e) => {
-							debug!("Node {:?} returned error: {}", node, e);
+							debug!("Get block {:?}: node {:?} could not be contacted: {}", hash, node, e);
 							continue;
 						}
 					};
 					let (header, stream) = match res.into_parts() {
 						(Ok(BlockRpc::PutBlock { hash: _, header }), Some(stream)) => (header, stream),
-						_ => {
-							debug!("Node {:?} returned a malformed response", node);
+						(Ok(_), _) => {
+							debug!("Get block {:?}: node {:?} returned a malformed response", hash, node);
+							continue;
+						}
+						(Err(e), _) => {
+							debug!("Get block {:?}: node {:?} returned error: {}", hash, node, e);
 							continue;
 						}
 					};
-					match read_stream_to_end(stream).await {
-						Ok(bytes) => return Ok(DataBlock::from_parts(header, bytes)),
+					match f(header, stream).await {
+						Ok(ret) => return Ok(ret),
 						Err(e) => {
-							debug!("Error reading stream from node {:?}: {}", node, e);
+							debug!("Get block {:?}: error reading stream from node {:?}: {}", hash, node, e);
 						}
 					}
 				}
+				// TODO: sleep less long (fail early), initiate a second request earlier
+				// if the first one doesn't succeed rapidly
+				// TODO: keep first request running when initiating a new one and take the
+				// one that finishes earlier
 				_ = tokio::time::sleep(self.system.rpc.rpc_timeout()) => {
-					debug!("Node {:?} didn't return block in time, trying next.", node);
+					debug!("Get block {:?}: node {:?} didn't return block in time, trying next.", hash, node);
 				}
 			};
 		}

-		Err(Error::Message(format!(
-			"Unable to read block {:?}: no node returned a valid block",
-			hash
-		)))
+		let msg = format!("Get block {:?}: no node returned a valid block", hash);
+		debug!("{}", msg);
+		Err(Error::Message(msg))
 	}

 	// ---- Public interface ----
@@ -471,8 +491,6 @@ impl BlockManager {
 	pub(crate) async fn write_block(&self, hash: &Hash, data: &DataBlock) -> Result<(), Error> {
 		let tracer = opentelemetry::global::tracer("garage");

-		let write_size = data.inner_buffer().len() as u64;
-
 		self.lock_mutate(hash)
 			.await
 			.write_block(hash, data, self)
@@ -482,8 +500,6 @@ impl BlockManager {
 			))
 			.await?;

-		self.metrics.bytes_written.add(write_size);
-
 		Ok(())
 	}

@@ -510,36 +526,42 @@ impl BlockManager {

 	/// Read block from disk, verifying it's integrity
 	pub(crate) async fn read_block(&self, hash: &Hash) -> Result<DataBlock, Error> {
-		let data = self
-			.read_block_internal(hash)
-			.bound_record_duration(&self.metrics.block_read_duration)
-			.await?;
-
-		self.metrics
-			.bytes_read
-			.add(data.inner_buffer().len() as u64);
-
-		Ok(data)
+		let tracer = opentelemetry::global::tracer("garage");
+		async {
+			match self.find_block(hash).await {
+				Some(p) => self.read_block_from(hash, &p).await,
+				None => {
+					// Not found but maybe we should have had it ??
+					self.resync
+						.put_to_resync(hash, 2 * self.system.rpc.rpc_timeout())?;
+					return Err(Error::Message(format!(
+						"block {:?} not found on node",
+						hash
+					)));
+				}
+			}
+		}
+		.bound_record_duration(&self.metrics.block_read_duration)
+		.with_context(Context::current_with_span(
+			tracer.start("BlockManager::read_block"),
+		))
+		.await
 	}

-	async fn read_block_internal(&self, hash: &Hash) -> Result<DataBlock, Error> {
-		let mut path = self.block_path(hash);
-		let compressed = match self.is_block_compressed(hash).await {
-			Ok(c) => c,
-			Err(e) => {
-				// Not found but maybe we should have had it ??
-				self.resync
-					.put_to_resync(hash, 2 * self.system.rpc.rpc_timeout())?;
-				return Err(Into::into(e));
-			}
+	pub(crate) async fn read_block_from(
+		&self,
+		hash: &Hash,
+		block_path: &DataBlockPath,
+	) -> Result<DataBlock, Error> {
+		let (path, compressed) = match block_path {
+			DataBlockPath::Plain(p) => (p, false),
+			DataBlockPath::Compressed(p) => (p, true),
 		};
-		if compressed {
-			path.set_extension("zst");
-		}
-		let mut f = fs::File::open(&path).await?;

+		let mut f = fs::File::open(&path).await?;
 		let mut data = vec![];
 		f.read_to_end(&mut data).await?;
+		self.metrics.bytes_read.add(data.len() as u64);
 		drop(f);

 		let data = if compressed {
@@ -551,29 +573,27 @@ impl BlockManager {
 		if data.verify(*hash).is_err() {
 			self.metrics.corruption_counter.add(1);

+			warn!(
+				"Block {:?} is corrupted. Renaming to .corrupted and resyncing.",
+				hash
+			);
 			self.lock_mutate(hash)
 				.await
-				.move_block_to_corrupted(hash, self)
+				.move_block_to_corrupted(block_path)
 				.await?;
 			self.resync.put_to_resync(hash, Duration::from_millis(0))?;
+
 			return Err(Error::CorruptData(*hash));
 		}

 		Ok(data)
 	}

-	/// Check if this node has a block and whether it needs it
-	pub(crate) async fn check_block_status(&self, hash: &Hash) -> Result<BlockStatus, Error> {
-		self.lock_mutate(hash)
-			.await
-			.check_block_status(hash, self)
-			.await
-	}
-
 	/// Check if this node should have a block, but don't actually have it
 	async fn need_block(&self, hash: &Hash) -> Result<bool, Error> {
-		let BlockStatus { exists, needed } = self.check_block_status(hash).await?;
-		Ok(needed.is_nonzero() && !exists)
+		let rc = self.rc.get_block_rc(hash)?;
+		let exists = self.find_block(hash).await.is_some();
+		Ok(rc.is_nonzero() && !exists)
 	}

 	/// Delete block if it is not needed anymore
@@ -584,59 +604,65 @@ impl BlockManager {
 			.await
 	}

-	/// Utility: gives the path of the directory in which a block should be found
-	fn block_dir(&self, hash: &Hash) -> PathBuf {
-		let mut path = self.data_dir.clone();
-		path.push(hex::encode(&hash.as_slice()[0..1]));
-		path.push(hex::encode(&hash.as_slice()[1..2]));
-		path
-	}
+	/// Find the path where a block is currently stored
+	pub(crate) async fn find_block(&self, hash: &Hash) -> Option<DataBlockPath> {
+		let data_layout = self.data_layout.load_full();
+		let dirs = Some(data_layout.primary_block_dir(hash))
+			.into_iter()
+			.chain(data_layout.secondary_block_dirs(hash));
+		let filename = hex::encode(hash.as_ref());

-	/// Utility: give the full path where a block should be found, minus extension if block is
-	/// compressed
-	fn block_path(&self, hash: &Hash) -> PathBuf {
-		let mut path = self.block_dir(hash);
-		path.push(hex::encode(hash.as_ref()));
-		path
-	}
+		for dir in dirs {
+			let mut path = dir;
+			path.push(&filename);

-	/// Utility: check if block is stored compressed. Error if block is not stored
-	async fn is_block_compressed(&self, hash: &Hash) -> Result<bool, Error> {
-		let mut path = self.block_path(hash);
-
-		// If compression is disabled on node - check for the raw block
-		// first and then a compressed one (as compression may have been
-		// previously enabled).
-		match self.compression_level {
-			None => {
+			if self.compression_level.is_none() {
+				// If compression is disabled on node - check for the raw block
+				// first and then a compressed one (as compression may have been
+				// previously enabled).
 				if fs::metadata(&path).await.is_ok() {
-					return Ok(false);
+					return Some(DataBlockPath::Plain(path));
 				}
-
 				path.set_extension("zst");
-
-				fs::metadata(&path).await.map(|_| true).map_err(Into::into)
-			}
-			_ => {
-				path.set_extension("zst");
-
 				if fs::metadata(&path).await.is_ok() {
-					return Ok(true);
+					return Some(DataBlockPath::Compressed(path));
+				}
+			} else {
+				path.set_extension("zst");
+				if fs::metadata(&path).await.is_ok() {
+					return Some(DataBlockPath::Compressed(path));
 				}
-
 				path.set_extension("");
-
-				fs::metadata(&path).await.map(|_| false).map_err(Into::into)
+				if fs::metadata(&path).await.is_ok() {
+					return Some(DataBlockPath::Plain(path));
+				}
 			}
 		}
+
+		None
+	}
+
+	/// Rewrite a block at the primary location for its path and delete the old path.
+	/// Returns the number of bytes read/written
+	pub(crate) async fn fix_block_location(
+		&self,
+		hash: &Hash,
+		wrong_path: DataBlockPath,
+	) -> Result<usize, Error> {
+		self.lock_mutate(hash)
+			.await
+			.fix_block_location(hash, wrong_path, self)
+			.await
 	}

 	async fn lock_mutate(&self, hash: &Hash) -> MutexGuard<'_, BlockManagerLocked> {
 		let tracer = opentelemetry::global::tracer("garage");
-		self.mutation_lock[hash.as_slice()[0] as usize]
+		let ilock = u16::from_be_bytes([hash.as_slice()[0], hash.as_slice()[1]]) as usize
+			% self.mutation_lock.len();
+		self.mutation_lock[ilock]
 			.lock()
 			.with_context(Context::current_with_span(
-				tracer.start("Acquire mutation_lock"),
+				tracer.start(format!("Acquire mutation_lock #{}", ilock)),
 			))
 			.await
 	}
@@ -649,7 +675,7 @@ impl StreamingEndpointHandler<BlockRpc> for BlockManager {
 			BlockRpc::PutBlock { hash, header } => Resp::new(
 				self.handle_put_block(*hash, *header, message.take_stream())
 					.await
-					.map(|_| BlockRpc::Ok),
+					.map(|()| BlockRpc::Ok),
 			),
 			BlockRpc::GetBlock(h, order_tag) => self.handle_get_block(h, *order_tag).await,
 			BlockRpc::NeedBlockQuery(h) => {
@@ -660,62 +686,70 @@ impl StreamingEndpointHandler<BlockRpc> for BlockManager {
 	}
 }

-pub(crate) struct BlockStatus {
-	pub(crate) exists: bool,
-	pub(crate) needed: RcEntry,
-}
-
 impl BlockManagerLocked {
-	async fn check_block_status(
-		&self,
-		hash: &Hash,
-		mgr: &BlockManager,
-	) -> Result<BlockStatus, Error> {
-		let exists = mgr.is_block_compressed(hash).await.is_ok();
-		let needed = mgr.rc.get_block_rc(hash)?;
-
-		Ok(BlockStatus { exists, needed })
-	}
-
 	async fn write_block(
 		&self,
 		hash: &Hash,
 		data: &DataBlock,
 		mgr: &BlockManager,
+	) -> Result<(), Error> {
+		let existing_path = mgr.find_block(hash).await;
+		self.write_block_inner(hash, data, mgr, existing_path).await
+	}
+
+	async fn write_block_inner(
+		&self,
+		hash: &Hash,
+		data: &DataBlock,
+		mgr: &BlockManager,
+		existing_path: Option<DataBlockPath>,
 	) -> Result<(), Error> {
 		let compressed = data.is_compressed();
 		let data = data.inner_buffer();

-		let mut path = mgr.block_dir(hash);
-		let directory = path.clone();
-		path.push(hex::encode(hash));
+		let directory = mgr.data_layout.load().primary_block_dir(hash);

-		fs::create_dir_all(&directory).await?;
+		let mut tgt_path = directory.clone();
+		tgt_path.push(hex::encode(hash));
+		if compressed {
+			tgt_path.set_extension("zst");
+		}

-		let to_delete = match (mgr.is_block_compressed(hash).await, compressed) {
-			(Ok(true), _) => return Ok(()),
-			(Ok(false), false) => return Ok(()),
-			(Ok(false), true) => {
-				let path_to_delete = path.clone();
-				path.set_extension("zst");
-				Some(path_to_delete)
-			}
-			(Err(_), compressed) => {
-				if compressed {
-					path.set_extension("zst");
-				}
-				None
-			}
+		let to_delete = match (existing_path, compressed) {
+			// If the block is stored in the wrong directory,
+			// write it again at the correct path and delete the old path
+			(Some(DataBlockPath::Plain(p)), false) if p != tgt_path => Some(p),
+			(Some(DataBlockPath::Compressed(p)), true) if p != tgt_path => Some(p),
+
+			// If the block is already stored not compressed but we have a compressed
+			// copy, write the compressed copy and delete the uncompressed one
+			(Some(DataBlockPath::Plain(plain_path)), true) => Some(plain_path),
+
+			// If the block is already stored compressed,
+			// keep the stored copy, we have nothing to do
+			(Some(DataBlockPath::Compressed(_)), _) => return Ok(()),
+
+			// If the block is already stored not compressed,
+			// and we don't have a compressed copy either,
+			// keep the stored copy, we have nothing to do
+			(Some(DataBlockPath::Plain(_)), false) => return Ok(()),
+
+			// If the block isn't stored already, just store what is given to us
+			(None, _) => None,
 		};
+		assert!(to_delete.as_ref() != Some(&tgt_path));

-		let mut path_tmp = path.clone();
+		let mut path_tmp = tgt_path.clone();
 		let tmp_extension = format!("tmp{}", hex::encode(thread_rng().gen::<[u8; 4]>()));
 		path_tmp.set_extension(tmp_extension);

+		fs::create_dir_all(&directory).await?;
+
 		let mut delete_on_drop = DeleteOnDrop(Some(path_tmp.clone()));

 		let mut f = fs::File::create(&path_tmp).await?;
 		f.write_all(data).await?;
+		mgr.metrics.bytes_written.add(data.len() as u64);

 		if mgr.data_fsync {
 			f.sync_all().await?;
@@ -723,7 +757,7 @@ impl BlockManagerLocked {

 		drop(f);

-		fs::rename(path_tmp, path).await?;
+		fs::rename(path_tmp, tgt_path).await?;

 		delete_on_drop.cancel();

@@ -749,36 +783,49 @@ impl BlockManagerLocked {
 		Ok(())
 	}

-	async fn move_block_to_corrupted(&self, hash: &Hash, mgr: &BlockManager) -> Result<(), Error> {
-		warn!(
-			"Block {:?} is corrupted. Renaming to .corrupted and resyncing.",
-			hash
-		);
-		let mut path = mgr.block_path(hash);
-		let mut path2 = path.clone();
-		if mgr.is_block_compressed(hash).await? {
-			path.set_extension("zst");
-			path2.set_extension("zst.corrupted");
-		} else {
-			path2.set_extension("corrupted");
-		}
+	async fn move_block_to_corrupted(&self, block_path: &DataBlockPath) -> Result<(), Error> {
+		let (path, path2) = match block_path {
+			DataBlockPath::Plain(p) => {
+				let mut p2 = p.clone();
+				p2.set_extension("corrupted");
+				(p, p2)
+			}
+			DataBlockPath::Compressed(p) => {
+				let mut p2 = p.clone();
+				p2.set_extension("zst.corrupted");
+				(p, p2)
+			}
+		};
+
 		fs::rename(path, path2).await?;
 		Ok(())
 	}

 	async fn delete_if_unneeded(&self, hash: &Hash, mgr: &BlockManager) -> Result<(), Error> {
-		let BlockStatus { exists, needed } = self.check_block_status(hash, mgr).await?;
-
-		if exists && needed.is_deletable() {
-			let mut path = mgr.block_path(hash);
-			if mgr.is_block_compressed(hash).await? {
-				path.set_extension("zst");
+		let rc = mgr.rc.get_block_rc(hash)?;
+		if rc.is_deletable() {
+			while let Some(path) = mgr.find_block(hash).await {
+				let path = match path {
+					DataBlockPath::Plain(p) | DataBlockPath::Compressed(p) => p,
+				};
+				fs::remove_file(path).await?;
+				mgr.metrics.delete_counter.add(1);
 			}
-			fs::remove_file(path).await?;
-			mgr.metrics.delete_counter.add(1);
 		}
 		Ok(())
 	}
+
+	async fn fix_block_location(
+		&self,
+		hash: &Hash,
+		wrong_path: DataBlockPath,
+		mgr: &BlockManager,
+	) -> Result<usize, Error> {
+		let data = mgr.read_block_from(hash, &wrong_path).await?;
+		self.write_block_inner(hash, &data, mgr, Some(wrong_path))
+			.await?;
+		Ok(data.inner_buffer().len())
+	}
 }

 async fn read_stream_to_end(mut stream: ByteStream) -> Result<Bytes, Error> {
@@ -56,7 +56,7 @@ impl BlockRc {
 	/// deletion time has passed
 	pub(crate) fn clear_deleted_block_rc(&self, hash: &Hash) -> Result<(), Error> {
 		let now = now_msec();
-		self.rc.db().transaction(|mut tx| {
+		self.rc.db().transaction(|tx| {
 			let rcval = RcEntry::parse_opt(tx.get(&self.rc, hash)?);
 			match rcval {
 				RcEntry::Deletable { at_time } if now > at_time => {
@@ -64,7 +64,7 @@ impl BlockRc {
 				}
 				_ => (),
 			};
-			tx.commit(())
+			Ok(())
 		})?;
 		Ok(())
 	}
@@ -17,6 +17,7 @@ use garage_util::persister::PersisterShared;
 use garage_util::time::*;
 use garage_util::tranquilizer::Tranquilizer;

+use crate::block::*;
 use crate::manager::*;

 // Full scrub every 25 days with a random element of 10 days mixed in below
@@ -136,7 +137,7 @@ impl Worker for RepairWorker {
 				// Lists all blocks on disk and adds them to the resync queue.
 				// This allows us to find blocks we are storing but don't actually need,
 				// so that we can offload them if necessary and then delete them locally.
-				if let Some(hash) = bi.next().await? {
+				if let Some((_path, hash)) = bi.next().await? {
 					self.manager
 						.resync
 						.put_to_resync(&hash, Duration::from_secs(0))?;
@@ -175,7 +176,9 @@ mod v081 {
 }

 mod v082 {
+	use garage_util::data::Hash;
 	use serde::{Deserialize, Serialize};
+	use std::path::PathBuf;

 	use super::v081;

@@ -185,6 +188,27 @@ mod v082 {
 		pub(crate) time_last_complete_scrub: u64,
 		pub(crate) time_next_run_scrub: u64,
 		pub(crate) corruptions_detected: u64,
+		#[serde(default)]
+		pub(crate) checkpoint: Option<BlockStoreIterator>,
+	}
+
+	#[derive(Serialize, Deserialize, Clone)]
+	pub struct BlockStoreIterator {
+		pub todo: Vec<BsiTodo>,
+	}
+
+	#[derive(Serialize, Deserialize, Clone)]
+	pub enum BsiTodo {
+		Directory {
+			path: PathBuf,
+			progress_min: u64,
+			progress_max: u64,
+		},
+		File {
+			path: PathBuf,
+			hash: Hash,
+			progress: u64,
+		},
 	}

 	impl garage_util::migrate::Migrate for ScrubWorkerPersisted {
@@ -199,6 +223,7 @@ mod v082 {
 				time_last_complete_scrub: old.time_last_complete_scrub,
 				time_next_run_scrub: randomize_next_scrub_run_time(old.time_last_complete_scrub),
 				corruptions_detected: old.corruptions_detected,
+				checkpoint: None,
 			}
 		}
 	}
@@ -235,14 +260,23 @@ impl Default for ScrubWorkerPersisted {
 			time_next_run_scrub: randomize_next_scrub_run_time(now_msec()),
 			tranquility: INITIAL_SCRUB_TRANQUILITY,
 			corruptions_detected: 0,
+			checkpoint: None,
 		}
 	}
 }

 #[derive(Default)]
 enum ScrubWorkerState {
-	Running(BlockStoreIterator),
-	Paused(BlockStoreIterator, u64), // u64 = time when to resume scrub
+	Running {
+		iterator: BlockStoreIterator,
+		// time of the last checkpoint
+		t_cp: u64,
+	},
+	Paused {
+		iterator: BlockStoreIterator,
+		// time at which the scrub should be resumed
+		t_resume: u64,
+	},
 	#[default]
 	Finished,
 }
@@ -261,10 +295,17 @@ impl ScrubWorker {
 		rx_cmd: mpsc::Receiver<ScrubWorkerCommand>,
 		persister: PersisterShared<ScrubWorkerPersisted>,
 	) -> Self {
+		let work = match persister.get_with(|x| x.checkpoint.clone()) {
+			None => ScrubWorkerState::Finished,
+			Some(iterator) => ScrubWorkerState::Running {
+				iterator,
+				t_cp: now_msec(),
+			},
+		};
 		Self {
 			manager,
 			rx_cmd,
-			work: ScrubWorkerState::Finished,
+			work,
 			tranquilizer: Tranquilizer::new(30),
 			persister,
 		}
@@ -277,7 +318,16 @@ impl ScrubWorker {
 					ScrubWorkerState::Finished => {
 						info!("Scrub worker initializing, now performing datastore scrub");
 						let iterator = BlockStoreIterator::new(&self.manager);
-						ScrubWorkerState::Running(iterator)
+						if let Err(e) = self
+							.persister
+							.set_with(|x| x.checkpoint = Some(iterator.clone()))
+						{
+							error!("Could not save scrub checkpoint: {}", e);
+						}
+						ScrubWorkerState::Running {
+							iterator,
+							t_cp: now_msec(),
+						}
 					}
 					work => {
 						error!("Cannot start scrub worker: already running!");
@@ -287,8 +337,18 @@ impl ScrubWorker {
 			}
 			ScrubWorkerCommand::Pause(dur) => {
 				self.work = match std::mem::take(&mut self.work) {
-					ScrubWorkerState::Running(it) | ScrubWorkerState::Paused(it, _) => {
-						ScrubWorkerState::Paused(it, now_msec() + dur.as_millis() as u64)
+					ScrubWorkerState::Running { iterator, .. }
+					| ScrubWorkerState::Paused { iterator, .. } => {
+						if let Err(e) = self
+							.persister
+							.set_with(|x| x.checkpoint = Some(iterator.clone()))
+						{
+							error!("Could not save scrub checkpoint: {}", e);
+						}
+						ScrubWorkerState::Paused {
+							iterator,
+							t_resume: now_msec() + dur.as_millis() as u64,
+						}
 					}
 					work => {
 						error!("Cannot pause scrub worker: not running!");
@@ -298,7 +358,10 @@ impl ScrubWorker {
 			}
 			ScrubWorkerCommand::Resume => {
 				self.work = match std::mem::take(&mut self.work) {
-					ScrubWorkerState::Paused(it, _) => ScrubWorkerState::Running(it),
+					ScrubWorkerState::Paused { iterator, .. } => ScrubWorkerState::Running {
+						iterator,
+						t_cp: now_msec(),
+					},
 					work => {
 						error!("Cannot resume scrub worker: not paused!");
 						work
@@ -307,7 +370,10 @@ impl ScrubWorker {
 			}
 			ScrubWorkerCommand::Cancel => {
 				self.work = match std::mem::take(&mut self.work) {
-					ScrubWorkerState::Running(_) | ScrubWorkerState::Paused(_, _) => {
+					ScrubWorkerState::Running { .. } | ScrubWorkerState::Paused { .. } => {
+						if let Err(e) = self.persister.set_with(|x| x.checkpoint = None) {
+							error!("Could not save scrub checkpoint: {}", e);
+						}
 						ScrubWorkerState::Finished
 					}
 					work => {
@@ -343,12 +409,15 @@ impl Worker for ScrubWorker {
 			..Default::default()
 		};
 		match &self.work {
-			ScrubWorkerState::Running(bsi) => {
-				s.progress = Some(format!("{:.2}%", bsi.progress() * 100.));
+			ScrubWorkerState::Running { iterator, .. } => {
+				s.progress = Some(format!("{:.2}%", iterator.progress() * 100.));
 			}
-			ScrubWorkerState::Paused(bsi, rt) => {
-				s.progress = Some(format!("{:.2}%", bsi.progress() * 100.));
-				s.freeform = vec![format!("Scrub paused, resumes at {}", msec_to_rfc3339(*rt))];
+			ScrubWorkerState::Paused { iterator, t_resume } => {
+				s.progress = Some(format!("{:.2}%", iterator.progress() * 100.));
+				s.freeform = vec![format!(
+					"Scrub paused, resumes at {}",
+					msec_to_rfc3339(*t_resume)
+				)];
 			}
 			ScrubWorkerState::Finished => {
 				s.freeform = vec![
@@ -374,9 +443,11 @@ impl Worker for ScrubWorker {
 		};

 		match &mut self.work {
-			ScrubWorkerState::Running(bsi) => {
+			ScrubWorkerState::Running { iterator, t_cp } => {
 				self.tranquilizer.reset();
-				if let Some(hash) = bsi.next().await? {
+				let now = now_msec();
+
+				if let Some((_path, hash)) = iterator.next().await? {
 					match self.manager.read_block(&hash).await {
 						Err(Error::CorruptData(_)) => {
 							error!("Found corrupt data block during scrub: {:?}", hash);
@@ -385,16 +456,23 @@ impl Worker for ScrubWorker {
 						Err(e) => return Err(e),
 						_ => (),
 					};
+
+					if now - *t_cp > 60 * 1000 {
+						self.persister
+							.set_with(|p| p.checkpoint = Some(iterator.clone()))?;
+						*t_cp = now;
+					}
+
 					Ok(self
 						.tranquilizer
 						.tranquilize_worker(self.persister.get_with(|p| p.tranquility)))
 				} else {
-					let now = now_msec();
 					let next_scrub_timestamp = randomize_next_scrub_run_time(now);

 					self.persister.set_with(|p| {
 						p.time_last_complete_scrub = now;
 						p.time_next_run_scrub = next_scrub_timestamp;
+						p.checkpoint = None;
 					})?;
 					self.work = ScrubWorkerState::Finished;
 					self.tranquilizer.clear();
@@ -413,8 +491,8 @@ impl Worker for ScrubWorker {

 	async fn wait_for_work(&mut self) -> WorkerState {
 		let (wait_until, command) = match &self.work {
-			ScrubWorkerState::Running(_) => return WorkerState::Busy,
-			ScrubWorkerState::Paused(_, resume_time) => (*resume_time, ScrubWorkerCommand::Resume),
+			ScrubWorkerState::Running { .. } => return WorkerState::Busy,
+			ScrubWorkerState::Paused { t_resume, .. } => (*t_resume, ScrubWorkerCommand::Resume),
 			ScrubWorkerState::Finished => (
 				self.persister.get_with(|p| p.time_next_run_scrub),
 				ScrubWorkerCommand::Start,
@@ -437,110 +515,250 @@ impl Worker for ScrubWorker {
 		}

 		match &self.work {
-			ScrubWorkerState::Running(_) => WorkerState::Busy,
+			ScrubWorkerState::Running { .. } => WorkerState::Busy,
 			_ => WorkerState::Idle,
 		}
 	}
 }

+// ---- ---- ----
+// THIRD KIND OF REPAIR: REBALANCING DATA BLOCKS
+// between multiple storage locations.
+// This is a one-shot repair operation that can be launched,
+// checks everything, and then exits.
+// ---- ---- ----
+
+pub struct RebalanceWorker {
+	manager: Arc<BlockManager>,
+	block_iter: BlockStoreIterator,
+	t_started: u64,
+	t_finished: Option<u64>,
+	moved: usize,
+	moved_bytes: u64,
+}
+
+impl RebalanceWorker {
+	pub fn new(manager: Arc<BlockManager>) -> Self {
+		let block_iter = BlockStoreIterator::new(&manager);
+		Self {
+			manager,
+			block_iter,
+			t_started: now_msec(),
+			t_finished: None,
+			moved: 0,
+			moved_bytes: 0,
+		}
+	}
+}
+
+#[async_trait]
+impl Worker for RebalanceWorker {
+	fn name(&self) -> String {
+		"Block rebalance worker".into()
+	}
+
+	fn status(&self) -> WorkerStatus {
+		let t_cur = self.t_finished.unwrap_or_else(|| now_msec());
+		let rate = self.moved_bytes / std::cmp::max(1, (t_cur - self.t_started) / 1000);
+		let mut freeform = vec![
+			format!("Blocks moved: {}", self.moved),
+			format!(
+				"Bytes moved: {} ({}/s)",
+				bytesize::ByteSize::b(self.moved_bytes),
+				bytesize::ByteSize::b(rate)
+			),
+			format!("Started: {}", msec_to_rfc3339(self.t_started)),
+		];
+		if let Some(t_fin) = self.t_finished {
+			freeform.push(format!("Finished: {}", msec_to_rfc3339(t_fin)))
+		}
+		WorkerStatus {
+			progress: Some(format!("{:.2}%", self.block_iter.progress() * 100.)),
+			freeform,
+			..Default::default()
+		}
+	}
+
+	async fn work(&mut self, _must_exit: &mut watch::Receiver<bool>) -> Result<WorkerState, Error> {
+		if let Some((path, hash)) = self.block_iter.next().await? {
+			let prim_loc = self.manager.data_layout.load().primary_block_dir(&hash);
+			if path.ancestors().all(|x| x != prim_loc) {
+				let block_path = match path.extension() {
+					None => DataBlockPath::Plain(path.clone()),
+					Some(x) if x.to_str() == Some("zst") => DataBlockPath::Compressed(path.clone()),
+					_ => {
+						warn!("not rebalancing file: {}", path.to_string_lossy());
+						return Ok(WorkerState::Busy);
+					}
+				};
+				// block is not in its primary location,
+				// move it there (reading and re-writing does the trick)
+				debug!("rebalance: moving block {:?} => {:?}", block_path, prim_loc);
+				let block_len = self.manager.fix_block_location(&hash, block_path).await?;
+				self.moved += 1;
+				self.moved_bytes += block_len as u64;
+			}
+			Ok(WorkerState::Busy)
+		} else {
+			// all blocks are in their primary location:
+			// - the ones we moved now are
+			// - the ones written in the meantime always were, because we only
+			//   write to primary locations
+			// so we can safely remove all secondary locations from the data layout
+			let new_layout = self
+				.manager
+				.data_layout
+				.load_full()
+				.without_secondary_locations();
+			self.manager
+				.data_layout_persister
+				.save_async(&new_layout)
+				.await?;
+			self.manager.data_layout.store(Arc::new(new_layout));
+			self.t_finished = Some(now_msec());
+			Ok(WorkerState::Done)
+		}
+	}
+
+	async fn wait_for_work(&mut self) -> WorkerState {
+		unreachable!()
+	}
+}
+
 // ---- ---- ----
 // UTILITY FOR ENUMERATING THE BLOCK STORE
 // ---- ---- ----

-struct BlockStoreIterator {
-	path: Vec<ReadingDir>,
-}
-
-enum ReadingDir {
-	Pending(PathBuf),
-	Read {
-		subpaths: Vec<fs::DirEntry>,
-		pos: usize,
-	},
-}
+const PROGRESS_FP: u64 = 1_000_000_000;

 impl BlockStoreIterator {
 	fn new(manager: &BlockManager) -> Self {
-		let root_dir = manager.data_dir.clone();
-		Self {
-			path: vec![ReadingDir::Pending(root_dir)],
+		let data_layout = manager.data_layout.load_full();
+
+		let mut dir_cap = vec![0; data_layout.data_dirs.len()];
+		for prim in data_layout.part_prim.iter() {
+			dir_cap[*prim as usize] += 1;
 		}
+		for sec_vec in data_layout.part_sec.iter() {
+			for sec in sec_vec.iter() {
+				dir_cap[*sec as usize] += 1;
+			}
+		}
+		let sum_cap = dir_cap.iter().sum::<usize>() as u64;
+
+		let mut cum_cap = 0;
+		let mut todo = vec![];
+		for (dir, cap) in data_layout.data_dirs.iter().zip(dir_cap.into_iter()) {
+			let progress_min = (cum_cap * PROGRESS_FP) / sum_cap;
+			let progress_max = ((cum_cap + cap as u64) * PROGRESS_FP) / sum_cap;
+			cum_cap += cap as u64;
+
+			todo.push(BsiTodo::Directory {
+				path: dir.path.clone(),
+				progress_min,
+				progress_max,
+			});
+		}
+		// entries are processed back-to-front (because of .pop()),
+		// so reverse entries to process them in increasing progress bounds
+		todo.reverse();
+
+		let ret = Self { todo };
+		debug_assert!(ret.progress_invariant());
+
+		ret
 	}

 	/// Returns progress done, between 0 and 1
 	fn progress(&self) -> f32 {
-		if self.path.is_empty() {
-			1.0
-		} else {
-			let mut ret = 0.0;
-			let mut next_div = 1;
-			for p in self.path.iter() {
-				match p {
-					ReadingDir::Pending(_) => break,
-					ReadingDir::Read { subpaths, pos } => {
-						next_div *= subpaths.len();
-						ret += ((*pos - 1) as f32) / (next_div as f32);
+		self.todo
+			.last()
+			.map(|x| match x {
+				BsiTodo::Directory { progress_min, .. } => *progress_min,
+				BsiTodo::File { progress, .. } => *progress,
+			})
+			.map(|x| x as f32 / PROGRESS_FP as f32)
+			.unwrap_or(1.0)
+	}
+
+	async fn next(&mut self) -> Result<Option<(PathBuf, Hash)>, Error> {
+		loop {
+			match self.todo.pop() {
+				None => return Ok(None),
+				Some(BsiTodo::Directory {
+					path,
+					progress_min,
+					progress_max,
+				}) => {
+					let istart = self.todo.len();
+
+					let mut reader = fs::read_dir(&path).await?;
+					while let Some(ent) = reader.next_entry().await? {
+						let name = if let Ok(n) = ent.file_name().into_string() {
+							n
+						} else {
+							continue;
+						};
+						let ft = ent.file_type().await?;
+						if ft.is_dir() && hex::decode(&name).is_ok() {
+							self.todo.push(BsiTodo::Directory {
+								path: ent.path(),
+								progress_min: 0,
+								progress_max: 0,
+							});
+						} else if ft.is_file() {
+							let filename = name.split_once('.').map(|(f, _)| f).unwrap_or(&name);
+							if filename.len() == 64 {
+								if let Ok(h) = hex::decode(filename) {
+									let mut hash = [0u8; 32];
+									hash.copy_from_slice(&h);
+									self.todo.push(BsiTodo::File {
+										path: ent.path(),
+										hash: hash.into(),
+										progress: 0,
+									});
+								}
+							}
+						}
 					}
+
+					let count = self.todo.len() - istart;
+					for (i, ent) in self.todo[istart..].iter_mut().enumerate() {
+						let p1 = progress_min
+							+ ((progress_max - progress_min) * i as u64) / count as u64;
+						let p2 = progress_min
+							+ ((progress_max - progress_min) * (i + 1) as u64) / count as u64;
+						match ent {
+							BsiTodo::Directory {
+								progress_min,
+								progress_max,
+								..
+							} => {
+								*progress_min = p1;
+								*progress_max = p2;
+							}
+							BsiTodo::File { progress, .. } => {
+								*progress = p1;
+							}
+						}
+					}
+					self.todo[istart..].reverse();
+					debug_assert!(self.progress_invariant());
+				}
+				Some(BsiTodo::File { path, hash, .. }) => {
+					return Ok(Some((path, hash)));
 				}
 			}
-			ret
 		}
 	}

-	async fn next(&mut self) -> Result<Option<Hash>, Error> {
-		loop {
-			let last_path = match self.path.last_mut() {
-				None => return Ok(None),
-				Some(lp) => lp,
-			};
-
-			if let ReadingDir::Pending(path) = last_path {
-				let mut reader = fs::read_dir(&path).await?;
-				let mut subpaths = vec![];
-				while let Some(ent) = reader.next_entry().await? {
-					subpaths.push(ent);
-				}
-				*last_path = ReadingDir::Read { subpaths, pos: 0 };
-			}
-
-			let (subpaths, pos) = match *last_path {
-				ReadingDir::Read {
-					ref subpaths,
-					ref mut pos,
-				} => (subpaths, pos),
-				ReadingDir::Pending(_) => unreachable!(),
-			};
-
-			let data_dir_ent = match subpaths.get(*pos) {
-				None => {
-					self.path.pop();
-					continue;
-				}
-				Some(ent) => {
-					*pos += 1;
-					ent
-				}
-			};
-
-			let name = data_dir_ent.file_name();
-			let name = if let Ok(n) = name.into_string() {
-				n
-			} else {
-				continue;
-			};
-			let ent_type = data_dir_ent.file_type().await?;
-
-			let name = name.strip_suffix(".zst").unwrap_or(&name);
-			if name.len() == 2 && hex::decode(name).is_ok() && ent_type.is_dir() {
-				let path = data_dir_ent.path();
-				self.path.push(ReadingDir::Pending(path));
-			} else if name.len() == 64 {
-				if let Ok(h) = hex::decode(name) {
-					let mut hash = [0u8; 32];
-					hash.copy_from_slice(&h);
-					return Ok(Some(hash.into()));
-				}
-			}
-		}
+	// for debug_assert!
+	fn progress_invariant(&self) -> bool {
+		let iter = self.todo.iter().map(|x| match x {
+			BsiTodo::Directory { progress_min, .. } => progress_min,
+			BsiTodo::File { progress, .. } => progress,
+		});
+		let iter_1 = iter.clone().skip(1);
+		iter.zip(iter_1).all(|(prev, next)| prev >= next)
 	}
 }
@@ -41,7 +41,7 @@ pub(crate) const RESYNC_RETRY_DELAY: Duration = Duration::from_secs(60);
 pub(crate) const RESYNC_RETRY_DELAY_MAX_BACKOFF_POWER: u64 = 6;

 // No more than 4 resync workers can be running in the system
-pub(crate) const MAX_RESYNC_WORKERS: usize = 4;
+pub(crate) const MAX_RESYNC_WORKERS: usize = 8;
 // Resync tranquility is initially set to 2, but can be changed in the CLI
 // and the updated version is persisted over Garage restarts
 const INITIAL_RESYNC_TRANQUILITY: u32 = 2;
@@ -359,20 +359,23 @@ impl BlockResyncManager {
 	}

 	async fn resync_block(&self, manager: &BlockManager, hash: &Hash) -> Result<(), Error> {
-		let BlockStatus { exists, needed } = manager.check_block_status(hash).await?;
+		let existing_path = manager.find_block(hash).await;
+		let exists = existing_path.is_some();
+		let rc = manager.rc.get_block_rc(hash)?;

-		if exists != needed.is_needed() || exists != needed.is_nonzero() {
+		if exists != rc.is_needed() || exists != rc.is_nonzero() {
 			debug!(
 				"Resync block {:?}: exists {}, nonzero rc {}, deletable {}",
 				hash,
 				exists,
-				needed.is_nonzero(),
-				needed.is_deletable(),
+				rc.is_nonzero(),
+				rc.is_deletable(),
 			);
 		}

-		if exists && needed.is_deletable() {
+		if exists && rc.is_deletable() {
 			info!("Resync block {:?}: offloading and deleting", hash);
+			let existing_path = existing_path.unwrap();

 			let mut who = manager.replication.write_nodes(hash);
 			if who.len() < manager.replication.write_quorum() {
@@ -419,7 +422,7 @@ impl BlockResyncManager {
 						.add(1, &[KeyValue::new("to", format!("{:?}", node))]);
 				}

-				let block = manager.read_block(hash).await?;
+				let block = manager.read_block_from(hash, &existing_path).await?;
 				let (header, bytes) = block.into_parts();
 				let put_block_message = Req::new(BlockRpc::PutBlock {
 					hash: *hash,
@@ -451,7 +454,7 @@ impl BlockResyncManager {
 			manager.rc.clear_deleted_block_rc(hash)?;
 		}

-		if needed.is_nonzero() && !exists {
+		if rc.is_nonzero() && !exists {
 			info!(
 				"Resync block {:?}: fetching absent but needed block (refcount > 0)",
 				hash
@@ -1,6 +1,6 @@
 [package]
 name = "garage_db"
-version = "0.8.3"
+version = "0.8.4"
 authors = ["Alex Auvolat <alex@adnab.me>"]
 edition = "2018"
 license = "AGPL-3.0"
@@ -85,7 +85,7 @@ impl CountedTree {
 		let old_some = expected_old.is_some();
 		let new_some = new.is_some();

-		let tx_res = self.0.tree.db().transaction(|mut tx| {
+		let tx_res = self.0.tree.db().transaction(|tx| {
 			let old_val = tx.get(&self.0.tree, &key)?;
 			let is_same = match (&old_val, &expected_old) {
 				(None, None) => true,
@@ -101,9 +101,9 @@ impl CountedTree {
 						tx.remove(&self.0.tree, &key)?;
 					}
 				}
-				tx.commit(())
+				Ok(())
 			} else {
-				tx.abort(())
+				Err(TxError::Abort(()))
 			}
 		});

@@ -22,10 +22,15 @@ use std::sync::Arc;

 use err_derive::Error;

+pub(crate) type OnCommit = Vec<Box<dyn FnOnce()>>;
+
 #[derive(Clone)]
 pub struct Db(pub(crate) Arc<dyn IDb>);

-pub struct Transaction<'a>(&'a mut dyn ITx);
+pub struct Transaction<'a> {
+	tx: &'a mut dyn ITx,
+	on_commit: OnCommit,
+}

 #[derive(Clone)]
 pub struct Tree(Arc<dyn IDb>, usize);
@@ -85,7 +90,7 @@ impl Db {

 	pub fn transaction<R, E, F>(&self, fun: F) -> TxResult<R, E>
 	where
-		F: Fn(Transaction<'_>) -> TxResult<R, E>,
+		F: Fn(&mut Transaction<'_>) -> TxResult<R, E>,
 	{
 		let f = TxFn {
 			function: fun,
@@ -98,14 +103,17 @@ impl Db {
 			.expect("Transaction did not store result");

 		match tx_res {
-			Ok(()) => {
-				assert!(matches!(ret, Ok(_)));
-				ret
-			}
-			Err(TxError::Abort(())) => {
-				assert!(matches!(ret, Err(TxError::Abort(_))));
-				ret
-			}
+			Ok(on_commit) => match ret {
+				Ok(value) => {
+					on_commit.into_iter().for_each(|f| f());
+					Ok(value)
+				}
+				_ => unreachable!(),
+			},
+			Err(TxError::Abort(())) => match ret {
+				Err(TxError::Abort(e)) => Err(TxError::Abort(e)),
+				_ => unreachable!(),
+			},
 			Err(TxError::Db(e2)) => match ret {
 				// Ok was stored -> the error occured when finalizing
 				// transaction
@@ -139,7 +147,7 @@ impl Db {

 			let ex_tree = other.open_tree(&name)?;

-			let tx_res = self.transaction(|mut tx| {
+			let tx_res = self.transaction(|tx| {
 				let mut i = 0;
 				for item in ex_tree.iter().map_err(TxError::Abort)? {
 					let (k, v) = item.map_err(TxError::Abort)?;
@@ -149,7 +157,7 @@ impl Db {
 						println!("{}: imported {}", name, i);
 					}
 				}
-				tx.commit(i)
+				Ok(i)
 			});
 			let total = match tx_res {
 				Err(TxError::Db(e)) => return Err(e),
@@ -249,11 +257,11 @@ impl Tree {
 impl<'a> Transaction<'a> {
 	#[inline]
 	pub fn get<T: AsRef<[u8]>>(&self, tree: &Tree, key: T) -> TxOpResult<Option<Value>> {
-		self.0.get(tree.1, key.as_ref())
+		self.tx.get(tree.1, key.as_ref())
 	}
 	#[inline]
 	pub fn len(&self, tree: &Tree) -> TxOpResult<usize> {
-		self.0.len(tree.1)
+		self.tx.len(tree.1)
 	}

 	/// Returns the old value if there was one
@@ -264,21 +272,21 @@ impl<'a> Transaction<'a> {
 		key: T,
 		value: U,
 	) -> TxOpResult<Option<Value>> {
-		self.0.insert(tree.1, key.as_ref(), value.as_ref())
+		self.tx.insert(tree.1, key.as_ref(), value.as_ref())
 	}
 	/// Returns the old value if there was one
 	#[inline]
 	pub fn remove<T: AsRef<[u8]>>(&mut self, tree: &Tree, key: T) -> TxOpResult<Option<Value>> {
-		self.0.remove(tree.1, key.as_ref())
+		self.tx.remove(tree.1, key.as_ref())
 	}

 	#[inline]
 	pub fn iter(&self, tree: &Tree) -> TxOpResult<TxValueIter<'_>> {
-		self.0.iter(tree.1)
+		self.tx.iter(tree.1)
 	}
 	#[inline]
 	pub fn iter_rev(&self, tree: &Tree) -> TxOpResult<TxValueIter<'_>> {
-		self.0.iter_rev(tree.1)
+		self.tx.iter_rev(tree.1)
 	}

 	#[inline]
@@ -289,7 +297,7 @@ impl<'a> Transaction<'a> {
 	{
 		let sb = range.start_bound();
 		let eb = range.end_bound();
-		self.0.range(tree.1, get_bound(sb), get_bound(eb))
+		self.tx.range(tree.1, get_bound(sb), get_bound(eb))
 	}
 	#[inline]
 	pub fn range_rev<K, R>(&self, tree: &Tree, range: R) -> TxOpResult<TxValueIter<'_>>
@@ -299,19 +307,12 @@ impl<'a> Transaction<'a> {
 	{
 		let sb = range.start_bound();
 		let eb = range.end_bound();
-		self.0.range_rev(tree.1, get_bound(sb), get_bound(eb))
-	}
-
-	// ----
-
-	#[inline]
-	pub fn abort<R, E>(self, e: E) -> TxResult<R, E> {
-		Err(TxError::Abort(e))
+		self.tx.range_rev(tree.1, get_bound(sb), get_bound(eb))
 	}

 	#[inline]
-	pub fn commit<R, E>(self, r: R) -> TxResult<R, E> {
-		Ok(r)
+	pub fn on_commit<F: FnOnce() + 'static>(&mut self, f: F) {
+		self.on_commit.push(Box::new(f));
 	}
 }

@@ -348,7 +349,7 @@ pub(crate) trait IDb: Send + Sync {
 		high: Bound<&'r [u8]>,
 	) -> Result<ValueIter<'_>>;

-	fn transaction(&self, f: &dyn ITxFn) -> TxResult<(), ()>;
+	fn transaction(&self, f: &dyn ITxFn) -> TxResult<OnCommit, ()>;
 }

 pub(crate) trait ITx {
@@ -380,14 +381,14 @@ pub(crate) trait ITxFn {
 }

 pub(crate) enum TxFnResult {
-	Ok,
+	Ok(OnCommit),
 	Abort,
 	DbErr,
 }

 struct TxFn<F, R, E>
 where
-	F: Fn(Transaction<'_>) -> TxResult<R, E>,
+	F: Fn(&mut Transaction<'_>) -> TxResult<R, E>,
 {
 	function: F,
 	result: Cell<Option<TxResult<R, E>>>,
@@ -395,12 +396,16 @@ where

 impl<F, R, E> ITxFn for TxFn<F, R, E>
 where
-	F: Fn(Transaction<'_>) -> TxResult<R, E>,
+	F: Fn(&mut Transaction<'_>) -> TxResult<R, E>,
 {
 	fn try_on(&self, tx: &mut dyn ITx) -> TxFnResult {
-		let res = (self.function)(Transaction(tx));
+		let mut tx = Transaction {
+			tx,
+			on_commit: vec![],
+		};
+		let res = (self.function)(&mut tx);
 		let res2 = match &res {
-			Ok(_) => TxFnResult::Ok,
+			Ok(_) => TxFnResult::Ok(tx.on_commit),
 			Err(TxError::Abort(_)) => TxFnResult::Abort,
 			Err(TxError::Db(_)) => TxFnResult::DbErr,
 		};
@@ -9,8 +9,8 @@ use heed::types::ByteSlice;
 use heed::{BytesDecode, Env, RoTxn, RwTxn, UntypedDatabase as Database};

 use crate::{
-	Db, Error, IDb, ITx, ITxFn, Result, TxError, TxFnResult, TxOpError, TxOpResult, TxResult,
-	TxValueIter, Value, ValueIter,
+	Db, Error, IDb, ITx, ITxFn, OnCommit, Result, TxError, TxFnResult, TxOpError, TxOpResult,
+	TxResult, TxValueIter, Value, ValueIter,
 };

 pub use heed;
@@ -186,7 +186,7 @@ impl IDb for LmdbDb {

 	// ----

-	fn transaction(&self, f: &dyn ITxFn) -> TxResult<(), ()> {
+	fn transaction(&self, f: &dyn ITxFn) -> TxResult<OnCommit, ()> {
 		let trees = self.trees.read().unwrap();
 		let mut tx = LmdbTx {
 			trees: &trees.0[..],
@@ -199,9 +199,9 @@ impl IDb for LmdbDb {

 		let res = f.try_on(&mut tx);
 		match res {
-			TxFnResult::Ok => {
+			TxFnResult::Ok(on_commit) => {
 				tx.tx.commit().map_err(Error::from).map_err(TxError::Db)?;
-				Ok(())
+				Ok(on_commit)
 			}
 			TxFnResult::Abort => {
 				tx.tx.abort().map_err(Error::from).map_err(TxError::Db)?;
@@ -10,8 +10,8 @@ use sled::transaction::{
 };

 use crate::{
-	Db, Error, IDb, ITx, ITxFn, Result, TxError, TxFnResult, TxOpError, TxOpResult, TxResult,
-	TxValueIter, Value, ValueIter,
+	Db, Error, IDb, ITx, ITxFn, OnCommit, Result, TxError, TxFnResult, TxOpError, TxOpResult,
+	TxResult, TxValueIter, Value, ValueIter,
 };

 pub use sled;
@@ -166,7 +166,7 @@ impl IDb for SledDb {

 	// ----

-	fn transaction(&self, f: &dyn ITxFn) -> TxResult<(), ()> {
+	fn transaction(&self, f: &dyn ITxFn) -> TxResult<OnCommit, ()> {
 		let trees = self.trees.read().unwrap();
 		let res = trees.0.transaction(|txtrees| {
 			let mut tx = SledTx {
@@ -174,9 +174,9 @@ impl IDb for SledDb {
 				err: Cell::new(None),
 			};
 			match f.try_on(&mut tx) {
-				TxFnResult::Ok => {
+				TxFnResult::Ok(on_commit) => {
 					assert!(tx.err.into_inner().is_none());
-					Ok(())
+					Ok(on_commit)
 				}
 				TxFnResult::Abort => {
 					assert!(tx.err.into_inner().is_none());
@@ -189,7 +189,7 @@ impl IDb for SledDb {
 			}
 		});
 		match res {
-			Ok(()) => Ok(()),
+			Ok(on_commit) => Ok(on_commit),
 			Err(TransactionError::Abort(())) => Err(TxError::Abort(())),
 			Err(TransactionError::Storage(s)) => Err(TxError::Db(s.into())),
 		}
@@ -9,8 +9,8 @@ use std::sync::{Arc, Mutex, MutexGuard};
 use rusqlite::{params, Connection, Rows, Statement, Transaction};

 use crate::{
-	Db, Error, IDb, ITx, ITxFn, Result, TxError, TxFnResult, TxOpError, TxOpResult, TxResult,
-	TxValueIter, Value, ValueIter,
+	Db, Error, IDb, ITx, ITxFn, OnCommit, Result, TxError, TxFnResult, TxOpError, TxOpResult,
+	TxResult, TxValueIter, Value, ValueIter,
 };

 pub use rusqlite;
@@ -261,7 +261,7 @@ impl IDb for SqliteDb {

 	// ----

-	fn transaction(&self, f: &dyn ITxFn) -> TxResult<(), ()> {
+	fn transaction(&self, f: &dyn ITxFn) -> TxResult<OnCommit, ()> {
 		trace!("transaction: lock db");
 		let mut this = self.0.lock().unwrap();
 		trace!("transaction: lock acquired");
@@ -277,9 +277,9 @@ impl IDb for SqliteDb {
 			trees: &this_mut_ref.trees,
 		};
 		let res = match f.try_on(&mut tx) {
-			TxFnResult::Ok => {
+			TxFnResult::Ok(on_commit) => {
 				tx.tx.commit().map_err(Error::from).map_err(TxError::Db)?;
-				Ok(())
+				Ok(on_commit)
 			}
 			TxFnResult::Abort => {
 				tx.tx.rollback().map_err(Error::from).map_err(TxError::Db)?;
@@ -13,26 +13,26 @@ fn test_suite(db: Db) {
 	assert!(tree.insert(ka, va).unwrap().is_none());
 	assert_eq!(tree.get(ka).unwrap().unwrap(), va);

-	let res = db.transaction::<_, (), _>(|mut tx| {
+	let res = db.transaction::<_, (), _>(|tx| {
 		assert_eq!(tx.get(&tree, ka).unwrap().unwrap(), va);

 		assert_eq!(tx.insert(&tree, ka, vb).unwrap().unwrap(), va);

 		assert_eq!(tx.get(&tree, ka).unwrap().unwrap(), vb);

-		tx.commit(12)
+		Ok(12)
 	});
 	assert!(matches!(res, Ok(12)));
 	assert_eq!(tree.get(ka).unwrap().unwrap(), vb);

-	let res = db.transaction::<(), _, _>(|mut tx| {
+	let res = db.transaction::<(), _, _>(|tx| {
 		assert_eq!(tx.get(&tree, ka).unwrap().unwrap(), vb);

 		assert_eq!(tx.insert(&tree, ka, vc).unwrap().unwrap(), vb);

 		assert_eq!(tx.get(&tree, ka).unwrap().unwrap(), vc);

-		tx.abort(42)
+		Err(TxError::Abort(42))
 	});
 	assert!(matches!(res, Err(TxError::Abort(42))));
 	assert_eq!(tree.get(ka).unwrap().unwrap(), vb);
@@ -1,6 +1,6 @@
 [package]
 name = "garage"
-version = "0.8.3"
+version = "0.8.4"
 authors = ["Alex Auvolat <alex@adnab.me>"]
 edition = "2018"
 license = "AGPL-3.0"
@@ -174,16 +174,12 @@ pub async fn cmd_show_layout(
 	let layout = fetch_layout(rpc_cli, rpc_host).await?;

 	println!("==== CURRENT CLUSTER LAYOUT ====");
-	if !print_cluster_layout(&layout) {
-		println!("No nodes currently have a role in the cluster.");
-		println!("See `garage status` to view available nodes.");
-	}
+	print_cluster_layout(&layout, "No nodes currently have a role in the cluster.\nSee `garage status` to view available nodes.");
 	println!();
 	println!("Current cluster layout version: {}", layout.version);

 	let has_role_changes = print_staging_role_changes(&layout);
-	let has_param_changes = print_staging_parameters_changes(&layout);
-	if has_role_changes || has_param_changes {
+	if has_role_changes {
 		let v = layout.version;
 		let res_apply = layout.apply_staged_changes(Some(v + 1));

@@ -193,9 +189,7 @@ pub async fn cmd_show_layout(
 			Ok((layout, msg)) => {
 				println!();
 				println!("==== NEW CLUSTER LAYOUT AFTER APPLYING CHANGES ====");
-				if !print_cluster_layout(&layout) {
-					println!("No nodes have a role in the new layout.");
-				}
+				print_cluster_layout(&layout, "No nodes have a role in the new layout.");
 				println!();

 				for line in msg.iter() {
@@ -267,28 +261,35 @@ pub async fn cmd_config_layout(
 	let mut did_something = false;
 	match config_opt.redundancy {
 		None => (),
-		Some(r) => {
-			if r > layout.replication_factor {
-				println!(
-					"The zone redundancy must be smaller or equal to the \
-                replication factor ({}).",
-					layout.replication_factor
-				);
-			} else if r < 1 {
-				println!("The zone redundancy must be at least 1.");
-			} else {
-				layout
-					.staging_parameters
-					.update(LayoutParameters { zone_redundancy: r });
-				println!("The new zone redundancy has been saved ({}).", r);
+		Some(r_str) => {
+			let r = r_str
+				.parse::<ZoneRedundancy>()
+				.ok_or_message("invalid zone redundancy value")?;
+			if let ZoneRedundancy::AtLeast(r_int) = r {
+				if r_int > layout.replication_factor {
+					return Err(Error::Message(format!(
+						"The zone redundancy must be smaller or equal to the \
+                    replication factor ({}).",
+						layout.replication_factor
+					)));
+				} else if r_int < 1 {
+					return Err(Error::Message(
+						"The zone redundancy must be at least 1.".into(),
+					));
+				}
 			}
+
+			layout
+				.staging_parameters
+				.update(LayoutParameters { zone_redundancy: r });
+			println!("The zone redundancy parameter has been set to '{}'.", r);
 			did_something = true;
 		}
 	}

 	if !did_something {
 		return Err(Error::Message(
-			"Please specify an action for `garage layout config` to do".into(),
+			"Please specify an action for `garage layout config`".into(),
 		));
 	}

@@ -326,7 +327,7 @@ pub async fn send_layout(
 	Ok(())
 }

-pub fn print_cluster_layout(layout: &ClusterLayout) -> bool {
+pub fn print_cluster_layout(layout: &ClusterLayout, empty_msg: &str) {
 	let mut table = vec!["ID\tTags\tZone\tCapacity\tUsable capacity".to_string()];
 	for (id, _, role) in layout.roles.items().iter() {
 		let role = match &role.0 {
@@ -356,61 +357,54 @@ pub fn print_cluster_layout(layout: &ClusterLayout) -> bool {
 			));
 		};
 	}
-	println!();
-	println!("Parameters of the layout computation:");
-	println!("Zone redundancy: {}", layout.parameters.zone_redundancy);
-	println!();
-	if table.len() == 1 {
-		false
-	} else {
+	if table.len() > 1 {
 		format_table(table);
-		true
-	}
-}
-
-pub fn print_staging_parameters_changes(layout: &ClusterLayout) -> bool {
-	let has_changes = *layout.staging_parameters.get() != layout.parameters;
-	if has_changes {
-		println!();
-		println!("==== NEW LAYOUT PARAMETERS ====");
-		println!(
-			"Zone redundancy: {}",
-			layout.staging_parameters.get().zone_redundancy
-		);
 		println!();
+		println!("Zone redundancy: {}", layout.parameters.zone_redundancy);
+	} else {
+		println!("{}", empty_msg);
 	}
-	has_changes
 }

 pub fn print_staging_role_changes(layout: &ClusterLayout) -> bool {
-	let has_changes = layout
+	let has_role_changes = layout
 		.staging_roles
 		.items()
 		.iter()
 		.any(|(k, _, v)| layout.roles.get(k) != Some(v));
+	let has_layout_changes = *layout.staging_parameters.get() != layout.parameters;

-	if has_changes {
+	if has_role_changes || has_layout_changes {
 		println!();
 		println!("==== STAGED ROLE CHANGES ====");
-		let mut table = vec!["ID\tTags\tZone\tCapacity".to_string()];
-		for (id, _, role) in layout.staging_roles.items().iter() {
-			if layout.roles.get(id) == Some(role) {
-				continue;
-			}
-			if let Some(role) = &role.0 {
-				let tags = role.tags.join(",");
-				table.push(format!(
-					"{:?}\t{}\t{}\t{}",
-					id,
-					tags,
-					role.zone,
-					role.capacity_string()
-				));
-			} else {
-				table.push(format!("{:?}\tREMOVED", id));
+		if has_role_changes {
+			let mut table = vec!["ID\tTags\tZone\tCapacity".to_string()];
+			for (id, _, role) in layout.staging_roles.items().iter() {
+				if layout.roles.get(id) == Some(role) {
+					continue;
+				}
+				if let Some(role) = &role.0 {
+					let tags = role.tags.join(",");
+					table.push(format!(
+						"{:?}\t{}\t{}\t{}",
+						id,
+						tags,
+						role.zone,
+						role.capacity_string()
+					));
+				} else {
+					table.push(format!("{:?}\tREMOVED", id));
+				}
 			}
+			format_table(table);
+			println!();
+		}
+		if has_layout_changes {
+			println!(
+				"Zone redundancy: {}",
+				layout.staging_parameters.get().zone_redundancy
+			);
 		}
-		format_table(table);
 		true
 	} else {
 		false
@@ -143,9 +143,9 @@ pub struct RemoveRoleOpt {

 #[derive(StructOpt, Debug)]
 pub struct ConfigLayoutOpt {
-	/// Zone redundancy parameter
+	/// Zone redundancy parameter ('none'/'max' or integer)
 	#[structopt(short = "r", long = "redundancy")]
-	pub(crate) redundancy: Option<usize>,
+	pub(crate) redundancy: Option<String>,
 }

 #[derive(StructOpt, Debug)]
@@ -471,6 +471,9 @@ pub enum RepairWhat {
 		#[structopt(subcommand)]
 		cmd: ScrubCmd,
 	},
+	/// Rebalance data blocks among storage locations
+	#[structopt(name = "rebalance", version = garage_version())]
+	Rebalance,
 }

 #[derive(Serialize, Deserialize, StructOpt, Debug, Eq, PartialEq, Clone)]
@@ -70,6 +70,12 @@ pub async fn launch_online_repair(
 			info!("Sending command to scrub worker: {:?}", cmd);
 			garage.block_manager.send_scrub_command(cmd).await?;
 		}
+		RepairWhat::Rebalance => {
+			info!("Rebalancing the stored blocks among storage locations");
+			bg.spawn_worker(garage_block::repair::RebalanceWorker::new(
+				garage.block_manager.clone(),
+			));
+		}
 	}
 	Ok(())
 }
@@ -52,7 +52,7 @@ impl Instance {
 			r#"
 metadata_dir = "{path}/meta"
 data_dir = "{path}/data"
-db_engine = "sled"
+db_engine = "lmdb"

 replication_mode = "1"

@@ -44,6 +44,7 @@ async fn test_items_and_indices() {
 		let content = format!("{}: hello world", sk).into_bytes();
 		let content2 = format!("{}: hello universe", sk).into_bytes();
 		let content3 = format!("{}: concurrent value", sk).into_bytes();
+		eprintln!("test iteration {}: {}", i, sk);

 		// Put initially, no causality token
 		let res = ctx
@@ -89,7 +90,7 @@ async fn test_items_and_indices() {
 		assert_eq!(res_body, content);

 		// ReadIndex -- now there should be some stuff
-		tokio::time::sleep(Duration::from_secs(1)).await;
+		tokio::time::sleep(Duration::from_millis(100)).await;
 		let res = ctx
 			.k2v
 			.request
@@ -158,7 +159,7 @@ async fn test_items_and_indices() {
 		assert_eq!(res_body, content2);

 		// ReadIndex -- now there should be some stuff
-		tokio::time::sleep(Duration::from_secs(1)).await;
+		tokio::time::sleep(Duration::from_millis(100)).await;
 		let res = ctx
 			.k2v
 			.request
@@ -230,7 +231,7 @@ async fn test_items_and_indices() {
 		);

 		// ReadIndex -- now there should be some stuff
-		tokio::time::sleep(Duration::from_secs(1)).await;
+		tokio::time::sleep(Duration::from_millis(100)).await;
 		let res = ctx
 			.k2v
 			.request
@@ -299,7 +300,7 @@ async fn test_items_and_indices() {
 		assert_eq!(res.status(), StatusCode::NO_CONTENT);

 		// ReadIndex -- now there should be some stuff
-		tokio::time::sleep(Duration::from_secs(1)).await;
+		tokio::time::sleep(Duration::from_millis(100)).await;
 		let res = ctx
 			.k2v
 			.request
@@ -1,6 +1,6 @@
 [package]
 name = "garage_model"
-version = "0.8.3"
+version = "0.8.4"
 authors = ["Alex Auvolat <alex@adnab.me>"]
 edition = "2018"
 license = "AGPL-3.0"
@@ -23,6 +23,7 @@ garage_util.workspace = true
 async-trait = "0.1.7"
 arc-swap = "1.0"
 blake2 = "0.10"
+chrono = "0.4"
 err-derive = "0.3"
 hex = "0.4"
 base64 = "0.21"
@@ -48,6 +48,9 @@ mod v08 {
 		pub website_config: crdt::Lww<Option<WebsiteConfig>>,
 		/// CORS rules
 		pub cors_config: crdt::Lww<Option<Vec<CorsRule>>>,
+		/// Lifecycle configuration
+		#[serde(default)]
+		pub lifecycle_config: crdt::Lww<Option<Vec<LifecycleRule>>>,
 		/// Bucket quotas
 		#[serde(default)]
 		pub quotas: crdt::Lww<BucketQuotas>,
@@ -69,6 +72,42 @@ mod v08 {
 		pub expose_headers: Vec<String>,
 	}

+	/// Lifecycle configuration rule
+	#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
+	pub struct LifecycleRule {
+		/// The ID of the rule
+		pub id: Option<String>,
+		/// Whether the rule is active
+		pub enabled: bool,
+		/// The filter to check whether rule applies to a given object
+		pub filter: LifecycleFilter,
+		/// Number of days after which incomplete multipart uploads are aborted
+		pub abort_incomplete_mpu_days: Option<usize>,
+		/// Expiration policy for stored objects
+		pub expiration: Option<LifecycleExpiration>,
+	}
+
+	/// A lifecycle filter is a set of conditions that must all be true.
+	/// For each condition, if it is None, it is not verified (always true),
+	/// and if it is Some(x), then it is verified for value x
+	#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize, Default)]
+	pub struct LifecycleFilter {
+		/// If Some(x), object key has to start with prefix x
+		pub prefix: Option<String>,
+		/// If Some(x), object size has to be more than x
+		pub size_gt: Option<u64>,
+		/// If Some(x), object size has to be less than x
+		pub size_lt: Option<u64>,
+	}
+
+	#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
+	pub enum LifecycleExpiration {
+		/// Objects expire x days after they were created
+		AfterDays(usize),
+		/// Objects expire at date x (must be in yyyy-mm-dd format)
+		AtDate(String),
+	}
+
 	#[derive(Default, PartialEq, Eq, PartialOrd, Ord, Clone, Debug, Serialize, Deserialize)]
 	pub struct BucketQuotas {
 		/// Maximum size in bytes (bucket size = sum of sizes of objects in the bucket)
@@ -88,7 +127,7 @@ impl AutoCrdt for BucketQuotas {

 impl BucketParams {
 	/// Create an empty BucketParams with no authorized keys and no website accesss
-	pub fn new() -> Self {
+	fn new() -> Self {
 		BucketParams {
 			creation_date: now_msec(),
 			authorized_keys: crdt::Map::new(),
@@ -96,6 +135,7 @@ impl BucketParams {
 			local_aliases: crdt::LwwMap::new(),
 			website_config: crdt::Lww::new(None),
 			cors_config: crdt::Lww::new(None),
+			lifecycle_config: crdt::Lww::new(None),
 			quotas: crdt::Lww::new(BucketQuotas::default()),
 		}
 	}
@@ -111,10 +151,25 @@ impl Crdt for BucketParams {

 		self.website_config.merge(&o.website_config);
 		self.cors_config.merge(&o.cors_config);
+		self.lifecycle_config.merge(&o.lifecycle_config);
 		self.quotas.merge(&o.quotas);
 	}
 }

+pub fn parse_lifecycle_date(date: &str) -> Result<chrono::NaiveDate, &'static str> {
+	use chrono::prelude::*;
+
+	if let Ok(datetime) = NaiveDateTime::parse_from_str(date, "%Y-%m-%dT%H:%M:%SZ") {
+		if datetime.time() == NaiveTime::MIN {
+			Ok(datetime.date())
+		} else {
+			Err("date must be at midnight")
+		}
+	} else {
+		NaiveDate::parse_from_str(date, "%Y-%m-%d").map_err(|_| "date has invalid format")
+	}
+}
+
 impl Default for Bucket {
 	fn default() -> Self {
 		Self::new()
@@ -7,6 +7,7 @@ use garage_db as db;
 use garage_util::background::*;
 use garage_util::config::*;
 use garage_util::error::*;
+use garage_util::persister::PersisterShared;

 use garage_rpc::replication_mode::ReplicationMode;
 use garage_rpc::system::System;
@@ -17,6 +18,7 @@ use garage_table::replication::TableShardedReplication;
 use garage_table::*;

 use crate::s3::block_ref_table::*;
+use crate::s3::lifecycle_worker;
 use crate::s3::mpu_table::*;
 use crate::s3::object_table::*;
 use crate::s3::version_table::*;
@@ -67,6 +69,9 @@ pub struct Garage {
 	/// Table containing S3 block references (not blocks themselves)
 	pub block_ref_table: Arc<Table<BlockRefTable, TableShardedReplication>>,

+	/// Persister for lifecycle worker info
+	pub lifecycle_persister: PersisterShared<lifecycle_worker::LifecycleWorkerPersisted>,
+
 	#[cfg(feature = "k2v")]
 	pub k2v: GarageK2V,
 }
@@ -87,8 +92,22 @@ impl Garage {
 		// Create meta dir and data dir if they don't exist already
 		std::fs::create_dir_all(&config.metadata_dir)
 			.ok_or_message("Unable to create Garage metadata directory")?;
-		std::fs::create_dir_all(&config.data_dir)
-			.ok_or_message("Unable to create Garage data directory")?;
+		match &config.data_dir {
+			DataDirEnum::Single(data_dir) => {
+				std::fs::create_dir_all(data_dir).ok_or_message(format!(
+					"Unable to create Garage data directory: {}",
+					data_dir.to_string_lossy()
+				))?;
+			}
+			DataDirEnum::Multiple(data_dirs) => {
+				for dir in data_dirs {
+					std::fs::create_dir_all(&dir.path).ok_or_message(format!(
+						"Unable to create Garage data directory: {}",
+						dir.path.to_string_lossy()
+					))?;
+				}
+			}
+		}

 		info!("Opening database...");
 		let mut db_path = config.metadata_dir.clone();
@@ -105,7 +124,7 @@ impl Garage {
 				info!("Opening Sled database at: {}", db_path.display());
 				let db = db::sled_adapter::sled::Config::default()
 					.path(&db_path)
-					.cache_capacity(config.sled_cache_capacity)
+					.cache_capacity(config.sled_cache_capacity as u64)
 					.flush_every_ms(Some(config.sled_flush_every_ms))
 					.open()
 					.ok_or_message("Unable to open sled DB")?;
@@ -144,7 +163,10 @@ impl Garage {
 				info!("Opening LMDB database at: {}", db_path.display());
 				std::fs::create_dir_all(&db_path)
 					.ok_or_message("Unable to create LMDB data directory")?;
-				let map_size = garage_db::lmdb_adapter::recommended_map_size();
+				let map_size = match config.lmdb_map_size {
+					v if v == usize::default() => garage_db::lmdb_adapter::recommended_map_size(),
+					v => v - (v % 4096),
+				};

 				use db::lmdb_adapter::heed;
 				let mut env_builder = heed::EnvOpenOptions::new();
@@ -163,6 +185,7 @@ impl Garage {
 							"OutOfMemory error while trying to open LMDB database. This can happen \
 							if your operating system is not allowing you to use sufficient virtual \
 							memory address space. Please check that no limit is set (ulimit -v). \
+							You may also try to set a smaller `lmdb_map_size` configuration parameter. \
 							On 32-bit machines, you should probably switch to another database engine.".into()))
 					}
 					x => x.ok_or_message("Unable to open LMDB DB")?,
@@ -199,6 +222,9 @@ impl Garage {
 		let replication_mode = ReplicationMode::parse(&config.replication_mode)
 			.ok_or_message("Invalid replication_mode in config file.")?;

+		info!("Initialize background variable system...");
+		let mut bg_vars = vars::BgVars::new();
+
 		info!("Initialize membership management system...");
 		let system = System::new(network_key, replication_mode, &config)?;

@@ -229,7 +255,8 @@ impl Garage {
 			config.compression_level,
 			data_rep_param,
 			system.clone(),
-		);
+		)?;
+		block_manager.register_bg_vars(&mut bg_vars);

 		// ---- admin tables ----
 		info!("Initialize bucket_table...");
@@ -296,14 +323,15 @@ impl Garage {
 			&db,
 		);

+		info!("Load lifecycle worker state...");
+		let lifecycle_persister =
+			PersisterShared::new(&system.metadata_dir, "lifecycle_worker_state");
+		lifecycle_worker::register_bg_vars(&lifecycle_persister, &mut bg_vars);
+
 		// ---- K2V ----
 		#[cfg(feature = "k2v")]
 		let k2v = GarageK2V::new(system.clone(), &db, meta_rep_param);

-		// Initialize bg vars
-		let mut bg_vars = vars::BgVars::new();
-		block_manager.register_bg_vars(&mut bg_vars);
-
 		// -- done --
 		Ok(Arc::new(Self {
 			config,
@@ -321,12 +349,13 @@ impl Garage {
 			mpu_counter_table,
 			version_table,
 			block_ref_table,
+			lifecycle_persister,
 			#[cfg(feature = "k2v")]
 			k2v,
 		}))
 	}

-	pub fn spawn_workers(&self, bg: &BackgroundRunner) {
+	pub fn spawn_workers(self: &Arc<Self>, bg: &BackgroundRunner) {
 		self.block_manager.spawn_workers(bg);

 		self.bucket_table.spawn_workers(bg);
@@ -340,6 +369,11 @@ impl Garage {
 		self.version_table.spawn_workers(bg);
 		self.block_ref_table.spawn_workers(bg);

+		bg.spawn_worker(lifecycle_worker::LifecycleWorker::new(
+			self.clone(),
+			self.lifecycle_persister.clone(),
+		));
+
 		#[cfg(feature = "k2v")]
 		self.k2v.spawn_workers(bg);
 	}
@@ -294,7 +294,7 @@ impl<T: CountedItem> IndexCounter<T> {
 				let counter_entry = local_counter.into_counter_entry(self.this_node);
 				self.local_counter
 					.db()
-					.transaction(|mut tx| self.table.queue_insert(&mut tx, &counter_entry))?;
+					.transaction(|tx| self.table.queue_insert(tx, &counter_entry))?;

 				next_start = Some(local_counter_k);
 			}
@@ -360,7 +360,7 @@ impl<T: CountedItem> IndexCounter<T> {
 				let counter_entry = local_counter.into_counter_entry(self.this_node);
 				self.local_counter
 					.db()
-					.transaction(|mut tx| self.table.queue_insert(&mut tx, &counter_entry))?;
+					.transaction(|tx| self.table.queue_insert(tx, &counter_entry))?;

 				next_start = Some(counted_entry_k);
 			}
@@ -78,6 +78,7 @@ impl Migrate {
 					local_aliases: LwwMap::new(),
 					website_config: Lww::new(website),
 					cors_config: Lww::new(None),
+					lifecycle_config: Lww::new(None),
 					quotas: Lww::new(Default::default()),
 				}),
 			})
@@ -0,0 +1,410 @@
+use std::sync::Arc;
+
+use async_trait::async_trait;
+use chrono::prelude::*;
+use std::time::{Duration, Instant};
+use tokio::sync::watch;
+
+use garage_util::background::*;
+use garage_util::data::*;
+use garage_util::error::Error;
+use garage_util::persister::PersisterShared;
+use garage_util::time::*;
+
+use garage_table::EmptyKey;
+
+use crate::bucket_table::*;
+use crate::s3::object_table::*;
+
+use crate::garage::Garage;
+
+mod v090 {
+	use serde::{Deserialize, Serialize};
+
+	#[derive(Serialize, Deserialize, Default, Clone)]
+	pub struct LifecycleWorkerPersisted {
+		pub last_completed: Option<String>,
+	}
+
+	impl garage_util::migrate::InitialFormat for LifecycleWorkerPersisted {
+		const VERSION_MARKER: &'static [u8] = b"G09lwp";
+	}
+}
+
+pub use v090::*;
+
+pub struct LifecycleWorker {
+	garage: Arc<Garage>,
+
+	state: State,
+
+	persister: PersisterShared<LifecycleWorkerPersisted>,
+}
+
+enum State {
+	Completed(NaiveDate),
+	Running {
+		date: NaiveDate,
+		pos: Vec<u8>,
+		counter: usize,
+		objects_expired: usize,
+		mpu_aborted: usize,
+		last_bucket: Option<Bucket>,
+	},
+}
+
+#[derive(Clone, Copy, Eq, PartialEq)]
+enum Skip {
+	SkipBucket,
+	NextObject,
+}
+
+pub fn register_bg_vars(
+	persister: &PersisterShared<LifecycleWorkerPersisted>,
+	vars: &mut vars::BgVars,
+) {
+	vars.register_ro(persister, "lifecycle-last-completed", |p| {
+		p.get_with(|x| x.last_completed.clone().unwrap_or("never".to_string()))
+	});
+}
+
+impl LifecycleWorker {
+	pub fn new(garage: Arc<Garage>, persister: PersisterShared<LifecycleWorkerPersisted>) -> Self {
+		let today = today();
+		let last_completed = persister.get_with(|x| {
+			x.last_completed
+				.as_deref()
+				.and_then(|x| x.parse::<NaiveDate>().ok())
+		});
+		let state = match last_completed {
+			Some(d) if d >= today => State::Completed(d),
+			_ => State::start(today),
+		};
+		Self {
+			garage,
+			state,
+			persister,
+		}
+	}
+}
+
+impl State {
+	fn start(date: NaiveDate) -> Self {
+		info!("Starting lifecycle worker for {}", date);
+		State::Running {
+			date,
+			pos: vec![],
+			counter: 0,
+			objects_expired: 0,
+			mpu_aborted: 0,
+			last_bucket: None,
+		}
+	}
+}
+
+#[async_trait]
+impl Worker for LifecycleWorker {
+	fn name(&self) -> String {
+		"object lifecycle worker".to_string()
+	}
+
+	fn status(&self) -> WorkerStatus {
+		match &self.state {
+			State::Completed(d) => WorkerStatus {
+				freeform: vec![format!("Last completed: {}", d)],
+				..Default::default()
+			},
+			State::Running {
+				date,
+				counter,
+				objects_expired,
+				mpu_aborted,
+				..
+			} => {
+				let n_objects = self
+					.garage
+					.object_table
+					.data
+					.store
+					.fast_len()
+					.unwrap_or(None);
+				let progress = match n_objects {
+					None => "...".to_string(),
+					Some(total) => format!(
+						"~{:.2}%",
+						100. * std::cmp::min(*counter, total) as f32 / total as f32
+					),
+				};
+				WorkerStatus {
+					progress: Some(progress),
+					freeform: vec![
+						format!("Started: {}", date),
+						format!("Objects expired: {}", objects_expired),
+						format!("Multipart uploads aborted: { }", mpu_aborted),
+					],
+					..Default::default()
+				}
+			}
+		}
+	}
+
+	async fn work(&mut self, _must_exit: &mut watch::Receiver<bool>) -> Result<WorkerState, Error> {
+		match &mut self.state {
+			State::Completed(_) => Ok(WorkerState::Idle),
+			State::Running {
+				date,
+				counter,
+				objects_expired,
+				mpu_aborted,
+				pos,
+				last_bucket,
+			} => {
+				// Process a batch of 100 items before yielding to bg task scheduler
+				for _ in 0..100 {
+					let (object_bytes, next_pos) = match self
+						.garage
+						.object_table
+						.data
+						.store
+						.get_gt(&pos)?
+					{
+						None => {
+							info!("Lifecycle worker finished for {}, objects expired: {}, mpu aborted: {}", date, *objects_expired, *mpu_aborted);
+							self.persister
+								.set_with(|x| x.last_completed = Some(date.to_string()))?;
+							self.state = State::Completed(*date);
+							return Ok(WorkerState::Idle);
+						}
+						Some((k, v)) => (v, k),
+					};
+
+					let object = self.garage.object_table.data.decode_entry(&object_bytes)?;
+					let skip = process_object(
+						&self.garage,
+						*date,
+						&object,
+						objects_expired,
+						mpu_aborted,
+						last_bucket,
+					)
+					.await?;
+
+					*counter += 1;
+					if skip == Skip::SkipBucket {
+						let bucket_id_len = object.bucket_id.as_slice().len();
+						assert_eq!(
+							next_pos.get(..bucket_id_len),
+							Some(object.bucket_id.as_slice())
+						);
+						let last_bucket_pos = [&next_pos[..bucket_id_len], &[0xFFu8][..]].concat();
+						*pos = std::cmp::max(next_pos, last_bucket_pos);
+					} else {
+						*pos = next_pos;
+					}
+				}
+
+				Ok(WorkerState::Busy)
+			}
+		}
+	}
+
+	async fn wait_for_work(&mut self) -> WorkerState {
+		match &self.state {
+			State::Completed(d) => {
+				let next_day = d.succ_opt().expect("no next day");
+				let next_start = midnight_ts(next_day);
+				loop {
+					let now = now_msec();
+					if now < next_start {
+						tokio::time::sleep_until(
+							(Instant::now() + Duration::from_millis(next_start - now)).into(),
+						)
+						.await;
+					} else {
+						break;
+					}
+				}
+				self.state = State::start(std::cmp::max(next_day, today()));
+			}
+			State::Running { .. } => (),
+		}
+		WorkerState::Busy
+	}
+}
+
+async fn process_object(
+	garage: &Arc<Garage>,
+	now_date: NaiveDate,
+	object: &Object,
+	objects_expired: &mut usize,
+	mpu_aborted: &mut usize,
+	last_bucket: &mut Option<Bucket>,
+) -> Result<Skip, Error> {
+	if !object
+		.versions()
+		.iter()
+		.any(|x| x.is_data() || x.is_uploading(None))
+	{
+		return Ok(Skip::NextObject);
+	}
+
+	let bucket = match last_bucket.take() {
+		Some(b) if b.id == object.bucket_id => b,
+		_ => {
+			match garage
+				.bucket_table
+				.get(&EmptyKey, &object.bucket_id)
+				.await?
+			{
+				Some(b) => b,
+				None => {
+					warn!(
+						"Lifecycle worker: object in non-existent bucket {:?}",
+						object.bucket_id
+					);
+					return Ok(Skip::SkipBucket);
+				}
+			}
+		}
+	};
+
+	let lifecycle_policy: &[LifecycleRule] = bucket
+		.state
+		.as_option()
+		.and_then(|s| s.lifecycle_config.get().as_deref())
+		.unwrap_or_default();
+
+	if lifecycle_policy.iter().all(|x| !x.enabled) {
+		return Ok(Skip::SkipBucket);
+	}
+
+	let db = garage.object_table.data.store.db();
+
+	for rule in lifecycle_policy.iter() {
+		if !rule.enabled {
+			continue;
+		}
+
+		if let Some(pfx) = &rule.filter.prefix {
+			if !object.key.starts_with(pfx) {
+				continue;
+			}
+		}
+
+		if let Some(expire) = &rule.expiration {
+			if let Some(current_version) = object.versions().iter().rev().find(|v| v.is_data()) {
+				let version_date = next_date(current_version.timestamp);
+
+				let current_version_data = match &current_version.state {
+					ObjectVersionState::Complete(c) => c,
+					_ => unreachable!(),
+				};
+
+				let size_match = check_size_filter(current_version_data, &rule.filter);
+				let date_match = match expire {
+					LifecycleExpiration::AfterDays(n_days) => {
+						(now_date - version_date) >= chrono::Duration::days(*n_days as i64)
+					}
+					LifecycleExpiration::AtDate(exp_date) => {
+						if let Ok(exp_date) = parse_lifecycle_date(exp_date) {
+							now_date >= exp_date
+						} else {
+							warn!("Invalid expiration date stored in bucket {:?} lifecycle config: {}", bucket.id, exp_date);
+							false
+						}
+					}
+				};
+
+				if size_match && date_match {
+					// Delete expired version
+					let deleted_object = Object::new(
+						object.bucket_id,
+						object.key.clone(),
+						vec![ObjectVersion {
+							uuid: gen_uuid(),
+							timestamp: std::cmp::max(now_msec(), current_version.timestamp + 1),
+							state: ObjectVersionState::Complete(ObjectVersionData::DeleteMarker),
+						}],
+					);
+					info!(
+						"Lifecycle: expiring 1 object in bucket {:?}",
+						object.bucket_id
+					);
+					db.transaction(|tx| garage.object_table.queue_insert(tx, &deleted_object))?;
+					*objects_expired += 1;
+				}
+			}
+		}
+
+		if let Some(abort_mpu_days) = &rule.abort_incomplete_mpu_days {
+			let aborted_versions = object
+				.versions()
+				.iter()
+				.filter_map(|v| {
+					let version_date = next_date(v.timestamp);
+					if (now_date - version_date) >= chrono::Duration::days(*abort_mpu_days as i64)
+						&& matches!(&v.state, ObjectVersionState::Uploading { .. })
+					{
+						Some(ObjectVersion {
+							state: ObjectVersionState::Aborted,
+							..*v
+						})
+					} else {
+						None
+					}
+				})
+				.collect::<Vec<_>>();
+			if !aborted_versions.is_empty() {
+				// Insert aborted mpu info
+				let n_aborted = aborted_versions.len();
+				info!(
+					"Lifecycle: aborting {} incomplete upload(s) in bucket {:?}",
+					n_aborted, object.bucket_id
+				);
+				let aborted_object =
+					Object::new(object.bucket_id, object.key.clone(), aborted_versions);
+				db.transaction(|tx| garage.object_table.queue_insert(tx, &aborted_object))?;
+				*mpu_aborted += n_aborted;
+			}
+		}
+	}
+
+	*last_bucket = Some(bucket);
+	Ok(Skip::NextObject)
+}
+
+fn check_size_filter(version_data: &ObjectVersionData, filter: &LifecycleFilter) -> bool {
+	let size = match version_data {
+		ObjectVersionData::Inline(meta, _) | ObjectVersionData::FirstBlock(meta, _) => meta.size,
+		_ => unreachable!(),
+	};
+	if let Some(size_gt) = filter.size_gt {
+		if !(size > size_gt) {
+			return false;
+		}
+	}
+	if let Some(size_lt) = filter.size_lt {
+		if !(size < size_lt) {
+			return false;
+		}
+	}
+	true
+}
+
+fn midnight_ts(date: NaiveDate) -> u64 {
+	date.and_hms_opt(0, 0, 0)
+		.expect("midnight does not exist")
+		.timestamp_millis() as u64
+}
+
+fn next_date(ts: u64) -> NaiveDate {
+	NaiveDateTime::from_timestamp_millis(ts as i64)
+		.expect("bad timestamp")
+		.date()
+		.succ_opt()
+		.expect("no next day")
+}
+
+fn today() -> NaiveDate {
+	Utc::now().naive_utc().date()
+}
@@ -2,3 +2,5 @@ pub mod block_ref_table;
 pub mod mpu_table;
 pub mod object_table;
 pub mod version_table;
+
+pub mod lifecycle_worker;
@@ -1,6 +1,6 @@
 [package]
 name = "garage_rpc"
-version = "0.8.3"
+version = "0.8.4"
 authors = ["Alex Auvolat <alex@adnab.me>"]
 edition = "2018"
 license = "AGPL-3.0"
@@ -14,6 +14,7 @@ path = "lib.rs"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html

 [dependencies]
+format_table.workspace = true
 garage_db.workspace = true
 garage_util.workspace = true

@@ -26,7 +27,7 @@ tracing = "0.1"
 rand = "0.8"
 itertools="0.10"
 sodiumoxide = { version = "0.2.5-0", package = "kuska-sodiumoxide" }
-systemstat = "0.2.3"
+nix = { version = "0.27", default-features = false, features = ["fs"] }

 async-trait = "0.1.7"
 serde = { version = "1.0", default-features = false, features = ["derive", "rc"] }
@@ -48,7 +49,7 @@ tokio = { version = "1.0", default-features = false, features = ["rt", "rt-multi
 tokio-stream = { version = "0.1", features = ["net"] }
 opentelemetry = "0.17"

-netapp = { version = "0.5.3", features = ["telemetry"] }
+netapp = { version = "=0.5.2", features = ["telemetry"] }

 [features]
 kubernetes-discovery = [ "kube", "k8s-openapi", "schemars" ]
@@ -1,7 +1,7 @@
 //! This module deals with graph algorithms.
 //! It is used in layout.rs to build the partition to node assignment.

-use rand::prelude::SliceRandom;
+use rand::prelude::{SeedableRng, SliceRandom};
 use std::cmp::{max, min};
 use std::collections::HashMap;
 use std::collections::VecDeque;
@@ -143,7 +143,11 @@ impl Graph<FlowEdge> {
 	/// This function shuffles the order of the edge lists. It keeps the ids of the
 	/// reversed edges consistent.
 	fn shuffle_edges(&mut self) {
-		let mut rng = rand::thread_rng();
+		// We use deterministic randomness so that the layout calculation algorihtm
+		// will output the same thing every time it is run. This way, the results
+		// pre-calculated in `garage layout show` will match exactly those used
+		// in practice with `garage layout apply`
+		let mut rng = rand::rngs::StdRng::from_seed([0x12u8; 32]);
 		for i in 0..self.graph.len() {
 			self.graph[i].shuffle(&mut rng);
 			// We need to update the ids of the reverse edges.
@@ -189,7 +193,7 @@ impl Graph<FlowEdge> {
 			let mut fifo = VecDeque::new();
 			fifo.push_back((idsource, 0));
 			while let Some((id, lvl)) = fifo.pop_front() {
-				if level[id] == None {
+				if level[id].is_none() {
 					// it means id has not yet been reached
 					level[id] = Some(lvl);
 					for edge in self.graph[id].iter() {
@@ -199,7 +203,7 @@ impl Graph<FlowEdge> {
 					}
 				}
 			}
-			if level[idsink] == None {
+			if level[idsink].is_none() {
 				// There is no residual flow
 				break;
 			}
@@ -383,7 +387,7 @@ fn cycles_of_1_forest(forest: &[Option<usize>]) -> Vec<Vec<usize>> {
 	for t in 0..forest.len() {
 		let mut id = t;
 		// while we are on a valid undiscovered node
-		while time_of_discovery[id] == None {
+		while time_of_discovery[id].is_none() {
 			time_of_discovery[id] = Some(t);
 			if let Some(i) = forest[id] {
 				id = i;
@@ -391,7 +395,7 @@ fn cycles_of_1_forest(forest: &[Option<usize>]) -> Vec<Vec<usize>> {
 				break;
 			}
 		}
-		if forest[id] != None && time_of_discovery[id] == Some(t) {
+		if forest[id].is_some() && time_of_discovery[id] == Some(t) {
 			// We discovered an id that we explored at this iteration t.
 			// It means we are on a cycle
 			let mut cy = vec![id; 1];
@@ -1,6 +1,7 @@
 use std::cmp::Ordering;
 use std::collections::HashMap;
 use std::collections::HashSet;
+use std::fmt;

 use bytesize::ByteSize;
 use itertools::Itertools;
@@ -115,7 +116,16 @@ mod v09 {
 	/// algorithm. It is stored as a Crdt.
 	#[derive(PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Debug, Serialize, Deserialize)]
 	pub struct LayoutParameters {
-		pub zone_redundancy: usize,
+		pub zone_redundancy: ZoneRedundancy,
+	}
+
+	/// Zone redundancy: if set to AtLeast(x), the layout calculation will aim to store copies
+	/// of each partition on at least that number of different zones.
+	/// Otherwise, copies will be stored on the maximum possible number of zones.
+	#[derive(PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Debug, Serialize, Deserialize)]
+	pub enum ZoneRedundancy {
+		AtLeast(usize),
+		Maximum,
 	}

 	impl garage_util::migrate::Migrate for ClusterLayout {
@@ -125,7 +135,6 @@ mod v09 {

 		fn migrate(previous: Self::Previous) -> Self {
 			use itertools::Itertools;
-			use std::collections::HashSet;

 			// In the old layout, capacities are in an arbitrary unit,
 			// but in the new layout they are in bytes.
@@ -152,17 +161,10 @@ mod v09 {
 				.min()
 				.unwrap_or(0);

-			// Determine zone redundancy parameter
-			let zone_redundancy = std::cmp::min(
-				previous.replication_factor,
-				roles
-					.items()
-					.iter()
-					.filter_map(|(_, _, r)| r.0.as_ref().map(|p| p.zone.as_str()))
-					.collect::<HashSet<&str>>()
-					.len(),
-			);
-			let parameters = LayoutParameters { zone_redundancy };
+			// By default, zone_redundancy is maximum possible value
+			let parameters = LayoutParameters {
+				zone_redundancy: ZoneRedundancy::Maximum,
+			};

 			let mut res = Self {
 				version: previous.version,
@@ -193,7 +195,7 @@ mod v09 {
 				..
 			})) = role
 			{
-				*cap = *cap * mul;
+				*cap *= mul;
 			}
 			new_roles.merge_raw(node, *ts, &role);
 		}
@@ -224,15 +226,39 @@ impl NodeRole {
 	}
 }

+impl fmt::Display for ZoneRedundancy {
+	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+		match self {
+			ZoneRedundancy::Maximum => write!(f, "maximum"),
+			ZoneRedundancy::AtLeast(x) => write!(f, "{}", x),
+		}
+	}
+}
+
+impl core::str::FromStr for ZoneRedundancy {
+	type Err = &'static str;
+	fn from_str(s: &str) -> Result<Self, Self::Err> {
+		match s {
+			"none" | "max" | "maximum" => Ok(ZoneRedundancy::Maximum),
+			x => {
+				let v = x
+					.parse::<usize>()
+					.map_err(|_| "zone redundancy must be 'none'/'max' or an integer")?;
+				Ok(ZoneRedundancy::AtLeast(v))
+			}
+		}
+	}
+}
+
 // Implementation of the ClusterLayout methods unrelated to the assignment algorithm.
 impl ClusterLayout {
 	pub fn new(replication_factor: usize) -> Self {
-		// We set the default zone redundancy to be equal to the replication factor,
-		// i.e. as strict as possible.
+		// We set the default zone redundancy to be Maximum, meaning that the maximum
+		// possible value will be used depending on the cluster topology
 		let parameters = LayoutParameters {
-			zone_redundancy: replication_factor,
+			zone_redundancy: ZoneRedundancy::Maximum,
 		};
-		let staging_parameters = Lww::<LayoutParameters>::new(parameters.clone());
+		let staging_parameters = Lww::<LayoutParameters>::new(parameters);

 		let empty_lwwmap = LwwMap::new();

@@ -296,7 +322,7 @@ To know the correct value of the new layout version, invoke `garage layout show`

 		self.roles.merge(&self.staging_roles);
 		self.roles.retain(|(_, _, v)| v.0.is_some());
-		self.parameters = self.staging_parameters.get().clone();
+		self.parameters = *self.staging_parameters.get();

 		self.staging_roles.clear();
 		self.staging_hash = self.calculate_staging_hash();
@@ -325,7 +351,7 @@ To know the correct value of the new layout version, invoke `garage layout show`
 		}

 		self.staging_roles.clear();
-		self.staging_parameters.update(self.parameters.clone());
+		self.staging_parameters.update(self.parameters);
 		self.staging_hash = self.calculate_staging_hash();

 		self.version += 1;
@@ -356,7 +382,7 @@ To know the correct value of the new layout version, invoke `garage layout show`
 		let mut result = Vec::<Uuid>::new();
 		for uuid in self.node_id_vec.iter() {
 			match self.node_role(uuid) {
-				Some(role) if role.capacity != None => result.push(*uuid),
+				Some(role) if role.capacity.is_some() => result.push(*uuid),
 				_ => (),
 			}
 		}
@@ -418,6 +444,23 @@ To know the correct value of the new layout version, invoke `garage layout show`
 		Ok(total_capacity)
 	}

+	/// Returns the effective value of the zone_redundancy parameter
+	fn effective_zone_redundancy(&self) -> usize {
+		match self.parameters.zone_redundancy {
+			ZoneRedundancy::AtLeast(v) => v,
+			ZoneRedundancy::Maximum => {
+				let n_zones = self
+					.roles
+					.items()
+					.iter()
+					.filter_map(|(_, _, role)| role.0.as_ref().map(|x| x.zone.as_str()))
+					.collect::<HashSet<&str>>()
+					.len();
+				std::cmp::min(n_zones, self.replication_factor)
+			}
+		}
+	}
+
 	/// Check a cluster layout for internal consistency
 	/// (assignment, roles, parameters, partition size)
 	/// returns true if consistent, false if error
@@ -471,6 +514,7 @@ To know the correct value of the new layout version, invoke `garage layout show`
 		}

 		// Check that every partition is associated to distinct nodes
+		let zone_redundancy = self.effective_zone_redundancy();
 		let rf = self.replication_factor;
 		for p in 0..(1 << PARTITION_BITS) {
 			let nodes_of_p = self.ring_assignment_data[rf * p..rf * (p + 1)].to_vec();
@@ -485,11 +529,10 @@ To know the correct value of the new layout version, invoke `garage layout show`
 						.expect("Zone not found.")
 				})
 				.collect::<Vec<_>>();
-			let redundancy = self.parameters.zone_redundancy;
-			if zones_of_p.iter().unique().count() < redundancy {
+			if zones_of_p.iter().unique().count() < zone_redundancy {
 				return Err(format!(
 					"nodes of partition are in less than {} distinct zones",
-					redundancy
+					zone_redundancy
 				));
 			}
 		}
@@ -518,7 +561,7 @@ To know the correct value of the new layout version, invoke `garage layout show`
 		// algorithm.
 		let cl2 = self.clone();
 		let (_, zone_to_id) = cl2.generate_nongateway_zone_ids().unwrap();
-		match cl2.compute_optimal_partition_size(&zone_to_id) {
+		match cl2.compute_optimal_partition_size(&zone_to_id, zone_redundancy) {
 			Ok(s) if s != self.partition_size => {
 				return Err(format!(
 					"partition_size ({}) is different than optimal value ({})",
@@ -533,6 +576,8 @@ To know the correct value of the new layout version, invoke `garage layout show`
 	}
 }

+// ====================================================================================
+
 // Implementation of the ClusterLayout methods related to the assignment algorithm.
 impl ClusterLayout {
 	/// This function calculates a new partition-to-node assignment.
@@ -549,13 +594,15 @@ impl ClusterLayout {
 		// changes in the layout. We retrieve the old_assignment reframed with new ids
 		let old_assignment_opt = self.update_node_id_vec()?;

+		let zone_redundancy = self.effective_zone_redundancy();
+
 		let mut msg = Message::new();
 		msg.push("==== COMPUTATION OF A NEW PARTITION ASSIGNATION ====".into());
 		msg.push("".into());
 		msg.push(format!(
 			"Partitions are \
        replicated {} times on at least {} distinct zones.",
-			self.replication_factor, self.parameters.zone_redundancy
+			self.replication_factor, zone_redundancy
 		));

 		// We generate for once numerical ids for the zones of non gateway nodes,
@@ -570,12 +617,12 @@ impl ClusterLayout {
 				nb_nongateway_nodes, self.replication_factor
 			)));
 		}
-		if id_to_zone.len() < self.parameters.zone_redundancy {
+		if id_to_zone.len() < zone_redundancy {
 			return Err(Error::Message(format!(
 				"The number of zones with non-gateway \
            nodes ({}) is smaller than the redundancy parameter ({})",
 				id_to_zone.len(),
-				self.parameters.zone_redundancy
+				zone_redundancy
 			)));
 		}

@@ -583,18 +630,18 @@ impl ClusterLayout {
 		// Capacities should be given in a unit so that partition size is at least 100.
 		// In this case, integer rounding plays a marginal role in the percentages of
 		// optimality.
-		let partition_size = self.compute_optimal_partition_size(&zone_to_id)?;
+		let partition_size = self.compute_optimal_partition_size(&zone_to_id, zone_redundancy)?;

-		if old_assignment_opt != None {
+		msg.push("".into());
+		if old_assignment_opt.is_some() {
 			msg.push(format!(
-				"Optimal size of a partition: {} (was {} in the previous layout).",
+				"Optimal partition size:                     {} ({} in previous layout)",
 				ByteSize::b(partition_size).to_string_as(false),
 				ByteSize::b(self.partition_size).to_string_as(false)
 			));
 		} else {
 			msg.push(format!(
-				"Given the replication and redundancy constraints, the \
-                optimal size of a partition is {}.",
+				"Optimal partition size:                     {}",
 				ByteSize::b(partition_size).to_string_as(false)
 			));
 		}
@@ -610,7 +657,8 @@ impl ClusterLayout {

 		// We compute a first flow/assignment that is heuristically close to the previous
 		// assignment
-		let mut gflow = self.compute_candidate_assignment(&zone_to_id, &old_assignment_opt)?;
+		let mut gflow =
+			self.compute_candidate_assignment(&zone_to_id, &old_assignment_opt, zone_redundancy)?;
 		if let Some(assoc) = &old_assignment_opt {
 			// We minimize the distance to the previous assignment.
 			self.minimize_rebalance_load(&mut gflow, &zone_to_id, assoc)?;
@@ -618,7 +666,6 @@ impl ClusterLayout {

 		// We display statistics of the computation
 		msg.extend(self.output_stat(&gflow, &old_assignment_opt, &zone_to_id, &id_to_zone)?);
-		msg.push("".to_string());

 		// We update the layout structure
 		self.update_ring_from_flow(id_to_zone.len(), &gflow)?;
@@ -645,7 +692,7 @@ impl ClusterLayout {
 			.roles
 			.items()
 			.iter()
-			.filter(|(_, _, v)| matches!(&v.0, Some(r) if r.capacity != None))
+			.filter(|(_, _, v)| matches!(&v.0, Some(r) if r.capacity.is_some()))
 			.map(|(k, _, _)| *k)
 			.collect();

@@ -661,7 +708,7 @@ impl ClusterLayout {
 			.roles
 			.items()
 			.iter()
-			.filter(|(_, _, v)| matches!(v, NodeRoleV(Some(r)) if r.capacity == None))
+			.filter(|(_, _, v)| matches!(v, NodeRoleV(Some(r)) if r.capacity.is_none()))
 			.map(|(k, _, _)| *k)
 			.collect();

@@ -723,7 +770,7 @@ impl ClusterLayout {

 		for uuid in self.nongateway_nodes().iter() {
 			let r = self.node_role(uuid).unwrap();
-			if !zone_to_id.contains_key(&r.zone) && r.capacity != None {
+			if !zone_to_id.contains_key(&r.zone) && r.capacity.is_some() {
 				zone_to_id.insert(r.zone.clone(), id_to_zone.len());
 				id_to_zone.push(r.zone.clone());
 			}
@@ -736,9 +783,10 @@ impl ClusterLayout {
 	fn compute_optimal_partition_size(
 		&self,
 		zone_to_id: &HashMap<String, usize>,
+		zone_redundancy: usize,
 	) -> Result<u64, Error> {
 		let empty_set = HashSet::<(usize, usize)>::new();
-		let mut g = self.generate_flow_graph(1, zone_to_id, &empty_set)?;
+		let mut g = self.generate_flow_graph(1, zone_to_id, &empty_set, zone_redundancy)?;
 		g.compute_maximal_flow()?;
 		if g.get_flow_value()? < (NB_PARTITIONS * self.replication_factor) as i64 {
 			return Err(Error::Message(
@@ -751,7 +799,12 @@ impl ClusterLayout {
 		let mut s_down = 1;
 		let mut s_up = self.get_total_capacity()?;
 		while s_down + 1 < s_up {
-			g = self.generate_flow_graph((s_down + s_up) / 2, zone_to_id, &empty_set)?;
+			g = self.generate_flow_graph(
+				(s_down + s_up) / 2,
+				zone_to_id,
+				&empty_set,
+				zone_redundancy,
+			)?;
 			g.compute_maximal_flow()?;
 			if g.get_flow_value()? < (NB_PARTITIONS * self.replication_factor) as i64 {
 				s_up = (s_down + s_up) / 2;
@@ -790,18 +843,18 @@ impl ClusterLayout {
 		partition_size: u64,
 		zone_to_id: &HashMap<String, usize>,
 		exclude_assoc: &HashSet<(usize, usize)>,
+		zone_redundancy: usize,
 	) -> Result<Graph<FlowEdge>, Error> {
 		let vertices =
 			ClusterLayout::generate_graph_vertices(zone_to_id.len(), self.nongateway_nodes().len());
 		let mut g = Graph::<FlowEdge>::new(&vertices);
 		let nb_zones = zone_to_id.len();
-		let redundancy = self.parameters.zone_redundancy;
 		for p in 0..NB_PARTITIONS {
-			g.add_edge(Vertex::Source, Vertex::Pup(p), redundancy as u64)?;
+			g.add_edge(Vertex::Source, Vertex::Pup(p), zone_redundancy as u64)?;
 			g.add_edge(
 				Vertex::Source,
 				Vertex::Pdown(p),
-				(self.replication_factor - redundancy) as u64,
+				(self.replication_factor - zone_redundancy) as u64,
 			)?;
 			for z in 0..nb_zones {
 				g.add_edge(Vertex::Pup(p), Vertex::PZ(p, z), 1)?;
@@ -830,6 +883,7 @@ impl ClusterLayout {
 		&self,
 		zone_to_id: &HashMap<String, usize>,
 		prev_assign_opt: &Option<Vec<Vec<usize>>>,
+		zone_redundancy: usize,
 	) -> Result<Graph<FlowEdge>, Error> {
 		// We list the (partition,node) associations that are not used in the
 		// previous assignment
@@ -847,7 +901,12 @@ impl ClusterLayout {
 		}

 		// We compute the best flow using only the edges used in the previous assignment
-		let mut g = self.generate_flow_graph(self.partition_size, zone_to_id, &exclude_edge)?;
+		let mut g = self.generate_flow_graph(
+			self.partition_size,
+			zone_to_id,
+			&exclude_edge,
+			zone_redundancy,
+		)?;
 		g.compute_maximal_flow()?;

 		// We add the excluded edges and compute the maximal flow with the full graph.
@@ -931,29 +990,33 @@ impl ClusterLayout {
 		let used_cap = self.partition_size * NB_PARTITIONS as u64 * self.replication_factor as u64;
 		let total_cap = self.get_total_capacity()?;
 		let percent_cap = 100.0 * (used_cap as f32) / (total_cap as f32);
-		msg.push("".into());
 		msg.push(format!(
-			"Usable capacity / Total cluster capacity: {} / {} ({:.1} %)",
+			"Usable capacity / total cluster capacity:   {} / {} ({:.1} %)",
 			ByteSize::b(used_cap).to_string_as(false),
 			ByteSize::b(total_cap).to_string_as(false),
 			percent_cap
 		));
-		msg.push("".into());
-		msg.push(
-			"If the percentage is too low, it might be that the \
-        replication/redundancy constraints force the use of nodes/zones with small \
-        storage capacities. \
-        You might want to rebalance the storage capacities or relax the constraints. \
-        See the detailed statistics below and look for saturated nodes/zones."
-				.into(),
-		);
 		msg.push(format!(
-			"Recall that because of the replication factor, the actual available \
-                         storage capacity is {} / {} = {}.",
-			ByteSize::b(used_cap).to_string_as(false),
+			"Effective capacity (replication factor {}):  {}",
 			self.replication_factor,
 			ByteSize::b(used_cap / self.replication_factor as u64).to_string_as(false)
 		));
+		if percent_cap < 80. {
+			msg.push("".into());
+			msg.push(
+				"If the percentage is too low, it might be that the \
+            cluster topology and redundancy constraints are forcing the use of nodes/zones with small \
+            storage capacities."
+					.into(),
+			);
+			msg.push(
+				"You might want to move storage capacity between zones or relax the redundancy constraint."
+					.into(),
+			);
+			msg.push(
+				"See the detailed statistics below and look for saturated nodes/zones.".into(),
+			);
+		}

 		// We define and fill in the following tables
 		let storing_nodes = self.nongateway_nodes();
@@ -992,25 +1055,25 @@ impl ClusterLayout {
 			}
 		}

-		if *prev_assign_opt == None {
+		if prev_assign_opt.is_none() {
 			new_partitions = stored_partitions.clone();
-			new_partitions_zone = stored_partitions_zone.clone();
+			//new_partitions_zone = stored_partitions_zone.clone();
 		}

 		// We display the statistics

 		msg.push("".into());
-		if *prev_assign_opt != None {
+		if prev_assign_opt.is_some() {
 			let total_new_partitions: usize = new_partitions.iter().sum();
 			msg.push(format!(
 				"A total of {} new copies of partitions need to be \
                             transferred.",
 				total_new_partitions
 			));
+			msg.push("".into());
 		}
-		msg.push("".into());
-		msg.push("==== DETAILED STATISTICS BY ZONES AND NODES ====".into());

+		let mut table = vec![];
 		for z in 0..id_to_zone.len() {
 			let mut nodes_of_z = Vec::<usize>::new();
 			for n in 0..storing_nodes.len() {
@@ -1020,15 +1083,9 @@ impl ClusterLayout {
 			}
 			let replicated_partitions: usize =
 				nodes_of_z.iter().map(|n| stored_partitions[*n]).sum();
-			msg.push("".into());
-
-			msg.push(format!(
-				"Zone {}: {} distinct partitions stored ({} new, \
-                {} partition copies) ",
-				id_to_zone[z],
-				stored_partitions_zone[z],
-				new_partitions_zone[z],
-				replicated_partitions
+			table.push(format!(
+				"{}\tTags\tPartitions\tCapacity\tUsable capacity",
+				id_to_zone[z]
 			));

 			let available_cap_z: u64 = self.partition_size * replicated_partitions as u64;
@@ -1037,33 +1094,35 @@ impl ClusterLayout {
 				total_cap_z += self.get_node_capacity(&self.node_id_vec[*n])?;
 			}
 			let percent_cap_z = 100.0 * (available_cap_z as f32) / (total_cap_z as f32);
-			msg.push(format!(
-				"  Usable capacity / Total capacity: {} / {} ({:.1}%).",
-				ByteSize::b(available_cap_z).to_string_as(false),
-				ByteSize::b(total_cap_z).to_string_as(false),
-				percent_cap_z
-			));

 			for n in nodes_of_z.iter() {
 				let available_cap_n = stored_partitions[*n] as u64 * self.partition_size;
 				let total_cap_n = self.get_node_capacity(&self.node_id_vec[*n])?;
-				let tags_n = (self
-					.node_role(&self.node_id_vec[*n])
-					.ok_or("Node not found."))?
-				.tags_string();
-				msg.push(format!(
-					"  Node {:?}: {} partitions ({} new) ; \
-                                 usable/total capacity: {} / {} ({:.1}%) ; tags:{}",
+				let tags_n = (self.node_role(&self.node_id_vec[*n]).ok_or("<??>"))?.tags_string();
+				table.push(format!(
+					"  {:?}\t{}\t{} ({} new)\t{}\t{} ({:.1}%)",
 					self.node_id_vec[*n],
+					tags_n,
 					stored_partitions[*n],
 					new_partitions[*n],
-					ByteSize::b(available_cap_n).to_string_as(false),
 					ByteSize::b(total_cap_n).to_string_as(false),
+					ByteSize::b(available_cap_n).to_string_as(false),
 					(available_cap_n as f32) / (total_cap_n as f32) * 100.0,
-					tags_n
 				));
 			}
+
+			table.push(format!(
+				"  TOTAL\t\t{} ({} unique)\t{}\t{} ({:.1}%)",
+				replicated_partitions,
+				stored_partitions_zone[z],
+				//new_partitions_zone[z],
+				ByteSize::b(total_cap_z).to_string_as(false),
+				ByteSize::b(available_cap_z).to_string_as(false),
+				percent_cap_z
+			));
+			table.push("".into());
 		}
+		msg.push(format_table::format_table_to_string(table));

 		Ok(msg)
 	}
@@ -1125,7 +1184,7 @@ mod tests {

 		let mut curr_zone = 0;

-		let redundancy = cl.parameters.zone_redundancy;
+		let redundancy = cl.effective_zone_redundancy();

 		for replic in 0..cl.replication_factor {
 			for p in 0..NB_PARTITIONS {
@@ -1177,8 +1236,9 @@ mod tests {
 			);
 			cl.staging_roles.merge(&update);
 		}
-		cl.staging_parameters
-			.update(LayoutParameters { zone_redundancy });
+		cl.staging_parameters.update(LayoutParameters {
+			zone_redundancy: ZoneRedundancy::AtLeast(zone_redundancy),
+		});
 		cl.staging_hash = cl.calculate_staging_hash();
 	}

@@ -22,9 +22,9 @@ use netapp::peering::fullmesh::FullMeshPeeringStrategy;
 use netapp::util::parse_and_resolve_peer_addr_async;
 use netapp::{NetApp, NetworkKey, NodeID, NodeKey};

-use garage_util::config::Config;
 #[cfg(feature = "kubernetes-discovery")]
 use garage_util::config::KubernetesDiscoveryConfig;
+use garage_util::config::{Config, DataDirEnum};
 use garage_util::data::*;
 use garage_util::error::*;
 use garage_util::persister::Persister;
@@ -119,7 +119,7 @@ pub struct System {
 	/// Path to metadata directory
 	pub metadata_dir: PathBuf,
 	/// Path to data directory
-	pub data_dir: PathBuf,
+	pub data_dir: DataDirEnum,
 }

 #[derive(Debug, Clone, Serialize, Deserialize)]
@@ -668,7 +668,7 @@ impl System {

 		let prev_layout_check = layout.check().is_ok();
 		if layout.merge(adv) {
-			if prev_layout_check && !layout.check().is_ok() {
+			if prev_layout_check && layout.check().is_err() {
 				error!("New cluster layout is invalid, discarding.");
 				return Err(Error::Message(
 					"New cluster layout is invalid, discarding.".into(),
@@ -724,7 +724,7 @@ impl System {

 	async fn discovery_loop(self: &Arc<Self>, mut stop_signal: watch::Receiver<bool>) {
 		while !*stop_signal.borrow() {
-			let not_configured = !self.ring.borrow().layout.check().is_ok();
+			let not_configured = self.ring.borrow().layout.check().is_err();
 			let no_peers = self.fullmesh.get_peer_list().len() < self.replication_factor;
 			let expected_n_nodes = self.ring.borrow().layout.num_nodes();
 			let bad_peers = self
@@ -890,20 +890,47 @@ impl NodeStatus {
 		}
 	}

-	fn update_disk_usage(&mut self, meta_dir: &Path, data_dir: &Path, metrics: &SystemMetrics) {
-		use systemstat::{Platform, System};
-		let mounts = System::new().mounts().unwrap_or_default();
-
-		let mount_avail = |path: &Path| {
-			mounts
-				.iter()
-				.filter(|x| path.starts_with(&x.fs_mounted_on))
-				.max_by_key(|x| x.fs_mounted_on.len())
-				.map(|x| (x.avail.as_u64(), x.total.as_u64()))
+	fn update_disk_usage(
+		&mut self,
+		meta_dir: &Path,
+		data_dir: &DataDirEnum,
+		metrics: &SystemMetrics,
+	) {
+		use nix::sys::statvfs::statvfs;
+		let mount_avail = |path: &Path| match statvfs(path) {
+			Ok(x) => {
+				let avail = x.blocks_available() * x.fragment_size() as u64;
+				let total = x.blocks() * x.fragment_size() as u64;
+				Some((x.filesystem_id(), avail, total))
+			}
+			Err(_) => None,
 		};

-		self.meta_disk_avail = mount_avail(meta_dir);
-		self.data_disk_avail = mount_avail(data_dir);
+		self.meta_disk_avail = mount_avail(meta_dir).map(|(_, a, t)| (a, t));
+		self.data_disk_avail = match data_dir {
+			DataDirEnum::Single(dir) => mount_avail(dir).map(|(_, a, t)| (a, t)),
+			DataDirEnum::Multiple(dirs) => (|| {
+				// TODO: more precise calculation that takes into account
+				// how data is going to be spread among partitions
+				let mut mounts = HashMap::new();
+				for dir in dirs.iter() {
+					if dir.capacity.is_none() {
+						continue;
+					}
+					match mount_avail(&dir.path) {
+						Some((fsid, avail, total)) => {
+							mounts.insert(fsid, (avail, total));
+						}
+						None => return None,
+					}
+				}
+				Some(
+					mounts
+						.into_iter()
+						.fold((0, 0), |(x, y), (_, (a, b))| (x + a, y + b)),
+				)
+			})(),
+		};

 		if let Some((avail, total)) = self.meta_disk_avail {
 			metrics
@@ -1,6 +1,6 @@
 [package]
 name = "garage_table"
-version = "0.8.3"
+version = "0.8.4"
 authors = ["Alex Auvolat <alex@adnab.me>"]
 edition = "2018"
 license = "AGPL-3.0"
@@ -34,7 +34,7 @@ pub struct TableData<F: TableSchema, R: TableReplication> {
 	pub(crate) merkle_todo_notify: Notify,

 	pub(crate) insert_queue: db::Tree,
-	pub(crate) insert_queue_notify: Notify,
+	pub(crate) insert_queue_notify: Arc<Notify>,

 	pub(crate) gc_todo: CountedTree,

@@ -80,7 +80,7 @@ impl<F: TableSchema, R: TableReplication> TableData<F, R> {
 			merkle_todo,
 			merkle_todo_notify: Notify::new(),
 			insert_queue,
-			insert_queue_notify: Notify::new(),
+			insert_queue_notify: Arc::new(Notify::new()),
 			gc_todo,
 			metrics,
 		})
@@ -203,14 +203,14 @@ impl<F: TableSchema, R: TableReplication> TableData<F, R> {
 	) -> Result<Option<F::E>, Error> {
 		let tree_key = self.tree_key(partition_key, sort_key);

-		let changed = self.store.db().transaction(|mut tx| {
+		let changed = self.store.db().transaction(|tx| {
 			let (old_entry, old_bytes, new_entry) = match tx.get(&self.store, &tree_key)? {
 				Some(old_bytes) => {
 					let old_entry = self.decode_entry(&old_bytes).map_err(db::TxError::Abort)?;
-					let new_entry = update_fn(&mut tx, Some(old_entry.clone()))?;
+					let new_entry = update_fn(tx, Some(old_entry.clone()))?;
 					(Some(old_entry), Some(old_bytes), new_entry)
 				}
-				None => (None, None, update_fn(&mut tx, None)?),
+				None => (None, None, update_fn(tx, None)?),
 			};

 			// Changed can be true in two scenarios
@@ -233,7 +233,7 @@ impl<F: TableSchema, R: TableReplication> TableData<F, R> {
 				tx.insert(&self.store, &tree_key, new_bytes)?;

 				self.instance
-					.updated(&mut tx, old_entry.as_ref(), Some(&new_entry))?;
+					.updated(tx, old_entry.as_ref(), Some(&new_entry))?;

 				Ok(Some((new_entry, new_bytes_hash)))
 			} else {
@@ -270,14 +270,14 @@ impl<F: TableSchema, R: TableReplication> TableData<F, R> {
 		let removed = self
 			.store
 			.db()
-			.transaction(|mut tx| match tx.get(&self.store, k)? {
+			.transaction(|tx| match tx.get(&self.store, k)? {
 				Some(cur_v) if cur_v == v => {
 					let old_entry = self.decode_entry(v).map_err(db::TxError::Abort)?;

 					tx.remove(&self.store, k)?;
 					tx.insert(&self.merkle_todo, k, vec![])?;

-					self.instance.updated(&mut tx, Some(&old_entry), None)?;
+					self.instance.updated(tx, Some(&old_entry), None)?;
 					Ok(true)
 				}
 				_ => Ok(false),
@@ -298,14 +298,14 @@ impl<F: TableSchema, R: TableReplication> TableData<F, R> {
 		let removed = self
 			.store
 			.db()
-			.transaction(|mut tx| match tx.get(&self.store, k)? {
+			.transaction(|tx| match tx.get(&self.store, k)? {
 				Some(cur_v) if blake2sum(&cur_v[..]) == vhash => {
 					let old_entry = self.decode_entry(&cur_v[..]).map_err(db::TxError::Abort)?;

 					tx.remove(&self.store, k)?;
 					tx.insert(&self.merkle_todo, k, vec![])?;

-					self.instance.updated(&mut tx, Some(&old_entry), None)?;
+					self.instance.updated(tx, Some(&old_entry), None)?;
 					Ok(true)
 				}
 				_ => Ok(false),
@@ -339,7 +339,9 @@ impl<F: TableSchema, R: TableReplication> TableData<F, R> {
 			.map_err(Error::RmpEncode)
 			.map_err(db::TxError::Abort)?;
 		tx.insert(&self.insert_queue, &tree_key, new_entry)?;
-		self.insert_queue_notify.notify_one();
+
+		let notif = self.insert_queue_notify.clone();
+		tx.on_commit(move || notif.notify_one());

 		Ok(())
 	}
@@ -108,9 +108,9 @@ impl<F: TableSchema, R: TableReplication> MerkleUpdater<F, R> {
 		self.data
 			.merkle_tree
 			.db()
-			.transaction(|mut tx| self.update_item_rec(&mut tx, k, &khash, &key, new_vhash))?;
+			.transaction(|tx| self.update_item_rec(tx, k, &khash, &key, new_vhash))?;

-		let deleted = self.data.merkle_todo.db().transaction(|mut tx| {
+		let deleted = self.data.merkle_todo.db().transaction(|tx| {
 			let remove = matches!(tx.get(&self.data.merkle_todo, k)?, Some(ov) if ov == vhash_by);
 			if remove {
 				tx.remove(&self.data.merkle_todo, k)?;
@@ -12,7 +12,7 @@ use crate::replication::*;
 use crate::schema::*;
 use crate::table::*;

-const BATCH_SIZE: usize = 100;
+const BATCH_SIZE: usize = 1024;

 pub(crate) struct InsertQueueWorker<F, R>(pub(crate) Arc<Table<F, R>>)
 where
@@ -53,7 +53,7 @@ impl<F: TableSchema, R: TableReplication> Worker for InsertQueueWorker<F, R> {

 		self.0.insert_many(values).await?;

-		self.0.data.insert_queue.db().transaction(|mut tx| {
+		self.0.data.insert_queue.db().transaction(|tx| {
 			for (k, v) in kv_pairs.iter() {
 				if let Some(v2) = tx.get(&self.0.data.insert_queue, k)? {
 					if &v2 == v {
@@ -1,6 +1,6 @@
 [package]
 name = "garage_util"
-version = "0.8.3"
+version = "0.8.4"
 authors = ["Alex Auvolat <alex@adnab.me>"]
 edition = "2018"
 license = "AGPL-3.0"
@@ -20,6 +20,7 @@ arc-swap = "1.0"
 async-trait = "0.1"
 blake2 = "0.10"
 bytes = "1.0"
+bytesize = "1.2"
 digest = "0.10"
 err-derive = "0.3"
 hexdump = "0.1"
@@ -31,7 +32,7 @@ rand = "0.8"
 sha2 = "0.10"

 chrono = "0.4"
-rmp-serde = "1.1.2"
+rmp-serde = "0.15"
 serde = { version = "1.0", default-features = false, features = ["derive", "rc"] }
 serde_json = "1.0"
 toml = "0.6"
@@ -1,4 +1,5 @@
 //! Contains type and functions related to Garage configuration file
+use std::convert::TryFrom;
 use std::io::Read;
 use std::net::SocketAddr;
 use std::path::PathBuf;
@@ -13,7 +14,7 @@ pub struct Config {
 	/// Path where to store metadata. Should be fast, but low volume
 	pub metadata_dir: PathBuf,
 	/// Path where to store data. Can be slower, but need higher volume
-	pub data_dir: PathBuf,
+	pub data_dir: DataDirEnum,

 	/// Whether to fsync after all metadata transactions (disabled by default)
 	#[serde(default)]
@@ -23,7 +24,10 @@ pub struct Config {
 	pub data_fsync: bool,

 	/// Size of data blocks to save to disk
-	#[serde(default = "default_block_size")]
+	#[serde(
+		deserialize_with = "deserialize_capacity",
+		default = "default_block_size"
+	)]
 	pub block_size: usize,

 	/// Replication mode. Supported values:
@@ -73,12 +77,19 @@ pub struct Config {
 	pub db_engine: String,

 	/// Sled cache size, in bytes
-	#[serde(default = "default_sled_cache_capacity")]
-	pub sled_cache_capacity: u64,
+	#[serde(
+		deserialize_with = "deserialize_capacity",
+		default = "default_sled_cache_capacity"
+	)]
+	pub sled_cache_capacity: usize,
 	/// Sled flush interval in milliseconds
 	#[serde(default = "default_sled_flush_every_ms")]
 	pub sled_flush_every_ms: u64,

+	/// LMDB map size
+	#[serde(deserialize_with = "deserialize_capacity", default)]
+	pub lmdb_map_size: usize,
+
 	// -- APIs
 	/// Configuration for S3 api
 	pub s3_api: S3ApiConfig,
@@ -94,6 +105,26 @@ pub struct Config {
 	pub admin: AdminConfig,
 }

+/// Value for data_dir: either a single directory or a list of dirs with attributes
+#[derive(Deserialize, Debug, Clone)]
+#[serde(untagged)]
+pub enum DataDirEnum {
+	Single(PathBuf),
+	Multiple(Vec<DataDir>),
+}
+
+#[derive(Deserialize, Debug, Clone)]
+pub struct DataDir {
+	/// Path to the data directory
+	pub path: PathBuf,
+	/// Capacity of the drive (required if read_only is false)
+	#[serde(default)]
+	pub capacity: Option<String>,
+	/// Whether this is a legacy read-only path (capacity should be None)
+	#[serde(default)]
+	pub read_only: bool,
+}
+
 /// Configuration for S3 api
 #[derive(Deserialize, Debug, Clone)]
 pub struct S3ApiConfig {
@@ -193,7 +224,7 @@ fn default_db_engine() -> String {
 	"lmdb".into()
 }

-fn default_sled_cache_capacity() -> u64 {
+fn default_sled_cache_capacity() -> usize {
 	128 * 1024 * 1024
 }
 fn default_sled_flush_every_ms() -> u64 {
@@ -273,8 +304,6 @@ fn deserialize_compression<'de, D>(deserializer: D) -> Result<Option<i32>, D::Er
 where
 	D: de::Deserializer<'de>,
 {
-	use std::convert::TryFrom;
-
 	struct OptionVisitor;

 	impl<'de> serde::de::Visitor<'de> for OptionVisitor {
@@ -319,6 +348,50 @@ where
 	deserializer.deserialize_any(OptionVisitor)
 }

+fn deserialize_capacity<'de, D>(deserializer: D) -> Result<usize, D::Error>
+where
+	D: de::Deserializer<'de>,
+{
+	struct CapacityVisitor;
+
+	impl<'de> serde::de::Visitor<'de> for CapacityVisitor {
+		type Value = usize;
+		fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result {
+			formatter.write_str("int or '<capacity>'")
+		}
+
+		fn visit_str<E>(self, value: &str) -> Result<Self::Value, E>
+		where
+			E: de::Error,
+		{
+			value
+				.parse::<bytesize::ByteSize>()
+				.map(|x| x.as_u64())
+				.map_err(|e| E::custom(format!("invalid capacity value: {}", e)))
+				.and_then(|v| {
+					usize::try_from(v)
+						.map_err(|_| E::custom("capacity value out of bound".to_owned()))
+				})
+		}
+
+		fn visit_i64<E>(self, v: i64) -> Result<Self::Value, E>
+		where
+			E: de::Error,
+		{
+			usize::try_from(v).map_err(|_| E::custom("capacity value out of bound".to_owned()))
+		}
+
+		fn visit_u64<E>(self, v: u64) -> Result<Self::Value, E>
+		where
+			E: de::Error,
+		{
+			usize::try_from(v).map_err(|_| E::custom("capacity value out of bound".to_owned()))
+		}
+	}
+
+	deserializer.deserialize_any(CapacityVisitor)
+}
+
 #[cfg(test)]
 mod tests {
 	use crate::error::Error;
@@ -1,6 +1,6 @@
 [package]
 name = "garage_web"
-version = "0.8.3"
+version = "0.8.4"
 authors = ["Alex Auvolat <alex@adnab.me>", "Quentin Dufour <quentin@dufour.io>"]
 edition = "2018"
 license = "AGPL-3.0"
Author	SHA1	Message	Date
Alex	aa7eadc799	Merge pull request 'New layout: fixes and UX improvements' (#634 ) from new-layout-ux into next Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/634	2023-09-27 09:04:32 +00:00
Alex Auvolat	0e5925fff6	layout doc: reformulate	2023-09-22 16:14:47 +02:00
Alex Auvolat	8d07888fa2	layout doc: write explanations for bizarre scenarios	2023-09-22 16:07:46 +02:00
Alex Auvolat	405aa42b7d	layout doc: update old text	2023-09-22 10:06:31 +02:00
Alex Auvolat	b4a0e636d8	new layout doc: add examples of unexpected layout, to explain	2023-09-22 09:49:07 +02:00
Alex	1d986bd889	Merge pull request 'Refactor db transactions and add on_commit for table.queue_insert' (#637 ) from k2v-indices-lmdb into next Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/637	2023-09-21 14:03:35 +00:00
Alex Auvolat	0635250b2b	garage_table/queue_insert: delay worker notification to after transaction commit (fix #583 )	2023-09-21 15:37:28 +02:00
Alex Auvolat	f97168f805	garage_db: refactor transactions and add on_commit mechanism	2023-09-21 15:35:31 +02:00
Alex Auvolat	3ecc17f8c5	new layout: use deterministic randomness for reproducible results	2023-09-21 11:21:35 +02:00
Alex Auvolat	013b026d56	update cargo.nix	2023-09-18 12:18:56 +02:00
Alex Auvolat	0088599f52	new layout: fix clippy lints	2023-09-18 12:17:07 +02:00
Alex Auvolat	749b4865d0	new layout: improve display and fix comments	2023-09-18 12:07:45 +02:00
Alex Auvolat	015ccb39aa	new layout: make zone_redundancy optionnal (if not set, is maximum)	2023-09-18 11:59:08 +02:00
Alex Auvolat	2e229d4430	new layout: improve output display	2023-09-12 17:24:51 +02:00
Alex Auvolat	fd7d8fec59	Merge branch 'main' into next	2023-09-11 23:09:20 +02:00
Alex	143a349f55	Merge pull request 'fix 32-bit build' (#632 ) from fix-32bit into main Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/632	2023-09-11 21:08:26 +00:00
Alex Auvolat	9cfe55ab60	fix 32-bit build	2023-09-11 20:01:29 +02:00
Alex Auvolat	51abbb02d8	Merge branch 'main' into next	2023-09-11 20:00:02 +02:00
Alex	2548a247f2	Merge pull request 'use statvfs instead of mount list to determine free data/meta space (fix #611 )' (#631 ) from fix-free-space into main Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/631	2023-09-11 17:29:23 +00:00
Alex Auvolat	d5bb50d738	use statvfs instead of mount list to determine free data/meta space (fix #611 )	2023-09-11 19:08:24 +02:00
Alex	fc635f7072	Merge pull request 'make lmdb's map_size configurable (fix #628 )' (#630 ) from configurable-map-size into main Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/630	2023-09-11 16:48:14 +00:00
Alex Auvolat	f8b3883611	config: make block_size and sled_cache_capacity expressable as strings	2023-09-11 18:34:59 +02:00
Alex Auvolat	51b9731a08	make lmdb's map_size configurable (fix #628 )	2023-09-11 18:03:44 +02:00
Alex Auvolat	ad6b1cc0be	Merge branch 'main' into next	2023-09-11 13:14:18 +02:00
Alex	7228fbfd4f	Merge pull request 'multi-hdd support (fix #218 )' (#625 ) from multihdd into next Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/625	2023-09-11 10:52:01 +00:00
Alex Auvolat	ba7ac52c19	block repair: simpler/more robust iterator progress calculation	2023-09-11 12:31:34 +02:00
Alex Auvolat	9526328d38	scrub: clear saved checkpoint when canceling scrub	2023-09-11 12:10:48 +02:00
Alex Auvolat	7f9ba49c71	block manager: remove data_dir field	2023-09-11 11:57:36 +02:00
Alex Auvolat	de5d792181	block manager: fix indentation (why not detected by cargo fmt?)	2023-09-11 11:52:57 +02:00
Alex Auvolat	be91ef6294	block manager: fix bug where rebalance didn't delete old copies	2023-09-07 16:04:03 +02:00
Alex Auvolat	2657b5c1b9	block manager: fix bugs	2023-09-07 15:30:56 +02:00
Alex Auvolat	eb972a8422	doc: update multi-hdd section	2023-09-07 14:48:36 +02:00
Alex Auvolat	2f112ac682	correct free data space accounting for multiple data dirs on same fs	2023-09-07 14:42:20 +02:00
Alex Auvolat	6a067e30ee	doc: documentation of rebalance repair	2023-09-07 13:49:12 +02:00
Alex Auvolat	6b008b5bd3	block manager: add rebalance operation to rebalance multi-hdd setups	2023-09-07 13:44:11 +02:00
Alex Auvolat	6595efd82f	Document multi-hdd support	2023-09-07 13:23:02 +02:00
Alex Auvolat	bca347a1e8	doc: update page on upgradin clusters	2023-09-07 12:52:44 +02:00
Alex Auvolat	99ed18350f	block manager: refactor and fix monitoring/statistics	2023-09-07 12:41:36 +02:00
Alex Auvolat	f38a31b330	block manager: avoid incorrect data_dir configs and avoid losing files	2023-09-06 17:49:30 +02:00
Alex Auvolat	e30865984a	block manager: scrub checkpointing	2023-09-06 16:35:28 +02:00
Alex Auvolat	55c514999e	block manager: fixes in layout	2023-09-06 16:35:28 +02:00
Alex Auvolat	a44f486931	block manager: refactoring & increase max worker count to 8	2023-09-06 16:35:28 +02:00
Alex Auvolat	3a74844df0	block manager: fix dir_not_empty	2023-09-06 16:35:28 +02:00
Alex Auvolat	93114a9747	block manager: refactoring	2023-09-06 16:35:28 +02:00
Alex Auvolat	fd00a47ddc	table queue: increase batch size	2023-09-06 16:35:28 +02:00
Alex Auvolat	1b8c265c14	block manager: get rid of check_block_status	2023-09-06 16:35:28 +02:00
Alex Auvolat	3199cab4c8	update cargo.nix	2023-09-06 16:35:28 +02:00
Alex Auvolat	a09f86729c	block manager: move blocks in write_block if necessary	2023-09-06 16:35:28 +02:00
Alex Auvolat	887b3233f4	block manager: use data paths from layout	2023-09-06 16:35:28 +02:00
Alex Auvolat	6c420c0880	block manager: multi-directory layout computation	2023-09-06 16:35:28 +02:00
Alex Auvolat	71c0188055	block manager: skeleton for multi-hdd support	2023-09-06 16:35:28 +02:00
Alex Auvolat	4b4f2000f4	lifecycle: fix SkipBucket bug	2023-09-06 16:34:07 +02:00
Alex	5f86b48f97	Merge pull request 'Revert netapp to 0.5.2 to avoid rmp-serde upgrade that breaks things' (#627 ) from hold-netapp-0.5.2 into main Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/627	2023-09-05 22:08:40 +00:00
Alex Auvolat	51eac97260	update version to 0.8.4	2023-09-05 23:28:12 +02:00
Alex Auvolat	e78566591b	Revert netapp update, hold to version 0.5.2 that uses rmp-serde 0.15	2023-09-05 23:23:23 +02:00
Alex	3f461d8891	Merge pull request 'object lifecycles (fix #309 )' (#620 ) from bucket-lifecycle into next Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/620	2023-09-04 09:45:10 +00:00
Alex Auvolat	8e0c020bb9	lifecycle worker: correct small clippy lints	2023-09-04 11:33:44 +02:00
Alex Auvolat	1cdc321e28	lifecycle worker: don't get stuck on non-existent bucket	2023-08-31 11:36:30 +02:00
Alex Auvolat	f579d6d9b4	lifecycle worker: fix potential inifinite loop	2023-08-31 11:29:54 +02:00
Alex Auvolat	a00a52633f	lifecycle worker: add log message when starting	2023-08-31 11:25:14 +02:00
Alex Auvolat	adbf5925de	lifecycle worker: use queue_insert and process objects in batches	2023-08-31 11:19:26 +02:00
Alex Auvolat	1cfcc61de8	lifecycle worker: mitigate potential bugs + refactoring	2023-08-31 00:28:37 +02:00
Alex Auvolat	be03a4610f	s3api: remove redundant serde rename attribute	2023-08-31 00:00:26 +02:00
Alex Auvolat	b2f679675e	lifecycle worker: take into account disabled rules	2023-08-30 23:52:09 +02:00
Alex Auvolat	5fad4c4658	update cargo.nix	2023-08-30 23:47:42 +02:00
Alex Auvolat	01c327a07a	lifecycle worker: avoid building chrono's serde feature	2023-08-30 23:46:15 +02:00
Alex Auvolat	f0a395e2e5	s3 bucket apis: remove redundant call	2023-08-30 23:39:28 +02:00
Alex Auvolat	d94f1c9178	reference manual: remove obsolete caveat about multipart uploads	2023-08-30 23:27:02 +02:00
Alex Auvolat	5c923d48d7	reference manual: document support for lifecycle configuration	2023-08-30 23:24:28 +02:00
Alex Auvolat	a1d57283c0	bucket_table: bucketparams::new doesn't need to be pub	2023-08-30 20:07:14 +02:00
Alex Auvolat	d2e94e36d6	lifecycle config: add missing line in merge() and remove tracing	2023-08-30 20:05:53 +02:00
Alex Auvolat	75ccc5a95c	lifecycle config: store date as given, try to debug	2023-08-30 20:02:07 +02:00
Alex Auvolat	7200954318	lifecycle worker: add logging	2023-08-30 14:54:52 +02:00
Alex Auvolat	0f1849e1ac	lifecycle worker: launch with the rest of Garage	2023-08-30 14:51:08 +02:00
Alex Auvolat	da8b224e24	lifecycle worker: skip entire bucket when no lifecycle config is set	2023-08-30 14:38:19 +02:00
Alex Auvolat	2996dc875f	lifecycle worker: implement main functionality	2023-08-30 14:29:03 +02:00
Alex Auvolat	a2e0e34db5	lifecycle: skeleton for lifecycle worker	2023-08-30 12:41:11 +02:00
Alex Auvolat	f7b409f114	use a NaiveDate in data model, it serializes to string (iso 8601 format)	2023-08-30 11:24:01 +02:00
Alex Auvolat	abf011c290	lifecycle: implement validation into garage's internal data structure	2023-08-29 18:22:03 +02:00
Alex Auvolat	8041d9a827	s3: add xml structures to serialize/deserialize lifecycle configs	2023-08-29 17:44:17 +02:00
Alex Auvolat	0b83e0558e	bucket_table: data model for lifecycle configuration	2023-08-29 17:00:41 +02:00