wip: dump waaaaaay too much stuff

Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/447

.gitattributes

@@ -1 +0,0 @@
-*.pdf filter=lfs diff=lfs merge=lfs -text

Cargo.lock

@@ -1172,10 +1172,15 @@ dependencies = [
  "clap 3.1.18",
  "err-derive",
  "heed",
+ "hex",
  "hexdump",
  "mktemp",
+ "nettext",
  "pretty_env_logger",
+ "rmp-serde",
  "rusqlite",
+ "serde",
+ "serde-transcode",
  "sled",
  "tracing",
 ]
@@ -1243,6 +1248,7 @@ dependencies = [
 name = "garage_table"
 version = "0.8.0"
 dependencies = [
+ "arc-swap",
  "async-trait",
  "bytes",
  "futures",
@@ -2140,6 +2146,17 @@ dependencies = [
  "tokio-util 0.7.0",
 ]
 
+[[package]]
+name = "nettext"
+version = "0.3.2"
+dependencies = [
+ "base64",
+ "err-derive",
+ "hex",
+ "nom",
+ "serde",
+]
+
 [[package]]
 name = "nom"
 version = "7.1.1"
@@ -3281,6 +3298,15 @@ dependencies = [
  "serde_derive",
 ]
 
+[[package]]
+name = "serde-transcode"
+version = "1.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "590c0e25c2a5bb6e85bf5c1bce768ceb86b316e7a01bdf07d2cb4ec2271990e2"
+dependencies = [
+ "serde",
+]
+
 [[package]]
 name = "serde-value"
 version = "0.7.0"

Cargo.nix

@@ -32,7 +32,7 @@ args@{
   ignoreLockHash,
 }:
 let
-  nixifiedLockHash = "90b29705f5037c7e1b33f4650841f1266f2e86fa03d5d0c87ad80be7619985c7";
+  nixifiedLockHash = "a1d84930f23d3d8abc8abbed59b8ce3c9adf9f25d06bc1f39cbdf5bd90aceead";
   workspaceSrc = if args.workspaceSrc == null then ./. else args.workspaceSrc;
   currentLockHash = builtins.hashFile "sha256" (workspaceSrc + /Cargo.lock);
   lockHashIgnored = if ignoreLockHash
@@ -1769,6 +1769,7 @@ in
     registry = "unknown";
     src = fetchCrateLocal (workspaceSrc + "/src/table");
     dependencies = {
+      arc_swap = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".arc-swap."1.5.0" { inherit profileName; }).out;
       async_trait = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".async-trait."0.1.52" { profileName = "__noProfile"; }).out;
       bytes = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.2.0" { inherit profileName; }).out;
       futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.21" { inherit profileName; }).out;
@@ -5398,7 +5399,6 @@ in
       [ "attributes" ]
       [ "default" ]
       [ "log" ]
-      [ "log-always" ]
       [ "std" ]
       [ "tracing-attributes" ]
     ];

default.nix

@@ -11,14 +11,14 @@ let
 
   build_debug_and_release = (target: {
     debug = (compile {
-      inherit target git_version;
+      inherit system target git_version pkgsSrc cargo2nixOverlay;
       release = false;
     }).workspace.garage {
       compileMode = "build";
     };
 
     release = (compile {
-      inherit target git_version;
+      inherit system target git_version pkgsSrc cargo2nixOverlay;
       release = true;
     }).workspace.garage {
       compileMode = "build";
@@ -39,7 +39,7 @@ in {
   };
   test = {
     amd64 = test (compile {
-      inherit git_version;
+      inherit system git_version pkgsSrc cargo2nixOverlay;
       target = "x86_64-unknown-linux-musl";
       features = [
         "garage/bundled-libs"
@@ -52,7 +52,7 @@ in {
   };
   clippy = {
     amd64 = (compile {
-      inherit git_version;
+      inherit system git_version pkgsSrc cargo2nixOverlay;
       target = "x86_64-unknown-linux-musl";
       compiler = "clippy";
     }).workspace.garage {

doc/book/cookbook/real-world.md

@@ -109,7 +109,7 @@ especially you must consider the following folders/files:
   this folder will be your main data storage and must be on a large storage (e.g. large HDD)
 
 
-A valid `/etc/garage/garage.toml` for our cluster would look as follows:
+A valid `/etc/garage.toml` for our cluster would look as follows:
 
 ```toml
 metadata_dir = "/var/lib/garage/meta"

doc/book/design/goals.md

@@ -12,7 +12,7 @@ as pictures, video, images, documents, etc., in a redundant multi-node
 setting. S3 is versatile enough to also be used to publish a static
 website.
 
-Garage is an opinionated object storage solutoin, we focus on the following **desirable properties**:
+Garage is an opinionated object storage solution, we focus on the following **desirable properties**:
 
   - **Internet enabled**: made for multi-sites (eg. datacenters, offices, households, etc.) interconnected through regular Internet connections.
   - **Self-contained & lightweight**: works everywhere and integrates well in existing environments to target [hyperconverged infrastructures](https://en.wikipedia.org/wiki/Hyper-converged_infrastructure).

doc/book/reference-manual/features.md

@@ -83,7 +83,7 @@ This feature is totally invisible to S3 clients and does not break compatibility
 ### Cluster administration API
 
 Garage provides a fully-fledged REST API to administer your cluster programatically.
-Functionnality included in the admin API include: setting up and monitoring
+Functionality included in the admin API include: setting up and monitoring
 cluster nodes, managing access credentials, and managing storage buckets and bucket aliases.
 A full reference of the administration API is available [here](@/documentation/reference-manual/admin-api.md).
 

doc/drafts/admin-api.md

@@ -0,0 +1,686 @@
++++
+title = "Administration API"
+weight = 60
++++
+
+The Garage administration API is accessible through a dedicated server whose
+listen address is specified in the `[admin]` section of the configuration
+file (see [configuration file
+reference](@/documentation/reference-manual/configuration.md))
+
+**WARNING.** At this point, there is no comittement to stability of the APIs described in this document.
+We will bump the version numbers prefixed to each API endpoint at each time the syntax
+or semantics change, meaning that code that relies on these endpoint will break
+when changes are introduced.
+
+The Garage administration API was introduced in version 0.7.2, this document
+does not apply to older versions of Garage.
+
+
+## Access control
+
+The admin API uses two different tokens for acces control, that are specified in the config file's `[admin]` section:
+
+- `metrics_token`: the token for accessing the Metrics endpoint (if this token
+  is not set in the config file, the Metrics endpoint can be accessed without
+  access control);
+
+- `admin_token`: the token for accessing all of the other administration
+  endpoints (if this token is not set in the config file, access to these
+  endpoints is disabled entirely).
+
+These tokens are used as simple HTTP bearer tokens. In other words, to
+authenticate access to an admin API endpoint, add the following HTTP header
+to your request:
+
+```
+Authorization: Bearer <token>
+```
+
+## Administration API endpoints
+
+### Metrics-related endpoints
+
+#### Metrics `GET /metrics`
+
+Returns internal Garage metrics in Prometheus format.
+
+#### Health `GET /health`
+
+Used for simple health checks in a cluster setting with an orchestrator.
+Returns an HTTP status 200 if the node is ready to answer user's requests,
+and an HTTP status 503 (Service Unavailable) if there are some partitions
+for which a quorum of nodes is not available.
+A simple textual message is also returned in a body with content-type `text/plain`.
+See `/v0/health` for an API that also returns JSON output.
+
+### Cluster operations
+
+#### GetClusterStatus `GET /v0/status`
+
+Returns the cluster's current status in JSON, including:
+
+- ID of the node being queried and its version of the Garage daemon
+- Live nodes
+- Currently configured cluster layout
+- Staged changes to the cluster layout
+
+Example response body:
+
+```json
+{
+  "node": "ec79480e0ce52ae26fd00c9da684e4fa56658d9c64cdcecb094e936de0bfe71f",
+  "garage_version": "git:v0.8.0",
+  "knownNodes": {
+    "ec79480e0ce52ae26fd00c9da684e4fa56658d9c64cdcecb094e936de0bfe71f": {
+      "addr": "10.0.0.11:3901",
+      "is_up": true,
+      "last_seen_secs_ago": 9,
+      "hostname": "node1"
+    },
+    "4a6ae5a1d0d33bf895f5bb4f0a418b7dc94c47c0dd2eb108d1158f3c8f60b0ff": {
+      "addr": "10.0.0.12:3901",
+      "is_up": true,
+      "last_seen_secs_ago": 1,
+      "hostname": "node2"
+    },
+    "23ffd0cdd375ebff573b20cc5cef38996b51c1a7d6dbcf2c6e619876e507cf27": {
+      "addr": "10.0.0.21:3901",
+      "is_up": true,
+      "last_seen_secs_ago": 7,
+      "hostname": "node3"
+    },
+    "e2ee7984ee65b260682086ec70026165903c86e601a4a5a501c1900afe28d84b": {
+      "addr": "10.0.0.22:3901",
+      "is_up": true,
+      "last_seen_secs_ago": 1,
+      "hostname": "node4"
+    }
+  },
+  "layout": {
+    "version": 12,
+    "roles": {
+      "ec79480e0ce52ae26fd00c9da684e4fa56658d9c64cdcecb094e936de0bfe71f": {
+        "zone": "dc1",
+        "capacity": 4,
+        "tags": [
+          "node1"
+        ]
+      },
+      "4a6ae5a1d0d33bf895f5bb4f0a418b7dc94c47c0dd2eb108d1158f3c8f60b0ff": {
+        "zone": "dc1",
+        "capacity": 6,
+        "tags": [
+          "node2"
+        ]
+      },
+      "23ffd0cdd375ebff573b20cc5cef38996b51c1a7d6dbcf2c6e619876e507cf27": {
+        "zone": "dc2",
+        "capacity": 10,
+        "tags": [
+          "node3"
+        ]
+      }
+    },
+    "stagedRoleChanges": {
+      "e2ee7984ee65b260682086ec70026165903c86e601a4a5a501c1900afe28d84b": {
+        "zone": "dc2",
+        "capacity": 5,
+        "tags": [
+          "node4"
+        ]
+      }
+    }
+  }
+}
+```
+
+#### GetClusterHealth `GET /v0/health`
+
+Returns the cluster's current health in JSON format, with the following variables:
+
+- `status`: one of `Healthy`, `Degraded` or `Unavailable`:
+  - Healthy: Garage node is connected to all storage nodes
+  - Degraded: Garage node is not connected to all storage nodes, but a quorum of write nodes is available for all partitions
+  - Unavailable: a quorum of write nodes is not available for some partitions
+- `known_nodes`: the number of nodes this Garage node has had a TCP connection to since the daemon started
+- `connected_nodes`: the nubmer of nodes this Garage node currently has an open connection to
+- `storage_nodes`: the number of storage nodes currently registered in the cluster layout
+- `storage_nodes_ok`: the number of storage nodes to which a connection is currently open
+- `partitions`: the total number of partitions of the data (currently always 256)
+- `partitions_quorum`: the number of partitions for which a quorum of write nodes is available
+- `partitions_all_ok`: the number of partitions for which we are connected to all storage nodes responsible of storing it
+
+Contrarily to `GET /health`, this endpoint always returns a 200 OK HTTP response code.
+
+Example response body:
+
+```json
+{
+    "status": "Degraded",
+    "known_nodes": 3,
+    "connected_nodes": 2,
+    "storage_nodes": 3,
+    "storage_nodes_ok": 2,
+    "partitions": 256,
+    "partitions_quorum": 256,
+    "partitions_all_ok": 0
+}
+```
+
+#### ConnectClusterNodes `POST /v0/connect`
+
+Instructs this Garage node to connect to other Garage nodes at specified addresses.
+
+Example request body:
+
+```json
+[
+    "ec79480e0ce52ae26fd00c9da684e4fa56658d9c64cdcecb094e936de0bfe71f@10.0.0.11:3901",
+    "4a6ae5a1d0d33bf895f5bb4f0a418b7dc94c47c0dd2eb108d1158f3c8f60b0ff@10.0.0.12:3901"
+]
+```
+
+The format of the string for a node to connect to is: `<node ID>@<ip address>:<port>`, same as in the `garage node connect` CLI call.
+
+Example response:
+
+```json
+[
+    {
+        "success": true,
+        "error": null
+    },
+    {
+        "success": false,
+        "error": "Handshake error"
+    }
+]
+```
+
+#### GetClusterLayout `GET /v0/layout`
+
+Returns the cluster's current layout in JSON, including:
+
+- Currently configured cluster layout
+- Staged changes to the cluster layout
+
+(the info returned by this endpoint is a subset of the info returned by GetClusterStatus)
+
+Example response body:
+
+```json
+{
+  "version": 12,
+  "roles": {
+    "ec79480e0ce52ae26fd00c9da684e4fa56658d9c64cdcecb094e936de0bfe71f": {
+      "zone": "dc1",
+      "capacity": 4,
+      "tags": [
+        "node1"
+      ]
+    },
+    "4a6ae5a1d0d33bf895f5bb4f0a418b7dc94c47c0dd2eb108d1158f3c8f60b0ff": {
+      "zone": "dc1",
+      "capacity": 6,
+      "tags": [
+        "node2"
+      ]
+    },
+    "23ffd0cdd375ebff573b20cc5cef38996b51c1a7d6dbcf2c6e619876e507cf27": {
+      "zone": "dc2",
+      "capacity": 10,
+      "tags": [
+        "node3"
+      ]
+    }
+  },
+  "stagedRoleChanges": {
+    "e2ee7984ee65b260682086ec70026165903c86e601a4a5a501c1900afe28d84b": {
+      "zone": "dc2",
+      "capacity": 5,
+      "tags": [
+        "node4"
+      ]
+    }
+  }
+}
+```
+
+#### UpdateClusterLayout `POST /v0/layout`
+
+Send modifications to the cluster layout. These modifications will
+be included in the staged role changes, visible in subsequent calls
+of `GetClusterLayout`. Once the set of staged changes is satisfactory,
+the user may call `ApplyClusterLayout` to apply the changed changes,
+or `Revert ClusterLayout` to clear all of the staged changes in
+the layout.
+
+Request body format:
+
+```json
+{
+  <node_id>: {
+    "capacity": <new_capacity>,
+    "zone": <new_zone>,
+    "tags": [
+      <new_tag>,
+      ...
+    ]
+  },
+  <node_id_to_remove>: null,
+  ...
+}
+```
+
+Contrary to the CLI that may update only a subset of the fields
+`capacity`, `zone` and `tags`, when calling this API all of these
+values must be specified.
+
+
+#### ApplyClusterLayout `POST /v0/layout/apply`
+
+Applies to the cluster the layout changes currently registered as
+staged layout changes.
+
+Request body format:
+
+```json
+{
+  "version": 13
+}
+```
+
+Similarly to the CLI, the body must include the version of the new layout
+that will be created, which MUST be 1 + the value of the currently
+existing layout in the cluster.
+
+#### RevertClusterLayout `POST /v0/layout/revert`
+
+Clears all of the staged layout changes.
+
+Request body format:
+
+```json
+{
+  "version": 13
+}
+```
+
+Reverting the staged changes is done by incrementing the version number
+and clearing the contents of the staged change list.
+Similarly to the CLI, the body must include the incremented
+version number, which MUST be 1 + the value of the currently
+existing layout in the cluster.
+
+
+### Access key operations
+
+#### ListKeys `GET /v0/key`
+
+Returns all API access keys in the cluster.
+
+Example response:
+
+```json
+[
+  {
+    "id": "GK31c2f218a2e44f485b94239e",
+    "name": "test"
+  },
+  {
+    "id": "GKe10061ac9c2921f09e4c5540",
+    "name": "test2"
+  }
+]
+```
+
+#### CreateKey `POST /v0/key`
+
+Creates a new API access key.
+
+Request body format:
+
+```json
+{
+    "name": "NameOfMyKey"
+}
+```
+
+#### ImportKey `POST /v0/key/import`
+
+Imports an existing API key.
+
+Request body format:
+
+```json
+{
+    "accessKeyId": "GK31c2f218a2e44f485b94239e",
+    "secretAccessKey": "b892c0665f0ada8a4755dae98baa3b133590e11dae3bcc1f9d769d67f16c3835",
+    "name": "NameOfMyKey"
+}
+```
+
+#### GetKeyInfo `GET /v0/key?id=<acces key id>`
+#### GetKeyInfo `GET /v0/key?search=<pattern>`
+
+Returns information about the requested API access key.
+
+If `id` is set, the key is looked up using its exact identifier (faster).
+If `search` is set, the key is looked up using its name or prefix
+of identifier (slower, all keys are enumerated to do this).
+
+Example response:
+
+```json
+{
+  "name": "test",
+  "accessKeyId": "GK31c2f218a2e44f485b94239e",
+  "secretAccessKey": "b892c0665f0ada8a4755dae98baa3b133590e11dae3bcc1f9d769d67f16c3835",
+  "permissions": {
+    "createBucket": false
+  },
+  "buckets": [
+    {
+      "id": "70dc3bed7fe83a75e46b66e7ddef7d56e65f3c02f9f80b6749fb97eccb5e1033",
+      "globalAliases": [
+        "test2"
+      ],
+      "localAliases": [],
+      "permissions": {
+        "read": true,
+        "write": true,
+        "owner": false
+      }
+    },
+    {
+      "id": "d7452a935e663fc1914f3a5515163a6d3724010ce8dfd9e4743ca8be5974f995",
+      "globalAliases": [
+        "test3"
+      ],
+      "localAliases": [],
+      "permissions": {
+        "read": true,
+        "write": true,
+        "owner": false
+      }
+    },
+    {
+      "id": "e6a14cd6a27f48684579ec6b381c078ab11697e6bc8513b72b2f5307e25fff9b",
+      "globalAliases": [],
+      "localAliases": [
+        "test"
+      ],
+      "permissions": {
+        "read": true,
+        "write": true,
+        "owner": true
+      }
+    },
+    {
+      "id": "96470e0df00ec28807138daf01915cfda2bee8eccc91dea9558c0b4855b5bf95",
+      "globalAliases": [
+        "alex"
+      ],
+      "localAliases": [],
+      "permissions": {
+        "read": true,
+        "write": true,
+        "owner": true
+      }
+    }
+  ]
+}
+```
+
+#### DeleteKey `DELETE /v0/key?id=<acces key id>`
+
+Deletes an API access key.
+
+#### UpdateKey `POST /v0/key?id=<acces key id>`
+
+Updates information about the specified API access key.
+
+Request body format:
+
+```json
+{
+    "name": "NameOfMyKey",
+    "allow": {
+        "createBucket": true,
+    },
+    "deny": {}
+}
+```
+
+All fields (`name`, `allow` and `deny`) are optionnal.
+If they are present, the corresponding modifications are applied to the key, otherwise nothing is changed.
+The possible flags in `allow` and `deny` are: `createBucket`.
+
+
+### Bucket operations
+
+#### ListBuckets `GET /v0/bucket`
+
+Returns all storage buckets in the cluster.
+
+Example response:
+
+```json
+[
+  {
+    "id": "70dc3bed7fe83a75e46b66e7ddef7d56e65f3c02f9f80b6749fb97eccb5e1033",
+    "globalAliases": [
+      "test2"
+    ],
+    "localAliases": []
+  },
+  {
+    "id": "96470e0df00ec28807138daf01915cfda2bee8eccc91dea9558c0b4855b5bf95",
+    "globalAliases": [
+      "alex"
+    ],
+    "localAliases": []
+  },
+  {
+    "id": "d7452a935e663fc1914f3a5515163a6d3724010ce8dfd9e4743ca8be5974f995",
+    "globalAliases": [
+      "test3"
+    ],
+    "localAliases": []
+  },
+  {
+    "id": "e6a14cd6a27f48684579ec6b381c078ab11697e6bc8513b72b2f5307e25fff9b",
+    "globalAliases": [],
+    "localAliases": [
+      {
+        "accessKeyId": "GK31c2f218a2e44f485b94239e",
+        "alias": "test"
+      }
+    ]
+  }
+]
+```
+
+#### GetBucketInfo `GET /v0/bucket?id=<bucket id>`
+#### GetBucketInfo `GET /v0/bucket?globalAlias=<alias>`
+
+Returns information about the requested storage bucket.
+
+If `id` is set, the bucket is looked up using its exact identifier.
+If `globalAlias` is set, the bucket is looked up using its global alias.
+(both are fast)
+
+Example response:
+
+```json
+{
+    "id": "afa8f0a22b40b1247ccd0affb869b0af5cff980924a20e4b5e0720a44deb8d39",
+        "globalAliases": [],
+        "websiteAccess": false,
+        "websiteConfig": null,
+        "keys": [
+        {
+            "accessKeyId": "GK31c2f218a2e44f485b94239e",
+            "name": "Imported key",
+            "permissions": {
+                "read": true,
+                "write": true,
+                "owner": true
+            },
+            "bucketLocalAliases": [
+                "debug"
+            ]
+        }
+        ],
+        "objects": 14827,
+        "bytes": 13189855625,
+        "unfinshedUploads": 0,
+        "quotas": {
+            "maxSize": null,
+            "maxObjects": null
+        }
+}
+```
+
+#### CreateBucket `POST /v0/bucket`
+
+Creates a new storage bucket.
+
+Request body format:
+
+```json
+{
+    "globalAlias": "NameOfMyBucket"
+}
+```
+
+OR
+
+```json
+{
+    "localAlias": {
+        "accessKeyId": "GK31c2f218a2e44f485b94239e",
+        "alias": "NameOfMyBucket",
+        "allow": {
+            "read": true,
+            "write": true,
+            "owner": false
+        }
+    }
+}
+```
+
+OR
+
+```json
+{}
+```
+
+Creates a new bucket, either with a global alias, a local one,
+or no alias at all.
+
+Technically, you can also specify both `globalAlias` and `localAlias` and that would create
+two aliases, but I don't see why you would want to do that.
+
+#### DeleteBucket `DELETE /v0/bucket?id=<bucket id>`
+
+Deletes a storage bucket. A bucket cannot be deleted if it is not empty.
+
+Warning: this will delete all aliases associated with the bucket!
+
+#### UpdateBucket `PUT /v0/bucket?id=<bucket id>`
+
+Updates configuration of the given bucket.
+
+Request body format:
+
+```json
+{
+    "websiteAccess": {
+        "enabled": true,
+        "indexDocument": "index.html",
+        "errorDocument": "404.html"
+    },
+    "quotas": {
+        "maxSize": 19029801,
+        "maxObjects": null,
+    }
+}
+```
+
+All fields (`websiteAccess` and `quotas`) are optionnal.
+If they are present, the corresponding modifications are applied to the bucket, otherwise nothing is changed.
+
+In `websiteAccess`: if `enabled` is `true`, `indexDocument` must be specified.
+The field `errorDocument` is optional, if no error document is set a generic
+error message is displayed when errors happen. Conversely, if `enabled` is
+`false`, neither `indexDocument` nor `errorDocument` must be specified.
+
+In `quotas`: new values of `maxSize` and `maxObjects` must both be specified, or set to `null`
+to remove the quotas. An absent value will be considered the same as a `null`. It is not possible
+to change only one of the two quotas.
+
+### Operations on permissions for keys on buckets
+
+#### BucketAllowKey `POST /v0/bucket/allow`
+
+Allows a key to do read/write/owner operations on a bucket.
+
+Request body format:
+
+```json
+{
+    "bucketId": "e6a14cd6a27f48684579ec6b381c078ab11697e6bc8513b72b2f5307e25fff9b",
+    "accessKeyId": "GK31c2f218a2e44f485b94239e",
+    "permissions": {
+        "read": true,
+        "write": true,
+        "owner": true
+    },
+}
+```
+
+Flags in `permissions` which have the value `true` will be activated.
+Other flags will remain unchanged.
+
+#### BucketDenyKey `POST /v0/bucket/deny`
+
+Denies a key from doing read/write/owner operations on a bucket.
+
+Request body format:
+
+```json
+{
+    "bucketId": "e6a14cd6a27f48684579ec6b381c078ab11697e6bc8513b72b2f5307e25fff9b",
+    "accessKeyId": "GK31c2f218a2e44f485b94239e",
+    "permissions": {
+        "read": false,
+        "write": false,
+        "owner": true
+    },
+}
+```
+
+Flags in `permissions` which have the value `true` will be deactivated.
+Other flags will remain unchanged.
+
+
+### Operations on bucket aliases
+
+#### GlobalAliasBucket `PUT /v0/bucket/alias/global?id=<bucket id>&alias=<global alias>`
+
+Empty body. Creates a global alias for a bucket.
+
+#### GlobalUnaliasBucket `DELETE /v0/bucket/alias/global?id=<bucket id>&alias=<global alias>`
+
+Removes a global alias for a bucket.
+
+#### LocalAliasBucket `PUT /v0/bucket/alias/local?id=<bucket id>&accessKeyId=<access key ID>&alias=<local alias>`
+
+Empty body. Creates a local alias for a bucket in the namespace of a specific access key.
+
+#### LocalUnaliasBucket `DELETE /v0/bucket/alias/local?id=<bucket id>&accessKeyId<access key ID>&alias=<local alias>`
+
+Removes a local alias for a bucket in the namespace of a specific access key.
+

doc/talks/2022-11-19-Capitole-du-Libre/.gitignore

@@ -0,0 +1,10 @@
+*.aux
+*.bbl
+*.blg
+*.log
+*.nav
+*.out
+*.snm
+*.synctex.gz
+*.toc
+*.dvi

doc/talks/2022-11-19-Capitole-du-Libre/Makefile

@@ -0,0 +1,8 @@
+all:
+	pdflatex présentation.tex
+
+clean:
+	rm -f *.aux *.bbl *.blg *.log *.nav *.out *.snm *.synctex.gz *.toc *.dvi présentation.pdf
+
+clean_sauf_pdf:
+	rm -f *.aux *.bbl *.blg *.log *.nav *.out *.snm *.synctex.gz *.toc *.dvi

doc/talks/2022-11-19-Capitole-du-Libre/présentation.tex

@@ -0,0 +1,340 @@
+\documentclass[11pt, aspectratio=1610]{beamer}
+\usetheme{Warsaw}
+\usepackage[utf8]{inputenc}
+\usepackage[french]{babel}
+\usepackage{amsmath}
+\usepackage{amsfonts}
+\usepackage{amssymb}
+\usepackage{tikz}
+\usepackage{graphicx}
+\usepackage{xcolor}
+\usepackage{setspace}
+\usepackage{todonotes}
+\presetkeys{todonotes}{inline}{}
+\renewcommand{\baselinestretch}{1.25}
+
+\definecolor{orange_garage}{RGB}{255,147,41}
+\definecolor{gris_garage}{RGB}{78,78,78}
+
+\author[Association Deuxfleurs]{~\linebreak Vincent Giraud}
+\title[De l'auto-hébergement à l'entre-hébergement avec Garage]{De l'auto-hébergement à l'entre-hébergement :\\Garage, pour conserver ses données ensemble}
+%\setbeamercovered{transparent} 
+%\setbeamertemplate{navigation symbols}{} 
+\date{Capitole du Libre 2022\linebreak
+
+\scriptsize Samedi 19 novembre 2022\linebreak
+} 
+
+\setbeamercolor{palette primary}{fg=gris_garage,bg=orange_garage}
+\setbeamercolor{palette secondary}{fg=gris_garage,bg=gris_garage}
+\setbeamercolor{palette tiertary}{fg=white,bg=gris_garage}
+\setbeamercolor{palette quaternary}{fg=white,bg=gris_garage}
+\setbeamercolor{navigation symbols}{fg=black, bg=white}
+\setbeamercolor{navigation symbols dimmed}{fg=darkgray, bg=white}
+\setbeamercolor{itemize item}{fg=gris_garage}
+\setbeamertemplate{itemize item}[circle]
+
+\addtobeamertemplate{navigation symbols}{}{%
+    \usebeamerfont{footline}%
+    \usebeamercolor[fg]{footline}%
+    \hspace{1em}%
+    \insertframenumber/\inserttotalframenumber
+}
+
+\setbeamertemplate{headline}
+{%
+  \leavevmode%
+  \begin{beamercolorbox}[wd=.5\paperwidth,ht=2.5ex,dp=1.125ex]{section in head/foot}%
+    \hbox to .5\paperwidth{\hfil\insertsectionhead\hfil}
+  \end{beamercolorbox}%
+  \begin{beamercolorbox}[wd=.5\paperwidth,ht=2.5ex,dp=1.125ex]{subsection in head/foot}%
+    \hbox to .5\paperwidth{\hfil\insertsubsectionhead\hfil}
+  \end{beamercolorbox}%
+}
+\addtobeamertemplate{footnote}{}{\vspace{2ex}}
+
+\begin{document}
+\begin{frame}
+\titlepage
+\end{frame}
+
+\section{Introduction}
+\subsection{Présentation}
+\begin{frame}
+\begin{columns}
+\column{0.5 \linewidth}
+\begin{center}
+\includegraphics[width=3.5cm]{deuxfleurs-logo.png}\linebreak
+
+\texttt{https://deuxfleurs.fr}
+\end{center}
+\column{0.4 \linewidth}
+\begin{center}
+Deuxfleurs est une association militant en faveur d'un internet plus convivial, avec une organisation et des rapports de force repensés.\linebreak
+
+Nous faisons partie du CHATONS\footnote[frame]{Collectif des Hébergeurs Alternatifs, Transparents, Ouverts, Neutres et Solidaires} depuis avril 2022.
+
+\includegraphics[width=2cm]{logo_chatons.png}
+\end{center}
+\end{columns}
+\end{frame}
+
+\subsection{Héberger à la maison}
+\begin{frame}
+\begin{columns}
+\begin{column}{0.5 \linewidth}
+\begin{center}
+Pour échapper au contrôle et au giron des opérateurs de clouds, héberger ses données à la maison présente de nombreux avantages...
+\end{center}
+
+\vspace{0.5cm}
+
+\begin{itemize}[<+(1)->]
+\item On récupère la souveraineté sur ses données
+\item On gagne en vie privée
+\item On gagne en libertés
+\item On est responsabilisé face à ses besoins
+\end{itemize}
+\end{column}
+\vrule{}
+\begin{column}{0.5 \linewidth}
+\begin{center}
+\onslide<6->{... mais aussi bien des contraintes...}
+\end{center}
+
+\vspace{0.5cm}
+
+\begin{itemize}[<+(2)->]
+\item On repose sur une connexion internet pour particulier
+\item Un certain savoir-faire et moultes compétences sont requis
+\item Assurer la résilience de ses services est difficile
+\item Bien sauvegarder ses données, et ceci au-delà de son site géographique, n'est pas évident
+\end{itemize}
+\end{column}
+\end{columns}
+\end{frame}
+
+\subsection{Sauvegarder pour se parer à tout imprévu}
+\begin{frame}
+\begin{center}
+Sauvegarder pour se parer contre les pannes matérielles est une chose...
+
+Sauvegarder pour se parer contre les cambriolages et les incendies en est une autre !\linebreak
+
+\vspace{1cm}
+\onslide<2->{Répartir géographiquement ses données devient alors nécessaire.}
+\end{center}
+\end{frame}
+
+\section{Les solutions à explorer}
+\subsection{L'entre-hébergement}
+\begin{frame}
+\begin{center}
+On a vu récemment se développer au sein du CHATONS la notion d'entre-hébergement : en plus de renforcer l'intégrité des sauvegardes, on va améliorer la disponibilité pendant les coupures de liaison internet, de courant, ou pendant les déménagements d'administrateurs par exemple.\linebreak
+
+\vspace{1cm}
+\onslide<2->
+{
+Dans le cadre du collectif, il s'agit de partager ses volumes de données entre hébergeurs.\linebreak
+
+Pour assurer la confidentialité, on peut chiffrer les données au niveau applicatif.
+}
+\end{center}
+\end{frame}
+
+\subsection{S3 contre les systèmes de fichiers}
+\begin{frame}
+\begin{center}
+Dans le cadre de l'administration de services en ligne, les systèmes de fichiers recèlent certaines difficultés.\linebreak
+
+\vspace{1cm}
+Le standard S3 apporte des facilités; on réduit le stockage à un paradigme de clé-valeur basé essentiellement sur deux opérations seulement: lire ou écrire une clé.
+\end{center}
+\end{frame}
+
+\section{Garage}
+\subsection{Présentation}
+\begin{frame}
+\begin{columns}
+\column{0.5 \linewidth}
+\begin{center}
+Garage essaye de répondre à l'ensemble de ces besoins.\linebreak
+
+\vspace{0.5cm}
+Il s'agit d'un logiciel libre permettant de distribuer un service S3 sur diverses machines éloignées.
+\end{center}
+\column{0.5 \linewidth}
+\begin{center}
+\includegraphics[width=4cm]{garage-logo.png}\linebreak
+
+\texttt{https://garagehq.deuxfleurs.fr/}
+\end{center}
+\end{columns}
+\end{frame}
+
+\subsection{Gestion des zones}
+\begin{frame}
+\begin{center}
+Garage va prendre en compte les zones géographiques au moment de répliquer les données.\linebreak
+
+\vspace{1cm}
+\includegraphics[width=13.25cm]{zones.png}
+\end{center}
+\end{frame}
+
+\subsection{Comment ça marche ?}
+\begin{frame}
+\begin{columns}
+\column{0.5 \linewidth}
+\input{schéma europe}
+\column{0.5 \linewidth}
+\begin{center}
+Chaque objet est dupliqué sur plusieurs zones différentes.\linebreak
+
+\onslide<5->{Lorsqu'un nouvel hébergeur rejoint le réseau, la charge se voit équilibrée.}\linebreak
+
+\onslide<12->{Si une zone devient indisponible, les autres continuent d'assurer le service.}\linebreak
+\end{center}
+\end{columns}
+\end{frame}
+
+\subsection{Financement}
+\begin{frame}
+\begin{center}
+Dans le cadre du programme \textit{Horizon 2021} de l'Union Européenne, nous avons reçu une subvention de la part de l'initiative NGI Pointer\footnote[frame]{Next Generation Internet Program for Open Internet Renovation}.\linebreak
+
+\includegraphics[width=3cm]{drapeau_européen.png}\hspace{1cm}
+\includegraphics[width=3cm]{NGI.png}\linebreak
+
+Nous avons ainsi pu financer le développement de Garage pendant 1 an.
+\end{center}
+\end{frame}
+
+\subsection{Licence}
+\begin{frame}
+\begin{center}
+De par nos valeurs, nous avons attribué la licence AGPL version 3 à Garage, notamment afin qu'il reste parmi les biens communs.\linebreak
+
+\vspace{0.5cm}
+\includegraphics[width=5cm]{agpl-v3-logo.png}\linebreak
+\end{center}
+\end{frame}
+
+\subsection{Langage utilisé}
+\begin{frame}
+\begin{center}
+Nous avons décidé d'écrire Garage à l'aide du langage Rust, afin d'obtenir une compilation vers des binaires natifs et efficaces.\linebreak
+
+\includegraphics[width=3.5cm]{rust-logo.png}\linebreak
+
+Ce choix permet également de bénéficier des avantages reconnus de Rust en termes de sécurité.
+\end{center}
+\end{frame}
+
+\subsection{Matériel requis}
+\begin{frame}
+\begin{center}
+Garage peut ainsi être performant sur des machines limitées. Les prérequis sont minimes : n'importe quelle machine avec un processeur qui a moins d'une décennie, 1~gigaoctet de mémoire vive, et 16~gigaoctets de stockage suffit.\linebreak
+
+\vspace{1cm}
+
+Cet aspect est déterminant : il permet en effet d'héberger sur du matériel acheté d'occasion, pour réduire l'impact écologique de nos infrastructures.
+\end{center}
+\end{frame}
+
+\subsection{Performances}
+\begin{frame}
+\begin{center}
+\includegraphics[width=13.25cm]{rpc-amplification.png}
+\end{center}
+\end{frame}
+
+\begin{frame}
+\begin{center}
+\includegraphics[width=11cm]{rpc-complexity.png}
+\end{center}
+\end{frame}
+
+\subsection{Services}
+\begin{frame}
+\begin{center}
+Puisqu'il suit le standard S3, beaucoup de services populaires sont par conséquence compatibles avec Garage :\linebreak
+
+\begin{columns}
+\column{0.2 \linewidth}
+\begin{center}
+\includegraphics[width=2.5cm]{nextcloud-logo.png}
+\end{center}
+\column{0.2 \linewidth}
+\begin{center}
+\includegraphics[width=2.5cm]{peertube-logo.png}
+\end{center}
+\column{0.2 \linewidth}
+\begin{center}
+\includegraphics[width=2.5cm]{matrix-logo.png}
+\end{center}
+\column{0.2 \linewidth}
+\begin{center}
+\includegraphics[width=2.5cm]{mastodon-logo.png}
+\end{center}
+\end{columns}
+~\linebreak
+
+Et comme souvent avec S3, on peut assimiler un bucket à un site, et utiliser le serveur pour héberger des sites web statiques.
+\end{center}
+\end{frame}
+
+\section{Intégration chez Deuxfleurs}
+\subsection{Matériel}
+\begin{frame}
+\begin{center}
+\includegraphics[width=13cm]{neptune.jpg}\linebreak
+
+En pratique, nos serveurs ne sont effectivement que des machines achetées d'occasion (très souvent des anciens ordinateurs destinés à la bureautique en entreprise).
+\end{center}
+\end{frame}
+
+\subsection{Environnement logiciel}
+\begin{frame}
+\begin{center}
+Pour faciliter la reproduction d'un environnement connu, NixOS est installé sur nos machines.\linebreak
+
+\vspace{1cm}
+Pour s’accommoder des réseaux qu'on trouve derrière des routeurs pour particuliers, on s'aide de notre logiciel Diplonat\footnote[frame]{\texttt{https://git.deuxfleurs.fr/Deuxfleurs/diplonat}}.
+\end{center}
+\end{frame}
+
+\section{Au-delà...}
+\subsection{... de Deuxfleurs}
+\begin{frame}
+\begin{center}
+\includegraphics[width=10cm]{tedomum.png}
+\end{center}
+\end{frame}
+
+\subsection{... de Garage}
+\begin{frame}
+\begin{center}
+Nous avons récemment lancé le développement d'Aérogramme\footnote[frame]{\texttt{https://git.deuxfleurs.fr/Deuxfleurs/aerogramme}}.\linebreak
+
+\vspace{1cm}
+Il s'agit d'un serveur de stockage de courriels chiffrés.\linebreak
+
+\vspace{1cm}
+Il est conçu pour pouvoir travailler avec Garage.
+\end{center}
+\end{frame}
+
+\section{Fin}
+\subsection{Contacts}
+\begin{frame}
+\begin{center}
+\begin{tikzpicture}
+\node (ronce) {\includegraphics[width=0.95\textwidth]{ronce.jpg}};
+\node[white] at (-5.1,3.6) {Intéressé(e) ?};
+\node[white, align=center] at (4.2,-2.6) {Contactez-nous !\\\texttt{coucou@deuxfleurs.fr}\\\texttt{\#forum:deuxfleurs.fr}};
+\end{tikzpicture}
+\end{center}
+\end{frame}
+\end{document}
+

doc/talks/2022-11-19-Capitole-du-Libre/schéma europe.tex

@@ -0,0 +1,52 @@
+\begin{tikzpicture}
+\node (carte) {\includegraphics[width=\textwidth]{carte-Europe.pdf}};
+
+% \personnage{position X}{position Y}{facteur d'échelle}
+\newcommand{\personnage}[4]
+{
+\fill[#4] ({#1-(0.4 * #3)},{#2-(0.9 * #3)}) .. controls ({#1-(0.4 * #3)},#2) and ({#1+(0.4 * #3)},#2) .. ({#1+(0.4 * #3)},{#2-(0.9 * #3)}) -- ({#1-(0.4 * #3)},{#2-(0.9 * #3)});
+\fill[#4] (#1,#2) circle ({0.25 * #3});
+}
+
+\onslide<1-11>{\personnage{-2.25}{-0.75}{0.75}{green}}
+\onslide<1-11>{\draw (-1.9,-1.6) rectangle ++(1,1.2);}
+\onslide<2-11>{\draw[fill=green] (-1.8,-1.525) rectangle ++(0.35,0.3) node[pos=0.5] {\tiny 1};}
+\onslide<4-5>{\draw[fill=red] (-1.8,-1.15) rectangle ++(0.35,0.3) node[pos=0.5] {\tiny 3};}
+\onslide<7-11>{\draw[fill=yellow] (-1.8,-1.15) rectangle ++(0.35,0.3) node[pos=0.5] {\tiny 4};}
+\onslide<9-11>{\draw[fill=red] (-1.8,-0.775) rectangle ++(0.35,0.3) node[pos=0.5] {\tiny 6};}
+\onslide<3-11>{\draw[fill=blue] (-1.35,-1.525) rectangle ++(0.35,0.3) node[pos=0.5, white] {\tiny 2};}
+\onslide<8-11>{\draw[fill=blue] (-1.35,-1.15) rectangle ++(0.35,0.3) node[pos=0.5, white] {\tiny 5};}
+\onslide<11-11>{\draw[fill=yellow] (-1.35,-0.775) rectangle ++(0.35,0.3) node[pos=0.5] {\tiny 8};}
+
+\personnage{1.65}{1.5}{0.75}{blue}
+\draw (0.3,0.7) rectangle ++(1,1.2);
+\onslide<2->{\draw[fill=green] (0.4,0.775) rectangle ++(0.35,0.3) node[pos=0.5] {\tiny 1};}
+\onslide<4->{\draw[fill=red] (0.4,1.15) rectangle ++(0.35,0.3) node[pos=0.5] {\tiny 3};}
+\onslide<10->{\draw[fill=green] (0.4,1.525) rectangle ++(0.35,0.3) node[pos=0.5] {\tiny 7};}
+\onslide<3->{\draw[fill=blue] (0.85,0.775) rectangle ++(0.35,0.3) node[pos=0.5, white] {\tiny 2};}
+\onslide<9->{\draw[fill=red] (0.85,1.15) rectangle ++(0.35,0.3) node[pos=0.5] {\tiny 6};}
+\onslide<11->{\draw[fill=yellow] (0.85,1.525) rectangle ++(0.35,0.3) node[pos=0.5] {\tiny 8};}
+
+\personnage{1.85}{-2.3}{0.75}{red}
+\draw (0.5,-3.15) rectangle ++(1,1.2);
+\onslide<2->{\draw[fill=green] (0.6,-3.075) rectangle ++(0.35,0.3) node[pos=0.5] {\tiny 1};}
+\onslide<4-5>{\draw[fill=red] (0.6,-2.7) rectangle ++(0.35,0.3) node[pos=0.5] {\tiny 3};}
+\onslide<7->{\draw[fill=yellow] (0.6,-2.7) rectangle ++(0.35,0.3) node[pos=0.5] {\tiny 4};}
+\onslide<9->{\draw[fill=red] (0.6,-2.325) rectangle ++(0.35,0.3) node[pos=0.5] {\tiny 6};}
+\onslide<3-5>{\draw[fill=blue] (1.05,-3.075) rectangle ++(0.35,0.3) node[pos=0.5, white] {\tiny 2};}
+\onslide<6->{\draw[fill=red] (1.05,-3.075) rectangle ++(0.35,0.3) node[pos=0.5] {\tiny 3};}
+\onslide<8->{\draw[fill=blue] (1.05,-2.7) rectangle ++(0.35,0.3) node[pos=0.5, white] {\tiny 5};}
+\onslide<10->{\draw[fill=green] (1.05,-2.325) rectangle ++(0.35,0.3) node[pos=0.5] {\tiny 7};}
+
+\onslide<5->{\personnage{1.05}{-0.15}{0.75}{yellow}}
+\onslide<5->{\draw (-0.35,-1) rectangle ++(1,1.2);}
+\onslide<6->{\draw[fill=blue] (-0.25,-0.925) rectangle ++(0.35,0.3) node[pos=0.5, white] {\tiny 2};}
+\onslide<7->{\draw[fill=yellow] (-0.25,-0.55) rectangle ++(0.35,0.3) node[pos=0.5] {\tiny 4};}
+\onslide<10->{\draw[fill=green] (-0.25,-0.175) rectangle ++(0.35,0.3) node[pos=0.5] {\tiny 7};}
+\onslide<6->{\draw[fill=red] (0.2,-0.925) rectangle ++(0.35,0.3) node[pos=0.5] {\tiny 3};}
+\onslide<8->{\draw[fill=blue] (0.2,-0.55) rectangle ++(0.35,0.3) node[pos=0.5,white] {\tiny 5};}
+\onslide<11->{\draw[fill=yellow] (0.2,-0.175) rectangle ++(0.35,0.3) node[pos=0.5] {\tiny 8};}
+
+\onslide<12->{\draw[line width=0.25cm] (-2.15,-0.5) -- ++(1,-1);}
+\onslide<12->{\draw[line width=0.25cm] (-2.15,-1.5) -- ++(1,1);}
+\end{tikzpicture}

flake.lock

@@ -0,0 +1,108 @@
+{
+  "nodes": {
+    "cargo2nix": {
+      "inputs": {
+        "flake-compat": "flake-compat",
+        "flake-utils": "flake-utils",
+        "nixpkgs": [
+          "nixpkgs"
+        ],
+        "rust-overlay": "rust-overlay"
+      },
+      "locked": {
+        "lastModified": 1666087781,
+        "narHash": "sha256-trKVdjMZ8mNkGfLcY5LsJJGtdV3xJDZnMVrkFjErlcs=",
+        "owner": "Alexis211",
+        "repo": "cargo2nix",
+        "rev": "a7a61179b66054904ef6a195d8da736eaaa06c36",
+        "type": "github"
+      },
+      "original": {
+        "owner": "Alexis211",
+        "repo": "cargo2nix",
+        "rev": "a7a61179b66054904ef6a195d8da736eaaa06c36",
+        "type": "github"
+      }
+    },
+    "flake-compat": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1650374568,
+        "narHash": "sha256-Z+s0J8/r907g149rllvwhb4pKi8Wam5ij0st8PwAh+E=",
+        "owner": "edolstra",
+        "repo": "flake-compat",
+        "rev": "b4a34015c698c7793d592d66adbab377907a2be8",
+        "type": "github"
+      },
+      "original": {
+        "owner": "edolstra",
+        "repo": "flake-compat",
+        "type": "github"
+      }
+    },
+    "flake-utils": {
+      "locked": {
+        "lastModified": 1659877975,
+        "narHash": "sha256-zllb8aq3YO3h8B/U0/J1WBgAL8EX5yWf5pMj3G0NAmc=",
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "rev": "c0e246b9b83f637f4681389ecabcb2681b4f3af0",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "type": "github"
+      }
+    },
+    "nixpkgs": {
+      "locked": {
+        "lastModified": 1665657542,
+        "narHash": "sha256-mojxNyzbvmp8NtVtxqiHGhRfjCALLfk9i/Uup68Y5q8=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "a3073c49bc0163fea6a121c276f526837672b555",
+        "type": "github"
+      },
+      "original": {
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "a3073c49bc0163fea6a121c276f526837672b555",
+        "type": "github"
+      }
+    },
+    "root": {
+      "inputs": {
+        "cargo2nix": "cargo2nix",
+        "nixpkgs": "nixpkgs"
+      }
+    },
+    "rust-overlay": {
+      "inputs": {
+        "flake-utils": [
+          "cargo2nix",
+          "flake-utils"
+        ],
+        "nixpkgs": [
+          "cargo2nix",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1664247556,
+        "narHash": "sha256-J4vazHU3609ekn7dr+3wfqPo5WGlZVAgV7jfux352L0=",
+        "owner": "oxalica",
+        "repo": "rust-overlay",
+        "rev": "524db9c9ea7bc7743bb74cdd45b6d46ea3fcc2ab",
+        "type": "github"
+      },
+      "original": {
+        "owner": "oxalica",
+        "repo": "rust-overlay",
+        "type": "github"
+      }
+    }
+  },
+  "root": "root",
+  "version": 7
+}

flake.nix

@@ -0,0 +1,28 @@
+{
+  description = "Garage, an S3-compatible distributed object store for self-hosted deployments";
+
+  inputs.nixpkgs.url = "github:NixOS/nixpkgs/a3073c49bc0163fea6a121c276f526837672b555";
+  inputs.cargo2nix = {
+    # As of 2022-10-18: two small patches over unstable branch, one for clippy and one to fix feature detection
+    url = "github:Alexis211/cargo2nix/a7a61179b66054904ef6a195d8da736eaaa06c36";
+    inputs.nixpkgs.follows = "nixpkgs";
+  };
+
+  outputs = { self, nixpkgs, cargo2nix }: let
+    git_version = self.lastModifiedDate;
+    compile = import ./nix/compile.nix;
+    forAllSystems = nixpkgs.lib.genAttrs nixpkgs.lib.systems.flakeExposed;
+  in
+  {
+    packages = forAllSystems (system: {
+      default = (compile {
+        inherit system git_version;
+        pkgsSrc = nixpkgs;
+        cargo2nixOverlay = cargo2nix.overlays.default;
+        release = true;
+      }).workspace.garage {
+        compileMode = "build";
+      };
+    });
+  };
+}

nix/compile.nix

@@ -1,25 +1,32 @@
 {
-  system ? builtins.currentSystem,
-  target,
+  system,
+  target ? null,
+  pkgsSrc,
+  cargo2nixOverlay,
   compiler ? "rustc",
   release ? false,
   git_version ? null,
   features ? null,
 }:
 
-with import ./common.nix;
-
 let
   log = v: builtins.trace v v;
 
-  pkgs = import pkgsSrc {
-    inherit system;
-    crossSystem = {
-      config = target;
-      isStatic = true;
-    };
-    overlays = [ cargo2nixOverlay ];
-  };
+  pkgs =
+    if target != null then
+      import pkgsSrc {
+        inherit system;
+        crossSystem = {
+          config = target;
+          isStatic = true;
+        };
+        overlays = [ cargo2nixOverlay ];
+      }
+    else
+      import pkgsSrc {
+        inherit system;
+        overlays = [ cargo2nixOverlay ];
+      };
 
   /*
    Cargo2nix is built for rustOverlay which installs Rust from Mozilla releases.
@@ -34,7 +41,7 @@ let
    NixOS ships them in separate ones. We reunite them with symlinkJoin.
   */
   toolchainOptions =
-    if target == "x86_64-unknown-linux-musl" || target == "aarch64-unknown-linux-musl" then {
+    if target == null || target == "x86_64-unknown-linux-musl" || target == "aarch64-unknown-linux-musl" then {
       rustVersion = "1.63.0";
       extraRustComponents = [ "clippy" ];
     } else {

script/helm/garage/Chart.yaml

@@ -15,7 +15,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.1.0
+version: 0.2.0
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to

script/helm/garage/templates/ingress.yaml

@@ -18,6 +18,9 @@ metadata:
   name: {{ $fullName }}-s3-api
   labels:
     {{- include "garage.labels" . | nindent 4 }}
+    {{- with .Values.ingress.s3.api.labels }}
+    {{- toYaml . | nindent 4 }}
+    {{- end }}
   {{- with .Values.ingress.s3.api.annotations }}
   annotations:
     {{- toYaml . | nindent 4 }}
@@ -80,6 +83,9 @@ metadata:
   name: {{ $fullName }}-s3-web
   labels:
     {{- include "garage.labels" . | nindent 4 }}
+    {{- with .Values.ingress.s3.web.labels }}
+    {{- toYaml . | nindent 4 }}
+    {{- end }}
   {{- with .Values.ingress.s3.web.annotations }}
   annotations:
     {{- toYaml . | nindent 4 }}

script/helm/garage/values.yaml

@@ -85,14 +85,15 @@ service:
 ingress:
   s3:
     api:
-      enabled: true
+      enabled: false
       # Rely either on the className or the annotation below but not both
       # replace "nginx" by an Ingress controller
       # you can find examples here https://kubernetes.io/docs/concepts/services-networking/ingress-controllers
-      className: "nginx"
-      annotations:
+      # className: "nginx"
+      annotations: {}
         # kubernetes.io/ingress.class: "nginx"
         # kubernetes.io/tls-acme: "true"
+      labels: {}
       hosts:
         - host: "s3.garage.tld" # garage S3 API endpoint
           paths:
@@ -107,11 +108,15 @@ ingress:
       #    hosts:
       #      - kubernetes.docker.internal
     web:
-      enabled: true
-      className: "nginx"
+      enabled: false
+      # Rely either on the className or the annotation below but not both
+      # replace "nginx" by an Ingress controller
+      # you can find examples here https://kubernetes.io/docs/concepts/services-networking/ingress-controllers
+      # className: "nginx"
       annotations: {}
         # kubernetes.io/ingress.class: nginx
         # kubernetes.io/tls-acme: "true"
+      labels: {}
       hosts:
        - host: "*.web.garage.tld" # wildcard website access with bucket name prefix
          paths:

shell.nix

@@ -1,5 +1,5 @@
 {
-  system ? builtins.currentSystem,
+    system ? builtins.currentSystem,
 }:
 
 with import ./nix/common.nix;
@@ -71,13 +71,25 @@ function refresh_cache {
   for attr in clippy.amd64 test.amd64 pkgs.{amd64,i386,arm,arm64}.{debug,release}; do
     echo "Updating cache for ''${attr}"
     derivation=$(nix-instantiate --attr ''${attr})
-    nix copy \
+    nix copy -j8 \
       --to 's3://nix?endpoint=garage.deuxfleurs.fr&region=garage&secret-key=/tmp/nix-signing-key.sec' \
       $(nix-store -qR ''${derivation%\!bin})
   done
   rm /tmp/nix-signing-key.sec
 }
 
+function refresh_flake_cache {
+  pass show deuxfleurs/nix_priv_key > /tmp/nix-signing-key.sec
+  for attr in packages.x86_64-linux.default; do
+    echo "Updating cache for ''${attr}"
+    derivation=$(nix path-info --derivation ".#''${attr}")
+    nix copy -j8 \
+      --to 's3://nix?endpoint=garage.deuxfleurs.fr&region=garage&secret-key=/tmp/nix-signing-key.sec' \
+      $(nix-store -qR ''${derivation})
+  done
+  rm /tmp/nix-signing-key.sec
+}
+
 function to_s3 {
   aws \
       --endpoint-url https://garage.deuxfleurs.fr \

src/api/admin/api_server.rs

@@ -15,6 +15,7 @@ use opentelemetry_prometheus::PrometheusExporter;
 use prometheus::{Encoder, TextEncoder};
 
 use garage_model::garage::Garage;
+use garage_rpc::system::ClusterHealthStatus;
 use garage_util::error::Error as GarageError;
 
 use crate::generic_server::*;
@@ -76,6 +77,31 @@ impl AdminApiServer {
 			.body(Body::empty())?)
 	}
 
+	fn handle_health(&self) -> Result<Response<Body>, Error> {
+		let health = self.garage.system.health();
+
+		let (status, status_str) = match health.status {
+			ClusterHealthStatus::Healthy => (StatusCode::OK, "Garage is fully operational"),
+			ClusterHealthStatus::Degraded => (
+				StatusCode::OK,
+				"Garage is operational but some storage nodes are unavailable",
+			),
+			ClusterHealthStatus::Unavailable => (
+				StatusCode::SERVICE_UNAVAILABLE,
+				"Quorum is not available for some/all partitions, reads and writes will fail",
+			),
+		};
+		let status_str = format!(
+			"{}\nConsult the full health check API endpoint at /v0/health for more details\n",
+			status_str
+		);
+
+		Ok(Response::builder()
+			.status(status)
+			.header(http::header::CONTENT_TYPE, "text/plain")
+			.body(Body::from(status_str))?)
+	}
+
 	fn handle_metrics(&self) -> Result<Response<Body>, Error> {
 		#[cfg(feature = "metrics")]
 		{
@@ -124,6 +150,7 @@ impl ApiHandler for AdminApiServer {
 	) -> Result<Response<Body>, Error> {
 		let expected_auth_header =
 			match endpoint.authorization_type() {
+				Authorization::None => None,
 				Authorization::MetricsToken => self.metrics_token.as_ref(),
 				Authorization::AdminToken => match &self.admin_token {
 					None => return Err(Error::forbidden(
@@ -147,8 +174,10 @@ impl ApiHandler for AdminApiServer {
 
 		match endpoint {
 			Endpoint::Options => self.handle_options(&req),
+			Endpoint::Health => self.handle_health(),
 			Endpoint::Metrics => self.handle_metrics(),
 			Endpoint::GetClusterStatus => handle_get_cluster_status(&self.garage).await,
+			Endpoint::GetClusterHealth => handle_get_cluster_health(&self.garage).await,
 			Endpoint::ConnectClusterNodes => handle_connect_cluster_nodes(&self.garage, req).await,
 			// Layout
 			Endpoint::GetClusterLayout => handle_get_cluster_layout(&self.garage).await,

src/api/admin/cluster.rs

@@ -43,6 +43,11 @@ pub async fn handle_get_cluster_status(garage: &Arc<Garage>) -> Result<Response<
 	Ok(json_ok_response(&res)?)
 }
 
+pub async fn handle_get_cluster_health(garage: &Arc<Garage>) -> Result<Response<Body>, Error> {
+	let health = garage.system.health();
+	Ok(json_ok_response(&health)?)
+}
+
 pub async fn handle_connect_cluster_nodes(
 	garage: &Arc<Garage>,
 	req: Request<Body>,

src/api/admin/router.rs

@@ -6,6 +6,7 @@ use crate::admin::error::*;
 use crate::router_macros::*;
 
 pub enum Authorization {
+	None,
 	MetricsToken,
 	AdminToken,
 }
@@ -16,8 +17,10 @@ router_match! {@func
 #[derive(Debug, Clone, PartialEq, Eq)]
 pub enum Endpoint {
 	Options,
+	Health,
 	Metrics,
 	GetClusterStatus,
+	GetClusterHealth,
 	ConnectClusterNodes,
 	// Layout
 	GetClusterLayout,
@@ -88,8 +91,10 @@ impl Endpoint {
 
 		let res = router_match!(@gen_path_parser (req.method(), path, query) [
 			OPTIONS _ => Options,
+			GET "/health" => Health,
 			GET "/metrics" => Metrics,
 			GET "/v0/status" => GetClusterStatus,
+			GET "/v0/health" => GetClusterHealth,
 			POST "/v0/connect" => ConnectClusterNodes,
 			// Layout endpoints
 			GET "/v0/layout" => GetClusterLayout,
@@ -130,6 +135,7 @@ impl Endpoint {
 	/// Get the kind of authorization which is required to perform the operation.
 	pub fn authorization_type(&self) -> Authorization {
 		match self {
+			Self::Health => Authorization::None,
 			Self::Metrics => Authorization::MetricsToken,
 			_ => Authorization::AdminToken,
 		}
@@ -137,6 +143,7 @@ impl Endpoint {
 }
 
 generateQueryParameters! {
+	"format" => format,
 	"id" => id,
 	"search" => search,
 	"globalAlias" => global_alias,

src/api/s3/bucket.rs

@@ -161,6 +161,15 @@ pub async fn handle_create_bucket(
 			return Err(CommonError::BucketAlreadyExists.into());
 		}
 	} else {
+		// Check user is allowed to create bucket
+		if !key_params.allow_create_bucket.get() {
+			return Err(CommonError::Forbidden(format!(
+				"Access key {} is not allowed to create buckets",
+				api_key.key_id
+			))
+			.into());
+		}
+
 		// Create the bucket!
 		if !is_valid_bucket_name(&bucket_name) {
 			return Err(Error::bad_request(format!(

src/block/manager.rs

@@ -3,6 +3,7 @@ use std::pin::Pin;
 use std::sync::Arc;
 use std::time::Duration;
 
+use arc_swap::ArcSwapOption;
 use async_trait::async_trait;
 use bytes::Bytes;
 use serde::{Deserialize, Serialize};
@@ -22,6 +23,7 @@ use garage_rpc::rpc_helper::netapp::stream::{stream_asyncread, ByteStream};
 
 use garage_db as db;
 
+use garage_util::background::BackgroundRunner;
 use garage_util::data::*;
 use garage_util::error::*;
 use garage_util::metrics::RecordDuration;
@@ -87,7 +89,16 @@ pub struct BlockManager {
 
 	pub(crate) metrics: BlockManagerMetrics,
 
-	tx_scrub_command: mpsc::Sender<ScrubWorkerCommand>,
+	tx_scrub_command: ArcSwapOption<mpsc::Sender<ScrubWorkerCommand>>,
+}
+
+#[derive(Serialize, Deserialize, Clone, Debug)]
+pub struct BlockResyncErrorInfo {
+	pub hash: Hash,
+	pub refcount: u64,
+	pub error_count: u64,
+	pub last_try: u64,
+	pub next_try: u64,
 }
 
 // This custom struct contains functions that must only be ran
@@ -114,9 +125,8 @@ impl BlockManager {
 			.netapp
 			.endpoint("garage_block/manager.rs/Rpc".to_string());
 
-		let metrics = BlockManagerMetrics::new(resync.queue.clone(), resync.errors.clone());
-
-		let (scrub_tx, scrub_rx) = mpsc::channel(1);
+		let metrics =
+			BlockManagerMetrics::new(rc.rc.clone(), resync.queue.clone(), resync.errors.clone());
 
 		let block_manager = Arc::new(Self {
 			replication,
@@ -128,21 +138,24 @@ impl BlockManager {
 			system,
 			endpoint,
 			metrics,
-			tx_scrub_command: scrub_tx,
+			tx_scrub_command: ArcSwapOption::new(None),
 		});
 		block_manager.endpoint.set_handler(block_manager.clone());
 
+		block_manager
+	}
+
+	pub fn spawn_workers(self: &Arc<Self>, bg: &BackgroundRunner) {
 		// Spawn a bunch of resync workers
 		for index in 0..MAX_RESYNC_WORKERS {
-			let worker = ResyncWorker::new(index, block_manager.clone());
-			block_manager.system.background.spawn_worker(worker);
+			let worker = ResyncWorker::new(index, self.clone());
+			bg.spawn_worker(worker);
 		}
 
 		// Spawn scrub worker
-		let scrub_worker = ScrubWorker::new(block_manager.clone(), scrub_rx);
-		block_manager.system.background.spawn_worker(scrub_worker);
-
-		block_manager
+		let (scrub_tx, scrub_rx) = mpsc::channel(1);
+		self.tx_scrub_command.store(Some(Arc::new(scrub_tx)));
+		bg.spawn_worker(ScrubWorker::new(self.clone(), scrub_rx));
 	}
 
 	/// Ask nodes that might have a (possibly compressed) block for it
@@ -309,9 +322,42 @@ impl BlockManager {
 		Ok(self.rc.rc.len()?)
 	}
 
+	/// Get number of items in the refcount table
+	pub fn rc_fast_len(&self) -> Result<Option<usize>, Error> {
+		Ok(self.rc.rc.fast_len()?)
+	}
+
 	/// Send command to start/stop/manager scrub worker
-	pub async fn send_scrub_command(&self, cmd: ScrubWorkerCommand) {
-		let _ = self.tx_scrub_command.send(cmd).await;
+	pub async fn send_scrub_command(&self, cmd: ScrubWorkerCommand) -> Result<(), Error> {
+		let tx = self.tx_scrub_command.load();
+		let tx = tx.as_ref().ok_or_message("scrub worker is not running")?;
+		tx.send(cmd).await.ok_or_message("send error")?;
+		Ok(())
+	}
+
+	/// Get the reference count of a block
+	pub fn get_block_rc(&self, hash: &Hash) -> Result<u64, Error> {
+		Ok(self.rc.get_block_rc(hash)?.as_u64())
+	}
+
+	/// List all resync errors
+	pub fn list_resync_errors(&self) -> Result<Vec<BlockResyncErrorInfo>, Error> {
+		let mut blocks = Vec::with_capacity(self.resync.errors.len());
+		for ent in self.resync.errors.iter()? {
+			let (hash, cnt) = ent?;
+			let cnt = ErrorCounter::decode(&cnt);
+			blocks.push(BlockResyncErrorInfo {
+				hash: Hash::try_from(&hash).unwrap(),
+				refcount: 0,
+				error_count: cnt.errors,
+				last_try: cnt.last_try,
+				next_try: cnt.next_try(),
+			});
+		}
+		for block in blocks.iter_mut() {
+			block.refcount = self.get_block_rc(&block.hash)?;
+		}
+		Ok(blocks)
 	}
 
 	//// ----- Managing the reference counter ----

src/block/metrics.rs

@@ -1,9 +1,11 @@
 use opentelemetry::{global, metrics::*};
 
+use garage_db as db;
 use garage_db::counted_tree_hack::CountedTree;
 
 /// TableMetrics reference all counter used for metrics
 pub struct BlockManagerMetrics {
+	pub(crate) _rc_size: ValueObserver<u64>,
 	pub(crate) _resync_queue_len: ValueObserver<u64>,
 	pub(crate) _resync_errored_blocks: ValueObserver<u64>,
 
@@ -23,9 +25,17 @@ pub struct BlockManagerMetrics {
 }
 
 impl BlockManagerMetrics {
-	pub fn new(resync_queue: CountedTree, resync_errors: CountedTree) -> Self {
+	pub fn new(rc_tree: db::Tree, resync_queue: CountedTree, resync_errors: CountedTree) -> Self {
 		let meter = global::meter("garage_model/block");
 		Self {
+			_rc_size: meter
+				.u64_value_observer("block.rc_size", move |observer| {
+					if let Ok(Some(v)) = rc_tree.fast_len() {
+						observer.observe(v as u64, &[])
+					}
+				})
+				.with_description("Number of blocks known to the reference counter")
+				.init(),
 			_resync_queue_len: meter
 				.u64_value_observer("block.resync_queue_length", move |observer| {
 					observer.observe(resync_queue.len() as u64, &[])

src/block/rc.rs

@@ -169,4 +169,11 @@ impl RcEntry {
 	pub(crate) fn is_needed(&self) -> bool {
 		!self.is_deletable()
 	}
+
+	pub(crate) fn as_u64(&self) -> u64 {
+		match self {
+			RcEntry::Present { count } => *count,
+			_ => 0,
+		}
+	}
 }

src/block/repair.rs

@@ -53,7 +53,7 @@ impl Worker for RepairWorker {
 		"Block repair worker".into()
 	}
 
-	fn info(&self) -> Option<String> {
+	fn status(&self) -> WorkerStatus {
 		match self.block_iter.as_ref() {
 			None => {
 				let idx_bytes = self
@@ -66,9 +66,20 @@ impl Worker for RepairWorker {
 				} else {
 					idx_bytes
 				};
-				Some(format!("Phase 1: {}", hex::encode(idx_bytes)))
+				WorkerStatus {
+					progress: Some("0.00%".into()),
+					freeform: vec![format!(
+						"Currently in phase 1, iterator position: {}",
+						hex::encode(idx_bytes)
+					)],
+					..Default::default()
+				}
 			}
-			Some(bi) => Some(format!("Phase 2: {:.2}% done", bi.progress() * 100.)),
+			Some(bi) => WorkerStatus {
+				progress: Some(format!("{:.2}%", bi.progress() * 100.)),
+				freeform: vec!["Currently in phase 2".into()],
+				..Default::default()
+			},
 		}
 	}
 
@@ -137,7 +148,7 @@ impl Worker for RepairWorker {
 		}
 	}
 
-	async fn wait_for_work(&mut self, _must_exit: &watch::Receiver<bool>) -> WorkerState {
+	async fn wait_for_work(&mut self) -> WorkerState {
 		unreachable!()
 	}
 }
@@ -271,29 +282,28 @@ impl Worker for ScrubWorker {
 		"Block scrub worker".into()
 	}
 
-	fn info(&self) -> Option<String> {
-		let s = match &self.work {
-			ScrubWorkerState::Running(bsi) => format!(
-				"{:.2}% done (tranquility = {})",
-				bsi.progress() * 100.,
-				self.persisted.tranquility
-			),
-			ScrubWorkerState::Paused(bsi, rt) => {
-				format!(
-					"Paused, {:.2}% done, resumes at {}",
-					bsi.progress() * 100.,
-					msec_to_rfc3339(*rt)
-				)
-			}
-			ScrubWorkerState::Finished => format!(
-				"Last completed scrub: {}",
-				msec_to_rfc3339(self.persisted.time_last_complete_scrub)
-			),
+	fn status(&self) -> WorkerStatus {
+		let mut s = WorkerStatus {
+			persistent_errors: Some(self.persisted.corruptions_detected),
+			tranquility: Some(self.persisted.tranquility),
+			..Default::default()
 		};
-		Some(format!(
-			"{} ; corruptions detected: {}",
-			s, self.persisted.corruptions_detected
-		))
+		match &self.work {
+			ScrubWorkerState::Running(bsi) => {
+				s.progress = Some(format!("{:.2}%", bsi.progress() * 100.));
+			}
+			ScrubWorkerState::Paused(bsi, rt) => {
+				s.progress = Some(format!("{:.2}%", bsi.progress() * 100.));
+				s.freeform = vec![format!("Scrub paused, resumes at {}", msec_to_rfc3339(*rt))];
+			}
+			ScrubWorkerState::Finished => {
+				s.freeform = vec![format!(
+					"Last scrub completed at {}",
+					msec_to_rfc3339(self.persisted.time_last_complete_scrub)
+				)];
+			}
+		}
+		s
 	}
 
 	async fn work(&mut self, _must_exit: &mut watch::Receiver<bool>) -> Result<WorkerState, Error> {
@@ -331,7 +341,7 @@ impl Worker for ScrubWorker {
 		}
 	}
 
-	async fn wait_for_work(&mut self, _must_exit: &watch::Receiver<bool>) -> WorkerState {
+	async fn wait_for_work(&mut self) -> WorkerState {
 		let (wait_until, command) = match &self.work {
 			ScrubWorkerState::Running(_) => return WorkerState::Busy,
 			ScrubWorkerState::Paused(_, resume_time) => (*resume_time, ScrubWorkerCommand::Resume),

src/block/resync.rs

@@ -123,6 +123,24 @@ impl BlockResyncManager {
 		Ok(self.errors.len())
 	}
 
+	/// Clear the error counter for a block and put it in queue immediately
+	pub fn clear_backoff(&self, hash: &Hash) -> Result<(), Error> {
+		let now = now_msec();
+		if let Some(ec) = self.errors.get(hash)? {
+			let mut ec = ErrorCounter::decode(&ec);
+			if ec.errors > 0 {
+				ec.last_try = now - ec.delay_msec();
+				self.errors.insert(hash, ec.encode())?;
+				self.put_to_resync_at(hash, now)?;
+				return Ok(());
+			}
+		}
+		Err(Error::Message(format!(
+			"Block {:?} was not in an errored state",
+			hash
+		)))
+	}
+
 	// ---- Resync loop ----
 
 	// This part manages a queue of blocks that need to be
@@ -257,7 +275,7 @@ impl BlockResyncManager {
 
 				if let Err(e) = &res {
 					manager.metrics.resync_error_counter.add(1);
-					warn!("Error when resyncing {:?}: {}", hash, e);
+					error!("Error when resyncing {:?}: {}", hash, e);
 
 					let err_counter = match self.errors.get(hash.as_slice())? {
 						Some(ec) => ErrorCounter::decode(&ec).add1(now + 1),
@@ -477,27 +495,22 @@ impl Worker for ResyncWorker {
 		format!("Block resync worker #{}", self.index + 1)
 	}
 
-	fn info(&self) -> Option<String> {
+	fn status(&self) -> WorkerStatus {
 		let persisted = self.manager.resync.persisted.load();
 
 		if self.index >= persisted.n_workers {
-			return Some("(unused)".into());
+			return WorkerStatus {
+				freeform: vec!["This worker is currently disabled".into()],
+				..Default::default()
+			};
 		}
 
-		let mut ret = vec![];
-		ret.push(format!("tranquility = {}", persisted.tranquility));
-
-		let qlen = self.manager.resync.queue_len().unwrap_or(0);
-		if qlen > 0 {
-			ret.push(format!("{} blocks in queue", qlen));
+		WorkerStatus {
+			queue_length: Some(self.manager.resync.queue_len().unwrap_or(0) as u64),
+			tranquility: Some(persisted.tranquility),
+			persistent_errors: Some(self.manager.resync.errors_len().unwrap_or(0) as u64),
+			..Default::default()
 		}
-
-		let elen = self.manager.resync.errors_len().unwrap_or(0);
-		if elen > 0 {
-			ret.push(format!("{} blocks in error state", elen));
-		}
-
-		Some(ret.join(", "))
 	}
 
 	async fn work(&mut self, _must_exit: &mut watch::Receiver<bool>) -> Result<WorkerState, Error> {
@@ -527,7 +540,7 @@ impl Worker for ResyncWorker {
 		}
 	}
 
-	async fn wait_for_work(&mut self, _must_exit: &watch::Receiver<bool>) -> WorkerState {
+	async fn wait_for_work(&mut self) -> WorkerState {
 		while self.index >= self.manager.resync.persisted.load().n_workers {
 			self.manager.resync.notify.notified().await
 		}
@@ -545,9 +558,9 @@ impl Worker for ResyncWorker {
 /// and the time of the last try.
 /// Used to implement exponential backoff.
 #[derive(Clone, Copy, Debug)]
-struct ErrorCounter {
-	errors: u64,
-	last_try: u64,
+pub(crate) struct ErrorCounter {
+	pub(crate) errors: u64,
+	pub(crate) last_try: u64,
 }
 
 impl ErrorCounter {
@@ -558,12 +571,13 @@ impl ErrorCounter {
 		}
 	}
 
-	fn decode(data: &[u8]) -> Self {
+	pub(crate) fn decode(data: &[u8]) -> Self {
 		Self {
 			errors: u64::from_be_bytes(data[0..8].try_into().unwrap()),
 			last_try: u64::from_be_bytes(data[8..16].try_into().unwrap()),
 		}
 	}
+
 	fn encode(&self) -> Vec<u8> {
 		[
 			u64::to_be_bytes(self.errors),
@@ -583,7 +597,8 @@ impl ErrorCounter {
 		(RESYNC_RETRY_DELAY.as_millis() as u64)
 			<< std::cmp::min(self.errors - 1, RESYNC_RETRY_DELAY_MAX_BACKOFF_POWER)
 	}
-	fn next_try(&self) -> u64 {
+
+	pub(crate) fn next_try(&self) -> u64 {
 		self.last_try + self.delay_msec()
 	}
 }

src/db/Cargo.toml

@@ -18,6 +18,7 @@ required-features = ["cli"]
 
 [dependencies]
 err-derive = "0.3"
+hex = "0.4"
 hexdump = "0.1"
 tracing = "0.1.30"
 
@@ -25,6 +26,11 @@ heed = { version = "0.11", default-features = false, features = ["lmdb"], option
 rusqlite = { version = "0.27", optional = true }
 sled = { version = "0.34", optional = true }
 
+rmp-serde = { version = "0.15", optional = true }
+serde = { version = "1.0", default-features = false, features = ["derive", "rc"], optional = true }
+serde-transcode = { version = "1.1", optional = true }
+nettext = { version = "0.3.2", default-features = false, features = ["serde"], optional = true, path = "../../../nettext" }
+
 # cli deps
 clap = { version = "3.1.18", optional = true, features = ["derive", "env"] }
 pretty_env_logger = { version = "0.4", optional = true }
@@ -38,3 +44,4 @@ bundled-libs = [ "rusqlite/bundled" ]
 cli = ["clap", "pretty_env_logger"]
 lmdb = [ "heed" ]
 sqlite = [ "rusqlite" ]
+debuglog = [ "serde", "rmp-serde", "nettext", "serde-transcode" ]

src/db/lib.rs

@@ -173,14 +173,31 @@ impl Tree {
 		Db(self.0.clone())
 	}
 
+	#[inline]
+	pub fn name(&self) -> Option<String> {
+		self.0.tree_name(self.1)
+	}
+
 	#[inline]
 	pub fn get<T: AsRef<[u8]>>(&self, key: T) -> Result<Option<Value>> {
-		self.0.get(self.1, key.as_ref())
+		let res = self.0.get(self.1, key.as_ref())?;
+		#[cfg(feature = "debuglog")]
+		debuglog(
+			self.name(),
+			"GET",
+			key.as_ref(),
+			res.as_deref().unwrap_or(b"-"),
+		);
+		Ok(res)
 	}
 	#[inline]
 	pub fn len(&self) -> Result<usize> {
 		self.0.len(self.1)
 	}
+	#[inline]
+	pub fn fast_len(&self) -> Result<Option<usize>> {
+		self.0.fast_len(self.1)
+	}
 
 	#[inline]
 	pub fn first(&self) -> Result<Option<(Value, Value)>> {
@@ -200,7 +217,10 @@ impl Tree {
 		key: T,
 		value: U,
 	) -> Result<Option<Value>> {
-		self.0.insert(self.1, key.as_ref(), value.as_ref())
+		let res = self.0.insert(self.1, key.as_ref(), value.as_ref())?;
+		#[cfg(feature = "debuglog")]
+		debuglog(self.name(), "PUT", key.as_ref(), value.as_ref());
+		Ok(res)
 	}
 	/// Returns the old value if there was one
 	#[inline]
@@ -263,7 +283,10 @@ impl<'a> Transaction<'a> {
 		key: T,
 		value: U,
 	) -> TxOpResult<Option<Value>> {
-		self.0.insert(tree.1, key.as_ref(), value.as_ref())
+		let res = self.0.insert(tree.1, key.as_ref(), value.as_ref())?;
+		#[cfg(feature = "debuglog")]
+		debuglog(tree.name(), "txPUT", key.as_ref(), value.as_ref());
+		Ok(res)
 	}
 	/// Returns the old value if there was one
 	#[inline]
@@ -320,9 +343,13 @@ pub(crate) trait IDb: Send + Sync {
 	fn engine(&self) -> String;
 	fn open_tree(&self, name: &str) -> Result<usize>;
 	fn list_trees(&self) -> Result<Vec<String>>;
+	fn tree_name(&self, tree: usize) -> Option<String>;
 
 	fn get(&self, tree: usize, key: &[u8]) -> Result<Option<Value>>;
 	fn len(&self, tree: usize) -> Result<usize>;
+	fn fast_len(&self, _tree: usize) -> Result<Option<usize>> {
+		Ok(None)
+	}
 
 	fn insert(&self, tree: usize, key: &[u8], value: &[u8]) -> Result<Option<Value>>;
 	fn remove(&self, tree: usize, key: &[u8]) -> Result<Option<Value>>;
@@ -414,3 +441,29 @@ fn get_bound<K: AsRef<[u8]>>(b: Bound<&K>) -> Bound<&[u8]> {
 		Bound::Unbounded => Bound::Unbounded,
 	}
 }
+
+#[cfg(feature = "debuglog")]
+fn debuglog(tree: Option<String>, action: &str, k: &[u8], v: &[u8]) {
+	let key = String::from_utf8(nettext::switch64::encode(k, false)).unwrap();
+	let tree = tree.as_deref().unwrap_or("(?)");
+	if let Ok(vstr) = std::str::from_utf8(v) {
+		eprintln!("{} {} {} S:{}", tree, action, key, vstr);
+	} else {
+		let mut vread = &v[..];
+		let mut vder = rmp_serde::decode::Deserializer::new(&mut vread);
+		let mut vser = nettext::serde::Serializer {
+			string_format: nettext::BytesEncoding::Switch64 {
+				allow_whitespace: true,
+			},
+			bytes_format: nettext::BytesEncoding::Hex { split: true },
+		};
+		if let Some(venc) = serde_transcode::transcode(&mut vder, &mut vser)
+			.ok()
+			.and_then(|x| String::from_utf8(x.encode_concise()).ok())
+		{
+			eprintln!("{} {} {} N:{}", tree, action, key, venc);
+		} else {
+			eprintln!("{} {} {} X:{}", tree, action, key, hex::encode(v));
+		}
+	}
+}

src/db/lmdb_adapter.rs

@@ -121,6 +121,10 @@ impl IDb for LmdbDb {
 		Ok(tree.len(&tx)?.try_into().unwrap())
 	}
 
+	fn fast_len(&self, tree: usize) -> Result<Option<usize>> {
+		Ok(Some(self.len(tree)?))
+	}
+
 	fn insert(&self, tree: usize, key: &[u8], value: &[u8]) -> Result<Option<Value>> {
 		let tree = self.get_tree(tree)?;
 		let mut tx = self.db.write_txn()?;

src/db/sled_adapter.rs

@@ -88,6 +88,15 @@ impl IDb for SledDb {
 		Ok(trees)
 	}
 
+	fn tree_name(&self, tree: usize) -> Option<String> {
+		self.trees
+			.read()
+			.unwrap()
+			.0
+			.get(tree)
+			.and_then(|x| String::from_utf8(x.name().to_vec()).ok())
+	}
+
 	// ----
 
 	fn get(&self, tree: usize, key: &[u8]) -> Result<Option<Value>> {

src/db/sqlite_adapter.rs

@@ -144,6 +144,10 @@ impl IDb for SqliteDb {
 		}
 	}
 
+	fn fast_len(&self, tree: usize) -> Result<Option<usize>> {
+		Ok(Some(self.len(tree)?))
+	}
+
 	fn insert(&self, tree: usize, key: &[u8], value: &[u8]) -> Result<Option<Value>> {
 		trace!("insert {}: lock db", tree);
 		let this = self.0.lock().unwrap();

src/garage/Cargo.toml

@@ -36,7 +36,7 @@ bytesize = "1.1"
 timeago = "0.3"
 parse_duration = "2.1"
 hex = "0.4"
-tracing = { version = "0.1.30", features = ["log-always"] }
+tracing = { version = "0.1.30" }
 tracing-subscriber = { version = "0.3", features = ["env-filter"] }
 rand = "0.8"
 async-trait = "0.1.7"

src/garage/admin.rs

@@ -5,9 +5,11 @@ use std::sync::Arc;
 use async_trait::async_trait;
 use serde::{Deserialize, Serialize};
 
+use garage_util::background::BackgroundRunner;
 use garage_util::crdt::*;
 use garage_util::data::*;
 use garage_util::error::Error as GarageError;
+use garage_util::formater::format_table_to_string;
 use garage_util::time::*;
 
 use garage_table::replication::*;
@@ -15,6 +17,7 @@ use garage_table::*;
 
 use garage_rpc::*;
 
+use garage_block::manager::BlockResyncErrorInfo;
 use garage_block::repair::ScrubWorkerCommand;
 
 use garage_model::bucket_alias_table::*;
@@ -24,6 +27,8 @@ use garage_model::helper::error::{Error, OkOrBadRequest};
 use garage_model::key_table::*;
 use garage_model::migrate::Migrate;
 use garage_model::permission::*;
+use garage_model::s3::object_table::*;
+use garage_model::s3::version_table::Version;
 
 use crate::cli::*;
 use crate::repair::online::launch_online_repair;
@@ -38,7 +43,8 @@ pub enum AdminRpc {
 	LaunchRepair(RepairOpt),
 	Migrate(MigrateOpt),
 	Stats(StatsOpt),
-	Worker(WorkerOpt),
+	Worker(WorkerOperation),
+	BlockOperation(BlockOperation),
 
 	// Replies
 	Ok(String),
@@ -54,6 +60,13 @@ pub enum AdminRpc {
 		HashMap<usize, garage_util::background::WorkerInfo>,
 		WorkerListOpt,
 	),
+	WorkerInfo(usize, garage_util::background::WorkerInfo),
+	BlockErrorList(Vec<BlockResyncErrorInfo>),
+	BlockInfo {
+		hash: Hash,
+		refcount: u64,
+		versions: Vec<Result<Version, Uuid>>,
+	},
 }
 
 impl Rpc for AdminRpc {
@@ -62,17 +75,24 @@ impl Rpc for AdminRpc {
 
 pub struct AdminRpcHandler {
 	garage: Arc<Garage>,
+	background: Arc<BackgroundRunner>,
 	endpoint: Arc<Endpoint<AdminRpc, Self>>,
 }
 
 impl AdminRpcHandler {
-	pub fn new(garage: Arc<Garage>) -> Arc<Self> {
+	pub fn new(garage: Arc<Garage>, background: Arc<BackgroundRunner>) -> Arc<Self> {
 		let endpoint = garage.system.netapp.endpoint(ADMIN_RPC_PATH.into());
-		let admin = Arc::new(Self { garage, endpoint });
+		let admin = Arc::new(Self {
+			garage,
+			background,
+			endpoint,
+		});
 		admin.endpoint.set_handler(admin.clone());
 		admin
 	}
 
+	// ================ BUCKET COMMANDS ====================
+
 	async fn handle_bucket_cmd(&self, cmd: &BucketOperation) -> Result<AdminRpc, Error> {
 		match cmd {
 			BucketOperation::List => self.handle_list_buckets().await,
@@ -551,6 +571,8 @@ impl AdminRpcHandler {
 		Ok(AdminRpc::Ok(ret))
 	}
 
+	// ================ KEY COMMANDS ====================
+
 	async fn handle_key_cmd(&self, cmd: &KeyOperation) -> Result<AdminRpc, Error> {
 		match cmd {
 			KeyOperation::List => self.handle_list_keys().await,
@@ -688,6 +710,8 @@ impl AdminRpcHandler {
 		Ok(AdminRpc::KeyInfo(key, relevant_buckets))
 	}
 
+	// ================ MIGRATION COMMANDS ====================
+
 	async fn handle_migrate(self: &Arc<Self>, opt: MigrateOpt) -> Result<AdminRpc, Error> {
 		if !opt.yes {
 			return Err(Error::BadRequest(
@@ -704,6 +728,8 @@ impl AdminRpcHandler {
 		Ok(AdminRpc::Ok("Migration successfull.".into()))
 	}
 
+	// ================ REPAIR COMMANDS ====================
+
 	async fn handle_launch_repair(self: &Arc<Self>, opt: RepairOpt) -> Result<AdminRpc, Error> {
 		if !opt.yes {
 			return Err(Error::BadRequest(
@@ -739,7 +765,7 @@ impl AdminRpcHandler {
 				)))
 			}
 		} else {
-			launch_online_repair(self.garage.clone(), opt).await;
+			launch_online_repair(&self.garage, &self.background, opt).await?;
 			Ok(AdminRpc::Ok(format!(
 				"Repair launched on {:?}",
 				self.garage.system.id
@@ -747,6 +773,8 @@ impl AdminRpcHandler {
 		}
 	}
 
+	// ================ STATS COMMANDS ====================
+
 	async fn handle_stats(&self, opt: StatsOpt) -> Result<AdminRpc, Error> {
 		if opt.all_nodes {
 			let mut ret = String::new();
@@ -763,11 +791,12 @@ impl AdminRpcHandler {
 				match self
 					.endpoint
 					.call(&node_id, AdminRpc::Stats(opt), PRIO_NORMAL)
-					.await?
+					.await
 				{
-					Ok(AdminRpc::Ok(s)) => writeln!(&mut ret, "{}", s).unwrap(),
-					Ok(x) => writeln!(&mut ret, "Bad answer: {:?}", x).unwrap(),
-					Err(e) => writeln!(&mut ret, "Error: {}", e).unwrap(),
+					Ok(Ok(AdminRpc::Ok(s))) => writeln!(&mut ret, "{}", s).unwrap(),
+					Ok(Ok(x)) => writeln!(&mut ret, "Bad answer: {:?}", x).unwrap(),
+					Ok(Err(e)) => writeln!(&mut ret, "Remote error: {}", e).unwrap(),
+					Err(e) => writeln!(&mut ret, "Network error: {}", e).unwrap(),
 				}
 			}
 			Ok(AdminRpc::Ok(ret))
@@ -787,6 +816,7 @@ impl AdminRpcHandler {
 				.unwrap_or_else(|| "(unknown)".into()),
 		)
 		.unwrap();
+
 		writeln!(&mut ret, "\nDatabase engine: {}", self.garage.db.engine()).unwrap();
 
 		// Gather ring statistics
@@ -805,21 +835,38 @@ impl AdminRpcHandler {
 			writeln!(&mut ret, "  {:?} {}", n, c).unwrap();
 		}
 
-		self.gather_table_stats(&mut ret, &self.garage.bucket_table, &opt)?;
-		self.gather_table_stats(&mut ret, &self.garage.key_table, &opt)?;
-		self.gather_table_stats(&mut ret, &self.garage.object_table, &opt)?;
-		self.gather_table_stats(&mut ret, &self.garage.version_table, &opt)?;
-		self.gather_table_stats(&mut ret, &self.garage.block_ref_table, &opt)?;
+		// Gather table statistics
+		let mut table = vec!["  Table\tItems\tMklItems\tMklTodo\tGcTodo".into()];
+		table.push(self.gather_table_stats(&self.garage.bucket_table, opt.detailed)?);
+		table.push(self.gather_table_stats(&self.garage.key_table, opt.detailed)?);
+		table.push(self.gather_table_stats(&self.garage.object_table, opt.detailed)?);
+		table.push(self.gather_table_stats(&self.garage.version_table, opt.detailed)?);
+		table.push(self.gather_table_stats(&self.garage.block_ref_table, opt.detailed)?);
+		write!(
+			&mut ret,
+			"\nTable stats:\n{}",
+			format_table_to_string(table)
+		)
+		.unwrap();
 
+		// Gather block manager statistics
 		writeln!(&mut ret, "\nBlock manager stats:").unwrap();
-		if opt.detailed {
-			writeln!(
-				&mut ret,
-				"  number of RC entries (~= number of blocks): {}",
-				self.garage.block_manager.rc_len()?
-			)
-			.unwrap();
-		}
+		let rc_len = if opt.detailed {
+			self.garage.block_manager.rc_len()?.to_string()
+		} else {
+			self.garage
+				.block_manager
+				.rc_fast_len()?
+				.map(|x| x.to_string())
+				.unwrap_or_else(|| "NC".into())
+		};
+
+		writeln!(
+			&mut ret,
+			"  number of RC entries (~= number of blocks): {}",
+			rc_len
+		)
+		.unwrap();
 		writeln!(
 			&mut ret,
 			"  resync queue length: {}",
@@ -833,67 +880,83 @@ impl AdminRpcHandler {
 		)
 		.unwrap();
 
+		if !opt.detailed {
+			writeln!(&mut ret, "\nIf values are missing (marked as NC), consider adding the --detailed flag - this will be slow.").unwrap();
+		}
+
 		Ok(ret)
 	}
 
 	fn gather_table_stats<F, R>(
 		&self,
-		to: &mut String,
 		t: &Arc<Table<F, R>>,
-		opt: &StatsOpt,
-	) -> Result<(), Error>
+		detailed: bool,
+	) -> Result<String, Error>
 	where
 		F: TableSchema + 'static,
 		R: TableReplication + 'static,
 	{
-		writeln!(to, "\nTable stats for {}", F::TABLE_NAME).unwrap();
-		if opt.detailed {
-			writeln!(
-				to,
-				"  number of items: {}",
-				t.data.store.len().map_err(GarageError::from)?
+		let (data_len, mkl_len) = if detailed {
+			(
+				t.data.store.len().map_err(GarageError::from)?.to_string(),
+				t.merkle_updater.merkle_tree_len()?.to_string(),
 			)
-			.unwrap();
-			writeln!(
-				to,
-				"  Merkle tree size: {}",
-				t.merkle_updater.merkle_tree_len()?
+		} else {
+			(
+				t.data
+					.store
+					.fast_len()
+					.map_err(GarageError::from)?
+					.map(|x| x.to_string())
+					.unwrap_or_else(|| "NC".into()),
+				t.merkle_updater
+					.merkle_tree_fast_len()?
+					.map(|x| x.to_string())
+					.unwrap_or_else(|| "NC".into()),
 			)
-			.unwrap();
-		}
-		writeln!(
-			to,
-			"  Merkle updater todo queue length: {}",
-			t.merkle_updater.todo_len()?
-		)
-		.unwrap();
-		writeln!(to, "  GC todo queue length: {}", t.data.gc_todo_len()?).unwrap();
+		};
 
-		Ok(())
+		Ok(format!(
+			"  {}\t{}\t{}\t{}\t{}",
+			F::TABLE_NAME,
+			data_len,
+			mkl_len,
+			t.merkle_updater.todo_len()?,
+			t.data.gc_todo_len()?
+		))
 	}
 
-	// ----
+	// ================ WORKER COMMANDS ====================
 
-	async fn handle_worker_cmd(&self, opt: WorkerOpt) -> Result<AdminRpc, Error> {
-		match opt.cmd {
-			WorkerCmd::List { opt } => {
-				let workers = self.garage.background.get_worker_info();
-				Ok(AdminRpc::WorkerList(workers, opt))
+	async fn handle_worker_cmd(&self, cmd: &WorkerOperation) -> Result<AdminRpc, Error> {
+		match cmd {
+			WorkerOperation::List { opt } => {
+				let workers = self.background.get_worker_info();
+				Ok(AdminRpc::WorkerList(workers, *opt))
 			}
-			WorkerCmd::Set { opt } => match opt {
+			WorkerOperation::Info { tid } => {
+				let info = self
+					.background
+					.get_worker_info()
+					.get(tid)
+					.ok_or_bad_request(format!("No worker with TID {}", tid))?
+					.clone();
+				Ok(AdminRpc::WorkerInfo(*tid, info))
+			}
+			WorkerOperation::Set { opt } => match opt {
 				WorkerSetCmd::ScrubTranquility { tranquility } => {
-					let scrub_command = ScrubWorkerCommand::SetTranquility(tranquility);
+					let scrub_command = ScrubWorkerCommand::SetTranquility(*tranquility);
 					self.garage
 						.block_manager
 						.send_scrub_command(scrub_command)
-						.await;
+						.await?;
 					Ok(AdminRpc::Ok("Scrub tranquility updated".into()))
 				}
-				WorkerSetCmd::ResyncNWorkers { n_workers } => {
+				WorkerSetCmd::ResyncWorkerCount { worker_count } => {
 					self.garage
 						.block_manager
 						.resync
-						.set_n_workers(n_workers)
+						.set_n_workers(*worker_count)
 						.await?;
 					Ok(AdminRpc::Ok("Number of resync workers updated".into()))
 				}
@@ -901,13 +964,154 @@ impl AdminRpcHandler {
 					self.garage
 						.block_manager
 						.resync
-						.set_tranquility(tranquility)
+						.set_tranquility(*tranquility)
 						.await?;
 					Ok(AdminRpc::Ok("Resync tranquility updated".into()))
 				}
 			},
 		}
 	}
+
+	// ================ BLOCK COMMANDS ====================
+
+	async fn handle_block_cmd(&self, cmd: &BlockOperation) -> Result<AdminRpc, Error> {
+		match cmd {
+			BlockOperation::ListErrors => Ok(AdminRpc::BlockErrorList(
+				self.garage.block_manager.list_resync_errors()?,
+			)),
+			BlockOperation::Info { hash } => {
+				let hash = hex::decode(hash).ok_or_bad_request("invalid hash")?;
+				let hash = Hash::try_from(&hash).ok_or_bad_request("invalid hash")?;
+				let refcount = self.garage.block_manager.get_block_rc(&hash)?;
+				let block_refs = self
+					.garage
+					.block_ref_table
+					.get_range(&hash, None, None, 10000, Default::default())
+					.await?;
+				let mut versions = vec![];
+				for br in block_refs {
+					if let Some(v) = self
+						.garage
+						.version_table
+						.get(&br.version, &EmptyKey)
+						.await?
+					{
+						versions.push(Ok(v));
+					} else {
+						versions.push(Err(br.version));
+					}
+				}
+				Ok(AdminRpc::BlockInfo {
+					hash,
+					refcount,
+					versions,
+				})
+			}
+			BlockOperation::RetryNow { all, blocks } => {
+				if *all {
+					if !blocks.is_empty() {
+						return Err(Error::BadRequest(
+							"--all was specified, cannot also specify blocks".into(),
+						));
+					}
+					let blocks = self.garage.block_manager.list_resync_errors()?;
+					for b in blocks.iter() {
+						self.garage.block_manager.resync.clear_backoff(&b.hash)?;
+					}
+					Ok(AdminRpc::Ok(format!(
+						"{} blocks returned in queue for a retry now (check logs to see results)",
+						blocks.len()
+					)))
+				} else {
+					for hash in blocks {
+						let hash = hex::decode(hash).ok_or_bad_request("invalid hash")?;
+						let hash = Hash::try_from(&hash).ok_or_bad_request("invalid hash")?;
+						self.garage.block_manager.resync.clear_backoff(&hash)?;
+					}
+					Ok(AdminRpc::Ok(format!(
+						"{} blocks returned in queue for a retry now (check logs to see results)",
+						blocks.len()
+					)))
+				}
+			}
+			BlockOperation::Purge { yes, blocks } => {
+				if !yes {
+					return Err(Error::BadRequest(
+						"Pass the --yes flag to confirm block purge operation.".into(),
+					));
+				}
+
+				let mut obj_dels = 0;
+				let mut ver_dels = 0;
+
+				for hash in blocks {
+					let hash = hex::decode(hash).ok_or_bad_request("invalid hash")?;
+					let hash = Hash::try_from(&hash).ok_or_bad_request("invalid hash")?;
+					let block_refs = self
+						.garage
+						.block_ref_table
+						.get_range(&hash, None, None, 10000, Default::default())
+						.await?;
+
+					for br in block_refs {
+						let version = match self
+							.garage
+							.version_table
+							.get(&br.version, &EmptyKey)
+							.await?
+						{
+							Some(v) => v,
+							None => continue,
+						};
+
+						if let Some(object) = self
+							.garage
+							.object_table
+							.get(&version.bucket_id, &version.key)
+							.await?
+						{
+							let ov = object.versions().iter().rev().find(|v| v.is_complete());
+							if let Some(ov) = ov {
+								if ov.uuid == br.version {
+									let del_uuid = gen_uuid();
+									let deleted_object = Object::new(
+										version.bucket_id,
+										version.key.clone(),
+										vec![ObjectVersion {
+											uuid: del_uuid,
+											timestamp: ov.timestamp + 1,
+											state: ObjectVersionState::Complete(
+												ObjectVersionData::DeleteMarker,
+											),
+										}],
+									);
+									self.garage.object_table.insert(&deleted_object).await?;
+									obj_dels += 1;
+								}
+							}
+						}
+
+						if !version.deleted.get() {
+							let deleted_version = Version::new(
+								version.uuid,
+								version.bucket_id,
+								version.key.clone(),
+								true,
+							);
+							self.garage.version_table.insert(&deleted_version).await?;
+							ver_dels += 1;
+						}
+					}
+				}
+				Ok(AdminRpc::Ok(format!(
+					"{} blocks were purged: {} object deletion markers added, {} versions marked deleted",
+					blocks.len(),
+					obj_dels,
+					ver_dels
+				)))
+			}
+		}
+	}
 }
 
 #[async_trait]
@@ -923,7 +1127,8 @@ impl EndpointHandler<AdminRpc> for AdminRpcHandler {
 			AdminRpc::Migrate(opt) => self.handle_migrate(opt.clone()).await,
 			AdminRpc::LaunchRepair(opt) => self.handle_launch_repair(opt.clone()).await,
 			AdminRpc::Stats(opt) => self.handle_stats(opt.clone()).await,
-			AdminRpc::Worker(opt) => self.handle_worker_cmd(opt.clone()).await,
+			AdminRpc::Worker(wo) => self.handle_worker_cmd(wo).await,
+			AdminRpc::BlockOperation(bo) => self.handle_block_cmd(bo).await,
 			m => Err(GarageError::unexpected_rpc_message(m).into()),
 		}
 	}

src/garage/cli/cmd.rs

@@ -41,6 +41,9 @@ pub async fn cli_command_dispatch(
 		}
 		Command::Stats(so) => cmd_admin(admin_rpc_endpoint, rpc_host, AdminRpc::Stats(so)).await,
 		Command::Worker(wo) => cmd_admin(admin_rpc_endpoint, rpc_host, AdminRpc::Worker(wo)).await,
+		Command::Block(bo) => {
+			cmd_admin(admin_rpc_endpoint, rpc_host, AdminRpc::BlockOperation(bo)).await
+		}
 		_ => unreachable!(),
 	}
 }
@@ -186,7 +189,20 @@ pub async fn cmd_admin(
 			print_key_info(&key, &rb);
 		}
 		AdminRpc::WorkerList(wi, wlo) => {
-			print_worker_info(wi, wlo);
+			print_worker_list(wi, wlo);
+		}
+		AdminRpc::WorkerInfo(tid, wi) => {
+			print_worker_info(tid, wi);
+		}
+		AdminRpc::BlockErrorList(el) => {
+			print_block_error_list(el);
+		}
+		AdminRpc::BlockInfo {
+			hash,
+			refcount,
+			versions,
+		} => {
+			print_block_info(hash, refcount, versions);
 		}
 		r => {
 			error!("Unexpected response: {:?}", r);

src/garage/cli/structs.rs

@@ -49,7 +49,11 @@ pub enum Command {
 
 	/// Manage background workers
 	#[structopt(name = "worker", version = garage_version())]
-	Worker(WorkerOpt),
+	Worker(WorkerOperation),
+
+	/// Low-level debug operations on data blocks
+	#[structopt(name = "block", version = garage_version())]
+	Block(BlockOperation),
 }
 
 #[derive(StructOpt, Debug)]
@@ -502,20 +506,17 @@ pub struct StatsOpt {
 	pub detailed: bool,
 }
 
-#[derive(Serialize, Deserialize, StructOpt, Debug, Clone)]
-pub struct WorkerOpt {
-	#[structopt(subcommand)]
-	pub cmd: WorkerCmd,
-}
-
 #[derive(Serialize, Deserialize, StructOpt, Debug, Eq, PartialEq, Clone)]
-pub enum WorkerCmd {
+pub enum WorkerOperation {
 	/// List all workers on Garage node
 	#[structopt(name = "list", version = garage_version())]
 	List {
 		#[structopt(flatten)]
 		opt: WorkerListOpt,
 	},
+	/// Get detailed information about a worker
+	#[structopt(name = "info", version = garage_version())]
+	Info { tid: usize },
 	/// Set worker parameter
 	#[structopt(name = "set", version = garage_version())]
 	Set {
@@ -540,9 +541,41 @@ pub enum WorkerSetCmd {
 	#[structopt(name = "scrub-tranquility", version = garage_version())]
 	ScrubTranquility { tranquility: u32 },
 	/// Set number of concurrent block resync workers
-	#[structopt(name = "resync-n-workers", version = garage_version())]
-	ResyncNWorkers { n_workers: usize },
+	#[structopt(name = "resync-worker-count", version = garage_version())]
+	ResyncWorkerCount { worker_count: usize },
 	/// Set tranquility of block resync operations
 	#[structopt(name = "resync-tranquility", version = garage_version())]
 	ResyncTranquility { tranquility: u32 },
 }
+
+#[derive(Serialize, Deserialize, StructOpt, Debug, Eq, PartialEq, Clone)]
+pub enum BlockOperation {
+	/// List all blocks that currently have a resync error
+	#[structopt(name = "list-errors", version = garage_version())]
+	ListErrors,
+	/// Get detailed information about a single block
+	#[structopt(name = "info", version = garage_version())]
+	Info {
+		/// Hash of the block for which to retrieve information
+		hash: String,
+	},
+	/// Retry now the resync of one or many blocks
+	#[structopt(name = "retry-now", version = garage_version())]
+	RetryNow {
+		/// Retry all blocks that have a resync error
+		#[structopt(long = "all")]
+		all: bool,
+		/// Hashes of the block to retry to resync now
+		blocks: Vec<String>,
+	},
+	/// Delete all objects referencing a missing block
+	#[structopt(name = "purge", version = garage_version())]
+	Purge {
+		/// Mandatory to confirm this operation
+		#[structopt(long = "yes")]
+		yes: bool,
+		/// Hashes of the block to purge
+		#[structopt(required = true)]
+		blocks: Vec<String>,
+	},
+}

src/garage/cli/util.rs

@@ -3,14 +3,17 @@ use std::time::Duration;
 
 use garage_util::background::*;
 use garage_util::crdt::*;
-use garage_util::data::Uuid;
+use garage_util::data::*;
 use garage_util::error::*;
 use garage_util::formater::format_table;
 use garage_util::time::*;
 
+use garage_block::manager::BlockResyncErrorInfo;
+
 use garage_model::bucket_table::*;
 use garage_model::key_table::*;
 use garage_model::s3::object_table::{BYTES, OBJECTS, UNFINISHED_UPLOADS};
+use garage_model::s3::version_table::Version;
 
 use crate::cli::structs::WorkerListOpt;
 
@@ -241,7 +244,7 @@ pub fn find_matching_node(
 	}
 }
 
-pub fn print_worker_info(wi: HashMap<usize, WorkerInfo>, wlo: WorkerListOpt) {
+pub fn print_worker_list(wi: HashMap<usize, WorkerInfo>, wlo: WorkerListOpt) {
 	let mut wi = wi.into_iter().collect::<Vec<_>>();
 	wi.sort_by_key(|(tid, info)| {
 		(
@@ -254,7 +257,7 @@ pub fn print_worker_info(wi: HashMap<usize, WorkerInfo>, wlo: WorkerListOpt) {
 		)
 	});
 
-	let mut table = vec![];
+	let mut table = vec!["TID\tState\tName\tTranq\tDone\tQueue\tErrors\tConsec\tLast".to_string()];
 	for (tid, info) in wi.iter() {
 		if wlo.busy && !matches!(info.state, WorkerState::Busy | WorkerState::Throttled(_)) {
 			continue;
@@ -263,33 +266,147 @@ pub fn print_worker_info(wi: HashMap<usize, WorkerInfo>, wlo: WorkerListOpt) {
 			continue;
 		}
 
-		table.push(format!("{}\t{}\t{}", tid, info.state, info.name));
-		if let Some(i) = &info.info {
-			table.push(format!("\t\t  {}", i));
-		}
 		let tf = timeago::Formatter::new();
-		let (err_ago, err_msg) = info
+		let err_ago = info
 			.last_error
 			.as_ref()
-			.map(|(m, t)| {
-				(
-					tf.convert(Duration::from_millis(now_msec() - t)),
-					m.as_str(),
-				)
-			})
-			.unwrap_or(("(?) ago".into(), "(?)"));
-		if info.consecutive_errors > 0 {
+			.map(|(_, t)| tf.convert(Duration::from_millis(now_msec() - t)))
+			.unwrap_or_default();
+		let (total_err, consec_err) = if info.errors > 0 {
+			(info.errors.to_string(), info.consecutive_errors.to_string())
+		} else {
+			("-".into(), "-".into())
+		};
+
+		table.push(format!(
+			"{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}",
+			tid,
+			info.state,
+			info.name,
+			info.status
+				.tranquility
+				.as_ref()
+				.map(ToString::to_string)
+				.unwrap_or_else(|| "-".into()),
+			info.status.progress.as_deref().unwrap_or("-"),
+			info.status
+				.queue_length
+				.as_ref()
+				.map(ToString::to_string)
+				.unwrap_or_else(|| "-".into()),
+			total_err,
+			consec_err,
+			err_ago,
+		));
+	}
+	format_table(table);
+}
+
+pub fn print_worker_info(tid: usize, info: WorkerInfo) {
+	let mut table = vec![];
+	table.push(format!("Task id:\t{}", tid));
+	table.push(format!("Worker name:\t{}", info.name));
+	match info.state {
+		WorkerState::Throttled(t) => {
 			table.push(format!(
-				"\t\t  {} consecutive errors ({} total), last {}",
-				info.consecutive_errors, info.errors, err_ago,
+				"Worker state:\tBusy (throttled, paused for {:.3}s)",
+				t
 			));
-			table.push(format!("\t\t  {}", err_msg));
-		} else if info.errors > 0 {
-			table.push(format!("\t\t  ({} errors, last {})", info.errors, err_ago,));
-			if wlo.errors {
-				table.push(format!("\t\t  {}", err_msg));
+		}
+		s => {
+			table.push(format!("Worker state:\t{}", s));
+		}
+	};
+	if let Some(tql) = info.status.tranquility {
+		table.push(format!("Tranquility:\t{}", tql));
+	}
+
+	table.push("".into());
+	table.push(format!("Total errors:\t{}", info.errors));
+	table.push(format!("Consecutive errs:\t{}", info.consecutive_errors));
+	if let Some((s, t)) = info.last_error {
+		table.push(format!("Last error:\t{}", s));
+		let tf = timeago::Formatter::new();
+		table.push(format!(
+			"Last error time:\t{}",
+			tf.convert(Duration::from_millis(now_msec() - t))
+		));
+	}
+
+	table.push("".into());
+	if let Some(p) = info.status.progress {
+		table.push(format!("Progress:\t{}", p));
+	}
+	if let Some(ql) = info.status.queue_length {
+		table.push(format!("Queue length:\t{}", ql));
+	}
+	if let Some(pe) = info.status.persistent_errors {
+		table.push(format!("Persistent errors:\t{}", pe));
+	}
+
+	for (i, s) in info.status.freeform.iter().enumerate() {
+		if i == 0 {
+			if table.last() != Some(&"".into()) {
+				table.push("".into());
 			}
+			table.push(format!("Message:\t{}", s));
+		} else {
+			table.push(format!("\t{}", s));
 		}
 	}
 	format_table(table);
 }
+
+pub fn print_block_error_list(el: Vec<BlockResyncErrorInfo>) {
+	let now = now_msec();
+	let tf = timeago::Formatter::new();
+	let mut tf2 = timeago::Formatter::new();
+	tf2.ago("");
+
+	let mut table = vec!["Hash\tRC\tErrors\tLast error\tNext try".into()];
+	for e in el {
+		table.push(format!(
+			"{}\t{}\t{}\t{}\tin {}",
+			hex::encode(e.hash.as_slice()),
+			e.refcount,
+			e.error_count,
+			tf.convert(Duration::from_millis(now - e.last_try)),
+			tf2.convert(Duration::from_millis(e.next_try - now))
+		));
+	}
+	format_table(table);
+}
+
+pub fn print_block_info(hash: Hash, refcount: u64, versions: Vec<Result<Version, Uuid>>) {
+	println!("Block hash: {}", hex::encode(hash.as_slice()));
+	println!("Refcount: {}", refcount);
+	println!();
+
+	let mut table = vec!["Version\tBucket\tKey\tDeleted".into()];
+	let mut nondeleted_count = 0;
+	for v in versions.iter() {
+		match v {
+			Ok(ver) => {
+				table.push(format!(
+					"{:?}\t{:?}\t{}\t{:?}",
+					ver.uuid,
+					ver.bucket_id,
+					ver.key,
+					ver.deleted.get()
+				));
+				if !ver.deleted.get() {
+					nondeleted_count += 1;
+				}
+			}
+			Err(vh) => {
+				table.push(format!("{:?}\t\t\tyes", vh));
+			}
+		}
+	}
+	format_table(table);
+
+	if refcount != nondeleted_count {
+		println!();
+		println!("Warning: refcount does not match number of non-deleted versions");
+	}
+}

src/garage/main.rs

@@ -127,9 +127,16 @@ async fn main() {
 		std::process::abort();
 	}));
 
+	// Parse arguments and dispatch command line
+	let opt = Opt::from_clap(&Opt::clap().version(version.as_str()).get_matches());
+
 	// Initialize logging as well as other libraries used in Garage
 	if std::env::var("RUST_LOG").is_err() {
-		std::env::set_var("RUST_LOG", "netapp=info,garage=info")
+		let default_log = match &opt.cmd {
+			Command::Server => "netapp=info,garage=info",
+			_ => "netapp=warn,garage=warn",
+		};
+		std::env::set_var("RUST_LOG", default_log)
 	}
 	tracing_subscriber::fmt()
 		.with_writer(std::io::stderr)
@@ -137,9 +144,6 @@ async fn main() {
 		.init();
 	sodiumoxide::init().expect("Unable to init sodiumoxide");
 
-	// Parse arguments and dispatch command line
-	let opt = Opt::from_clap(&Opt::clap().version(version.as_str()).get_matches());
-
 	let res = match opt.cmd {
 		Command::Server => server::run_server(opt.config_file).await,
 		Command::OfflineRepair(repair_opt) => {
@@ -182,9 +186,9 @@ async fn cli_command(opt: Opt) -> Result<(), Error> {
 	let netapp = NetApp::new(GARAGE_VERSION_TAG, network_key, sk);
 
 	// Find and parse the address of the target host
-	let (id, addr) = if let Some(h) = opt.rpc_host {
+	let (id, addr, is_default_addr) = if let Some(h) = opt.rpc_host {
 		let (id, addrs) = parse_and_resolve_peer_addr(&h).ok_or_else(|| format!("Invalid RPC remote node identifier: {}. Expected format is <pubkey>@<IP or hostname>:<port>.", h))?;
-		(id, addrs[0])
+		(id, addrs[0], false)
 	} else {
 		let node_id = garage_rpc::system::read_node_id(&config.as_ref().unwrap().metadata_dir)
 			.err_context(READ_KEY_ERROR)?;
@@ -195,24 +199,26 @@ async fn cli_command(opt: Opt) -> Result<(), Error> {
 				.ok_or_message("unable to resolve rpc_public_addr specified in config file")?
 				.next()
 				.ok_or_message("unable to resolve rpc_public_addr specified in config file")?;
-			(node_id, a)
+			(node_id, a, false)
 		} else {
 			let default_addr = SocketAddr::new(
 				"127.0.0.1".parse().unwrap(),
 				config.as_ref().unwrap().rpc_bind_addr.port(),
 			);
-			warn!(
-				"Trying to contact Garage node at default address {}",
-				default_addr
-			);
-			warn!("If this doesn't work, consider adding rpc_public_addr in your config file or specifying the -h command line parameter.");
-			(node_id, default_addr)
+			(node_id, default_addr, true)
 		}
 	};
 
 	// Connect to target host
-	netapp.clone().try_connect(addr, id).await
-		.err_context("Unable to connect to destination RPC host. Check that you are using the same value of rpc_secret as them, and that you have their correct public key.")?;
+	if let Err(e) = netapp.clone().try_connect(addr, id).await {
+		if is_default_addr {
+			warn!(
+				"Tried to contact Garage node at default address {}, which didn't work. If that address is wrong, consider setting rpc_public_addr in your config file.",
+				addr
+			);
+		}
+		Err(e).err_context("Unable to connect to destination RPC host. Check that you are using the same value of rpc_secret as them, and that you have their correct public key.")?;
+	}
 
 	let system_rpc_endpoint = netapp.endpoint::<SystemRpc, ()>(SYSTEM_RPC_PATH.into());
 	let admin_rpc_endpoint = netapp.endpoint::<AdminRpc, ()>(ADMIN_RPC_PATH.into());

src/garage/repair/offline.rs

@@ -1,8 +1,5 @@
 use std::path::PathBuf;
 
-use tokio::sync::watch;
-
-use garage_util::background::*;
 use garage_util::config::*;
 use garage_util::error::*;
 
@@ -20,12 +17,8 @@ pub async fn offline_repair(config_file: PathBuf, opt: OfflineRepairOpt) -> Resu
 	info!("Loading configuration...");
 	let config = read_config(config_file)?;
 
-	info!("Initializing background runner...");
-	let (done_tx, done_rx) = watch::channel(false);
-	let (background, await_background_done) = BackgroundRunner::new(16, done_rx);
-
 	info!("Initializing Garage main data store...");
-	let garage = Garage::new(config.clone(), background)?;
+	let garage = Garage::new(config)?;
 
 	info!("Launching repair operation...");
 	match opt.what {
@@ -43,13 +36,7 @@ pub async fn offline_repair(config_file: PathBuf, opt: OfflineRepairOpt) -> Resu
 		}
 	}
 
-	info!("Repair operation finished, shutting down Garage internals...");
-	done_tx.send(true).unwrap();
-	drop(garage);
-
-	await_background_done.await?;
-
-	info!("Cleaning up...");
+	info!("Repair operation finished, shutting down...");
 
 	Ok(())
 }

src/garage/repair/online.rs

@@ -15,35 +15,33 @@ use garage_util::error::Error;
 
 use crate::*;
 
-pub async fn launch_online_repair(garage: Arc<Garage>, opt: RepairOpt) {
+pub async fn launch_online_repair(
+	garage: &Arc<Garage>,
+	bg: &BackgroundRunner,
+	opt: RepairOpt,
+) -> Result<(), Error> {
 	match opt.what {
 		RepairWhat::Tables => {
 			info!("Launching a full sync of tables");
-			garage.bucket_table.syncer.add_full_sync();
-			garage.object_table.syncer.add_full_sync();
-			garage.version_table.syncer.add_full_sync();
-			garage.block_ref_table.syncer.add_full_sync();
-			garage.key_table.syncer.add_full_sync();
+			garage.bucket_table.syncer.add_full_sync()?;
+			garage.object_table.syncer.add_full_sync()?;
+			garage.version_table.syncer.add_full_sync()?;
+			garage.block_ref_table.syncer.add_full_sync()?;
+			garage.key_table.syncer.add_full_sync()?;
 		}
 		RepairWhat::Versions => {
 			info!("Repairing the versions table");
-			garage
-				.background
-				.spawn_worker(RepairVersionsWorker::new(garage.clone()));
+			bg.spawn_worker(RepairVersionsWorker::new(garage.clone()));
 		}
 		RepairWhat::BlockRefs => {
 			info!("Repairing the block refs table");
-			garage
-				.background
-				.spawn_worker(RepairBlockrefsWorker::new(garage.clone()));
+			bg.spawn_worker(RepairBlockrefsWorker::new(garage.clone()));
 		}
 		RepairWhat::Blocks => {
 			info!("Repairing the stored blocks");
-			garage
-				.background
-				.spawn_worker(garage_block::repair::RepairWorker::new(
-					garage.block_manager.clone(),
-				));
+			bg.spawn_worker(garage_block::repair::RepairWorker::new(
+				garage.block_manager.clone(),
+			));
 		}
 		RepairWhat::Scrub { cmd } => {
 			let cmd = match cmd {
@@ -56,9 +54,10 @@ pub async fn launch_online_repair(garage: Arc<Garage>, opt: RepairOpt) {
 				}
 			};
 			info!("Sending command to scrub worker: {:?}", cmd);
-			garage.block_manager.send_scrub_command(cmd).await;
+			garage.block_manager.send_scrub_command(cmd).await?;
 		}
 	}
+	Ok(())
 }
 
 // ----
@@ -85,24 +84,22 @@ impl Worker for RepairVersionsWorker {
 		"Version repair worker".into()
 	}
 
-	fn info(&self) -> Option<String> {
-		Some(format!("{} items done", self.counter))
+	fn status(&self) -> WorkerStatus {
+		WorkerStatus {
+			progress: Some(self.counter.to_string()),
+			..Default::default()
+		}
 	}
 
 	async fn work(&mut self, _must_exit: &mut watch::Receiver<bool>) -> Result<WorkerState, Error> {
-		let item_bytes = match self.garage.version_table.data.store.get_gt(&self.pos)? {
-			Some((k, v)) => {
-				self.pos = k;
-				v
-			}
+		let (item_bytes, next_pos) = match self.garage.version_table.data.store.get_gt(&self.pos)? {
+			Some((k, v)) => (v, k),
 			None => {
 				info!("repair_versions: finished, done {}", self.counter);
 				return Ok(WorkerState::Done);
 			}
 		};
 
-		self.counter += 1;
-
 		let version = rmp_serde::decode::from_read_ref::<_, Version>(&item_bytes)?;
 		if !version.deleted.get() {
 			let object = self
@@ -131,10 +128,13 @@ impl Worker for RepairVersionsWorker {
 			}
 		}
 
+		self.counter += 1;
+		self.pos = next_pos;
+
 		Ok(WorkerState::Busy)
 	}
 
-	async fn wait_for_work(&mut self, _must_exit: &watch::Receiver<bool>) -> WorkerState {
+	async fn wait_for_work(&mut self) -> WorkerState {
 		unreachable!()
 	}
 }
@@ -163,23 +163,22 @@ impl Worker for RepairBlockrefsWorker {
 		"Block refs repair worker".into()
 	}
 
-	fn info(&self) -> Option<String> {
-		Some(format!("{} items done", self.counter))
+	fn status(&self) -> WorkerStatus {
+		WorkerStatus {
+			progress: Some(self.counter.to_string()),
+			..Default::default()
+		}
 	}
 
 	async fn work(&mut self, _must_exit: &mut watch::Receiver<bool>) -> Result<WorkerState, Error> {
-		let item_bytes = match self.garage.block_ref_table.data.store.get_gt(&self.pos)? {
-			Some((k, v)) => {
-				self.pos = k;
-				v
-			}
-			None => {
-				info!("repair_block_ref: finished, done {}", self.counter);
-				return Ok(WorkerState::Done);
-			}
-		};
-
-		self.counter += 1;
+		let (item_bytes, next_pos) =
+			match self.garage.block_ref_table.data.store.get_gt(&self.pos)? {
+				Some((k, v)) => (v, k),
+				None => {
+					info!("repair_block_ref: finished, done {}", self.counter);
+					return Ok(WorkerState::Done);
+				}
+			};
 
 		let block_ref = rmp_serde::decode::from_read_ref::<_, BlockRef>(&item_bytes)?;
 		if !block_ref.deleted.get() {
@@ -206,10 +205,13 @@ impl Worker for RepairBlockrefsWorker {
 			}
 		}
 
+		self.counter += 1;
+		self.pos = next_pos;
+
 		Ok(WorkerState::Busy)
 	}
 
-	async fn wait_for_work(&mut self, _must_exit: &watch::Receiver<bool>) -> WorkerState {
+	async fn wait_for_work(&mut self) -> WorkerState {
 		unreachable!()
 	}
 }

src/garage/server.rs

@@ -35,12 +35,15 @@ pub async fn run_server(config_file: PathBuf) -> Result<(), Error> {
 	#[cfg(feature = "metrics")]
 	let metrics_exporter = opentelemetry_prometheus::exporter().init();
 
+	info!("Initializing Garage main data store...");
+	let garage = Garage::new(config.clone())?;
+
 	info!("Initializing background runner...");
 	let watch_cancel = watch_shutdown_signal();
-	let (background, await_background_done) = BackgroundRunner::new(16, watch_cancel.clone());
+	let (background, await_background_done) = BackgroundRunner::new(watch_cancel.clone());
 
-	info!("Initializing Garage main data store...");
-	let garage = Garage::new(config.clone(), background)?;
+	info!("Spawning Garage workers...");
+	garage.spawn_workers(&background);
 
 	if config.admin.trace_sink.is_some() {
 		info!("Initialize tracing...");
@@ -63,7 +66,7 @@ pub async fn run_server(config_file: PathBuf) -> Result<(), Error> {
 	let run_system = tokio::spawn(garage.system.clone().run(watch_cancel.clone()));
 
 	info!("Create admin RPC handler...");
-	AdminRpcHandler::new(garage.clone());
+	AdminRpcHandler::new(garage.clone(), background.clone());
 
 	// ---- Launch public-facing API servers ----
 

src/garage/tests/bucket.rs

@@ -1,4 +1,5 @@
 use crate::common;
+use crate::common::ext::CommandExt;
 use aws_sdk_s3::model::BucketLocationConstraint;
 use aws_sdk_s3::output::DeleteBucketOutput;
 
@@ -8,6 +9,27 @@ async fn test_bucket_all() {
 	let bucket_name = "hello";
 
 	{
+		// Check bucket cannot be created if not authorized
+		ctx.garage
+			.command()
+			.args(["key", "deny"])
+			.args(["--create-bucket", &ctx.garage.key.id])
+			.quiet()
+			.expect_success_output("Could not deny key to create buckets");
+
+		// Try create bucket, should fail
+		let r = ctx.client.create_bucket().bucket(bucket_name).send().await;
+		assert!(r.is_err());
+	}
+	{
+		// Now allow key to create bucket
+		ctx.garage
+			.command()
+			.args(["key", "allow"])
+			.args(["--create-bucket", &ctx.garage.key.id])
+			.quiet()
+			.expect_success_output("Could not deny key to create buckets");
+
 		// Create bucket
 		//@TODO check with an invalid bucket name + with an already existing bucket
 		let r = ctx

src/garage/tests/s3/streaming_signature.rs

@@ -1,6 +1,7 @@
 use std::collections::HashMap;
 
 use crate::common;
+use crate::common::ext::CommandExt;
 use common::custom_requester::BodySignature;
 use hyper::Method;
 
@@ -105,6 +106,13 @@ async fn test_create_bucket_streaming() {
 	let ctx = common::context();
 	let bucket = "createbucket-streaming";
 
+	ctx.garage
+		.command()
+		.args(["key", "allow"])
+		.args(["--create-bucket", &ctx.garage.key.id])
+		.quiet()
+		.expect_success_output("Could not allow key to create buckets");
+
 	{
 		// create bucket
 		let _ = ctx

src/model/garage.rs

@@ -8,10 +8,10 @@ use garage_util::background::*;
 use garage_util::config::*;
 use garage_util::error::*;
 
+use garage_rpc::replication_mode::ReplicationMode;
 use garage_rpc::system::System;
 
 use garage_block::manager::*;
-use garage_table::replication::ReplicationMode;
 use garage_table::replication::TableFullReplication;
 use garage_table::replication::TableShardedReplication;
 use garage_table::*;
@@ -34,10 +34,11 @@ pub struct Garage {
 	/// The parsed configuration Garage is running
 	pub config: Config,
 
+	/// The replication mode of this cluster
+	pub replication_mode: ReplicationMode,
+
 	/// The local database
 	pub db: db::Db,
-	/// A background job runner
-	pub background: Arc<BackgroundRunner>,
 	/// The membership manager
 	pub system: Arc<System>,
 	/// The block manager
@@ -75,7 +76,7 @@ pub struct GarageK2V {
 
 impl Garage {
 	/// Create and run garage
-	pub fn new(config: Config, background: Arc<BackgroundRunner>) -> Result<Arc<Self>, Error> {
+	pub fn new(config: Config) -> Result<Arc<Self>, Error> {
 		// Create meta dir and data dir if they don't exist already
 		std::fs::create_dir_all(&config.metadata_dir)
 			.ok_or_message("Unable to create Garage metadata directory")?;
@@ -164,12 +165,7 @@ impl Garage {
 			.expect("Invalid replication_mode in config file.");
 
 		info!("Initialize membership management system...");
-		let system = System::new(
-			network_key,
-			background.clone(),
-			replication_mode.replication_factor(),
-			&config,
-		)?;
+		let system = System::new(network_key, replication_mode, &config)?;
 
 		let data_rep_param = TableShardedReplication {
 			system: system.clone(),
@@ -227,7 +223,6 @@ impl Garage {
 		info!("Initialize version_table...");
 		let version_table = Table::new(
 			VersionTable {
-				background: background.clone(),
 				block_ref_table: block_ref_table.clone(),
 			},
 			meta_rep_param.clone(),
@@ -242,7 +237,6 @@ impl Garage {
 		#[allow(clippy::redundant_clone)]
 		let object_table = Table::new(
 			ObjectTable {
-				background: background.clone(),
 				version_table: version_table.clone(),
 				object_counter_table: object_counter_table.clone(),
 			},
@@ -258,8 +252,8 @@ impl Garage {
 		// -- done --
 		Ok(Arc::new(Self {
 			config,
+			replication_mode,
 			db,
-			background,
 			system,
 			block_manager,
 			bucket_table,
@@ -274,6 +268,22 @@ impl Garage {
 		}))
 	}
 
+	pub fn spawn_workers(&self, bg: &BackgroundRunner) {
+		self.block_manager.spawn_workers(bg);
+
+		self.bucket_table.spawn_workers(bg);
+		self.bucket_alias_table.spawn_workers(bg);
+		self.key_table.spawn_workers(bg);
+
+		self.object_table.spawn_workers(bg);
+		self.object_counter_table.spawn_workers(bg);
+		self.version_table.spawn_workers(bg);
+		self.block_ref_table.spawn_workers(bg);
+
+		#[cfg(feature = "k2v")]
+		self.k2v.spawn_workers(bg);
+	}
+
 	pub fn bucket_helper(&self) -> helper::bucket::BucketHelper {
 		helper::bucket::BucketHelper(self)
 	}
@@ -308,4 +318,9 @@ impl GarageK2V {
 			rpc,
 		}
 	}
+
+	pub fn spawn_workers(&self, bg: &BackgroundRunner) {
+		self.item_table.spawn_workers(bg);
+		self.counter_table.spawn_workers(bg);
+	}
 }

src/model/index_counter.rs

@@ -1,17 +1,15 @@
 use core::ops::Bound;
-use std::collections::{hash_map, BTreeMap, HashMap};
+use std::collections::{BTreeMap, HashMap};
 use std::marker::PhantomData;
 use std::sync::Arc;
 
-use async_trait::async_trait;
 use serde::{Deserialize, Serialize};
-use tokio::sync::{mpsc, watch};
 
 use garage_db as db;
 
 use garage_rpc::ring::Ring;
 use garage_rpc::system::System;
-use garage_util::background::*;
+use garage_util::background::BackgroundRunner;
 use garage_util::data::*;
 use garage_util::error::*;
 use garage_util::time::*;
@@ -142,7 +140,6 @@ impl<T: CountedItem> TableSchema for CounterTable<T> {
 pub struct IndexCounter<T: CountedItem> {
 	this_node: Uuid,
 	local_counter: db::Tree,
-	propagate_tx: mpsc::UnboundedSender<(T::CP, T::CS, LocalCounterEntry<T>)>,
 	pub table: Arc<Table<CounterTable<T>, TableShardedReplication>>,
 }
 
@@ -152,16 +149,11 @@ impl<T: CountedItem> IndexCounter<T> {
 		replication: TableShardedReplication,
 		db: &db::Db,
 	) -> Arc<Self> {
-		let background = system.background.clone();
-
-		let (propagate_tx, propagate_rx) = mpsc::unbounded_channel();
-
-		let this = Arc::new(Self {
+		Arc::new(Self {
 			this_node: system.id,
 			local_counter: db
 				.open_tree(format!("local_counter_v2:{}", T::COUNTER_TABLE_NAME))
 				.expect("Unable to open local counter tree"),
-			propagate_tx,
 			table: Table::new(
 				CounterTable {
 					_phantom_t: Default::default(),
@@ -170,16 +162,11 @@ impl<T: CountedItem> IndexCounter<T> {
 				system,
 				db,
 			),
-		});
+		})
+	}
 
-		background.spawn_worker(IndexPropagatorWorker {
-			index_counter: this.clone(),
-			propagate_rx,
-			buf: HashMap::new(),
-			errors: 0,
-		});
-
-		this
+	pub fn spawn_workers(&self, bg: &BackgroundRunner) {
+		self.table.spawn_workers(bg);
 	}
 
 	pub fn count(
@@ -232,12 +219,8 @@ impl<T: CountedItem> IndexCounter<T> {
 			.map_err(db::TxError::Abort)?;
 		tx.insert(&self.local_counter, &tree_key[..], new_entry_bytes)?;
 
-		if let Err(e) = self.propagate_tx.send((pk.clone(), sk.clone(), entry)) {
-			error!(
-				"Could not propagate updated counter values, failed to send to channel: {}",
-				e
-			);
-		}
+		let dist_entry = entry.into_counter_entry(self.this_node);
+		self.table.queue_insert(tx, &dist_entry)?;
 
 		Ok(())
 	}
@@ -250,23 +233,6 @@ impl<T: CountedItem> IndexCounter<T> {
 		TS: TableSchema<E = T>,
 		TR: TableReplication,
 	{
-		let save_counter_entry = |entry: CounterEntry<T>| -> Result<(), Error> {
-			let entry_k = self
-				.table
-				.data
-				.tree_key(entry.partition_key(), entry.sort_key());
-			self.table
-				.data
-				.update_entry_with(&entry_k, |ent| match ent {
-					Some(mut ent) => {
-						ent.merge(&entry);
-						ent
-					}
-					None => entry.clone(),
-				})?;
-			Ok(())
-		};
-
 		// 1. Set all old local counters to zero
 		let now = now_msec();
 		let mut next_start: Option<Vec<u8>> = None;
@@ -302,7 +268,9 @@ impl<T: CountedItem> IndexCounter<T> {
 					.insert(&local_counter_k, &local_counter_bytes)?;
 
 				let counter_entry = local_counter.into_counter_entry(self.this_node);
-				save_counter_entry(counter_entry)?;
+				self.local_counter
+					.db()
+					.transaction(|mut tx| self.table.queue_insert(&mut tx, &counter_entry))?;
 
 				next_start = Some(local_counter_k);
 			}
@@ -367,7 +335,9 @@ impl<T: CountedItem> IndexCounter<T> {
 					.insert(&local_counter_key, local_counter_bytes)?;
 
 				let counter_entry = local_counter.into_counter_entry(self.this_node);
-				save_counter_entry(counter_entry)?;
+				self.local_counter
+					.db()
+					.transaction(|mut tx| self.table.queue_insert(&mut tx, &counter_entry))?;
 
 				next_start = Some(counted_entry_k);
 			}
@@ -378,97 +348,7 @@ impl<T: CountedItem> IndexCounter<T> {
 	}
 }
 
-struct IndexPropagatorWorker<T: CountedItem> {
-	index_counter: Arc<IndexCounter<T>>,
-	propagate_rx: mpsc::UnboundedReceiver<(T::CP, T::CS, LocalCounterEntry<T>)>,
-
-	buf: HashMap<Vec<u8>, CounterEntry<T>>,
-	errors: usize,
-}
-
-impl<T: CountedItem> IndexPropagatorWorker<T> {
-	fn add_ent(&mut self, pk: T::CP, sk: T::CS, counters: LocalCounterEntry<T>) {
-		let tree_key = self.index_counter.table.data.tree_key(&pk, &sk);
-		let dist_entry = counters.into_counter_entry(self.index_counter.this_node);
-		match self.buf.entry(tree_key) {
-			hash_map::Entry::Vacant(e) => {
-				e.insert(dist_entry);
-			}
-			hash_map::Entry::Occupied(mut e) => {
-				e.get_mut().merge(&dist_entry);
-			}
-		}
-	}
-}
-
-#[async_trait]
-impl<T: CountedItem> Worker for IndexPropagatorWorker<T> {
-	fn name(&self) -> String {
-		format!("{} index counter propagator", T::COUNTER_TABLE_NAME)
-	}
-
-	fn info(&self) -> Option<String> {
-		if !self.buf.is_empty() {
-			Some(format!("{} items in queue", self.buf.len()))
-		} else {
-			None
-		}
-	}
-
-	async fn work(&mut self, must_exit: &mut watch::Receiver<bool>) -> Result<WorkerState, Error> {
-		// This loop batches updates to counters to be sent all at once.
-		// They are sent once the propagate_rx channel has been emptied (or is closed).
-		let closed = loop {
-			match self.propagate_rx.try_recv() {
-				Ok((pk, sk, counters)) => {
-					self.add_ent(pk, sk, counters);
-				}
-				Err(mpsc::error::TryRecvError::Empty) => break false,
-				Err(mpsc::error::TryRecvError::Disconnected) => break true,
-			}
-		};
-
-		if !self.buf.is_empty() {
-			let entries_k = self.buf.keys().take(100).cloned().collect::<Vec<_>>();
-			let entries = entries_k.iter().map(|k| self.buf.get(k).unwrap());
-			if let Err(e) = self.index_counter.table.insert_many(entries).await {
-				self.errors += 1;
-				if self.errors >= 2 && *must_exit.borrow() {
-					error!("({}) Could not propagate {} counter values: {}, these counters will not be updated correctly.", T::COUNTER_TABLE_NAME, self.buf.len(), e);
-					return Ok(WorkerState::Done);
-				}
-				// Propagate error up to worker manager, it will log it, increment a counter,
-				// and sleep for a certain delay (with exponential backoff), waiting for
-				// things to go back to normal
-				return Err(e);
-			} else {
-				for k in entries_k {
-					self.buf.remove(&k);
-				}
-				self.errors = 0;
-			}
-
-			return Ok(WorkerState::Busy);
-		} else if closed {
-			return Ok(WorkerState::Done);
-		} else {
-			return Ok(WorkerState::Idle);
-		}
-	}
-
-	async fn wait_for_work(&mut self, _must_exit: &watch::Receiver<bool>) -> WorkerState {
-		match self.propagate_rx.recv().await {
-			Some((pk, sk, counters)) => {
-				self.add_ent(pk, sk, counters);
-				WorkerState::Busy
-			}
-			None => match self.buf.is_empty() {
-				false => WorkerState::Busy,
-				true => WorkerState::Done,
-			},
-		}
-	}
-}
+// ----
 
 #[derive(PartialEq, Clone, Debug, Serialize, Deserialize)]
 struct LocalCounterEntry<T: CountedItem> {

src/model/k2v/rpc.rs

@@ -273,14 +273,9 @@ impl K2VRpcHandler {
 	}
 
 	fn local_insert(&self, item: &InsertedItem) -> Result<Option<K2VItem>, Error> {
-		let tree_key = self
-			.item_table
-			.data
-			.tree_key(&item.partition, &item.sort_key);
-
 		self.item_table
 			.data
-			.update_entry_with(&tree_key[..], |ent| {
+			.update_entry_with(&item.partition, &item.sort_key, |ent| {
 				let mut ent = ent.unwrap_or_else(|| {
 					K2VItem::new(
 						item.partition.bucket_id,

src/model/s3/object_table.rs

@@ -4,7 +4,6 @@ use std::sync::Arc;
 
 use garage_db as db;
 
-use garage_util::background::BackgroundRunner;
 use garage_util::data::*;
 
 use garage_table::crdt::*;
@@ -221,7 +220,6 @@ impl Crdt for Object {
 }
 
 pub struct ObjectTable {
-	pub background: Arc<BackgroundRunner>,
 	pub version_table: Arc<Table<VersionTable, TableShardedReplication>>,
 	pub object_counter_table: Arc<IndexCounter<Object>>,
 }
@@ -255,34 +253,34 @@ impl TableSchema for ObjectTable {
 			);
 		}
 
-		// 2. Spawn threads that propagates deletions to version table
-		let version_table = self.version_table.clone();
-		let old = old.cloned();
-		let new = new.cloned();
-
-		self.background.spawn(async move {
-			if let (Some(old_v), Some(new_v)) = (old, new) {
-				// Propagate deletion of old versions
-				for v in old_v.versions.iter() {
-					let newly_deleted = match new_v
-						.versions
-						.binary_search_by(|nv| nv.cmp_key().cmp(&v.cmp_key()))
-					{
-						Err(_) => true,
-						Ok(i) => {
-							new_v.versions[i].state == ObjectVersionState::Aborted
-								&& v.state != ObjectVersionState::Aborted
-						}
-					};
-					if newly_deleted {
-						let deleted_version =
-							Version::new(v.uuid, old_v.bucket_id, old_v.key.clone(), true);
-						version_table.insert(&deleted_version).await?;
+		// 2. Enqueue propagation deletions to version table
+		if let (Some(old_v), Some(new_v)) = (old, new) {
+			// Propagate deletion of old versions
+			for v in old_v.versions.iter() {
+				let newly_deleted = match new_v
+					.versions
+					.binary_search_by(|nv| nv.cmp_key().cmp(&v.cmp_key()))
+				{
+					Err(_) => true,
+					Ok(i) => {
+						new_v.versions[i].state == ObjectVersionState::Aborted
+							&& v.state != ObjectVersionState::Aborted
+					}
+				};
+				if newly_deleted {
+					let deleted_version =
+						Version::new(v.uuid, old_v.bucket_id, old_v.key.clone(), true);
+					let res = self.version_table.queue_insert(tx, &deleted_version);
+					if let Err(e) = db::unabort(res)? {
+						error!(
+							"Unable to enqueue version deletion propagation: {}. A repair will be needed.",
+							e
+						);
 					}
 				}
 			}
-			Ok(())
-		});
+		}
+
 		Ok(())
 	}
 

src/model/s3/version_table.rs

@@ -3,7 +3,6 @@ use std::sync::Arc;
 
 use garage_db as db;
 
-use garage_util::background::BackgroundRunner;
 use garage_util::data::*;
 
 use garage_table::crdt::*;
@@ -127,7 +126,6 @@ impl Crdt for Version {
 }
 
 pub struct VersionTable {
-	pub background: Arc<BackgroundRunner>,
 	pub block_ref_table: Arc<Table<BlockRefTable, TableShardedReplication>>,
 }
 
@@ -141,33 +139,26 @@ impl TableSchema for VersionTable {
 
 	fn updated(
 		&self,
-		_tx: &mut db::Transaction,
+		tx: &mut db::Transaction,
 		old: Option<&Self::E>,
 		new: Option<&Self::E>,
 	) -> db::TxOpResult<()> {
-		let block_ref_table = self.block_ref_table.clone();
-		let old = old.cloned();
-		let new = new.cloned();
-
-		self.background.spawn(async move {
-			if let (Some(old_v), Some(new_v)) = (old, new) {
-				// Propagate deletion of version blocks
-				if new_v.deleted.get() && !old_v.deleted.get() {
-					let deleted_block_refs = old_v
-						.blocks
-						.items()
-						.iter()
-						.map(|(_k, vb)| BlockRef {
-							block: vb.hash,
-							version: old_v.uuid,
-							deleted: true.into(),
-						})
-						.collect::<Vec<_>>();
-					block_ref_table.insert_many(&deleted_block_refs[..]).await?;
+		if let (Some(old_v), Some(new_v)) = (old, new) {
+			// Propagate deletion of version blocks
+			if new_v.deleted.get() && !old_v.deleted.get() {
+				let deleted_block_refs = old_v.blocks.items().iter().map(|(_k, vb)| BlockRef {
+					block: vb.hash,
+					version: old_v.uuid,
+					deleted: true.into(),
+				});
+				for block_ref in deleted_block_refs {
+					let res = self.block_ref_table.queue_insert(tx, &block_ref);
+					if let Err(e) = db::unabort(res)? {
+						error!("Unable to enqueue block ref deletion propagation: {}. A repair will be needed.", e);
+					}
 				}
 			}
-			Ok(())
-		});
+		}
 
 		Ok(())
 	}

src/rpc/layout.rs

@@ -254,9 +254,11 @@ To know the correct value of the new layout version, invoke `garage layout show`
 		match self.initial_partition_assignation() {
 			Some(initial_partitions) => {
 				for (part, ipart) in partitions.iter_mut().zip(initial_partitions.iter()) {
-					for (id, info) in ipart.nodes.iter() {
-						if part.nodes.len() < self.replication_factor {
-							part.add(None, n_zones, id, info.unwrap());
+					for _ in 0..2 {
+						for (id, info) in ipart.nodes.iter() {
+							if part.nodes.len() < self.replication_factor {
+								part.add(None, n_zones, id, info.unwrap());
+							}
 						}
 					}
 					assert!(part.nodes.len() == self.replication_factor);

src/rpc/lib.rs

@@ -9,6 +9,7 @@ mod consul;
 mod kubernetes;
 
 pub mod layout;
+pub mod replication_mode;
 pub mod ring;
 pub mod system;
 

src/rpc/replication_mode.rs

@@ -1,3 +1,4 @@
+#[derive(Clone, Copy)]
 pub enum ReplicationMode {
 	None,
 	TwoWay,

src/rpc/rpc_helper.rs

@@ -5,7 +5,6 @@ use std::time::Duration;
 use futures::future::join_all;
 use futures::stream::futures_unordered::FuturesUnordered;
 use futures::stream::StreamExt;
-use futures_util::future::FutureExt;
 use tokio::select;
 use tokio::sync::watch;
 
@@ -24,7 +23,6 @@ pub use netapp::message::{
 use netapp::peering::fullmesh::FullMeshPeeringStrategy;
 pub use netapp::{self, NetApp, NodeID};
 
-use garage_util::background::BackgroundRunner;
 use garage_util::data::*;
 use garage_util::error::Error;
 use garage_util::metrics::RecordDuration;
@@ -94,7 +92,6 @@ pub struct RpcHelper(Arc<RpcHelperInner>);
 struct RpcHelperInner {
 	our_node_id: Uuid,
 	fullmesh: Arc<FullMeshPeeringStrategy>,
-	background: Arc<BackgroundRunner>,
 	ring: watch::Receiver<Arc<Ring>>,
 	metrics: RpcMetrics,
 	rpc_timeout: Duration,
@@ -104,7 +101,6 @@ impl RpcHelper {
 	pub(crate) fn new(
 		our_node_id: Uuid,
 		fullmesh: Arc<FullMeshPeeringStrategy>,
-		background: Arc<BackgroundRunner>,
 		ring: watch::Receiver<Arc<Ring>>,
 		rpc_timeout: Option<Duration>,
 	) -> Self {
@@ -113,7 +109,6 @@ impl RpcHelper {
 		Self(Arc::new(RpcHelperInner {
 			our_node_id,
 			fullmesh,
-			background,
 			ring,
 			metrics,
 			rpc_timeout: rpc_timeout.unwrap_or(DEFAULT_TIMEOUT),
@@ -377,16 +372,13 @@ impl RpcHelper {
 
 			if !resp_stream.is_empty() {
 				// Continue remaining requests in background.
-				// Continue the remaining requests immediately using tokio::spawn
-				// but enqueue a task in the background runner
-				// to ensure that the process won't exit until the requests are done
-				// (if we had just enqueued the resp_stream.collect directly in the background runner,
-				// the requests might have been put on hold in the background runner's queue,
-				// in which case they might timeout or otherwise fail)
-				let wait_finished_fut = tokio::spawn(async move {
+				// Note: these requests can get interrupted on process shutdown,
+				// we must not count on them being executed for certain.
+				// For all background things that have to happen with certainty,
+				// they have to be put in a proper queue that is persisted to disk.
+				tokio::spawn(async move {
 					resp_stream.collect::<Vec<Result<_, _>>>().await;
 				});
-				self.0.background.spawn(wait_finished_fut.map(|_| Ok(())));
 			}
 		}
 

src/rpc/system.rs

@@ -21,7 +21,6 @@ use netapp::peering::fullmesh::FullMeshPeeringStrategy;
 use netapp::util::parse_and_resolve_peer_addr_async;
 use netapp::{NetApp, NetworkKey, NodeID, NodeKey};
 
-use garage_util::background::BackgroundRunner;
 use garage_util::config::Config;
 #[cfg(feature = "kubernetes-discovery")]
 use garage_util::config::KubernetesDiscoveryConfig;
@@ -35,6 +34,7 @@ use crate::consul::ConsulDiscovery;
 #[cfg(feature = "kubernetes-discovery")]
 use crate::kubernetes::*;
 use crate::layout::*;
+use crate::replication_mode::*;
 use crate::ring::*;
 use crate::rpc_helper::*;
 
@@ -49,8 +49,6 @@ pub const GARAGE_VERSION_TAG: u64 = 0x6761726167650008; // garage 0x0008
 /// RPC endpoint used for calls related to membership
 pub const SYSTEM_RPC_PATH: &str = "garage_rpc/membership.rs/SystemRpc";
 
-pub const CONNECT_ERROR_MESSAGE: &str = "Error establishing RPC connection to remote node. This can happen if the remote node is not reachable on the network, but also if the two nodes are not configured with the same rpc_secret";
-
 /// RPC messages related to membership
 #[derive(Debug, Serialize, Deserialize, Clone)]
 pub enum SystemRpc {
@@ -102,15 +100,13 @@ pub struct System {
 	#[cfg(feature = "kubernetes-discovery")]
 	kubernetes_discovery: Option<KubernetesDiscoveryConfig>,
 
+	replication_mode: ReplicationMode,
 	replication_factor: usize,
 
 	/// The ring
 	pub ring: watch::Receiver<Arc<Ring>>,
 	update_ring: Mutex<watch::Sender<Arc<Ring>>>,
 
-	/// The job runner of this node
-	pub background: Arc<BackgroundRunner>,
-
 	/// Path to metadata directory
 	pub metadata_dir: PathBuf,
 }
@@ -136,6 +132,37 @@ pub struct KnownNodeInfo {
 	pub status: NodeStatus,
 }
 
+#[derive(Debug, Clone, Copy, Serialize, Deserialize)]
+pub struct ClusterHealth {
+	/// The current health status of the cluster (see below)
+	pub status: ClusterHealthStatus,
+	/// Number of nodes already seen once in the cluster
+	pub known_nodes: usize,
+	/// Number of nodes currently connected
+	pub connected_nodes: usize,
+	/// Number of storage nodes declared in the current layout
+	pub storage_nodes: usize,
+	/// Number of storage nodes currently connected
+	pub storage_nodes_ok: usize,
+	/// Number of partitions in the layout
+	pub partitions: usize,
+	/// Number of partitions for which we have a quorum of connected nodes
+	pub partitions_quorum: usize,
+	/// Number of partitions for which all storage nodes are connected
+	pub partitions_all_ok: usize,
+}
+
+#[derive(Debug, Clone, Copy, Serialize, Deserialize)]
+pub enum ClusterHealthStatus {
+	/// All nodes are available
+	Healthy,
+	/// Some storage nodes are unavailable, but quorum is stil
+	/// achieved for all partitions
+	Degraded,
+	/// Quorum is not available for some partitions
+	Unavailable,
+}
+
 pub fn read_node_id(metadata_dir: &Path) -> Result<NodeID, Error> {
 	let mut pubkey_file = metadata_dir.to_path_buf();
 	pubkey_file.push("node_key.pub");
@@ -199,10 +226,11 @@ impl System {
 	/// Create this node's membership manager
 	pub fn new(
 		network_key: NetworkKey,
-		background: Arc<BackgroundRunner>,
-		replication_factor: usize,
+		replication_mode: ReplicationMode,
 		config: &Config,
 	) -> Result<Arc<Self>, Error> {
+		let replication_factor = replication_mode.replication_factor();
+
 		let node_key =
 			gen_node_key(&config.metadata_dir).expect("Unable to read or generate node ID");
 		info!(
@@ -319,11 +347,11 @@ impl System {
 			rpc: RpcHelper::new(
 				netapp.id.into(),
 				fullmesh,
-				background.clone(),
 				ring.clone(),
 				config.rpc_timeout_msec.map(Duration::from_millis),
 			),
 			system_endpoint,
+			replication_mode,
 			replication_factor,
 			rpc_listen_addr: config.rpc_bind_addr,
 			#[cfg(any(feature = "consul-discovery", feature = "kubernetes-discovery"))]
@@ -336,7 +364,6 @@ impl System {
 
 			ring,
 			update_ring: Mutex::new(update_ring),
-			background,
 			metadata_dir: config.metadata_dir.clone(),
 		});
 		sys.system_endpoint.set_handler(sys.clone());
@@ -408,17 +435,14 @@ impl System {
 				))
 			})?;
 		let mut errors = vec![];
-		for ip in addrs.iter() {
-			match self
-				.netapp
-				.clone()
-				.try_connect(*ip, pubkey)
-				.await
-				.err_context(CONNECT_ERROR_MESSAGE)
-			{
+		for addr in addrs.iter() {
+			match self.netapp.clone().try_connect(*addr, pubkey).await {
 				Ok(()) => return Ok(()),
 				Err(e) => {
-					errors.push((*ip, e));
+					errors.push((
+						*addr,
+						Error::Message(connect_error_message(*addr, pubkey, e)),
+					));
 				}
 			}
 		}
@@ -429,6 +453,67 @@ impl System {
 		}
 	}
 
+	pub fn health(&self) -> ClusterHealth {
+		let ring: Arc<_> = self.ring.borrow().clone();
+		let quorum = self.replication_mode.write_quorum();
+		let replication_factor = self.replication_factor;
+
+		let nodes = self
+			.get_known_nodes()
+			.into_iter()
+			.map(|n| (n.id, n))
+			.collect::<HashMap<Uuid, _>>();
+		let connected_nodes = nodes.iter().filter(|(_, n)| n.is_up).count();
+
+		let storage_nodes = ring
+			.layout
+			.roles
+			.items()
+			.iter()
+			.filter(|(_, _, v)| matches!(v, NodeRoleV(Some(r)) if r.capacity.is_some()))
+			.collect::<Vec<_>>();
+		let storage_nodes_ok = storage_nodes
+			.iter()
+			.filter(|(x, _, _)| nodes.get(x).map(|n| n.is_up).unwrap_or(false))
+			.count();
+
+		let partitions = ring.partitions();
+		let partitions_n_up = partitions
+			.iter()
+			.map(|(_, h)| {
+				let pn = ring.get_nodes(h, ring.replication_factor);
+				pn.iter()
+					.filter(|x| nodes.get(x).map(|n| n.is_up).unwrap_or(false))
+					.count()
+			})
+			.collect::<Vec<usize>>();
+		let partitions_all_ok = partitions_n_up
+			.iter()
+			.filter(|c| **c == replication_factor)
+			.count();
+		let partitions_quorum = partitions_n_up.iter().filter(|c| **c >= quorum).count();
+
+		let status =
+			if partitions_quorum == partitions.len() && storage_nodes_ok == storage_nodes.len() {
+				ClusterHealthStatus::Healthy
+			} else if partitions_quorum == partitions.len() {
+				ClusterHealthStatus::Degraded
+			} else {
+				ClusterHealthStatus::Unavailable
+			};
+
+		ClusterHealth {
+			status,
+			known_nodes: nodes.len(),
+			connected_nodes,
+			storage_nodes: storage_nodes.len(),
+			storage_nodes_ok,
+			partitions: partitions.len(),
+			partitions_quorum,
+			partitions_all_ok,
+		}
+	}
+
 	// ---- INTERNALS ----
 
 	#[cfg(feature = "consul-discovery")]
@@ -481,7 +566,7 @@ impl System {
 	}
 
 	/// Save network configuration to disc
-	async fn save_cluster_layout(self: Arc<Self>) -> Result<(), Error> {
+	async fn save_cluster_layout(&self) -> Result<(), Error> {
 		let ring: Arc<Ring> = self.ring.borrow().clone();
 		self.persist_cluster_layout
 			.save_async(&ring.layout)
@@ -533,11 +618,7 @@ impl System {
 		if info.cluster_layout_version > local_info.cluster_layout_version
 			|| info.cluster_layout_staging_hash != local_info.cluster_layout_staging_hash
 		{
-			let self2 = self.clone();
-			self.background.spawn_cancellable(async move {
-				self2.pull_cluster_layout(from).await;
-				Ok(())
-			});
+			tokio::spawn(self.clone().pull_cluster_layout(from));
 		}
 
 		self.node_status
@@ -579,18 +660,21 @@ impl System {
 			drop(update_ring);
 
 			let self2 = self.clone();
-			self.background.spawn_cancellable(async move {
-				self2
+			tokio::spawn(async move {
+				if let Err(e) = self2
 					.rpc
 					.broadcast(
 						&self2.system_endpoint,
 						SystemRpc::AdvertiseClusterLayout(layout),
 						RequestStrategy::with_priority(PRIO_HIGH),
 					)
-					.await?;
-				Ok(())
+					.await
+				{
+					warn!("Error while broadcasting new cluster layout: {}", e);
+				}
 			});
-			self.background.spawn(self.clone().save_cluster_layout());
+
+			self.save_cluster_layout().await?;
 		}
 
 		Ok(SystemRpc::Ok)
@@ -676,12 +760,12 @@ impl System {
 				}
 
 				for (node_id, node_addr) in ping_list {
-					tokio::spawn(
-						self.netapp
-							.clone()
-							.try_connect(node_addr, node_id)
-							.map(|r| r.err_context(CONNECT_ERROR_MESSAGE)),
-					);
+					let self2 = self.clone();
+					tokio::spawn(async move {
+						if let Err(e) = self2.netapp.clone().try_connect(node_addr, node_id).await {
+							error!("{}", connect_error_message(node_addr, node_id, e));
+						}
+					});
 				}
 			}
 
@@ -690,11 +774,10 @@ impl System {
 			}
 
 			#[cfg(feature = "consul-discovery")]
-			self.background.spawn(self.clone().advertise_to_consul());
+			background::spawn(self.clone().advertise_to_consul());
 
 			#[cfg(feature = "kubernetes-discovery")]
-			self.background
-				.spawn(self.clone().advertise_to_kubernetes());
+			background::spawn(self.clone().advertise_to_kubernetes());
 
 			let restart_at = tokio::time::sleep(DISCOVERY_INTERVAL);
 			select! {
@@ -784,3 +867,11 @@ async fn resolve_peers(peers: &[String]) -> Vec<(NodeID, SocketAddr)> {
 
 	ret
 }
+
+fn connect_error_message(
+	addr: SocketAddr,
+	pubkey: ed25519::PublicKey,
+	e: netapp::error::Error,
+) -> String {
+	format!("Error establishing RPC connection to remote node: {}@{}.\nThis can happen if the remote node is not reachable on the network, but also if the two nodes are not configured with the same rpc_secret.\n{}", hex::encode(pubkey), addr, e)
+}

src/table/Cargo.toml

@@ -21,6 +21,7 @@ garage_util = { version = "0.8.0", path = "../util" }
 opentelemetry = "0.17"
 
 async-trait = "0.1.7"
+arc-swap = "1.0"
 bytes = "1.0"
 hex = "0.4"
 hexdump = "0.1"

src/table/data.rs

@@ -31,6 +31,10 @@ pub struct TableData<F: TableSchema, R: TableReplication> {
 	pub(crate) merkle_tree: db::Tree,
 	pub(crate) merkle_todo: db::Tree,
 	pub(crate) merkle_todo_notify: Notify,
+
+	pub(crate) insert_queue: db::Tree,
+	pub(crate) insert_queue_notify: Notify,
+
 	pub(crate) gc_todo: CountedTree,
 
 	pub(crate) metrics: TableMetrics,
@@ -53,12 +57,22 @@ where
 			.open_tree(&format!("{}:merkle_todo", F::TABLE_NAME))
 			.expect("Unable to open DB Merkle TODO tree");
 
+		let insert_queue = db
+			.open_tree(&format!("{}:insert_queue", F::TABLE_NAME))
+			.expect("Unable to open insert queue DB tree");
+
 		let gc_todo = db
 			.open_tree(&format!("{}:gc_todo_v2", F::TABLE_NAME))
-			.expect("Unable to open DB tree");
+			.expect("Unable to open GC DB tree");
 		let gc_todo = CountedTree::new(gc_todo).expect("Cannot count gc_todo_v2");
 
-		let metrics = TableMetrics::new(F::TABLE_NAME, merkle_todo.clone(), gc_todo.clone());
+		let metrics = TableMetrics::new(
+			F::TABLE_NAME,
+			store.clone(),
+			merkle_tree.clone(),
+			merkle_todo.clone(),
+			gc_todo.clone(),
+		);
 
 		Arc::new(Self {
 			system,
@@ -68,6 +82,8 @@ where
 			merkle_tree,
 			merkle_todo,
 			merkle_todo_notify: Notify::new(),
+			insert_queue,
+			insert_queue_notify: Notify::new(),
 			gc_todo,
 			metrics,
 		})
@@ -167,9 +183,8 @@ where
 
 	pub(crate) fn update_entry(&self, update_bytes: &[u8]) -> Result<(), Error> {
 		let update = self.decode_entry(update_bytes)?;
-		let tree_key = self.tree_key(update.partition_key(), update.sort_key());
 
-		self.update_entry_with(&tree_key[..], |ent| match ent {
+		self.update_entry_with(update.partition_key(), update.sort_key(), |ent| match ent {
 			Some(mut ent) => {
 				ent.merge(&update);
 				ent
@@ -181,11 +196,14 @@ where
 
 	pub fn update_entry_with(
 		&self,
-		tree_key: &[u8],
+		partition_key: &F::P,
+		sort_key: &F::S,
 		f: impl Fn(Option<F::E>) -> F::E,
 	) -> Result<Option<F::E>, Error> {
+		let tree_key = self.tree_key(partition_key, sort_key);
+
 		let changed = self.store.db().transaction(|mut tx| {
-			let (old_entry, old_bytes, new_entry) = match tx.get(&self.store, tree_key)? {
+			let (old_entry, old_bytes, new_entry) = match tx.get(&self.store, &tree_key)? {
 				Some(old_bytes) => {
 					let old_entry = self.decode_entry(&old_bytes).map_err(db::TxError::Abort)?;
 					let new_entry = f(Some(old_entry.clone()));
@@ -194,23 +212,23 @@ where
 				None => (None, None, f(None)),
 			};
 
-			// Scenario 1: the value changed, so of course there is a change
-			let value_changed = Some(&new_entry) != old_entry.as_ref();
-
+			// Changed can be true in two scenarios
+			// Scenario 1: the actual represented value changed,
+			//   so of course the messagepack encoding changed as well
 			// Scenario 2: the value didn't change but due to a migration in the
-			// data format, the messagepack encoding changed. In this case
-			// we have to write the migrated value in the table and update
-			// the associated Merkle tree entry.
+			//   data format, the messagepack encoding changed. In this case,
+			//   we also have to write the migrated value in the table and update
+			//   the associated Merkle tree entry.
 			let new_bytes = rmp_to_vec_all_named(&new_entry)
 				.map_err(Error::RmpEncode)
 				.map_err(db::TxError::Abort)?;
-			let encoding_changed = Some(&new_bytes[..]) != old_bytes.as_ref().map(|x| &x[..]);
+			let changed = Some(&new_bytes[..]) != old_bytes.as_deref();
 			drop(old_bytes);
 
-			if value_changed || encoding_changed {
-				let new_bytes_hash = blake2sum(&new_bytes[..]);
-				tx.insert(&self.merkle_todo, tree_key, new_bytes_hash.as_slice())?;
-				tx.insert(&self.store, tree_key, new_bytes)?;
+			if changed {
+				let new_bytes_hash = blake2sum(&new_bytes);
+				tx.insert(&self.merkle_todo, &tree_key, new_bytes_hash.as_slice())?;
+				tx.insert(&self.store, &tree_key, new_bytes)?;
 
 				self.instance
 					.updated(&mut tx, old_entry.as_ref(), Some(&new_entry))?;
@@ -236,7 +254,7 @@ where
 				let pk_hash = Hash::try_from(&tree_key[..32]).unwrap();
 				let nodes = self.replication.write_nodes(&pk_hash);
 				if nodes.first() == Some(&self.system.id) {
-					GcTodoEntry::new(tree_key.to_vec(), new_bytes_hash).save(&self.gc_todo)?;
+					GcTodoEntry::new(tree_key, new_bytes_hash).save(&self.gc_todo)?;
 				}
 			}
 
@@ -252,10 +270,11 @@ where
 			.db()
 			.transaction(|mut tx| match tx.get(&self.store, k)? {
 				Some(cur_v) if cur_v == v => {
+					let old_entry = self.decode_entry(v).map_err(db::TxError::Abort)?;
+
 					tx.remove(&self.store, k)?;
 					tx.insert(&self.merkle_todo, k, vec![])?;
 
-					let old_entry = self.decode_entry(v).map_err(db::TxError::Abort)?;
 					self.instance.updated(&mut tx, Some(&old_entry), None)?;
 					Ok(true)
 				}
@@ -279,10 +298,11 @@ where
 			.db()
 			.transaction(|mut tx| match tx.get(&self.store, k)? {
 				Some(cur_v) if blake2sum(&cur_v[..]) == vhash => {
+					let old_entry = self.decode_entry(&cur_v[..]).map_err(db::TxError::Abort)?;
+
 					tx.remove(&self.store, k)?;
 					tx.insert(&self.merkle_todo, k, vec![])?;
 
-					let old_entry = self.decode_entry(&cur_v[..]).map_err(db::TxError::Abort)?;
 					self.instance.updated(&mut tx, Some(&old_entry), None)?;
 					Ok(true)
 				}
@@ -296,6 +316,32 @@ where
 		Ok(removed)
 	}
 
+	// ---- Insert queue functions ----
+
+	pub(crate) fn queue_insert(
+		&self,
+		tx: &mut db::Transaction,
+		ins: &F::E,
+	) -> db::TxResult<(), Error> {
+		let tree_key = self.tree_key(ins.partition_key(), ins.sort_key());
+
+		let new_entry = match tx.get(&self.insert_queue, &tree_key)? {
+			Some(old_v) => {
+				let mut entry = self.decode_entry(&old_v).map_err(db::TxError::Abort)?;
+				entry.merge(ins);
+				rmp_to_vec_all_named(&entry)
+			}
+			None => rmp_to_vec_all_named(ins),
+		};
+		let new_entry = new_entry
+			.map_err(Error::RmpEncode)
+			.map_err(db::TxError::Abort)?;
+		tx.insert(&self.insert_queue, &tree_key, new_entry)?;
+		self.insert_queue_notify.notify_one();
+
+		Ok(())
+	}
+
 	// ---- Utility functions ----
 
 	pub fn tree_key(&self, p: &F::P, s: &F::S) -> Vec<u8> {

src/table/gc.rs

@@ -54,24 +54,25 @@ where
 	F: TableSchema + 'static,
 	R: TableReplication + 'static,
 {
-	pub(crate) fn launch(system: Arc<System>, data: Arc<TableData<F, R>>) -> Arc<Self> {
+	pub(crate) fn new(system: Arc<System>, data: Arc<TableData<F, R>>) -> Arc<Self> {
 		let endpoint = system
 			.netapp
 			.endpoint(format!("garage_table/gc.rs/Rpc:{}", F::TABLE_NAME));
 
 		let gc = Arc::new(Self {
-			system: system.clone(),
+			system,
 			data,
 			endpoint,
 		});
-
 		gc.endpoint.set_handler(gc.clone());
 
-		system.background.spawn_worker(GcWorker::new(gc.clone()));
-
 		gc
 	}
 
+	pub(crate) fn spawn_workers(self: &Arc<Self>, bg: &BackgroundRunner) {
+		bg.spawn_worker(GcWorker::new(self.clone()));
+	}
+
 	async fn gc_loop_iter(&self) -> Result<Option<Duration>, Error> {
 		let now = now_msec();
 
@@ -330,12 +331,10 @@ where
 		format!("{} GC", F::TABLE_NAME)
 	}
 
-	fn info(&self) -> Option<String> {
-		let l = self.gc.data.gc_todo_len().unwrap_or(0);
-		if l > 0 {
-			Some(format!("{} items in queue", l))
-		} else {
-			None
+	fn status(&self) -> WorkerStatus {
+		WorkerStatus {
+			queue_length: Some(self.gc.data.gc_todo_len().unwrap_or(0) as u64),
+			..Default::default()
 		}
 	}
 
@@ -349,10 +348,7 @@ where
 		}
 	}
 
-	async fn wait_for_work(&mut self, must_exit: &watch::Receiver<bool>) -> WorkerState {
-		if *must_exit.borrow() {
-			return WorkerState::Done;
-		}
+	async fn wait_for_work(&mut self) -> WorkerState {
 		tokio::time::sleep(self.wait_delay).await;
 		WorkerState::Busy
 	}

src/table/lib.rs

@@ -4,16 +4,18 @@
 #[macro_use]
 extern crate tracing;
 
-mod metrics;
 pub mod schema;
 pub mod util;
 
 pub mod data;
+pub mod replication;
+pub mod table;
+
 mod gc;
 mod merkle;
-pub mod replication;
+mod metrics;
+mod queue;
 mod sync;
-pub mod table;
 
 pub use schema::*;
 pub use table::*;

src/table/merkle.rs

@@ -3,6 +3,7 @@ use std::time::Duration;
 
 use async_trait::async_trait;
 use serde::{Deserialize, Serialize};
+use tokio::select;
 use tokio::sync::watch;
 
 use garage_db as db;
@@ -69,17 +70,17 @@ where
 	F: TableSchema + 'static,
 	R: TableReplication + 'static,
 {
-	pub(crate) fn launch(background: &BackgroundRunner, data: Arc<TableData<F, R>>) -> Arc<Self> {
+	pub(crate) fn new(data: Arc<TableData<F, R>>) -> Arc<Self> {
 		let empty_node_hash = blake2sum(&rmp_to_vec_all_named(&MerkleNode::Empty).unwrap()[..]);
 
-		let ret = Arc::new(Self {
+		Arc::new(Self {
 			data,
 			empty_node_hash,
-		});
+		})
+	}
 
-		background.spawn_worker(MerkleWorker(ret.clone()));
-
-		ret
+	pub(crate) fn spawn_workers(self: &Arc<Self>, background: &BackgroundRunner) {
+		background.spawn_worker(MerkleWorker(self.clone()));
 	}
 
 	fn updater_loop_iter(&self) -> Result<WorkerState, Error> {
@@ -293,6 +294,10 @@ where
 		Ok(self.data.merkle_tree.len()?)
 	}
 
+	pub fn merkle_tree_fast_len(&self) -> Result<Option<usize>, Error> {
+		Ok(self.data.merkle_tree.fast_len()?)
+	}
+
 	pub fn todo_len(&self) -> Result<usize, Error> {
 		Ok(self.data.merkle_todo.len()?)
 	}
@@ -310,15 +315,13 @@ where
 	R: TableReplication + 'static,
 {
 	fn name(&self) -> String {
-		format!("{} Merkle tree updater", F::TABLE_NAME)
+		format!("{} Merkle", F::TABLE_NAME)
 	}
 
-	fn info(&self) -> Option<String> {
-		let l = self.0.todo_len().unwrap_or(0);
-		if l > 0 {
-			Some(format!("{} items in queue", l))
-		} else {
-			None
+	fn status(&self) -> WorkerStatus {
+		WorkerStatus {
+			queue_length: Some(self.0.todo_len().unwrap_or(0) as u64),
+			..Default::default()
 		}
 	}
 
@@ -337,11 +340,11 @@ where
 		.unwrap()
 	}
 
-	async fn wait_for_work(&mut self, must_exit: &watch::Receiver<bool>) -> WorkerState {
-		if *must_exit.borrow() {
-			return WorkerState::Done;
+	async fn wait_for_work(&mut self) -> WorkerState {
+		select! {
+			_ = tokio::time::sleep(Duration::from_secs(60)) => (),
+			_ = self.0.data.merkle_todo_notify.notified() => (),
 		}
-		tokio::time::sleep(Duration::from_secs(10)).await;
 		WorkerState::Busy
 	}
 }

src/table/metrics.rs

@@ -5,6 +5,8 @@ use garage_db::counted_tree_hack::CountedTree;
 
 /// TableMetrics reference all counter used for metrics
 pub struct TableMetrics {
+	pub(crate) _table_size: ValueObserver<u64>,
+	pub(crate) _merkle_tree_size: ValueObserver<u64>,
 	pub(crate) _merkle_todo_len: ValueObserver<u64>,
 	pub(crate) _gc_todo_len: ValueObserver<u64>,
 
@@ -20,9 +22,43 @@ pub struct TableMetrics {
 	pub(crate) sync_items_received: Counter<u64>,
 }
 impl TableMetrics {
-	pub fn new(table_name: &'static str, merkle_todo: db::Tree, gc_todo: CountedTree) -> Self {
+	pub fn new(
+		table_name: &'static str,
+		store: db::Tree,
+		merkle_tree: db::Tree,
+		merkle_todo: db::Tree,
+		gc_todo: CountedTree,
+	) -> Self {
 		let meter = global::meter(table_name);
 		TableMetrics {
+			_table_size: meter
+				.u64_value_observer(
+					"table.size",
+					move |observer| {
+						if let Ok(Some(v)) = store.fast_len() {
+							observer.observe(
+								v as u64,
+								&[KeyValue::new("table_name", table_name)],
+							);
+						}
+					},
+				)
+				.with_description("Number of items in table")
+				.init(),
+			_merkle_tree_size: meter
+				.u64_value_observer(
+					"table.merkle_tree_size",
+					move |observer| {
+						if let Ok(Some(v)) = merkle_tree.fast_len() {
+							observer.observe(
+								v as u64,
+								&[KeyValue::new("table_name", table_name)],
+							);
+						}
+					},
+				)
+				.with_description("Number of nodes in table's Merkle tree")
+				.init(),
 			_merkle_todo_len: meter
 				.u64_value_observer(
 					"table.merkle_updater_todo_queue_length",

src/table/queue.rs

@@ -0,0 +1,81 @@
+use std::sync::Arc;
+use std::time::Duration;
+
+use async_trait::async_trait;
+use tokio::select;
+use tokio::sync::watch;
+
+use garage_util::background::*;
+use garage_util::error::Error;
+
+use crate::replication::*;
+use crate::schema::*;
+use crate::table::*;
+
+const BATCH_SIZE: usize = 100;
+
+pub(crate) struct InsertQueueWorker<F, R>(pub(crate) Arc<Table<F, R>>)
+where
+	F: TableSchema + 'static,
+	R: TableReplication + 'static;
+
+#[async_trait]
+impl<F, R> Worker for InsertQueueWorker<F, R>
+where
+	F: TableSchema + 'static,
+	R: TableReplication + 'static,
+{
+	fn name(&self) -> String {
+		format!("{} queue", F::TABLE_NAME)
+	}
+
+	fn status(&self) -> WorkerStatus {
+		WorkerStatus {
+			queue_length: Some(self.0.data.insert_queue.len().unwrap_or(0) as u64),
+			..Default::default()
+		}
+	}
+
+	async fn work(&mut self, _must_exit: &mut watch::Receiver<bool>) -> Result<WorkerState, Error> {
+		let mut kv_pairs = vec![];
+		let mut values = vec![];
+
+		for entry_kv in self.0.data.insert_queue.iter()? {
+			let (k, v) = entry_kv?;
+
+			values.push(self.0.data.decode_entry(&v)?);
+			kv_pairs.push((k, v));
+
+			if kv_pairs.len() > BATCH_SIZE {
+				break;
+			}
+		}
+
+		if kv_pairs.is_empty() {
+			return Ok(WorkerState::Idle);
+		}
+
+		self.0.insert_many(values).await?;
+
+		self.0.data.insert_queue.db().transaction(|mut tx| {
+			for (k, v) in kv_pairs.iter() {
+				if let Some(v2) = tx.get(&self.0.data.insert_queue, k)? {
+					if &v2 == v {
+						tx.remove(&self.0.data.insert_queue, k)?;
+					}
+				}
+			}
+			Ok(())
+		})?;
+
+		Ok(WorkerState::Busy)
+	}
+
+	async fn wait_for_work(&mut self) -> WorkerState {
+		select! {
+			_ = tokio::time::sleep(Duration::from_secs(600)) => (),
+			_ = self.0.data.insert_queue_notify.notified() => (),
+		}
+		WorkerState::Busy
+	}
+}

src/table/replication/mod.rs

@@ -1,10 +1,8 @@
 mod parameters;
 
 mod fullcopy;
-mod mode;
 mod sharded;
 
 pub use fullcopy::TableFullReplication;
-pub use mode::ReplicationMode;
 pub use parameters::*;
 pub use sharded::TableShardedReplication;

src/table/sync.rs

@@ -2,6 +2,7 @@ use std::collections::VecDeque;
 use std::sync::Arc;
 use std::time::{Duration, Instant};
 
+use arc_swap::ArcSwapOption;
 use async_trait::async_trait;
 use futures_util::stream::*;
 use opentelemetry::KeyValue;
@@ -13,7 +14,7 @@ use tokio::sync::{mpsc, watch};
 
 use garage_util::background::*;
 use garage_util::data::*;
-use garage_util::error::Error;
+use garage_util::error::{Error, OkOrMessage};
 
 use garage_rpc::ring::*;
 use garage_rpc::system::System;
@@ -32,7 +33,7 @@ pub struct TableSyncer<F: TableSchema + 'static, R: TableReplication + 'static>
 	data: Arc<TableData<F, R>>,
 	merkle: Arc<MerkleUpdater<F, R>>,
 
-	add_full_sync_tx: mpsc::UnboundedSender<()>,
+	add_full_sync_tx: ArcSwapOption<mpsc::UnboundedSender<()>>,
 	endpoint: Arc<Endpoint<SyncRpc, Self>>,
 }
 
@@ -65,7 +66,7 @@ where
 	F: TableSchema + 'static,
 	R: TableReplication + 'static,
 {
-	pub(crate) fn launch(
+	pub(crate) fn new(
 		system: Arc<System>,
 		data: Arc<TableData<F, R>>,
 		merkle: Arc<MerkleUpdater<F, R>>,
@@ -74,34 +75,40 @@ where
 			.netapp
 			.endpoint(format!("garage_table/sync.rs/Rpc:{}", F::TABLE_NAME));
 
-		let (add_full_sync_tx, add_full_sync_rx) = mpsc::unbounded_channel();
-
 		let syncer = Arc::new(Self {
-			system: system.clone(),
+			system,
 			data,
 			merkle,
-			add_full_sync_tx,
+			add_full_sync_tx: ArcSwapOption::new(None),
 			endpoint,
 		});
-
 		syncer.endpoint.set_handler(syncer.clone());
 
-		system.background.spawn_worker(SyncWorker {
-			syncer: syncer.clone(),
-			ring_recv: system.ring.clone(),
-			ring: system.ring.borrow().clone(),
-			add_full_sync_rx,
-			todo: vec![],
-			next_full_sync: Instant::now() + Duration::from_secs(20),
-		});
-
 		syncer
 	}
 
-	pub fn add_full_sync(&self) {
-		if self.add_full_sync_tx.send(()).is_err() {
-			error!("({}) Could not add full sync", F::TABLE_NAME);
-		}
+	pub(crate) fn spawn_workers(self: &Arc<Self>, bg: &BackgroundRunner) {
+		let (add_full_sync_tx, add_full_sync_rx) = mpsc::unbounded_channel();
+		self.add_full_sync_tx
+			.store(Some(Arc::new(add_full_sync_tx)));
+
+		bg.spawn_worker(SyncWorker {
+			syncer: self.clone(),
+			ring_recv: self.system.ring.clone(),
+			ring: self.system.ring.borrow().clone(),
+			add_full_sync_rx,
+			todo: vec![],
+			next_full_sync: Instant::now() + Duration::from_secs(20),
+		});
+	}
+
+	pub fn add_full_sync(&self) -> Result<(), Error> {
+		let tx = self.add_full_sync_tx.load();
+		let tx = tx
+			.as_ref()
+			.ok_or_message("table sync worker is not running")?;
+		tx.send(()).ok_or_message("send error")?;
+		Ok(())
 	}
 
 	// ----
@@ -570,12 +577,10 @@ impl<F: TableSchema + 'static, R: TableReplication + 'static> Worker for SyncWor
 		format!("{} sync", F::TABLE_NAME)
 	}
 
-	fn info(&self) -> Option<String> {
-		let l = self.todo.len();
-		if l > 0 {
-			Some(format!("{} partitions remaining", l))
-		} else {
-			None
+	fn status(&self) -> WorkerStatus {
+		WorkerStatus {
+			queue_length: Some(self.todo.len() as u64),
+			..Default::default()
 		}
 	}
 
@@ -588,10 +593,7 @@ impl<F: TableSchema + 'static, R: TableReplication + 'static> Worker for SyncWor
 		}
 	}
 
-	async fn wait_for_work(&mut self, must_exit: &watch::Receiver<bool>) -> WorkerState {
-		if *must_exit.borrow() {
-			return WorkerState::Done;
-		}
+	async fn wait_for_work(&mut self) -> WorkerState {
 		select! {
 			s = self.add_full_sync_rx.recv() => {
 				if let Some(()) = s {

src/table/table.rs

@@ -14,6 +14,7 @@ use opentelemetry::{
 
 use garage_db as db;
 
+use garage_util::background::BackgroundRunner;
 use garage_util::data::*;
 use garage_util::error::Error;
 use garage_util::metrics::RecordDuration;
@@ -25,6 +26,7 @@ use crate::crdt::Crdt;
 use crate::data::*;
 use crate::gc::*;
 use crate::merkle::*;
+use crate::queue::InsertQueueWorker;
 use crate::replication::*;
 use crate::schema::*;
 use crate::sync::*;
@@ -35,6 +37,7 @@ pub struct Table<F: TableSchema + 'static, R: TableReplication + 'static> {
 	pub data: Arc<TableData<F, R>>,
 	pub merkle_updater: Arc<MerkleUpdater<F, R>>,
 	pub syncer: Arc<TableSyncer<F, R>>,
+	gc: Arc<TableGc<F, R>>,
 	endpoint: Arc<Endpoint<TableRpc<F>, Self>>,
 }
 
@@ -75,15 +78,16 @@ where
 
 		let data = TableData::new(system.clone(), instance, replication, db);
 
-		let merkle_updater = MerkleUpdater::launch(&system.background, data.clone());
+		let merkle_updater = MerkleUpdater::new(data.clone());
 
-		let syncer = TableSyncer::launch(system.clone(), data.clone(), merkle_updater.clone());
-		TableGc::launch(system.clone(), data.clone());
+		let syncer = TableSyncer::new(system.clone(), data.clone(), merkle_updater.clone());
+		let gc = TableGc::new(system.clone(), data.clone());
 
 		let table = Arc::new(Self {
 			system,
 			data,
 			merkle_updater,
+			gc,
 			syncer,
 			endpoint,
 		});
@@ -93,6 +97,13 @@ where
 		table
 	}
 
+	pub fn spawn_workers(self: &Arc<Self>, bg: &BackgroundRunner) {
+		self.merkle_updater.spawn_workers(bg);
+		self.syncer.spawn_workers(bg);
+		self.gc.spawn_workers(bg);
+		bg.spawn_worker(InsertQueueWorker(self.clone()));
+	}
+
 	pub async fn insert(&self, e: &F::E) -> Result<(), Error> {
 		let tracer = opentelemetry::global::tracer("garage_table");
 		let span = tracer.start(format!("{} insert", F::TABLE_NAME));
@@ -128,6 +139,11 @@ where
 		Ok(())
 	}
 
+	/// Insert item locally
+	pub fn queue_insert(&self, tx: &mut db::Transaction, e: &F::E) -> db::TxResult<(), Error> {
+		self.data.queue_insert(tx, e)
+	}
+
 	pub async fn insert_many<I, IE>(&self, entries: I) -> Result<(), Error>
 	where
 		I: IntoIterator<Item = IE> + Send + Sync,
@@ -259,9 +275,11 @@ where
 			if not_all_same {
 				let self2 = self.clone();
 				let ent2 = ret_entry.clone();
-				self.system
-					.background
-					.spawn_cancellable(async move { self2.repair_on_read(&who[..], ent2).await });
+				tokio::spawn(async move {
+					if let Err(e) = self2.repair_on_read(&who[..], ent2).await {
+						warn!("Error doing repair on read: {}", e);
+					}
+				});
 			}
 		}
 
@@ -358,11 +376,12 @@ where
 				.into_iter()
 				.map(|k| ret.get(&k).unwrap().clone())
 				.collect::<Vec<_>>();
-			self.system.background.spawn_cancellable(async move {
+			tokio::spawn(async move {
 				for v in to_repair {
-					self2.repair_on_read(&who[..], v).await?;
+					if let Err(e) = self2.repair_on_read(&who[..], v).await {
+						warn!("Error doing repair on read: {}", e);
+					}
 				}
-				Ok(())
 			});
 		}
 

src/table/util.rs

@@ -49,3 +49,9 @@ impl EnumerationOrder {
 		}
 	}
 }
+
+impl Default for EnumerationOrder {
+	fn default() -> Self {
+		EnumerationOrder::Forward
+	}
+}

src/util/background/job_worker.rs

@@ -1,48 +0,0 @@
-//! Job worker: a generic worker that just processes incoming
-//! jobs one by one
-
-use std::sync::Arc;
-
-use async_trait::async_trait;
-use tokio::sync::{mpsc, Mutex};
-
-use crate::background::worker::*;
-use crate::background::*;
-
-pub(crate) struct JobWorker {
-	pub(crate) index: usize,
-	pub(crate) job_chan: Arc<Mutex<mpsc::UnboundedReceiver<(Job, bool)>>>,
-	pub(crate) next_job: Option<Job>,
-}
-
-#[async_trait]
-impl Worker for JobWorker {
-	fn name(&self) -> String {
-		format!("Job worker #{}", self.index)
-	}
-
-	async fn work(&mut self, _must_exit: &mut watch::Receiver<bool>) -> Result<WorkerState, Error> {
-		match self.next_job.take() {
-			None => return Ok(WorkerState::Idle),
-			Some(job) => {
-				job.await?;
-				Ok(WorkerState::Busy)
-			}
-		}
-	}
-
-	async fn wait_for_work(&mut self, must_exit: &watch::Receiver<bool>) -> WorkerState {
-		loop {
-			match self.job_chan.lock().await.recv().await {
-				Some((job, cancellable)) => {
-					if cancellable && *must_exit.borrow() {
-						continue;
-					}
-					self.next_job = Some(job);
-					return WorkerState::Busy;
-				}
-				None => return WorkerState::Done,
-			}
-		}
-	}
-}

src/util/background/mod.rs

@@ -1,27 +1,18 @@
 //! Job runner for futures and async functions
 
-pub mod job_worker;
 pub mod worker;
 
-use core::future::Future;
-
 use std::collections::HashMap;
-use std::pin::Pin;
 use std::sync::Arc;
 
 use serde::{Deserialize, Serialize};
-use tokio::sync::{mpsc, watch, Mutex};
+use tokio::sync::{mpsc, watch};
 
-use crate::error::Error;
 use worker::WorkerProcessor;
 pub use worker::{Worker, WorkerState};
 
-pub(crate) type JobOutput = Result<(), Error>;
-pub(crate) type Job = Pin<Box<dyn Future<Output = JobOutput> + Send>>;
-
 /// Job runner for futures and async functions
 pub struct BackgroundRunner {
-	send_job: mpsc::UnboundedSender<(Job, bool)>,
 	send_worker: mpsc::UnboundedSender<Box<dyn Worker>>,
 	worker_info: Arc<std::sync::Mutex<HashMap<usize, WorkerInfo>>>,
 }
@@ -29,19 +20,27 @@ pub struct BackgroundRunner {
 #[derive(Clone, Serialize, Deserialize, Debug)]
 pub struct WorkerInfo {
 	pub name: String,
-	pub info: Option<String>,
+	pub status: WorkerStatus,
 	pub state: WorkerState,
 	pub errors: usize,
 	pub consecutive_errors: usize,
 	pub last_error: Option<(String, u64)>,
 }
 
+/// WorkerStatus is a struct returned by the worker with a bunch of canonical
+/// fields to indicate their status to CLI users. All fields are optional.
+#[derive(Clone, Serialize, Deserialize, Debug, Default)]
+pub struct WorkerStatus {
+	pub tranquility: Option<u32>,
+	pub progress: Option<String>,
+	pub queue_length: Option<u64>,
+	pub persistent_errors: Option<u64>,
+	pub freeform: Vec<String>,
+}
+
 impl BackgroundRunner {
 	/// Create a new BackgroundRunner
-	pub fn new(
-		n_runners: usize,
-		stop_signal: watch::Receiver<bool>,
-	) -> (Arc<Self>, tokio::task::JoinHandle<()>) {
+	pub fn new(stop_signal: watch::Receiver<bool>) -> (Arc<Self>, tokio::task::JoinHandle<()>) {
 		let (send_worker, worker_out) = mpsc::unbounded_channel::<Box<dyn Worker>>();
 
 		let worker_info = Arc::new(std::sync::Mutex::new(HashMap::new()));
@@ -52,24 +51,7 @@ impl BackgroundRunner {
 			worker_processor.run().await;
 		});
 
-		let (send_job, queue_out) = mpsc::unbounded_channel();
-		let queue_out = Arc::new(Mutex::new(queue_out));
-
-		for i in 0..n_runners {
-			let queue_out = queue_out.clone();
-
-			send_worker
-				.send(Box::new(job_worker::JobWorker {
-					index: i,
-					job_chan: queue_out.clone(),
-					next_job: None,
-				}))
-				.ok()
-				.unwrap();
-		}
-
 		let bgrunner = Arc::new(Self {
-			send_job,
 			send_worker,
 			worker_info,
 		});
@@ -80,31 +62,6 @@ impl BackgroundRunner {
 		self.worker_info.lock().unwrap().clone()
 	}
 
-	/// Spawn a task to be run in background
-	pub fn spawn<T>(&self, job: T)
-	where
-		T: Future<Output = JobOutput> + Send + 'static,
-	{
-		let boxed: Job = Box::pin(job);
-		self.send_job
-			.send((boxed, false))
-			.ok()
-			.expect("Could not put job in queue");
-	}
-
-	/// Spawn a task to be run in background. It may get discarded before running if spawned while
-	/// the runner is stopping
-	pub fn spawn_cancellable<T>(&self, job: T)
-	where
-		T: Future<Output = JobOutput> + Send + 'static,
-	{
-		let boxed: Job = Box::pin(job);
-		self.send_job
-			.send((boxed, true))
-			.ok()
-			.expect("Could not put job in queue");
-	}
-
 	pub fn spawn_worker<W>(&self, worker: W)
 	where
 		W: Worker + 'static,

src/util/background/worker.rs

@@ -1,6 +1,6 @@
 use std::collections::HashMap;
 use std::sync::Arc;
-use std::time::{Duration, Instant};
+use std::time::Duration;
 
 use async_trait::async_trait;
 use futures::future::*;
@@ -10,10 +10,14 @@ use serde::{Deserialize, Serialize};
 use tokio::select;
 use tokio::sync::{mpsc, watch};
 
-use crate::background::WorkerInfo;
+use crate::background::{WorkerInfo, WorkerStatus};
 use crate::error::Error;
 use crate::time::now_msec;
 
+// All workers that haven't exited for this time after an exit signal was recieved
+// will be interrupted in the middle of whatever they are doing.
+const EXIT_DEADLINE: Duration = Duration::from_secs(8);
+
 #[derive(PartialEq, Copy, Clone, Serialize, Deserialize, Debug)]
 pub enum WorkerState {
 	Busy,
@@ -26,7 +30,7 @@ impl std::fmt::Display for WorkerState {
 	fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
 		match self {
 			WorkerState::Busy => write!(f, "Busy"),
-			WorkerState::Throttled(t) => write!(f, "Thr:{:.3}", t),
+			WorkerState::Throttled(_) => write!(f, "Busy*"),
 			WorkerState::Idle => write!(f, "Idle"),
 			WorkerState::Done => write!(f, "Done"),
 		}
@@ -37,8 +41,8 @@ impl std::fmt::Display for WorkerState {
 pub trait Worker: Send {
 	fn name(&self) -> String;
 
-	fn info(&self) -> Option<String> {
-		None
+	fn status(&self) -> WorkerStatus {
+		Default::default()
 	}
 
 	/// Work: do a basic unit of work, if one is available (otherwise, should return
@@ -50,10 +54,8 @@ pub trait Worker: Send {
 	async fn work(&mut self, must_exit: &mut watch::Receiver<bool>) -> Result<WorkerState, Error>;
 
 	/// Wait for work: await for some task to become available.  This future can be interrupted in
-	/// the middle for any reason.  This future doesn't have to await on must_exit.changed(), we
-	/// are doing it for you.  Therefore it only receives a read refernce to must_exit which allows
-	/// it to check if we are exiting.
-	async fn wait_for_work(&mut self, must_exit: &watch::Receiver<bool>) -> WorkerState;
+	/// the middle for any reason, for example if an interrupt signal was recieved.
+	async fn wait_for_work(&mut self) -> WorkerState;
 }
 
 pub(crate) struct WorkerProcessor {
@@ -93,11 +95,9 @@ impl WorkerProcessor {
 						let task_id = next_task_id;
 						next_task_id += 1;
 						let stop_signal = self.stop_signal.clone();
-						let stop_signal_worker = self.stop_signal.clone();
 						let mut worker = WorkerHandler {
 								task_id,
 								stop_signal,
-								stop_signal_worker,
 								worker: new_worker,
 								state: WorkerState::Busy,
 								errors: 0,
@@ -119,7 +119,7 @@ impl WorkerProcessor {
 						match wi.get_mut(&worker.task_id) {
 							Some(i) => {
 								i.state = worker.state;
-								i.info = worker.worker.info();
+								i.status = worker.worker.status();
 								i.errors = worker.errors;
 								i.consecutive_errors = worker.consecutive_errors;
 								if worker.last_error.is_some() {
@@ -130,7 +130,7 @@ impl WorkerProcessor {
 								wi.insert(worker.task_id, WorkerInfo {
 									name: worker.worker.name(),
 									state: worker.state,
-									info: worker.worker.info(),
+									status: worker.worker.status(),
 									errors: worker.errors,
 									consecutive_errors: worker.consecutive_errors,
 									last_error: worker.last_error.take(),
@@ -153,26 +153,14 @@ impl WorkerProcessor {
 		}
 
 		// We are exiting, drain everything
-		let drain_half_time = Instant::now() + Duration::from_secs(5);
 		let drain_everything = async move {
-			while let Some(mut worker) = workers.next().await {
-				if worker.state == WorkerState::Done {
-					info!(
-						"Worker {} (TID {}) exited",
-						worker.worker.name(),
-						worker.task_id
-					);
-				} else if Instant::now() > drain_half_time {
-					warn!("Worker {} (TID {}) interrupted between two iterations in state {:?} (this should be fine)", worker.worker.name(), worker.task_id, worker.state);
-				} else {
-					workers.push(
-						async move {
-							worker.step().await;
-							worker
-						}
-						.boxed(),
-					);
-				}
+			while let Some(worker) = workers.next().await {
+				info!(
+					"Worker {} (TID {}) exited (last state: {:?})",
+					worker.worker.name(),
+					worker.task_id,
+					worker.state
+				);
 			}
 		};
 
@@ -180,7 +168,7 @@ impl WorkerProcessor {
 			_ = drain_everything => {
 				info!("All workers exited peacefully \\o/");
 			}
-			_ = tokio::time::sleep(Duration::from_secs(9)) => {
+			_ = tokio::time::sleep(EXIT_DEADLINE) => {
 				error!("Some workers could not exit in time, we are cancelling some things in the middle");
 			}
 		}
@@ -190,7 +178,6 @@ impl WorkerProcessor {
 struct WorkerHandler {
 	task_id: usize,
 	stop_signal: watch::Receiver<bool>,
-	stop_signal_worker: watch::Receiver<bool>,
 	worker: Box<dyn Worker>,
 	state: WorkerState,
 	errors: usize,
@@ -225,33 +212,19 @@ impl WorkerHandler {
 			},
 			WorkerState::Throttled(delay) => {
 				// Sleep for given delay and go back to busy state
-				if !*self.stop_signal.borrow() {
-					select! {
-						_ = tokio::time::sleep(Duration::from_secs_f32(delay)) => (),
-						_ = self.stop_signal.changed() => (),
+				select! {
+					_ = tokio::time::sleep(Duration::from_secs_f32(delay)) => {
+						self.state = WorkerState::Busy;
 					}
+					_ = self.stop_signal.changed() => (),
 				}
-				self.state = WorkerState::Busy;
 			}
 			WorkerState::Idle => {
-				if *self.stop_signal.borrow() {
-					select! {
-						new_st = self.worker.wait_for_work(&self.stop_signal_worker) => {
-							self.state = new_st;
-						}
-						_ = tokio::time::sleep(Duration::from_secs(1)) => {
-							// stay in Idle state
-						}
-					}
-				} else {
-					select! {
-						new_st = self.worker.wait_for_work(&self.stop_signal_worker) => {
-							self.state = new_st;
-						}
-						_ = self.stop_signal.changed() => {
-							// stay in Idle state
-						}
+				select! {
+					new_st = self.worker.wait_for_work() => {
+						self.state = new_st;
 					}
+					_ = self.stop_signal.changed() => (),
 				}
 			}
 			WorkerState::Done => unreachable!(),

src/util/formater.rs

@@ -1,4 +1,4 @@
-pub fn format_table(data: Vec<String>) {
+pub fn format_table_to_string(data: Vec<String>) -> String {
 	let data = data
 		.iter()
 		.map(|s| s.split('\t').collect::<Vec<_>>())
@@ -24,5 +24,9 @@ pub fn format_table(data: Vec<String>) {
 		out.push('\n');
 	}
 
-	print!("{}", out);
+	out
+}
+
+pub fn format_table(data: Vec<String>) {
+	print!("{}", format_table_to_string(data));
 }