Improve clarity

.drone.yml

@@ -36,7 +36,7 @@ steps:
       path: /etc/nix
     commands:
       - nix-shell --arg release false --run "cargo fmt -- --check"
-      - nix-shell --arg release false --run "cargo clippy -- --deny warnings"
+      - nix-shell --arg release false --run "cargo clippy -- --allow clippy::needless_borrow --allow  clippy::needless-return --deny warnings"
 
   - name: build
     image: nixpkgs/nix:nixos-21.05
@@ -276,115 +276,115 @@ trigger:
 node:
   nix: 1
 
-# ---
+---
-# kind: pipeline
+kind: pipeline
-# type: docker
+type: docker
-# name: release-linux-i686
+name: release-linux-i686
-# 
+
-# volumes:
+volumes:
-# - name: nix_store
+- name: nix_store
-#   host:
+  host:
-#     path: /var/lib/drone/nix
+    path: /var/lib/drone/nix
-# - name: nix_config
+- name: nix_config
-#   temp: {}
+  temp: {}
-# 
+
-# environment:
+environment:
-#   TARGET: i686-unknown-linux-musl
+  TARGET: i686-unknown-linux-musl
-# 
+
-# steps:
+steps:
-#   - name: setup nix
+  - name: setup nix
-#     image: nixpkgs/nix:nixos-21.05
+    image: nixpkgs/nix:nixos-21.05
-#     volumes:
+    volumes:
-#     - name: nix_store
+    - name: nix_store
-#       path: /nix
+      path: /nix
-#     - name: nix_config
+    - name: nix_config
-#       path: /etc/nix
+      path: /etc/nix
-#     commands:
+    commands:
-#       - cp nix/nix.conf /etc/nix/nix.conf
+      - cp nix/nix.conf /etc/nix/nix.conf
-#       - nix-build --no-build-output --no-out-link shell.nix -A inputDerivation
+      - nix-build --no-build-output --no-out-link shell.nix -A inputDerivation
-# 
+
-#   - name: build
+  - name: build
-#     image: nixpkgs/nix:nixos-21.05
+    image: nixpkgs/nix:nixos-21.05
-#     volumes:
+    volumes:
-#     - name: nix_store
+    - name: nix_store
-#       path: /nix
+      path: /nix
-#     - name: nix_config
+    - name: nix_config
-#       path: /etc/nix
+      path: /etc/nix
-#     commands:
+    commands:
-#       - nix-build --no-build-output --argstr target $TARGET --arg release true --argstr git_version $DRONE_COMMIT
+      - nix-build --no-build-output --argstr target $TARGET --arg release true --argstr git_version $DRONE_COMMIT
-# 
+
-#   - name: integration
+  - name: integration
-#     image: nixpkgs/nix:nixos-21.05
+    image: nixpkgs/nix:nixos-21.05
-#     volumes:
+    volumes:
-#     - name: nix_store
+    - name: nix_store
-#       path: /nix
+      path: /nix
-#     - name: nix_config
+    - name: nix_config
-#       path: /etc/nix
+      path: /etc/nix
-#     commands:
+    commands:
-#       - nix-shell --run ./script/test-smoke.sh || (cat /tmp/garage.log; false)
+      - nix-shell --run ./script/test-smoke.sh || (cat /tmp/garage.log; false)
-# 
+
-#   - name: update cache
+  - name: update cache
-#     image: nixpkgs/nix:nixos-21.05
+    image: nixpkgs/nix:nixos-21.05
-#     environment:
+    environment:
-#       AWS_ACCESS_KEY_ID:
+      AWS_ACCESS_KEY_ID:
-#         from_secret: cache_aws_access_key_id
+        from_secret: cache_aws_access_key_id
-#       AWS_SECRET_ACCESS_KEY:
+      AWS_SECRET_ACCESS_KEY:
-#         from_secret: cache_aws_secret_access_key
+        from_secret: cache_aws_secret_access_key
-#       NIX_PRIV_KEY:
+      NIX_PRIV_KEY:
-#         from_secret: nix_priv_key
+        from_secret: nix_priv_key
-#     volumes:
+    volumes:
-#     - name: nix_store
+    - name: nix_store
-#       path: /nix
+      path: /nix
-#     - name: nix_config
+    - name: nix_config
-#       path: /etc/nix
+      path: /etc/nix
-#     commands:
+    commands:
-#       - (umask 377 && echo $NIX_PRIV_KEY > /etc/nix/signing-key.sec)
+      - (umask 377 && echo $NIX_PRIV_KEY > /etc/nix/signing-key.sec)
-#       - |
+      - |
-#         nix copy --to 's3://nix?endpoint=garage.deuxfleurs.fr&region=garage&secret-key=/etc/nix/signing-key.sec' \
+        nix copy --to 's3://nix?endpoint=garage.deuxfleurs.fr&region=garage&secret-key=/etc/nix/signing-key.sec' \
-#           $(nix-store -qR --include-outputs \
+          $(nix-store -qR --include-outputs \
-#             $(nix-instantiate --argstr target $TARGET --arg release true))
+            $(nix-instantiate --argstr target $TARGET --arg release true))
-# 
+
-#   - name: push static binary
+  - name: push static binary
-#     image: nixpkgs/nix:nixos-21.05
+    image: nixpkgs/nix:nixos-21.05
-#     volumes:
+    volumes:
-#     - name: nix_store
+    - name: nix_store
-#       path: /nix
+      path: /nix
-#     - name: nix_config
+    - name: nix_config
-#       path: /etc/nix
+      path: /etc/nix
-#     environment:
+    environment:
-#       AWS_ACCESS_KEY_ID:
+      AWS_ACCESS_KEY_ID:
-#         from_secret: garagehq_aws_access_key_id
+        from_secret: garagehq_aws_access_key_id
-#       AWS_SECRET_ACCESS_KEY:
+      AWS_SECRET_ACCESS_KEY:
-#         from_secret: garagehq_aws_secret_access_key
+        from_secret: garagehq_aws_secret_access_key
-#     commands:
+    commands:
-#       - nix-shell --arg rust false --arg integration false --run "to_s3"
+      - nix-shell --arg rust false --arg integration false --run "to_s3"
-# 
+
-#   - name: docker build and publish
+  - name: docker build and publish
-#     image: nixpkgs/nix:nixos-21.05
+    image: nixpkgs/nix:nixos-21.05
-#     volumes:
+    volumes:
-#     - name: nix_store
+    - name: nix_store
-#       path: /nix
+      path: /nix
-#     - name: nix_config
+    - name: nix_config
-#       path: /etc/nix
+      path: /etc/nix
-#     environment:
+    environment:
-#       DOCKER_AUTH:
+      DOCKER_AUTH:
-#         from_secret: docker_auth
+        from_secret: docker_auth
-#       DOCKER_PLATFORM: "linux/386"
+      DOCKER_PLATFORM: "linux/386"
-#       CONTAINER_NAME: "dxflrs/386_garage"
+      CONTAINER_NAME: "dxflrs/386_garage"
-#       HOME: "/kaniko"
+      HOME: "/kaniko"
-#     commands:
+    commands:
-#       - mkdir -p /kaniko/.docker
+      - mkdir -p /kaniko/.docker
-#       - echo $DOCKER_AUTH > /kaniko/.docker/config.json
+      - echo $DOCKER_AUTH > /kaniko/.docker/config.json
-#       - export CONTAINER_TAG=${DRONE_TAG:-$DRONE_COMMIT}
+      - export CONTAINER_TAG=${DRONE_TAG:-$DRONE_COMMIT}
-#       - nix-shell --arg rust false --arg integration false --run "to_docker"
+      - nix-shell --arg rust false --arg integration false --run "to_docker"
-# 
+
-# trigger:
+trigger:
-#   event:
+  event:
-#   - promote
+  - promote
-#   - cron
+  - cron
-# 
+
-# node:
+node:
-#   nix: 1
+  nix: 1
 
 ---
 kind: pipeline
@@ -486,105 +486,105 @@ trigger:
 node:
   nix: 1
 
-# ---
+---
-# kind: pipeline
+kind: pipeline
-# type: docker
+type: docker
-# name: release-linux-armv6l
+name: release-linux-armv6l
-# 
+
-# volumes:
+volumes:
-# - name: nix_store
+- name: nix_store
-#   host:
+  host:
-#     path: /var/lib/drone/nix
+    path: /var/lib/drone/nix
-# - name: nix_config
+- name: nix_config
-#   temp: {}
+  temp: {}
-# 
+
-# environment:
+environment:
-#   TARGET: armv6l-unknown-linux-musleabihf
+  TARGET: armv6l-unknown-linux-musleabihf
-# 
+
-# steps:
+steps:
-#   - name: setup nix
+  - name: setup nix
-#     image: nixpkgs/nix:nixos-21.05
+    image: nixpkgs/nix:nixos-21.05
-#     volumes:
+    volumes:
-#     - name: nix_store
+    - name: nix_store
-#       path: /nix
+      path: /nix
-#     - name: nix_config
+    - name: nix_config
-#       path: /etc/nix
+      path: /etc/nix
-#     commands:
+    commands:
-#       - cp nix/nix.conf /etc/nix/nix.conf
+      - cp nix/nix.conf /etc/nix/nix.conf
-#       - nix-build --no-build-output --no-out-link --arg rust false --arg integration false -A inputDerivation
+      - nix-build --no-build-output --no-out-link --arg rust false --arg integration false -A inputDerivation
-# 
+
-#   - name: build
+  - name: build
-#     image: nixpkgs/nix:nixos-21.05
+    image: nixpkgs/nix:nixos-21.05
-#     volumes:
+    volumes:
-#     - name: nix_store
+    - name: nix_store
-#       path: /nix
+      path: /nix
-#     - name: nix_config
+    - name: nix_config
-#       path: /etc/nix
+      path: /etc/nix
-#     commands:
+    commands:
-#       - nix-build --no-build-output --argstr target $TARGET --arg release true --argstr git_version $DRONE_COMMIT
+      - nix-build --no-build-output --argstr target $TARGET --arg release true --argstr git_version $DRONE_COMMIT
-# 
+
-#   - name: update cache
+  - name: update cache
-#     image: nixpkgs/nix:nixos-21.05
+    image: nixpkgs/nix:nixos-21.05
-#     environment:
+    environment:
-#       AWS_ACCESS_KEY_ID:
+      AWS_ACCESS_KEY_ID:
-#         from_secret: cache_aws_access_key_id
+        from_secret: cache_aws_access_key_id
-#       AWS_SECRET_ACCESS_KEY:
+      AWS_SECRET_ACCESS_KEY:
-#         from_secret: cache_aws_secret_access_key
+        from_secret: cache_aws_secret_access_key
-#       NIX_PRIV_KEY:
+      NIX_PRIV_KEY:
-#         from_secret: nix_priv_key
+        from_secret: nix_priv_key
-#     volumes:
+    volumes:
-#     - name: nix_store
+    - name: nix_store
-#       path: /nix
+      path: /nix
-#     - name: nix_config
+    - name: nix_config
-#       path: /etc/nix
+      path: /etc/nix
-#     commands:
+    commands:
-#       - (umask 377 && echo $NIX_PRIV_KEY > /etc/nix/signing-key.sec)
+      - (umask 377 && echo $NIX_PRIV_KEY > /etc/nix/signing-key.sec)
-#       - |
+      - |
-#         nix copy --to 's3://nix?endpoint=garage.deuxfleurs.fr&region=garage&secret-key=/etc/nix/signing-key.sec' \
+        nix copy --to 's3://nix?endpoint=garage.deuxfleurs.fr&region=garage&secret-key=/etc/nix/signing-key.sec' \
-#           $(nix-store -qR --include-outputs \
+          $(nix-store -qR --include-outputs \
-#             $(nix-instantiate --argstr target $TARGET --arg release true))
+            $(nix-instantiate --argstr target $TARGET --arg release true))
-# 
+
-#   - name: push static binary
+  - name: push static binary
-#     image: nixpkgs/nix:nixos-21.05
+    image: nixpkgs/nix:nixos-21.05
-#     volumes:
+    volumes:
-#     - name: nix_store
+    - name: nix_store
-#       path: /nix
+      path: /nix
-#     - name: nix_config
+    - name: nix_config
-#       path: /etc/nix
+      path: /etc/nix
-#     environment:
+    environment:
-#       AWS_ACCESS_KEY_ID:
+      AWS_ACCESS_KEY_ID:
-#         from_secret: garagehq_aws_access_key_id
+        from_secret: garagehq_aws_access_key_id
-#       AWS_SECRET_ACCESS_KEY:
+      AWS_SECRET_ACCESS_KEY:
-#         from_secret: garagehq_aws_secret_access_key
+        from_secret: garagehq_aws_secret_access_key
-#     commands:
+    commands:
-#       - nix-shell --arg integration false --arg rust false --run "to_s3"
+      - nix-shell --arg integration false --arg rust false --run "to_s3"
-# 
+
-#   - name: docker build and publish
+  - name: docker build and publish
-#     image: nixpkgs/nix:nixos-21.05
+    image: nixpkgs/nix:nixos-21.05
-#     volumes:
+    volumes:
-#     - name: nix_store
+    - name: nix_store
-#       path: /nix
+      path: /nix
-#     - name: nix_config
+    - name: nix_config
-#       path: /etc/nix
+      path: /etc/nix
-#     environment:
+    environment:
-#       DOCKER_AUTH:
+      DOCKER_AUTH:
-#         from_secret: docker_auth
+        from_secret: docker_auth
-#       DOCKER_PLATFORM: "linux/arm"
+      DOCKER_PLATFORM: "linux/arm"
-#       CONTAINER_NAME: "dxflrs/arm_garage"
+      CONTAINER_NAME: "dxflrs/arm_garage"
-#       HOME: "/kaniko"
+      HOME: "/kaniko"
-#     commands:
+    commands:
-#       - mkdir -p /kaniko/.docker
+      - mkdir -p /kaniko/.docker
-#       - echo $DOCKER_AUTH > /kaniko/.docker/config.json
+      - echo $DOCKER_AUTH > /kaniko/.docker/config.json
-#       - export CONTAINER_TAG=${DRONE_TAG:-$DRONE_COMMIT}
+      - export CONTAINER_TAG=${DRONE_TAG:-$DRONE_COMMIT}
-#       - nix-shell --arg rust false --arg integration false --run "to_docker"
+      - nix-shell --arg rust false --arg integration false --run "to_docker"
-# 
+
-# trigger:
+trigger:
-#   event:
+  event:
-#   - promote
+  - promote
-#   - cron
+  - cron
-# 
+
-# node:
+node:
-#   nix: 1
+  nix: 1
 
 ---
 kind: pipeline
@@ -613,9 +613,9 @@ steps:
 
 depends_on:
   - release-linux-x86_64
-  #- release-linux-i686
+  - release-linux-i686
   - release-linux-aarch64
-  #- release-linux-armv6l
+  - release-linux-armv6l
 
 trigger:
   event:

Cargo.lock

@@ -874,8 +874,7 @@ dependencies = [
 [[package]]
 name = "netapp"
 version = "0.3.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
+source = "git+https://git.deuxfleurs.fr/lx/netapp#57327f10e2536a89004f3a1def83ed16243c1a3e"
-checksum = "d202ff23744499a8e5d89a3199ca7b9a8aef9980a23b1b09d6508a8db2a118d2"
 dependencies = [
  "arc-swap",
  "async-trait",

Cargo.nix

@@ -246,7 +246,7 @@ in
     registry = "registry+https://github.com/rust-lang/crates.io-index";
     src = fetchCratesIo { inherit name version; sha256 = "95059428f66df56b63431fdb4e1947ed2190586af5c5a8a8b71122bdf5a7f469"; };
     dependencies = {
-      ${ if hostPlatform.config == "aarch64-apple-darwin" || hostPlatform.parsed.cpu.name == "aarch64" && hostPlatform.parsed.kernel.name == "linux" then "libc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.103" { inherit profileName; };
+      ${ if hostPlatform.parsed.cpu.name == "aarch64" && hostPlatform.parsed.kernel.name == "linux" || hostPlatform.config == "aarch64-apple-darwin" then "libc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.103" { inherit profileName; };
     };
   });
   
@@ -626,7 +626,7 @@ in
       hex = rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; };
       sodiumoxide = rustPackages."registry+https://github.com/rust-lang/crates.io-index".kuska-sodiumoxide."0.2.5-0" { inherit profileName; };
       log = rustPackages."registry+https://github.com/rust-lang/crates.io-index".log."0.4.14" { inherit profileName; };
-      netapp = rustPackages."registry+https://github.com/rust-lang/crates.io-index".netapp."0.3.0" { inherit profileName; };
+      netapp = rustPackages."git+https://git.deuxfleurs.fr/lx/netapp".netapp."0.3.0" { inherit profileName; };
       pretty_env_logger = rustPackages."registry+https://github.com/rust-lang/crates.io-index".pretty_env_logger."0.4.0" { inherit profileName; };
       rand = rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand."0.8.4" { inherit profileName; };
       rmp_serde = rustPackages."registry+https://github.com/rust-lang/crates.io-index".rmp-serde."0.15.5" { inherit profileName; };
@@ -687,7 +687,7 @@ in
       garage_util = rustPackages."unknown".garage_util."0.4.0" { inherit profileName; };
       hex = rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; };
       log = rustPackages."registry+https://github.com/rust-lang/crates.io-index".log."0.4.14" { inherit profileName; };
-      netapp = rustPackages."registry+https://github.com/rust-lang/crates.io-index".netapp."0.3.0" { inherit profileName; };
+      netapp = rustPackages."git+https://git.deuxfleurs.fr/lx/netapp".netapp."0.3.0" { inherit profileName; };
       rand = rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand."0.8.4" { inherit profileName; };
       rmp_serde = rustPackages."registry+https://github.com/rust-lang/crates.io-index".rmp-serde."0.15.5" { inherit profileName; };
       serde = rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde."1.0.130" { inherit profileName; };
@@ -714,7 +714,7 @@ in
       hyper = rustPackages."registry+https://github.com/rust-lang/crates.io-index".hyper."0.14.13" { inherit profileName; };
       sodiumoxide = rustPackages."registry+https://github.com/rust-lang/crates.io-index".kuska-sodiumoxide."0.2.5-0" { inherit profileName; };
       log = rustPackages."registry+https://github.com/rust-lang/crates.io-index".log."0.4.14" { inherit profileName; };
-      netapp = rustPackages."registry+https://github.com/rust-lang/crates.io-index".netapp."0.3.0" { inherit profileName; };
+      netapp = rustPackages."git+https://git.deuxfleurs.fr/lx/netapp".netapp."0.3.0" { inherit profileName; };
       rand = rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand."0.8.4" { inherit profileName; };
       rmp_serde = rustPackages."registry+https://github.com/rust-lang/crates.io-index".rmp-serde."0.15.5" { inherit profileName; };
       serde = rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde."1.0.130" { inherit profileName; };
@@ -761,7 +761,7 @@ in
       http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.5" { inherit profileName; };
       hyper = rustPackages."registry+https://github.com/rust-lang/crates.io-index".hyper."0.14.13" { inherit profileName; };
       log = rustPackages."registry+https://github.com/rust-lang/crates.io-index".log."0.4.14" { inherit profileName; };
-      netapp = rustPackages."registry+https://github.com/rust-lang/crates.io-index".netapp."0.3.0" { inherit profileName; };
+      netapp = rustPackages."git+https://git.deuxfleurs.fr/lx/netapp".netapp."0.3.0" { inherit profileName; };
       rand = rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand."0.8.4" { inherit profileName; };
       rmp_serde = rustPackages."registry+https://github.com/rust-lang/crates.io-index".rmp-serde."0.15.5" { inherit profileName; };
       serde = rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde."1.0.130" { inherit profileName; };
@@ -1228,11 +1228,15 @@ in
     };
   });
   
-  "registry+https://github.com/rust-lang/crates.io-index".netapp."0.3.0" = overridableMkRustCrate (profileName: rec {
+  "git+https://git.deuxfleurs.fr/lx/netapp".netapp."0.3.0" = overridableMkRustCrate (profileName: rec {
     name = "netapp";
     version = "0.3.0";
-    registry = "registry+https://github.com/rust-lang/crates.io-index";
+    registry = "git+https://git.deuxfleurs.fr/lx/netapp";
-    src = fetchCratesIo { inherit name version; sha256 = "d202ff23744499a8e5d89a3199ca7b9a8aef9980a23b1b09d6508a8db2a118d2"; };
+    src = fetchCrateGit {
+      url = https://git.deuxfleurs.fr/lx/netapp;
+      name = "netapp";
+      version = "0.3.0";
+      rev = "57327f10e2536a89004f3a1def83ed16243c1a3e";};
     features = builtins.concatLists [
       [ "default" ]
     ];

doc/book/src/SUMMARY.md

@@ -30,4 +30,3 @@
 
 - [Working Documents](./working_documents/index.md)
   - [Load Balancing Data](./working_documents/load_balancing.md)
-  - [Migrating from 0.3 to 0.4](./working_documents/migration_04.md)

doc/book/src/cookbook/real_world.md

@@ -11,7 +11,7 @@ to get familiar with Garage's command line and usage patterns.
 
 ## Prerequisites
 
-To run a real-world deployment, make sure the following conditions are met:
+To run a real-world deployment, make sure you the following conditions are met:
 
 - You have at least three machines with sufficient storage space available.
 
@@ -52,6 +52,7 @@ For example:
 sudo docker pull lxpz/garage_amd64:v0.4.0
 ```
 
+
 ## Deploying and configuring Garage
 
 On each machine, we will have a similar setup,
@@ -78,6 +79,10 @@ rpc_bind_addr = "[::]:3901"
 rpc_public_addr = "<this node's public IP>:3901"
 rpc_secret = "<RPC secret>"
 
+bootstrap_peers = [
+    # We will fill this in later
+]
+
 [s3_api]
 s3_region = "garage"
 api_bind_addr = "[::]:3900"
@@ -97,6 +102,33 @@ Check the following for your configuration files:
 - Make sure `rpc_secret` is the same value on all nodes. It should be a 32-bytes hex-encoded secret key.
   You can generate such a key with `openssl rand -hex 32`.
 
+You will now have to run `garage node-id` on all nodes to generate node keys.
+This will print keys as follows:
+
+```bash
+Mercury$ garage node-id
+563e1ac825ee3323aa441e72c26d1030d6d4414aeb3dd25287c531e7fc2bc95d@[fc00:1::1]:3901
+
+Venus$ garage node-id
+86f0f26ae4afbd59aaf9cfb059eefac844951efd5b8caeec0d53f4ed6c85f332@[fc00:1::2]:3901
+
+etc.
+```
+
+You can then add these nodes to the `bootstrap_peers` list of at least one of your nodes:
+
+```toml
+bootstrap_peers = [
+    "563e1ac825ee3323aa441e72c26d1030d6d4414aeb3dd25287c531e7fc2bc95d@[fc00:1::1]:3901",
+    "86f0f26ae4afbd59aaf9cfb059eefac844951efd5b8caeec0d53f4ed6c85f332@[fc00:1::2]:3901",
+    ...
+]
+```
+
+Check the [configuration file reference documentation](../reference_manual/configuration.md)
+to learn more about all available configuration options.
+
+
 ## Starting Garage using Docker
 
 On each machine, you can run the daemon with:
@@ -122,6 +154,7 @@ but please check the relase notes before doing so!
 To upgrade, simply stop and remove this container and
 start again the command with a new version of Garage.
 
+
 ## Controling the daemon
 
 The `garage` binary has two purposes:
@@ -133,51 +166,11 @@ If your configuration file is at `/etc/garage.toml`, the `garage` binary should
 
 You can test your `garage` CLI utility by running a simple command such as:
 
-```bash
+```
 garage status
 ```
 
-At this point, nodes are not yet talking to one another.
+You should get something like that as result:
-Your output should therefore look like follows:
-
-```
-Mercury$ garage node-id
-==== HEALTHY NODES ====
-ID                  Hostname  Address           Tag                   Zone  Capacity
-563e1ac825ee3323…   Mercury   [fc00:1::1]:3901  NO ROLE ASSIGNED
-```
-
-
-## Connecting nodes together
-
-When your Garage nodes first start, they will generate a local node identifier
-(based on a public/private key pair).
-
-To obtain the node identifier of a node, once it is generated,
-run `garage node-id`.
-This will print keys as follows:
-
-```bash
-Mercury$ garage node-id
-563e1ac825ee3323aa441e72c26d1030d6d4414aeb3dd25287c531e7fc2bc95d@[fc00:1::1]:3901
-
-Venus$ garage node-id
-86f0f26ae4afbd59aaf9cfb059eefac844951efd5b8caeec0d53f4ed6c85f332@[fc00:1::2]:3901
-
-etc.
-```
-
-You can then instruct nodes to connect to one another as follows:
-
-```bash
-# Instruct Venus to connect to Mercury (this will establish communication both ways)
-Venus$ garage node connect 563e1ac825ee3323aa441e72c26d1030d6d4414aeb3dd25287c531e7fc2bc95d@[fc00:1::1]:3901
-```
-
-You don't nead to instruct all node to connect to all other nodes:
-nodes will discover one another transitively.
-
-Now if your run `garage status` on any node, you should have an output that looks as follows:
 
 ```
 ==== HEALTHY NODES ====
@@ -188,13 +181,13 @@ ID                  Hostname  Address           Tag                   Zone  Capa
 212f7572f0c89da9…   Mars      [fc00:F::1]:3901  NO ROLE ASSIGNED
 ```
 
-## Giving roles to nodes
+
+## Configuring a cluster
 
 We will now inform Garage of the disk space available on each node of the cluster
 as well as the zone (e.g. datacenter) in which each machine is located.
 
-For our example, we will suppose we have the following infrastructure
+For our example, we will suppose we have the following infrastructure (Capacity, Identifier and Datacenter are specific values to Garage described in the following):
-(Capacity, Identifier and Zone are specific values to Garage described in the following):
 
 | Location | Name    | Disk Space | `Capacity` | `Identifier` | `Zone` |
 |----------|---------|------------|------------|--------------|--------------|

doc/book/src/intro.md

@@ -5,8 +5,7 @@
 </p>
 
 <p align="center" style="text-align:center;">
-	[ <a href="https://garagehq.deuxfleurs.fr/_releases.html">Download</a>
+	[ <a href="https://git.deuxfleurs.fr/Deuxfleurs/garage">Git repository</a>
-	| <a href="https://git.deuxfleurs.fr/Deuxfleurs/garage">Git repository</a>
 	| <a href="https://matrix.to/#/%23garage:deuxfleurs.fr">Matrix channel</a>
 	| <a href="https://drone.deuxfleurs.fr/Deuxfleurs/garage">Drone CI</a>
 	]

doc/book/src/working_documents/migration_04.md

@@ -1,105 +0,0 @@
-# Migrating from 0.3 to 0.4
-
-**Migrating from 0.3 to 0.4 is unsupported. This document is only intended to
-document the process internally for the Deuxfleurs cluster where we have to do
-it. Do not try it yourself, you will lose your data and we will not help you.**
-
-**Migrating from 0.2 to 0.4 will break everything for sure. Never try it.**
-
-The internal data format of Garage hasn't changed much between 0.3 and 0.4.
-The Sled database is still the same, and the data directory as well.
-
-The following has changed, all in the meta directory:
-
-- `node_id` in 0.3 contains the identifier of the current node. In 0.4, this
-  file does nothing and should be deleted. It is replaced by `node_key` (the
-  secret key) and `node_key.pub` (the associated public key). A node's
-  identifier on the ring is its public key.
-
-- `peer_info` in 0.3 contains the list of peers saved automatically by Garage.
-  The format has changed and it is now stored in `peer_list` (`peer_info`
-  should be deleted).
-
-When migrating, all node identifiers will change. This also means that the
-affectation of data partitions on the ring will change, and lots of data will
-have to be rebalanced.
-
-- If your cluster has only 3 nodes, all nodes store everything, therefore nothing has to be rebalanced.
-
-- If your cluster has only 4 nodes, for any partition there will always be at
-  least 2 nodes that stored data before that still store it after. Therefore
-  the migration should in theory be transparent and Garage should continue to
-  work during the rebalance.
-
-- If your cluster has 5 or more nodes, data will disappear during the
-  migration. Do not migrate (fortunately we don't have this scenario at
-  Deuxfleurs), or if you do, make Garage unavailable until things stabilize
-  (disable web and api access).
-
-
-The migration steps are as follows:
-
-1. Prepare a new configuration file for 0.4. For each node, point to the same
-   meta and data directories as Garage 0.3. Basically, the things that change
-   are the following:
-
-  - No more `rpc_tls` section
-  - You have to generate a shared `rpc_secret` and put it in all config files
-  - `bootstrap_peers` has a different syntax as it has to contain node keys.
-    Leave it empty and use `garage node-id` and `garage node connect` instead (new features of 0.4)
-  - put the publicly accessible RPC address of your node in `rpc_public_addr` if possible (its optional but recommended)
-  - If you are using Consul, change the `consul_service_name` to NOT be the name advertised by Nomad.
-    Now Garage is responsible for advertising its own service itself.
-
-2. Disable api and web access for some time (Garage does not support disabling
-   these endpoints but you can change the port number or stop your reverse
-   proxy for instance).
-
-3. Do `garage repair -a --yes tables` and `garage repair -a --yes blocks`,
-   check the logs and check that all data seems to be synced correctly between
-   nodes.
-
-4. Save somewhere the output of `garage status`. We will need this to remember
-   how to reconfigure nodes in 0.4.
-
-5. Turn off Garage 0.3
-
-6. Backup metadata folders if you can (i.e. if you have space to do it
-   somewhere). Backuping data folders could also be usefull but that's much
-   harder to do. If your filesystem supports snapshots, this could be a good
-   time to use them.
-
-7. Turn on Garage 0.4
-
-8. At this point, running `garage status` should indicate that all nodes of the
-   previous cluster are "unavailable". The nodes have new identifiers that
-   should appear in healthy nodes once they can talk to one another (use
-   `garage node connect` if necessary`). They should have NO ROLE ASSIGNED at
-   the moment.
-
-9. Prepare a script with several `garage node configure` commands that replace
-   each of the v0.3 node ID with the corresponding v0.4 node ID, with the same
-   zone/tag/capacity. For example if your node `drosera` had identifier `c24e`
-   before and now has identifier `789a`, and it was configured with capacity
-   `2` in zone `dc1`, put the following command in your script:
-
-```bash
-garage node configure 789a -z dc1 -c 2 -t drosera --replace c24e
-```
-
-10. Run your reconfiguration script. Check that the new output of `garage
-	status` contains the correct node IDs with the correct values for capacity
-	and zone. Old nodes should no longer be mentioned.
-
-11. If your cluster has 4 nodes or less, and you are feeling adventurous, you
-	can reenable Web and API access now. Things will probably work.
-
-12. Garage might already be resyncing stuff. Issue a `garage repair -a --yes
-	tables` and `garage repair -a --yes blocks` to force it to do so.
-
-13. Wait for resyncing activity to stop in the logs. Do steps 12 and 13 two or
-	three times, until you see that when you issue the repair commands, nothing
-	gets resynced any longer.
-
-14. Your upgraded cluster should be in a working state. Re-enable API and Web
-	access and check that everything went well.

src/api/api_server.rs

@@ -39,7 +39,7 @@ pub async fn run_api_server(
 		}
 	});
 
-	let server = Server::bind(addr).serve(service);
+	let server = Server::bind(&addr).serve(service);
 
 	let graceful = server.with_graceful_shutdown(shutdown_signal);
 	info!("API server listening on http://{}", addr);
@@ -88,8 +88,8 @@ async fn handler_inner(garage: Arc<Garage>, req: Request<Body>) -> Result<Respon
 
 	let (bucket, key) = parse_bucket_key(&path)?;
 	let allowed = match req.method() {
-		&Method::HEAD | &Method::GET => api_key.allow_read(bucket),
+		&Method::HEAD | &Method::GET => api_key.allow_read(&bucket),
-		_ => api_key.allow_write(bucket),
+		_ => api_key.allow_write(&bucket),
 	};
 	if !allowed {
 		return Err(Error::Forbidden(
@@ -109,11 +109,11 @@ async fn handler_inner(garage: Arc<Garage>, req: Request<Body>) -> Result<Respon
 		match *req.method() {
 			Method::HEAD => {
 				// HeadObject query
-				Ok(handle_head(garage, &req, bucket, key).await?)
+				Ok(handle_head(garage, &req, &bucket, &key).await?)
 			}
 			Method::GET => {
 				// GetObject query
-				Ok(handle_get(garage, &req, bucket, key).await?)
+				Ok(handle_get(garage, &req, &bucket, &key).await?)
 			}
 			Method::PUT => {
 				if params.contains_key(&"partnumber".to_string())
@@ -125,8 +125,8 @@ async fn handler_inner(garage: Arc<Garage>, req: Request<Body>) -> Result<Respon
 					Ok(handle_put_part(
 						garage,
 						req,
-						bucket,
+						&bucket,
-						key,
+						&key,
 						part_number,
 						upload_id,
 						content_sha256,
@@ -136,43 +136,46 @@ async fn handler_inner(garage: Arc<Garage>, req: Request<Body>) -> Result<Respon
 					// CopyObject query
 					let copy_source = req.headers().get("x-amz-copy-source").unwrap().to_str()?;
 					let copy_source =
-						percent_encoding::percent_decode_str(copy_source).decode_utf8()?;
+						percent_encoding::percent_decode_str(&copy_source).decode_utf8()?;
 					let (source_bucket, source_key) = parse_bucket_key(&copy_source)?;
-					if !api_key.allow_read(source_bucket) {
+					if !api_key.allow_read(&source_bucket) {
 						return Err(Error::Forbidden(format!(
 							"Reading from bucket {} not allowed for this key",
 							source_bucket
 						)));
 					}
 					let source_key = source_key.ok_or_bad_request("No source key specified")?;
-					Ok(handle_copy(garage, &req, bucket, key, source_bucket, source_key).await?)
+					Ok(
+						handle_copy(garage, &req, &bucket, &key, &source_bucket, &source_key)
+							.await?,
+					)
 				} else {
 					// PutObject query
-					Ok(handle_put(garage, req, bucket, key, content_sha256).await?)
+					Ok(handle_put(garage, req, &bucket, &key, content_sha256).await?)
 				}
 			}
 			Method::DELETE => {
 				if params.contains_key(&"uploadid".to_string()) {
 					// AbortMultipartUpload query
 					let upload_id = params.get("uploadid").unwrap();
-					Ok(handle_abort_multipart_upload(garage, bucket, key, upload_id).await?)
+					Ok(handle_abort_multipart_upload(garage, &bucket, &key, upload_id).await?)
 				} else {
 					// DeleteObject query
-					Ok(handle_delete(garage, bucket, key).await?)
+					Ok(handle_delete(garage, &bucket, &key).await?)
 				}
 			}
 			Method::POST => {
 				if params.contains_key(&"uploads".to_string()) {
 					// CreateMultipartUpload call
-					Ok(handle_create_multipart_upload(garage, &req, bucket, key).await?)
+					Ok(handle_create_multipart_upload(garage, &req, &bucket, &key).await?)
 				} else if params.contains_key(&"uploadid".to_string()) {
 					// CompleteMultipartUpload call
 					let upload_id = params.get("uploadid").unwrap();
 					Ok(handle_complete_multipart_upload(
 						garage,
 						req,
-						bucket,
+						&bucket,
-						key,
+						&key,
 						upload_id,
 						content_sha256,
 					)

src/api/s3_delete.rs

@@ -55,8 +55,7 @@ async fn handle_delete_internal(
 	);
 
 	garage.object_table.insert(&object).await?;
-
+	return Ok((deleted_version, version_uuid));
-	Ok((deleted_version, version_uuid))
 }
 
 pub async fn handle_delete(
@@ -83,7 +82,7 @@ pub async fn handle_delete_objects(
 	let body = hyper::body::to_bytes(req.into_body()).await?;
 	verify_signed_content(content_sha256, &body[..])?;
 
-	let cmd_xml = roxmltree::Document::parse(std::str::from_utf8(&body)?)?;
+	let cmd_xml = roxmltree::Document::parse(&std::str::from_utf8(&body)?)?;
 	let cmd = parse_delete_objects_xml(&cmd_xml).ok_or_bad_request("Invalid delete XML query")?;
 
 	let mut ret_deleted = Vec::new();

src/api/s3_get.rs

@@ -106,12 +106,12 @@ pub async fn handle_head(
 		_ => unreachable!(),
 	};
 
-	if let Some(cached) = try_answer_cached(version, version_meta, req) {
+	if let Some(cached) = try_answer_cached(&version, version_meta, req) {
 		return Ok(cached);
 	}
 
 	let body: Body = Body::empty();
-	let response = object_headers(version, version_meta)
+	let response = object_headers(&version, version_meta)
 		.header("Content-Length", format!("{}", version_meta.size))
 		.status(StatusCode::OK)
 		.body(body)
@@ -149,7 +149,7 @@ pub async fn handle_get(
 		ObjectVersionData::FirstBlock(meta, _) => meta,
 	};
 
-	if let Some(cached) = try_answer_cached(last_v, last_v_meta, req) {
+	if let Some(cached) = try_answer_cached(&last_v, last_v_meta, req) {
 		return Ok(cached);
 	}
 
@@ -179,7 +179,7 @@ pub async fn handle_get(
 		.await;
 	}
 
-	let resp_builder = object_headers(last_v, last_v_meta)
+	let resp_builder = object_headers(&last_v, last_v_meta)
 		.header("Content-Length", format!("{}", last_v_meta.size))
 		.status(StatusCode::OK);
 
@@ -190,7 +190,7 @@ pub async fn handle_get(
 			Ok(resp_builder.body(body)?)
 		}
 		ObjectVersionData::FirstBlock(_, first_block_hash) => {
-			let read_first_block = garage.block_manager.rpc_get_block(first_block_hash);
+			let read_first_block = garage.block_manager.rpc_get_block(&first_block_hash);
 			let get_next_blocks = garage.version_table.get(&last_v.uuid, &EmptyKey);
 
 			let (first_block, version) = futures::try_join!(read_first_block, get_next_blocks)?;

src/api/s3_put.rs

@@ -193,8 +193,8 @@ async fn read_and_put_blocks(
 
 	let mut next_offset = first_block.len();
 	let mut put_curr_version_block = put_block_meta(
-		garage,
+		&garage,
-		version,
+		&version,
 		part_number,
 		0,
 		first_block_hash,
@@ -213,8 +213,8 @@ async fn read_and_put_blocks(
 			let block_hash = blake2sum(&block[..]);
 			let block_len = block.len();
 			put_curr_version_block = put_block_meta(
-				garage,
+				&garage,
-				version,
+				&version,
 				part_number,
 				next_offset as u64,
 				block_hash,
@@ -437,7 +437,7 @@ pub async fn handle_complete_multipart_upload(
 	let body = hyper::body::to_bytes(req.into_body()).await?;
 	verify_signed_content(content_sha256, &body[..])?;
 
-	let body_xml = roxmltree::Document::parse(std::str::from_utf8(&body)?)?;
+	let body_xml = roxmltree::Document::parse(&std::str::from_utf8(&body)?)?;
 	let body_list_of_parts = parse_complete_multpart_upload_body(&body_xml)
 		.ok_or_bad_request("Invalid CompleteMultipartUpload XML")?;
 	debug!(

src/api/signature.rs

@@ -70,7 +70,7 @@ pub async fn check_signature(
 	let canonical_request = canonical_request(
 		request.method(),
 		&request.uri().path().to_string(),
-		&canonical_query_string(request.uri()),
+		&canonical_query_string(&request.uri()),
 		&headers,
 		&authorization.signed_headers,
 		&authorization.content_sha256,
@@ -252,7 +252,7 @@ fn canonical_request(
 		method.as_str(),
 		url_path,
 		canonical_query_string,
-		&canonical_header_string(headers, signed_headers),
+		&canonical_header_string(&headers, signed_headers),
 		"",
 		signed_headers,
 		content_sha256,

src/garage/Cargo.toml

@@ -41,5 +41,4 @@ futures = "0.3"
 futures-util = "0.3"
 tokio = { version = "1.0", default-features = false, features = ["rt", "rt-multi-thread", "io-util", "net", "time", "macros", "sync", "signal", "fs"] }
 
-#netapp = { version = "0.3.0", git = "https://git.deuxfleurs.fr/lx/netapp" }
+netapp = { version = "0.3.0", git = "https://git.deuxfleurs.fr/lx/netapp" }
-netapp = "0.3.0"

src/garage/cli/cmd.rs

@@ -93,17 +93,14 @@ pub async fn cmd_status(rpc_cli: &Endpoint<SystemRpc, ()>, rpc_host: NodeID) ->
 		for adv in status.iter().filter(|adv| !adv.is_up) {
 			if let Some(cfg) = config.members.get(&adv.id) {
 				failed_nodes.push(format!(
-					"{id:?}\t{host}\t{addr}\t[{tag}]\t{zone}\t{capacity}\t{last_seen}",
+					"{id:?}\t{host}\t{addr}\t[{tag}]\t{zone}\t{capacity}\t{last_seen}s ago",
 					id = adv.id,
 					host = adv.status.hostname,
 					addr = adv.addr,
 					tag = cfg.tag,
 					zone = cfg.zone,
 					capacity = cfg.capacity_string(),
-					last_seen = adv
+					last_seen = "FIXME", // FIXME was (now_msec() - adv.last_seen) / 1000,
-						.last_seen_secs_ago
-						.map(|s| format!("{}s ago", s))
-						.unwrap_or_else(|| "never seen".into()),
 				));
 			}
 		}

src/garage/cli/init.rs

@@ -1,7 +1,5 @@
 use std::path::PathBuf;
 
-use log::warn;
-
 use garage_util::error::*;
 
 pub const READ_KEY_ERROR: &str = "Unable to read node key. It will be generated by your garage node the first time is it launched. Ensure that your garage node is currently running. (The node key is supposed to be stored in your metadata directory.)";
@@ -24,11 +22,11 @@ pub fn node_id_command(config_file: PathBuf, quiet: bool) -> Result<(), Error> {
 		println!("{}", idstr);
 
 		if !quiet {
-			warn!("WARNING: I don't know the public address to reach this node.");
+			eprintln!("WARNING: I don't know the public address to reach this node.");
-			warn!("In all of the instructions below, replace 127.0.0.1:{} by the appropriate address and port.", config.rpc_bind_addr.port());
+			eprintln!("In all of the instructions below, replace 127.0.0.1:3901 by the appropriate address and port.");
 		}
 
-		format!("{}@127.0.0.1:{}", idstr, config.rpc_bind_addr.port())
+		format!("{}@127.0.0.1:3901", idstr)
 	};
 
 	if !quiet {

src/garage/cli/util.rs

@@ -54,7 +54,7 @@ pub fn format_table(data: Vec<String>) {
 			out.push_str(col);
 			(0..col_len - col.chars().count() + 2).for_each(|_| out.push(' '));
 		}
-		out.push_str(row[row.len() - 1]);
+		out.push_str(&row[row.len() - 1]);
 		out.push('\n');
 	}
 

src/garage/main.rs

@@ -9,7 +9,6 @@ mod cli;
 mod repair;
 mod server;
 
-use std::net::SocketAddr;
 use std::path::PathBuf;
 
 use structopt::StructOpt;
@@ -47,9 +46,6 @@ struct Opt {
 
 #[tokio::main]
 async fn main() {
-	if std::env::var("RUST_LOG").is_err() {
-		std::env::set_var("RUST_LOG", "garage=info")
-	}
 	pretty_env_logger::init();
 	sodiumoxide::init().expect("Unable to init sodiumoxide");
 
@@ -103,23 +99,12 @@ async fn cli_command(opt: Opt) -> Result<(), Error> {
 	let (id, addr) = if let Some(h) = opt.rpc_host {
 		let (id, addrs) = parse_and_resolve_peer_addr(&h).ok_or_else(|| format!("Invalid RPC remote node identifier: {}. Expected format is <pubkey>@<IP or hostname>:<port>.", h))?;
 		(id, addrs[0])
-	} else {
+	} else if let Some(a) = config.as_ref().map(|c| c.rpc_public_addr).flatten() {
-		let node_id = garage_rpc::system::read_node_id(&config.as_ref().unwrap().metadata_dir)
+		let node_id = garage_rpc::system::read_node_id(&config.unwrap().metadata_dir)
 			.err_context(READ_KEY_ERROR)?;
-		if let Some(a) = config.as_ref().map(|c| c.rpc_public_addr).flatten() {
+		(node_id, a)
-			(node_id, a)
+	} else {
-		} else {
+		return Err(Error::Message("No RPC host provided".into()));
-			let default_addr = SocketAddr::new(
-				"127.0.0.1".parse().unwrap(),
-				config.as_ref().unwrap().rpc_bind_addr.port(),
-			);
-			warn!(
-				"Trying to contact Garage node at default address {}",
-				default_addr
-			);
-			warn!("If this doesn't work, consider adding rpc_public_addr in your config file or specifying the -h command line parameter.");
-			(node_id, default_addr)
-		}
 	};
 
 	// Connect to target host

src/model/Cargo.toml

@@ -33,5 +33,4 @@ futures = "0.3"
 futures-util = "0.3"
 tokio = { version = "1.0", default-features = false, features = ["rt", "rt-multi-thread", "io-util", "net", "time", "macros", "sync", "signal", "fs"] }
 
-#netapp = { version = "0.3.0", git = "https://git.deuxfleurs.fr/lx/netapp" }
+netapp = { version = "0.3.0", git = "https://git.deuxfleurs.fr/lx/netapp" }
-netapp = "0.3.0"

src/model/block.rs

@@ -27,7 +27,7 @@ use crate::garage::Garage;
 /// Size under which data will be stored inlined in database instead of as files
 pub const INLINE_THRESHOLD: usize = 3072;
 
-pub const BACKGROUND_WORKERS: u64 = 2;
+pub const BACKGROUND_WORKERS: u64 = 1;
 
 const BLOCK_RW_TIMEOUT: Duration = Duration::from_secs(42);
 const BLOCK_GC_TIMEOUT: Duration = Duration::from_secs(60);
@@ -70,8 +70,8 @@ pub struct BlockManager {
 	pub replication: TableShardedReplication,
 	/// Directory in which block are stored
 	pub data_dir: PathBuf,
-
+	/// Lock to prevent concurrent edition of the directory
-	mutation_lock: Mutex<BlockManagerLocked>,
+	pub data_dir_lock: Mutex<()>,
 
 	rc: sled::Tree,
 
@@ -83,11 +83,6 @@ pub struct BlockManager {
 	pub(crate) garage: ArcSwapOption<Garage>,
 }
 
-// This custom struct contains functions that must only be ran
-// when the lock is held. We ensure that it is the case by storing
-// it INSIDE a Mutex.
-struct BlockManagerLocked();
-
 impl BlockManager {
 	pub fn new(
 		db: &sled::Db,
@@ -107,12 +102,10 @@ impl BlockManager {
 			.netapp
 			.endpoint("garage_model/block.rs/Rpc".to_string());
 
-		let manager_locked = BlockManagerLocked();
-
 		let block_manager = Arc::new(Self {
 			replication,
 			data_dir,
-			mutation_lock: Mutex::new(manager_locked),
+			data_dir_lock: Mutex::new(()),
 			rc,
 			resync_queue,
 			resync_notify: Notify::new(),
@@ -125,144 +118,38 @@ impl BlockManager {
 		block_manager
 	}
 
-	// ---- Public interface ----
+	pub fn spawn_background_worker(self: Arc<Self>) {
-
+		// Launch 2 simultaneous workers for background resync loop preprocessing <= TODO actually this
-	/// Ask nodes that might have a block for it
+		// launches only one worker with current value of BACKGROUND_WORKERS
-	pub async fn rpc_get_block(&self, hash: &Hash) -> Result<Vec<u8>, Error> {
+		for i in 0..BACKGROUND_WORKERS {
-		let who = self.replication.read_nodes(hash);
+			let bm2 = self.clone();
-		let resps = self
+			let background = self.system.background.clone();
-			.system
+			tokio::spawn(async move {
-			.rpc
+				tokio::time::sleep(Duration::from_secs(10 * (i + 1))).await;
-			.try_call_many(
+				background.spawn_worker(format!("block resync worker {}", i), move |must_exit| {
-				&self.endpoint,
+					bm2.resync_loop(must_exit)
-				&who[..],
+				});
-				BlockRpc::GetBlock(*hash),
+			});
-				RequestStrategy::with_priority(PRIO_NORMAL)
-					.with_quorum(1)
-					.with_timeout(BLOCK_RW_TIMEOUT)
-					.interrupt_after_quorum(true),
-			)
-			.await?;
-
-		for resp in resps {
-			if let BlockRpc::PutBlock(msg) = resp {
-				return Ok(msg.data);
-			}
 		}
-		Err(Error::Message(format!(
-			"Unable to read block {:?}: no valid blocks returned",
-			hash
-		)))
 	}
 
-	/// Send block to nodes that should have it
-	pub async fn rpc_put_block(&self, hash: Hash, data: Vec<u8>) -> Result<(), Error> {
-		let who = self.replication.write_nodes(&hash);
-		self.system
-			.rpc
-			.try_call_many(
-				&self.endpoint,
-				&who[..],
-				BlockRpc::PutBlock(PutBlockMessage { hash, data }),
-				RequestStrategy::with_priority(PRIO_NORMAL)
-					.with_quorum(self.replication.write_quorum())
-					.with_timeout(BLOCK_RW_TIMEOUT),
-			)
-			.await?;
-		Ok(())
-	}
-
-	/// Launch the repair procedure on the data store
-	///
-	/// This will list all blocks locally present, as well as those
-	/// that are required because of refcount > 0, and will try
-	/// to fix any mismatch between the two.
-	pub async fn repair_data_store(&self, must_exit: &watch::Receiver<bool>) -> Result<(), Error> {
-		// 1. Repair blocks from RC table
-		let garage = self.garage.load_full().unwrap();
-		let mut last_hash = None;
-		for (i, entry) in garage.block_ref_table.data.store.iter().enumerate() {
-			let (_k, v_bytes) = entry?;
-			let block_ref = rmp_serde::decode::from_read_ref::<_, BlockRef>(v_bytes.as_ref())?;
-			if Some(&block_ref.block) == last_hash.as_ref() {
-				continue;
-			}
-			if !block_ref.deleted.get() {
-				last_hash = Some(block_ref.block);
-				self.put_to_resync(&block_ref.block, Duration::from_secs(0))?;
-			}
-			if i & 0xFF == 0 && *must_exit.borrow() {
-				return Ok(());
-			}
-		}
-
-		// 2. Repair blocks actually on disk
-		self.repair_aux_read_dir_rec(&self.data_dir, must_exit)
-			.await?;
-
-		Ok(())
-	}
-
-	/// Get lenght of resync queue
-	pub fn resync_queue_len(&self) -> usize {
-		self.resync_queue.len()
-	}
-
-	/// Get number of items in the refcount table
-	pub fn rc_len(&self) -> usize {
-		self.rc.len()
-	}
-
-	//// ----- Managing the reference counter ----
-
-	/// Increment the number of time a block is used, putting it to resynchronization if it is
-	/// required, but not known
-	pub fn block_incref(&self, hash: &Hash) -> Result<(), Error> {
-		let old_rc = self.rc.fetch_and_update(&hash, |old| {
-			let old_v = old.map(u64_from_be_bytes).unwrap_or(0);
-			Some(u64::to_be_bytes(old_v + 1).to_vec())
-		})?;
-		let old_rc = old_rc.map(u64_from_be_bytes).unwrap_or(0);
-		if old_rc == 0 {
-			self.put_to_resync(hash, BLOCK_RW_TIMEOUT)?;
-		}
-		Ok(())
-	}
-
-	/// Decrement the number of time a block is used
-	pub fn block_decref(&self, hash: &Hash) -> Result<(), Error> {
-		let new_rc = self.rc.update_and_fetch(&hash, |old| {
-			let old_v = old.map(u64_from_be_bytes).unwrap_or(0);
-			if old_v > 1 {
-				Some(u64::to_be_bytes(old_v - 1).to_vec())
-			} else {
-				None
-			}
-		})?;
-		if new_rc.is_none() {
-			self.put_to_resync(hash, BLOCK_GC_TIMEOUT)?;
-		}
-		Ok(())
-	}
-
-	/// Read a block's reference count
-	pub fn get_block_rc(&self, hash: &Hash) -> Result<u64, Error> {
-		Ok(self
-			.rc
-			.get(hash.as_ref())?
-			.map(u64_from_be_bytes)
-			.unwrap_or(0))
-	}
-
-	// ---- Reading and writing blocks locally ----
-
 	/// Write a block to disk
 	async fn write_block(&self, hash: &Hash, data: &[u8]) -> Result<BlockRpc, Error> {
-		self.mutation_lock
+		let _lock = self.data_dir_lock.lock().await;
-			.lock()
+
-			.await
+		let mut path = self.block_dir(hash);
-			.write_block(hash, data, self)
+		fs::create_dir_all(&path).await?;
-			.await
+
+		path.push(hex::encode(hash));
+		if fs::metadata(&path).await.is_ok() {
+			return Ok(BlockRpc::Ok);
+		}
+
+		let mut f = fs::File::create(path).await?;
+		f.write_all(data).await?;
+		drop(f);
+
+		Ok(BlockRpc::Ok)
 	}
 
 	/// Read block from disk, verifying it's integrity
@@ -282,11 +169,15 @@ impl BlockManager {
 		drop(f);
 
 		if blake2sum(&data[..]) != *hash {
-			self.mutation_lock
+			let _lock = self.data_dir_lock.lock().await;
-				.lock()
+			warn!(
-				.await
+				"Block {:?} is corrupted. Renaming to .corrupted and resyncing.",
-				.move_block_to_corrupted(hash, self)
+				hash
-				.await?;
+			);
+			let mut path2 = path.clone();
+			path2.set_extension(".corrupted");
+			fs::rename(path, path2).await?;
+			self.put_to_resync(&hash, Duration::from_millis(0))?;
 			return Err(Error::CorruptData(*hash));
 		}
 
@@ -295,44 +186,60 @@ impl BlockManager {
 
 	/// Check if this node should have a block, but don't actually have it
 	async fn need_block(&self, hash: &Hash) -> Result<bool, Error> {
-		let BlockStatus { exists, needed } = self
+		let needed = self
-			.mutation_lock
+			.rc
-			.lock()
+			.get(hash.as_ref())?
-			.await
+			.map(|x| u64_from_be_bytes(x) > 0)
-			.check_block_status(hash, self)
+			.unwrap_or(false);
-			.await?;
+		if needed {
-		Ok(needed && !exists)
+			let path = self.block_path(hash);
+			let exists = fs::metadata(&path).await.is_ok();
+			Ok(!exists)
+		} else {
+			Ok(false)
+		}
 	}
 
-	/// Utility: gives the path of the directory in which a block should be found
 	fn block_dir(&self, hash: &Hash) -> PathBuf {
 		let mut path = self.data_dir.clone();
 		path.push(hex::encode(&hash.as_slice()[0..1]));
 		path.push(hex::encode(&hash.as_slice()[1..2]));
 		path
 	}
-
-	/// Utility: give the full path where a block should be found
 	fn block_path(&self, hash: &Hash) -> PathBuf {
 		let mut path = self.block_dir(hash);
 		path.push(hex::encode(hash.as_ref()));
 		path
 	}
 
-	// ---- Resync loop ----
+	/// Increment the number of time a block is used, putting it to resynchronization if it is
-
+	/// required, but not known
-	pub fn spawn_background_worker(self: Arc<Self>) {
+	pub fn block_incref(&self, hash: &Hash) -> Result<(), Error> {
-		// Launch 2 simultaneous workers for background resync loop preprocessing
+		let old_rc = self.rc.fetch_and_update(&hash, |old| {
-		for i in 0..BACKGROUND_WORKERS {
+			let old_v = old.map(u64_from_be_bytes).unwrap_or(0);
-			let bm2 = self.clone();
+			Some(u64::to_be_bytes(old_v + 1).to_vec())
-			let background = self.system.background.clone();
+		})?;
-			tokio::spawn(async move {
+		let old_rc = old_rc.map(u64_from_be_bytes).unwrap_or(0);
-				tokio::time::sleep(Duration::from_secs(10 * (i + 1))).await;
+		if old_rc == 0 {
-				background.spawn_worker(format!("block resync worker {}", i), move |must_exit| {
+			self.put_to_resync(&hash, BLOCK_RW_TIMEOUT)?;
-					bm2.resync_loop(must_exit)
-				});
-			});
 		}
+		Ok(())
+	}
+
+	/// Decrement the number of time a block is used
+	pub fn block_decref(&self, hash: &Hash) -> Result<(), Error> {
+		let new_rc = self.rc.update_and_fetch(&hash, |old| {
+			let old_v = old.map(u64_from_be_bytes).unwrap_or(0);
+			if old_v > 1 {
+				Some(u64::to_be_bytes(old_v - 1).to_vec())
+			} else {
+				None
+			}
+		})?;
+		if new_rc.is_none() {
+			self.put_to_resync(&hash, BLOCK_GC_TIMEOUT)?;
+		}
+		Ok(())
 	}
 
 	fn put_to_resync(&self, hash: &Hash, delay: Duration) -> Result<(), Error> {
@@ -389,12 +296,16 @@ impl BlockManager {
 	}
 
 	async fn resync_block(&self, hash: &Hash) -> Result<(), Error> {
-		let BlockStatus { exists, needed } = self
+		let lock = self.data_dir_lock.lock().await;
-			.mutation_lock
+
-			.lock()
+		let path = self.block_path(hash);
-			.await
+
-			.check_block_status(hash, self)
+		let exists = fs::metadata(&path).await.is_ok();
-			.await?;
+		let needed = self
+			.rc
+			.get(hash.as_ref())?
+			.map(|x| u64_from_be_bytes(x) > 0)
+			.unwrap_or(false);
 
 		if exists != needed {
 			info!(
@@ -406,7 +317,7 @@ impl BlockManager {
 		if exists && !needed {
 			trace!("Offloading block {:?}", hash);
 
-			let mut who = self.replication.write_nodes(hash);
+			let mut who = self.replication.write_nodes(&hash);
 			if who.len() < self.replication.write_quorum() {
 				return Err(Error::Message("Not trying to offload block because we don't have a quorum of nodes to write to".to_string()));
 			}
@@ -467,24 +378,95 @@ impl BlockManager {
 				who.len()
 			);
 
-			self.mutation_lock
+			fs::remove_file(path).await?;
-				.lock()
-				.await
-				.delete_if_unneeded(hash, self)
-				.await?;
 		}
 
 		if needed && !exists {
+			drop(lock);
+
 			// TODO find a way to not do this if they are sending it to us
 			// Let's suppose this isn't an issue for now with the BLOCK_RW_TIMEOUT delay
 			// between the RC being incremented and this part being called.
-			let block_data = self.rpc_get_block(hash).await?;
+			let block_data = self.rpc_get_block(&hash).await?;
 			self.write_block(hash, &block_data[..]).await?;
 		}
 
 		Ok(())
 	}
 
+	/// Ask nodes that might have a block for it
+	pub async fn rpc_get_block(&self, hash: &Hash) -> Result<Vec<u8>, Error> {
+		let who = self.replication.read_nodes(&hash);
+		let resps = self
+			.system
+			.rpc
+			.try_call_many(
+				&self.endpoint,
+				&who[..],
+				BlockRpc::GetBlock(*hash),
+				RequestStrategy::with_priority(PRIO_NORMAL)
+					.with_quorum(1)
+					.with_timeout(BLOCK_RW_TIMEOUT)
+					.interrupt_after_quorum(true),
+			)
+			.await?;
+
+		for resp in resps {
+			if let BlockRpc::PutBlock(msg) = resp {
+				return Ok(msg.data);
+			}
+		}
+		Err(Error::Message(format!(
+			"Unable to read block {:?}: no valid blocks returned",
+			hash
+		)))
+	}
+
+	/// Send block to nodes that should have it
+	pub async fn rpc_put_block(&self, hash: Hash, data: Vec<u8>) -> Result<(), Error> {
+		let who = self.replication.write_nodes(&hash);
+		self.system
+			.rpc
+			.try_call_many(
+				&self.endpoint,
+				&who[..],
+				BlockRpc::PutBlock(PutBlockMessage { hash, data }),
+				RequestStrategy::with_priority(PRIO_NORMAL)
+					.with_quorum(self.replication.write_quorum())
+					.with_timeout(BLOCK_RW_TIMEOUT),
+			)
+			.await?;
+		Ok(())
+	}
+
+	pub async fn repair_data_store(&self, must_exit: &watch::Receiver<bool>) -> Result<(), Error> {
+		// 1. Repair blocks from RC table
+		let garage = self.garage.load_full().unwrap();
+		let mut last_hash = None;
+		let mut i = 0usize;
+		for entry in garage.block_ref_table.data.store.iter() {
+			let (_k, v_bytes) = entry?;
+			let block_ref = rmp_serde::decode::from_read_ref::<_, BlockRef>(v_bytes.as_ref())?;
+			if Some(&block_ref.block) == last_hash.as_ref() {
+				continue;
+			}
+			if !block_ref.deleted.get() {
+				last_hash = Some(block_ref.block);
+				self.put_to_resync(&block_ref.block, Duration::from_secs(0))?;
+			}
+			i += 1;
+			if i & 0xFF == 0 && *must_exit.borrow() {
+				return Ok(());
+			}
+		}
+
+		// 2. Repair blocks actually on disk
+		self.repair_aux_read_dir_rec(&self.data_dir, must_exit)
+			.await?;
+
+		Ok(())
+	}
+
 	fn repair_aux_read_dir_rec<'a>(
 		&'a self,
 		path: &'a Path,
@@ -529,6 +511,15 @@ impl BlockManager {
 		}
 		.boxed()
 	}
+
+	/// Get lenght of resync queue
+	pub fn resync_queue_len(&self) -> usize {
+		self.resync_queue.len()
+	}
+
+	pub fn rc_len(&self) -> usize {
+		self.rc.len()
+	}
 }
 
 #[async_trait]
@@ -547,74 +538,6 @@ impl EndpointHandler<BlockRpc> for BlockManager {
 	}
 }
 
-struct BlockStatus {
-	exists: bool,
-	needed: bool,
-}
-
-impl BlockManagerLocked {
-	async fn check_block_status(
-		&self,
-		hash: &Hash,
-		mgr: &BlockManager,
-	) -> Result<BlockStatus, Error> {
-		let path = mgr.block_path(hash);
-
-		let exists = fs::metadata(&path).await.is_ok();
-		let needed = mgr.get_block_rc(hash)? > 0;
-
-		Ok(BlockStatus { exists, needed })
-	}
-
-	async fn write_block(
-		&self,
-		hash: &Hash,
-		data: &[u8],
-		mgr: &BlockManager,
-	) -> Result<BlockRpc, Error> {
-		let mut path = mgr.block_dir(hash);
-		fs::create_dir_all(&path).await?;
-
-		path.push(hex::encode(hash));
-		if fs::metadata(&path).await.is_ok() {
-			return Ok(BlockRpc::Ok);
-		}
-
-		let mut path2 = path.clone();
-		path2.set_extension("tmp");
-		let mut f = fs::File::create(&path2).await?;
-		f.write_all(data).await?;
-		drop(f);
-
-		fs::rename(path2, path).await?;
-
-		Ok(BlockRpc::Ok)
-	}
-
-	async fn move_block_to_corrupted(&self, hash: &Hash, mgr: &BlockManager) -> Result<(), Error> {
-		warn!(
-			"Block {:?} is corrupted. Renaming to .corrupted and resyncing.",
-			hash
-		);
-		let path = mgr.block_path(hash);
-		let mut path2 = path.clone();
-		path2.set_extension(".corrupted");
-		fs::rename(path, path2).await?;
-		mgr.put_to_resync(hash, Duration::from_millis(0))?;
-		Ok(())
-	}
-
-	async fn delete_if_unneeded(&self, hash: &Hash, mgr: &BlockManager) -> Result<(), Error> {
-		let BlockStatus { exists, needed } = self.check_block_status(hash, mgr).await?;
-
-		if exists && !needed {
-			let path = mgr.block_path(hash);
-			fs::remove_file(path).await?;
-		}
-		Ok(())
-	}
-}
-
 fn u64_from_be_bytes<T: AsRef<[u8]>>(bytes: T) -> u64 {
 	assert!(bytes.as_ref().len() == 8);
 	let mut x8 = [0u8; 8];

src/rpc/Cargo.toml

@@ -33,7 +33,6 @@ futures-util = "0.3"
 tokio = { version = "1.0", default-features = false, features = ["rt", "rt-multi-thread", "io-util", "net", "time", "macros", "sync", "signal", "fs"] }
 tokio-stream = { version = "0.1", features = ["net"] }
 
-#netapp = { version = "0.3.0", git = "https://git.deuxfleurs.fr/lx/netapp" }
+netapp = { version = "0.3.0", git = "https://git.deuxfleurs.fr/lx/netapp" }
-netapp = "0.3.0"
 hyper = { version = "0.14", features = ["client", "http1", "runtime", "tcp"] }
 

src/rpc/system.rs

@@ -4,7 +4,7 @@ use std::io::{Read, Write};
 use std::net::SocketAddr;
 use std::path::Path;
 use std::sync::{Arc, RwLock};
-use std::time::{Duration, Instant};
+use std::time::Duration;
 
 use arc_swap::ArcSwap;
 use async_trait::async_trait;
@@ -112,7 +112,6 @@ pub struct KnownNodeInfo {
 	pub id: Uuid,
 	pub addr: SocketAddr,
 	pub is_up: bool,
-	pub last_seen_secs_ago: Option<u64>,
 	pub status: NodeStatus,
 }
 
@@ -355,7 +354,6 @@ impl System {
 				id: n.id.into(),
 				addr: n.addr,
 				is_up: n.is_up(),
-				last_seen_secs_ago: n.last_seen.map(|t| (Instant::now() - t).as_secs()),
 				status: node_status
 					.get(&n.id.into())
 					.cloned()
@@ -543,7 +541,7 @@ impl EndpointHandler<SystemRpc> for System {
 			SystemRpc::Connect(node) => self.handle_connect(node).await,
 			SystemRpc::PullConfig => Ok(self.handle_pull_config()),
 			SystemRpc::AdvertiseStatus(adv) => self.handle_advertise_status(from.into(), adv).await,
-			SystemRpc::AdvertiseConfig(adv) => self.clone().handle_advertise_config(adv).await,
+			SystemRpc::AdvertiseConfig(adv) => self.clone().handle_advertise_config(&adv).await,
 			SystemRpc::GetKnownNodes => Ok(self.handle_get_known_nodes()),
 			_ => Err(Error::BadRpc("Unexpected RPC message".to_string())),
 		}

src/table/crdt/lww_map.rs

@@ -103,7 +103,7 @@ where
 	}
 	/// Get a reference to the value assigned to a key
 	pub fn get(&self, k: &K) -> Option<&V> {
-		match self.vals.binary_search_by(|(k2, _, _)| k2.cmp(k)) {
+		match self.vals.binary_search_by(|(k2, _, _)| k2.cmp(&k)) {
 			Ok(i) => Some(&self.vals[i].2),
 			Err(_) => None,
 		}
@@ -132,14 +132,14 @@ where
 {
 	fn merge(&mut self, other: &Self) {
 		for (k, ts2, v2) in other.vals.iter() {
-			match self.vals.binary_search_by(|(k2, _, _)| k2.cmp(k)) {
+			match self.vals.binary_search_by(|(k2, _, _)| k2.cmp(&k)) {
 				Ok(i) => {
 					let (_, ts1, _v1) = &self.vals[i];
 					if ts2 > ts1 {
 						self.vals[i].1 = *ts2;
 						self.vals[i].2 = v2.clone();
 					} else if ts1 == ts2 {
-						self.vals[i].2.merge(v2);
+						self.vals[i].2.merge(&v2);
 					}
 				}
 				Err(i) => {

src/table/crdt/map.rs

@@ -49,7 +49,7 @@ where
 
 	/// Get a reference to the value assigned to a key
 	pub fn get(&self, k: &K) -> Option<&V> {
-		match self.vals.binary_search_by(|(k2, _)| k2.cmp(k)) {
+		match self.vals.binary_search_by(|(k2, _)| k2.cmp(&k)) {
 			Ok(i) => Some(&self.vals[i].1),
 			Err(_) => None,
 		}
@@ -76,9 +76,9 @@ where
 {
 	fn merge(&mut self, other: &Self) {
 		for (k, v2) in other.vals.iter() {
-			match self.vals.binary_search_by(|(k2, _)| k2.cmp(k)) {
+			match self.vals.binary_search_by(|(k2, _)| k2.cmp(&k)) {
 				Ok(i) => {
-					self.vals[i].1.merge(v2);
+					self.vals[i].1.merge(&v2);
 				}
 				Err(i) => {
 					self.vals.insert(i, (k.clone(), v2.clone()));

src/table/merkle.rs

@@ -167,7 +167,7 @@ where
 		// Calculate an update to apply to this node
 		// This update is an Option<_>, so that it is None if the update is a no-op
 		// and we can thus skip recalculating and re-storing everything
-		let mutate = match self.read_node_txn(tx, key)? {
+		let mutate = match self.read_node_txn(tx, &key)? {
 			MerkleNode::Empty => new_vhash.map(|vhv| MerkleNode::Leaf(k.to_vec(), vhv)),
 			MerkleNode::Intermediate(mut children) => {
 				let key2 = key.next_key(khash);
@@ -270,7 +270,7 @@ where
 		};
 
 		if let Some(new_node) = mutate {
-			let hash = self.put_node_txn(tx, key, &new_node)?;
+			let hash = self.put_node_txn(tx, &key, &new_node)?;
 			Ok(Some(hash))
 		} else {
 			Ok(None)

src/table/replication/sharded.rs

@@ -27,7 +27,7 @@ pub struct TableShardedReplication {
 impl TableReplication for TableShardedReplication {
 	fn read_nodes(&self, hash: &Hash) -> Vec<Uuid> {
 		let ring = self.system.ring.borrow();
-		ring.get_nodes(hash, self.replication_factor)
+		ring.get_nodes(&hash, self.replication_factor)
 	}
 	fn read_quorum(&self) -> usize {
 		self.read_quorum
@@ -35,7 +35,7 @@ impl TableReplication for TableShardedReplication {
 
 	fn write_nodes(&self, hash: &Hash) -> Vec<Uuid> {
 		let ring = self.system.ring.borrow();
-		ring.get_nodes(hash, self.replication_factor)
+		ring.get_nodes(&hash, self.replication_factor)
 	}
 	fn write_quorum(&self) -> usize {
 		self.write_quorum

src/table/sync.rs

@@ -266,7 +266,7 @@ where
 				let nodes = self
 					.data
 					.replication
-					.write_nodes(begin)
+					.write_nodes(&begin)
 					.into_iter()
 					.collect::<Vec<_>>();
 				if nodes.contains(&self.system.id) {
@@ -530,7 +530,7 @@ where
 				Ok(SyncRpc::RootCkDifferent(hash != *h))
 			}
 			SyncRpc::GetNode(k) => {
-				let node = self.merkle.read_node(k)?;
+				let node = self.merkle.read_node(&k)?;
 				Ok(SyncRpc::Node(k.clone(), node))
 			}
 			SyncRpc::Items(items) => {

src/util/Cargo.toml

@@ -32,8 +32,6 @@ toml = "0.5"
 futures = "0.3"
 tokio = { version = "1.0", default-features = false, features = ["rt", "rt-multi-thread", "io-util", "net", "time", "macros", "sync", "signal", "fs"] }
 
-#netapp = { version = "0.3.0", git = "https://git.deuxfleurs.fr/lx/netapp" }
+netapp = { version = "0.3.0", git = "https://git.deuxfleurs.fr/lx/netapp" }
-netapp = "0.3.0"
-
 http = "0.2"
 hyper = "0.14"

src/web/web_server.rs

@@ -36,7 +36,7 @@ pub async fn run_web_server(
 		}
 	});
 
-	let server = Server::bind(addr).serve(service);
+	let server = Server::bind(&addr).serve(service);
 	let graceful = server.with_graceful_shutdown(shutdown_signal);
 	info!("Web server listening on http://{}", addr);
 
@@ -95,12 +95,12 @@ async fn serve_file(garage: Arc<Garage>, req: Request<Body>) -> Result<Response<
 	// Get path
 	let path = req.uri().path().to_string();
 	let index = &garage.config.s3_web.index;
-	let key = path_to_key(&path, index)?;
+	let key = path_to_key(&path, &index)?;
 
 	info!("Selected bucket: \"{}\", selected key: \"{}\"", bucket, key);
 
 	let res = match *req.method() {
-		Method::HEAD => handle_head(garage, &req, bucket, &key).await?,
+		Method::HEAD => handle_head(garage, &req, &bucket, &key).await?,
 		Method::GET => handle_get(garage, &req, bucket, &key).await?,
 		_ => return Err(Error::BadRequest("HTTP method not supported".to_string())),
 	};
@@ -173,7 +173,7 @@ fn host_to_bucket<'a>(host: &'a str, root: &str) -> &'a str {
 /// When a path ends with "/", we append the index name to match traditional web server behavior
 /// which is also AWS S3 behavior.
 fn path_to_key<'a>(path: &'a str, index: &str) -> Result<Cow<'a, str>, Error> {
-	let path_utf8 = percent_encoding::percent_decode_str(path).decode_utf8()?;
+	let path_utf8 = percent_encoding::percent_decode_str(&path).decode_utf8()?;
 
 	if !path_utf8.starts_with('/') {
 		return Err(Error::BadRequest(