update Cargo.nix

to reduce noise in logs

Cargo.lock

@@ -840,7 +840,7 @@ dependencies = [
  "http",
  "hyper",
  "kuska-sodiumoxide",
- "netapp 0.4.1",
+ "netapp 0.4.2",
  "pretty_env_logger",
  "rand 0.8.5",
  "rmp-serde 0.15.5",
@@ -975,7 +975,7 @@ dependencies = [
  "garage_table 0.7.0",
  "garage_util 0.7.0",
  "hex",
- "netapp 0.4.1",
+ "netapp 0.4.2",
  "opentelemetry",
  "rand 0.8.5",
  "rmp-serde 0.15.5",
@@ -1031,7 +1031,7 @@ dependencies = [
  "k8s-openapi",
  "kube",
  "kuska-sodiumoxide",
- "netapp 0.4.1",
+ "netapp 0.4.2",
  "openssl",
  "opentelemetry",
  "pnet_datalink",
@@ -1126,7 +1126,7 @@ dependencies = [
  "hex",
  "http",
  "hyper",
- "netapp 0.4.1",
+ "netapp 0.4.2",
  "opentelemetry",
  "rand 0.8.5",
  "rmp-serde 0.15.5",
@@ -1833,9 +1833,9 @@ dependencies = [
 
 [[package]]
 name = "netapp"
-version = "0.4.1"
+version = "0.4.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "aafa16daee87ca3251e6d2639c6bfb88c40590627459fd5d44034f60742bb5d1"
+checksum = "d1a19af9ad24e6cdb166e2c5882a7ff9326cab8d45c9d82379fa638c0cbc84df"
 dependencies = [
  "arc-swap",
  "async-trait",

Cargo.nix

@@ -1193,7 +1193,7 @@ in
       git_version = rustPackages."registry+https://github.com/rust-lang/crates.io-index".git-version."0.3.5" { inherit profileName; };
       hex = rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; };
       sodiumoxide = rustPackages."registry+https://github.com/rust-lang/crates.io-index".kuska-sodiumoxide."0.2.5-0" { inherit profileName; };
-      netapp = rustPackages."registry+https://github.com/rust-lang/crates.io-index".netapp."0.4.1" { inherit profileName; };
+      netapp = rustPackages."registry+https://github.com/rust-lang/crates.io-index".netapp."0.4.2" { inherit profileName; };
       pretty_env_logger = rustPackages."registry+https://github.com/rust-lang/crates.io-index".pretty_env_logger."0.4.0" { inherit profileName; };
       rand = rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand."0.8.5" { inherit profileName; };
       rmp_serde = rustPackages."registry+https://github.com/rust-lang/crates.io-index".rmp-serde."0.15.5" { inherit profileName; };
@@ -1348,7 +1348,7 @@ in
       garage_table = rustPackages."unknown".garage_table."0.7.0" { inherit profileName; };
       garage_util = rustPackages."unknown".garage_util."0.7.0" { inherit profileName; };
       hex = rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; };
-      netapp = rustPackages."registry+https://github.com/rust-lang/crates.io-index".netapp."0.4.1" { inherit profileName; };
+      netapp = rustPackages."registry+https://github.com/rust-lang/crates.io-index".netapp."0.4.2" { inherit profileName; };
       opentelemetry = rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry."0.17.0" { inherit profileName; };
       rand = rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand."0.8.5" { inherit profileName; };
       rmp_serde = rustPackages."registry+https://github.com/rust-lang/crates.io-index".rmp-serde."0.15.5" { inherit profileName; };
@@ -1415,7 +1415,7 @@ in
       ${ if rootFeatures' ? "garage_rpc" then "k8s_openapi" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".k8s-openapi."0.13.1" { inherit profileName; };
       ${ if rootFeatures' ? "garage_rpc" then "kube" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".kube."0.62.0" { inherit profileName; };
       ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web" then "sodiumoxide" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".kuska-sodiumoxide."0.2.5-0" { inherit profileName; };
-      ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web" then "netapp" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".netapp."0.4.1" { inherit profileName; };
+      ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web" then "netapp" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".netapp."0.4.2" { inherit profileName; };
       ${ if rootFeatures' ? "garage_rpc" then "openssl" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".openssl."0.10.38" { inherit profileName; };
       ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web" then "opentelemetry" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry."0.17.0" { inherit profileName; };
       ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web" then "pnet_datalink" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".pnet_datalink."0.28.0" { inherit profileName; };
@@ -1518,7 +1518,7 @@ in
       hex = rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; };
       http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.6" { inherit profileName; };
       hyper = rustPackages."registry+https://github.com/rust-lang/crates.io-index".hyper."0.14.18" { inherit profileName; };
-      netapp = rustPackages."registry+https://github.com/rust-lang/crates.io-index".netapp."0.4.1" { inherit profileName; };
+      netapp = rustPackages."registry+https://github.com/rust-lang/crates.io-index".netapp."0.4.2" { inherit profileName; };
       opentelemetry = rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry."0.17.0" { inherit profileName; };
       rand = rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand."0.8.5" { inherit profileName; };
       rmp_serde = rustPackages."registry+https://github.com/rust-lang/crates.io-index".rmp-serde."0.15.5" { inherit profileName; };
@@ -2456,11 +2456,11 @@ in
     };
   });
   
-  "registry+https://github.com/rust-lang/crates.io-index".netapp."0.4.1" = overridableMkRustCrate (profileName: rec {
+  "registry+https://github.com/rust-lang/crates.io-index".netapp."0.4.2" = overridableMkRustCrate (profileName: rec {
     name = "netapp";
-    version = "0.4.1";
+    version = "0.4.2";
     registry = "registry+https://github.com/rust-lang/crates.io-index";
-    src = fetchCratesIo { inherit name version; sha256 = "aafa16daee87ca3251e6d2639c6bfb88c40590627459fd5d44034f60742bb5d1"; };
+    src = fetchCratesIo { inherit name version; sha256 = "d1a19af9ad24e6cdb166e2c5882a7ff9326cab8d45c9d82379fa638c0cbc84df"; };
     features = builtins.concatLists [
       [ "default" ]
       (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "opentelemetry")
@@ -2659,23 +2659,23 @@ in
     features = builtins.concatLists [
       [ "async-trait" ]
       [ "crossbeam-channel" ]
-      (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "dashmap")
+      [ "dashmap" ]
       [ "default" ]
-      (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "fnv")
+      [ "fnv" ]
-      (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "metrics")
+      [ "metrics" ]
       [ "percent-encoding" ]
       [ "pin-project" ]
       [ "rand" ]
-      (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "rt-tokio")
+      [ "rt-tokio" ]
-      (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "tokio")
+      [ "tokio" ]
-      (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "tokio-stream")
+      [ "tokio-stream" ]
       [ "trace" ]
     ];
     dependencies = {
       async_trait = buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".async-trait."0.1.52" { profileName = "__noProfile"; };
       crossbeam_channel = rustPackages."registry+https://github.com/rust-lang/crates.io-index".crossbeam-channel."0.5.4" { inherit profileName; };
-      ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web" then "dashmap" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".dashmap."4.0.2" { inherit profileName; };
+      dashmap = rustPackages."registry+https://github.com/rust-lang/crates.io-index".dashmap."4.0.2" { inherit profileName; };
-      ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web" then "fnv" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".fnv."1.0.7" { inherit profileName; };
+      fnv = rustPackages."registry+https://github.com/rust-lang/crates.io-index".fnv."1.0.7" { inherit profileName; };
       futures_channel = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-channel."0.3.21" { inherit profileName; };
       futures_executor = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-executor."0.3.21" { inherit profileName; };
       futures_util = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.21" { inherit profileName; };
@@ -2685,8 +2685,8 @@ in
       pin_project = rustPackages."registry+https://github.com/rust-lang/crates.io-index".pin-project."1.0.10" { inherit profileName; };
       rand = rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand."0.8.5" { inherit profileName; };
       thiserror = rustPackages."registry+https://github.com/rust-lang/crates.io-index".thiserror."1.0.30" { inherit profileName; };
-      ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web" then "tokio" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.17.0" { inherit profileName; };
+      tokio = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.17.0" { inherit profileName; };
-      ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web" then "tokio_stream" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio-stream."0.1.8" { inherit profileName; };
+      tokio_stream = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio-stream."0.1.8" { inherit profileName; };
     };
   });
   
@@ -3342,7 +3342,7 @@ in
     ];
     dependencies = {
       ${ if hostPlatform.parsed.kernel.name == "android" || hostPlatform.parsed.kernel.name == "linux" then "libc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.121" { inherit profileName; };
-      ${ if hostPlatform.parsed.kernel.name == "android" || hostPlatform.parsed.kernel.name == "linux" || hostPlatform.parsed.kernel.name == "dragonfly" || hostPlatform.parsed.kernel.name == "freebsd" || hostPlatform.parsed.kernel.name == "illumos" || hostPlatform.parsed.kernel.name == "netbsd" || hostPlatform.parsed.kernel.name == "openbsd" || hostPlatform.parsed.kernel.name == "solaris" then "once_cell" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".once_cell."1.10.0" { inherit profileName; };
+      ${ if hostPlatform.parsed.kernel.name == "dragonfly" || hostPlatform.parsed.kernel.name == "freebsd" || hostPlatform.parsed.kernel.name == "illumos" || hostPlatform.parsed.kernel.name == "netbsd" || hostPlatform.parsed.kernel.name == "openbsd" || hostPlatform.parsed.kernel.name == "solaris" || hostPlatform.parsed.kernel.name == "android" || hostPlatform.parsed.kernel.name == "linux" then "once_cell" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".once_cell."1.10.0" { inherit profileName; };
       ${ if hostPlatform.parsed.cpu.name == "i686" || hostPlatform.parsed.cpu.name == "x86_64" || (hostPlatform.parsed.cpu.name == "aarch64" || hostPlatform.parsed.cpu.name == "armv6l" || hostPlatform.parsed.cpu.name == "armv7l") && (hostPlatform.parsed.kernel.name == "android" || hostPlatform.parsed.kernel.name == "fuchsia" || hostPlatform.parsed.kernel.name == "linux") then "spin" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".spin."0.5.2" { inherit profileName; };
       untrusted = rustPackages."registry+https://github.com/rust-lang/crates.io-index".untrusted."0.7.1" { inherit profileName; };
       ${ if hostPlatform.parsed.cpu.name == "wasm32" && hostPlatform.parsed.vendor.name == "unknown" && hostPlatform.parsed.kernel.name == "unknown" && hostPlatform.parsed.abi.name == "" then "web_sys" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".web-sys."0.3.56" { inherit profileName; };
@@ -3819,7 +3819,7 @@ in
     ];
     dependencies = {
       bitflags = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bitflags."1.3.2" { inherit profileName; };
-      ${ if hostPlatform.parsed.kernel.name == "linux" || hostPlatform.parsed.kernel.name == "android" then "libc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.121" { inherit profileName; };
+      ${ if hostPlatform.parsed.kernel.name == "android" || hostPlatform.parsed.kernel.name == "linux" then "libc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.121" { inherit profileName; };
       ${ if !(hostPlatform.parsed.kernel.name == "linux" || hostPlatform.parsed.kernel.name == "android") then "parking_lot" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".parking_lot."0.11.2" { inherit profileName; };
       ${ if !(hostPlatform.parsed.kernel.name == "linux" || hostPlatform.parsed.kernel.name == "android") then "parking_lot_core" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".parking_lot_core."0.8.5" { inherit profileName; };
       static_init_macro = buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".static_init_macro."1.0.2" { profileName = "__noProfile"; };
@@ -4790,10 +4790,10 @@ in
     ];
     dependencies = {
       ${ if hostPlatform.config == "aarch64-uwp-windows-msvc" || hostPlatform.config == "aarch64-pc-windows-msvc" then "windows_aarch64_msvc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_aarch64_msvc."0.32.0" { inherit profileName; };
-      ${ if hostPlatform.config == "i686-pc-windows-gnu" || hostPlatform.config == "i686-uwp-windows-gnu" then "windows_i686_gnu" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_i686_gnu."0.32.0" { inherit profileName; };
+      ${ if hostPlatform.config == "i686-uwp-windows-gnu" || hostPlatform.config == "i686-pc-windows-gnu" then "windows_i686_gnu" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_i686_gnu."0.32.0" { inherit profileName; };
       ${ if hostPlatform.config == "i686-pc-windows-msvc" || hostPlatform.config == "i686-uwp-windows-msvc" then "windows_i686_msvc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_i686_msvc."0.32.0" { inherit profileName; };
       ${ if hostPlatform.config == "x86_64-uwp-windows-gnu" || hostPlatform.config == "x86_64-pc-windows-gnu" then "windows_x86_64_gnu" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_x86_64_gnu."0.32.0" { inherit profileName; };
-      ${ if hostPlatform.config == "x86_64-uwp-windows-msvc" || hostPlatform.config == "x86_64-pc-windows-msvc" then "windows_x86_64_msvc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_x86_64_msvc."0.32.0" { inherit profileName; };
+      ${ if hostPlatform.config == "x86_64-pc-windows-msvc" || hostPlatform.config == "x86_64-uwp-windows-msvc" then "windows_x86_64_msvc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_x86_64_msvc."0.32.0" { inherit profileName; };
     };
   });
   

doc/book/connect/cli.md

@@ -13,7 +13,8 @@ These tools are particularly suitable for debug, backups, website deployments or
 | [rclone](#rclone)     | ✅       |    |
 | [s3cmd](#s3cmd)     | ✅       |    |
 | [(Cyber)duck](#cyberduck)     | ✅       |    |
-| [WinSCP (libs3)](#winscp)     | ✅       | No instructions yet   |
+| [WinSCP (libs3)](#winscp)     | ✅       | CLI instructions only   |
+| [sftpgo](#sftpgo)     | ✅       |    |
 
 
 ## Minio client
@@ -281,5 +282,59 @@ duck --delete garage:/my-files/an-object.txt
 
 ## WinSCP (libs3) {#winscp}
 
-*No instruction yet. You can find ones in french [in our wiki](https://wiki.deuxfleurs.fr/fr/Guide/Garage/WinSCP).*
+*You can find instructions on how to use the GUI in french [in our wiki](https://wiki.deuxfleurs.fr/fr/Guide/Garage/WinSCP).*
+
+How to use `winscp.com`, the CLI interface of WinSCP:
+
+```
+open s3://GKxxxxx:yyyyyyy@127.0.0.1:4443 -certificate=* -rawsettings S3DefaultRegion=garage S3UrlStyle=1
+ls
+ls my-files/
+get my-files/an-object.txt Z:\tmp\object.txt
+put Z:\tmp\object.txt my-files/another-object.txt
+rm my-files/an-object
+exit
+```
+
+Notes:
+  - It seems WinSCP supports only TLS connections for S3
+  - `-certificate=*` allows self-signed certificates, remove it if you have valid certificates
+
+
+## sftpgo {#sftpgo}
+
+sftpgo needs a database to work, by default it uses sqlite and does not require additional configuration.
+You can then directly init it:
+
+```
+sftpgo initprovider
+```
+
+Then you can directly launch the daemon that will listen by default on `:8080 (http)` and `:2022 (ssh)`:
+
+```
+sftpgo serve
+```
+
+Go to the admin web interface (http://[::1]:8080/web/admin/), create the required admin account, then create a user account.
+Choose a username (eg: `ada`) and a password.
+
+In the filesystem section, choose:
+  - Storage: AWS S3 (Compatible)
+  - Bucket: *your bucket name*
+  - Region: `garage` (or the one you defined in `config.toml`)
+  - Access key: *your access key*
+  - Access secret: *your secret key*
+  - Endpoint: *your endpoint*, eg. `https://garage.example.tld`, note that the protocol (`https` here) must be specified. Non standard ports and `http` have not been tested yet.
+  - Keep the default values for other fields
+  - Tick "Use path-style addressing". It should work without ticking it if you have correctly configured your instance to use URL vhost-style.
+
+Now you can access your bucket through SFTP:
+
+```
+sftp -P2022 ada@[::1]
+ls
+```
+
+And through the web interface at http://[::1]:8080/web/client
 

doc/book/cookbook/upgrading.md

@@ -0,0 +1,50 @@
++++
+title = "Upgrading Garage"
+weight = 40
++++
+
+Garage is a stateful clustered application, where all nodes are communicating together and share data structures.
+It makes upgrade more difficult than stateless applications so you must be more careful when upgrading.
+On a new version release, there is 2 possibilities:
+  - protocols and data structures remained the same ➡️ this is a **straightforward upgrade**
+  - protocols or data structures changed  ➡️  this is an **advanced upgrade**
+
+You can quickly now what type of update you will have to operate by looking at the version identifier.
+Following the [SemVer ](https://semver.org/) terminology, if only the *patch* number changed, it will only need a straightforward upgrade.
+Example: an upgrade from v0.6.0 from v0.6.1 is a straightforward upgrade.
+If the *minor* or *major* number changed however, you will have to do an advanced upgrade. Example: from v0.6.1 to v0.7.0.
+
+Migrations are designed to be run only between contiguous versions (from a *major*.*minor* perspective, *patches* can be skipped).
+Example: migrations from v0.6.1 to v0.7.0 and from v0.6.0 to v0.7.0 are supported but migrations from v0.5.0 to v0.7.0 are not supported.
+
+## Straightforward upgrades
+
+Straightforward upgrades do not imply cluster downtime.
+Before upgrading, you should still read [the changelog](https://git.deuxfleurs.fr/Deuxfleurs/garage/releases) and ideally test your deployment on a staging cluster before.
+
+When you are ready, start by checking the health of your cluster.
+You can force some checks with `garage repair`, we recommend at least running `garage repair --all-nodes --yes` that is very quick to run (less than a minute).
+You will see that the command correctly terminated in the logs of your daemon.
+
+Finally, you can simply upgrades nodes one by one. 
+For each node: stop it, install the new binary, edit the configuration if needed, restart it. 
+
+## Advanced upgrades
+
+Advanced upgrades will imply cluster downtime.
+Before upgrading, you must read [the changelog](https://git.deuxfleurs.fr/Deuxfleurs/garage/releases) and you must test your deployment on a staging cluster before.
+
+From a high level perspective, an advanced upgrade looks like this:
+  1. Make sure the health of your cluster is good (see `garage repair`)
+  2. Disable API access (comment the configuration in your reverse proxy)
+  3. Check that your cluster is idle
+  4. Stop the whole cluster
+  5. Backup the metadata folder of all your nodes, so that you will be able to restore it quickly if the upgrade fails (blocks being immutable, they should not be impacted)
+  6. Install the new binary, update the configuration
+  7. Start the whole cluster
+  8. If needed, run the corresponding migration from `garage migrate`
+  9. Make sure the health of your cluster is good
+  10. Enable API access (uncomment the configuration in your reverse proxy)
+  11. Monitor your cluster while load comes back, check that all your applications are happy with this new version
+
+We write guides for each advanced upgrade, they are stored under the "Working Documents" section of this documentation.

doc/book/reference-manual/configuration.md

@@ -44,6 +44,10 @@ root_domain = ".s3.garage"
 [s3_web]
 bind_addr = "[::]:3902"
 root_domain = ".web.garage"
+
+[admin]
+api_bind_addr = "0.0.0.0:3903"
+trace_sink = "http://localhost:4317"
 ```
 
 The following gives details about each available configuration option.
@@ -84,20 +88,47 @@ might use more storage space that is optimally possible.
 
 Garage supports the following replication modes:
 
-- `none` or `1`: data stored on Garage is stored on a single node. There is no redundancy,
+- `none` or `1`: data stored on Garage is stored on a single node. There is no
-  and data will be unavailable as soon as one node fails or its network is disconnected.
+  redundancy, and data will be unavailable as soon as one node fails or its
-  Do not use this for anything else than test deployments.
+  network is disconnected.  Do not use this for anything else than test
+  deployments.
 
-- `2`: data stored on Garage will be stored on two different nodes, if possible in different
+- `2`: data stored on Garage will be stored on two different nodes, if possible
-  zones. Garage tolerates one node failure before losing data. Data should be available
+  in different zones. Garage tolerates one node failure, or several nodes
-  read-only when one node is down, but write operations will fail.
+  failing but all in a single zone (in a deployment with at least two zones),
-  Use this only if you really have to.
+  before losing data. Data remains available in read-only mode when one node is
+  down, but write operations will fail.
 
-- `3`: data stored on Garage will be stored on three different nodes, if possible each in
+  - `2-dangerous`: a variant of mode `2`, where written objects are written to
-  a different zones.
+    the second replica asynchronously. This means that Garage will return `200
-  Garage tolerates two node failure before losing data. Data should be available
+    OK` to a PutObject request before the second copy is fully written (or even
-  read-only when two nodes are down, and writes should be possible if only a single node
+    before it even starts being written).  This means that data can more easily
-  is down.
+    be lost if the node crashes before a second copy can be completed.  This
+    also means that written objects might not be visible immediately in read
+    operations.  In other words, this mode severely breaks the consistency and
+    durability guarantees of standard Garage cluster operation.  Benefits of
+    this mode: you can still write to your cluster when one node is
+    unavailable.
+
+- `3`: data stored on Garage will be stored on three different nodes, if
+  possible each in a different zones.  Garage tolerates two node failure, or
+  several node failures but in no more than two zones (in a deployment with at
+  least three zones), before losing data. As long as only a single node fails,
+  or node failures are only in a single zone, reading and writing data to
+  Garage can continue normally.
+
+  - `3-degraded`: a variant of replication mode `3`, that lowers the read
+    quorum to `1`, to allow you to read data from your cluster when several
+    nodes (or nodes in several zones) are unavailable.  In this mode, Garage
+    does not provide read-after-write consistency anymore.  The write quorum is
+    still 2, ensuring that data successfully written to Garage is stored on at
+    least two nodes.
+
+  - `3-dangerous`: a variant of replication mode `3` that lowers both the read
+    and write quorums to `1`, to allow you to both read and write to your
+    cluster when several nodes (or nodes in several zones) are unavailable.  It
+    is the least consistent mode of operation proposed by Garage, and also one
+    that should probably never be used.
 
 Note that in modes `2` and `3`,
 if at least the same number of zones are available, an arbitrary number of failures in 
@@ -106,8 +137,35 @@ any given zone is tolerated as copies of data will be spread over several zones.
 **Make sure `replication_mode` is the same in the configuration files of all nodes.
 Never run a Garage cluster where that is not the case.**
 
-Changing the `replication_mode` of a cluster might work (make sure to shut down all nodes
+The quorums associated with each replication mode are described below:
-and changing it everywhere at the time), but is not officially supported.
+
+| `replication_mode` | Number of replicas | Write quorum | Read quorum | Read-after-write consistency? |
+| ------------------ | ------------------ | ------------ | ----------- | ----------------------------- |
+| `none` or `1`      | 1                  | 1            | 1           | yes                           |
+| `2`                | 2                  | 2            | 1           | yes                           |
+| `2-dangerous`      | 2                  | 1            | 1           | NO                            |
+| `3`                | 3                  | 2            | 2           | yes                           |
+| `3-degraded`       | 3                  | 2            | 1           | NO                            |
+| `3-dangerous`      | 3                  | 1            | 1           | NO                            |
+
+Changing the `replication_mode` between modes with the same number of replicas
+(e.g. from `3` to `3-degraded`, or from `2-dangerous` to `2`), can be done easily by
+just changing the `replication_mode` parameter in your config files and restarting all your
+Garage nodes.
+
+It is also technically possible to change the replication mode to a mode with a
+different numbers of replicas, although it's a dangerous operation that is not
+officially supported.  This requires you to delete the existing cluster layout
+and create a new layout from scratch, meaning that a full rebalancing of your
+cluster's data will be needed.  To do it, shut down your cluster entirely,
+delete the `custer_layout` files in the meta directories of all your nodes,
+update all your configuration files with the new `replication_mode` parameter,
+restart your cluster, and then create a new layout with all the nodes you want
+to keep.  Rebalancing data will take some time, and data might temporarily
+appear unavailable to your users.  It is recommended to shut down public access
+to the cluster while rebalancing is in progress.  In theory, no data should be
+lost as rebalancing is a routine operation for Garage, although we cannot
+guarantee you that everything will go right in such an extreme scenario.
 
 ### `compression_level`
 
@@ -260,3 +318,21 @@ For instance, if `root_domain` is `web.garage.eu`, a bucket called `deuxfleurs.f
 will be accessible either with hostname `deuxfleurs.fr.web.garage.eu`
 or with hostname `deuxfleurs.fr`.
 
+
+## The `[admin]` section
+
+Garage has a few administration capabilities, in particular to allow remote monitoring. These features are detailed below.
+
+### `api_bind_addr`
+
+If specified, Garage will bind an HTTP server to this port and address, on
+which it will listen to requests for administration features. Currently,
+this endpoint only exposes Garage metrics in the Prometheus format at
+`/metrics`. This endpoint is not authenticated. In the future, bucket and
+access key management might be possible by REST calls to this endpoint.
+
+### `trace_sink`
+
+Optionnally, the address of an Opentelemetry collector.  If specified,
+Garage will send traces in the Opentelemetry format to this endpoint. These
+trace allow to inspect Garage's operation when it handles S3 API requests.

doc/book/working-documents/migration-06.md

@@ -4,12 +4,12 @@ weight = 15
 +++
 
 **This guide explains how to migrate to 0.6 if you have an existing 0.5 cluster.
-We don't recommend trying to migrate directly from 0.4 or older to 0.6.**
+We don't recommend trying to migrate to 0.6 directly from 0.4 or older.**
 
 **We make no guarantee that this migration will work perfectly:
 back up all your data before attempting it!**
 
-Garage v0.6 (not yet released) introduces a new data model for buckets,
+Garage v0.6 introduces a new data model for buckets,
 that allows buckets to have many names (aliases).
 Buckets can also have "private" aliases (called local aliases),
 which are only visible when using a certain access key.

doc/book/working-documents/migration-07.md

@@ -0,0 +1,31 @@
++++
+title = "Migrating from 0.6 to 0.7"
+weight = 14
++++
+**This guide explains how to migrate to 0.7 if you have an existing 0.6 cluster.
+We don't recommend trying to migrate to 0.7 directly from 0.5 or older.**
+
+**We make no guarantee that this migration will work perfectly:
+back up all your data before attempting it!**
+
+Garage v0.7 introduces a cluster protocol change to support request tracing through OpenTelemetry.
+No data structure is changed, so no data migration is required.
+
+The migration steps are as follows:
+
+1. Do `garage repair --all-nodes --yes tables` and `garage repair --all-nodes --yes blocks`,
+   check the logs and check that all data seems to be synced correctly between
+   nodes. If you have time, do additional checks (`scrub`, `block_refs`, etc.)
+2. Disable api and web access. Garage does not support disabling
+   these endpoints but you can change the port number or stop your reverse
+   proxy for instance.
+3. Check once again that your cluster is healty. Run again `garage repair --all-nodes --yes tables` which is quick.
+   Also check your queues are empty, run `garage stats` to query them.
+4. Turn off Garage v0.6
+5. Backup the metadata folder of all your nodes: `cd /var/lib/garage ; tar -acf meta-v0.6.tar.zst meta/`
+6. Install Garage v0.7, edit the configuration if you plan to use OpenTelemetry or the Kubernetes integration
+7. Turn on Garage v0.7
+8. Do `garage repair --all-nodes --yes tables` and `garage repair --all-nodes --yes blocks`
+9. Your upgraded cluster should be in a working state. Re-enable API and Web
+    access and check that everything went well.
+10. Monitor your cluster in the next hours to see if it works well under your production load, report any issue.

script/k8s/README.md

@@ -0,0 +1,13 @@
+Spawn a cluster with minikube
+
+```bash
+minikube start
+minikube kubectl -- apply -f config.yaml
+minikube kubectl -- apply -f daemon.yaml
+minikube dashboard
+
+minikube kubectl -- exec -it garage-0 --container garage -- /garage status
+# etc.
+```
+
+

script/k8s/admin.yaml

@@ -0,0 +1,12 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: garage-admin
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: cluster-admin
+subjects:
+- apiGroup: rbac.authorization.k8s.io
+  kind: User
+  name: system:serviceaccount:default:default

script/k8s/config.yaml

@@ -0,0 +1,30 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: garage-config
+  namespace: default
+data:
+  garage.toml: |-
+    metadata_dir = "/tmp/meta"
+    data_dir = "/tmp/data"
+
+    replication_mode = "3"
+
+    rpc_bind_addr = "[::]:3901"
+    rpc_secret = "1799bccfd7411eddcf9ebd316bc1f5287ad12a68094e1c6ac6abde7e6feae1ec"
+
+    bootstrap_peers = []
+
+    kubernetes_namespace = "default"
+    kubernetes_service_name = "garage-daemon"
+    kubernetes_skip_crd = false
+
+    [s3_api]
+    s3_region = "garage"
+    api_bind_addr = "[::]:3900"
+    root_domain = ".s3.garage.tld"
+
+    [s3_web]
+    bind_addr = "[::]:3902"
+    root_domain = ".web.garage.tld"
+    index = "index.html"

script/k8s/daemon.yaml

@@ -0,0 +1,52 @@
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: garage
+spec:
+  selector:
+    matchLabels:
+      app: garage
+  serviceName: "garage"
+  replicas: 3 
+  template:
+    metadata:
+      labels:
+        app: garage
+    spec:
+      terminationGracePeriodSeconds: 10
+      containers:
+      - name: garage
+        image: dxflrs/amd64_garage:v0.7.0-rc1
+        ports:
+        - containerPort: 3900
+          name: s3-api
+        - containerPort: 3902
+          name: web-api
+        volumeMounts:
+        - name: fast
+          mountPath: /mnt/fast
+        - name: slow
+          mountPath: /mnt/slow
+        - name: etc
+          mountPath: /etc/garage.toml
+          subPath: garage.toml
+      volumes:
+      - name: etc
+        configMap:
+          name: garage-config
+  volumeClaimTemplates:
+  - metadata:
+      name: fast
+    spec:
+      accessModes: [ "ReadWriteOnce" ]
+      resources:
+        requests:
+          storage: 100Mi
+  - metadata:
+      name: slow
+    spec:
+      accessModes: [ "ReadWriteOnce" ]
+      resources:
+        requests:
+          storage: 100Mi
+  

script/telemetry/README.md

@@ -0,0 +1,21 @@
+Configure your `[admin-api]` endpoint:
+
+```
+[admin]
+api_bind_addr = "0.0.0.0:3903"
+trace_sink = "http://localhost:4317"
+```
+
+Start the test stack:
+
+```
+cd telemetry
+docker-compose up
+```
+
+Access the web interfaces:
+  - [Kibana](http://localhost:5601) - Click on the hamburger menu, in the Observability section, click APM
+  - [Grafana](http://localhost:3000) - Set a password, then on the left menu, click Dashboard -> Browse. On the new page click Import -> Choose the test dashboard we ship `grafana-garage-dashboard-elasticsearch.json` 
+
+
+

script/telemetry/elastic/grafana/provisioning/datasources/elastic.yaml

@@ -4,17 +4,16 @@ datasources:
   - name: DS_ELASTICSEARCH
     type: elasticsearch
     access: proxy
-    url: http://elastic:9700
+    url: http://localhost:9200
     password: ''
     user: ''
-    database: metricbeat-*
+    database: apm-*
     basicAuth: false
     isDefault: true
     jsonData:
-      esVersion: 70
+      esVersion: 7.10.0
       logLevelField: ''
       logMessageField: ''
       maxConcurrentShardRequests: 5
       timeField: "@timestamp"
-      timeInterval: 10s
     readOnly: false

script/telemetry/elastic/otel-config.yaml

@@ -13,7 +13,7 @@ receivers:
         - job_name: "garage"
           scrape_interval: 5s
           static_configs:
-            - targets: ["localhost:3909"]
+            - targets: ["localhost:3903"]
 
 exporters:
   logging:

src/admin/metrics.rs

@@ -28,7 +28,7 @@ async fn serve_req(
 	req: Request<Body>,
 	admin_server: Arc<AdminServer>,
 ) -> Result<Response<Body>, hyper::Error> {
-	info!("Receiving request at path {}", req.uri());
+	debug!("Receiving request at path {}", req.uri());
 	let request_start = SystemTime::now();
 
 	admin_server.metrics.http_counter.add(1);

src/api/signature/payload.rs

@@ -235,7 +235,7 @@ pub fn canonical_request(
 ) -> String {
 	[
 		method.as_str(),
-		&uri.path().to_string(),
+		uri.path(),
 		&canonical_query_string(uri),
 		&canonical_header_string(headers, signed_headers),
 		"",

src/block/manager.rs

@@ -46,6 +46,9 @@ const NEED_BLOCK_QUERY_TIMEOUT: Duration = Duration::from_secs(5);
 // and the time when it is retried, with exponential backoff
 // (multiplied by 2, 4, 8, 16, etc. for every consecutive failure).
 const RESYNC_RETRY_DELAY: Duration = Duration::from_secs(60);
+// The minimum retry delay is 60 seconds = 1 minute
+// The maximum retry delay is 60 seconds * 2^6 = 60 seconds << 6 = 64 minutes (~1 hour)
+const RESYNC_RETRY_DELAY_MAX_BACKOFF_POWER: u64 = 6;
 
 // The delay between the moment when the reference counter
 // drops to zero, and the moment where we allow ourselves
@@ -265,6 +268,11 @@ impl BlockManager {
 		self.resync_queue.len()
 	}
 
+	/// Get number of blocks that have an error
+	pub fn resync_errors_len(&self) -> usize {
+		self.resync_errors.len()
+	}
+
 	/// Get number of items in the refcount table
 	pub fn rc_len(&self) -> usize {
 		self.rc.rc.len()
@@ -980,7 +988,8 @@ impl ErrorCounter {
 	}
 
 	fn delay_msec(&self) -> u64 {
-		(RESYNC_RETRY_DELAY.as_millis() as u64) << std::cmp::min(self.errors - 1, 10)
+		(RESYNC_RETRY_DELAY.as_millis() as u64)
+			<< std::cmp::min(self.errors - 1, RESYNC_RETRY_DELAY_MAX_BACKOFF_POWER)
 	}
 	fn next_try(&self) -> u64 {
 		self.last_try + self.delay_msec()

src/garage/Cargo.toml

@@ -52,7 +52,7 @@ tokio = { version = "1.0", default-features = false, features = ["rt", "rt-multi
 
 #netapp = { version = "0.3.2", git = "https://git.deuxfleurs.fr/lx/netapp" }
 #netapp = { version = "0.4", path = "../../../netapp" }
-netapp = "0.4.1"
+netapp = "0.4"
 
 [dev-dependencies]
 aws-sdk-s3 = "0.8"

src/garage/admin.rs

@@ -728,6 +728,12 @@ impl AdminRpcHandler {
 			self.garage.block_manager.resync_queue_len()
 		)
 		.unwrap();
+		writeln!(
+			&mut ret,
+			"  blocks with resync errors: {}",
+			self.garage.block_manager.resync_errors_len()
+		)
+		.unwrap();
 
 		ret
 	}

src/model/Cargo.toml

@@ -41,4 +41,4 @@ opentelemetry = "0.17"
 
 #netapp = { version = "0.3.0", git = "https://git.deuxfleurs.fr/lx/netapp" }
 #netapp = { version = "0.4", path = "../../../netapp" }
-netapp = "0.4.1"
+netapp = "0.4"

src/rpc/Cargo.toml

@@ -48,7 +48,7 @@ opentelemetry = "0.17"
 
 #netapp = { version = "0.3.0", git = "https://git.deuxfleurs.fr/lx/netapp" }
 #netapp = { version = "0.4", path = "../../../netapp", features = ["telemetry"] }
-netapp = { version = "0.4.1", features = ["telemetry"] }
+netapp = { version = "0.4.2", features = ["telemetry"] }
 
 hyper = { version = "0.14", features = ["client", "http1", "runtime", "tcp"] }
 

src/table/replication/mode.rs

@@ -1,7 +1,10 @@
 pub enum ReplicationMode {
 	None,
 	TwoWay,
+	TwoWayDangerous,
 	ThreeWay,
+	ThreeWayDegraded,
+	ThreeWayDangerous,
 }
 
 impl ReplicationMode {
@@ -9,7 +12,10 @@ impl ReplicationMode {
 		match v {
 			"none" | "1" => Some(Self::None),
 			"2" => Some(Self::TwoWay),
+			"2-dangerous" => Some(Self::TwoWayDangerous),
 			"3" => Some(Self::ThreeWay),
+			"3-degraded" => Some(Self::ThreeWayDegraded),
+			"3-dangerous" => Some(Self::ThreeWayDangerous),
 			_ => None,
 		}
 	}
@@ -24,16 +30,17 @@ impl ReplicationMode {
 	pub fn replication_factor(&self) -> usize {
 		match self {
 			Self::None => 1,
-			Self::TwoWay => 2,
+			Self::TwoWay | Self::TwoWayDangerous => 2,
-			Self::ThreeWay => 3,
+			Self::ThreeWay | Self::ThreeWayDegraded | Self::ThreeWayDangerous => 3,
 		}
 	}
 
 	pub fn read_quorum(&self) -> usize {
 		match self {
 			Self::None => 1,
-			Self::TwoWay => 1,
+			Self::TwoWay | Self::TwoWayDangerous => 1,
 			Self::ThreeWay => 2,
+			Self::ThreeWayDegraded | Self::ThreeWayDangerous => 1,
 		}
 	}
 
@@ -41,7 +48,9 @@ impl ReplicationMode {
 		match self {
 			Self::None => 1,
 			Self::TwoWay => 2,
-			Self::ThreeWay => 2,
+			Self::TwoWayDangerous => 1,
+			Self::ThreeWay | Self::ThreeWayDegraded => 2,
+			Self::ThreeWayDangerous => 1,
 		}
 	}
 }

src/util/Cargo.toml

@@ -35,9 +35,9 @@ tokio = { version = "1.0", default-features = false, features = ["rt", "rt-multi
 
 #netapp = { version = "0.3.0", git = "https://git.deuxfleurs.fr/lx/netapp" }
 #netapp = { version = "0.4", path = "../../../netapp" }
-netapp = "0.4.1"
+netapp = "0.4"
 
 http = "0.2"
 hyper = "0.14"
 
-opentelemetry = "0.17"
+opentelemetry = { version = "0.17", features = [ "rt-tokio", "metrics", "trace" ] }