Prepare for v0.3.0 and add migration path from v0.2.1.x

- Explicit "replication_mode" configuration parameters that takes
  either "none", "2" or "3" as values, instead of letting user configure
  replication factor themselves. These are presets whose corresponding
  replication/quorum values can be found in replication/mode.rs

- Explicit support for single-node and two-node deployments
  (number of nodes must be at least "replication_mode", with "none"
  we can have only one node)

- Ring is now stored much more compactly with 256*8 + n*32 bytes,
  instead of 256*32 bytes

- Support for gateway-only nodes that do not store data
  (these nodes still need a metadata_directory to store the list
  of bucket and keys since those are stored on all nodes; it also
  technically needs a data_directory to start but it will stay
  empty unless we have bugs)

.drone.yml

@@ -1,12 +1,20 @@
+---
 kind: pipeline
 name: default
 
 workspace:
   base: /drone/garage
 
+volumes:
+- name: cargo_home
+  temp: {}
+
 steps:
   - name: restore-cache
     image: meltwater/drone-cache:dev
+    volumes:
+    - name: cargo_home
+      path: /drone/cargo
     environment:
       AWS_ACCESS_KEY_ID:
         from_secret: cache_aws_access_key_id
@@ -17,38 +25,54 @@ steps:
       restore: true
       archive_format: "gzip"
       bucket: drone-cache
-      cache_key: '{{ .Repo.Name }}_{{ checksum "garage/Cargo.lock" }}_{{ arch }}_{{ os }}_gzip'
+      cache_key: '{{ .Repo.Name }}_{{ checksum "Cargo.lock" }}_{{ arch }}_{{ os }}_gzip'
       region: garage
       mount:
+        - '/drone/cargo'
         - 'target'
-        - '/drone/cargo/registry/index'
-        - '/drone/cargo/registry/cache'
-        - '/drone/cargo/bin'
-        - '/drone/cargo/git/db'
       path_style: true
       endpoint: https://garage.deuxfleurs.fr
+    when:
+      branch:
+      - nonexistent_skip_this_step
 
-  - name: build
-    image: rust:buster
+  - name: code quality
+    image: superboum/garage_builder_amd64:4
+    volumes:
+    - name: cargo_home
+      path: /drone/cargo
+    environment:
+      CARGO_HOME: /drone/cargo
+    commands:
+      - cargo fmt -- --check
+      - cargo clippy -- --deny warnings
+
+  - name: build
+    image: superboum/garage_builder_amd64:4
+    volumes:
+    - name: cargo_home
+      path: /drone/cargo
     environment:
       CARGO_HOME: /drone/cargo
     commands:
-      - apt-get update
-      - apt-get install --yes libsodium-dev
       - pwd
       - cargo build
 
   - name: cargo-test
-    image: rust:buster
+    image: superboum/garage_builder_amd64:4
+    volumes:
+    - name: cargo_home
+      path: /drone/cargo
     environment:
       CARGO_HOME: /drone/cargo
     commands:
-      - apt-get update
-      - apt-get install --yes libsodium-dev
       - cargo test
 
   - name: rebuild-cache
     image: meltwater/drone-cache:dev
+    volumes:
+    - name: cargo_home
+      path: /drone/cargo
     environment:
       AWS_ACCESS_KEY_ID:
         from_secret: cache_aws_access_key_id
@@ -59,23 +83,62 @@ steps:
       rebuild: true
       archive_format: "gzip"
       bucket: drone-cache
-      cache_key: '{{ .Repo.Name }}_{{ checksum "garage/Cargo.lock" }}_{{ arch }}_{{ os }}_gzip'
+      cache_key: '{{ .Repo.Name }}_{{ checksum "Cargo.lock" }}_{{ arch }}_{{ os }}_gzip'
       region: garage
       mount:
+        - '/drone/cargo'
         - 'target'
-        - '/drone/cargo/registry/index'
-        - '/drone/cargo/registry/cache'
-        - '/drone/cargo/git/db'
-        - '/drone/cargo/bin'
       path_style: true
       endpoint: https://garage.deuxfleurs.fr
+    when:
+      branch:
+      - nonexistent_skip_this_step
 
   - name: smoke-test
-    image: rust:buster
+    image: superboum/garage_builder_amd64:4
+    volumes:
+    - name: cargo_home
+      path: /drone/cargo
     environment:
       CARGO_HOME: /drone/cargo
     commands:
-      - apt-get update
-      - apt-get install --yes libsodium-dev awscli python-pip
-      - pip install s3cmd
       - ./script/test-smoke.sh || (cat /tmp/garage.log; false)
+
+---
+kind: pipeline
+name: website
+
+steps:
+  - name: build
+    image: hrektts/mdbook
+    commands:
+      - cd doc/book
+      - mdbook build
+
+  - name: upload
+    image: plugins/s3
+    settings:
+      bucket: garagehq.deuxfleurs.fr
+      access_key:
+        from_secret: garagehq_aws_access_key_id
+      secret_key:
+        from_secret: garagehq_aws_secret_access_key
+      source: doc/book/book/**/*
+      strip_prefix: doc/book/book/
+      target: /
+      path_style: true
+      endpoint: https://garage.deuxfleurs.fr
+      region: garage
+    when:
+      event:
+        - push
+      branch:
+        - main
+      repo:
+        - Deuxfleurs/garage
+
+---
+kind: signature
+hmac: e919f8a66d20ebfeeec56b291a8a0fdd59a482601da987fcf533d96d24768744
+
+...

Cargo.lock

@@ -350,16 +350,16 @@ dependencies = [
 
 [[package]]
 name = "garage"
-version = "0.2.1"
+version = "0.3.0"
 dependencies = [
  "bytes",
  "futures",
  "futures-util",
  "garage_api",
  "garage_model",
- "garage_rpc",
+ "garage_rpc 0.3.0",
  "garage_table",
- "garage_util",
+ "garage_util 0.3.0",
  "garage_web",
  "git-version",
  "hex",
@@ -376,7 +376,7 @@ dependencies = [
 
 [[package]]
 name = "garage_api"
-version = "0.2.1"
+version = "0.3.0"
 dependencies = [
  "base64",
  "bytes",
@@ -387,7 +387,7 @@ dependencies = [
  "futures-util",
  "garage_model",
  "garage_table",
- "garage_util",
+ "garage_util 0.3.0",
  "hex",
  "hmac",
  "http",
@@ -397,7 +397,9 @@ dependencies = [
  "log",
  "md-5",
  "percent-encoding",
+ "quick-xml",
  "roxmltree",
+ "serde",
  "sha2",
  "tokio",
  "url",
@@ -405,14 +407,14 @@ dependencies = [
 
 [[package]]
 name = "garage_model"
-version = "0.2.1"
+version = "0.3.0"
 dependencies = [
  "arc-swap",
  "futures",
  "futures-util",
- "garage_rpc",
+ "garage_rpc 0.3.0",
  "garage_table",
- "garage_util",
+ "garage_util 0.3.0",
  "hex",
  "log",
  "rand",
@@ -426,12 +428,40 @@ dependencies = [
 [[package]]
 name = "garage_rpc"
 version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "48c182633cebe4abed9594afb14770fc45402513765d38a4b19659ae0ccb2a2f"
 dependencies = [
  "arc-swap",
  "bytes",
  "futures",
  "futures-util",
- "garage_util",
+ "garage_util 0.2.1",
+ "gethostname",
+ "hex",
+ "http",
+ "hyper",
+ "hyper-rustls",
+ "log",
+ "rmp-serde",
+ "rustls",
+ "serde",
+ "serde_json",
+ "tokio",
+ "tokio-rustls",
+ "tokio-stream",
+ "webpki",
+]
+
+[[package]]
+name = "garage_rpc"
+version = "0.3.0"
+dependencies = [
+ "arc-swap",
+ "bytes",
+ "futures",
+ "futures-util",
+ "garage_rpc 0.2.1",
+ "garage_util 0.3.0",
  "gethostname",
  "hex",
  "http",
@@ -450,13 +480,13 @@ dependencies = [
 
 [[package]]
 name = "garage_table"
-version = "0.2.1"
+version = "0.3.0"
 dependencies = [
  "bytes",
  "futures",
  "futures-util",
- "garage_rpc",
- "garage_util",
+ "garage_rpc 0.3.0",
+ "garage_util 0.3.0",
  "hexdump",
  "log",
  "rand",
@@ -470,6 +500,33 @@ dependencies = [
 [[package]]
 name = "garage_util"
 version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "aef76d3779e406a16fdcaffe8d86b8ae2943a549d2b33f2c20930838764464c0"
+dependencies = [
+ "blake2",
+ "chrono",
+ "err-derive",
+ "futures",
+ "hex",
+ "http",
+ "hyper",
+ "log",
+ "rand",
+ "rmp-serde",
+ "rustls",
+ "serde",
+ "serde_json",
+ "sha2",
+ "sled",
+ "tokio",
+ "toml",
+ "webpki",
+ "xxhash-rust",
+]
+
+[[package]]
+name = "garage_util"
+version = "0.3.0"
 dependencies = [
  "blake2",
  "chrono",
@@ -494,14 +551,14 @@ dependencies = [
 
 [[package]]
 name = "garage_web"
-version = "0.2.1"
+version = "0.3.0"
 dependencies = [
  "err-derive",
  "futures",
  "garage_api",
  "garage_model",
  "garage_table",
- "garage_util",
+ "garage_util 0.3.0",
  "http",
  "hyper",
  "idna",
@@ -1034,6 +1091,16 @@ version = "1.2.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0"
 
+[[package]]
+name = "quick-xml"
+version = "0.21.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0452695941410a58c8ce4391707ba9bad26a247173bd9886a05a5e8a8babec75"
+dependencies = [
+ "memchr",
+ "serde",
+]
+
 [[package]]
 name = "quote"
 version = "1.0.9"

Makefile

@@ -1,19 +1,5 @@
-BIN=target/release/garage
-DOCKER=lxpz/garage_amd64
-
 all:
-	clear; RUSTFLAGS="-C link-arg=-fuse-ld=lld -C target-cpu=x86-64 -C target-feature=+sse2" cargo build --no-default-features
+	clear; cargo build
 
-$(BIN):
+release:
 	RUSTFLAGS="-C link-arg=-fuse-ld=lld -C target-cpu=x86-64 -C target-feature=+sse2" cargo build --release --no-default-features
-
-$(BIN).stripped: $(BIN)
-	cp $^ $@
-	strip $@
-
-docker: $(BIN).stripped
-	docker build -t $(DOCKER):$(TAG) .
-	docker push $(DOCKER):$(TAG)
-	docker tag $(DOCKER):$(TAG) $(DOCKER):latest
-	docker push $(DOCKER):latest
-	

README.md

@@ -1,8 +1,8 @@
-Garage [![Build Status](https://drone.deuxfleurs.fr/api/badges/Deuxfleurs/garage/status.svg)](https://drone.deuxfleurs.fr/Deuxfleurs/garage)
+Garage [![Build Status](https://drone.deuxfleurs.fr/api/badges/Deuxfleurs/garage/status.svg?ref=refs/heads/main)](https://drone.deuxfleurs.fr/Deuxfleurs/garage)
 ===
 
 <p align="center" style="text-align:center;">
-	<a href="https://git.deuxfleurs.fr/Deuxfleurs/garage">
+	<a href="https://garagehq.deuxfleurs.fr">
 	<img alt="Garage logo" src="doc/logo/garage.png" height="200" />
 	</a>
 </p>

config.dev.toml

@@ -1,22 +0,0 @@
-block_size = 1048576			# objects are split in blocks of maximum this number of bytes
-
-metadata_dir = "/tmp/garage-meta"
-data_dir = "/tmp/garage-data"
-
-rpc_bind_addr = "[::]:3901"		# the port other Garage nodes will use to talk to this node
-
-bootstrap_peers = []
-
-max_concurrent_rpc_requests = 12
-data_replication_factor = 3
-meta_replication_factor = 3
-meta_epidemic_fanout = 3
-
-[s3_api]
-api_bind_addr = "[::1]:3900"	# the S3 API port, HTTP without TLS. Add a reverse proxy for the TLS part.
-s3_region = "garage"				# set this to anything. S3 API calls will fail if they are not made against the region set here.
-
-[s3_web]
-bind_addr = "[::1]:3902"
-root_domain = ".garage.tld"
-index = "index.html"

doc/book/src/SUMMARY.md

@@ -14,6 +14,7 @@
   - [Host a website](./cookbook/website.md)
   - [Integrate as a media backend]()
   - [Operate a cluster]()
+  - [Recovering from failures](./cookbook/recovering.md)
 
 - [Reference Manual](./reference_manual/index.md)
   - [Garage CLI]()

doc/book/src/cookbook/index.md

@@ -2,4 +2,4 @@
 
 A cookbook, when you cook, is a collection of recipes.
 Similarly, Garage's cookbook contains a collection of recipes that are known to works well!
-This chapter could also be referred as "Tutorials" or "Best practises".
+This chapter could also be referred as "Tutorials" or "Best practices".

doc/book/src/cookbook/recovering.md

@@ -0,0 +1,99 @@
+# Recovering from failures
+
+Garage is meant to work on old, second-hand hardware.
+In particular, this makes it likely that some of your drives will fail, and some manual intervention will be needed.
+Fear not! For Garage is fully equipped to handle drive failures, in most common cases.
+
+## A note on availability of Garage
+
+With nodes dispersed in 3 datacenters or more, here are the guarantees Garage provides with the default replication strategy (3 copies of all data, which is the recommended value):
+
+- The cluster remains fully functional as long as the machines that fail are in only one datacenter. This includes a whole datacenter going down due to power/Internet outage.
+- No data is lost as long as the machines that fail are in at most two datacenters.
+
+Of course this only works if your Garage nodes are correctly configured to be aware of the datacenter in which they are located.
+Make sure this is the case using `garage status` to check on the state of your cluster's configuration.
+
+
+## First option: removing a node
+
+If you don't have spare parts (HDD, SDD) to replace the failed component, and if there are enough remaining nodes in your cluster
+(at least 3), you can simply remove the failed node from Garage's configuration.
+Note that if you **do** intend to replace the failed parts by new ones, using this method followed by adding back the node is **not recommended** (although it should work),
+and you should instead use one of the methods detailed in the next sections.
+
+Removing a node is done with the following command:
+
+```
+garage node remove --yes <node_id>
+```
+
+(you can get the `node_id` of the failed node by running `garage status`)
+
+This will repartition the data and ensure that 3 copies of everything are present on the nodes that remain available.
+
+
+
+## Replacement scenario 1: only data is lost, metadata is fine
+
+The recommended deployment for Garage uses an SSD to store metadata, and an HDD to store blocks of data.
+In the case where only a single HDD crashes, the blocks of data are lost but the metadata is still fine.
+
+This is very easy to recover by setting up a new HDD to replace the failed one.
+The node does not need to be fully replaced and the configuration doesn't need to change.
+We just need to tell Garage to get back all the data blocks and store them on the new HDD.
+
+First, set up a new HDD to store Garage's data directory on the failed node, and restart Garage using
+the existing configuration.  Then, run:
+
+```
+garage repair -a --yes blocks
+```
+
+This will re-synchronize blocks of data that are missing to the new HDD, reading them from copies located on other nodes.
+
+You can check on the advancement of this process by doing the following command: 
+
+```
+garage stats -a
+```
+
+Look out for the following output:
+
+```
+Block manager stats:
+  resync queue length: 26541
+```
+
+This indicates that one of the Garage node is in the process of retrieving missing data from other nodes.
+This number decreases to zero when the node is fully synchronized.
+
+
+## Replacement scenario 2: metadata (and possibly data) is lost
+
+This scenario covers the case where a full node fails, i.e. both the metadata directory and
+the data directory are lost, as well as the case where only the metadata directory is lost.
+
+To replace the lost node, we will start from an empty metadata directory, which means
+Garage will generate a new node ID for the replacement node.
+We will thus need to remove the previous node ID from Garage's configuration and replace it by the ID of the new node.
+
+If your data directory is stored on a separate drive and is still fine, you can keep it, but it is not necessary to do so.
+In all cases, the data will be rebalanced and the replacement node will not store the same pieces of data
+as were originally stored on the one that failed. So if you keep the data files, the rebalancing
+might be faster but most of the pieces will be deleted anyway from the disk and replaced by other ones.
+
+First, set up a new drive to store the metadata directory for the replacement node (a SSD is recommended),
+and for the data directory if necessary. You can then start Garage on the new node.
+The restarted node should generate a new node ID, and it should be shown as `NOT CONFIGURED` in `garage status`.
+The ID of the lost node should be shown in `garage status` in the section for disconnected/unavailable nodes.
+
+Then, replace the broken node by the new one, using:
+
+```
+garage node configure --replace <old_node_id> \
+		-c <capacity> -d <datacenter> -t <node_tag> <new_node_id>
+```
+
+Garage will then start synchronizing all required data on the new node.
+This process can be monitored using the `garage stats -a` command.

doc/book/src/getting_started/binary.md

@@ -7,14 +7,14 @@ We did not test other architecture/operating system but, as long as your archite
 ## From Docker
 
 Our docker image is currently named `lxpz/garage_amd64` and is stored on the [Docker Hub](https://hub.docker.com/r/lxpz/garage_amd64/tags?page=1&ordering=last_updated).
-We encourage you to use a fixed tag (eg. `v0.1.1d`) and not the `latest` tag.
-For this example, we will use the latest published version at the time of the writing which is `v0.1.1d` but it's up to you
+We encourage you to use a fixed tag (eg. `v0.2.1`) and not the `latest` tag.
+For this example, we will use the latest published version at the time of the writing which is `v0.2.1` but it's up to you
 to check [the most recent versions on the Docker Hub](https://hub.docker.com/r/lxpz/garage_amd64/tags?page=1&ordering=last_updated).
 
 For example:
 
 ```
-sudo docker pull lxpz/garage_amd64:v0.1.1d
+sudo docker pull lxpz/garage_amd64:v0.2.1
 ```
 
 ## From source

doc/book/src/getting_started/bucket.md

@@ -40,22 +40,18 @@ garagectl key new --name nextcloud-app-key
 
 You will have the following output (this one is fake, `key_id` and `secret_key` were generated with the openssl CLI tool):
 
-```javascript
-Key { 
-  key_id: "GK3515373e4c851ebaad366558", 
-  secret_key: "7d37d093435a41f2aab8f13c19ba067d9776c90215f56614adad6ece597dbb34", 
-  name: "nextcloud-app-key", 
-  name_timestamp: 1603280506694, 
-  deleted: false, 
-  authorized_buckets: [] 
-}
+```
+Key name: nextcloud-app-key
+Key ID: GK3515373e4c851ebaad366558
+Secret key: 7d37d093435a41f2aab8f13c19ba067d9776c90215f56614adad6ece597dbb34
+Authorized buckets:
 ```
 
-Check that everything works as intended (be careful, info works only with your key identifier and not with its friendly name!):
+Check that everything works as intended:
 
 ```
 garagectl key list
-garagectl key info GK3515373e4c851ebaad366558
+garagectl key info nextcloud-app-key
 ```
 
 ## Allow a key to access a bucket
@@ -67,7 +63,7 @@ garagectl bucket allow \
   --read \
   --write 
   nextcloud-bucket \
-  --key GK3515373e4c851ebaad366558
+  --key nextcloud-app-key
 ```
 
 You can check at any times allowed keys on your bucket with:

doc/book/src/getting_started/cluster.md

@@ -11,20 +11,20 @@ As this part is not relevant for a test cluster, you can use this one-liner to c
 
 ```bash
 garagectl status | grep UNCONFIGURED | grep -Po '^[0-9a-f]+' | while read id; do 
-    garagectl node configure -d dc1 -n 10 $id
+    garagectl node configure -d dc1 -c 1 $id
 done
 ```
 
 ## Real-world cluster
 
-For our example, we will suppose we have the following infrastructure (Tokens, Identifier and Datacenter are specific values to garage described in the following):
+For our example, we will suppose we have the following infrastructure (Capacity, Identifier and Datacenter are specific values to garage described in the following):
 
-| Location | Name    | Disk Space | `Tokens` | `Identifier` | `Datacenter` |
-|----------|---------|------------|----------|--------------|--------------|
-| Paris    | Mercury | 1 To       | `100`    | `8781c5`     | `par1`       |
-| Paris    | Venus   | 2 To       | `200`    | `2a638e`     | `par1`       |
-| London   | Earth   | 2 To       | `200`    | `68143d`     | `lon1`       |
-| Brussels | Mars    | 1.5 To     | `150`    | `212f75`     | `bru1`       |
+| Location | Name    | Disk Space | `Capacity` | `Identifier` | `Datacenter` |
+|----------|---------|------------|------------|--------------|--------------|
+| Paris    | Mercury | 1 To       | `2`        | `8781c5`     | `par1`       |
+| Paris    | Venus   | 2 To       | `4`        | `2a638e`     | `par1`       |
+| London   | Earth   | 2 To       | `4`        | `68143d`     | `lon1`       |
+| Brussels | Mars    | 1.5 To     | `3`        | `212f75`     | `bru1`       |
 
 ### Identifier
 
@@ -45,14 +45,15 @@ garagectl status
 
 It will display the IP address associated with each node; from the IP address you will be able to recognize the node.
 
-### Tokens
+### Capacity
 
-Garage reasons on an arbitrary metric about disk storage that is named "tokens".
-The number of tokens must be proportional to the disk space dedicated to the node.
-Additionaly, ideally the number of tokens must be in the order of magnitude of 100
-to provide a good trade-off between data load balancing and performances (*this sentence must be verified, it may be wrong*).
+Garage reasons on an arbitrary metric about disk storage that is named the *capacity* of a node.
+The capacity configured in Garage must be proportional to the disk space dedicated to the node.
+Additionaly, the capacity values used in Garage should be as small as possible, with
+1 ideally representing the size of your smallest server.
 
-Here we chose 1 token = 10 Go but you are free to select the value that best fit your needs.
+Here we chose that 1 unit of capacity = 0.5 To, so that we can express servers of size
+1 To and 2 To, as wel as the intermediate size 1.5 To.
 
 ### Datacenter
 
@@ -65,8 +66,8 @@ Behind the scene, garage will try to store the same data on different sites to p
 Given the information above, we will configure our cluster as follow:
 
 ```
-garagectl node configure --datacenter par1 -n 100 -t mercury 8781c5
-garagectl node configure --datacenter par1 -n 200 -t venus 2a638e
-garagectl node configure --datacenter lon1 -n 200 -t earth 68143d
-garagectl node configure --datacenter bru1 -n 150 -t mars 212f75
+garagectl node configure --datacenter par1 -c 2 -t mercury 8781c5
+garagectl node configure --datacenter par1 -c 4 -t venus 2a638e
+garagectl node configure --datacenter lon1 -c 4 -t earth 68143d
+garagectl node configure --datacenter bru1 -c 3 -t mars 212f75
 ```

doc/book/src/getting_started/daemon.md

@@ -124,8 +124,8 @@ For our example, we will suppose the following infrastructure:
 |----------|---------|------------|------------|
 | Paris    | Mercury | fc00:1::1  | 1 To       |
 | Paris    | Venus   | fc00:1::2  | 2 To       |
-| London   | Earth   | fc00:1::2  | 2 To       |
-| Brussels | Mars    | fc00:B::1  | 1.5 To     |
+| London   | Earth   | fc00:B::1  | 2 To       |
+| Brussels | Mars    | fc00:F::1  | 1.5 To     |
 
 On each machine, we will have a similar setup, especially you must consider the following folders/files:
   - `/etc/garage/pki`: Garage certificates, must be generated on your computer and copied on the servers

doc/book/src/intro.md

@@ -1,62 +1,78 @@
-![Garage's Logo](img/logo.svg)
+<p align="center" style="text-align:center;">
+	<a href="https://garagehq.deuxfleurs.fr">
+	<img alt="Garage's Logo" src="img/logo.svg" height="200" />
+	</a>
+</p>
+
+<p align="center" style="text-align:center;">
+	[ <a href="https://git.deuxfleurs.fr/Deuxfleurs/garage">Git repository</a>
+	| <a href="https://matrix.to/#/%23garage:deuxfleurs.fr">Matrix channel</a>
+	| <a href="https://drone.deuxfleurs.fr/Deuxfleurs/garage">Drone CI</a>
+	]
+</p>
+
+```
+This very website is hosted using Garage. In other words: the doc is the PoC!
+```
 
 # The Garage Geo-Distributed Data Store
 
 Garage is a lightweight geo-distributed data store.
 It comes from the observation that despite numerous object stores
 many people have broken data management policies (backup/replication on a single site or none at all).
-To promote better data management policies, with focused on the following desirable properties:
+To promote better data management policies, we focused on the following **desirable properties**:
 
-  - **Self-contained & lightweight**: works everywhere and integrates well in existing environments to target hyperconverged infrastructures
-  - **Highly resilient**: highly resilient to network failures, network latency, disk failures, sysadmin failures
-  - **Simple**: simple to understand, simple to operate, simple to debug
-  - **Internet enabled**: made for multi-sites (eg. datacenter, offices, etc.) interconnected through a regular internet connection.
+  - **Self-contained & lightweight**: works everywhere and integrates well in existing environments to target [hyperconverged infrastructures](https://en.wikipedia.org/wiki/Hyper-converged_infrastructure).
+  - **Highly resilient**: highly resilient to network failures, network latency, disk failures, sysadmin failures.
+  - **Simple**: simple to understand, simple to operate, simple to debug.
+  - **Internet enabled**: made for multi-sites (eg. datacenters, offices, households, etc.) interconnected through regular Internet connections.
 
 We also noted that the pursuit of some other goals are detrimental to our initial goals.
-The following have been identified has non-goals, if it matters to you, you should not use Garage:
+The following has been identified as **non-goals** (if these points matter to you, you should not use Garage):
 
   - **Extreme performances**: high performances constrain a lot the design and the infrastructure; we seek performances through minimalism only.
   - **Feature extensiveness**: complete implementation of the S3 API or any other API to make garage a drop-in replacement is not targeted as it could lead to decisions impacting our desirable properties.
   - **Storage optimizations**: erasure coding or any other coding technique both increase the difficulty of placing data and synchronizing; we limit ourselves to duplication.
-  - **POSIX/Filesystem compatibility**: we do not aim at being POSIX compatible or to emulate any kind of filesystem. Indeed, in a distributed environment, such syncronizations are translated in network messages that impose severe constraints on the deployment.
+  - **POSIX/Filesystem compatibility**: we do not aim at being POSIX compatible or to emulate any kind of filesystem. Indeed, in a distributed environment, such synchronizations are translated in network messages that impose severe constraints on the deployment.
 
 ## Supported and planned protocols
 
 Garage speaks (or will speak) the following protocols:
 
   - [S3](https://docs.aws.amazon.com/AmazonS3/latest/API/Welcome.html) - *SUPPORTED* - Enable applications to store large blobs such as pictures, video, images, documents, etc. S3 is versatile enough to also be used to publish a static website.
-  - [IMAP](https://github.com/go-pluto/pluto) - *PLANNED* - email storage is quite complex to get good oerformances.
-To keep performances optimals, most imap servers only support on-disk storage.
+  - [IMAP](https://github.com/go-pluto/pluto) - *PLANNED* - email storage is quite complex to get good performances.
+To keep performances optimal, most IMAP servers only support on-disk storage.
 We plan to add logic to Garage to make it a viable solution for email storage.
   - *More to come*
 
 ## Use Cases
 
-**[Deuxfleurs](https://deuxfleurs.fr) :** Garage is used by Deuxfleurs which is a non-profit hosting organization.
-Especially, it is used to host their main website, this documentation and some of its members's blogs. Additionally,
-Garage is used as a [backend for Nextcloud](https://docs.nextcloud.com/server/20/admin_manual/configuration_files/primary_storage.html). Deuxfleurs also plans to use Garage as their [Matrix's media backend](https://github.com/matrix-org/synapse-s3-storage-provider) and has the backend of [OCIS](https://github.com/owncloud/ocis).
+**[Deuxfleurs](https://deuxfleurs.fr):** Garage is used by Deuxfleurs which is a non-profit hosting organization.
+Especially, it is used to host their main website, this documentation and some of its members' blogs. 
+Additionally, Garage is used as a [backend for Nextcloud](https://docs.nextcloud.com/server/20/admin_manual/configuration_files/primary_storage.html). 
+Deuxfleurs also plans to use Garage as their [Matrix's media backend](https://github.com/matrix-org/synapse-s3-storage-provider) and as the backend of [OCIS](https://github.com/owncloud/ocis).
 
 *Are you using Garage? [Open a pull request](https://git.deuxfleurs.fr/Deuxfleurs/garage/) to add your organization here!*
 
 ## Comparison to existing software
 
-**[Minio](https://min.io/) :** Minio shares our *self-contained & lightweight* goal but selected two of our non-goals: *storage optimizations* through erasure coding and *POSIX/Filesystem compatibility* through strong consistency.
-However, by pursuing these two non-goals, minio do not reach our desirable properties.
-First, it fails on the *simple* property: due to the erasure coding, minio has severe limitations on how drives can be added or deleted from a cluster.
-Second, it fails on the *interned enabled* property: due to its strong consistency, minio is latency sensitive.
-Furthermore, minio has no knowledge of "sites" and thus can not distribute data to minimize the failure of a given site.
+**[MinIO](https://min.io/):** MinIO shares our *Self-contained & lightweight* goal but selected two of our non-goals: *Storage optimizations* through erasure coding and *POSIX/Filesystem compatibility* through strong consistency.
+However, by pursuing these two non-goals, MinIO do not reach our desirable properties.
+Firstly, it fails on the *Simple* property: due to the erasure coding, MinIO has severe limitations on how drives can be added or deleted from a cluster.
+Secondly, it fails on the *Internet enabled* property: due to its strong consistency, MinIO is latency sensitive.
+Furthermore, MinIO has no knowledge of "sites" and thus can not distribute data to minimize the failure of a given site.
 
-**[Openstack Swift](https://docs.openstack.org/swift/latest/) :**
-OpenStack Swift at least fails on the *self-contained & lightweight* goal.
-Starting it requires around 8Gb of RAM, which is too much especially in an hyperconverged infrastructure.
-It seems also to be far from *Simple*.
+**[Openstack Swift](https://docs.openstack.org/swift/latest/):**
+OpenStack Swift at least fails on the *Self-contained & lightweight* goal.
+Starting it requires around 8GB of RAM, which is too much especially in an hyperconverged infrastructure.
+We also do not classify Swift as *Simple*.
 
-**[Ceph](https://ceph.io/ceph-storage/object-storage/) :**
+**[Ceph](https://ceph.io/ceph-storage/object-storage/):**
 This review holds for the whole Ceph stack, including the RADOS paper, Ceph Object Storage module, the RADOS Gateway, etc.
-At is core, Ceph has been designed to provide *POSIX/Filesystem compatibility* which requires strong consistency, which in turn
-makes Ceph latency sensitive and fails our *Internet enabled* goal. 
-Due to its industry oriented design, Ceph is also far from being *Simple* to operate and from being *self-contained & lightweight* which makes it hard to integrate it in an hyperconverged infrastructure.
-In a certain way, Ceph and Minio are closer togethers than they are from Garage or OpenStack Swift.
+At its core, Ceph has been designed to provide *POSIX/Filesystem compatibility* which requires strong consistency, which in turn
+makes Ceph latency-sensitive and fails our *Internet enabled* goal. 
+Due to its industry oriented design, Ceph is also far from being *Simple* to operate and from being *Self-contained & lightweight* which makes it hard to integrate it in an hyperconverged infrastructure.
+In a certain way, Ceph and MinIO are closer together than they are from Garage or OpenStack Swift.
 
 *More comparisons are available in our [Related Work](design/related_work.md) chapter.*
 
@@ -67,29 +83,31 @@ We reference here other places on the Internet where you can learn more about Ga
 
 ### Rust API (docs.rs)
 
-If you encounter a specific bug in Garage or plan to patch it, you may jump directly to the source code documentation!
+If you encounter a specific bug in Garage or plan to patch it, you may jump directly to the source code's documentation!
 
   - [garage\_api](https://docs.rs/garage_api/latest/garage_api/) - contains the S3 standard API endpoint
   - [garage\_model](https://docs.rs/garage_model/latest/garage_model/) - contains Garage's model built on the table abstraction
   - [garage\_rpc](https://docs.rs/garage_rpc/latest/garage_rpc/) - contains Garage's federation protocol
   - [garage\_table](https://docs.rs/garage_table/latest/garage_table/) - contains core Garage's CRDT datatypes
-  - [garage\_util](https://docs.rs/garage_util/latest/garage_util/) - contains garage entrypoints (daemon, cli)
+  - [garage\_util](https://docs.rs/garage_util/latest/garage_util/) - contains garage helpers
   - [garage\_web](https://docs.rs/garage_web/latest/garage_web/) - contains the S3 website endpoint
 
 ### Talks
 
 We love to talk and hear about Garage, that's why we keep a log here:
 
-  - [(fr, 2020-12-02) Garage : jouer dans la cour des grands quand on est un hébergeur associatif](https://git.deuxfleurs.fr/Deuxfleurs/garage/src/branch/master/doc/20201202_talk/talk.pdf)
+  - [(en, 2021-04-28) Distributed object storage is centralised](https://git.deuxfleurs.fr/Deuxfleurs/garage/raw/commit/b1f60579a13d3c5eba7f74b1775c84639ea9b51a/doc/talks/2021-04-28_spirals-team/talk.pdf)
+
+  - [(fr, 2020-12-02) Garage : jouer dans la cour des grands quand on est un hébergeur associatif](https://git.deuxfleurs.fr/Deuxfleurs/garage/raw/commit/b1f60579a13d3c5eba7f74b1775c84639ea9b51a/doc/talks/2020-12-02_wide-team/talk.pdf)
 
 *Did you write or talk about Garage? [Open a pull request](https://git.deuxfleurs.fr/Deuxfleurs/garage/) to add a link here!*
 
 ## Community
 
 If you want to discuss with us, you can join our Matrix channel at [#garage:deuxfleurs.fr](https://matrix.to/#/#garage:deuxfleurs.fr).
-Our code and our issue tracker, which is the place where you should report bugs, are managed on [Deuxfleurs' Gitea](https://git.deuxfleurs.fr/Deuxfleurs/garage).
+Our code repository and issue tracker, which is the place where you should report bugs, is managed on [Deuxfleurs' Gitea](https://git.deuxfleurs.fr/Deuxfleurs/garage).
 
 ## License
 
-Garage, all the source code, is released under the [AGPL v3 License](https://www.gnu.org/licenses/agpl-3.0.en.html).
+Garage's source code, is released under the [AGPL v3 License](https://www.gnu.org/licenses/agpl-3.0.en.html).
 Please note that if you patch Garage and then use it to provide any service over a network, you must share your code!

doc/talks/2021-04-28_spirals-team/.gitignore

@@ -0,0 +1,10 @@
+*.bbl
+*.aux
+*.bcf
+*.blg
+*.log
+*.run.xml
+*.synctex.gz
+build/
+main.pdf
+figures/crdt.pdf

doc/talks/2021-04-28_spirals-team/Makefile

@@ -0,0 +1,33 @@
+.PHONY: all viewpdf pdf clean
+
+TARGET       = main
+SOURCE_FILES = $(shell find . -type f -name "*.tex" -print)
+CLASS_FILES  = $(shell find . -type f -name "*.cls" -print)
+BIB_FILES    = $(shell find . -type f -name "*.bib" -print)
+FIGURES      = $(shell find . -path "./figures/*" -type f -print)
+BUILD_PATH   = build
+BUILD_FILES  = $(shell find $(BUILD_PATH) -type f -print)
+
+BIB_PROCESSOR := biber
+
+.PHONY: all pdf clean figures
+
+all: pdf
+
+pdf: $(TARGET).pdf
+
+clean:
+	@rm $(TARGET).pdf $(BUILD_FILES) > /dev/null 2>&1 || exit 0
+
+figures: figures/crdt.pdf
+
+$(TARGET).pdf: figures $(FIGURES) $(SOURCE_FILES) $(BIB_FILES) $(CLASS_FILES)
+	@mkdir -p $(BUILD_PATH) > /dev/null 2>&1 || exit 0
+	@pdflatex -interaction=nonstopmode -jobname=$(TARGET) -output-directory $(BUILD_PATH) $(TARGET).tex
+	@$(BIB_PROCESSOR) --output-directory $(BUILD_PATH) $(TARGET)
+	@pdflatex -interaction=nonstopmode -jobname=$(TARGET) -output-directory $(BUILD_PATH) $(TARGET).tex # For biber
+	@pdflatex -interaction=nonstopmode -jobname=$(TARGET) -output-directory $(BUILD_PATH) $(TARGET).tex # For biber
+	@ln -fs $(BUILD_PATH)/$(TARGET).pdf $(TARGET).pdf
+
+figures/crdt.pdf: figures/svg/crdt.svg
+	@inkscape -C --file=$< --export-pdf=$@

doc/talks/2021-04-28_spirals-team/README.md

@@ -0,0 +1 @@
+Presentation of Garage by Adrien on April, 28th 2021, for his research team [Spirals](https://team.inria.fr/spirals/).

doc/talks/2021-04-28_spirals-team/bibliography.bib

@@ -0,0 +1,27 @@
+
+@inproceedings{brewer_towards_2000,
+  title = {Towards {{Robust Distributed Systems}}},
+  booktitle = {{{ACM PODC}}},
+  author = {Brewer, Eric},
+  year = {2000}
+}
+
+
+
+@incollection{defago_conflict-free_2011,
+  title = {Conflict-{{Free Replicated Data Types}}},
+  booktitle = {Stabilization, {{Safety}}, and {{Security}} of {{Distributed Systems}}},
+  author = {Shapiro, Marc and Pregui{\c c}a, Nuno and Baquero, Carlos and Zawirski, Marek},
+  year = {2011},
+  address = {{Berlin, Heidelberg}},
+}
+
+
+@inproceedings{decandia_dynamo:_2007,
+  title = {Dynamo: {{Amazon}}'s {{Highly Available Key}}-Value {{Store}}},
+  booktitle = {{ACM SOSP}},
+  author = {DeCandia, Giuseppe and Hastorun, Deniz and Jampani, Madan and Kakulapati, Gunavardhan and Lakshman, Avinash and Pilchin, Alex and Sivasubramanian, Swaminathan and Vosshall, Peter and Vogels, Werner},
+  year = {2007},
+  address = {{New York, USA}},
+}
+

doc/talks/2021-04-28_spirals-team/conclusion.tex

@@ -0,0 +1,7 @@
+\section{Conclusion}
+
+\begin{frame}{The future is cooler when we bend it our way}
+
+Contributions welcome! :D
+
+\end{frame}

doc/talks/2021-04-28_spirals-team/escaping_the_cloud.tex

@@ -0,0 +1,124 @@
+\section{Escaping the cloud}
+
+% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+% \begin{frame}{Down to Earth with home-hosting}
+
+
+% \todo{Stanley Parabole reference?}
+
+% \end{frame}
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+\begin{frame}{Why?}
+
+\begin{itemize}
+	\item \textbf{Privacy}: no prying eyes besides your ISP
+	\item \textbf{Control} of your infrastructure
+	\item \textbf{Ecology}: reuse old hardware
+\end{itemize}
+
+\vfill
+\begin{block}{\emph{Tim Berners-Lee} (1994)}
+``Now, if someone tries to monopolize the Web, for example pushes proprietary variations on network protocols, then that would make me unhappy.''
+\end{block}
+
+\begin{itemize}
+	\item Make Tim Berners-Lee happy
+\end{itemize}
+
+\end{frame}
+
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+\begin{frame}{What?}
+
+\centering\Large
+A data store for commodity hardware on heterogenous household connections. 
+
+
+\vfill\raggedright\normalsize
+
+
+\begin{block}{Targetting user-facing services}
+\begin{itemize}
+	\item Static sites 
+	\item E-mails
+	\item Instant communication
+	%\item Video streaming % No need for a data store
+	\item Collaboration 
+\end{itemize}
+\end{block}
+\vfill
+
+Nothing fancy like sensors data streams, AI or IoT.
+
+\end{frame}
+
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+\begin{frame}{What?}
+
+
+\begin{block}{Requirements}
+\begin{itemize}
+	\item \textbf{No single point of failure} / flat hierarchy: 
+
+	Any node can die for extended periods of time.
+	\item \textbf{Multi-site}: cluster spans regions/countries.
+	\item \textbf{Acceptable performance}.
+	\item \textbf{Lightweight}: targets legacy hardware.
+	\item \textbf{Conceptually simple}: built for low-tech organisations.
+
+	Adding/maintaining cluster nodes should be easy.
+\end{itemize}
+\end{block}
+\vfill 
+
+\begin{block}{Non-goals}
+\begin{itemize}
+	\item \textbf{Super badass performance}.
+	\item \textbf{NAT traversal} etc.: we require full-mesh connectivity.
+\end{itemize}
+\end{block}
+\end{frame}
+
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+\begin{frame}{How?}
+
+\begin{itemize}
+	\item Theoretically possible with object storage \& CRDTs.
+	\vfill
+	\item Household uplinks are getting decent (optical fibers).
+\end{itemize}
+
+
+
+\end{frame}
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+\begin{frame}{Research Questions}
+
+\begin{itemize}
+	\item Decent performance despite bad inter-node connectivity.
+	\vfill
+
+	\item Tailoring workloads as a function of nodes' capabilities:
+
+	\begin{itemize}
+		\item Make use of low-end nodes (e.g. Raspberry Pis),
+		\item Avoid impeding global performance because of low-end nodes.
+	\end{itemize}
+	\vfill
+
+	\item Building CRDTs for target use-cases:
+
+	\begin{itemize}
+		\item Software engineering: DSL or native code?
+		\item Provide APIs to data store users? Risky?
+	\end{itemize}
+	\vfill
+
+	\item Cluster management: effortless UX, low perf. overhead.
+\end{itemize}
+\end{frame}

doc/talks/2021-04-28_spirals-team/figures/svg/conflict.svg

@@ -0,0 +1,930 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<!-- Created with Inkscape (http://www.inkscape.org/) -->
+
+<svg
+   xmlns:dc="http://purl.org/dc/elements/1.1/"
+   xmlns:cc="http://creativecommons.org/ns#"
+   xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+   xmlns:svg="http://www.w3.org/2000/svg"
+   xmlns="http://www.w3.org/2000/svg"
+   xmlns:xlink="http://www.w3.org/1999/xlink"
+   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
+   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
+   width="119.30722mm"
+   height="121.49291mm"
+   viewBox="0 0 119.30722 121.49291"
+   version="1.1"
+   id="svg8"
+   inkscape:version="0.92.5 (2060ec1f9f, 2020-04-08)"
+   sodipodi:docname="conflict.svg">
+  <defs
+     id="defs2">
+    <marker
+       inkscape:stockid="Arrow1Mend"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="marker2999"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         inkscape:connector-curvature="0"
+         id="path2997"
+         d="M 0,0 5,-5 -12.5,0 5,5 Z"
+         style="fill:#000000;fill-opacity:1;fill-rule:evenodd;stroke:#000000;stroke-width:1.00000003pt;stroke-opacity:1"
+         transform="matrix(-0.4,0,0,-0.4,-4,0)" />
+    </marker>
+    <marker
+       inkscape:stockid="Arrow1Mend"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="marker2964"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         inkscape:connector-curvature="0"
+         id="path2962"
+         d="M 0,0 5,-5 -12.5,0 5,5 Z"
+         style="fill:#000000;fill-opacity:1;fill-rule:evenodd;stroke:#000000;stroke-width:1.00000003pt;stroke-opacity:1"
+         transform="matrix(-0.4,0,0,-0.4,-4,0)" />
+    </marker>
+    <marker
+       inkscape:stockid="Arrow1Mend"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="marker1467"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         inkscape:connector-curvature="0"
+         id="path1465"
+         d="M 0,0 5,-5 -12.5,0 5,5 Z"
+         style="fill:#000000;fill-opacity:1;fill-rule:evenodd;stroke:#000000;stroke-width:1.00000003pt;stroke-opacity:1"
+         transform="matrix(-0.4,0,0,-0.4,-4,0)" />
+    </marker>
+    <marker
+       inkscape:isstock="true"
+       style="overflow:visible"
+       id="marker1449"
+       refX="0"
+       refY="0"
+       orient="auto"
+       inkscape:stockid="Arrow1Mend">
+      <path
+         inkscape:connector-curvature="0"
+         transform="matrix(-0.4,0,0,-0.4,-4,0)"
+         style="fill:#000000;fill-opacity:1;fill-rule:evenodd;stroke:#000000;stroke-width:1.00000003pt;stroke-opacity:1"
+         d="M 0,0 5,-5 -12.5,0 5,5 Z"
+         id="path1447" />
+    </marker>
+    <marker
+       inkscape:isstock="true"
+       style="overflow:visible"
+       id="marker12071"
+       refX="0"
+       refY="0"
+       orient="auto"
+       inkscape:stockid="Arrow1Mend"
+       inkscape:collect="always">
+      <path
+         transform="matrix(-0.4,0,0,-0.4,-4,0)"
+         style="fill:#000000;fill-opacity:1;fill-rule:evenodd;stroke:#000000;stroke-width:1.00000003pt;stroke-opacity:1"
+         d="M 0,0 5,-5 -12.5,0 5,5 Z"
+         id="path12069"
+         inkscape:connector-curvature="0" />
+    </marker>
+    <marker
+       inkscape:isstock="true"
+       style="overflow:visible"
+       id="marker9668"
+       refX="0"
+       refY="0"
+       orient="auto"
+       inkscape:stockid="Arrow1Mend"
+       inkscape:collect="always">
+      <path
+         transform="matrix(-0.4,0,0,-0.4,-4,0)"
+         style="fill:#000000;fill-opacity:1;fill-rule:evenodd;stroke:#000000;stroke-width:1.00000003pt;stroke-opacity:1"
+         d="M 0,0 5,-5 -12.5,0 5,5 Z"
+         id="path9666"
+         inkscape:connector-curvature="0" />
+    </marker>
+    <marker
+       inkscape:stockid="Arrow1Mend"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="marker6943"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         id="path6941"
+         d="M 0,0 5,-5 -12.5,0 5,5 Z"
+         style="fill:#000000;fill-opacity:1;fill-rule:evenodd;stroke:#000000;stroke-width:1.00000003pt;stroke-opacity:1"
+         transform="matrix(-0.4,0,0,-0.4,-4,0)"
+         inkscape:connector-curvature="0" />
+    </marker>
+    <marker
+       inkscape:isstock="true"
+       style="overflow:visible"
+       id="marker5445"
+       refX="0"
+       refY="0"
+       orient="auto"
+       inkscape:stockid="Arrow1Mend">
+      <path
+         transform="matrix(-0.4,0,0,-0.4,-4,0)"
+         style="fill:#000000;fill-opacity:1;fill-rule:evenodd;stroke:#000000;stroke-width:1.00000003pt;stroke-opacity:1"
+         d="M 0,0 5,-5 -12.5,0 5,5 Z"
+         id="path5443"
+         inkscape:connector-curvature="0" />
+    </marker>
+    <marker
+       inkscape:stockid="Arrow1Mend"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="marker1115"
+       style="overflow:visible"
+       inkscape:isstock="true"
+       inkscape:collect="always">
+      <path
+         id="path861"
+         d="M 0,0 5,-5 -12.5,0 5,5 Z"
+         style="fill:#000000;fill-opacity:1;fill-rule:evenodd;stroke:#000000;stroke-width:1.00000003pt;stroke-opacity:1"
+         transform="matrix(-0.4,0,0,-0.4,-4,0)"
+         inkscape:connector-curvature="0" />
+    </marker>
+    <marker
+       inkscape:stockid="Arrow1Mend"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="Arrow1Mend"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         id="path826"
+         d="M 0,0 5,-5 -12.5,0 5,5 Z"
+         style="fill:#000000;fill-opacity:1;fill-rule:evenodd;stroke:#000000;stroke-width:1.00000003pt;stroke-opacity:1"
+         transform="matrix(-0.4,0,0,-0.4,-4,0)"
+         inkscape:connector-curvature="0" />
+    </marker>
+    <marker
+       inkscape:stockid="Arrow1Lend"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="Arrow1Lend"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         id="path820"
+         d="M 0,0 5,-5 -12.5,0 5,5 Z"
+         style="fill:#000000;fill-opacity:1;fill-rule:evenodd;stroke:#000000;stroke-width:1.00000003pt;stroke-opacity:1"
+         transform="matrix(-0.8,0,0,-0.8,-10,0)"
+         inkscape:connector-curvature="0" />
+    </marker>
+    <marker
+       inkscape:stockid="Arrow1Lstart"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="marker1101"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         id="path1099"
+         d="M 0,0 5,-5 -12.5,0 5,5 Z"
+         style="fill:#000000;fill-opacity:1;fill-rule:evenodd;stroke:#000000;stroke-width:1.00000003pt;stroke-opacity:1"
+         transform="matrix(0.8,0,0,0.8,10,0)"
+         inkscape:connector-curvature="0" />
+    </marker>
+    <marker
+       inkscape:stockid="Arrow1Lstart"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="Arrow1Lstart"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         id="path817"
+         d="M 0,0 5,-5 -12.5,0 5,5 Z"
+         style="fill:#000000;fill-opacity:1;fill-rule:evenodd;stroke:#000000;stroke-width:1.00000003pt;stroke-opacity:1"
+         transform="matrix(0.8,0,0,0.8,10,0)"
+         inkscape:connector-curvature="0" />
+    </marker>
+  </defs>
+  <sodipodi:namedview
+     id="base"
+     pagecolor="#ffffff"
+     bordercolor="#666666"
+     borderopacity="1.0"
+     inkscape:pageopacity="0.0"
+     inkscape:pageshadow="2"
+     inkscape:zoom="1.4"
+     inkscape:cx="244.86596"
+     inkscape:cy="205.06317"
+     inkscape:document-units="mm"
+     inkscape:current-layer="layer5"
+     showgrid="false"
+     inkscape:window-width="1920"
+     inkscape:window-height="1055"
+     inkscape:window-x="0"
+     inkscape:window-y="0"
+     inkscape:window-maximized="1"
+     fit-margin-top="0"
+     fit-margin-left="0"
+     fit-margin-right="0"
+     fit-margin-bottom="0"
+     inkscape:snap-global="false" />
+  <metadata
+     id="metadata5">
+    <rdf:RDF>
+      <cc:Work
+         rdf:about="">
+        <dc:format>image/svg+xml</dc:format>
+        <dc:type
+           rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
+        <dc:title></dc:title>
+      </cc:Work>
+    </rdf:RDF>
+  </metadata>
+  <g
+     inkscape:groupmode="layer"
+     id="layer2"
+     inkscape:label="Common"
+     transform="translate(-43.663712,-16.067619)"
+     style="display:inline">
+    <path
+       style="fill:none;fill-rule:evenodd;stroke:#000000;stroke-width:0.5291667;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
+       d="m 99.856918,34.684511 h 4.913692"
+       id="path1257"
+       inkscape:connector-curvature="0" />
+    <path
+       style="fill:none;fill-rule:evenodd;stroke:#000000;stroke-width:0.79374999;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1;marker-end:url(#Arrow1Mend)"
+       d="M 102.31376,23.208557 V 135.9963"
+       id="path815"
+       inkscape:connector-curvature="0" />
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:4.93888855px;line-height:125%;font-family:Iwona;-inkscape-font-specification:'Iwona, Normal';font-variant-ligatures:normal;font-variant-caps:normal;font-variant-numeric:normal;font-feature-settings:normal;text-align:start;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:start;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.26458332px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+       x="90.524658"
+       y="19.57423"
+       id="text838"><tspan
+         sodipodi:role="line"
+         id="tspan836"
+         x="90.524658"
+         y="19.57423"
+         style="font-weight:bold;stroke-width:0.26458332px"># vaccines</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:4.93888855px;line-height:125%;font-family:Iwona;-inkscape-font-specification:'Iwona, Normal';font-variant-ligatures:normal;font-variant-caps:normal;font-variant-numeric:normal;font-feature-settings:normal;text-align:start;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:start;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.26458332px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+       x="46.748138"
+       y="19.57423"
+       id="text13775"><tspan
+         sodipodi:role="line"
+         id="tspan13773"
+         x="46.748138"
+         y="19.57423"
+         style="font-style:italic;font-variant:normal;font-weight:normal;font-stretch:normal;font-family:Iwona;-inkscape-font-specification:'Iwona Italic';stroke-width:0.26458332px">Paris</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:4.93888855px;line-height:125%;font-family:Iwona;-inkscape-font-specification:'Iwona, Normal';font-variant-ligatures:normal;font-variant-caps:normal;font-variant-numeric:normal;font-feature-settings:normal;text-align:start;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:start;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.26458332px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+       x="148.20457"
+       y="19.57423"
+       id="text13798"><tspan
+         sodipodi:role="line"
+         id="tspan13796"
+         x="148.20457"
+         y="19.57423"
+         style="font-style:italic;font-variant:normal;font-weight:normal;font-stretch:normal;font-family:Iwona;-inkscape-font-specification:'Iwona Italic';stroke-width:0.26458332px">Lille</tspan></text>
+  </g>
+  <g
+     inkscape:label="The problem"
+     inkscape:groupmode="layer"
+     id="layer1"
+     style="display:none"
+     transform="translate(-43.663712,-16.067619)">
+    <g
+       id="g13753"
+       transform="translate(0,12.170833)">
+      <path
+         inkscape:connector-curvature="0"
+         id="path850"
+         d="M 97.820644,51.619712 75.828475,64.316897"
+         style="fill:none;fill-rule:evenodd;stroke:#000000;stroke-width:0.52969581;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1;marker-end:url(#marker1115)" />
+      <text
+         y="-108.69946"
+         x="-183.00148"
+         transform="rotate(180,91.874093,54.229103)"
+         id="text7803"
+         style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:4.93888855px;line-height:125%;font-family:Iwona;-inkscape-font-specification:'Iwona, Normal';font-variant-ligatures:normal;font-variant-caps:normal;font-variant-numeric:normal;font-feature-settings:normal;text-align:start;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:start;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.26458332px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+         xml:space="preserve"><textPath
+           id="textPath7973"
+           xlink:href="#path850"><tspan
+   id="tspan7801"
+   style="font-weight:bold;stroke-width:0.26458332px">R</tspan></textPath></text>
+      <path
+         style="fill:none;fill-rule:evenodd;stroke:#000000;stroke-width:0.52969581;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1;marker-end:url(#marker9668)"
+         d="M 75.056888,72.78638 97.049057,85.483565"
+         id="path9498"
+         inkscape:connector-curvature="0" />
+      <text
+         transform="translate(9.4212508,3.3408691)"
+         y="2.986012"
+         x="8.5186663"
+         id="text10668"
+         style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:4.93888855px;line-height:125%;font-family:Iwona;-inkscape-font-specification:'Iwona, Normal';font-variant-ligatures:normal;font-variant-caps:normal;font-variant-numeric:normal;font-feature-settings:normal;text-align:start;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:start;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.26458332px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+         xml:space="preserve"><textPath
+           id="textPath10836"
+           xlink:href="#path9498"><tspan
+   id="tspan10666"
+   style="font-weight:bold;stroke-width:0.26458332px">W</tspan></textPath></text>
+    </g>
+    <g
+       id="g13763"
+       transform="translate(0,12.170833)">
+      <path
+         style="fill:none;fill-rule:evenodd;stroke:#000000;stroke-width:0.52969581;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1;marker-end:url(#marker12071)"
+         d="m 106.80689,51.619712 21.99218,12.697185"
+         id="path11899"
+         inkscape:connector-curvature="0" />
+      <path
+         inkscape:connector-curvature="0"
+         id="path11901"
+         d="M 129.57066,72.78638 107.57848,85.483565"
+         style="fill:none;fill-rule:evenodd;stroke:#000000;stroke-width:0.52969581;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1;marker-end:url(#marker9668)" />
+      <text
+         xml:space="preserve"
+         style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:4.93888855px;line-height:125%;font-family:Iwona;-inkscape-font-specification:'Iwona, Normal';font-variant-ligatures:normal;font-variant-caps:normal;font-variant-numeric:normal;font-feature-settings:normal;text-align:start;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:start;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.26458332px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+         id="text13737"
+         transform="rotate(-115.89278,108.31787,47.643413)"
+         x="-372.04132"
+         y="-217.15767"><textPath
+           xlink:href="#path850"
+           id="textPath13735"><tspan
+   style="font-weight:bold;stroke-width:0.26458332px"
+   id="tspan13733">R</tspan></textPath></text>
+      <text
+         transform="rotate(-58.856754,100.88692,38.870944)"
+         xml:space="preserve"
+         style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:4.93888855px;line-height:125%;font-family:Iwona;-inkscape-font-specification:'Iwona, Normal';font-variant-ligatures:normal;font-variant-caps:normal;font-variant-numeric:normal;font-feature-settings:normal;text-align:start;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:start;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.26458332px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+         id="text13743"
+         x="46.604572"
+         y="6.6609678"><textPath
+           xlink:href="#path9498"
+           id="textPath13741"><tspan
+   style="font-weight:bold;stroke-width:0.26458332px"
+   id="tspan13739">W</tspan></textPath></text>
+    </g>
+    <path
+       inkscape:connector-curvature="0"
+       id="path13768"
+       d="m 99.856918,112.89149 h 4.913692"
+       style="fill:none;fill-rule:evenodd;stroke:#000000;stroke-width:0.5291667;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1" />
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:4.93888855px;line-height:125%;font-family:Iwona;-inkscape-font-specification:'Iwona, Normal';font-variant-ligatures:normal;font-variant-caps:normal;font-variant-numeric:normal;font-feature-settings:normal;text-align:start;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:start;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.26458332px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+       x="43.564934"
+       y="53.51746"
+       id="text13802"><tspan
+         sodipodi:role="line"
+         id="tspan13800"
+         x="43.564934"
+         y="53.51746"
+         style="stroke-width:0.26458332px">We acquired 1000 doses!</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:4.93888855px;line-height:125%;font-family:Iwona;-inkscape-font-specification:'Iwona, Normal';font-variant-ligatures:normal;font-variant-caps:normal;font-variant-numeric:normal;font-feature-settings:normal;text-align:start;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:start;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.26458332px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+       x="49.979401"
+       y="82.649841"
+       id="text13806"><tspan
+         sodipodi:role="line"
+         id="tspan13804"
+         x="49.979401"
+         y="82.649841"
+         style="stroke-width:0.26458332px">x + 1000</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:4.93888855px;line-height:125%;font-family:Iwona;-inkscape-font-specification:'Iwona, Normal';font-variant-ligatures:normal;font-variant-caps:normal;font-variant-numeric:normal;font-feature-settings:normal;text-align:start;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:start;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.26458332px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+       x="136.842"
+       y="82.649841"
+       id="text13810"><tspan
+         sodipodi:role="line"
+         id="tspan13808"
+         x="136.842"
+         y="82.649841"
+         style="stroke-width:0.26458332px">x + 500</tspan></text>
+    <text
+       id="text13814"
+       y="53.51746"
+       x="112.88577"
+       style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:4.93888855px;line-height:125%;font-family:Iwona;-inkscape-font-specification:'Iwona, Normal';font-variant-ligatures:normal;font-variant-caps:normal;font-variant-numeric:normal;font-feature-settings:normal;text-align:start;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:start;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.26458332px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+       xml:space="preserve"><tspan
+         style="stroke-width:0.26458332px"
+         y="53.51746"
+         x="112.88577"
+         id="tspan13812"
+         sodipodi:role="line">We acquired 500 doses!</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:4.93888855px;line-height:125%;font-family:Iwona;-inkscape-font-specification:'Iwona, Normal';font-variant-ligatures:normal;font-variant-caps:normal;font-variant-numeric:normal;font-feature-settings:normal;text-align:start;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:start;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.26458332px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+       x="108.70142"
+       y="114.62009"
+       id="text13818"><tspan
+         sodipodi:role="line"
+         id="tspan13816"
+         x="108.70142"
+         y="114.62009"
+         style="font-weight:bold;stroke-width:0.26458332px">???</tspan></text>
+    <text
+       id="text3109"
+       y="36.388439"
+       x="108.80019"
+       style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:4.23333311px;line-height:125%;font-family:Iwona;-inkscape-font-specification:'Iwona, Normal';font-variant-ligatures:normal;font-variant-caps:normal;font-variant-numeric:normal;text-align:start;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:start;display:inline;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.26458332px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+       xml:space="preserve"><tspan
+         style="font-weight:normal;font-size:4.93888903px;stroke-width:0.26458332px"
+         y="36.388439"
+         x="108.80019"
+         id="tspan3107"
+         sodipodi:role="line">T0: 5000</tspan></text>
+  </g>
+  <g
+     inkscape:groupmode="layer"
+     id="layer3"
+     inkscape:label="ACID"
+     style="display:none">
+    <rect
+       ry="5.2916665"
+       y="67.923912"
+       x="61.174206"
+       height="39.498508"
+       width="54.42857"
+       id="rect14092"
+       style="opacity:1;vector-effect:none;fill:#f9f9f9;fill-opacity:0.97409327;fill-rule:nonzero;stroke:#000000;stroke-width:0.26458332;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:0.91191711;paint-order:fill markers stroke" />
+    <rect
+       style="opacity:1;vector-effect:none;fill:#f9f9f9;fill-opacity:0.97409327;fill-rule:nonzero;stroke:#000000;stroke-width:0.26458332;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:0.91191711;paint-order:fill markers stroke"
+       id="rect14090"
+       width="54.42857"
+       height="39.498508"
+       x="1.6973262"
+       y="21.886396"
+       ry="5.2916665" />
+    <path
+       inkscape:connector-curvature="0"
+       id="path14037"
+       d="m 56.193207,64.654408 h 4.913692"
+       style="fill:none;fill-rule:evenodd;stroke:#000000;stroke-width:0.5291667;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1" />
+    <path
+       style="fill:none;fill-rule:evenodd;stroke:#000000;stroke-width:0.5291667;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
+       d="m 56.193207,110.69192 h 4.913692"
+       id="path14040"
+       inkscape:connector-curvature="0" />
+    <g
+       id="g14075"
+       transform="translate(0,-1.9988201)">
+      <g
+         id="g13993"
+         transform="translate(-43.663711,-25.06345)"
+         style="display:inline">
+        <path
+           inkscape:connector-curvature="0"
+           id="path13977"
+           d="M 97.820644,51.619712 75.828475,64.316897"
+           style="fill:none;fill-rule:evenodd;stroke:#000000;stroke-width:0.52969581;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1;marker-end:url(#marker1115)" />
+        <text
+           y="-108.69946"
+           x="-183.00148"
+           transform="rotate(180,91.874093,54.229103)"
+           id="text13983"
+           style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:4.93888855px;line-height:125%;font-family:Iwona;-inkscape-font-specification:'Iwona, Normal';font-variant-ligatures:normal;font-variant-caps:normal;font-variant-numeric:normal;font-feature-settings:normal;text-align:start;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:start;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.26458332px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+           xml:space="preserve"><textPath
+             id="textPath13981"
+             xlink:href="#path850"><tspan
+   id="tspan13979"
+   style="font-weight:bold;stroke-width:0.26458332px">R</tspan></textPath></text>
+        <path
+           style="fill:none;fill-rule:evenodd;stroke:#000000;stroke-width:0.52969581;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1;marker-end:url(#marker9668)"
+           d="M 75.056888,72.78638 97.049057,85.483565"
+           id="path13985"
+           inkscape:connector-curvature="0" />
+        <text
+           transform="translate(9.4212508,3.3408691)"
+           y="2.986012"
+           x="8.5186663"
+           id="text13991"
+           style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:4.93888855px;line-height:125%;font-family:Iwona;-inkscape-font-specification:'Iwona, Normal';font-variant-ligatures:normal;font-variant-caps:normal;font-variant-numeric:normal;font-feature-settings:normal;text-align:start;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:start;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.26458332px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+           xml:space="preserve"><textPath
+             id="textPath13989"
+             xlink:href="#path9498"><tspan
+   id="tspan13987"
+   style="font-weight:bold;stroke-width:0.26458332px">W</tspan></textPath></text>
+      </g>
+      <text
+         id="text14044"
+         y="29.833508"
+         x="3.4844763"
+         style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:4.93888855px;line-height:125%;font-family:Iwona;-inkscape-font-specification:'Iwona, Normal';font-variant-ligatures:normal;font-variant-caps:normal;font-variant-numeric:normal;font-feature-settings:normal;text-align:start;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:start;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.26458332px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+         xml:space="preserve"><tspan
+           style="stroke-width:0.26458332px"
+           y="29.833508"
+           x="3.4844763"
+           id="tspan14042"
+           sodipodi:role="line">AcqDoses(1000)</tspan></text>
+    </g>
+    <g
+       id="g14088"
+       transform="translate(0,-0.41130478)">
+      <g
+         id="g14011"
+         transform="translate(-43.663711,19.386549)"
+         style="display:inline">
+        <path
+           style="fill:none;fill-rule:evenodd;stroke:#000000;stroke-width:0.52969581;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1;marker-end:url(#marker12071)"
+           d="m 106.80689,51.619712 21.99218,12.697185"
+           id="path13995"
+           inkscape:connector-curvature="0" />
+        <path
+           inkscape:connector-curvature="0"
+           id="path13997"
+           d="M 129.57066,72.78638 107.57848,85.483565"
+           style="fill:none;fill-rule:evenodd;stroke:#000000;stroke-width:0.52969581;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1;marker-end:url(#marker9668)" />
+        <text
+           xml:space="preserve"
+           style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:4.93888855px;line-height:125%;font-family:Iwona;-inkscape-font-specification:'Iwona, Normal';font-variant-ligatures:normal;font-variant-caps:normal;font-variant-numeric:normal;font-feature-settings:normal;text-align:start;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:start;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.26458332px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+           id="text14003"
+           transform="rotate(-115.89278,108.31787,47.643413)"
+           x="-372.04132"
+           y="-217.15767"><textPath
+             xlink:href="#path850"
+             id="textPath14001"><tspan
+   style="font-weight:bold;stroke-width:0.26458332px"
+   id="tspan13999">R</tspan></textPath></text>
+        <text
+           transform="rotate(-58.856754,100.88692,38.870944)"
+           xml:space="preserve"
+           style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:4.93888855px;line-height:125%;font-family:Iwona;-inkscape-font-specification:'Iwona, Normal';font-variant-ligatures:normal;font-variant-caps:normal;font-variant-numeric:normal;font-feature-settings:normal;text-align:start;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:start;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.26458332px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+           id="text14009"
+           x="46.604572"
+           y="6.6609678"><textPath
+             xlink:href="#path9498"
+             id="textPath14007"><tspan
+   style="font-weight:bold;stroke-width:0.26458332px"
+   id="tspan14005">W</tspan></textPath></text>
+      </g>
+      <text
+         id="text14044-8"
+         y="74.283508"
+         x="82.76487"
+         style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:4.93888855px;line-height:125%;font-family:Iwona;-inkscape-font-specification:'Iwona, Normal';font-variant-ligatures:normal;font-variant-caps:normal;font-variant-numeric:normal;font-feature-settings:normal;text-align:start;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:start;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.26458332px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+         xml:space="preserve"><tspan
+           style="stroke-width:0.26458332px"
+           y="74.283508"
+           x="82.76487"
+           id="tspan14042-8"
+           sodipodi:role="line">AcqDoses(500)</tspan></text>
+    </g>
+    <text
+       transform="translate(-43.663711,-16.067619)"
+       xml:space="preserve"
+       style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:4.23333311px;line-height:125%;font-family:Iwona;-inkscape-font-specification:'Iwona, Normal';font-variant-ligatures:normal;font-variant-caps:normal;font-variant-numeric:normal;text-align:start;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:start;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.26458332px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+       x="108.80019"
+       y="82.757004"
+       id="text832-6"><tspan
+         sodipodi:role="line"
+         id="tspan830-8"
+         x="108.80019"
+         y="82.757004"
+         style="font-weight:normal;font-size:4.93888903px;stroke-width:0.26458332px">T1: 6000</tspan></text>
+    <text
+       transform="translate(-43.663711,-16.067619)"
+       xml:space="preserve"
+       style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:4.23333311px;line-height:125%;font-family:Iwona;-inkscape-font-specification:'Iwona, Normal';font-variant-ligatures:normal;font-variant-caps:normal;font-variant-numeric:normal;text-align:start;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:start;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.26458332px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+       x="108.80019"
+       y="128.8701"
+       id="text832-3"><tspan
+         sodipodi:role="line"
+         id="tspan830-83"
+         x="108.80019"
+         y="128.8701"
+         style="font-weight:normal;font-size:4.93888903px;stroke-width:0.26458332px">T2: 6500</tspan></text>
+    <text
+       transform="translate(-43.663711,-16.067619)"
+       xml:space="preserve"
+       style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:4.23333311px;line-height:125%;font-family:Iwona;-inkscape-font-specification:'Iwona, Normal';font-variant-ligatures:normal;font-variant-caps:normal;font-variant-numeric:normal;text-align:start;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:start;display:inline;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.26458332px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+       x="108.80019"
+       y="36.388439"
+       id="text3117"><tspan
+         sodipodi:role="line"
+         id="tspan3115"
+         x="108.80019"
+         y="36.388439"
+         style="font-weight:normal;font-size:4.93888903px;stroke-width:0.26458332px">T0: 5000</tspan></text>
+  </g>
+  <g
+     inkscape:groupmode="layer"
+     id="layer4"
+     inkscape:label="BASE"
+     style="display:none">
+    <g
+       id="g1463"
+       transform="translate(0,-21.166667)">
+      <path
+         style="fill:none;fill-rule:evenodd;stroke:#000000;stroke-width:0.52969581;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1;marker-end:url(#marker1449)"
+         d="M 54.156933,47.722926 32.164764,60.420111"
+         id="path1369"
+         inkscape:connector-curvature="0" />
+      <text
+         xml:space="preserve"
+         style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:4.93888855px;line-height:125%;font-family:Iwona;-inkscape-font-specification:'Iwona, Normal';font-variant-ligatures:normal;font-variant-caps:normal;font-variant-numeric:normal;font-feature-settings:normal;text-align:start;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:start;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.26458332px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+         id="text1375"
+         transform="rotate(180,70.308075,52.117558)"
+         x="-279.42224"
+         y="-209.36409"><textPath
+           xlink:href="#path850"
+           id="textPath1373"><tspan
+   style="font-weight:bold;stroke-width:0.26458332px"
+   id="tspan1371">R</tspan></textPath></text>
+    </g>
+    <g
+       id="g1481"
+       transform="translate(0,-21.166667)">
+      <path
+         inkscape:connector-curvature="0"
+         id="path1377"
+         d="M 31.393177,68.889594 53.385346,81.586779"
+         style="fill:none;fill-rule:evenodd;stroke:#000000;stroke-width:0.52969581;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1;marker-end:url(#marker1467)" />
+      <text
+         transform="translate(-34.395836)"
+         xml:space="preserve"
+         style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:4.93888855px;line-height:125%;font-family:Iwona;-inkscape-font-specification:'Iwona, Normal';font-variant-ligatures:normal;font-variant-caps:normal;font-variant-numeric:normal;font-feature-settings:normal;text-align:start;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:start;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.26458332px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+         id="text1383"
+         x="17.939917"
+         y="6.3268809"><textPath
+           xlink:href="#path9498"
+           id="textPath1381"><tspan
+   style="font-weight:bold;stroke-width:0.26458332px"
+   id="tspan1379">W</tspan></textPath></text>
+    </g>
+    <g
+       style="display:inline"
+       transform="translate(-43.663711,-25.063453)"
+       id="g1403">
+      <path
+         inkscape:connector-curvature="0"
+         id="path1387"
+         d="m 106.80689,51.619712 21.99218,12.697185"
+         style="fill:none;fill-rule:evenodd;stroke:#000000;stroke-width:0.52969581;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1;marker-end:url(#marker12071)" />
+      <path
+         style="fill:none;fill-rule:evenodd;stroke:#000000;stroke-width:0.52969581;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1;marker-end:url(#marker9668)"
+         d="M 129.57066,72.78638 107.57848,85.483565"
+         id="path1389"
+         inkscape:connector-curvature="0" />
+      <text
+         y="-217.15767"
+         x="-372.04132"
+         transform="rotate(-115.89278,108.31787,47.643413)"
+         id="text1395"
+         style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:4.93888855px;line-height:125%;font-family:Iwona;-inkscape-font-specification:'Iwona, Normal';font-variant-ligatures:normal;font-variant-caps:normal;font-variant-numeric:normal;font-feature-settings:normal;text-align:start;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:start;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.26458332px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+         xml:space="preserve"><textPath
+           id="textPath1393"
+           xlink:href="#path850"><tspan
+   id="tspan1391"
+   style="font-weight:bold;stroke-width:0.26458332px">R</tspan></textPath></text>
+      <text
+         y="6.6609678"
+         x="46.604572"
+         id="text1401"
+         style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:4.93888855px;line-height:125%;font-family:Iwona;-inkscape-font-specification:'Iwona, Normal';font-variant-ligatures:normal;font-variant-caps:normal;font-variant-numeric:normal;font-feature-settings:normal;text-align:start;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:start;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.26458332px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+         xml:space="preserve"
+         transform="rotate(-58.856754,100.88692,38.870944)"><textPath
+           id="textPath1399"
+           xlink:href="#path9498"><tspan
+   id="tspan1397"
+   style="font-weight:bold;stroke-width:0.26458332px">W</tspan></textPath></text>
+    </g>
+    <path
+       inkscape:connector-curvature="0"
+       id="path1503"
+       d="m 56.193207,73.650233 h 4.913692"
+       style="fill:none;fill-rule:evenodd;stroke:#000000;stroke-width:0.5291667;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1" />
+    <text
+       id="text1507"
+       y="75.354156"
+       x="64.607315"
+       style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:4.23333311px;line-height:125%;font-family:Iwona;-inkscape-font-specification:'Iwona, Normal';font-variant-ligatures:normal;font-variant-caps:normal;font-variant-numeric:normal;text-align:start;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:start;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.26458332px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+       xml:space="preserve"><tspan
+         style="font-weight:normal;font-size:4.93888903px;stroke-width:0.26458332px"
+         y="75.354156"
+         x="64.607315"
+         id="tspan1505"
+         sodipodi:role="line">T1: [(T0, 6000),(T0, 5500)]</tspan></text>
+    <g
+       id="g1530"
+       transform="translate(0,38.100002)">
+      <path
+         inkscape:connector-curvature="0"
+         id="path1522"
+         d="M 54.156933,47.722926 32.164764,60.420111"
+         style="fill:none;fill-rule:evenodd;stroke:#000000;stroke-width:0.52969581;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1;marker-end:url(#marker1449)" />
+      <text
+         y="-209.36409"
+         x="-279.42224"
+         transform="rotate(180,70.308075,52.117558)"
+         id="text1528"
+         style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:4.93888855px;line-height:125%;font-family:Iwona;-inkscape-font-specification:'Iwona, Normal';font-variant-ligatures:normal;font-variant-caps:normal;font-variant-numeric:normal;font-feature-settings:normal;text-align:start;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:start;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.26458332px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+         xml:space="preserve"><textPath
+           id="textPath1526"
+           xlink:href="#path850"><tspan
+   id="tspan1524"
+   style="font-weight:bold;stroke-width:0.26458332px">R</tspan></textPath></text>
+    </g>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:4.93888855px;line-height:125%;font-family:Iwona;-inkscape-font-specification:'Iwona, Bold';font-variant-ligatures:normal;font-variant-caps:normal;font-variant-numeric:normal;text-align:start;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:start;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.26458332px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+       x="8.5044641"
+       y="109.58665"
+       id="text1534"><tspan
+         sodipodi:role="line"
+         id="tspan1532"
+         x="8.5044641"
+         y="109.58665"
+         style="font-weight:normal;stroke-width:0.26458332px">Now solve this!</tspan><tspan
+         sodipodi:role="line"
+         x="8.5044641"
+         y="115.76025"
+         style="font-weight:normal;stroke-width:0.26458332px"
+         id="tspan3159">And write back!</tspan></text>
+    <text
+       id="text1538"
+       y="44.886383"
+       x="6.3156896"
+       style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:4.93888855px;line-height:125%;font-family:Iwona;-inkscape-font-specification:'Iwona, Normal';font-variant-ligatures:normal;font-variant-caps:normal;font-variant-numeric:normal;font-feature-settings:normal;text-align:start;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:start;display:inline;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.26458332px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+       xml:space="preserve"><tspan
+         style="stroke-width:0.26458332px"
+         y="44.886383"
+         x="6.3156896"
+         id="tspan1536"
+         sodipodi:role="line">x + 1000</tspan></text>
+    <text
+       id="text1542"
+       y="44.886383"
+       x="93.178284"
+       style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:4.93888855px;line-height:125%;font-family:Iwona;-inkscape-font-specification:'Iwona, Normal';font-variant-ligatures:normal;font-variant-caps:normal;font-variant-numeric:normal;font-feature-settings:normal;text-align:start;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:start;display:inline;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.26458332px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+       xml:space="preserve"><tspan
+         style="stroke-width:0.26458332px"
+         y="44.886383"
+         x="93.178284"
+         id="tspan1540"
+         sodipodi:role="line">x + 500</tspan></text>
+    <text
+       transform="translate(-43.663711,-16.067619)"
+       id="text3103"
+       y="36.388439"
+       x="108.80019"
+       style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:4.23333311px;line-height:125%;font-family:Iwona;-inkscape-font-specification:'Iwona, Normal';font-variant-ligatures:normal;font-variant-caps:normal;font-variant-numeric:normal;text-align:start;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:start;display:inline;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.26458332px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+       xml:space="preserve"><tspan
+         style="font-weight:normal;font-size:4.93888903px;stroke-width:0.26458332px"
+         y="36.388439"
+         x="108.80019"
+         id="tspan3101"
+         sodipodi:role="line">T0: (⊥, 5000)</tspan></text>
+  </g>
+  <g
+     inkscape:groupmode="layer"
+     id="layer5"
+     inkscape:label="CRDT"
+     style="display:inline">
+    <g
+       style="display:inline"
+       transform="translate(0,-21.166667)"
+       id="g2920">
+      <path
+         style="fill:none;fill-rule:evenodd;stroke:#000000;stroke-width:0.52969581;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1;marker-end:url(#marker1467)"
+         d="M 31.393177,68.889594 53.385346,81.586779"
+         id="path2912"
+         inkscape:connector-curvature="0" />
+      <text
+         y="6.3268809"
+         x="17.939917"
+         id="text2918"
+         style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:4.93888855px;line-height:125%;font-family:Iwona;-inkscape-font-specification:'Iwona, Normal';font-variant-ligatures:normal;font-variant-caps:normal;font-variant-numeric:normal;font-feature-settings:normal;text-align:start;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:start;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.26458332px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+         xml:space="preserve"
+         transform="translate(-34.395836)"><textPath
+           id="textPath2916"
+           xlink:href="#path9498"><tspan
+   id="tspan2914"
+   style="font-weight:bold;stroke-width:0.26458332px">W</tspan></textPath></text>
+    </g>
+    <g
+       id="g3013">
+      <path
+         inkscape:connector-curvature="0"
+         id="path2924"
+         d="M 85.906949,47.722927 63.914769,60.420112"
+         style="fill:none;fill-rule:evenodd;stroke:#000000;stroke-width:0.52969581;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1;marker-end:url(#marker2999)" />
+      <text
+         transform="rotate(-58.856754,57.228059,65.935492)"
+         xml:space="preserve"
+         style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:4.93888855px;line-height:125%;font-family:Iwona;-inkscape-font-specification:'Iwona, Normal';font-variant-ligatures:normal;font-variant-caps:normal;font-variant-numeric:normal;font-feature-settings:normal;text-align:start;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:start;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.26458332px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+         id="text2936"
+         x="7.1585193"
+         y="48.970455"><textPath
+           xlink:href="#path9498"
+           id="textPath2934"><tspan
+   style="font-weight:bold;stroke-width:0.26458332px"
+   id="tspan2932">W</tspan></textPath></text>
+    </g>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:4.93888855px;line-height:125%;font-family:Iwona;-inkscape-font-specification:'Iwona, Bold';font-variant-ligatures:normal;font-variant-caps:normal;font-variant-numeric:normal;text-align:start;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:start;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.26458332px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+       x="15.648211"
+       y="44.007793"
+       id="text2976"><tspan
+         sodipodi:role="line"
+         id="tspan2974"
+         x="15.648211"
+         y="44.007793"
+         style="font-weight:normal;stroke-width:0.26458332px">incr(1000)</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:4.93888855px;line-height:125%;font-family:Iwona;-inkscape-font-specification:'Iwona, Bold';font-variant-ligatures:normal;font-variant-caps:normal;font-variant-numeric:normal;text-align:start;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:start;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.26458332px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+       x="82.927979"
+       y="44.007793"
+       id="text2980"><tspan
+         sodipodi:role="line"
+         id="tspan2978"
+         x="82.927979"
+         y="44.007793"
+         style="font-weight:normal;stroke-width:0.26458332px">incr(500)</tspan></text>
+    <path
+       inkscape:connector-curvature="0"
+       id="path2982"
+       d="m 56.193207,72.591896 h 4.913692"
+       style="fill:none;fill-rule:evenodd;stroke:#000000;stroke-width:0.5291667;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1" />
+    <text
+       id="text2986"
+       y="74.295815"
+       x="65.136482"
+       style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:4.23333311px;line-height:125%;font-family:Iwona;-inkscape-font-specification:'Iwona, Normal';font-variant-ligatures:normal;font-variant-caps:normal;font-variant-numeric:normal;text-align:start;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:start;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.26458332px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+       xml:space="preserve"><tspan
+         style="font-weight:normal;font-size:4.93888903px;stroke-width:0.26458332px"
+         y="74.295815"
+         x="65.136482"
+         sodipodi:role="line"
+         id="tspan2995">T1: (5000, </tspan><tspan
+         style="font-weight:normal;font-size:4.93888903px;stroke-width:0.26458332px"
+         y="79.777985"
+         x="65.136482"
+         sodipodi:role="line"
+         id="tspan1204">      {(a, 1000), (b, 500)})</tspan></text>
+    <g
+       style="display:inline"
+       transform="translate(0,38.100003)"
+       id="g3042">
+      <path
+         style="fill:none;fill-rule:evenodd;stroke:#000000;stroke-width:0.52969581;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1;marker-end:url(#marker1449)"
+         d="M 54.156933,47.722926 32.164764,60.420111"
+         id="path3034"
+         inkscape:connector-curvature="0" />
+      <text
+         xml:space="preserve"
+         style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:4.93888855px;line-height:125%;font-family:Iwona;-inkscape-font-specification:'Iwona, Normal';font-variant-ligatures:normal;font-variant-caps:normal;font-variant-numeric:normal;font-feature-settings:normal;text-align:start;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:start;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.26458332px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+         id="text3040"
+         transform="rotate(180,70.308075,52.117558)"
+         x="-279.42224"
+         y="-209.36409"><textPath
+           xlink:href="#path850"
+           id="textPath3038"><tspan
+   style="font-weight:bold;stroke-width:0.26458332px"
+   id="tspan3036">R</tspan></textPath></text>
+    </g>
+    <text
+       id="text3051"
+       y="102.21611"
+       x="17.764879"
+       style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:4.93888855px;line-height:125%;font-family:Iwona;-inkscape-font-specification:'Iwona, Bold';font-variant-ligatures:normal;font-variant-caps:normal;font-variant-numeric:normal;text-align:start;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:start;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.26458332px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+       xml:space="preserve"><tspan
+         style="font-weight:normal;stroke-width:0.26458332px"
+         y="102.21611"
+         x="17.764879"
+         id="tspan3049"
+         sodipodi:role="line">6500</tspan></text>
+    <text
+       transform="translate(-43.663711,-16.067619)"
+       id="text3097"
+       y="36.388439"
+       x="108.80019"
+       style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:4.23333311px;line-height:125%;font-family:Iwona;-inkscape-font-specification:'Iwona, Normal';font-variant-ligatures:normal;font-variant-caps:normal;font-variant-numeric:normal;text-align:start;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:start;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.26458332px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+       xml:space="preserve"><tspan
+         style="font-weight:normal;font-size:4.93888903px;stroke-width:0.26458332px"
+         y="36.388439"
+         x="108.80019"
+         id="tspan3095"
+         sodipodi:role="line">T0: (5000, ∅)</tspan></text>
+  </g>
+</svg>

doc/talks/2021-04-28_spirals-team/figures/svg/crdt.svg

@@ -0,0 +1,227 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<!-- Created with Inkscape (http://www.inkscape.org/) -->
+
+<svg
+   xmlns:dc="http://purl.org/dc/elements/1.1/"
+   xmlns:cc="http://creativecommons.org/ns#"
+   xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+   xmlns:svg="http://www.w3.org/2000/svg"
+   xmlns="http://www.w3.org/2000/svg"
+   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
+   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
+   width="82.010674mm"
+   height="17.561125mm"
+   viewBox="0 0 82.010674 17.561125"
+   version="1.1"
+   id="svg1579"
+   inkscape:version="0.92.5 (2060ec1f9f, 2020-04-08)"
+   sodipodi:docname="crdt.svg">
+  <defs
+     id="defs1573">
+    <marker
+       inkscape:isstock="true"
+       style="overflow:visible"
+       id="marker2645"
+       refX="0"
+       refY="0"
+       orient="auto"
+       inkscape:stockid="Arrow2Mend">
+      <path
+         transform="scale(-0.6)"
+         d="M 8.7185878,4.0337352 -2.2072895,0.01601326 8.7185884,-4.0017078 c -1.7454984,2.3720609 -1.7354408,5.6174519 -6e-7,8.035443 z"
+         style="fill:#000000;fill-opacity:1;fill-rule:evenodd;stroke:#000000;stroke-width:0.625;stroke-linejoin:round;stroke-opacity:1"
+         id="path2643"
+         inkscape:connector-curvature="0" />
+    </marker>
+    <marker
+       inkscape:isstock="true"
+       style="overflow:visible"
+       id="marker2605"
+       refX="0"
+       refY="0"
+       orient="auto"
+       inkscape:stockid="Arrow2Mstart">
+      <path
+         transform="scale(0.6)"
+         d="M 8.7185878,4.0337352 -2.2072895,0.01601326 8.7185884,-4.0017078 c -1.7454984,2.3720609 -1.7354408,5.6174519 -6e-7,8.035443 z"
+         style="fill:#000000;fill-opacity:1;fill-rule:evenodd;stroke:#000000;stroke-width:0.625;stroke-linejoin:round;stroke-opacity:1"
+         id="path2603"
+         inkscape:connector-curvature="0" />
+    </marker>
+    <marker
+       inkscape:stockid="Arrow2Mend"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="Arrow2Mend"
+       style="overflow:visible"
+       inkscape:isstock="true"
+       inkscape:collect="always">
+      <path
+         id="path2206"
+         style="fill:#000000;fill-opacity:1;fill-rule:evenodd;stroke:#000000;stroke-width:0.625;stroke-linejoin:round;stroke-opacity:1"
+         d="M 8.7185878,4.0337352 -2.2072895,0.01601326 8.7185884,-4.0017078 c -1.7454984,2.3720609 -1.7354408,5.6174519 -6e-7,8.035443 z"
+         transform="scale(-0.6)"
+         inkscape:connector-curvature="0" />
+    </marker>
+    <marker
+       inkscape:stockid="Arrow2Mstart"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="Arrow2Mstart"
+       style="overflow:visible"
+       inkscape:isstock="true"
+       inkscape:collect="always">
+      <path
+         id="path2203"
+         style="fill:#000000;fill-opacity:1;fill-rule:evenodd;stroke:#000000;stroke-width:0.625;stroke-linejoin:round;stroke-opacity:1"
+         d="M 8.7185878,4.0337352 -2.2072895,0.01601326 8.7185884,-4.0017078 c -1.7454984,2.3720609 -1.7354408,5.6174519 -6e-7,8.035443 z"
+         transform="scale(0.6)"
+         inkscape:connector-curvature="0" />
+    </marker>
+    <marker
+       inkscape:stockid="Arrow2Lstart"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="Arrow2Lstart"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         id="path2197"
+         style="fill:#000000;fill-opacity:1;fill-rule:evenodd;stroke:#000000;stroke-width:0.625;stroke-linejoin:round;stroke-opacity:1"
+         d="M 8.7185878,4.0337352 -2.2072895,0.01601326 8.7185884,-4.0017078 c -1.7454984,2.3720609 -1.7354408,5.6174519 -6e-7,8.035443 z"
+         transform="matrix(1.1,0,0,1.1,1.1,0)"
+         inkscape:connector-curvature="0" />
+    </marker>
+    <marker
+       inkscape:stockid="Arrow1Sstart"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="Arrow1Sstart"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         id="path2191"
+         d="M 0,0 5,-5 -12.5,0 5,5 Z"
+         style="fill:#000000;fill-opacity:1;fill-rule:evenodd;stroke:#000000;stroke-width:1.00000003pt;stroke-opacity:1"
+         transform="matrix(0.2,0,0,0.2,1.2,0)"
+         inkscape:connector-curvature="0" />
+    </marker>
+  </defs>
+  <sodipodi:namedview
+     id="base"
+     pagecolor="#ffffff"
+     bordercolor="#666666"
+     borderopacity="1.0"
+     inkscape:pageopacity="0.0"
+     inkscape:pageshadow="2"
+     inkscape:zoom="2.8"
+     inkscape:cx="148.82928"
+     inkscape:cy="-32.722297"
+     inkscape:document-units="mm"
+     inkscape:current-layer="layer1"
+     showgrid="false"
+     inkscape:snap-bbox="true"
+     fit-margin-top="0"
+     fit-margin-left="0"
+     fit-margin-right="0"
+     fit-margin-bottom="0"
+     inkscape:window-width="1920"
+     inkscape:window-height="1055"
+     inkscape:window-x="0"
+     inkscape:window-y="0"
+     inkscape:window-maximized="1" />
+  <metadata
+     id="metadata1576">
+    <rdf:RDF>
+      <cc:Work
+         rdf:about="">
+        <dc:format>image/svg+xml</dc:format>
+        <dc:type
+           rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
+        <dc:title />
+      </cc:Work>
+    </rdf:RDF>
+  </metadata>
+  <g
+     inkscape:label="Layer 1"
+     inkscape:groupmode="layer"
+     id="layer1"
+     transform="translate(-21.438257,-19.857903)">
+    <path
+       style="fill:none;fill-rule:evenodd;stroke:#000000;stroke-width:0.26458332;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1;marker-start:url(#Arrow2Mstart);marker-end:url(#Arrow2Mend)"
+       d="M 44.979166,28.87314 H 58.822544"
+       id="path2177"
+       inkscape:connector-curvature="0" />
+    <g
+       id="g2153">
+      <rect
+         rx="0"
+         ry="3.8793426"
+         y="19.990194"
+         x="21.570549"
+         height="17.296541"
+         width="23.537758"
+         id="rect2124"
+         style="opacity:1;vector-effect:none;fill:#f9f9f9;fill-opacity:0.97409327;fill-rule:nonzero;stroke:#000000;stroke-width:0.26458332;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:0.91191711;paint-order:fill markers stroke" />
+      <text
+         id="text2132"
+         y="29.799105"
+         x="29.366098"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:4.93888855px;line-height:125%;font-family:Iwona;-inkscape-font-specification:'Iwona, Bold';font-variant-ligatures:normal;font-variant-caps:normal;font-variant-numeric:normal;text-align:start;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:start;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.26458332px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+         xml:space="preserve"><tspan
+           style="font-weight:normal;stroke-width:0.26458332px"
+           y="29.799105"
+           x="29.366098"
+           id="tspan2130"
+           sodipodi:role="line">App</tspan></text>
+    </g>
+    <rect
+       rx="0"
+       ry="3.8793426"
+       y="19.990194"
+       x="58.612213"
+       height="17.296541"
+       width="23.537758"
+       id="rect2155"
+       style="opacity:1;vector-effect:none;fill:#f9f9f9;fill-opacity:0.97409327;fill-rule:nonzero;stroke:#000000;stroke-width:0.26458332;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:0.91191711;paint-order:fill markers stroke" />
+    <text
+       id="text2159"
+       y="30.367077"
+       x="63.842014"
+       style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:4.93888855px;line-height:125%;font-family:Iwona;-inkscape-font-specification:'Iwona, Bold';font-variant-ligatures:normal;font-variant-caps:normal;font-variant-numeric:normal;text-align:start;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:start;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.26458332px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+       xml:space="preserve"><tspan
+         style="font-weight:normal;stroke-width:0.26458332px"
+         y="30.367077"
+         x="63.842014"
+         id="tspan2157"
+         sodipodi:role="line">CRDT</tspan></text>
+    <g
+       id="g4137"
+       transform="translate(-15.875)">
+      <rect
+         rx="0"
+         ry="3.8793426"
+         y="19.990194"
+         x="95.653885"
+         height="17.296541"
+         width="23.537758"
+         id="rect2163"
+         style="opacity:1;vector-effect:none;fill:#f9f9f9;fill-opacity:0.97409327;fill-rule:nonzero;stroke:#000000;stroke-width:0.26458332;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:0.91191711;paint-order:fill markers stroke" />
+      <text
+         id="text2167"
+         y="30.367077"
+         x="96.969627"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:4.93888855px;line-height:125%;font-family:Iwona;-inkscape-font-specification:'Iwona, Bold';font-variant-ligatures:normal;font-variant-caps:normal;font-variant-numeric:normal;text-align:start;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:start;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.26458332px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+         xml:space="preserve"><tspan
+           style="font-weight:normal;stroke-width:0.26458332px"
+           y="30.367077"
+           x="96.969627"
+           id="tspan2165"
+           sodipodi:role="line">Datastore</tspan></text>
+    </g>
+  </g>
+</svg>

doc/talks/2021-04-28_spirals-team/garage.tex

@@ -0,0 +1,147 @@
+\section{Introducing Garage}
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+\begin{frame}{Brought to you by the Deuxfleurs association}
+
+\begin{block}{\textbf{deuxfleurs.fr} -- a libre hosting association with a vision}
+``Shifting the current structure of the Internet from a world of a few very large service providers, to a world where services are hosted by a variety of smaller organisations.''	
+\end{block}
+
+
+\begin{block}{Our goals}
+\begin{itemize}
+	\item To propose performant \& reliable libre services for the masses
+	\item To host and administer our infrastructure ourselves
+	\item To allow members to contribute storage/compute nodes
+	\item Resilience: for availability \& the sysadmins' sleep 
+	\item Conceptual simplicity to ease onboarding \& demistify hosting
+\end{itemize}
+\end{block}
+
+
+\end{frame}
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+\begin{frame}{The lacking state of the practice}
+
+\begin{block}{Object storage fitted our needs}
+\begin{itemize}
+	\item Distributed by design
+	\item Objects are replicated
+	\item Conceptually simple
+\end{itemize}
+\end{block}
+\vfill 
+
+\begin{block}{Existing object stores did not}
+	\begin{itemize}
+		\item Too specific / complex 
+		\item Resource hungry 
+		\item Hidden constraints 
+	\end{itemize}
+\end{block}
+\vfill
+
+We developed Garage, an object store with minimal functionality.
+
+It works, and serves our static sites and media.
+
+\end{frame}
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+\begin{frame}{Introducing Garage}
+
+\centering
+\url{garagehq.deuxfleurs.fr}
+
+\url{git.deuxfleurs.fr/Deuxfleurs/garage}
+
+\includegraphics[width=.4\columnwidth]{figures/garage_distributed.png}
+\vfill
+
+\raggedright
+\begin{itemize}
+	\item Distributed data store
+	\item Based on DynamoDB object store (P2P!)
+	\item Modular data types/protocols with CRDTs:
+	\begin{itemize}
+		\item Done: objects (media, static sites, backups...) via S3 API 
+		\item To do: e-mails via IMAP protocol, and more
+	\end{itemize}
+\end{itemize}
+
+\end{frame}
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+\begin{frame}[t]{The \textbf{RING}}
+
+\centering
+\fullcite{decandia_dynamo:_2007}
+\vspace{3ex}
+
+\only<1>{\includegraphics[width=.5\columnwidth]{figures/c1.pdf}}%
+\only<2>{\includegraphics[width=.5\columnwidth]{figures/c2.pdf}}%
+\only<3>{\includegraphics[width=.5\columnwidth]{figures/c3.pdf}}%
+\only<4>{\includegraphics[width=.5\columnwidth]{figures/c4.pdf}}%
+\vspace{5ex}
+
+%\raggedright
+\only<1>{Each node is assigned a unique ID on the circular address space.}%
+\only<2-3>{When a new object is added to the store...}%
+\only<3>{\\ It is assigned a unique ID (its \emph{key}) on the address space.}%
+\only<4>{The $R$ nodes after the object are in charge of replicating it.}%
+
+\end{frame}
+
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+\begin{frame}{Distributed metadata}
+
+\centering
+\includegraphics[width=.8\columnwidth]{figures/garage_tables.pdf}
+\vfill
+
+The objects, versions and blocks are all stored in the ring.
+
+\end{frame}
+
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+\begin{frame}{Written in Rust}
+
+\begin{columns}
+\column{.65\columnwidth}
+Entirely written in Rust!
+\column{.35\columnwidth}
+\centering
+\includegraphics[width=.85\columnwidth]{figures/rustacean-flat-happy.png}
+\end{columns}
+\vfill
+
+\begin{columns}
+\column[t]{.6\columnwidth}
+	\textbf{Pros:}
+	\begin{itemize}
+		\item Compiled and fast
+		\item Features prevent usual mistakes:
+
+		strongly typed, immutable by default, ownership instead of GC...
+
+		\item Best of several paradigms:
+
+		imperative, OO, functional
+
+		\item Good libraries for network programmings:
+
+		serialization, http, async/await...
+	\end{itemize}
+\column[t]{.4\columnwidth}
+	\textbf{Cons}:
+	\begin{itemize}
+		\item Steep learning curve
+		\item Long compilation times
+		\item Compiler rage
+	\end{itemize}
+\end{columns}
+
+\end{frame}

doc/talks/2021-04-28_spirals-team/header.tex

@@ -0,0 +1,134 @@
+%% Imports %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+\usepackage{graphicx}
+\usepackage[utf8]{inputenc}
+\usepackage{csquotes} % Elegant quotes (must be loaded before babel)
+\usepackage[english]{babel}
+\usepackage[T1]{fontenc}
+% \usepackage{amsfonts}
+% \usepackage{amssymb}
+%\usepackage{lmodern}
+\usepackage{iwona}
+\usepackage{color}
+\usepackage{xspace}
+\usepackage{amsmath}
+\usepackage{hanging}
+\usepackage{listings}
+\lstset{basicstyle=\scriptsize}
+\usepackage{tikz}
+\usepackage{tikzsymbols} % For emojis
+%\usepackage{setspace} % Activation disables footnoes
+\usepackage{perpage} % Reset footnote counter every page
+\MakePerPage{footnote}
+\usepackage{nameref} % For printing the sections' name
+\usepackage{hyperref}
+
+%% Biblio %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+\usepackage[style=alphabetic,giveninits=true,sorting=none,hyperref,backend=biber,maxnames=3]{biblatex}
+\addbibresource{bibliography.bib}
+
+%% Templating %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+\usepackage{pgf}
+\usepackage{xifthen, tikz}
+
+\usetheme{default} 
+
+\makeatletter
+
+
+%%% colors
+\definecolor{bleuroi}{RGB}{0, 8, 88}
+\definecolor{vertemeraude}{RGB}{0, 64, 44}
+\definecolor{lightgray}{RGB}{245,245,245}
+\definecolor{encre_de_chine}{RGB}{9,36,53}
+\definecolor{im_lost_in_your_eyes}{RGB}{110,138,159}
+\definecolor{metallic_grey}{RGB}{209,219,221}
+\definecolor{metallic_white}{RGB}{229,239,241}
+\definecolor{boss}{RGB}{229,72,27}
+\definecolor{toon_eyes}{RGB}{3,4,6}
+
+\setbeamercolor{frametitle}{bg=bleuroi,fg=white}
+\setbeamercolor*{normal text}{fg=toon_eyes,bg=white} 
+\setbeamercolor*{block title}{fg=white,bg=vertemeraude}
+\setbeamercolor*{block body}{fg=toon_eyes,bg=vertemeraude!5}
+\setbeamercolor{alerted text}{fg=vertemeraude}
+\setbeamercolor*{example text}{fg=toon_eyes,bg=white} 
+\setbeamercolor*{structure}{fg=bleuroi,bg=white}
+% sectionnavigation:
+\setbeamercolor*{section in head/foot}{fg=bleuroi,bg=white}
+
+% \let\oldtextbf\textbf
+% \renewcommand{\textbf}[1]{\textcolor{bleuroi}{\oldtextbf{#1}}}
+
+%%% Rounded boxes with no shading
+\pgfdeclareverticalshading[lower.bg,upper.bg]{bmb@transition}{200cm}{%
+  color(0pt)=(upper.bg); color(2pt)=(upper.bg); color(4pt)=(upper.bg)}
+\setbeamertemplate{blocks}[rounded][shadow=false]
+
+%%% Who on Earth uses navigation bars?
+\setbeamertemplate{navigation symbols}{}
+
+
+%%% Headline with sections
+\setbeamertemplate{headline}
+{%
+  \begin{beamercolorbox}[wd=\paperwidth,dp=.5ex,ht=2ex]{section in head/foot}
+    \insertsectionnavigationhorizontal{\paperwidth}{\hskip0pt plus1fill}{\hskip0pt plus1fill}\par % Centered
+    %\insertsectionnavigationhorizontal{\paperwidth}{}{\hfill\hfill} % Left aligned
+  \end{beamercolorbox}%
+}
+% Set the colors of the section bar
+\usesectionheadtemplate
+  {\colorbox{fg}{\color{bg} \insertsectionhead}}
+  {\color{fg!40!bg} \insertsectionhead}
+
+% \newlength\lpg@linewd
+% \setbox0=\hbox{\strut}
+% \newlength\strutht \strutht\ht0
+% \newlength\strutdp \strutdp\dp0
+
+\setbeamertemplate{frametitle}
+{
+  \begin{beamercolorbox}[wd=\paperwidth,dp=1ex,ht=2.6ex,leftskip=.7cm,rightskip=.7cm]{frametitle}
+    %\vskip-1ex
+    %[wd=\lpg@linewd, ht=\strutht, dp=\strutdp]{frametitle}
+    %\vskip-.8ex\hskip0.7cm
+    %\vskip-.8ex
+    \usebeamerfont{frametitle}\strut\insertframetitle
+    \ifx\insertframesubtitle\@empty
+    \else
+    \hfill \usebeamerfont{framesubtitle}\strut\insertframesubtitle
+    \fi%
+  \end{beamercolorbox}
+}
+
+
+%%% Page numbering in footline
+\setbeamertemplate{footline}{
+\hbox{\begin{beamercolorbox}[wd=1\paperwidth,ht=2.5ex,dp=1ex,right]{framenumber in head/foot}%
+    \usebeamerfont{framenumber in head/foot}%
+    %\insertframenumber\space/\space\inserttotalframenumber\hspace{0.3em}
+    \insertframenumber\hspace{0.5em}
+\end{beamercolorbox}}}
+
+%%% Reduce foot citation size
+\renewcommand{\footnotesize}{\tiny}
+
+%%% Display footnotes (not working, still no footnotes)
+% \setbeamertemplate{footnote}{%
+%   \hangpara{2em}{1}%
+%   \makebox[2em][l]{\insertfootnotemark}\footnotesize\insertfootnotetext\par%
+% }
+
+
+\makeatother
+
+%% Commentaires %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+\newcommand{\annote}[3]{{
+    \colorbox{#3}{\bfseries\sffamily\footnotesize\textcolor{white}{#2}}
+    \color{#3}
+    $\blacktriangleright$\textit{#1}$\blacktriangleleft$}
+}
+%% Uncomment for final version (removes notes)
+%\renewcommand{\annote}[3]{}
+
+\newcommand{\todo}[1]{\annote{#1}{TODO}{green}}

doc/talks/2021-04-28_spirals-team/introduction.tex

@@ -0,0 +1,99 @@
+\section{Introduction}
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+\begin{frame}{A very casual motivation}
+
+
+\begin{center}
+\Large
+I want to host \textbf{resilient web services} with \textbf{acceptable performance} on commodity hardware behind \textbf{household networks}.
+\end{center}
+\vfill
+
+\begin{block}{Keywords}
+\begin{columns}
+\column{.5\columnwidth}
+	\begin{itemize}
+		\item Decentralised networks
+		\item Edge computing
+	\end{itemize}
+\column{.5\columnwidth}
+	\begin{itemize}
+		\item Distributed storage
+		\item Privacy
+	\end{itemize}
+\end{columns}
+\end{block}
+
+\end{frame}
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+\begin{frame}{Context}
+
+\textbf{Resilience}: Ability to recover quickly from failures and changes.
+\vspace{1ex}
+
+Only achievable through distribution of the hosted applications across several physical locations.
+\vfill
+
+
+\begin{block}{Application = \textbf{computations} on \textbf{data}}
+\begin{itemize}
+	\item \textbf{Computation}: Stateless; easy to distribute \& orchestrate.
+	% where it is performed does not matter as long as the application's state is accessible R/W
+	%Computation units are "easy" to distribute and orchestrate.
+
+	\item \textbf{Data}: Stateful; hard to distribute \& full of trade-offs.
+\end{itemize}
+\end{block}
+
+\end{frame}
+
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+\begin{frame}{Concurrent writes example}{How to lose vaccines}
+
+\centering
+
+\includegraphics[width=.5\columnwidth]{figures/conflict_problem.pdf}
+\end{frame}
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+\begin{frame}{The problem}
+
+% \textbf{Path dependency}: existing data stores are built for data centres.
+
+% $\implies$ They assume good inter-node connectivity.
+% \vfill
+
+\begin{center}
+\Large
+Can we design an available data store tailored for adverse network conditions?
+\end{center}
+	
+\end{frame}
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+% Maybe more framing of the context. What kind of data storage? Object vs Block vs what?
+% \begin{frame}{``Stateless'', ``serverless'', and the elephant in the room}
+
+% It seems easy to deploy \& administer web services nowadays ...
+
+% Because the inherent complexity is shadowed by proprietary ``cloud'' solutions.
+
+% The IT crowd can gloss over ``statelessness'' \emph{ad nauseam} ...
+
+% But storing \emph{state} remains an open research problem.
+
+% Data storage is either:
+
+% \begin{itemize}
+% 	\item A single point of failure;
+% 	\item Delegated to proprietary solutions;
+% 	\item Pain.
+% \end{itemize}
+
+% Today, we will review networked storage's history, and discuss open research questions.
+
+% \end{frame}

doc/talks/2021-04-28_spirals-team/main.tex

@@ -0,0 +1,70 @@
+\documentclass[compress]{beamer}
+
+\input{header.tex}
+
+%% Metadata %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+\title{Distributed object storage is centralised}
+\subtitle{A quest for autonomy in the modern hosting ecology}
+\author{Adrien Luxey}
+\date{Wednesday, 28th April, 2021}
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+%% Begin document %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+\begin{document}
+
+%% Title page %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+{
+\setbeamertemplate{footline}{} 
+\setbeamertemplate{headline}{} 
+\begin{frame}
+\titlepage
+\end{frame}
+}
+
+%% Contents %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+\input{introduction.tex}
+\input{sota.tex}
+\input{escaping_the_cloud.tex}
+\input{garage.tex}
+\input{conclusion.tex}
+
+
+%% Bibliography %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+% \begin{frame}
+% \large
+% \raggedright
+% Références
+
+% \printbibliography
+% \end{frame}
+
+%% Ending %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+{
+\setbeamertemplate{footline}{} 
+\setbeamertemplate{headline}{} 
+\begin{frame}
+\centering
+
+\vspace{1cm}
+\Large Thank you for your attention.
+
+\vspace{2cm}
+\large Now let's chat!
+
+\vspace{1cm}
+\raggedleft
+\includegraphics[width=.2\textwidth]{figures/m_proxy_gray.pdf}
+\end{frame}
+}
+
+%% Appendices %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+% \appendix
+% \input{sprinkler_appendix.tex}
+% \input{cascade_appendix.tex}
+% \input{spores_appendix.tex}
+
+
+\end{document}

doc/talks/2021-04-28_spirals-team/sota.tex

@@ -0,0 +1,323 @@
+\section{State of the art}
+
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+\begin{frame}{The CAP theorem}{Consistency vs. Availability}
+
+\begin{block}{Eric Brewer's theorem}
+``A shared-state system can have \textbf{at most two} of the following properties at any given time:
+
+\begin{itemize}
+	\item \textbf{C}onsistency
+	\item \textbf{A}vailability
+	\item \textbf{P}artition tolerance''
+\end{itemize}
+\end{block}
+
+
+\begin{center}
+\Large 
+Under network partitions, a distributed data store has to sacrifice either availability or consistency.
+\end{center}
+\vfill
+
+\begin{itemize}
+	\item \textbf{Consistency-first}: Abort incoming queries;
+	\item \textbf{Availability-first}: Return possibly stale data.
+\end{itemize}
+
+\end{frame}
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+\begin{frame}{Consistency-first: the ACID model}{Consistency vs. Availability}
+
+\textbf{Transaction}: unit of work within an ACID data store. 
+%Comprises multiple operations.
+%E.g. bank transfer.
+%E.g. a bank transfer from A to B is a transaction involving two operations: withdraw money from A & credit B with the same money amount. 
+\vfill
+
+\begin{itemize}
+	\item \textbf{\underline{A}tomicity}: Transactions either complete entirely or fail.
+
+	No transaction ever seen as in-progress.
+
+	\item \textbf{\underline{C}onsistency}: Transactions always generate a valid state.
+
+	The database maintains its invariants across transactions.
+
+	\item \textbf{\underline{I}solation}: Concurrent transactions are seen as sequential.
+
+	Transactions are serializable, or sequentially consistent.
+
+	\item \textbf{\underline{D}urability}: Committed transactions are never forgotten.
+\end{itemize}
+\vfill\centering
+
+Reads are fast, writes are slow.
+
+\vfill\raggedright
+
+Example: relational databases.
+\end{frame}
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+\begin{frame}[fragile]{Concurrent writes in ACID}{Consistency vs. Availability}
+
+
+\begin{columns}
+\column{.5\columnwidth}
+	\begin{block}{}
+		\begin{lstlisting}
+transaction AcqDoses(y):
+  x <- SELECT #vaccines;
+  UPDATE #vaccines = (x + y);
+		\end{lstlisting}
+	\end{block}
+	\vspace{5ex}
+
+	Supports compound operations.
+\column{.5\columnwidth}
+\centering
+\includegraphics[width=\columnwidth]{figures/conflict_acid.pdf}
+\end{columns}
+
+\end{frame}
+
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+\begin{frame}{Availability-first: the BASE model}{Consistency vs. Availability}
+
+
+Some apps prefer availability, e.g. Amazon products' reviews.
+\vfill 
+
+The BASE model trades Consistency \& Isolation for Availability.
+
+
+%Some applications do not care about strong consistency and prefer being highly available (e.g. Amazon's product reviews).
+
+%In order to achieve higher availability, the BASE model relaxes consistency constraints of the ACID model: "eventual consistency".
+\vfill
+
+\begin{itemize}
+	\item \textbf{\underline{B}asic \underline{A}vailability}: 
+	The data store thrives to be available.
+
+	\item \textbf{\underline{S}oft-state}: 
+	Replicas can disagree on the valid state.
+
+	\item \textbf{\underline{E}ventual consistency}: 
+	In the absence of write queries, 
+	the data store will eventually converge to a single valid state.
+\end{itemize}
+\vfill\centering
+
+Writes are fast, reads are slow.
+
+\vfill\raggedright
+
+Examples: key-value \& object stores.
+
+\end{frame}
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+\begin{frame}{Concurrent writes in BASE}{Consistency vs. Availability}
+
+\begin{columns}
+\column{.5\columnwidth}
+	\begin{block}{Object}
+		\begin{itemize}
+			\item Unique key
+			\item Arbitrary value 
+			\item Metadata
+		\end{itemize}
+	\end{block}
+	\vspace{5ex} 
+
+	Conflict resolution = client's job!
+	\vspace{5ex}
+
+	No compound operations.
+\column{.5\columnwidth}
+	\centering
+	\includegraphics[width=\columnwidth]{figures/conflict_base.pdf}
+\end{columns}
+
+% KV storage is another example, distinction is minor here
+
+% Object = unique key, arbitrary value, metadata.
+
+% Object storage only provides semantics to investigate causal order of queries *for individual objects*. No compound operations, no transactions.
+
+% Much easier to distribute, and "scale-out".
+
+% Write is fast, read is slow (gotta collect all object versions).
+
+% \todo{vaccines example with BASE model}
+
+\end{frame}
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+\begin{frame}{Strong Eventual Consistency w/ CRDTs}{Consistency vs. Availability}
+
+\centering\small
+
+\fullcite{defago_conflict-free_2011}
+
+\vfill\raggedright\normalsize
+
+\begin{block}{Strong Eventual Consistency (SEC)}
+	\begin{itemize}
+		\item CRDTs specify distributed operations
+		\item Conflicts will be solved according to specification
+		\item Proven \& bound eventual convergence
+	\end{itemize}
+\end{block}
+
+\vfill\centering
+\includegraphics[width=.5\columnwidth]{figures/crdt.pdf}
+
+\end{frame}
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+\begin{frame}[fragile]{Concurrent writes with CRDTs}{Consistency vs. Availability}
+
+\begin{columns}
+\column{.5\columnwidth}
+	\begin{block}{}
+		\begin{lstlisting}
+CRDT Counter(x0):
+  history = {}
+  op. incr(y):
+    history U= {(UUID(), y)}
+  op. decr(y):
+    history U= {(UUID(), -y)}
+  op. read():
+    x = x0
+    for (_, y) in history:
+    	x += y
+    return x
+		\end{lstlisting}
+	\end{block}
+	\vspace{2ex}
+
+	Operations commute?
+
+	$\implies$ screw total order!
+\column{.5\columnwidth}
+	\centering
+	\includegraphics[width=\columnwidth]{figures/conflict_crdt.pdf}
+\end{columns}
+
+\end{frame}
+
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+\begin{frame}{A complex CRDT: the DAG}{Consistency vs. Availability}
+
+\centering
+\only<1>{\includegraphics[height=\textheight]{figures/dag_crdt.png}}%
+\only<2>{
+	Just to say I swept a lot under the rug.
+	\vfill
+
+	For details, go read:
+
+	\fullcite{defago_conflict-free_2011}
+	\vfill
+
+	For an implementation, check \textbf{AntidoteDB}.
+}
+
+\end{frame}
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+\begin{frame}{State of the practice}{Path dependency to the ``cloud''}
+
+\begin{block}{The BASE model is fashionable because}
+\centering 
+
+``\emph{High-performance} object storage for \emph{AI analytics} with PBs of \emph{IoT data streams} at the \emph{edge}, using \emph{5G}.''
+	% \begin{itemize}
+	% 	\item Highest performance 
+	% 	\item IoT data streams are inherently distributed
+	% \end{itemize}
+\end{block}
+
+\vfill\centering
+
+\includegraphics[width=.9\columnwidth]{figures/minio_edge.png}
+
+\vfill\raggedright
+
+
+%\begin{block}{}
+\begin{itemize}
+	\item Always backed by cloud: high performance network links.
+	\item Edge nodes always seen as clients or data sources, not peers.
+\end{itemize}
+%\end{block}
+
+% There is \textbf{always a central cloud cluster} in these use-cases.
+
+% Hidden constraint: \textbf{high performance inter-node connectivity}.
+
+
+
+\end{frame}
+
+
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+% \begin{frame}{A brief history of storage}
+
+% We keep it short because we'll follow chronological order in the next section too.
+
+% \end{frame}
+
+
+% \begin{frame}{In the beginning, there were \emph{monoliths}}
+
+% \includegraphics[width=.5\columnwidth]{figures/stonehenge.jpg}
+
+% Web applications used to be monolithic:
+
+% \begin{itemize}
+% 	\item One or two servers;
+% 	\item Availability was not an obsession;
+% 	\item Latency was acceptable.
+% \end{itemize}
+
+% Relational databases were queens.
+
+% \end{frame}
+
+
+% \begin{frame}{Then came \emph{expectations}}
+% Then, the whole world went online, and suddenly: expectations!
+
+% \begin{itemize}
+% 	\item ``Milliseconds matter.'' (Algolia slogan)
+% 	\item Critical networked services (healthcare, logistics) need 100\% availability 
+% \end{itemize}
+
+% $\implies$ Microservices \& horizontal scalability.
+
+% \todo{Develop on the `herd not sheep' paradigm a bit.}
+
+% \end{frame}
+
+
+% \begin{frame}{Distributing state/storage: the remaining unknown}
+
+% The microservices orchestration game works well for \emph{stateless} services.
+
+% However, any application requires \emph{state}, persistent data. 
+
+% And this is tough. As we will now see.
+
+% (Not that it's not well studied: distributed storage has always been fashionable.)
+	
+% \end{frame}

make_docker.sh

@@ -0,0 +1,20 @@
+#!/bin/sh
+
+BIN=target/release/garage
+DOCKER=lxpz/garage_amd64
+
+TAG=$1
+if [ -z "$1" ]; then
+	echo "Usage: $0 <tag>"
+	exit 1
+fi
+
+RUSTFLAGS="-C link-arg=-fuse-ld=lld -C target-cpu=x86-64 -C target-feature=+sse2" cargo build --release --no-default-features
+cp $BIN $BIN.stripped
+strip $BIN.stripped
+
+docker pull archlinux:latest
+docker build -t $DOCKER:$TAG .
+docker push $DOCKER:$TAG
+docker tag $DOCKER:$TAG $DOCKER:latest
+docker push $DOCKER:latest

script/builder_image/Dockerfile

@@ -0,0 +1,7 @@
+FROM rust:buster
+RUN apt-get update && \
+	apt-get install --yes libsodium-dev awscli python-pip wget rclone openssl socat && \
+	    rm -rf /var/lib/apt/lists/*
+RUN wget https://dl.min.io/client/mc/release/linux-amd64/mc -O /usr/local/bin/mc && chmod +x /usr/local/bin/mc
+RUN rustup component add rustfmt clippy
+RUN pip install s3cmd

script/builder_image/Makefile

@@ -0,0 +1,8 @@
+DOCKER=lxpz/garage_builder_amd64
+
+docker:
+	docker build -t $(DOCKER):$(TAG) .
+	docker push $(DOCKER):$(TAG)
+	docker tag $(DOCKER):$(TAG) $(DOCKER):latest
+	docker push $(DOCKER):latest
+	

script/dev-clean.sh

@@ -3,5 +3,6 @@
 set -ex
 
 killall -9 garage || echo "garage is not running"
+killall -9 socat || echo "socat is not running"
 rm -rf /tmp/garage*
 rm -rf /tmp/config.*.toml

script/dev-cluster.sh

@@ -31,9 +31,7 @@ bootstrap_peers = [
   "127.0.0.1:3903"
 ]
 max_concurrent_rpc_requests = 12
-data_replication_factor = 3
-meta_replication_factor = 3
-meta_epidemic_fanout = 3
+replication_mode = "3"
 
 [s3_api]
 api_bind_addr = "0.0.0.0:$((3910+$count))"	# the S3 API port, HTTP without TLS. Add a reverse proxy for the TLS part.
@@ -47,6 +45,21 @@ EOF
 
 echo -en "$LABEL configuration written to $CONF_PATH\n"
 
+if [ -z "$SKIP_HTTPS" ]; then
+  echo -en "$LABEL Starting dummy HTTPS reverse proxy\n"
+  mkdir -p /tmp/garagessl
+  openssl req \
+    -new \
+    -x509 \
+    -keyout /tmp/garagessl/test.key \
+    -out /tmp/garagessl/test.crt \
+    -nodes  \
+    -subj "/C=XX/ST=XX/L=XX/O=XX/OU=XX/CN=localhost/emailAddress=X@X.XX" \
+    -addext "subjectAltName = DNS:localhost, IP:127.0.0.1"
+  cat /tmp/garagessl/test.key /tmp/garagessl/test.crt > /tmp/garagessl/test.pem
+  socat openssl-listen:4443,reuseaddr,fork,cert=/tmp/garagessl/test.pem,verify=0 tcp4-connect:localhost:3911 &
+fi
+
 (garage server -c /tmp/config.$count.toml 2>&1|while read r; do echo -en "$LABEL $r\n"; done) &
 done
 

script/dev-configure.sh

@@ -18,6 +18,6 @@ garage status \
 	| grep UNCONFIGURED \
 	| grep -Po '^[0-9a-f]+' \
 	| while read id; do 
-	  garage node configure -d dc1 -c 1 $id
+	  garage node configure -z dc1 -c 1 $id
 	done
 

script/dev-env-aws.sh

@@ -1,14 +1,9 @@
 #!/bin/bash
 
-SCRIPT_FOLDER="`dirname \"${BASH_SOURCE[0]}\"`"
-REPO_FOLDER="${SCRIPT_FOLDER}/../"
-GARAGE_DEBUG="${REPO_FOLDER}/target/debug/"
-GARAGE_RELEASE="${REPO_FOLDER}/target/release/"
-PATH="${GARAGE_DEBUG}:${GARAGE_RELEASE}:$PATH"
-
 export AWS_ACCESS_KEY_ID=`cat /tmp/garage.s3 |cut -d' ' -f1`
 export AWS_SECRET_ACCESS_KEY=`cat /tmp/garage.s3 |cut -d' ' -f2`
 export AWS_DEFAULT_REGION='garage'
 
-alias awsgrg="aws s3 \
-  --endpoint-url http://127.0.0.1:3911"
+function aws { command aws --endpoint-url http://127.0.0.1:3911 $@ ; }
+
+aws --version

script/dev-env-duck.sh

@@ -0,0 +1,52 @@
+mkdir -p /tmp/garage.cyberduck.home/.duck/profiles
+
+DUCK_ACCESS_KEY=`cat /tmp/garage.s3 |cut -d' ' -f1`
+DUCK_SECRET_KEY=`cat /tmp/garage.s3 |cut -d' ' -f2`
+
+cat > /tmp/garage.cyberduck.home/.duck/credentials <<EOF
+https\://$DUCK_ACCESS_KEY@127.0.0.1\:4443=$DUCK_SECRET_KEY
+EOF
+
+cat > /tmp/garage.cyberduck.home/.duck/profiles/garage.cyberduckprofile <<EOF
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+    <dict>
+        <key>Protocol</key>
+        <string>s3</string>
+        <key>Vendor</key>
+        <string>garage</string>
+        <key>Scheme</key>
+        <string>https</string>
+        <key>Description</key>
+        <string>GarageS3</string>
+        <key>Default Hostname</key>
+        <string>127.0.0.1</string>
+        <key>Default Port</key>
+        <string>4443</string>
+        <key>Hostname Configurable</key>
+        <false/>
+        <key>Port Configurable</key>
+        <false/>
+        <key>Username Configurable</key>
+        <true/>
+        <key>Username Placeholder</key>
+        <string>Access Key ID (GK...)</string>
+        <key>Password Placeholder</key>
+        <string>Secret Key</string>
+        <key>Properties</key>
+        <array>
+            <string>s3service.disable-dns-buckets=true</string>
+        </array>
+        <key>Region</key>
+        <string>garage</string>
+        <key>Regions</key>
+        <array>
+            <string>garage</string>
+        </array>
+    </dict>
+</plist>
+EOF
+
+function duck { HOME=/tmp/garage.cyberduck.home/ command duck --username $DUCK_ACCESS_KEY $@ ; }
+

script/dev-env-mc.sh

@@ -0,0 +1,22 @@
+ACCESS_KEY=`cat /tmp/garage.s3 |cut -d' ' -f1`
+SECRET_KEY=`cat /tmp/garage.s3 |cut -d' ' -f2`
+
+mkdir -p /tmp/garage.mc/certs/CAs
+
+cat > /tmp/garage.mc/config.json <<EOF
+{
+  "version": "10",
+  "aliases": {
+    "garage": {
+      "url": "https://localhost:4443",
+      "accessKey": "$ACCESS_KEY",
+      "secretKey": "$SECRET_KEY",
+      "api": "S3v4",
+      "path": "auto"
+    }
+  }
+}
+EOF
+
+function mc { command mc --insecure --config-dir /tmp/garage.mc $@ ; }
+mc --version

script/dev-env-rclone.sh

@@ -0,0 +1,21 @@
+ACCESS_KEY=`cat /tmp/garage.s3 |cut -d' ' -f1`
+SECRET_KEY=`cat /tmp/garage.s3 |cut -d' ' -f2`
+
+cat > /tmp/garage.rclone.conf <<EOF
+[garage]
+type = s3
+provider = Other
+env_auth = false
+access_key_id = $ACCESS_KEY
+secret_access_key = $SECRET_KEY
+endpoint = http://127.0.0.1:3911
+bucket_acl = private
+force_path_style = true
+region = garage
+no_check_bucket = true
+EOF
+# It seems that region is mandatory as rclone does not support redirection
+
+
+function rclone { command rclone --config /tmp/garage.rclone.conf $@ ; }
+rclone --version

script/dev-env-s3cmd.sh

@@ -1,19 +1,15 @@
-#!/bin/bash
-
-SCRIPT_FOLDER="`dirname \"${BASH_SOURCE[0]}\"`"
-REPO_FOLDER="${SCRIPT_FOLDER}/../"
-GARAGE_DEBUG="${REPO_FOLDER}/target/debug/"
-GARAGE_RELEASE="${REPO_FOLDER}/target/release/"
-PATH="${GARAGE_DEBUG}:${GARAGE_RELEASE}:$PATH"
-
 ACCESS_KEY=`cat /tmp/garage.s3 |cut -d' ' -f1`
 SECRET_KEY=`cat /tmp/garage.s3 |cut -d' ' -f2`
 
-alias s3grg="s3cmd \
-  --host 127.0.0.1:3911 \
-  --host-bucket 127.0.0.1:3911 \
-  --access_key=$ACCESS_KEY \
-  --secret_key=$SECRET_KEY \
-  --region=garage \
-  --no-ssl"
+cat > /tmp/garage.s3cmd.cfg <<EOF
+[default]
+access_key = $ACCESS_KEY
+secret_key = $SECRET_KEY
+host_base = 127.0.0.1:3911
+host_bucket = 127.0.0.1:3911
+use_https = False
+EOF
 
+function s3cmd { command s3cmd --config=/tmp/garage.s3cmd.cfg $@ ; }
+s3cmd --version
+python3 --version

script/test-smoke.sh

@@ -1,76 +1,134 @@
 #!/bin/bash
 
 set -ex
-shopt -s expand_aliases
 
+export LC_ALL=C.UTF-8
+export LANG=C.UTF-8
 SCRIPT_FOLDER="`dirname \"$0\"`"
 REPO_FOLDER="${SCRIPT_FOLDER}/../"
+GARAGE_DEBUG="${REPO_FOLDER}/target/debug/"
+GARAGE_RELEASE="${REPO_FOLDER}/target/release/"
+PATH="${GARAGE_DEBUG}:${GARAGE_RELEASE}:$PATH"
 
-echo "setup"
+# @FIXME Duck is not ready for testing, we have a bug
+SKIP_DUCK=1
+
+echo "⏳ Setup"
 cargo build
 ${SCRIPT_FOLDER}/dev-clean.sh
 ${SCRIPT_FOLDER}/dev-cluster.sh > /tmp/garage.log 2>&1 &
 ${SCRIPT_FOLDER}/dev-configure.sh
 ${SCRIPT_FOLDER}/dev-bucket.sh
-source ${SCRIPT_FOLDER}/dev-env-aws.sh
-source ${SCRIPT_FOLDER}/dev-env-s3cmd.sh
 
+which garage
 garage status
 garage key list
 garage bucket list
 
-dd if=/dev/urandom of=/tmp/garage.1.rnd bs=1k count=2 # < INLINE_THRESHOLD = 3072 bytes
-dd if=/dev/urandom of=/tmp/garage.2.rnd bs=1M count=5
-dd if=/dev/urandom of=/tmp/garage.3.rnd bs=1M count=10
+dd if=/dev/urandom of=/tmp/garage.1.rnd bs=1k count=2 # No multipart, inline storage (< INLINE_THRESHOLD = 3072 bytes)
+dd if=/dev/urandom of=/tmp/garage.2.rnd bs=1M count=5 # No multipart but file will be chunked
+dd if=/dev/urandom of=/tmp/garage.3.rnd bs=1M count=10 # by default, AWS starts using multipart at 8MB
 
-echo "s3 api testing..."
-for idx in $(seq 1 3); do
-  # AWS sends	
-  awsgrg cp /tmp/garage.$idx.rnd s3://eprouvette/garage.$idx.aws
-  
-  awsgrg ls s3://eprouvette
-  
-  awsgrg cp s3://eprouvette/garage.$idx.aws /tmp/garage.$idx.dl
-  diff /tmp/garage.$idx.rnd /tmp/garage.$idx.dl
-  rm /tmp/garage.$idx.dl
-  
-  s3grg get s3://eprouvette/garage.$idx.aws /tmp/garage.$idx.dl
-  diff /tmp/garage.$idx.rnd /tmp/garage.$idx.dl
-  rm /tmp/garage.$idx.dl
-  
-  awsgrg rm s3://eprouvette/garage.$idx.aws
+echo "🧪 S3 API testing..."
 
-  # S3CMD sends
-  s3grg put /tmp/garage.$idx.rnd s3://eprouvette/garage.$idx.s3cmd
+# AWS
+if [ -z "$SKIP_AWS" ]; then
+  echo "🛠️ Testing with awscli"
+  source ${SCRIPT_FOLDER}/dev-env-aws.sh
+  aws s3 ls
+  for idx in $(seq 1 3); do
+    aws s3 cp "/tmp/garage.$idx.rnd" "s3://eprouvette/&+-é\"/garage.$idx.aws"
+    aws s3 ls s3://eprouvette
+    aws s3 cp "s3://eprouvette/&+-é\"/garage.$idx.aws" "/tmp/garage.$idx.dl"
+    diff /tmp/garage.$idx.rnd /tmp/garage.$idx.dl
+    rm /tmp/garage.$idx.dl
+    aws s3 rm "s3://eprouvette/&+-é\"/garage.$idx.aws"
+  done
+fi
+
+# S3CMD
+if [ -z "$SKIP_S3CMD" ]; then
+  echo "🛠️ Testing with s3cmd"
+  source ${SCRIPT_FOLDER}/dev-env-s3cmd.sh
+  s3cmd ls
+  for idx in $(seq 1 3); do
+    s3cmd put "/tmp/garage.$idx.rnd" "s3://eprouvette/&+-é\"/garage.$idx.s3cmd"
+    s3cmd ls s3://eprouvette
+    s3cmd get "s3://eprouvette/&+-é\"/garage.$idx.s3cmd" "/tmp/garage.$idx.dl"
+    diff /tmp/garage.$idx.rnd /tmp/garage.$idx.dl
+    rm /tmp/garage.$idx.dl
+    s3cmd rm "s3://eprouvette/&+-é\"/garage.$idx.s3cmd"
+  done
+fi
+
+# Minio Client
+if [ -z "$SKIP_MC" ]; then
+  echo "🛠️ Testing with mc (minio client)"
+  source ${SCRIPT_FOLDER}/dev-env-mc.sh
+  mc ls garage/
+  for idx in $(seq 1 3); do
+    mc cp "/tmp/garage.$idx.rnd" "garage/eprouvette/&+-é\"/garage.$idx.mc"
+    mc ls garage/eprouvette
+    mc cp "garage/eprouvette/&+-é\"/garage.$idx.mc" "/tmp/garage.$idx.dl"
+    diff /tmp/garage.$idx.rnd /tmp/garage.$idx.dl
+    rm /tmp/garage.$idx.dl
+    mc rm "garage/eprouvette/&+-é\"/garage.$idx.mc"
+  done
+fi
+
+# RClone
+if [ -z "$SKIP_RCLONE" ]; then
+  echo "🛠️ Testing with rclone"
+  source ${SCRIPT_FOLDER}/dev-env-rclone.sh
+  rclone lsd garage:
+  for idx in $(seq 1 3); do
+    cp /tmp/garage.$idx.rnd /tmp/garage.$idx.dl
+    rclone copy "/tmp/garage.$idx.dl" "garage:eprouvette/&+-é\"/"
+    rm /tmp/garage.$idx.dl
+    rclone ls garage:eprouvette
+    rclone copy "garage:eprouvette/&+-é\"/garage.$idx.dl" "/tmp/"
+    diff /tmp/garage.$idx.rnd /tmp/garage.$idx.dl
+    rm /tmp/garage.$idx.dl
+    rclone delete "garage:eprouvette/&+-é\"/garage.$idx.dl"
+  done
+fi
+
+# Duck (aka Cyberduck CLI)
+if [ -z "$SKIP_DUCK" ]; then
+  echo "🛠️ Testing with duck (aka cyberduck cli)"
+  source ${SCRIPT_FOLDER}/dev-env-duck.sh
+  duck --list garage:/
+  duck --mkdir "garage:/eprouvette/duck"
+  for idx in $(seq 1 3); do
+    duck --verbose --upload "garage:/eprouvette/duck/" "/tmp/garage.$idx.rnd" 
+    duck --list garage:/eprouvette/duck/
+    duck --download "garage:/eprouvette/duck/garage.$idx.rnd" "/tmp/garage.$idx.dl"
+    diff /tmp/garage.$idx.rnd /tmp/garage.$idx.dl
+    rm /tmp/garage.$idx.dl
+    duck --delete "garage:/eprouvette/duck/garage.$idx.dk"
+  done
+fi
 
-  s3grg ls s3://eprouvette
-	
-  s3grg get s3://eprouvette/garage.$idx.s3cmd /tmp/garage.$idx.dl
-  diff /tmp/garage.$idx.rnd /tmp/garage.$idx.dl
-  rm /tmp/garage.$idx.dl
-  
-  awsgrg cp s3://eprouvette/garage.$idx.s3cmd /tmp/garage.$idx.dl
-  diff /tmp/garage.$idx.rnd /tmp/garage.$idx.dl
-  rm /tmp/garage.$idx.dl
-	
-  s3grg rm s3://eprouvette/garage.$idx.s3cmd
-done
 rm /tmp/garage.{1,2,3}.rnd
 
-echo "website testing"
-echo "<h1>hello world</h1>" > /tmp/garage-index.html
-awsgrg cp /tmp/garage-index.html s3://eprouvette/index.html
-[ `curl -s -o /dev/null -w "%{http_code}" --header "Host: eprouvette.garage.tld"  http://127.0.0.1:3923/ ` == 404 ]
-garage bucket website --allow eprouvette
-[ `curl -s -o /dev/null -w "%{http_code}" --header "Host: eprouvette.garage.tld"  http://127.0.0.1:3923/ ` == 200 ]
-garage bucket website --deny eprouvette
-[ `curl -s -o /dev/null -w "%{http_code}" --header "Host: eprouvette.garage.tld"  http://127.0.0.1:3923/ ` == 404 ]
-awsgrg rm s3://eprouvette/index.html
-rm /tmp/garage-index.html
+if [ -z "$SKIP_AWS" ]; then
+  echo "🧪 Website Testing"
+  echo "<h1>hello world</h1>" > /tmp/garage-index.html
+  aws s3 cp /tmp/garage-index.html s3://eprouvette/index.html
+  [ `curl -s -o /dev/null -w "%{http_code}" --header "Host: eprouvette.garage.tld"  http://127.0.0.1:3923/ ` == 404 ]
+  garage bucket website --allow eprouvette
+  [ `curl -s -o /dev/null -w "%{http_code}" --header "Host: eprouvette.garage.tld"  http://127.0.0.1:3923/ ` == 200 ]
+  garage bucket website --deny eprouvette
+  [ `curl -s -o /dev/null -w "%{http_code}" --header "Host: eprouvette.garage.tld"  http://127.0.0.1:3923/ ` == 404 ]
+  aws s3 rm s3://eprouvette/index.html
+  rm /tmp/garage-index.html
+fi
 
-echo "teardown"
+echo "🏁 Teardown"
+AWS_ACCESS_KEY_ID=`cat /tmp/garage.s3 |cut -d' ' -f1`
+AWS_SECRET_ACCESS_KEY=`cat /tmp/garage.s3 |cut -d' ' -f2`
 garage bucket deny --read --write eprouvette --key $AWS_ACCESS_KEY_ID
 garage bucket delete --yes eprouvette
 garage key delete --yes $AWS_ACCESS_KEY_ID
 
-echo "success"
+echo "✅ Success"

src/api/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "garage_api"
-version = "0.2.1"
+version = "0.3.0"
 authors = ["Alex Auvolat <alex@adnab.me>"]
 edition = "2018"
 license = "AGPL-3.0"
@@ -13,9 +13,9 @@ path = "lib.rs"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-garage_model = { version = "0.2.1", path = "../model" }
-garage_table = { version = "0.2.1", path = "../table" }
-garage_util = { version = "0.2.1", path = "../util" }
+garage_model = { version = "0.3.0", path = "../model" }
+garage_table = { version = "0.3.0", path = "../table" }
+garage_util = { version = "0.3.0", path = "../util" }
 
 base64 = "0.13"
 bytes = "1.0"
@@ -38,4 +38,6 @@ http-range = "0.1"
 hyper = "0.14"
 percent-encoding = "2.1.0"
 roxmltree = "0.14"
+serde = { version = "1.0", features = ["derive"] }
+quick-xml = { version = "0.21", features = [ "serialize" ] }
 url = "2.1"

src/api/api_server.rs

@@ -14,12 +14,14 @@ use garage_model::garage::Garage;
 use crate::error::*;
 use crate::signature::check_signature;
 
+use crate::s3_bucket::*;
 use crate::s3_copy::*;
 use crate::s3_delete::*;
 use crate::s3_get::*;
 use crate::s3_list::*;
 use crate::s3_put::*;
 
+/// Run the S3 API server
 pub async fn run_api_server(
 	garage: Arc<Garage>,
 	shutdown_signal: impl Future<Output = ()>,
@@ -51,17 +53,21 @@ async fn handler(
 	req: Request<Body>,
 	addr: SocketAddr,
 ) -> Result<Response<Body>, GarageError> {
-	info!("{} {} {}", addr, req.method(), req.uri());
+	let uri = req.uri().clone();
+	info!("{} {} {}", addr, req.method(), uri);
 	debug!("{:?}", req);
-	match handler_inner(garage, req).await {
+	match handler_inner(garage.clone(), req).await {
 		Ok(x) => {
 			debug!("{} {:?}", x.status(), x.headers());
 			Ok(x)
 		}
 		Err(e) => {
-			let body: Body = Body::from(format!("{}\n", e));
-			let mut http_error = Response::new(body);
-			*http_error.status_mut() = e.http_status_code();
+			let body: Body = Body::from(e.aws_xml(&garage.config.s3_api.s3_region, uri.path()));
+			let http_error = Response::builder()
+				.status(e.http_status_code())
+				.header("Content-Type", "application/xml")
+				.body(body)?;
+
 			if e.http_status_code().is_server_error() {
 				warn!("Response: error {}, {}", e.http_status_code(), e);
 			} else {
@@ -75,18 +81,20 @@ async fn handler(
 async fn handler_inner(garage: Arc<Garage>, req: Request<Body>) -> Result<Response<Body>, Error> {
 	let path = req.uri().path().to_string();
 	let path = percent_encoding::percent_decode_str(&path).decode_utf8()?;
+	let (api_key, content_sha256) = check_signature(&garage, &req).await?;
+	if path == "/" {
+		return handle_list_buckets(&api_key);
+	}
 
 	let (bucket, key) = parse_bucket_key(&path)?;
-
-	let (api_key, content_sha256) = check_signature(&garage, &req).await?;
 	let allowed = match req.method() {
 		&Method::HEAD | &Method::GET => api_key.allow_read(&bucket),
 		_ => api_key.allow_write(&bucket),
 	};
 	if !allowed {
-		return Err(Error::Forbidden(format!(
-			"Operation is not allowed for this key."
-		)));
+		return Err(Error::Forbidden(
+			"Operation is not allowed for this key.".to_string(),
+		));
 	}
 
 	let mut params = HashMap::new();
@@ -98,16 +106,16 @@ async fn handler_inner(garage: Arc<Garage>, req: Request<Body>) -> Result<Respon
 	}
 
 	if let Some(key) = key {
-		match req.method() {
-			&Method::HEAD => {
+		match *req.method() {
+			Method::HEAD => {
 				// HeadObject query
 				Ok(handle_head(garage, &req, &bucket, &key).await?)
 			}
-			&Method::GET => {
+			Method::GET => {
 				// GetObject query
 				Ok(handle_get(garage, &req, &bucket, &key).await?)
 			}
-			&Method::PUT => {
+			Method::PUT => {
 				if params.contains_key(&"partnumber".to_string())
 					&& params.contains_key(&"uploadid".to_string())
 				{
@@ -146,7 +154,7 @@ async fn handler_inner(garage: Arc<Garage>, req: Request<Body>) -> Result<Respon
 					Ok(handle_put(garage, req, &bucket, &key, content_sha256).await?)
 				}
 			}
-			&Method::DELETE => {
+			Method::DELETE => {
 				if params.contains_key(&"uploadid".to_string()) {
 					// AbortMultipartUpload query
 					let upload_id = params.get("uploadid").unwrap();
@@ -156,7 +164,7 @@ async fn handler_inner(garage: Arc<Garage>, req: Request<Body>) -> Result<Respon
 					Ok(handle_delete(garage, &bucket, &key).await?)
 				}
 			}
-			&Method::POST => {
+			Method::POST => {
 				if params.contains_key(&"uploads".to_string()) {
 					// CreateMultipartUpload call
 					Ok(handle_create_multipart_upload(garage, &req, &bucket, &key).await?)
@@ -173,16 +181,16 @@ async fn handler_inner(garage: Arc<Garage>, req: Request<Body>) -> Result<Respon
 					)
 					.await?)
 				} else {
-					Err(Error::BadRequest(format!(
-						"Not a CreateMultipartUpload call, what is it?"
-					)))
+					Err(Error::BadRequest(
+						"Not a CreateMultipartUpload call, what is it?".to_string(),
+					))
 				}
 			}
-			_ => Err(Error::BadRequest(format!("Invalid method"))),
+			_ => Err(Error::BadRequest("Invalid method".to_string())),
 		}
 	} else {
-		match req.method() {
-			&Method::PUT => {
+		match *req.method() {
+			Method::PUT => {
 				// CreateBucket
 				// If we're here, the bucket already exists, so just answer ok
 				debug!(
@@ -197,24 +205,32 @@ async fn handler_inner(garage: Arc<Garage>, req: Request<Body>) -> Result<Respon
 					.unwrap();
 				Ok(response)
 			}
-			&Method::HEAD => {
+			Method::HEAD => {
 				// HeadBucket
 				let empty_body: Body = Body::from(vec![]);
 				let response = Response::builder().body(empty_body).unwrap();
 				Ok(response)
 			}
-			&Method::DELETE => {
+			Method::DELETE => {
 				// DeleteBucket query
 				Err(Error::Forbidden(
 					"Cannot delete buckets using S3 api, please talk to Garage directly".into(),
 				))
 			}
-			&Method::GET => {
-				// ListObjects or ListObjectsV2 query
-				let q = parse_list_objects_query(bucket, &params)?;
-				Ok(handle_list(garage, &q).await?)
+			Method::GET => {
+				if params.contains_key("location") {
+					// GetBucketLocation call
+					Ok(handle_get_bucket_location(garage)?)
+				} else if params.contains_key("versioning") {
+					// GetBucketVersioning
+					Ok(handle_get_bucket_versioning()?)
+				} else {
+					// ListObjects or ListObjectsV2 query
+					let q = parse_list_objects_query(bucket, &params)?;
+					Ok(handle_list(garage, &q).await?)
+				}
 			}
-			&Method::POST => {
+			Method::POST => {
 				if params.contains_key(&"delete".to_string()) {
 					// DeleteObjects
 					Ok(handle_delete_objects(garage, bucket, req, content_sha256).await?)
@@ -224,10 +240,10 @@ async fn handler_inner(garage: Arc<Garage>, req: Request<Body>) -> Result<Respon
 						std::str::from_utf8(&hyper::body::to_bytes(req.into_body()).await?)
 							.unwrap_or("<invalid utf8>")
 					);
-					Err(Error::BadRequest(format!("Unsupported call")))
+					Err(Error::BadRequest("Unsupported call".to_string()))
 				}
 			}
-			_ => Err(Error::BadRequest(format!("Invalid method"))),
+			_ => Err(Error::BadRequest("Invalid method".to_string())),
 		}
 	}
 }
@@ -242,7 +258,7 @@ fn parse_bucket_key(path: &str) -> Result<(&str, Option<&str>), Error> {
 	let (bucket, key) = match path.find('/') {
 		Some(i) => {
 			let key = &path[i + 1..];
-			if key.len() > 0 {
+			if !key.is_empty() {
 				(&path[..i], Some(key))
 			} else {
 				(&path[..i], None)
@@ -250,8 +266,8 @@ fn parse_bucket_key(path: &str) -> Result<(&str, Option<&str>), Error> {
 		}
 		None => (path, None),
 	};
-	if bucket.len() == 0 {
-		return Err(Error::BadRequest(format!("No bucket specified")));
+	if bucket.is_empty() {
+		return Err(Error::BadRequest("No bucket specified".to_string()));
 	}
 	Ok((bucket, key))
 }

src/api/encoding.rs

@@ -1,9 +1,6 @@
-pub fn xml_escape(s: &str) -> String {
-	s.replace("<", "&lt;")
-		.replace(">", "&gt;")
-		.replace("\"", "&quot;")
-}
+//! Module containing various helpers for encoding
 
+/// Encode &str for use in a URI
 pub fn uri_encode(string: &str, encode_slash: bool) -> String {
 	let mut result = String::with_capacity(string.len() * 2);
 	for c in string.chars() {
@@ -23,11 +20,3 @@ pub fn uri_encode(string: &str, encode_slash: bool) -> String {
 	}
 	result
 }
-
-pub fn xml_encode_key(k: &str, urlencode: bool) -> String {
-	if urlencode {
-		uri_encode(k, true)
-	} else {
-		xml_escape(k)
-	}
-}

src/api/error.rs

@@ -3,68 +3,125 @@ use hyper::StatusCode;
 
 use garage_util::error::Error as GarageError;
 
+use crate::s3_xml;
+
+/// Errors of this crate
 #[derive(Debug, Error)]
 pub enum Error {
 	// Category: internal error
+	/// Error related to deeper parts of Garage
 	#[error(display = "Internal error: {}", _0)]
 	InternalError(#[error(source)] GarageError),
 
+	/// Error related to Hyper
 	#[error(display = "Internal error (Hyper error): {}", _0)]
 	Hyper(#[error(source)] hyper::Error),
 
+	/// Error related to HTTP
 	#[error(display = "Internal error (HTTP error): {}", _0)]
-	HTTP(#[error(source)] http::Error),
+	Http(#[error(source)] http::Error),
 
 	// Category: cannot process
+	/// No proper api key was used, or the signature was invalid
 	#[error(display = "Forbidden: {}", _0)]
 	Forbidden(String),
 
+	/// Authorization Header Malformed
+	#[error(display = "Authorization header malformed, expected scope: {}", _0)]
+	AuthorizationHeaderMalformed(String),
+
+	/// The object requested don't exists
 	#[error(display = "Not found")]
 	NotFound,
 
 	// Category: bad request
+	/// The request contained an invalid UTF-8 sequence in its path or in other parameters
 	#[error(display = "Invalid UTF-8: {}", _0)]
-	InvalidUTF8Str(#[error(source)] std::str::Utf8Error),
+	InvalidUtf8Str(#[error(source)] std::str::Utf8Error),
 
+	/// The request used an invalid path
 	#[error(display = "Invalid UTF-8: {}", _0)]
-	InvalidUTF8String(#[error(source)] std::string::FromUtf8Error),
+	InvalidUtf8String(#[error(source)] std::string::FromUtf8Error),
 
+	/// Some base64 encoded data was badly encoded
 	#[error(display = "Invalid base64: {}", _0)]
 	InvalidBase64(#[error(source)] base64::DecodeError),
 
+	/// The client sent invalid XML data
 	#[error(display = "Invalid XML: {}", _0)]
-	InvalidXML(String),
+	InvalidXml(String),
 
+	/// The client sent a header with invalid value
 	#[error(display = "Invalid header value: {}", _0)]
 	InvalidHeader(#[error(source)] hyper::header::ToStrError),
 
+	/// The client sent a range header with invalid value
 	#[error(display = "Invalid HTTP range: {:?}", _0)]
 	InvalidRange(#[error(from)] http_range::HttpRangeParseError),
 
+	/// The client sent an invalid request
 	#[error(display = "Bad request: {}", _0)]
 	BadRequest(String),
 }
 
 impl From<roxmltree::Error> for Error {
 	fn from(err: roxmltree::Error) -> Self {
-		Self::InvalidXML(format!("{}", err))
+		Self::InvalidXml(format!("{}", err))
+	}
+}
+
+impl From<quick_xml::de::DeError> for Error {
+	fn from(err: quick_xml::de::DeError) -> Self {
+		Self::InvalidXml(format!("{}", err))
 	}
 }
 
 impl Error {
+	/// Get the HTTP status code that best represents the meaning of the error for the client
 	pub fn http_status_code(&self) -> StatusCode {
 		match self {
 			Error::NotFound => StatusCode::NOT_FOUND,
 			Error::Forbidden(_) => StatusCode::FORBIDDEN,
-			Error::InternalError(GarageError::RPC(_)) => StatusCode::SERVICE_UNAVAILABLE,
-			Error::InternalError(_) | Error::Hyper(_) | Error::HTTP(_) => {
+			Error::InternalError(GarageError::Rpc(_)) => StatusCode::SERVICE_UNAVAILABLE,
+			Error::InternalError(_) | Error::Hyper(_) | Error::Http(_) => {
 				StatusCode::INTERNAL_SERVER_ERROR
 			}
 			_ => StatusCode::BAD_REQUEST,
 		}
 	}
+
+	pub fn aws_code(&self) -> &'static str {
+		match self {
+			Error::NotFound => "NoSuchKey",
+			Error::Forbidden(_) => "AccessDenied",
+			Error::AuthorizationHeaderMalformed(_) => "AuthorizationHeaderMalformed",
+			Error::InternalError(GarageError::Rpc(_)) => "ServiceUnavailable",
+			Error::InternalError(_) | Error::Hyper(_) | Error::Http(_) => "InternalError",
+			_ => "InvalidRequest",
+		}
+	}
+
+	pub fn aws_xml(&self, garage_region: &str, path: &str) -> String {
+		let error = s3_xml::Error {
+			code: s3_xml::Value(self.aws_code().to_string()),
+			message: s3_xml::Value(format!("{}", self)),
+			resource: Some(s3_xml::Value(path.to_string())),
+			region: Some(s3_xml::Value(garage_region.to_string())),
+		};
+		s3_xml::to_xml_with_header(&error).unwrap_or_else(|_| {
+			r#"
+<?xml version="1.0" encoding="UTF-8"?>
+<Error>
+	<Code>InternalError</Code>
+	<Message>XML encoding of error failed</Message>
+</Error>
+			"#
+			.into()
+		})
+	}
 }
 
+/// Trait to map error to the Bad Request error code
 pub trait OkOrBadRequest {
 	type S2;
 	fn ok_or_bad_request(self, reason: &'static str) -> Self::S2;
@@ -88,11 +145,12 @@ impl<T> OkOrBadRequest for Option<T> {
 	fn ok_or_bad_request(self, reason: &'static str) -> Result<T, Error> {
 		match self {
 			Some(x) => Ok(x),
-			None => Err(Error::BadRequest(format!("{}", reason))),
+			None => Err(Error::BadRequest(reason.to_string())),
 		}
 	}
 }
 
+/// Trait to map an error to an Internal Error code
 pub trait OkOrInternalError {
 	type S2;
 	fn ok_or_internal_error(self, reason: &'static str) -> Self::S2;
@@ -119,10 +177,9 @@ impl<T> OkOrInternalError for Option<T> {
 	fn ok_or_internal_error(self, reason: &'static str) -> Result<T, Error> {
 		match self {
 			Some(x) => Ok(x),
-			None => Err(Error::InternalError(GarageError::Message(format!(
-				"{}",
-				reason
-			)))),
+			None => Err(Error::InternalError(GarageError::Message(
+				reason.to_string(),
+			))),
 		}
 	}
 }

src/api/lib.rs

@@ -1,15 +1,21 @@
+//! Crate for serving a S3 compatible API
 #[macro_use]
 extern crate log;
 
-pub mod error;
+mod error;
+pub use error::Error;
 
-pub mod encoding;
+mod encoding;
 
-pub mod api_server;
-pub mod signature;
+mod api_server;
+pub use api_server::run_api_server;
 
-pub mod s3_copy;
-pub mod s3_delete;
+mod signature;
+
+mod s3_bucket;
+mod s3_copy;
+mod s3_delete;
 pub mod s3_get;
-pub mod s3_list;
-pub mod s3_put;
+mod s3_list;
+mod s3_put;
+mod s3_xml;

src/api/s3_bucket.rs

@@ -0,0 +1,62 @@
+use std::sync::Arc;
+
+use hyper::{Body, Response};
+
+use garage_model::garage::Garage;
+use garage_model::key_table::Key;
+use garage_util::time::*;
+
+use crate::error::*;
+use crate::s3_xml;
+
+pub fn handle_get_bucket_location(garage: Arc<Garage>) -> Result<Response<Body>, Error> {
+	let loc = s3_xml::LocationConstraint {
+		xmlns: (),
+		region: garage.config.s3_api.s3_region.to_string(),
+	};
+	let xml = s3_xml::to_xml_with_header(&loc)?;
+
+	Ok(Response::builder()
+		.header("Content-Type", "application/xml")
+		.body(Body::from(xml.into_bytes()))?)
+}
+
+pub fn handle_get_bucket_versioning() -> Result<Response<Body>, Error> {
+	let versioning = s3_xml::VersioningConfiguration {
+		xmlns: (),
+		status: None,
+	};
+
+	let xml = s3_xml::to_xml_with_header(&versioning)?;
+
+	Ok(Response::builder()
+		.header("Content-Type", "application/xml")
+		.body(Body::from(xml.into_bytes()))?)
+}
+
+pub fn handle_list_buckets(api_key: &Key) -> Result<Response<Body>, Error> {
+	let list_buckets = s3_xml::ListAllMyBucketsResult {
+		owner: s3_xml::Owner {
+			display_name: s3_xml::Value(api_key.name.get().to_string()),
+			id: s3_xml::Value(api_key.key_id.to_string()),
+		},
+		buckets: s3_xml::BucketList {
+			entries: api_key
+				.authorized_buckets
+				.items()
+				.iter()
+				.map(|(name, ts, _)| s3_xml::Bucket {
+					creation_date: s3_xml::Value(msec_to_rfc3339(*ts)),
+					name: s3_xml::Value(name.to_string()),
+				})
+				.collect(),
+		},
+	};
+
+	let xml = s3_xml::to_xml_with_header(&list_buckets)?;
+	trace!("xml: {}", xml);
+
+	Ok(Response::builder()
+		.header("Content-Type", "application/xml")
+		.body(Body::from(xml))?)
+}

src/api/s3_copy.rs

@@ -1,4 +1,3 @@
-use std::fmt::Write;
 use std::sync::Arc;
 
 use hyper::{Body, Request, Response};
@@ -14,6 +13,7 @@ use garage_model::version_table::*;
 
 use crate::error::*;
 use crate::s3_put::get_headers;
+use crate::s3_xml;
 
 pub async fn handle_copy(
 	garage: Arc<Garage>,
@@ -33,8 +33,7 @@ pub async fn handle_copy(
 		.versions()
 		.iter()
 		.rev()
-		.filter(|v| v.is_complete())
-		.next()
+		.find(|v| v.is_complete())
 		.ok_or(Error::NotFound)?;
 
 	let source_last_state = match &source_last_v.state {
@@ -62,6 +61,8 @@ pub async fn handle_copy(
 		_ => old_meta.clone(),
 	};
 
+	let etag = new_meta.etag.to_string();
+
 	// Save object copy
 	match source_last_state {
 		ObjectVersionData::DeleteMarker => unreachable!(),
@@ -159,13 +160,13 @@ pub async fn handle_copy(
 	}
 
 	let last_modified = msec_to_rfc3339(new_timestamp);
-	let mut xml = String::new();
-	writeln!(&mut xml, r#"<?xml version="1.0" encoding="UTF-8"?>"#).unwrap();
-	writeln!(&mut xml, r#"<CopyObjectResult>"#).unwrap();
-	writeln!(&mut xml, "\t<LastModified>{}</LastModified>", last_modified).unwrap();
-	writeln!(&mut xml, "</CopyObjectResult>").unwrap();
+	let result = s3_xml::CopyObjectResult {
+		last_modified: s3_xml::Value(last_modified),
+		etag: s3_xml::Value(etag),
+	};
+	let xml = s3_xml::to_xml_with_header(&result)?;
 
 	Ok(Response::builder()
 		.header("Content-Type", "application/xml")
-		.body(Body::from(xml.into_bytes()))?)
+		.body(Body::from(xml))?)
 }

src/api/s3_delete.rs

@@ -1,4 +1,3 @@
-use std::fmt::Write;
 use std::sync::Arc;
 
 use hyper::{Body, Request, Response};
@@ -9,25 +8,27 @@ use garage_util::time::*;
 use garage_model::garage::Garage;
 use garage_model::object_table::*;
 
-use crate::encoding::*;
 use crate::error::*;
+use crate::s3_xml;
 use crate::signature::verify_signed_content;
 
 async fn handle_delete_internal(
 	garage: &Garage,
 	bucket: &str,
 	key: &str,
-) -> Result<(UUID, UUID), Error> {
+) -> Result<(Uuid, Uuid), Error> {
 	let object = garage
 		.object_table
 		.get(&bucket.to_string(), &key.to_string())
 		.await?
 		.ok_or(Error::NotFound)?; // No need to delete
 
-	let interesting_versions = object.versions().iter().filter(|v| match v.state {
-		ObjectVersionState::Aborted => false,
-		ObjectVersionState::Complete(ObjectVersionData::DeleteMarker) => false,
-		_ => true,
+	let interesting_versions = object.versions().iter().filter(|v| {
+		!matches!(
+			v.state,
+			ObjectVersionState::Aborted
+				| ObjectVersionState::Complete(ObjectVersionData::DeleteMarker)
+		)
 	});
 
 	let mut version_to_delete = None;
@@ -83,52 +84,45 @@ pub async fn handle_delete_objects(
 	let cmd_xml = roxmltree::Document::parse(&std::str::from_utf8(&body)?)?;
 	let cmd = parse_delete_objects_xml(&cmd_xml).ok_or_bad_request("Invalid delete XML query")?;
 
-	let mut retxml = String::new();
-	writeln!(&mut retxml, r#"<?xml version="1.0" encoding="UTF-8"?>"#).unwrap();
-	writeln!(&mut retxml, "<DeleteObjectsOutput>").unwrap();
+	let mut ret_deleted = Vec::new();
+	let mut ret_errors = Vec::new();
 
 	for obj in cmd.objects.iter() {
 		match handle_delete_internal(&garage, bucket, &obj.key).await {
 			Ok((deleted_version, delete_marker_version)) => {
-				writeln!(&mut retxml, "\t<Deleted>").unwrap();
-				writeln!(&mut retxml, "\t\t<Key>{}</Key>", xml_escape(&obj.key)).unwrap();
-				writeln!(
-					&mut retxml,
-					"\t\t<VersionId>{}</VersionId>",
-					hex::encode(deleted_version)
-				)
-				.unwrap();
-				writeln!(
-					&mut retxml,
-					"\t\t<DeleteMarkerVersionId>{}</DeleteMarkerVersionId>",
-					hex::encode(delete_marker_version)
-				)
-				.unwrap();
-				writeln!(&mut retxml, "\t</Deleted>").unwrap();
+				if cmd.quiet {
+					continue;
+				}
+				ret_deleted.push(s3_xml::Deleted {
+					key: s3_xml::Value(obj.key.clone()),
+					version_id: s3_xml::Value(hex::encode(deleted_version)),
+					delete_marker_version_id: s3_xml::Value(hex::encode(delete_marker_version)),
+				});
 			}
 			Err(e) => {
-				writeln!(&mut retxml, "\t<Error>").unwrap();
-				writeln!(&mut retxml, "\t\t<Code>{}</Code>", e.http_status_code()).unwrap();
-				writeln!(&mut retxml, "\t\t<Key>{}</Key>", xml_escape(&obj.key)).unwrap();
-				writeln!(
-					&mut retxml,
-					"\t\t<Message>{}</Message>",
-					xml_escape(&format!("{}", e))
-				)
-				.unwrap();
-				writeln!(&mut retxml, "\t</Error>").unwrap();
+				ret_errors.push(s3_xml::DeleteError {
+					code: s3_xml::Value(e.aws_code().to_string()),
+					key: Some(s3_xml::Value(obj.key.clone())),
+					message: s3_xml::Value(format!("{}", e)),
+					version_id: None,
+				});
 			}
 		}
 	}
 
-	writeln!(&mut retxml, "</DeleteObjectsOutput>").unwrap();
+	let xml = s3_xml::to_xml_with_header(&s3_xml::DeleteResult {
+		xmlns: (),
+		deleted: ret_deleted,
+		errors: ret_errors,
+	})?;
 
 	Ok(Response::builder()
 		.header("Content-Type", "application/xml")
-		.body(Body::from(retxml.into_bytes()))?)
+		.body(Body::from(xml))?)
 }
 
 struct DeleteRequest {
+	quiet: bool,
 	objects: Vec<DeleteObject>,
 }
 
@@ -137,7 +131,10 @@ struct DeleteObject {
 }
 
 fn parse_delete_objects_xml(xml: &roxmltree::Document) -> Option<DeleteRequest> {
-	let mut ret = DeleteRequest { objects: vec![] };
+	let mut ret = DeleteRequest {
+		quiet: false,
+		objects: vec![],
+	};
 
 	let root = xml.root();
 	let delete = root.first_child()?;
@@ -153,6 +150,12 @@ fn parse_delete_objects_xml(xml: &roxmltree::Document) -> Option<DeleteRequest>
 			ret.objects.push(DeleteObject {
 				key: key_str.to_string(),
 			});
+		} else if item.has_tag_name("Quiet") {
+			if item.text()? == "true" {
+				ret.quiet = true;
+			} else {
+				ret.quiet = false;
+			}
 		} else {
 			return None;
 		}

src/api/s3_get.rs

@@ -1,3 +1,4 @@
+//! Function related to GET and HEAD requests
 use std::sync::Arc;
 use std::time::{Duration, UNIX_EPOCH};
 
@@ -27,7 +28,7 @@ fn object_headers(
 			version_meta.headers.content_type.to_string(),
 		)
 		.header("Last-Modified", date_str)
-		.header("Accept-Ranges", format!("bytes"));
+		.header("Accept-Ranges", "bytes".to_string());
 
 	if !version_meta.etag.is_empty() {
 		resp = resp.header("ETag", format!("\"{}\"", version_meta.etag));
@@ -79,6 +80,7 @@ fn try_answer_cached(
 	}
 }
 
+/// Handle HEAD request
 pub async fn handle_head(
 	garage: Arc<Garage>,
 	req: &Request<Body>,
@@ -95,8 +97,7 @@ pub async fn handle_head(
 		.versions()
 		.iter()
 		.rev()
-		.filter(|v| v.is_data())
-		.next()
+		.find(|v| v.is_data())
 		.ok_or(Error::NotFound)?;
 
 	let version_meta = match &version.state {
@@ -118,6 +119,7 @@ pub async fn handle_head(
 	Ok(response)
 }
 
+/// Handle GET request
 pub async fn handle_get(
 	garage: Arc<Garage>,
 	req: &Request<Body>,
@@ -134,8 +136,7 @@ pub async fn handle_get(
 		.versions()
 		.iter()
 		.rev()
-		.filter(|v| v.is_complete())
-		.next()
+		.find(|v| v.is_complete())
 		.ok_or(Error::NotFound)?;
 
 	let last_v_data = match &last_v.state {
@@ -157,7 +158,9 @@ pub async fn handle_get(
 			let range_str = range.to_str()?;
 			let mut ranges = http_range::HttpRange::parse(range_str, last_v_meta.size)?;
 			if ranges.len() > 1 {
-				return Err(Error::BadRequest(format!("Multiple ranges not supported")));
+				return Err(Error::BadRequest(
+					"Multiple ranges not supported".to_string(),
+				));
 			} else {
 				ranges.pop()
 			}
@@ -224,7 +227,7 @@ pub async fn handle_get(
 	}
 }
 
-pub async fn handle_get_range(
+async fn handle_get_range(
 	garage: Arc<Garage>,
 	version: &ObjectVersion,
 	version_data: &ObjectVersionData,
@@ -233,7 +236,7 @@ pub async fn handle_get_range(
 	end: u64,
 ) -> Result<Response<Body>, Error> {
 	if end > version_meta.size {
-		return Err(Error::BadRequest(format!("Range not included in file")));
+		return Err(Error::BadRequest("Range not included in file".to_string()));
 	}
 
 	let resp_builder = object_headers(version, version_meta)
@@ -279,7 +282,7 @@ pub async fn handle_get_range(
 				}
 				// Keep only blocks that have an intersection with the requested range
 				if true_offset < end && true_offset + b.size > begin {
-					blocks.push((b.clone(), true_offset));
+					blocks.push((*b, true_offset));
 				}
 				true_offset += b.size;
 			}
@@ -300,9 +303,9 @@ pub async fn handle_get_range(
 						} else {
 							end - true_offset
 						};
-						Result::<Bytes, Error>::Ok(Bytes::from(
+						Result::<Bytes, Error>::Ok(
 							data.slice(start_in_block as usize..end_in_block as usize),
-						))
+						)
 					}
 				})
 				.buffered(2);

src/api/s3_list.rs

@@ -1,5 +1,4 @@
 use std::collections::{BTreeMap, BTreeSet, HashMap};
-use std::fmt::Write;
 use std::sync::Arc;
 
 use hyper::{Body, Response};
@@ -14,6 +13,7 @@ use garage_table::DeletedFilter;
 
 use crate::encoding::*;
 use crate::error::*;
+use crate::s3_xml;
 
 #[derive(Debug)]
 pub struct ListObjectsQuery {
@@ -50,7 +50,7 @@ pub fn parse_list_objects_query(
 					.ok_or_bad_request("Invalid value for max-keys")
 			})
 			.unwrap_or(Ok(1000))?,
-		prefix: params.get("prefix").cloned().unwrap_or(String::new()),
+		prefix: params.get("prefix").cloned().unwrap_or_default(),
 		marker: params.get("marker").cloned(),
 		continuation_token: params.get("continuation-token").cloned(),
 		start_after: params.get("start-after").cloned(),
@@ -72,10 +72,13 @@ pub async fn handle_list(
 		if let Some(ct) = &query.continuation_token {
 			String::from_utf8(base64::decode(ct.as_bytes())?)?
 		} else {
-			query.start_after.clone().unwrap_or(query.prefix.clone())
+			query
+				.start_after
+				.clone()
+				.unwrap_or_else(|| query.prefix.clone())
 		}
 	} else {
-		query.marker.clone().unwrap_or(query.prefix.clone())
+		query.marker.clone().unwrap_or_else(|| query.prefix.clone())
 	};
 
 	debug!(
@@ -155,131 +158,86 @@ pub async fn handle_list(
 			truncated = None;
 			break 'query_loop;
 		}
-		if objects.len() > 0 {
+		if !objects.is_empty() {
 			next_chunk_start = objects[objects.len() - 1].key.clone();
 		}
 	}
 
-	let mut xml = String::new();
-	writeln!(&mut xml, r#"<?xml version="1.0" encoding="UTF-8"?>"#).unwrap();
-	writeln!(
-		&mut xml,
-		r#"<ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">"#
-	)
-	.unwrap();
+	let mut result = s3_xml::ListBucketResult {
+		xmlns: (),
+		name: s3_xml::Value(query.bucket.to_string()),
+		prefix: uriencode_maybe(&query.prefix, query.urlencode_resp),
+		marker: None,
+		next_marker: None,
+		start_after: None,
+		continuation_token: None,
+		next_continuation_token: None,
+		max_keys: s3_xml::IntValue(query.max_keys as i64),
+		delimiter: query
+			.delimiter
+			.as_ref()
+			.map(|x| uriencode_maybe(x, query.urlencode_resp)),
+		encoding_type: match query.urlencode_resp {
+			true => Some(s3_xml::Value("url".to_string())),
+			false => None,
+		},
 
-	writeln!(&mut xml, "\t<Name>{}</Name>", query.bucket).unwrap();
-
-	// TODO: in V1, is this supposed to be urlencoded when encoding-type is URL??
-	writeln!(
-		&mut xml,
-		"\t<Prefix>{}</Prefix>",
-		xml_encode_key(&query.prefix, query.urlencode_resp),
-	)
-	.unwrap();
-
-	if let Some(delim) = &query.delimiter {
-		// TODO: in V1, is this supposed to be urlencoded when encoding-type is URL??
-		writeln!(
-			&mut xml,
-			"\t<Delimiter>{}</Delimiter>",
-			xml_encode_key(delim, query.urlencode_resp),
-		)
-		.unwrap();
-	}
-
-	writeln!(&mut xml, "\t<MaxKeys>{}</MaxKeys>", query.max_keys).unwrap();
-	if query.urlencode_resp {
-		writeln!(&mut xml, "\t<EncodingType>url</EncodingType>").unwrap();
-	}
-
-	writeln!(
-		&mut xml,
-		"\t<KeyCount>{}</KeyCount>",
-		result_keys.len() + result_common_prefixes.len()
-	)
-	.unwrap();
-	writeln!(
-		&mut xml,
-		"\t<IsTruncated>{}</IsTruncated>",
-		truncated.is_some()
-	)
-	.unwrap();
+		key_count: Some(s3_xml::IntValue(
+			result_keys.len() as i64 + result_common_prefixes.len() as i64,
+		)),
+		is_truncated: s3_xml::Value(format!("{}", truncated.is_some())),
+		contents: vec![],
+		common_prefixes: vec![],
+	};
 
 	if query.is_v2 {
 		if let Some(ct) = &query.continuation_token {
-			writeln!(&mut xml, "\t<ContinuationToken>{}</ContinuationToken>", ct).unwrap();
+			result.continuation_token = Some(s3_xml::Value(ct.to_string()));
 		}
 		if let Some(sa) = &query.start_after {
-			writeln!(
-				&mut xml,
-				"\t<StartAfter>{}</StartAfter>",
-				xml_encode_key(sa, query.urlencode_resp)
-			)
-			.unwrap();
+			result.start_after = Some(uriencode_maybe(sa, query.urlencode_resp));
 		}
 		if let Some(nct) = truncated {
-			writeln!(
-				&mut xml,
-				"\t<NextContinuationToken>{}</NextContinuationToken>",
-				base64::encode(nct.as_bytes())
-			)
-			.unwrap();
+			result.next_continuation_token = Some(s3_xml::Value(base64::encode(nct.as_bytes())));
 		}
 	} else {
 		// TODO: are these supposed to be urlencoded when encoding-type is URL??
 		if let Some(mkr) = &query.marker {
-			writeln!(
-				&mut xml,
-				"\t<Marker>{}</Marker>",
-				xml_encode_key(mkr, query.urlencode_resp)
-			)
-			.unwrap();
+			result.marker = Some(uriencode_maybe(mkr, query.urlencode_resp));
 		}
 		if let Some(next_marker) = truncated {
-			writeln!(
-				&mut xml,
-				"\t<NextMarker>{}</NextMarker>",
-				xml_encode_key(&next_marker, query.urlencode_resp)
-			)
-			.unwrap();
+			result.next_marker = Some(uriencode_maybe(&next_marker, query.urlencode_resp));
 		}
 	}
 
 	for (key, info) in result_keys.iter() {
-		let last_modif = msec_to_rfc3339(info.last_modified);
-		writeln!(&mut xml, "\t<Contents>").unwrap();
-		writeln!(
-			&mut xml,
-			"\t\t<Key>{}</Key>",
-			xml_encode_key(key, query.urlencode_resp),
-		)
-		.unwrap();
-		writeln!(&mut xml, "\t\t<LastModified>{}</LastModified>", last_modif).unwrap();
-		writeln!(&mut xml, "\t\t<Size>{}</Size>", info.size).unwrap();
-		if !info.etag.is_empty() {
-			writeln!(&mut xml, "\t\t<ETag>\"{}\"</ETag>", info.etag).unwrap();
-		}
-		writeln!(&mut xml, "\t\t<StorageClass>STANDARD</StorageClass>").unwrap();
-		writeln!(&mut xml, "\t</Contents>").unwrap();
+		result.contents.push(s3_xml::ListBucketItem {
+			key: uriencode_maybe(key, query.urlencode_resp),
+			last_modified: s3_xml::Value(msec_to_rfc3339(info.last_modified)),
+			size: s3_xml::IntValue(info.size as i64),
+			etag: s3_xml::Value(info.etag.to_string()),
+			storage_class: s3_xml::Value("STANDARD".to_string()),
+		});
 	}
 
 	for pfx in result_common_prefixes.iter() {
-		writeln!(&mut xml, "\t<CommonPrefixes>").unwrap();
 		//TODO: in V1, are these urlencoded when urlencode_resp is true ?? (proably)
-		writeln!(
-			&mut xml,
-			"\t\t<Prefix>{}</Prefix>",
-			xml_encode_key(pfx, query.urlencode_resp),
-		)
-		.unwrap();
-		writeln!(&mut xml, "\t</CommonPrefixes>").unwrap();
+		result.common_prefixes.push(s3_xml::CommonPrefix {
+			prefix: uriencode_maybe(pfx, query.urlencode_resp),
+		});
 	}
 
-	writeln!(&mut xml, "</ListBucketResult>").unwrap();
-	debug!("{}", xml);
+	let xml = s3_xml::to_xml_with_header(&result)?;
 
 	Ok(Response::builder()
 		.header("Content-Type", "application/xml")
 		.body(Body::from(xml.into_bytes()))?)
 }
+
+fn uriencode_maybe(s: &str, yes: bool) -> s3_xml::Value {
+	if yes {
+		s3_xml::Value(uri_encode(s, true))
+	} else {
+		s3_xml::Value(s.to_string())
+	}
+}

src/api/s3_put.rs

@@ -1,5 +1,4 @@
 use std::collections::{BTreeMap, VecDeque};
-use std::fmt::Write;
 use std::sync::Arc;
 
 use futures::stream::*;
@@ -18,8 +17,8 @@ use garage_model::garage::Garage;
 use garage_model::object_table::*;
 use garage_model::version_table::*;
 
-use crate::encoding::*;
 use crate::error::*;
+use crate::s3_xml;
 use crate::signature::verify_signed_content;
 
 pub async fn handle_put(
@@ -46,7 +45,7 @@ pub async fn handle_put(
 	let body = req.into_body();
 
 	let mut chunker = BodyChunker::new(body, garage.config.block_size);
-	let first_block = chunker.next().await?.unwrap_or(vec![]);
+	let first_block = chunker.next().await?.unwrap_or_default();
 
 	// If body is small enough, store it directly in the object table
 	// as "inline data". We can then return immediately.
@@ -160,16 +159,18 @@ fn ensure_checksum_matches(
 ) -> Result<(), Error> {
 	if let Some(expected_sha256) = content_sha256 {
 		if expected_sha256 != data_sha256sum {
-			return Err(Error::BadRequest(format!(
-				"Unable to validate x-amz-content-sha256"
-			)));
+			return Err(Error::BadRequest(
+				"Unable to validate x-amz-content-sha256".to_string(),
+			));
 		} else {
 			trace!("Successfully validated x-amz-content-sha256");
 		}
 	}
 	if let Some(expected_md5) = content_md5 {
 		if expected_md5.trim_matches('"') != base64::encode(data_md5sum) {
-			return Err(Error::BadRequest(format!("Unable to validate content-md5")));
+			return Err(Error::BadRequest(
+				"Unable to validate content-md5".to_string(),
+			));
 		} else {
 			trace!("Successfully validated content-md5");
 		}
@@ -291,7 +292,7 @@ impl BodyChunker {
 				self.read_all = true;
 			}
 		}
-		if self.buf.len() == 0 {
+		if self.buf.is_empty() {
 			Ok(None)
 		} else if self.buf.len() <= self.block_size {
 			let block = self.buf.drain(..).collect::<Vec<u8>>();
@@ -303,7 +304,7 @@ impl BodyChunker {
 	}
 }
 
-pub fn put_response(version_uuid: UUID, md5sum_hex: String) -> Response<Body> {
+pub fn put_response(version_uuid: Uuid, md5sum_hex: String) -> Response<Body> {
 	Response::builder()
 		.header("x-amz-version-id", hex::encode(version_uuid))
 		.header("ETag", format!("\"{}\"", md5sum_hex))
@@ -337,22 +338,13 @@ pub async fn handle_create_multipart_upload(
 	garage.version_table.insert(&version).await?;
 
 	// Send success response
-	let mut xml = String::new();
-	writeln!(&mut xml, r#"<?xml version="1.0" encoding="UTF-8"?>"#).unwrap();
-	writeln!(
-		&mut xml,
-		r#"<InitiateMultipartUploadResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">"#
-	)
-	.unwrap();
-	writeln!(&mut xml, "\t<Bucket>{}</Bucket>", bucket).unwrap();
-	writeln!(&mut xml, "\t<Key>{}</Key>", xml_escape(key)).unwrap();
-	writeln!(
-		&mut xml,
-		"\t<UploadId>{}</UploadId>",
-		hex::encode(version_uuid)
-	)
-	.unwrap();
-	writeln!(&mut xml, "</InitiateMultipartUploadResult>").unwrap();
+	let result = s3_xml::InitiateMultipartUploadResult {
+		xmlns: (),
+		bucket: s3_xml::Value(bucket.to_string()),
+		key: s3_xml::Value(key.to_string()),
+		upload_id: s3_xml::Value(hex::encode(version_uuid)),
+	};
+	let xml = s3_xml::to_xml_with_header(&result)?;
 
 	Ok(Response::new(Body::from(xml.into_bytes())))
 }
@@ -387,8 +379,8 @@ pub async fn handle_put_part(
 		futures::try_join!(garage.object_table.get(&bucket, &key), chunker.next(),)?;
 
 	// Check object is valid and multipart block can be accepted
-	let first_block = first_block.ok_or(Error::BadRequest(format!("Empty body")))?;
-	let object = object.ok_or(Error::BadRequest(format!("Object not found")))?;
+	let first_block = first_block.ok_or_else(|| Error::BadRequest("Empty body".to_string()))?;
+	let object = object.ok_or_else(|| Error::BadRequest("Object not found".to_string()))?;
 
 	if !object
 		.versions()
@@ -462,21 +454,21 @@ pub async fn handle_complete_multipart_upload(
 		garage.version_table.get(&version_uuid, &EmptyKey),
 	)?;
 
-	let object = object.ok_or(Error::BadRequest(format!("Object not found")))?;
+	let object = object.ok_or_else(|| Error::BadRequest("Object not found".to_string()))?;
 	let mut object_version = object
 		.versions()
 		.iter()
 		.find(|v| v.uuid == version_uuid && v.is_uploading())
 		.cloned()
-		.ok_or(Error::BadRequest(format!("Version not found")))?;
+		.ok_or_else(|| Error::BadRequest("Version not found".to_string()))?;
 
-	let version = version.ok_or(Error::BadRequest(format!("Version not found")))?;
-	if version.blocks.len() == 0 {
-		return Err(Error::BadRequest(format!("No data was uploaded")));
+	let version = version.ok_or_else(|| Error::BadRequest("Version not found".to_string()))?;
+	if version.blocks.is_empty() {
+		return Err(Error::BadRequest("No data was uploaded".to_string()));
 	}
 
 	let headers = match object_version.state {
-		ObjectVersionState::Uploading(headers) => headers.clone(),
+		ObjectVersionState::Uploading(headers) => headers,
 		_ => unreachable!(),
 	};
 
@@ -493,7 +485,9 @@ pub async fn handle_complete_multipart_upload(
 		.map(|x| (&x.part_number, &x.etag))
 		.eq(parts);
 	if !same_parts {
-		return Err(Error::BadRequest(format!("We don't have the same parts")));
+		return Err(Error::BadRequest(
+			"We don't have the same parts".to_string(),
+		));
 	}
 
 	// Calculate etag of final object
@@ -516,7 +510,7 @@ pub async fn handle_complete_multipart_upload(
 		ObjectVersionMeta {
 			headers,
 			size: total_size,
-			etag,
+			etag: etag.clone(),
 		},
 		version.blocks.items()[0].1.hash,
 	));
@@ -525,22 +519,14 @@ pub async fn handle_complete_multipart_upload(
 	garage.object_table.insert(&final_object).await?;
 
 	// Send response saying ok we're done
-	let mut xml = String::new();
-	writeln!(&mut xml, r#"<?xml version="1.0" encoding="UTF-8"?>"#).unwrap();
-	writeln!(
-		&mut xml,
-		r#"<CompleteMultipartUploadResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">"#
-	)
-	.unwrap();
-	writeln!(
-		&mut xml,
-		"\t<Location>{}</Location>",
-		garage.config.s3_api.s3_region
-	)
-	.unwrap();
-	writeln!(&mut xml, "\t<Bucket>{}</Bucket>", bucket).unwrap();
-	writeln!(&mut xml, "\t<Key>{}</Key>", xml_escape(&key)).unwrap();
-	writeln!(&mut xml, "</CompleteMultipartUploadResult>").unwrap();
+	let result = s3_xml::CompleteMultipartUploadResult {
+		xmlns: (),
+		location: None,
+		bucket: s3_xml::Value(bucket),
+		key: s3_xml::Value(key),
+		etag: s3_xml::Value(etag),
+	};
+	let xml = s3_xml::to_xml_with_header(&result)?;
 
 	Ok(Response::new(Body::from(xml.into_bytes())))
 }
@@ -557,7 +543,7 @@ pub async fn handle_abort_multipart_upload(
 		.object_table
 		.get(&bucket.to_string(), &key.to_string())
 		.await?;
-	let object = object.ok_or(Error::BadRequest(format!("Object not found")))?;
+	let object = object.ok_or_else(|| Error::BadRequest("Object not found".to_string()))?;
 
 	let object_version = object
 		.versions()
@@ -629,14 +615,14 @@ pub(crate) fn get_headers(req: &Request<Body>) -> Result<ObjectVersionHeaders, E
 	})
 }
 
-fn decode_upload_id(id: &str) -> Result<UUID, Error> {
+fn decode_upload_id(id: &str) -> Result<Uuid, Error> {
 	let id_bin = hex::decode(id).ok_or_bad_request("Invalid upload ID")?;
 	if id_bin.len() != 32 {
 		return None.ok_or_bad_request("Invalid upload ID");
 	}
 	let mut uuid = [0u8; 32];
 	uuid.copy_from_slice(&id_bin[..]);
-	Ok(UUID::from(uuid))
+	Ok(Uuid::from(uuid))
 }
 
 #[derive(Debug)]

src/api/s3_xml.rs

@@ -0,0 +1,629 @@
+use quick_xml::se::to_string;
+use serde::{Serialize, Serializer};
+
+use crate::Error as ApiError;
+
+pub fn to_xml_with_header<T: Serialize>(x: &T) -> Result<String, ApiError> {
+	let mut xml = r#"<?xml version="1.0" encoding="UTF-8"?>"#.to_string();
+	xml.push_str(&to_string(x)?);
+	Ok(xml)
+}
+
+fn xmlns_tag<S: Serializer>(_v: &(), s: S) -> Result<S::Ok, S::Error> {
+	s.serialize_str("http://s3.amazonaws.com/doc/2006-03-01/")
+}
+
+#[derive(Debug, Serialize, PartialEq)]
+pub struct Value(#[serde(rename = "$value")] pub String);
+
+#[derive(Debug, Serialize, PartialEq)]
+pub struct IntValue(#[serde(rename = "$value")] pub i64);
+
+#[derive(Debug, Serialize, PartialEq)]
+pub struct Bucket {
+	#[serde(rename = "CreationDate")]
+	pub creation_date: Value,
+	#[serde(rename = "Name")]
+	pub name: Value,
+}
+
+#[derive(Debug, Serialize, PartialEq)]
+pub struct DisplayName(#[serde(rename = "$value")] pub String);
+
+#[derive(Debug, Serialize, PartialEq)]
+pub struct Id(#[serde(rename = "$value")] pub String);
+
+#[derive(Debug, Serialize, PartialEq)]
+pub struct Owner {
+	#[serde(rename = "DisplayName")]
+	pub display_name: Value,
+	#[serde(rename = "ID")]
+	pub id: Value,
+}
+
+#[derive(Debug, Serialize, PartialEq)]
+pub struct BucketList {
+	#[serde(rename = "Bucket")]
+	pub entries: Vec<Bucket>,
+}
+
+#[derive(Debug, Serialize, PartialEq)]
+pub struct ListAllMyBucketsResult {
+	#[serde(rename = "Buckets")]
+	pub buckets: BucketList,
+	#[serde(rename = "Owner")]
+	pub owner: Owner,
+}
+
+#[derive(Debug, Serialize, PartialEq)]
+pub struct LocationConstraint {
+	#[serde(serialize_with = "xmlns_tag")]
+	pub xmlns: (),
+	#[serde(rename = "$value")]
+	pub region: String,
+}
+
+#[derive(Debug, Serialize, PartialEq)]
+pub struct Deleted {
+	#[serde(rename = "Key")]
+	pub key: Value,
+	#[serde(rename = "VersionId")]
+	pub version_id: Value,
+	#[serde(rename = "DeleteMarkerVersionId")]
+	pub delete_marker_version_id: Value,
+}
+
+#[derive(Debug, Serialize, PartialEq)]
+pub struct Error {
+	#[serde(rename = "Code")]
+	pub code: Value,
+	#[serde(rename = "Message")]
+	pub message: Value,
+	#[serde(rename = "Resource")]
+	pub resource: Option<Value>,
+	#[serde(rename = "Region")]
+	pub region: Option<Value>,
+}
+
+#[derive(Debug, Serialize, PartialEq)]
+pub struct DeleteError {
+	#[serde(rename = "Code")]
+	pub code: Value,
+	#[serde(rename = "Key")]
+	pub key: Option<Value>,
+	#[serde(rename = "Message")]
+	pub message: Value,
+	#[serde(rename = "VersionId")]
+	pub version_id: Option<Value>,
+}
+
+#[derive(Debug, Serialize, PartialEq)]
+pub struct DeleteResult {
+	#[serde(serialize_with = "xmlns_tag")]
+	pub xmlns: (),
+	#[serde(rename = "Deleted")]
+	pub deleted: Vec<Deleted>,
+	#[serde(rename = "Error")]
+	pub errors: Vec<DeleteError>,
+}
+
+#[derive(Debug, Serialize, PartialEq)]
+pub struct CopyObjectResult {
+	#[serde(rename = "LastModified")]
+	pub last_modified: Value,
+	#[serde(rename = "ETag")]
+	pub etag: Value,
+}
+
+#[derive(Debug, Serialize, PartialEq)]
+pub struct InitiateMultipartUploadResult {
+	#[serde(serialize_with = "xmlns_tag")]
+	pub xmlns: (),
+	#[serde(rename = "Bucket")]
+	pub bucket: Value,
+	#[serde(rename = "Key")]
+	pub key: Value,
+	#[serde(rename = "UploadId")]
+	pub upload_id: Value,
+}
+
+#[derive(Debug, Serialize, PartialEq)]
+pub struct CompleteMultipartUploadResult {
+	#[serde(serialize_with = "xmlns_tag")]
+	pub xmlns: (),
+	#[serde(rename = "Location")]
+	pub location: Option<Value>,
+	#[serde(rename = "Bucket")]
+	pub bucket: Value,
+	#[serde(rename = "Key")]
+	pub key: Value,
+	#[serde(rename = "ETag")]
+	pub etag: Value,
+}
+
+#[derive(Debug, Serialize, PartialEq)]
+pub struct ListBucketItem {
+	#[serde(rename = "Key")]
+	pub key: Value,
+	#[serde(rename = "LastModified")]
+	pub last_modified: Value,
+	#[serde(rename = "ETag")]
+	pub etag: Value,
+	#[serde(rename = "Size")]
+	pub size: IntValue,
+	#[serde(rename = "StorageClass")]
+	pub storage_class: Value,
+}
+
+#[derive(Debug, Serialize, PartialEq)]
+pub struct CommonPrefix {
+	#[serde(rename = "Prefix")]
+	pub prefix: Value,
+}
+
+#[derive(Debug, Serialize, PartialEq)]
+pub struct ListBucketResult {
+	#[serde(serialize_with = "xmlns_tag")]
+	pub xmlns: (),
+	#[serde(rename = "Name")]
+	pub name: Value,
+	#[serde(rename = "Prefix")]
+	pub prefix: Value,
+	#[serde(rename = "Marker")]
+	pub marker: Option<Value>,
+	#[serde(rename = "NextMarker")]
+	pub next_marker: Option<Value>,
+	#[serde(rename = "StartAfter")]
+	pub start_after: Option<Value>,
+	#[serde(rename = "ContinuationToken")]
+	pub continuation_token: Option<Value>,
+	#[serde(rename = "NextContinuationToken")]
+	pub next_continuation_token: Option<Value>,
+	#[serde(rename = "KeyCount")]
+	pub key_count: Option<IntValue>,
+	#[serde(rename = "MaxKeys")]
+	pub max_keys: IntValue,
+	#[serde(rename = "Delimiter")]
+	pub delimiter: Option<Value>,
+	#[serde(rename = "EncodingType")]
+	pub encoding_type: Option<Value>,
+	#[serde(rename = "IsTruncated")]
+	pub is_truncated: Value,
+	#[serde(rename = "Contents")]
+	pub contents: Vec<ListBucketItem>,
+	#[serde(rename = "CommonPrefixes")]
+	pub common_prefixes: Vec<CommonPrefix>,
+}
+
+#[derive(Debug, Serialize, PartialEq)]
+pub struct VersioningConfiguration {
+	#[serde(serialize_with = "xmlns_tag")]
+	pub xmlns: (),
+	#[serde(rename = "Status")]
+	pub status: Option<Value>,
+}
+
+#[cfg(test)]
+mod tests {
+	use super::*;
+
+	use garage_util::time::*;
+
+	#[test]
+	fn error_message() -> Result<(), ApiError> {
+		let error = Error {
+			code: Value("TestError".to_string()),
+			message: Value("A dummy error message".to_string()),
+			resource: Some(Value("/bucket/a/plop".to_string())),
+			region: Some(Value("garage".to_string())),
+		};
+		assert_eq!(
+			to_xml_with_header(&error)?,
+			"<?xml version=\"1.0\" encoding=\"UTF-8\"?>\
+<Error>\
+	<Code>TestError</Code>\
+	<Message>A dummy error message</Message>\
+	<Resource>/bucket/a/plop</Resource>\
+	<Region>garage</Region>\
+</Error>"
+		);
+		Ok(())
+	}
+
+	#[test]
+	fn list_all_my_buckets_result() -> Result<(), ApiError> {
+		let list_buckets = ListAllMyBucketsResult {
+			owner: Owner {
+				display_name: Value("owner_name".to_string()),
+				id: Value("qsdfjklm".to_string()),
+			},
+			buckets: BucketList {
+				entries: vec![
+					Bucket {
+						creation_date: Value(msec_to_rfc3339(0)),
+						name: Value("bucket_A".to_string()),
+					},
+					Bucket {
+						creation_date: Value(msec_to_rfc3339(3600 * 24 * 1000)),
+						name: Value("bucket_B".to_string()),
+					},
+				],
+			},
+		};
+		assert_eq!(
+			to_xml_with_header(&list_buckets)?,
+			"<?xml version=\"1.0\" encoding=\"UTF-8\"?>\
+<ListAllMyBucketsResult>\
+   <Buckets>\
+      <Bucket>\
+         <CreationDate>1970-01-01T00:00:00.000Z</CreationDate>\
+         <Name>bucket_A</Name>\
+      </Bucket>\
+      <Bucket>\
+         <CreationDate>1970-01-02T00:00:00.000Z</CreationDate>\
+         <Name>bucket_B</Name>\
+      </Bucket>\
+   </Buckets>\
+   <Owner>\
+      <DisplayName>owner_name</DisplayName>\
+      <ID>qsdfjklm</ID>\
+   </Owner>\
+</ListAllMyBucketsResult>"
+		);
+		Ok(())
+	}
+
+	#[test]
+	fn get_bucket_location_result() -> Result<(), ApiError> {
+		let get_bucket_location = LocationConstraint {
+			xmlns: (),
+			region: "garage".to_string(),
+		};
+		assert_eq!(
+			to_xml_with_header(&get_bucket_location)?,
+			"<?xml version=\"1.0\" encoding=\"UTF-8\"?>\
+<LocationConstraint xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">garage</LocationConstraint>"
+		);
+		Ok(())
+	}
+
+	#[test]
+	fn get_bucket_versioning_result() -> Result<(), ApiError> {
+		let get_bucket_versioning = VersioningConfiguration {
+			xmlns: (),
+			status: None,
+		};
+		assert_eq!(
+			to_xml_with_header(&get_bucket_versioning)?,
+			"<?xml version=\"1.0\" encoding=\"UTF-8\"?>\
+<VersioningConfiguration xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\"/>"
+		);
+		let get_bucket_versioning2 = VersioningConfiguration {
+			xmlns: (),
+			status: Some(Value("Suspended".to_string())),
+		};
+		assert_eq!(
+			to_xml_with_header(&get_bucket_versioning2)?,
+			"<?xml version=\"1.0\" encoding=\"UTF-8\"?>\
+<VersioningConfiguration xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\"><Status>Suspended</Status></VersioningConfiguration>"
+		);
+
+		Ok(())
+	}
+
+	#[test]
+	fn delete_result() -> Result<(), ApiError> {
+		let delete_result = DeleteResult {
+			xmlns: (),
+			deleted: vec![
+				Deleted {
+					key: Value("a/plop".to_string()),
+					version_id: Value("qsdfjklm".to_string()),
+					delete_marker_version_id: Value("wxcvbn".to_string()),
+				},
+				Deleted {
+					key: Value("b/plip".to_string()),
+					version_id: Value("1234".to_string()),
+					delete_marker_version_id: Value("4321".to_string()),
+				},
+			],
+			errors: vec![
+				DeleteError {
+					code: Value("NotFound".to_string()),
+					key: Some(Value("c/plap".to_string())),
+					message: Value("Object c/plap not found".to_string()),
+					version_id: None,
+				},
+				DeleteError {
+					code: Value("Forbidden".to_string()),
+					key: Some(Value("d/plep".to_string())),
+					message: Value("Not authorized".to_string()),
+					version_id: Some(Value("789".to_string())),
+				},
+			],
+		};
+		assert_eq!(
+			to_xml_with_header(&delete_result)?,
+			"<?xml version=\"1.0\" encoding=\"UTF-8\"?>\
+<DeleteResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">\
+    <Deleted>\
+        <Key>a/plop</Key>\
+        <VersionId>qsdfjklm</VersionId>\
+        <DeleteMarkerVersionId>wxcvbn</DeleteMarkerVersionId>\
+    </Deleted>\
+    <Deleted>\
+        <Key>b/plip</Key>\
+        <VersionId>1234</VersionId>\
+        <DeleteMarkerVersionId>4321</DeleteMarkerVersionId>\
+    </Deleted>\
+    <Error>\
+        <Code>NotFound</Code>\
+        <Key>c/plap</Key>\
+        <Message>Object c/plap not found</Message>\
+    </Error>\
+    <Error>\
+        <Code>Forbidden</Code>\
+        <Key>d/plep</Key>\
+        <Message>Not authorized</Message>\
+        <VersionId>789</VersionId>\
+    </Error>\
+</DeleteResult>"
+		);
+		Ok(())
+	}
+
+	#[test]
+	fn copy_object_result() -> Result<(), ApiError> {
+		let copy_result = CopyObjectResult {
+			last_modified: Value(msec_to_rfc3339(0)),
+			etag: Value("9b2cf535f27731c974343645a3985328".to_string()),
+		};
+		assert_eq!(
+			to_xml_with_header(&copy_result)?,
+			"<?xml version=\"1.0\" encoding=\"UTF-8\"?>\
+<CopyObjectResult>\
+    <LastModified>1970-01-01T00:00:00.000Z</LastModified>\
+    <ETag>9b2cf535f27731c974343645a3985328</ETag>\
+</CopyObjectResult>\
+			"
+		);
+		Ok(())
+	}
+
+	#[test]
+	fn initiate_multipart_upload_result() -> Result<(), ApiError> {
+		let result = InitiateMultipartUploadResult {
+			xmlns: (),
+			bucket: Value("mybucket".to_string()),
+			key: Value("a/plop".to_string()),
+			upload_id: Value("azerty".to_string()),
+		};
+		assert_eq!(
+			to_xml_with_header(&result)?,
+			"<?xml version=\"1.0\" encoding=\"UTF-8\"?>\
+<InitiateMultipartUploadResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">\
+	<Bucket>mybucket</Bucket>\
+	<Key>a/plop</Key>\
+	<UploadId>azerty</UploadId>\
+</InitiateMultipartUploadResult>"
+		);
+		Ok(())
+	}
+
+	#[test]
+	fn complete_multipart_upload_result() -> Result<(), ApiError> {
+		let result = CompleteMultipartUploadResult {
+			xmlns: (),
+			location: Some(Value("https://garage.tld/mybucket/a/plop".to_string())),
+			bucket: Value("mybucket".to_string()),
+			key: Value("a/plop".to_string()),
+			etag: Value("3858f62230ac3c915f300c664312c11f-9".to_string()),
+		};
+		assert_eq!(
+			to_xml_with_header(&result)?,
+			"<?xml version=\"1.0\" encoding=\"UTF-8\"?>\
+<CompleteMultipartUploadResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">\
+	<Location>https://garage.tld/mybucket/a/plop</Location>\
+	<Bucket>mybucket</Bucket>\
+	<Key>a/plop</Key>\
+	<ETag>3858f62230ac3c915f300c664312c11f-9</ETag>\
+</CompleteMultipartUploadResult>"
+		);
+		Ok(())
+	}
+
+	#[test]
+	fn list_objects_v1_1() -> Result<(), ApiError> {
+		let result = ListBucketResult {
+			xmlns: (),
+			name: Value("example-bucket".to_string()),
+			prefix: Value("".to_string()),
+			marker: Some(Value("".to_string())),
+			next_marker: None,
+			start_after: None,
+			continuation_token: None,
+			next_continuation_token: None,
+			key_count: None,
+			max_keys: IntValue(1000),
+			encoding_type: None,
+			delimiter: Some(Value("/".to_string())),
+			is_truncated: Value("false".to_string()),
+			contents: vec![ListBucketItem {
+				key: Value("sample.jpg".to_string()),
+				last_modified: Value(msec_to_rfc3339(0)),
+				etag: Value("bf1d737a4d46a19f3bced6905cc8b902".to_string()),
+				size: IntValue(142863),
+				storage_class: Value("STANDARD".to_string()),
+			}],
+			common_prefixes: vec![CommonPrefix {
+				prefix: Value("photos/".to_string()),
+			}],
+		};
+		assert_eq!(
+			to_xml_with_header(&result)?,
+			"<?xml version=\"1.0\" encoding=\"UTF-8\"?>\
+<ListBucketResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">\
+  <Name>example-bucket</Name>\
+  <Prefix></Prefix>\
+  <Marker></Marker>\
+  <MaxKeys>1000</MaxKeys>\
+  <Delimiter>/</Delimiter>\
+  <IsTruncated>false</IsTruncated>\
+  <Contents>\
+    <Key>sample.jpg</Key>\
+    <LastModified>1970-01-01T00:00:00.000Z</LastModified>\
+    <ETag>bf1d737a4d46a19f3bced6905cc8b902</ETag>\
+    <Size>142863</Size>\
+    <StorageClass>STANDARD</StorageClass>\
+  </Contents>\
+  <CommonPrefixes>\
+    <Prefix>photos/</Prefix>\
+  </CommonPrefixes>\
+</ListBucketResult>"
+		);
+		Ok(())
+	}
+
+	#[test]
+	fn list_objects_v1_2() -> Result<(), ApiError> {
+		let result = ListBucketResult {
+			xmlns: (),
+			name: Value("example-bucket".to_string()),
+			prefix: Value("photos/2006/".to_string()),
+			marker: Some(Value("".to_string())),
+			next_marker: None,
+			start_after: None,
+			continuation_token: None,
+			next_continuation_token: None,
+			key_count: None,
+			max_keys: IntValue(1000),
+			delimiter: Some(Value("/".to_string())),
+			encoding_type: None,
+			is_truncated: Value("false".to_string()),
+			contents: vec![],
+			common_prefixes: vec![
+				CommonPrefix {
+					prefix: Value("photos/2006/February/".to_string()),
+				},
+				CommonPrefix {
+					prefix: Value("photos/2006/January/".to_string()),
+				},
+			],
+		};
+		assert_eq!(
+			to_xml_with_header(&result)?,
+			"<?xml version=\"1.0\" encoding=\"UTF-8\"?>\
+<ListBucketResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">\
+  <Name>example-bucket</Name>\
+  <Prefix>photos/2006/</Prefix>\
+  <Marker></Marker>\
+  <MaxKeys>1000</MaxKeys>\
+  <Delimiter>/</Delimiter>\
+  <IsTruncated>false</IsTruncated>\
+  <CommonPrefixes>\
+    <Prefix>photos/2006/February/</Prefix>\
+  </CommonPrefixes>\
+  <CommonPrefixes>\
+    <Prefix>photos/2006/January/</Prefix>\
+  </CommonPrefixes>\
+</ListBucketResult>"
+		);
+		Ok(())
+	}
+
+	#[test]
+	fn list_objects_v2_1() -> Result<(), ApiError> {
+		let result = ListBucketResult {
+			xmlns: (),
+			name: Value("quotes".to_string()),
+			prefix: Value("E".to_string()),
+			marker: None,
+			next_marker: None,
+			start_after: Some(Value("ExampleGuide.pdf".to_string())),
+			continuation_token: None,
+			next_continuation_token: None,
+			key_count: None,
+			max_keys: IntValue(3),
+			delimiter: None,
+			encoding_type: None,
+			is_truncated: Value("false".to_string()),
+			contents: vec![ListBucketItem {
+				key: Value("ExampleObject.txt".to_string()),
+				last_modified: Value(msec_to_rfc3339(0)),
+				etag: Value("599bab3ed2c697f1d26842727561fd94".to_string()),
+				size: IntValue(857),
+				storage_class: Value("REDUCED_REDUNDANCY".to_string()),
+			}],
+			common_prefixes: vec![],
+		};
+		assert_eq!(
+			to_xml_with_header(&result)?,
+			"<?xml version=\"1.0\" encoding=\"UTF-8\"?>\
+<ListBucketResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">\
+  <Name>quotes</Name>\
+  <Prefix>E</Prefix>\
+  <StartAfter>ExampleGuide.pdf</StartAfter>\
+  <MaxKeys>3</MaxKeys>\
+  <IsTruncated>false</IsTruncated>\
+  <Contents>\
+    <Key>ExampleObject.txt</Key>\
+    <LastModified>1970-01-01T00:00:00.000Z</LastModified>\
+    <ETag>599bab3ed2c697f1d26842727561fd94</ETag>\
+    <Size>857</Size>\
+    <StorageClass>REDUCED_REDUNDANCY</StorageClass>\
+  </Contents>\
+</ListBucketResult>"
+		);
+		Ok(())
+	}
+
+	#[test]
+	fn list_objects_v2_2() -> Result<(), ApiError> {
+		let result = ListBucketResult {
+			xmlns: (),
+			name: Value("bucket".to_string()),
+			prefix: Value("".to_string()),
+			marker: None,
+			next_marker: None,
+			start_after: None,
+			continuation_token: Some(Value(
+				"1ueGcxLPRx1Tr/XYExHnhbYLgveDs2J/wm36Hy4vbOwM=".to_string(),
+			)),
+			next_continuation_token: Some(Value("qsdfjklm".to_string())),
+			key_count: Some(IntValue(112)),
+			max_keys: IntValue(1000),
+			delimiter: None,
+			encoding_type: None,
+			is_truncated: Value("false".to_string()),
+			contents: vec![ListBucketItem {
+				key: Value("happyfacex.jpg".to_string()),
+				last_modified: Value(msec_to_rfc3339(0)),
+				etag: Value("70ee1738b6b21e2c8a43f3a5ab0eee71".to_string()),
+				size: IntValue(1111),
+				storage_class: Value("STANDARD".to_string()),
+			}],
+			common_prefixes: vec![],
+		};
+		assert_eq!(
+			to_xml_with_header(&result)?,
+			"<?xml version=\"1.0\" encoding=\"UTF-8\"?>\
+<ListBucketResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">\
+  <Name>bucket</Name>\
+  <Prefix></Prefix>\
+  <ContinuationToken>1ueGcxLPRx1Tr/XYExHnhbYLgveDs2J/wm36Hy4vbOwM=</ContinuationToken>\
+  <NextContinuationToken>qsdfjklm</NextContinuationToken>\
+  <KeyCount>112</KeyCount>\
+  <MaxKeys>1000</MaxKeys>\
+  <IsTruncated>false</IsTruncated>\
+  <Contents>\
+    <Key>happyfacex.jpg</Key>\
+    <LastModified>1970-01-01T00:00:00.000Z</LastModified>\
+    <ETag>70ee1738b6b21e2c8a43f3a5ab0eee71</ETag>\
+    <Size>1111</Size>\
+    <StorageClass>STANDARD</StorageClass>\
+  </Contents>\
+</ListBucketResult>"
+		);
+		Ok(())
+	}
+}

src/api/signature.rs

@@ -43,13 +43,12 @@ pub async fn check_signature(
 	let date = headers
 		.get("x-amz-date")
 		.ok_or_bad_request("Missing X-Amz-Date field")?;
-	let date: NaiveDateTime = NaiveDateTime::parse_from_str(date, LONG_DATETIME)
-		.ok_or_bad_request("Invalid date")?
-		.into();
+	let date: NaiveDateTime =
+		NaiveDateTime::parse_from_str(date, LONG_DATETIME).ok_or_bad_request("Invalid date")?;
 	let date: DateTime<Utc> = DateTime::from_utc(date, Utc);
 
 	if Utc::now() - date > Duration::hours(24) {
-		return Err(Error::BadRequest(format!("Date is too old")));
+		return Err(Error::BadRequest("Date is too old".to_string()));
 	}
 
 	let scope = format!(
@@ -58,10 +57,7 @@ pub async fn check_signature(
 		garage.config.s3_api.s3_region
 	);
 	if authorization.scope != scope {
-		return Err(Error::BadRequest(format!(
-			"Invalid scope in authorization field, expected: {}",
-			scope
-		)));
+		return Err(Error::AuthorizationHeaderMalformed(scope.to_string()));
 	}
 
 	let key = garage
@@ -69,10 +65,7 @@ pub async fn check_signature(
 		.get(&EmptyKey, &authorization.key_id)
 		.await?
 		.filter(|k| !k.deleted.get())
-		.ok_or(Error::Forbidden(format!(
-			"No such key: {}",
-			authorization.key_id
-		)))?;
+		.ok_or_else(|| Error::Forbidden(format!("No such key: {}", authorization.key_id)))?;
 
 	let canonical_request = canonical_request(
 		request.method(),
@@ -98,7 +91,7 @@ pub async fn check_signature(
 		trace!("Canonical request: ``{}``", canonical_request);
 		trace!("String to sign: ``{}``", string_to_sign);
 		trace!("Expected: {}, got: {}", signature, authorization.signature);
-		return Err(Error::Forbidden(format!("Invalid signature")));
+		return Err(Error::Forbidden("Invalid signature".to_string()));
 	}
 
 	let content_sha256 = if authorization.content_sha256 == "UNSIGNED-PAYLOAD" {
@@ -108,7 +101,7 @@ pub async fn check_signature(
 			.ok_or_bad_request("Invalid content sha256 hash")?;
 		Some(
 			Hash::try_from(&bytes[..])
-				.ok_or(Error::BadRequest(format!("Invalid content sha256 hash")))?,
+				.ok_or_else(|| Error::BadRequest("Invalid content sha256 hash".to_string()))?,
 		)
 	};
 
@@ -176,9 +169,9 @@ fn parse_query_authorization(headers: &HashMap<String, String>) -> Result<Author
 		.get("x-amz-algorithm")
 		.ok_or_bad_request("X-Amz-Algorithm not found in query parameters")?;
 	if algo != "AWS4-HMAC-SHA256" {
-		return Err(Error::BadRequest(format!(
-			"Unsupported authorization method"
-		)));
+		return Err(Error::BadRequest(
+			"Unsupported authorization method".to_string(),
+		));
 	}
 
 	let cred = headers
@@ -296,9 +289,9 @@ pub fn verify_signed_content(content_sha256: Option<Hash>, body: &[u8]) -> Resul
 	let expected_sha256 =
 		content_sha256.ok_or_bad_request("Request content hash not signed, aborting.")?;
 	if expected_sha256 != sha256sum(body) {
-		return Err(Error::BadRequest(format!(
-			"Request content hash does not match signed hash"
-		)));
+		return Err(Error::BadRequest(
+			"Request content hash does not match signed hash".to_string(),
+		));
 	}
 	Ok(())
 }

src/garage/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "garage"
-version = "0.2.1"
+version = "0.3.0"
 authors = ["Alex Auvolat <alex@adnab.me>"]
 edition = "2018"
 license = "AGPL-3.0"
@@ -14,12 +14,12 @@ path = "main.rs"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-garage_api = { version = "0.2.1", path = "../api" }
-garage_model = { version = "0.2.1", path = "../model" }
-garage_rpc = { version = "0.2.1", path = "../rpc" }
-garage_table = { version = "0.2.1", path = "../table" }
-garage_util = { version = "0.2.1", path = "../util" }
-garage_web = { version = "0.2.1", path = "../web" }
+garage_api = { version = "0.3.0", path = "../api" }
+garage_model = { version = "0.3.0", path = "../model" }
+garage_rpc = { version = "0.3.0", path = "../rpc" }
+garage_table = { version = "0.3.0", path = "../table" }
+garage_util = { version = "0.3.0", path = "../util" }
+garage_web = { version = "0.3.0", path = "../web" }
 
 bytes = "1.0"
 git-version = "0.3.4"

src/garage/admin_rpc.rs

@@ -6,7 +6,7 @@ use serde::{Deserialize, Serialize};
 
 use garage_util::error::Error;
 
-use garage_table::crdt::CRDT;
+use garage_table::crdt::Crdt;
 use garage_table::replication::*;
 use garage_table::*;
 
@@ -25,7 +25,7 @@ pub const ADMIN_RPC_TIMEOUT: Duration = Duration::from_secs(30);
 pub const ADMIN_RPC_PATH: &str = "_admin";
 
 #[derive(Debug, Serialize, Deserialize)]
-pub enum AdminRPC {
+pub enum AdminRpc {
 	BucketOperation(BucketOperation),
 	KeyOperation(KeyOperation),
 	LaunchRepair(RepairOpt),
@@ -39,35 +39,35 @@ pub enum AdminRPC {
 	KeyInfo(Key),
 }
 
-impl RpcMessage for AdminRPC {}
+impl RpcMessage for AdminRpc {}
 
 pub struct AdminRpcHandler {
 	garage: Arc<Garage>,
-	rpc_client: Arc<RpcClient<AdminRPC>>,
+	rpc_client: Arc<RpcClient<AdminRpc>>,
 }
 
 impl AdminRpcHandler {
 	pub fn new(garage: Arc<Garage>) -> Arc<Self> {
-		let rpc_client = garage.system.clone().rpc_client::<AdminRPC>(ADMIN_RPC_PATH);
+		let rpc_client = garage.system.clone().rpc_client::<AdminRpc>(ADMIN_RPC_PATH);
 		Arc::new(Self { garage, rpc_client })
 	}
 
 	pub fn register_handler(self: Arc<Self>, rpc_server: &mut RpcServer) {
-		rpc_server.add_handler::<AdminRPC, _, _>(ADMIN_RPC_PATH.to_string(), move |msg, _addr| {
+		rpc_server.add_handler::<AdminRpc, _, _>(ADMIN_RPC_PATH.to_string(), move |msg, _addr| {
 			let self2 = self.clone();
 			async move {
 				match msg {
-					AdminRPC::BucketOperation(bo) => self2.handle_bucket_cmd(bo).await,
-					AdminRPC::KeyOperation(ko) => self2.handle_key_cmd(ko).await,
-					AdminRPC::LaunchRepair(opt) => self2.handle_launch_repair(opt).await,
-					AdminRPC::Stats(opt) => self2.handle_stats(opt).await,
-					_ => Err(Error::BadRPC(format!("Invalid RPC"))),
+					AdminRpc::BucketOperation(bo) => self2.handle_bucket_cmd(bo).await,
+					AdminRpc::KeyOperation(ko) => self2.handle_key_cmd(ko).await,
+					AdminRpc::LaunchRepair(opt) => self2.handle_launch_repair(opt).await,
+					AdminRpc::Stats(opt) => self2.handle_stats(opt).await,
+					_ => Err(Error::BadRpc("Invalid RPC".to_string())),
 				}
 			}
 		});
 	}
 
-	async fn handle_bucket_cmd(&self, cmd: BucketOperation) -> Result<AdminRPC, Error> {
+	async fn handle_bucket_cmd(&self, cmd: BucketOperation) -> Result<AdminRpc, Error> {
 		match cmd {
 			BucketOperation::List => {
 				let bucket_names = self
@@ -78,17 +78,17 @@ impl AdminRpcHandler {
 					.iter()
 					.map(|b| b.name.to_string())
 					.collect::<Vec<_>>();
-				Ok(AdminRPC::BucketList(bucket_names))
+				Ok(AdminRpc::BucketList(bucket_names))
 			}
 			BucketOperation::Info(query) => {
 				let bucket = self.get_existing_bucket(&query.name).await?;
-				Ok(AdminRPC::BucketInfo(bucket))
+				Ok(AdminRpc::BucketInfo(bucket))
 			}
 			BucketOperation::Create(query) => {
 				let bucket = match self.garage.bucket_table.get(&EmptyKey, &query.name).await? {
 					Some(mut bucket) => {
 						if !bucket.is_deleted() {
-							return Err(Error::BadRPC(format!(
+							return Err(Error::BadRpc(format!(
 								"Bucket {} already exists",
 								query.name
 							)));
@@ -101,7 +101,7 @@ impl AdminRpcHandler {
 					None => Bucket::new(query.name.clone()),
 				};
 				self.garage.bucket_table.insert(&bucket).await?;
-				Ok(AdminRPC::Ok(format!("Bucket {} was created.", query.name)))
+				Ok(AdminRpc::Ok(format!("Bucket {} was created.", query.name)))
 			}
 			BucketOperation::Delete(query) => {
 				let mut bucket = self.get_existing_bucket(&query.name).await?;
@@ -111,12 +111,12 @@ impl AdminRpcHandler {
 					.get_range(&query.name, None, Some(DeletedFilter::NotDeleted), 10)
 					.await?;
 				if !objects.is_empty() {
-					return Err(Error::BadRPC(format!("Bucket {} is not empty", query.name)));
+					return Err(Error::BadRpc(format!("Bucket {} is not empty", query.name)));
 				}
 				if !query.yes {
-					return Err(Error::BadRPC(format!(
-						"Add --yes flag to really perform this operation"
-					)));
+					return Err(Error::BadRpc(
+						"Add --yes flag to really perform this operation".to_string(),
+					));
 				}
 				// --- done checking, now commit ---
 				for (key_id, _, _) in bucket.authorized_keys() {
@@ -131,7 +131,7 @@ impl AdminRpcHandler {
 				}
 				bucket.state.update(BucketState::Deleted);
 				self.garage.bucket_table.insert(&bucket).await?;
-				Ok(AdminRPC::Ok(format!("Bucket {} was deleted.", query.name)))
+				Ok(AdminRpc::Ok(format!("Bucket {} was deleted.", query.name)))
 			}
 			BucketOperation::Allow(query) => {
 				let key = self.get_existing_key(&query.key_pattern).await?;
@@ -142,7 +142,7 @@ impl AdminRpcHandler {
 					.await?;
 				self.update_bucket_key(bucket, &key.key_id, allow_read, allow_write)
 					.await?;
-				Ok(AdminRPC::Ok(format!(
+				Ok(AdminRpc::Ok(format!(
 					"New permissions for {} on {}: read {}, write {}.",
 					&key.key_id, &query.bucket, allow_read, allow_write
 				)))
@@ -156,7 +156,7 @@ impl AdminRpcHandler {
 					.await?;
 				self.update_bucket_key(bucket, &key.key_id, allow_read, allow_write)
 					.await?;
-				Ok(AdminRPC::Ok(format!(
+				Ok(AdminRpc::Ok(format!(
 					"New permissions for {} on {}: read {}, write {}.",
 					&key.key_id, &query.bucket, allow_read, allow_write
 				)))
@@ -165,9 +165,9 @@ impl AdminRpcHandler {
 				let mut bucket = self.get_existing_bucket(&query.bucket).await?;
 
 				if !(query.allow ^ query.deny) {
-					return Err(Error::Message(format!(
-						"You must specify exactly one flag, either --allow or --deny"
-					)));
+					return Err(Error::Message(
+						"You must specify exactly one flag, either --allow or --deny".to_string(),
+					));
 				}
 
 				if let BucketState::Present(state) = bucket.state.get_mut() {
@@ -179,7 +179,7 @@ impl AdminRpcHandler {
 						format!("Website access denied for {}", &query.bucket)
 					};
 
-					Ok(AdminRPC::Ok(msg.to_string()))
+					Ok(AdminRpc::Ok(msg))
 				} else {
 					unreachable!();
 				}
@@ -187,7 +187,7 @@ impl AdminRpcHandler {
 		}
 	}
 
-	async fn handle_key_cmd(&self, cmd: KeyOperation) -> Result<AdminRPC, Error> {
+	async fn handle_key_cmd(&self, cmd: KeyOperation) -> Result<AdminRpc, Error> {
 		match cmd {
 			KeyOperation::List => {
 				let key_ids = self
@@ -203,29 +203,29 @@ impl AdminRpcHandler {
 					.iter()
 					.map(|k| (k.key_id.to_string(), k.name.get().clone()))
 					.collect::<Vec<_>>();
-				Ok(AdminRPC::KeyList(key_ids))
+				Ok(AdminRpc::KeyList(key_ids))
 			}
 			KeyOperation::Info(query) => {
 				let key = self.get_existing_key(&query.key_pattern).await?;
-				Ok(AdminRPC::KeyInfo(key))
+				Ok(AdminRpc::KeyInfo(key))
 			}
 			KeyOperation::New(query) => {
 				let key = Key::new(query.name);
 				self.garage.key_table.insert(&key).await?;
-				Ok(AdminRPC::KeyInfo(key))
+				Ok(AdminRpc::KeyInfo(key))
 			}
 			KeyOperation::Rename(query) => {
 				let mut key = self.get_existing_key(&query.key_pattern).await?;
 				key.name.update(query.new_name);
 				self.garage.key_table.insert(&key).await?;
-				Ok(AdminRPC::KeyInfo(key))
+				Ok(AdminRpc::KeyInfo(key))
 			}
 			KeyOperation::Delete(query) => {
 				let key = self.get_existing_key(&query.key_pattern).await?;
 				if !query.yes {
-					return Err(Error::BadRPC(format!(
-						"Add --yes flag to really perform this operation"
-					)));
+					return Err(Error::BadRpc(
+						"Add --yes flag to really perform this operation".to_string(),
+					));
 				}
 				// --- done checking, now commit ---
 				for (ab_name, _, _) in key.authorized_buckets.items().iter() {
@@ -240,25 +240,24 @@ impl AdminRpcHandler {
 				}
 				let del_key = Key::delete(key.key_id.to_string());
 				self.garage.key_table.insert(&del_key).await?;
-				Ok(AdminRPC::Ok(format!(
+				Ok(AdminRpc::Ok(format!(
 					"Key {} was deleted successfully.",
 					key.key_id
 				)))
 			}
 			KeyOperation::Import(query) => {
-				let prev_key = self.garage.key_table.get(&EmptyKey, &query.key_id)
-					.await?;
+				let prev_key = self.garage.key_table.get(&EmptyKey, &query.key_id).await?;
 				if prev_key.is_some() {
 					return Err(Error::Message(format!("Key {} already exists in data store. Even if it is deleted, we can't let you create a new key with the same ID. Sorry.", query.key_id)));
 				}
 				let imported_key = Key::import(&query.key_id, &query.secret_key, &query.name);
 				self.garage.key_table.insert(&imported_key).await?;
-				Ok(AdminRPC::KeyInfo(imported_key))
-
+				Ok(AdminRpc::KeyInfo(imported_key))
 			}
 		}
 	}
 
+	#[allow(clippy::ptr_arg)]
 	async fn get_existing_bucket(&self, bucket: &String) -> Result<Bucket, Error> {
 		self.garage
 			.bucket_table
@@ -266,10 +265,7 @@ impl AdminRpcHandler {
 			.await?
 			.filter(|b| !b.is_deleted())
 			.map(Ok)
-			.unwrap_or(Err(Error::BadRPC(format!(
-				"Bucket {} does not exist",
-				bucket
-			))))
+			.unwrap_or_else(|| Err(Error::BadRpc(format!("Bucket {} does not exist", bucket))))
 	}
 
 	async fn get_existing_key(&self, pattern: &str) -> Result<Key, Error> {
@@ -300,7 +296,7 @@ impl AdminRpcHandler {
 	async fn update_bucket_key(
 		&self,
 		mut bucket: Bucket,
-		key_id: &String,
+		key_id: &str,
 		allow_read: bool,
 		allow_write: bool,
 	) -> Result<(), Error> {
@@ -315,9 +311,9 @@ impl AdminRpcHandler {
 				},
 			));
 		} else {
-			return Err(Error::Message(format!(
-				"Bucket is deleted in update_bucket_key"
-			)));
+			return Err(Error::Message(
+				"Bucket is deleted in update_bucket_key".to_string(),
+			));
 		}
 		self.garage.bucket_table.insert(&bucket).await?;
 		Ok(())
@@ -327,14 +323,14 @@ impl AdminRpcHandler {
 	async fn update_key_bucket(
 		&self,
 		key: &Key,
-		bucket: &String,
+		bucket: &str,
 		allow_read: bool,
 		allow_write: bool,
 	) -> Result<(), Error> {
 		let mut key = key.clone();
 		let old_map = key.authorized_buckets.take_and_clear();
 		key.authorized_buckets.merge(&old_map.update_mutator(
-			bucket.clone(),
+			bucket.to_string(),
 			PermissionSet {
 				allow_read,
 				allow_write,
@@ -344,11 +340,11 @@ impl AdminRpcHandler {
 		Ok(())
 	}
 
-	async fn handle_launch_repair(self: &Arc<Self>, opt: RepairOpt) -> Result<AdminRPC, Error> {
+	async fn handle_launch_repair(self: &Arc<Self>, opt: RepairOpt) -> Result<AdminRpc, Error> {
 		if !opt.yes {
-			return Err(Error::BadRPC(format!(
-				"Please provide the --yes flag to initiate repair operations."
-			)));
+			return Err(Error::BadRpc(
+				"Please provide the --yes flag to initiate repair operations.".to_string(),
+			));
 		}
 		if opt.all_nodes {
 			let mut opt_to_send = opt.clone();
@@ -361,17 +357,17 @@ impl AdminRpcHandler {
 					.rpc_client
 					.call(
 						*node,
-						AdminRPC::LaunchRepair(opt_to_send.clone()),
+						AdminRpc::LaunchRepair(opt_to_send.clone()),
 						ADMIN_RPC_TIMEOUT,
 					)
 					.await
 					.is_err()
 				{
-					failures.push(node.clone());
+					failures.push(*node);
 				}
 			}
 			if failures.is_empty() {
-				Ok(AdminRPC::Ok(format!("Repair launched on all nodes")))
+				Ok(AdminRpc::Ok("Repair launched on all nodes".to_string()))
 			} else {
 				Err(Error::Message(format!(
 					"Could not launch repair on nodes: {:?} (launched successfully on other nodes)",
@@ -388,14 +384,14 @@ impl AdminRpcHandler {
 				.spawn_worker("Repair worker".into(), move |must_exit| async move {
 					repair.repair_worker(opt, must_exit).await
 				});
-			Ok(AdminRPC::Ok(format!(
+			Ok(AdminRpc::Ok(format!(
 				"Repair launched on {:?}",
 				self.garage.system.id
 			)))
 		}
 	}
 
-	async fn handle_stats(&self, opt: StatsOpt) -> Result<AdminRPC, Error> {
+	async fn handle_stats(&self, opt: StatsOpt) -> Result<AdminRpc, Error> {
 		if opt.all_nodes {
 			let mut ret = String::new();
 			let ring = self.garage.system.ring.borrow().clone();
@@ -408,21 +404,21 @@ impl AdminRpcHandler {
 				writeln!(&mut ret, "Stats for node {:?}:", node).unwrap();
 				match self
 					.rpc_client
-					.call(*node, AdminRPC::Stats(opt), ADMIN_RPC_TIMEOUT)
+					.call(*node, AdminRpc::Stats(opt), ADMIN_RPC_TIMEOUT)
 					.await
 				{
-					Ok(AdminRPC::Ok(s)) => writeln!(&mut ret, "{}", s).unwrap(),
+					Ok(AdminRpc::Ok(s)) => writeln!(&mut ret, "{}", s).unwrap(),
 					Ok(x) => writeln!(&mut ret, "Bad answer: {:?}", x).unwrap(),
 					Err(e) => writeln!(&mut ret, "Error: {}", e).unwrap(),
 				}
 			}
-			Ok(AdminRPC::Ok(ret))
+			Ok(AdminRpc::Ok(ret))
 		} else {
-			Ok(AdminRPC::Ok(self.gather_stats_local(opt)?))
+			Ok(AdminRpc::Ok(self.gather_stats_local(opt)))
 		}
 	}
 
-	fn gather_stats_local(&self, opt: StatsOpt) -> Result<String, Error> {
+	fn gather_stats_local(&self, opt: StatsOpt) -> String {
 		let mut ret = String::new();
 		writeln!(
 			&mut ret,
@@ -434,8 +430,8 @@ impl AdminRpcHandler {
 		// Gather ring statistics
 		let ring = self.garage.system.ring.borrow().clone();
 		let mut ring_nodes = HashMap::new();
-		for r in ring.ring.iter() {
-			for n in r.nodes.iter() {
+		for (_i, loc) in ring.partitions().iter() {
+			for n in ring.get_nodes(loc, ring.replication_factor).iter() {
 				if !ring_nodes.contains_key(n) {
 					ring_nodes.insert(*n, 0usize);
 				}
@@ -447,11 +443,11 @@ impl AdminRpcHandler {
 			writeln!(&mut ret, "  {:?} {}", n, c).unwrap();
 		}
 
-		self.gather_table_stats(&mut ret, &self.garage.bucket_table, &opt)?;
-		self.gather_table_stats(&mut ret, &self.garage.key_table, &opt)?;
-		self.gather_table_stats(&mut ret, &self.garage.object_table, &opt)?;
-		self.gather_table_stats(&mut ret, &self.garage.version_table, &opt)?;
-		self.gather_table_stats(&mut ret, &self.garage.block_ref_table, &opt)?;
+		self.gather_table_stats(&mut ret, &self.garage.bucket_table, &opt);
+		self.gather_table_stats(&mut ret, &self.garage.key_table, &opt);
+		self.gather_table_stats(&mut ret, &self.garage.object_table, &opt);
+		self.gather_table_stats(&mut ret, &self.garage.version_table, &opt);
+		self.gather_table_stats(&mut ret, &self.garage.block_ref_table, &opt);
 
 		writeln!(&mut ret, "\nBlock manager stats:").unwrap();
 		if opt.detailed {
@@ -469,15 +465,10 @@ impl AdminRpcHandler {
 		)
 		.unwrap();
 
-		Ok(ret)
+		ret
 	}
 
-	fn gather_table_stats<F, R>(
-		&self,
-		to: &mut String,
-		t: &Arc<Table<F, R>>,
-		opt: &StatsOpt,
-	) -> Result<(), Error>
+	fn gather_table_stats<F, R>(&self, to: &mut String, t: &Arc<Table<F, R>>, opt: &StatsOpt)
 	where
 		F: TableSchema + 'static,
 		R: TableReplication + 'static,
@@ -499,6 +490,5 @@ impl AdminRpcHandler {
 		)
 		.unwrap();
 		writeln!(to, "  GC todo queue length: {}", t.data.gc_todo_len()).unwrap();
-		Ok(())
 	}
 }

src/garage/cli.rs

@@ -5,8 +5,8 @@ use std::path::PathBuf;
 use serde::{Deserialize, Serialize};
 use structopt::StructOpt;
 
+use garage_util::data::Uuid;
 use garage_util::error::Error;
-use garage_util::data::UUID;
 use garage_util::time::*;
 
 use garage_rpc::membership::*;
@@ -72,15 +72,19 @@ pub struct ConfigureNodeOpt {
 	/// Node to configure (prefix of hexadecimal node id)
 	node_id: String,
 
-	/// Location (datacenter) of the node
-	#[structopt(short = "d", long = "datacenter")]
-	datacenter: Option<String>,
+	/// Location (zone or datacenter) of the node
+	#[structopt(short = "z", long = "zone")]
+	zone: Option<String>,
 
 	/// Capacity (in relative terms, use 1 to represent your smallest server)
 	#[structopt(short = "c", long = "capacity")]
 	capacity: Option<u32>,
 
-	/// Optionnal node tag
+	/// Gateway-only node
+	#[structopt(short = "g", long = "gateway")]
+	gateway: bool,
+
+	/// Optional node tag
 	#[structopt(short = "t", long = "tag")]
 	tag: Option<String>,
 
@@ -294,7 +298,7 @@ pub struct StatsOpt {
 pub async fn cli_cmd(
 	cmd: Command,
 	membership_rpc_cli: RpcAddrClient<Message>,
-	admin_rpc_cli: RpcAddrClient<AdminRPC>,
+	admin_rpc_cli: RpcAddrClient<AdminRpc>,
 	rpc_host: SocketAddr,
 ) -> Result<(), Error> {
 	match cmd {
@@ -306,11 +310,11 @@ pub async fn cli_cmd(
 			cmd_remove(membership_rpc_cli, rpc_host, remove_opt).await
 		}
 		Command::Bucket(bo) => {
-			cmd_admin(admin_rpc_cli, rpc_host, AdminRPC::BucketOperation(bo)).await
+			cmd_admin(admin_rpc_cli, rpc_host, AdminRpc::BucketOperation(bo)).await
 		}
-		Command::Key(ko) => cmd_admin(admin_rpc_cli, rpc_host, AdminRPC::KeyOperation(ko)).await,
-		Command::Repair(ro) => cmd_admin(admin_rpc_cli, rpc_host, AdminRPC::LaunchRepair(ro)).await,
-		Command::Stats(so) => cmd_admin(admin_rpc_cli, rpc_host, AdminRPC::Stats(so)).await,
+		Command::Key(ko) => cmd_admin(admin_rpc_cli, rpc_host, AdminRpc::KeyOperation(ko)).await,
+		Command::Repair(ro) => cmd_admin(admin_rpc_cli, rpc_host, AdminRpc::LaunchRepair(ro)).await,
+		Command::Stats(so) => cmd_admin(admin_rpc_cli, rpc_host, AdminRpc::Stats(so)).await,
 		_ => unreachable!(),
 	}
 }
@@ -339,7 +343,12 @@ pub async fn cmd_status(
 		if let Some(cfg) = config.members.get(&adv.id) {
 			println!(
 				"{:?}\t{}\t{}\t[{}]\t{}\t{}",
-				adv.id, adv.state_info.hostname, adv.addr, cfg.tag, cfg.datacenter, cfg.capacity
+				adv.id,
+				adv.state_info.hostname,
+				adv.addr,
+				cfg.tag,
+				cfg.zone,
+				cfg.capacity_string()
 			);
 		} else {
 			println!(
@@ -365,8 +374,8 @@ pub async fn cmd_status(
 					adv.state_info.hostname,
 					adv.addr,
 					cfg.tag,
-					cfg.datacenter,
-					cfg.capacity,
+					cfg.zone,
+					cfg.capacity_string(),
 					(now_msec() - adv.last_seen) / 1000,
 				);
 			}
@@ -375,7 +384,10 @@ pub async fn cmd_status(
 			if !status.iter().any(|x| x.id == *id) {
 				println!(
 					"{:?}\t{}\t{}\t{}\tnever seen",
-					id, cfg.tag, cfg.datacenter, cfg.capacity
+					id,
+					cfg.tag,
+					cfg.zone,
+					cfg.capacity_string(),
 				);
 			}
 		}
@@ -384,7 +396,10 @@ pub async fn cmd_status(
 	Ok(())
 }
 
-pub fn find_matching_node(cand: impl std::iter::Iterator<Item=UUID>, pattern: &str) -> Result<UUID, Error> {
+pub fn find_matching_node(
+	cand: impl std::iter::Iterator<Item = Uuid>,
+	pattern: &str,
+) -> Result<Uuid, Error> {
 	let mut candidates = vec![];
 	for c in cand {
 		if hex::encode(&c).starts_with(&pattern) {
@@ -428,25 +443,47 @@ pub async fn cmd_configure(
 	for replaced in args.replace.iter() {
 		let replaced_node = find_matching_node(config.members.keys().cloned(), replaced)?;
 		if config.members.remove(&replaced_node).is_none() {
-			return Err(Error::Message(format!("Cannot replace node {:?} as it is not in current configuration", replaced_node)));
+			return Err(Error::Message(format!(
+				"Cannot replace node {:?} as it is not in current configuration",
+				replaced_node
+			)));
 		}
 	}
 
+	if args.capacity.is_some() && args.gateway {
+		return Err(Error::Message(
+				"-c and -g are mutually exclusive, please configure node either with c>0 to act as a storage node or with -g to act as a gateway node".into()));
+	}
+	if args.capacity == Some(0) {
+		return Err(Error::Message("Invalid capacity value: 0".into()));
+	}
+
 	let new_entry = match config.members.get(&added_node) {
-		None => NetworkConfigEntry {
-			datacenter: args
-				.datacenter
-				.expect("Please specifiy a datacenter with the -d flag"),
-			capacity: args
-				.capacity
-				.expect("Please specifiy a capacity with the -c flag"),
-			tag: args.tag.unwrap_or("".to_string()),
-		},
-		Some(old) => NetworkConfigEntry {
-			datacenter: args.datacenter.unwrap_or(old.datacenter.to_string()),
-			capacity: args.capacity.unwrap_or(old.capacity),
-			tag: args.tag.unwrap_or(old.tag.to_string()),
-		},
+		None => {
+			let capacity = match args.capacity {
+				Some(c) => Some(c),
+				None if args.gateway => None,
+				_ => return Err(Error::Message(
+						"Please specify a capacity with the -c flag, or set node explicitly as gateway with -g".into())),
+			};
+			NetworkConfigEntry {
+				zone: args.zone.expect("Please specifiy a zone with the -z flag"),
+				capacity,
+				tag: args.tag.unwrap_or_default(),
+			}
+		}
+		Some(old) => {
+			let capacity = match args.capacity {
+				Some(c) => Some(c),
+				None if args.gateway => None,
+				_ => old.capacity,
+			};
+			NetworkConfigEntry {
+				zone: args.zone.unwrap_or_else(|| old.zone.to_string()),
+				capacity,
+				tag: args.tag.unwrap_or_else(|| old.tag.to_string()),
+			}
+		}
 	};
 
 	config.members.insert(added_node, new_entry);
@@ -498,30 +535,30 @@ pub async fn cmd_remove(
 }
 
 pub async fn cmd_admin(
-	rpc_cli: RpcAddrClient<AdminRPC>,
+	rpc_cli: RpcAddrClient<AdminRpc>,
 	rpc_host: SocketAddr,
-	args: AdminRPC,
+	args: AdminRpc,
 ) -> Result<(), Error> {
 	match rpc_cli.call(&rpc_host, args, ADMIN_RPC_TIMEOUT).await?? {
-		AdminRPC::Ok(msg) => {
+		AdminRpc::Ok(msg) => {
 			println!("{}", msg);
 		}
-		AdminRPC::BucketList(bl) => {
+		AdminRpc::BucketList(bl) => {
 			println!("List of buckets:");
 			for bucket in bl {
 				println!("{}", bucket);
 			}
 		}
-		AdminRPC::BucketInfo(bucket) => {
+		AdminRpc::BucketInfo(bucket) => {
 			print_bucket_info(&bucket);
 		}
-		AdminRPC::KeyList(kl) => {
+		AdminRpc::KeyList(kl) => {
 			println!("List of keys:");
 			for key in kl {
 				println!("{}\t{}", key.0, key.1);
 			}
 		}
-		AdminRPC::KeyInfo(key) => {
+		AdminRpc::KeyInfo(key) => {
 			print_key_info(&key);
 		}
 		r => {

src/garage/main.rs

@@ -1,4 +1,5 @@
 #![recursion_limit = "1024"]
+//! Garage CLI, used to interact with a running Garage instance, and to launch a Garage instance
 
 #[macro_use]
 extern crate log;
@@ -25,7 +26,7 @@ use cli::*;
 
 #[derive(StructOpt, Debug)]
 #[structopt(name = "garage")]
-pub struct Opt {
+struct Opt {
 	/// RPC connect to this host to execute client operations
 	#[structopt(short = "h", long = "rpc-host", default_value = "127.0.0.1:3901")]
 	pub rpc_host: SocketAddr,

src/garage/server.rs

@@ -8,10 +8,10 @@ use garage_util::background::*;
 use garage_util::config::*;
 use garage_util::error::Error;
 
-use garage_api::api_server;
+use garage_api::run_api_server;
 use garage_model::garage::Garage;
 use garage_rpc::rpc_server::RpcServer;
-use garage_web::web_server;
+use garage_web::run_web_server;
 
 use crate::admin_rpc::*;
 
@@ -25,7 +25,7 @@ async fn shutdown_signal(send_cancel: watch::Sender<bool>) -> Result<(), Error>
 	Ok(())
 }
 
-async fn wait_from(mut chan: watch::Receiver<bool>) -> () {
+async fn wait_from(mut chan: watch::Receiver<bool>) {
 	while !*chan.borrow() {
 		if chan.changed().await.is_err() {
 			return;
@@ -40,10 +40,15 @@ pub async fn run_server(config_file: PathBuf) -> Result<(), Error> {
 	info!("Opening database...");
 	let mut db_path = config.metadata_dir.clone();
 	db_path.push("db");
-	let db = sled::open(&db_path).expect("Unable to open sled DB");
+	let db = sled::Config::default()
+		.path(&db_path)
+		.cache_capacity(config.sled_cache_capacity)
+		.flush_every_ms(Some(config.sled_flush_every_ms))
+		.open()
+		.expect("Unable to open sled DB");
 
 	info!("Initialize RPC server...");
-	let mut rpc_server = RpcServer::new(config.rpc_bind_addr.clone(), config.rpc_tls.clone());
+	let mut rpc_server = RpcServer::new(config.rpc_bind_addr, config.rpc_tls.clone());
 
 	info!("Initializing background runner...");
 	let (send_cancel, watch_cancel) = watch::channel(false);
@@ -52,7 +57,7 @@ pub async fn run_server(config_file: PathBuf) -> Result<(), Error> {
 	info!("Initializing Garage main data store...");
 	let garage = Garage::new(config.clone(), db, background, &mut rpc_server);
 	let bootstrap = garage.system.clone().bootstrap(
-		&config.bootstrap_peers[..],
+		config.bootstrap_peers,
 		config.consul_host,
 		config.consul_service_name,
 	);
@@ -62,13 +67,13 @@ pub async fn run_server(config_file: PathBuf) -> Result<(), Error> {
 
 	info!("Initializing RPC and API servers...");
 	let run_rpc_server = Arc::new(rpc_server).run(wait_from(watch_cancel.clone()));
-	let api_server = api_server::run_api_server(garage.clone(), wait_from(watch_cancel.clone()));
-	let web_server = web_server::run_web_server(garage, wait_from(watch_cancel.clone()));
+	let api_server = run_api_server(garage.clone(), wait_from(watch_cancel.clone()));
+	let web_server = run_web_server(garage, wait_from(watch_cancel.clone()));
 
 	futures::try_join!(
-		bootstrap.map(|rv| {
+		bootstrap.map(|()| {
 			info!("Bootstrap done");
-			Ok(rv)
+			Ok(())
 		}),
 		run_rpc_server.map(|rv| {
 			info!("RPC server exited");

src/model/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "garage_model"
-version = "0.2.1"
+version = "0.3.0"
 authors = ["Alex Auvolat <alex@adnab.me>"]
 edition = "2018"
 license = "AGPL-3.0"
@@ -13,9 +13,9 @@ path = "lib.rs"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-garage_rpc = { version = "0.2.1", path = "../rpc" }
-garage_table = { version = "0.2.1", path = "../table" }
-garage_util = { version = "0.2.1", path = "../util" }
+garage_rpc = { version = "0.3.0", path = "../rpc" }
+garage_table = { version = "0.3.0", path = "../table" }
+garage_util = { version = "0.3.0", path = "../util" }
 
 arc-swap = "1.0"
 hex = "0.4"

src/model/block.rs

@@ -1,4 +1,4 @@
-use std::path::PathBuf;
+use std::path::{Path, PathBuf};
 use std::sync::Arc;
 use std::time::Duration;
 
@@ -18,12 +18,13 @@ use garage_rpc::membership::System;
 use garage_rpc::rpc_client::*;
 use garage_rpc::rpc_server::*;
 
-use garage_table::replication::{sharded::TableShardedReplication, TableReplication};
+use garage_table::replication::{TableReplication, TableShardedReplication};
 
 use crate::block_ref_table::*;
 
 use crate::garage::Garage;
 
+/// Size under which data will be stored inlined in database instead of as files
 pub const INLINE_THRESHOLD: usize = 3072;
 
 pub const BACKGROUND_WORKERS: u64 = 1;
@@ -33,28 +34,41 @@ const BLOCK_GC_TIMEOUT: Duration = Duration::from_secs(60);
 const NEED_BLOCK_QUERY_TIMEOUT: Duration = Duration::from_secs(5);
 const RESYNC_RETRY_TIMEOUT: Duration = Duration::from_secs(10);
 
+/// RPC messages used to share blocks of data between nodes
 #[derive(Debug, Serialize, Deserialize)]
 pub enum Message {
 	Ok,
+	/// Message to ask for a block of data, by hash
 	GetBlock(Hash),
+	/// Message to send a block of data, either because requested, of for first delivery of new
+	/// block
 	PutBlock(PutBlockMessage),
+	/// Ask other node if they should have this block, but don't actually have it
 	NeedBlockQuery(Hash),
+	/// Response : whether the node do require that block
 	NeedBlockReply(bool),
 }
 
+/// Structure used to send a block
 #[derive(Debug, Serialize, Deserialize)]
 pub struct PutBlockMessage {
+	/// Hash of the block
 	pub hash: Hash,
 
+	/// Content of the block
 	#[serde(with = "serde_bytes")]
 	pub data: Vec<u8>,
 }
 
 impl RpcMessage for Message {}
 
+/// The block manager, handling block exchange between nodes, and block storage on local node
 pub struct BlockManager {
+	/// Replication strategy, allowing to find on which node blocks should be located
 	pub replication: TableShardedReplication,
+	/// Directory in which block are stored
 	pub data_dir: PathBuf,
+	/// Lock to prevent concurrent edition of the directory
 	pub data_dir_lock: Mutex<()>,
 
 	rc: sled::Tree,
@@ -123,12 +137,13 @@ impl BlockManager {
 			Message::PutBlock(m) => self.write_block(&m.hash, &m.data).await,
 			Message::GetBlock(h) => self.read_block(h).await,
 			Message::NeedBlockQuery(h) => self.need_block(h).await.map(Message::NeedBlockReply),
-			_ => Err(Error::BadRPC(format!("Unexpected RPC message"))),
+			_ => Err(Error::BadRpc("Unexpected RPC message".to_string())),
 		}
 	}
 
 	pub fn spawn_background_worker(self: Arc<Self>) {
-		// Launch 2 simultaneous workers for background resync loop preprocessing
+		// Launch 2 simultaneous workers for background resync loop preprocessing <= TODO actually this
+		// launches only one worker with current value of BACKGROUND_WORKERS
 		for i in 0..BACKGROUND_WORKERS {
 			let bm2 = self.clone();
 			let background = self.system.background.clone();
@@ -141,7 +156,8 @@ impl BlockManager {
 		}
 	}
 
-	pub async fn write_block(&self, hash: &Hash, data: &[u8]) -> Result<Message, Error> {
+	/// Write a block to disk
+	async fn write_block(&self, hash: &Hash, data: &[u8]) -> Result<Message, Error> {
 		let _lock = self.data_dir_lock.lock().await;
 
 		let mut path = self.block_dir(hash);
@@ -159,7 +175,8 @@ impl BlockManager {
 		Ok(Message::Ok)
 	}
 
-	pub async fn read_block(&self, hash: &Hash) -> Result<Message, Error> {
+	/// Read block from disk, verifying it's integrity
+	async fn read_block(&self, hash: &Hash) -> Result<Message, Error> {
 		let path = self.block_path(hash);
 
 		let mut f = match fs::File::open(&path).await {
@@ -190,7 +207,8 @@ impl BlockManager {
 		Ok(Message::PutBlock(PutBlockMessage { hash: *hash, data }))
 	}
 
-	pub async fn need_block(&self, hash: &Hash) -> Result<bool, Error> {
+	/// Check if this node should have a block, but don't actually have it
+	async fn need_block(&self, hash: &Hash) -> Result<bool, Error> {
 		let needed = self
 			.rc
 			.get(hash.as_ref())?
@@ -217,6 +235,8 @@ impl BlockManager {
 		path
 	}
 
+	/// Increment the number of time a block is used, putting it to resynchronization if it is
+	/// required, but not known
 	pub fn block_incref(&self, hash: &Hash) -> Result<(), Error> {
 		let old_rc = self.rc.fetch_and_update(&hash, |old| {
 			let old_v = old.map(u64_from_be_bytes).unwrap_or(0);
@@ -229,6 +249,7 @@ impl BlockManager {
 		Ok(())
 	}
 
+	/// Decrement the number of time a block is used
 	pub fn block_decref(&self, hash: &Hash) -> Result<(), Error> {
 		let new_rc = self.rc.update_and_fetch(&hash, |old| {
 			let old_v = old.map(u64_from_be_bytes).unwrap_or(0);
@@ -259,8 +280,8 @@ impl BlockManager {
 			if let Err(e) = self.resync_iter(&mut must_exit).await {
 				warn!("Error in block resync loop: {}", e);
 				select! {
-					_ = tokio::time::sleep(Duration::from_secs(1)).fuse() => (),
-					_ = must_exit.changed().fuse() => (),
+					_ = tokio::time::sleep(Duration::from_secs(1)).fuse() => {},
+					_ = must_exit.changed().fuse() => {},
 				}
 			}
 		}
@@ -283,15 +304,15 @@ impl BlockManager {
 			} else {
 				let delay = tokio::time::sleep(Duration::from_millis(time_msec - now));
 				select! {
-					_ = delay.fuse() => (),
-					_ = self.resync_notify.notified().fuse() => (),
-					_ = must_exit.changed().fuse() => (),
+					_ = delay.fuse() => {},
+					_ = self.resync_notify.notified().fuse() => {},
+					_ = must_exit.changed().fuse() => {},
 				}
 			}
 		} else {
 			select! {
-				_ = self.resync_notify.notified().fuse() => (),
-				_ = must_exit.changed().fuse() => (),
+				_ = self.resync_notify.notified().fuse() => {},
+				_ = must_exit.changed().fuse() => {},
 			}
 		}
 		Ok(())
@@ -321,7 +342,7 @@ impl BlockManager {
 
 			let mut who = self.replication.write_nodes(&hash);
 			if who.len() < self.replication.write_quorum() {
-				return Err(Error::Message(format!("Not trying to offload block because we don't have a quorum of nodes to write to")));
+				return Err(Error::Message("Not trying to offload block because we don't have a quorum of nodes to write to".to_string()));
 			}
 			who.retain(|id| *id != self.system.id);
 
@@ -341,14 +362,14 @@ impl BlockManager {
 						}
 					}
 					_ => {
-						return Err(Error::Message(format!(
-							"Unexpected response to NeedBlockQuery RPC"
-						)));
+						return Err(Error::Message(
+							"Unexpected response to NeedBlockQuery RPC".to_string(),
+						));
 					}
 				}
 			}
 
-			if need_nodes.len() > 0 {
+			if !need_nodes.is_empty() {
 				trace!(
 					"Block {:?} needed by {} nodes, sending",
 					hash,
@@ -388,6 +409,7 @@ impl BlockManager {
 		Ok(())
 	}
 
+	/// Ask nodes that might have a block for it
 	pub async fn rpc_get_block(&self, hash: &Hash) -> Result<Vec<u8>, Error> {
 		let who = self.replication.read_nodes(&hash);
 		let resps = self
@@ -412,6 +434,7 @@ impl BlockManager {
 		)))
 	}
 
+	/// Send block to nodes that should have it
 	pub async fn rpc_put_block(&self, hash: Hash, data: Vec<u8>) -> Result<(), Error> {
 		let who = self.replication.write_nodes(&hash);
 		self.rpc_client
@@ -455,7 +478,7 @@ impl BlockManager {
 
 	fn repair_aux_read_dir_rec<'a>(
 		&'a self,
-		path: &'a PathBuf,
+		path: &'a Path,
 		must_exit: &'a watch::Receiver<bool>,
 	) -> BoxFuture<'a, Result<(), Error>> {
 		// Lists all blocks on disk and adds them to the resync queue.
@@ -498,6 +521,7 @@ impl BlockManager {
 		.boxed()
 	}
 
+	/// Get lenght of resync queue
 	pub fn resync_queue_len(&self) -> usize {
 		self.resync_queue.len()
 	}

src/model/block_ref_table.rs

@@ -3,28 +3,29 @@ use std::sync::Arc;
 
 use garage_util::data::*;
 
-use garage_table::crdt::CRDT;
+use garage_table::crdt::Crdt;
 use garage_table::*;
 
 use crate::block::*;
 
 #[derive(PartialEq, Clone, Debug, Serialize, Deserialize)]
 pub struct BlockRef {
-	// Primary key
+	/// Hash (blake2 sum) of the block, used as partition key
 	pub block: Hash,
 
-	// Sort key
-	pub version: UUID,
+	/// Id of the Version for the object containing this block, used as sorting key
+	pub version: Uuid,
 
 	// Keep track of deleted status
+	/// Is the Version that contains this block deleted
 	pub deleted: crdt::Bool,
 }
 
-impl Entry<Hash, UUID> for BlockRef {
+impl Entry<Hash, Uuid> for BlockRef {
 	fn partition_key(&self) -> &Hash {
 		&self.block
 	}
-	fn sort_key(&self) -> &UUID {
+	fn sort_key(&self) -> &Uuid {
 		&self.version
 	}
 	fn is_tombstone(&self) -> bool {
@@ -32,7 +33,7 @@ impl Entry<Hash, UUID> for BlockRef {
 	}
 }
 
-impl CRDT for BlockRef {
+impl Crdt for BlockRef {
 	fn merge(&mut self, other: &Self) {
 		self.deleted.merge(&other.deleted);
 	}
@@ -44,12 +45,12 @@ pub struct BlockRefTable {
 
 impl TableSchema for BlockRefTable {
 	type P = Hash;
-	type S = UUID;
+	type S = Uuid;
 	type E = BlockRef;
 	type Filter = DeletedFilter;
 
 	fn updated(&self, old: Option<Self::E>, new: Option<Self::E>) {
-		let block = &old.as_ref().or(new.as_ref()).unwrap().block;
+		let block = &old.as_ref().or_else(|| new.as_ref()).unwrap().block;
 		let was_before = old.as_ref().map(|x| !x.deleted.get()).unwrap_or(false);
 		let is_after = new.as_ref().map(|x| !x.deleted.get()).unwrap_or(false);
 		if is_after && !was_before {

src/model/bucket_table.rs

@@ -1,6 +1,6 @@
 use serde::{Deserialize, Serialize};
 
-use garage_table::crdt::CRDT;
+use garage_table::crdt::Crdt;
 use garage_table::*;
 
 use crate::key_table::PermissionSet;
@@ -12,19 +12,22 @@ use crate::key_table::PermissionSet;
 ///  - A bucket has 2 states, Present or Deleted and parameters make sense only if present.
 #[derive(PartialEq, Clone, Debug, Serialize, Deserialize)]
 pub struct Bucket {
-	// Primary key
+	/// Name of the bucket
 	pub name: String,
-
-	pub state: crdt::LWW<BucketState>,
+	/// State, and configuration if not deleted, of the bucket
+	pub state: crdt::Lww<BucketState>,
 }
 
+/// State of a bucket
 #[derive(PartialEq, Clone, Debug, Serialize, Deserialize)]
 pub enum BucketState {
+	/// The bucket is deleted
 	Deleted,
+	/// The bucket exists
 	Present(BucketParams),
 }
 
-impl CRDT for BucketState {
+impl Crdt for BucketState {
 	fn merge(&mut self, o: &Self) {
 		match o {
 			BucketState::Deleted => *self = BucketState::Deleted,
@@ -37,38 +40,54 @@ impl CRDT for BucketState {
 	}
 }
 
+/// Configuration for a bucket
 #[derive(PartialEq, Clone, Debug, Serialize, Deserialize)]
 pub struct BucketParams {
-	pub authorized_keys: crdt::LWWMap<String, PermissionSet>,
-	pub website: crdt::LWW<bool>,
+	/// Map of key with access to the bucket, and what kind of access they give
+	pub authorized_keys: crdt::LwwMap<String, PermissionSet>,
+	/// Is the bucket served as http
+	pub website: crdt::Lww<bool>,
 }
 
-impl CRDT for BucketParams {
+impl BucketParams {
+	/// Create an empty BucketParams with no authorized keys and no website accesss
+	pub fn new() -> Self {
+		BucketParams {
+			authorized_keys: crdt::LwwMap::new(),
+			website: crdt::Lww::new(false),
+		}
+	}
+}
+
+impl Crdt for BucketParams {
 	fn merge(&mut self, o: &Self) {
 		self.authorized_keys.merge(&o.authorized_keys);
 		self.website.merge(&o.website);
 	}
 }
 
-impl BucketParams {
-	pub fn new() -> Self {
-		BucketParams {
-			authorized_keys: crdt::LWWMap::new(),
-			website: crdt::LWW::new(false),
-		}
+impl Default for BucketParams {
+	fn default() -> Self {
+		Self::new()
 	}
 }
 
 impl Bucket {
+	/// Initializes a new instance of the Bucket struct
 	pub fn new(name: String) -> Self {
 		Bucket {
 			name,
-			state: crdt::LWW::new(BucketState::Present(BucketParams::new())),
+			state: crdt::Lww::new(BucketState::Present(BucketParams::new())),
 		}
 	}
+
+	/// Returns true if this represents a deleted bucket
 	pub fn is_deleted(&self) -> bool {
 		*self.state.get() == BucketState::Deleted
 	}
+
+	/// Return the list of authorized keys, when each was updated, and the permission associated to
+	/// the key
 	pub fn authorized_keys(&self) -> &[(String, u64, PermissionSet)] {
 		match self.state.get() {
 			BucketState::Deleted => &[],
@@ -86,7 +105,7 @@ impl Entry<EmptyKey, String> for Bucket {
 	}
 }
 
-impl CRDT for Bucket {
+impl Crdt for Bucket {
 	fn merge(&mut self, other: &Self) {
 		self.state.merge(&other.state);
 	}

src/model/garage.rs

@@ -7,8 +7,9 @@ use garage_rpc::membership::System;
 use garage_rpc::rpc_client::RpcHttpClient;
 use garage_rpc::rpc_server::RpcServer;
 
-use garage_table::replication::fullcopy::*;
-use garage_table::replication::sharded::*;
+use garage_table::replication::ReplicationMode;
+use garage_table::replication::TableFullReplication;
+use garage_table::replication::TableShardedReplication;
 use garage_table::*;
 
 use crate::block::*;
@@ -18,15 +19,23 @@ use crate::key_table::*;
 use crate::object_table::*;
 use crate::version_table::*;
 
+/// An entire Garage full of data
 pub struct Garage {
+	/// The parsed configuration Garage is running
 	pub config: Config,
 
+	/// The local database
 	pub db: sled::Db,
+	/// A background job runner
 	pub background: Arc<BackgroundRunner>,
+	/// The membership manager
 	pub system: Arc<System>,
+	/// The block manager
 	pub block_manager: Arc<BlockManager>,
 
+	/// Table containing informations about buckets
 	pub bucket_table: Arc<Table<BucketTable, TableFullReplication>>,
+	/// Table containing informations about api keys
 	pub key_table: Arc<Table<KeyTable, TableFullReplication>>,
 
 	pub object_table: Arc<Table<ObjectTable, TableShardedReplication>>,
@@ -35,12 +44,16 @@ pub struct Garage {
 }
 
 impl Garage {
+	/// Create and run garage
 	pub fn new(
 		config: Config,
 		db: sled::Db,
 		background: Arc<BackgroundRunner>,
 		rpc_server: &mut RpcServer,
 	) -> Arc<Self> {
+		let replication_mode = ReplicationMode::parse(&config.replication_mode)
+			.expect("Invalid replication_mode in config file.");
+
 		info!("Initialize membership management system...");
 		let rpc_http_client = Arc::new(
 			RpcHttpClient::new(config.max_concurrent_rpc_requests, &config.rpc_tls)
@@ -51,32 +64,33 @@ impl Garage {
 			rpc_http_client,
 			background.clone(),
 			rpc_server,
+			replication_mode.replication_factor(),
 		);
 
 		let data_rep_param = TableShardedReplication {
 			system: system.clone(),
-			replication_factor: config.data_replication_factor,
-			write_quorum: (config.data_replication_factor + 1) / 2,
+			replication_factor: replication_mode.replication_factor(),
+			write_quorum: replication_mode.write_quorum(),
 			read_quorum: 1,
 		};
 
 		let meta_rep_param = TableShardedReplication {
 			system: system.clone(),
-			replication_factor: config.meta_replication_factor,
-			write_quorum: (config.meta_replication_factor + 1) / 2,
-			read_quorum: (config.meta_replication_factor + 1) / 2,
+			replication_factor: replication_mode.replication_factor(),
+			write_quorum: replication_mode.write_quorum(),
+			read_quorum: replication_mode.read_quorum(),
 		};
 
 		let control_rep_param = TableFullReplication {
 			system: system.clone(),
-			max_faults: config.control_write_max_faults,
+			max_faults: replication_mode.control_write_max_faults(),
 		};
 
 		info!("Initialize block manager...");
 		let block_manager = BlockManager::new(
 			&db,
 			config.data_dir.clone(),
-			data_rep_param.clone(),
+			data_rep_param,
 			system.clone(),
 			rpc_server,
 		);
@@ -86,7 +100,7 @@ impl Garage {
 			BlockRefTable {
 				block_manager: block_manager.clone(),
 			},
-			data_rep_param.clone(),
+			meta_rep_param.clone(),
 			system.clone(),
 			&db,
 			"block_ref".to_string(),
@@ -112,7 +126,7 @@ impl Garage {
 				background: background.clone(),
 				version_table: version_table.clone(),
 			},
-			meta_rep_param.clone(),
+			meta_rep_param,
 			system.clone(),
 			&db,
 			"object".to_string(),
@@ -132,7 +146,7 @@ impl Garage {
 		info!("Initialize key_table_table...");
 		let key_table = Table::new(
 			KeyTable,
-			control_rep_param.clone(),
+			control_rep_param,
 			system.clone(),
 			&db,
 			"key".to_string(),
@@ -143,9 +157,9 @@ impl Garage {
 		let garage = Arc::new(Self {
 			config,
 			db,
-			system: system.clone(),
-			block_manager,
 			background,
+			system,
+			block_manager,
 			bucket_table,
 			key_table,
 			object_table,

src/model/key_table.rs

@@ -3,62 +3,71 @@ use serde::{Deserialize, Serialize};
 use garage_table::crdt::*;
 use garage_table::*;
 
+/// An api key
 #[derive(PartialEq, Clone, Debug, Serialize, Deserialize)]
 pub struct Key {
-	// Primary key
+	/// The id of the key (immutable), used as partition key
 	pub key_id: String,
 
-	// Associated secret key (immutable)
+	/// The secret_key associated
 	pub secret_key: String,
 
-	// Name
-	pub name: crdt::LWW<String>,
+	/// Name for the key
+	pub name: crdt::Lww<String>,
 
-	// Deletion
+	/// Is the key deleted
 	pub deleted: crdt::Bool,
 
-	// Authorized keys
-	pub authorized_buckets: crdt::LWWMap<String, PermissionSet>,
+	/// Buckets in which the key is authorized. Empty if `Key` is deleted
 	// CRDT interaction: deleted implies authorized_buckets is empty
+	pub authorized_buckets: crdt::LwwMap<String, PermissionSet>,
 }
 
 impl Key {
+	/// Initialize a new Key, generating a random identifier and associated secret key
 	pub fn new(name: String) -> Self {
 		let key_id = format!("GK{}", hex::encode(&rand::random::<[u8; 12]>()[..]));
 		let secret_key = hex::encode(&rand::random::<[u8; 32]>()[..]);
 		Self {
 			key_id,
 			secret_key,
-			name: crdt::LWW::new(name),
+			name: crdt::Lww::new(name),
 			deleted: crdt::Bool::new(false),
-			authorized_buckets: crdt::LWWMap::new(),
+			authorized_buckets: crdt::LwwMap::new(),
 		}
 	}
+
+	/// Import a key from it's parts
 	pub fn import(key_id: &str, secret_key: &str, name: &str) -> Self {
 		Self {
 			key_id: key_id.to_string(),
 			secret_key: secret_key.to_string(),
-			name: crdt::LWW::new(name.to_string()),
+			name: crdt::Lww::new(name.to_string()),
 			deleted: crdt::Bool::new(false),
-			authorized_buckets: crdt::LWWMap::new(),
+			authorized_buckets: crdt::LwwMap::new(),
 		}
 	}
+
+	/// Create a new Key which can me merged to mark an existing key deleted
 	pub fn delete(key_id: String) -> Self {
 		Self {
 			key_id,
 			secret_key: "".into(),
-			name: crdt::LWW::new("".to_string()),
+			name: crdt::Lww::new("".to_string()),
 			deleted: crdt::Bool::new(true),
-			authorized_buckets: crdt::LWWMap::new(),
+			authorized_buckets: crdt::LwwMap::new(),
 		}
 	}
-	/// Add an authorized bucket, only if it wasn't there before
+
+	/// Check if `Key` is allowed to read in bucket
 	pub fn allow_read(&self, bucket: &str) -> bool {
 		self.authorized_buckets
 			.get(&bucket.to_string())
 			.map(|x| x.allow_read)
 			.unwrap_or(false)
 	}
+
+	/// Check if `Key` is allowed to write in bucket
 	pub fn allow_write(&self, bucket: &str) -> bool {
 		self.authorized_buckets
 			.get(&bucket.to_string())
@@ -67,13 +76,16 @@ impl Key {
 	}
 }
 
+/// Permission given to a key in a bucket
 #[derive(PartialOrd, Ord, PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
 pub struct PermissionSet {
+	/// The key can be used to read the bucket
 	pub allow_read: bool,
+	/// The key can be used to write in the bucket
 	pub allow_write: bool,
 }
 
-impl AutoCRDT for PermissionSet {
+impl AutoCrdt for PermissionSet {
 	const WARN_IF_DIFFERENT: bool = true;
 }
 
@@ -86,7 +98,7 @@ impl Entry<EmptyKey, String> for Key {
 	}
 }
 
-impl CRDT for Key {
+impl Crdt for Key {
 	fn merge(&mut self, other: &Self) {
 		self.name.merge(&other.name);
 		self.deleted.merge(&other.deleted);

src/model/object_table.rs

@@ -6,24 +6,26 @@ use garage_util::background::BackgroundRunner;
 use garage_util::data::*;
 
 use garage_table::crdt::*;
-use garage_table::replication::sharded::*;
+use garage_table::replication::TableShardedReplication;
 use garage_table::*;
 
 use crate::version_table::*;
 
+/// An object
 #[derive(PartialEq, Clone, Debug, Serialize, Deserialize)]
 pub struct Object {
-	// Primary key
+	/// The bucket in which the object is stored, used as partition key
 	pub bucket: String,
 
-	// Sort key
+	/// The key at which the object is stored in its bucket, used as sorting key
 	pub key: String,
 
-	// Data
+	/// The list of currenty stored versions of the object
 	versions: Vec<ObjectVersion>,
 }
 
 impl Object {
+	/// Initialize an Object struct from parts
 	pub fn new(bucket: String, key: String, versions: Vec<ObjectVersion>) -> Self {
 		let mut ret = Self {
 			bucket,
@@ -36,7 +38,9 @@ impl Object {
 		}
 		ret
 	}
+
 	/// Adds a version if it wasn't already present
+	#[allow(clippy::result_unit_err)]
 	pub fn add_version(&mut self, new: ObjectVersion) -> Result<(), ()> {
 		match self
 			.versions
@@ -49,27 +53,36 @@ impl Object {
 			Ok(_) => Err(()),
 		}
 	}
+
+	/// Get a list of currently stored versions of `Object`
 	pub fn versions(&self) -> &[ObjectVersion] {
 		&self.versions[..]
 	}
 }
 
+/// Informations about a version of an object
 #[derive(PartialEq, Clone, Debug, Serialize, Deserialize)]
 pub struct ObjectVersion {
-	pub uuid: UUID,
+	/// Id of the version
+	pub uuid: Uuid,
+	/// Timestamp of when the object was created
 	pub timestamp: u64,
-
+	/// State of the version
 	pub state: ObjectVersionState,
 }
 
+/// State of an object version
 #[derive(PartialEq, Clone, Debug, Serialize, Deserialize)]
 pub enum ObjectVersionState {
+	/// The version is being received
 	Uploading(ObjectVersionHeaders),
+	/// The version is fully received
 	Complete(ObjectVersionData),
+	/// The version uploaded containded errors or the upload was explicitly aborted
 	Aborted,
 }
 
-impl CRDT for ObjectVersionState {
+impl Crdt for ObjectVersionState {
 	fn merge(&mut self, other: &Self) {
 		use ObjectVersionState::*;
 		match other {
@@ -90,46 +103,58 @@ impl CRDT for ObjectVersionState {
 	}
 }
 
+/// Data stored in object version
 #[derive(PartialEq, Eq, PartialOrd, Ord, Clone, Debug, Serialize, Deserialize)]
 pub enum ObjectVersionData {
+	/// The object was deleted, this Version is a tombstone to mark it as such
 	DeleteMarker,
+	/// The object is short, it's stored inlined
 	Inline(ObjectVersionMeta, #[serde(with = "serde_bytes")] Vec<u8>),
+	/// The object is not short, Hash of first block is stored here, next segments hashes are
+	/// stored in the version table
 	FirstBlock(ObjectVersionMeta, Hash),
 }
 
-impl AutoCRDT for ObjectVersionData {
+impl AutoCrdt for ObjectVersionData {
 	const WARN_IF_DIFFERENT: bool = true;
 }
 
+/// Metadata about the object version
 #[derive(PartialEq, Eq, PartialOrd, Ord, Clone, Debug, Serialize, Deserialize)]
 pub struct ObjectVersionMeta {
+	/// Headers to send to the client
 	pub headers: ObjectVersionHeaders,
+	/// Size of the object
 	pub size: u64,
+	/// etag of the object
 	pub etag: String,
 }
 
+/// Additional headers for an object
 #[derive(PartialEq, Eq, PartialOrd, Ord, Clone, Debug, Serialize, Deserialize)]
 pub struct ObjectVersionHeaders {
+	/// Content type of the object
 	pub content_type: String,
+	/// Any other http headers to send
 	pub other: BTreeMap<String, String>,
 }
 
 impl ObjectVersion {
-	fn cmp_key(&self) -> (u64, UUID) {
+	fn cmp_key(&self) -> (u64, Uuid) {
 		(self.timestamp, self.uuid)
 	}
+
+	/// Is the object version currently being uploaded
 	pub fn is_uploading(&self) -> bool {
-		match self.state {
-			ObjectVersionState::Uploading(_) => true,
-			_ => false,
-		}
+		matches!(self.state, ObjectVersionState::Uploading(_))
 	}
+
+	/// Is the object version completely received
 	pub fn is_complete(&self) -> bool {
-		match self.state {
-			ObjectVersionState::Complete(_) => true,
-			_ => false,
-		}
+		matches!(self.state, ObjectVersionState::Complete(_))
 	}
+
+	/// Is the object version available (received and not a tombstone)
 	pub fn is_data(&self) -> bool {
 		match self.state {
 			ObjectVersionState::Complete(ObjectVersionData::DeleteMarker) => false,
@@ -147,11 +172,13 @@ impl Entry<String, String> for Object {
 		&self.key
 	}
 	fn is_tombstone(&self) -> bool {
-		self.versions.len() == 1 && self.versions[0].state == ObjectVersionState::Complete(ObjectVersionData::DeleteMarker)
+		self.versions.len() == 1
+			&& self.versions[0].state
+				== ObjectVersionState::Complete(ObjectVersionData::DeleteMarker)
 	}
 }
 
-impl CRDT for Object {
+impl Crdt for Object {
 	fn merge(&mut self, other: &Self) {
 		// Merge versions from other into here
 		for other_v in other.versions.iter() {
@@ -175,8 +202,7 @@ impl CRDT for Object {
 			.iter()
 			.enumerate()
 			.rev()
-			.filter(|(_, v)| v.is_complete())
-			.next()
+			.find(|(_, v)| v.is_complete())
 			.map(|(vi, _)| vi);
 
 		if let Some(last_vi) = last_complete {

src/model/version_table.rs

@@ -5,31 +5,37 @@ use garage_util::background::BackgroundRunner;
 use garage_util::data::*;
 
 use garage_table::crdt::*;
-use garage_table::replication::sharded::*;
+use garage_table::replication::TableShardedReplication;
 use garage_table::*;
 
 use crate::block_ref_table::*;
 
+/// A version of an object
 #[derive(PartialEq, Clone, Debug, Serialize, Deserialize)]
 pub struct Version {
-	// Primary key
-	pub uuid: UUID,
+	/// UUID of the version, used as partition key
+	pub uuid: Uuid,
 
 	// Actual data: the blocks for this version
 	// In the case of a multipart upload, also store the etags
 	// of individual parts and check them when doing CompleteMultipartUpload
+	/// Is this version deleted
 	pub deleted: crdt::Bool,
+	/// list of blocks of data composing the version
 	pub blocks: crdt::Map<VersionBlockKey, VersionBlock>,
+	/// Etag of each part in case of a multipart upload, empty otherwise
 	pub parts_etags: crdt::Map<u64, String>,
 
 	// Back link to bucket+key so that we can figure if
 	// this was deleted later on
+	/// Bucket in which the related object is stored
 	pub bucket: String,
+	/// Key in which the related object is stored
 	pub key: String,
 }
 
 impl Version {
-	pub fn new(uuid: UUID, bucket: String, key: String, deleted: bool) -> Self {
+	pub fn new(uuid: Uuid, bucket: String, key: String, deleted: bool) -> Self {
 		Self {
 			uuid,
 			deleted: deleted.into(),
@@ -43,7 +49,9 @@ impl Version {
 
 #[derive(PartialEq, Eq, Clone, Copy, Debug, Serialize, Deserialize)]
 pub struct VersionBlockKey {
+	/// Number of the part
 	pub part_number: u64,
+	/// Offset of this sub-segment in its part
 	pub offset: u64,
 }
 
@@ -61,13 +69,16 @@ impl PartialOrd for VersionBlockKey {
 	}
 }
 
+/// Informations about a single block
 #[derive(PartialEq, Eq, Ord, PartialOrd, Clone, Copy, Debug, Serialize, Deserialize)]
 pub struct VersionBlock {
+	/// Blake2 sum of the block
 	pub hash: Hash,
+	/// Size of the block
 	pub size: u64,
 }
 
-impl AutoCRDT for VersionBlock {
+impl AutoCrdt for VersionBlock {
 	const WARN_IF_DIFFERENT: bool = true;
 }
 
@@ -83,7 +94,7 @@ impl Entry<Hash, EmptyKey> for Version {
 	}
 }
 
-impl CRDT for Version {
+impl Crdt for Version {
 	fn merge(&mut self, other: &Self) {
 		self.deleted.merge(&other.deleted);
 

src/rpc/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "garage_rpc"
-version = "0.2.1"
+version = "0.3.0"
 authors = ["Alex Auvolat <alex@adnab.me>"]
 edition = "2018"
 license = "AGPL-3.0"
@@ -13,7 +13,9 @@ path = "lib.rs"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-garage_util = { version = "0.2.1", path = "../util" }
+garage_util = { version = "0.3.0", path = "../util" }
+
+garage_rpc_021 = { package = "garage_rpc", version = "0.2.1" }
 
 arc-swap = "1.0"
 bytes = "1.0"

src/rpc/lib.rs

@@ -1,7 +1,9 @@
+//! Crate containing rpc related functions and types used in Garage
+
 #[macro_use]
 extern crate log;
 
-pub mod consul;
+mod consul;
 pub(crate) mod tls_util;
 
 pub mod membership;

src/rpc/membership.rs

@@ -1,8 +1,9 @@
+//! Module containing structs related to membership management
 use std::collections::HashMap;
 use std::fmt::Write as FmtWrite;
 use std::io::{Read, Write};
 use std::net::{IpAddr, SocketAddr};
-use std::path::PathBuf;
+use std::path::{Path, PathBuf};
 use std::sync::atomic::{AtomicUsize, Ordering};
 use std::sync::Arc;
 use std::time::Duration;
@@ -11,13 +12,13 @@ use futures::future::join_all;
 use futures::select;
 use futures_util::future::*;
 use serde::{Deserialize, Serialize};
-use tokio::io::AsyncWriteExt;
 use tokio::sync::watch;
 use tokio::sync::Mutex;
 
 use garage_util::background::BackgroundRunner;
 use garage_util::data::*;
 use garage_util::error::Error;
+use garage_util::persister::Persister;
 use garage_util::time::*;
 
 use crate::consul::get_consul_nodes;
@@ -26,27 +27,36 @@ use crate::rpc_client::*;
 use crate::rpc_server::*;
 
 const PING_INTERVAL: Duration = Duration::from_secs(10);
-const CONSUL_INTERVAL: Duration = Duration::from_secs(60);
+const DISCOVERY_INTERVAL: Duration = Duration::from_secs(60);
 const PING_TIMEOUT: Duration = Duration::from_secs(2);
 const MAX_FAILURES_BEFORE_CONSIDERED_DOWN: usize = 5;
 
+/// RPC endpoint used for calls related to membership
 pub const MEMBERSHIP_RPC_PATH: &str = "_membership";
 
+/// RPC messages related to membership
 #[derive(Debug, Serialize, Deserialize)]
 pub enum Message {
+	/// Response to successfull advertisements
 	Ok,
+	/// Message sent to detect other nodes status
 	Ping(PingMessage),
+	/// Ask other node for the nodes it knows. Answered with AdvertiseNodesUp
 	PullStatus,
+	/// Ask other node its config. Answered with AdvertiseConfig
 	PullConfig,
+	/// Advertisement of nodes the host knows up. Sent spontanously or in response to PullStatus
 	AdvertiseNodesUp(Vec<AdvertisedNode>),
+	/// Advertisement of nodes config. Sent spontanously or in response to PullConfig
 	AdvertiseConfig(NetworkConfig),
 }
 
 impl RpcMessage for Message {}
 
+/// A ping, containing informations about status and config
 #[derive(Debug, Serialize, Deserialize)]
 pub struct PingMessage {
-	id: UUID,
+	id: Uuid,
 	rpc_port: u16,
 
 	status_hash: Hash,
@@ -55,21 +65,29 @@ pub struct PingMessage {
 	state_info: StateInfo,
 }
 
+/// A node advertisement
 #[derive(Clone, Debug, Serialize, Deserialize)]
 pub struct AdvertisedNode {
-	pub id: UUID,
+	/// Id of the node this advertisement relates to
+	pub id: Uuid,
+	/// IP and port of the node
 	pub addr: SocketAddr,
 
+	/// Is the node considered up
 	pub is_up: bool,
+	/// When was the node last seen up, in milliseconds since UNIX epoch
 	pub last_seen: u64,
 
 	pub state_info: StateInfo,
 }
 
+/// This node's membership manager
 pub struct System {
-	pub id: UUID,
+	/// The id of this node
+	pub id: Uuid,
 
-	metadata_dir: PathBuf,
+	persist_config: Persister<NetworkConfig>,
+	persist_status: Persister<Vec<AdvertisedNode>>,
 	rpc_local_port: u16,
 
 	state_info: StateInfo,
@@ -77,29 +95,45 @@ pub struct System {
 	rpc_http_client: Arc<RpcHttpClient>,
 	rpc_client: Arc<RpcClient<Message>>,
 
+	replication_factor: usize,
 	pub(crate) status: watch::Receiver<Arc<Status>>,
+	/// The ring
 	pub ring: watch::Receiver<Arc<Ring>>,
 
-	update_lock: Mutex<(watch::Sender<Arc<Status>>, watch::Sender<Arc<Ring>>)>,
+	update_lock: Mutex<Updaters>,
 
+	/// The job runner of this node
 	pub background: Arc<BackgroundRunner>,
 }
 
+struct Updaters {
+	update_status: watch::Sender<Arc<Status>>,
+	update_ring: watch::Sender<Arc<Ring>>,
+}
+
+/// The status of each nodes, viewed by this node
 #[derive(Debug, Clone)]
 pub struct Status {
-	pub nodes: HashMap<UUID, Arc<StatusEntry>>,
+	/// Mapping of each node id to its known status
+	pub nodes: HashMap<Uuid, Arc<StatusEntry>>,
+	/// Hash of `nodes`, used to detect when nodes have different views of the cluster
 	pub hash: Hash,
 }
 
+/// The status of a single node
 #[derive(Debug)]
 pub struct StatusEntry {
+	/// The IP and port used to connect to this node
 	pub addr: SocketAddr,
+	/// Last time this node was seen
 	pub last_seen: u64,
+	/// Number of consecutive pings sent without reply to this node
 	pub num_failures: AtomicUsize,
 	pub state_info: StateInfo,
 }
 
 impl StatusEntry {
+	/// is the node associated to this entry considered up
 	pub fn is_up(&self) -> bool {
 		self.num_failures.load(Ordering::SeqCst) < MAX_FAILURES_BEFORE_CONSIDERED_DOWN
 	}
@@ -144,17 +178,36 @@ impl Status {
 		debug!("END --");
 		self.hash = blake2sum(nodes_txt.as_bytes());
 	}
+
+	fn to_serializable_membership(&self, system: &System) -> Vec<AdvertisedNode> {
+		let mut mem = vec![];
+		for (node, status) in self.nodes.iter() {
+			let state_info = if *node == system.id {
+				system.state_info.clone()
+			} else {
+				status.state_info.clone()
+			};
+			mem.push(AdvertisedNode {
+				id: *node,
+				addr: status.addr,
+				is_up: status.is_up(),
+				last_seen: status.last_seen,
+				state_info,
+			});
+		}
+		mem
+	}
 }
 
-fn gen_node_id(metadata_dir: &PathBuf) -> Result<UUID, Error> {
-	let mut id_file = metadata_dir.clone();
+fn gen_node_id(metadata_dir: &Path) -> Result<Uuid, Error> {
+	let mut id_file = metadata_dir.to_path_buf();
 	id_file.push("node_id");
 	if id_file.as_path().exists() {
 		let mut f = std::fs::File::open(id_file.as_path())?;
 		let mut d = vec![];
 		f.read_to_end(&mut d)?;
 		if d.len() != 32 {
-			return Err(Error::Message(format!("Corrupt node_id file")));
+			return Err(Error::Message("Corrupt node_id file".to_string()));
 		}
 
 		let mut id = [0u8; 32];
@@ -169,43 +222,42 @@ fn gen_node_id(metadata_dir: &PathBuf) -> Result<UUID, Error> {
 	}
 }
 
-fn read_network_config(metadata_dir: &PathBuf) -> Result<NetworkConfig, Error> {
-	let mut path = metadata_dir.clone();
-	path.push("network_config");
-
-	let mut file = std::fs::OpenOptions::new()
-		.read(true)
-		.open(path.as_path())?;
-
-	let mut net_config_bytes = vec![];
-	file.read_to_end(&mut net_config_bytes)?;
-
-	let net_config = rmp_serde::decode::from_read_ref(&net_config_bytes[..])
-		.expect("Unable to parse network configuration file (has version format changed?).");
-
-	Ok(net_config)
-}
-
 impl System {
+	/// Create this node's membership manager
 	pub fn new(
 		metadata_dir: PathBuf,
 		rpc_http_client: Arc<RpcHttpClient>,
 		background: Arc<BackgroundRunner>,
 		rpc_server: &mut RpcServer,
+		replication_factor: usize,
 	) -> Arc<Self> {
 		let id = gen_node_id(&metadata_dir).expect("Unable to read or generate node ID");
 		info!("Node ID: {}", hex::encode(&id));
 
-		let net_config = match read_network_config(&metadata_dir) {
+		let persist_config = Persister::new(&metadata_dir, "network_config");
+		let persist_status = Persister::new(&metadata_dir, "peer_info");
+
+		let net_config = match persist_config.load() {
 			Ok(x) => x,
 			Err(e) => {
-				info!(
-					"No valid previous network configuration stored ({}), starting fresh.",
-					e
-				);
-				NetworkConfig::new()
+				match Persister::<garage_rpc_021::ring::NetworkConfig>::new(
+					&metadata_dir,
+					"network_config",
+				)
+				.load()
+				{
+					Ok(old_config) => NetworkConfig::migrate_from_021(old_config),
+					Err(e2) => {
+						info!(
+							"No valid previous network configuration stored ({}, {}), starting fresh.",
+							e, e2
+						);
+						NetworkConfig::new()
+					}
+				}
 			}
 		};
+
 		let mut status = Status {
 			nodes: HashMap::new(),
 			hash: Hash::default(),
@@ -216,10 +268,10 @@ impl System {
 		let state_info = StateInfo {
 			hostname: gethostname::gethostname()
 				.into_string()
-				.unwrap_or("<invalid utf-8>".to_string()),
+				.unwrap_or_else(|_| "<invalid utf-8>".to_string()),
 		};
 
-		let ring = Ring::new(net_config);
+		let ring = Ring::new(net_config, replication_factor);
 		let (update_ring, ring) = watch::channel(Arc::new(ring));
 
 		let rpc_path = MEMBERSHIP_RPC_PATH.to_string();
@@ -231,14 +283,19 @@ impl System {
 
 		let sys = Arc::new(System {
 			id,
-			metadata_dir,
+			persist_config,
+			persist_status,
 			rpc_local_port: rpc_server.bind_addr.port(),
 			state_info,
 			rpc_http_client,
 			rpc_client,
+			replication_factor,
 			status,
 			ring,
-			update_lock: Mutex::new((update_status, update_ring)),
+			update_lock: Mutex::new(Updaters {
+				update_status,
+				update_ring,
+			}),
 			background,
 		});
 		sys.clone().register_handler(rpc_server, rpc_path);
@@ -252,17 +309,18 @@ impl System {
 				match msg {
 					Message::Ping(ping) => self2.handle_ping(&addr, &ping).await,
 
-					Message::PullStatus => self2.handle_pull_status(),
-					Message::PullConfig => self2.handle_pull_config(),
+					Message::PullStatus => Ok(self2.handle_pull_status()),
+					Message::PullConfig => Ok(self2.handle_pull_config()),
 					Message::AdvertiseNodesUp(adv) => self2.handle_advertise_nodes_up(&adv).await,
 					Message::AdvertiseConfig(adv) => self2.handle_advertise_config(&adv).await,
 
-					_ => Err(Error::BadRPC(format!("Unexpected RPC message"))),
+					_ => Err(Error::BadRpc("Unexpected RPC message".to_string())),
 				}
 			}
 		});
 	}
 
+	/// Get an RPC client
 	pub fn rpc_client<M: RpcMessage + 'static>(self: &Arc<Self>, path: &str) -> Arc<RpcClient<M>> {
 		RpcClient::new(
 			RpcAddrClient::new(self.rpc_http_client.clone(), path.to_string()),
@@ -271,15 +329,13 @@ impl System {
 		)
 	}
 
+	/// Save network configuration to disc
 	async fn save_network_config(self: Arc<Self>) -> Result<(), Error> {
-		let mut path = self.metadata_dir.clone();
-		path.push("network_config");
-
 		let ring = self.ring.borrow().clone();
-		let data = rmp_to_vec_all_named(&ring.config)?;
-
-		let mut f = tokio::fs::File::create(path.as_path()).await?;
-		f.write_all(&data[..]).await?;
+		self.persist_config
+			.save_async(&ring.config)
+			.await
+			.expect("Cannot save current cluster configuration");
 		Ok(())
 	}
 
@@ -306,31 +362,27 @@ impl System {
 		self.rpc_client.call_many(&to[..], msg, timeout).await;
 	}
 
+	/// Perform bootstraping, starting the ping loop
 	pub async fn bootstrap(
 		self: Arc<Self>,
-		peers: &[SocketAddr],
+		peers: Vec<SocketAddr>,
 		consul_host: Option<String>,
 		consul_service_name: Option<String>,
 	) {
-		let bootstrap_peers = peers.iter().map(|ip| (*ip, None)).collect::<Vec<_>>();
-		self.clone().ping_nodes(bootstrap_peers).await;
+		let self2 = self.clone();
+		self.background
+			.spawn_worker("discovery loop".to_string(), |stop_signal| {
+				self2.discovery_loop(peers, consul_host, consul_service_name, stop_signal)
+			});
 
 		let self2 = self.clone();
 		self.background
-			.spawn_worker(format!("ping loop"), |stop_signal| {
+			.spawn_worker("ping loop".to_string(), |stop_signal| {
 				self2.ping_loop(stop_signal)
 			});
-
-		if let (Some(consul_host), Some(consul_service_name)) = (consul_host, consul_service_name) {
-			let self2 = self.clone();
-			self.background
-				.spawn_worker(format!("Consul loop"), |stop_signal| {
-					self2.consul_loop(stop_signal, consul_host, consul_service_name)
-				});
-		}
 	}
 
-	async fn ping_nodes(self: Arc<Self>, peers: Vec<(SocketAddr, Option<UUID>)>) {
+	async fn ping_nodes(self: Arc<Self>, peers: Vec<(SocketAddr, Option<Uuid>)>) {
 		let ping_msg = self.make_ping();
 		let ping_resps = join_all(peers.iter().map(|(addr, id_option)| {
 			let sys = self.clone();
@@ -378,12 +430,13 @@ impl System {
 				}
 			} else if let Some(id) = id_option {
 				if let Some(st) = status.nodes.get_mut(id) {
+					// we need to increment failure counter as call was done using by_addr so the
+					// counter was not auto-incremented
 					st.num_failures.fetch_add(1, Ordering::SeqCst);
 					if !st.is_up() {
 						warn!("Node {:?} seems to be down.", id);
 						if !ring.config.members.contains_key(id) {
 							info!("Removing node {:?} from status (not in config and not responding to pings anymore)", id);
-							drop(st);
 							status.nodes.remove(&id);
 							has_changes = true;
 						}
@@ -394,12 +447,10 @@ impl System {
 		if has_changes {
 			status.recalculate_hash();
 		}
-		if let Err(e) = update_locked.0.send(Arc::new(status)) {
-			error!("In ping_nodes: could not save status update ({})", e);
-		}
+		self.update_status(&update_locked, status).await;
 		drop(update_locked);
 
-		if to_advertise.len() > 0 {
+		if !to_advertise.is_empty() {
 			self.broadcast(Message::AdvertiseNodesUp(to_advertise), PING_TIMEOUT)
 				.await;
 		}
@@ -420,7 +471,7 @@ impl System {
 		let status_hash = status.hash;
 		let config_version = self.ring.borrow().config.version;
 
-		update_locked.0.send(Arc::new(status))?;
+		self.update_status(&update_locked, status).await;
 		drop(update_locked);
 
 		if is_new || status_hash != ping.status_hash {
@@ -435,29 +486,13 @@ impl System {
 		Ok(self.make_ping())
 	}
 
-	fn handle_pull_status(&self) -> Result<Message, Error> {
-		let status = self.status.borrow().clone();
-		let mut mem = vec![];
-		for (node, status) in status.nodes.iter() {
-			let state_info = if *node == self.id {
-				self.state_info.clone()
-			} else {
-				status.state_info.clone()
-			};
-			mem.push(AdvertisedNode {
-				id: *node,
-				addr: status.addr,
-				is_up: status.is_up(),
-				last_seen: status.last_seen,
-				state_info,
-			});
-		}
-		Ok(Message::AdvertiseNodesUp(mem))
+	fn handle_pull_status(&self) -> Message {
+		Message::AdvertiseNodesUp(self.status.borrow().to_serializable_membership(self))
 	}
 
-	fn handle_pull_config(&self) -> Result<Message, Error> {
+	fn handle_pull_config(&self) -> Message {
 		let ring = self.ring.borrow().clone();
-		Ok(Message::AdvertiseConfig(ring.config.clone()))
+		Message::AdvertiseConfig(ring.config.clone())
 	}
 
 	async fn handle_advertise_nodes_up(
@@ -502,10 +537,10 @@ impl System {
 		if has_changed {
 			status.recalculate_hash();
 		}
-		update_lock.0.send(Arc::new(status))?;
+		self.update_status(&update_lock, status).await;
 		drop(update_lock);
 
-		if to_ping.len() > 0 {
+		if !to_ping.is_empty() {
 			self.background
 				.spawn_cancellable(self.clone().ping_nodes(to_ping).map(Ok));
 		}
@@ -521,8 +556,8 @@ impl System {
 		let ring: Arc<Ring> = self.ring.borrow().clone();
 
 		if adv.version > ring.config.version {
-			let ring = Ring::new(adv.clone());
-			update_lock.1.send(Arc::new(ring))?;
+			let ring = Ring::new(adv.clone(), self.replication_factor);
+			update_lock.update_ring.send(Arc::new(ring))?;
 			drop(update_lock);
 
 			self.background.spawn_cancellable(
@@ -537,7 +572,7 @@ impl System {
 	}
 
 	async fn ping_loop(self: Arc<Self>, mut stop_signal: watch::Receiver<bool>) {
-		loop {
+		while !*stop_signal.borrow() {
 			let restart_at = tokio::time::sleep(PING_INTERVAL);
 
 			let status = self.status.borrow().clone();
@@ -551,45 +586,74 @@ impl System {
 			self.clone().ping_nodes(ping_addrs).await;
 
 			select! {
-				_ = restart_at.fuse() => (),
-				_ = stop_signal.changed().fuse() => {
-					if *stop_signal.borrow() {
-						return;
+				_ = restart_at.fuse() => {},
+				_ = stop_signal.changed().fuse() => {},
+			}
+		}
+	}
+
+	async fn discovery_loop(
+		self: Arc<Self>,
+		bootstrap_peers: Vec<SocketAddr>,
+		consul_host: Option<String>,
+		consul_service_name: Option<String>,
+		mut stop_signal: watch::Receiver<bool>,
+	) {
+		let consul_config = match (consul_host, consul_service_name) {
+			(Some(ch), Some(csn)) => Some((ch, csn)),
+			_ => None,
+		};
+
+		while !*stop_signal.borrow() {
+			let not_configured = self.ring.borrow().config.members.is_empty();
+			let no_peers = self.status.borrow().nodes.len() < 3;
+			let bad_peers = self
+				.status
+				.borrow()
+				.nodes
+				.iter()
+				.filter(|(_, v)| v.is_up())
+				.count() != self.ring.borrow().config.members.len();
+
+			if not_configured || no_peers || bad_peers {
+				info!("Doing a bootstrap/discovery step (not_configured: {}, no_peers: {}, bad_peers: {})", not_configured, no_peers, bad_peers);
+
+				let mut ping_list = bootstrap_peers
+					.iter()
+					.map(|ip| (*ip, None))
+					.collect::<Vec<_>>();
+
+				if let Ok(peers) = self.persist_status.load_async().await {
+					ping_list.extend(peers.iter().map(|x| (x.addr, Some(x.id))));
+				}
+
+				if let Some((consul_host, consul_service_name)) = &consul_config {
+					match get_consul_nodes(consul_host, consul_service_name).await {
+						Ok(node_list) => {
+							ping_list.extend(node_list.iter().map(|a| (*a, None)));
+						}
+						Err(e) => {
+							warn!("Could not retrieve node list from Consul: {}", e);
+						}
 					}
 				}
-			}
-		}
-	}
 
-	async fn consul_loop(
-		self: Arc<Self>,
-		mut stop_signal: watch::Receiver<bool>,
-		consul_host: String,
-		consul_service_name: String,
-	) {
-		while !*stop_signal.borrow() {
-			let restart_at = tokio::time::sleep(CONSUL_INTERVAL);
-
-			match get_consul_nodes(&consul_host, &consul_service_name).await {
-				Ok(mut node_list) => {
-					let ping_addrs = node_list.drain(..).map(|a| (a, None)).collect::<Vec<_>>();
-					self.clone().ping_nodes(ping_addrs).await;
-				}
-				Err(e) => {
-					warn!("Could not retrieve node list from Consul: {}", e);
-				}
+				self.clone().ping_nodes(ping_list).await;
 			}
 
+			let restart_at = tokio::time::sleep(DISCOVERY_INTERVAL);
 			select! {
-				_ = restart_at.fuse() => (),
-				_ = stop_signal.changed().fuse() => (),
+				_ = restart_at.fuse() => {},
+				_ = stop_signal.changed().fuse() => {},
 			}
 		}
 	}
 
+	// for some reason fixing this is causing compilation error, see https://github.com/rust-lang/rust-clippy/issues/7052
+	#[allow(clippy::manual_async_fn)]
 	fn pull_status(
 		self: Arc<Self>,
-		peer: UUID,
+		peer: Uuid,
 	) -> impl futures::future::Future<Output = ()> + Send + 'static {
 		async move {
 			let resp = self
@@ -602,7 +666,7 @@ impl System {
 		}
 	}
 
-	async fn pull_config(self: Arc<Self>, peer: UUID) {
+	async fn pull_config(self: Arc<Self>, peer: Uuid) {
 		let resp = self
 			.rpc_client
 			.call(peer, Message::PullConfig, PING_TIMEOUT)
@@ -611,4 +675,32 @@ impl System {
 			let _: Result<_, _> = self.handle_advertise_config(&config).await;
 		}
 	}
+
+	async fn update_status(self: &Arc<Self>, updaters: &Updaters, status: Status) {
+		if status.hash != self.status.borrow().hash {
+			let mut list = status.to_serializable_membership(&self);
+
+			// Combine with old peer list to make sure no peer is lost
+			if let Ok(old_list) = self.persist_status.load_async().await {
+				for pp in old_list {
+					if !list.iter().any(|np| pp.id == np.id) {
+						list.push(pp);
+					}
+				}
+			}
+
+			if !list.is_empty() {
+				info!("Persisting new peer list ({} peers)", list.len());
+				self.persist_status
+					.save_async(&list)
+					.await
+					.expect("Unable to persist peer list");
+			}
+		}
+
+		updaters
+			.update_status
+			.send(Arc::new(status))
+			.expect("Could not update internal membership status");
+	}
 }

src/rpc/ring.rs

@@ -1,3 +1,5 @@
+//! Module containing types related to computing nodes which should receive a copy of data blocks
+//! and metadata
 use std::collections::{HashMap, HashSet};
 use std::convert::TryInto;
 
@@ -5,26 +7,27 @@ use serde::{Deserialize, Serialize};
 
 use garage_util::data::*;
 
-// A partition number is encoded on 16 bits,
-// i.e. we have up to 2**16 partitions.
-// (in practice we have exactly 2**PARTITION_BITS partitions)
+/// A partition id, which is stored on 16 bits
+/// i.e. we have up to 2**16 partitions.
+/// (in practice we have exactly 2**PARTITION_BITS partitions)
 pub type Partition = u16;
 
 // TODO: make this constant parametrizable in the config file
 // For deployments with many nodes it might make sense to bump
 // it up to 10.
 // Maximum value : 16
+/// How many bits from the hash are used to make partitions. Higher numbers means more fairness in
+/// presence of numerous nodes, but exponentially bigger ring. Max 16
 pub const PARTITION_BITS: usize = 8;
 
 const PARTITION_MASK_U16: u16 = ((1 << PARTITION_BITS) - 1) << (16 - PARTITION_BITS);
 
-// TODO: make this constant paraetrizable in the config file
-// (most deployments use a replication factor of 3, so...)
-pub const MAX_REPLICATION: usize = 3;
-
+/// The user-defined configuration of the cluster's nodes
 #[derive(Clone, Debug, Serialize, Deserialize)]
 pub struct NetworkConfig {
-	pub members: HashMap<UUID, NetworkConfigEntry>,
+	/// Map of each node's id to it's configuration
+	pub members: HashMap<Uuid, NetworkConfigEntry>,
+	/// Version of this config
 	pub version: u64,
 }
 
@@ -35,41 +38,112 @@ impl NetworkConfig {
 			version: 0,
 		}
 	}
+
+	pub(crate) fn migrate_from_021(old: garage_rpc_021::ring::NetworkConfig) -> Self {
+		let members = old
+			.members
+			.into_iter()
+			.map(|(id, conf)| {
+				(
+					Hash::try_from(id.as_slice()).unwrap(),
+					NetworkConfigEntry {
+						zone: conf.datacenter,
+						capacity: if conf.capacity == 0 {
+							None
+						} else {
+							Some(conf.capacity)
+						},
+						tag: conf.tag,
+					},
+				)
+			})
+			.collect();
+		Self {
+			members,
+			version: old.version,
+		}
+	}
 }
 
+/// The overall configuration of one (possibly remote) node
 #[derive(Clone, Debug, Serialize, Deserialize)]
 pub struct NetworkConfigEntry {
-	pub datacenter: String,
-	pub capacity: u32,
+	/// Datacenter at which this entry belong. This infromation might be used to perform a better
+	/// geodistribution
+	pub zone: String,
+	/// The (relative) capacity of the node
+	/// If this is set to None, the node does not participate in storing data for the system
+	/// and is only active as an API gateway to other nodes
+	pub capacity: Option<u32>,
+	/// A tag to recognize the entry, not used for other things than display
 	pub tag: String,
 }
 
-#[derive(Clone)]
-pub struct Ring {
-	pub config: NetworkConfig,
-	pub ring: Vec<RingEntry>,
+impl NetworkConfigEntry {
+	pub fn capacity_string(&self) -> String {
+		match self.capacity {
+			Some(c) => format!("{}", c),
+			None => "gateway".to_string(),
+		}
+	}
 }
 
+/// A ring distributing fairly objects to nodes
+#[derive(Clone)]
+pub struct Ring {
+	/// The replication factor for this ring
+	pub replication_factor: usize,
+
+	/// The network configuration used to generate this ring
+	pub config: NetworkConfig,
+
+	// Internal order of nodes used to make a more compact representation of the ring
+	nodes: Vec<Uuid>,
+
+	// The list of entries in the ring
+	ring: Vec<RingEntry>,
+}
+
+// Type to store compactly the id of a node in the system
+// Change this to u16 the day we want to have more than 256 nodes in a cluster
+type CompactNodeType = u8;
+
+// The maximum number of times an object might get replicated
+// This must be at least 3 because Garage supports 3-way replication
+// Here we use 6 so that the size of a ring entry is 8 bytes
+// (2 bytes partition id, 6 bytes node numbers as u8s)
+const MAX_REPLICATION: usize = 6;
+
+/// An entry in the ring
 #[derive(Clone, Debug)]
-pub struct RingEntry {
-	pub location: Hash,
-	pub nodes: [UUID; MAX_REPLICATION],
+struct RingEntry {
+	// The two first bytes of the first hash that goes in this partition
+	// (the next bytes are zeroes)
+	hash_prefix: u16,
+	// The nodes that store this partition, stored as a list of positions in the `nodes`
+	// field of the Ring structure
+	// Only items 0 up to ring.replication_factor - 1 are used, others are zeros
+	nodes_buf: [CompactNodeType; MAX_REPLICATION],
 }
 
 impl Ring {
-	pub(crate) fn new(config: NetworkConfig) -> Self {
+	// TODO this function MUST be refactored, it's 100 lines long, with a 50 lines loop, going up to 6
+	// levels of imbrication. It is basically impossible to test, maintain, or understand for an
+	// outsider.
+	pub(crate) fn new(config: NetworkConfig, replication_factor: usize) -> Self {
 		// Create a vector of partition indices (0 to 2**PARTITION_BITS-1)
 		let partitions_idx = (0usize..(1usize << PARTITION_BITS)).collect::<Vec<_>>();
 
-		let datacenters = config
+		let zones = config
 			.members
 			.iter()
-			.map(|(_id, info)| info.datacenter.as_str())
+			.filter(|(_id, info)| info.capacity.is_some())
+			.map(|(_id, info)| info.zone.as_str())
 			.collect::<HashSet<&str>>();
-		let n_datacenters = datacenters.len();
+		let n_zones = zones.len();
 
 		// Prepare ring
-		let mut partitions: Vec<Vec<(&UUID, &NetworkConfigEntry)>> = partitions_idx
+		let mut partitions: Vec<Vec<(&Uuid, &NetworkConfigEntry)>> = partitions_idx
 			.iter()
 			.map(|_i| Vec::new())
 			.collect::<Vec<_>>();
@@ -78,6 +152,7 @@ impl Ring {
 		let mut queues = config
 			.members
 			.iter()
+			.filter(|(_id, info)| info.capacity.is_some())
 			.map(|(node_id, node_info)| {
 				let mut parts = partitions_idx
 					.iter()
@@ -96,11 +171,13 @@ impl Ring {
 		let max_capacity = config
 			.members
 			.iter()
-			.map(|(_, node_info)| node_info.capacity)
+			.filter_map(|(_, node_info)| node_info.capacity)
 			.fold(0, std::cmp::max);
 
+		assert!(replication_factor <= MAX_REPLICATION);
+
 		// Fill up ring
-		for rep in 0..MAX_REPLICATION {
+		for rep in 0..replication_factor {
 			queues.sort_by_key(|(ni, _np, _q, _p)| {
 				let queue_data = [&u16::to_be_bytes(rep as u16)[..], ni.as_slice()].concat();
 				fasthash(&queue_data[..])
@@ -115,21 +192,19 @@ impl Ring {
 				let remaining0 = remaining;
 				for i_round in 0..max_capacity {
 					for (node_id, node_info, q, pos) in queues.iter_mut() {
-						if i_round >= node_info.capacity {
+						if i_round >= node_info.capacity.unwrap() {
 							continue;
 						}
-						for pos2 in *pos..q.len() {
-							let qv = q[pos2];
+						for (pos2, &qv) in q.iter().enumerate().skip(*pos) {
 							if partitions[qv].len() != rep {
 								continue;
 							}
-							let p_dcs = partitions[qv]
+							let p_zns = partitions[qv]
 								.iter()
-								.map(|(_id, info)| info.datacenter.as_str())
+								.map(|(_id, info)| info.zone.as_str())
 								.collect::<HashSet<&str>>();
-							if (p_dcs.len() < n_datacenters
-								&& !p_dcs.contains(&node_info.datacenter.as_str()))
-								|| (p_dcs.len() == n_datacenters
+							if (p_zns.len() < n_zones && !p_zns.contains(&node_info.zone.as_str()))
+								|| (p_zns.len() == n_zones
 									&& !partitions[qv].iter().any(|(id, _i)| id == node_id))
 							{
 								partitions[qv].push((node_id, node_info));
@@ -144,72 +219,109 @@ impl Ring {
 					// No progress made, exit
 					warn!("Could not build ring, not enough nodes configured.");
 					return Self {
+						replication_factor,
 						config,
+						nodes: vec![],
 						ring: vec![],
 					};
 				}
 			}
 		}
 
+		// Make a canonical order for nodes
+		let nodes = config
+			.members
+			.iter()
+			.filter(|(_id, info)| info.capacity.is_some())
+			.map(|(id, _)| *id)
+			.collect::<Vec<_>>();
+		let nodes_rev = nodes
+			.iter()
+			.enumerate()
+			.map(|(i, id)| (*id, i as CompactNodeType))
+			.collect::<HashMap<Uuid, CompactNodeType>>();
+
 		let ring = partitions
 			.iter()
 			.enumerate()
 			.map(|(i, nodes)| {
 				let top = (i as u16) << (16 - PARTITION_BITS);
-				let mut hash = [0u8; 32];
-				hash[0..2].copy_from_slice(&u16::to_be_bytes(top)[..]);
-				let nodes = nodes.iter().map(|(id, _info)| **id).collect::<Vec<UUID>>();
+				let nodes = nodes
+					.iter()
+					.map(|(id, _info)| *nodes_rev.get(id).unwrap())
+					.collect::<Vec<CompactNodeType>>();
+				assert!(nodes.len() == replication_factor);
+				let mut nodes_buf = [0u8; MAX_REPLICATION];
+				nodes_buf[..replication_factor].copy_from_slice(&nodes[..]);
 				RingEntry {
-					location: hash.into(),
-					nodes: nodes.try_into().unwrap(),
+					hash_prefix: top,
+					nodes_buf,
 				}
 			})
 			.collect::<Vec<_>>();
 
-		// eprintln!("RING: --");
-		// for e in ring.iter() {
-		// 	eprintln!("{:?}", e);
-		// }
-		// eprintln!("END --");
-
-		Self { config, ring }
+		Self {
+			replication_factor,
+			config,
+			nodes,
+			ring,
+		}
 	}
 
-	pub fn partition_of(&self, from: &Hash) -> Partition {
-		let top = u16::from_be_bytes(from.as_slice()[0..2].try_into().unwrap());
+	/// Get the partition in which data would fall on
+	pub fn partition_of(&self, position: &Hash) -> Partition {
+		let top = u16::from_be_bytes(position.as_slice()[0..2].try_into().unwrap());
 		top >> (16 - PARTITION_BITS)
 	}
 
+	/// Get the list of partitions and the first hash of a partition key that would fall in it
 	pub fn partitions(&self) -> Vec<(Partition, Hash)> {
 		let mut ret = vec![];
 
 		for (i, entry) in self.ring.iter().enumerate() {
-			ret.push((i as u16, entry.location));
+			let mut location = [0u8; 32];
+			location[..2].copy_from_slice(&u16::to_be_bytes(entry.hash_prefix)[..]);
+			ret.push((i as u16, location.into()));
 		}
-		if ret.len() > 0 {
+		if !ret.is_empty() {
 			assert_eq!(ret[0].1, [0u8; 32].into());
 		}
 
 		ret
 	}
 
-	pub fn walk_ring(&self, from: &Hash, n: usize) -> Vec<UUID> {
+	/// Walk the ring to find the n servers in which data should be replicated
+	pub fn get_nodes(&self, position: &Hash, n: usize) -> Vec<Uuid> {
 		if self.ring.len() != 1 << PARTITION_BITS {
 			warn!("Ring not yet ready, read/writes will be lost!");
 			return vec![];
 		}
 
-		let top = u16::from_be_bytes(from.as_slice()[0..2].try_into().unwrap());
-		let partition_idx = (top >> (16 - PARTITION_BITS)) as usize;
-		assert_eq!(partition_idx, self.partition_of(from) as usize);
-
+		let partition_idx = self.partition_of(position) as usize;
 		let partition = &self.ring[partition_idx];
 
-		let partition_top =
-			u16::from_be_bytes(partition.location.as_slice()[0..2].try_into().unwrap());
-		assert_eq!(partition_top & PARTITION_MASK_U16, top & PARTITION_MASK_U16);
+		let top = u16::from_be_bytes(position.as_slice()[0..2].try_into().unwrap());
+		// Check that we haven't messed up our partition table, i.e. that this partition
+		// table entrey indeed corresponds to the item we are storing
+		assert_eq!(
+			partition.hash_prefix & PARTITION_MASK_U16,
+			top & PARTITION_MASK_U16
+		);
 
-		assert!(n <= partition.nodes.len());
-		partition.nodes[..n].iter().cloned().collect::<Vec<_>>()
+		assert!(n <= self.replication_factor);
+		partition.nodes_buf[..n]
+			.iter()
+			.map(|i| self.nodes[*i as usize])
+			.collect::<Vec<_>>()
+	}
+}
+
+#[cfg(test)]
+mod tests {
+	use super::*;
+
+	#[test]
+	fn test_ring_entry_size() {
+		assert_eq!(std::mem::size_of::<RingEntry>(), 8);
 	}
 }