Fix S3 ListObjects result and replace println!s by debug!s

Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/7

Cargo.toml

@@ -5,6 +5,7 @@ members = [
 	"src/table",
 	"src/model",
 	"src/api",
+	"src/web",
 	"src/garage",
 ]
 

Makefile

@@ -3,10 +3,12 @@ DOCKER=lxpz/garage_amd64
 
 all:
 	#cargo fmt || true
-	RUSTFLAGS="-C link-arg=-fuse-ld=lld" cargo build
+	#RUSTFLAGS="-C link-arg=-fuse-ld=lld" cargo build
+	cargo build
 
 $(BIN):
-	RUSTFLAGS="-C link-arg=-fuse-ld=lld" cargo build --release
+	#RUSTFLAGS="-C link-arg=-fuse-ld=lld" cargo build --release
+	cargo build --release
 
 $(BIN).stripped: $(BIN)
 	cp $^ $@

README.md

@@ -16,6 +16,28 @@ Non-goals include:
 
 Our main use case is to provide a distributed storage layer for small-scale self hosted services such as [Deuxfleurs](https://deuxfleurs.fr).
 
+## Development
+
+We propose the following quickstart to setup a full dev. environment as quickly as possible:
+
+  1. Setup a rust/cargo environment. eg. `dnf install rust cargo`
+  2. Install awscli v2 by following the guide [here](https://docs.aws.amazon.com/cli/latest/userguide/install-cliv2.html).
+  3. Run `cargo build` to build the project
+  4. Run `./script/dev-cluster.sh` to launch a test cluster (feel free to read the script)
+  5. Run `./script/dev-configure.sh` to configure your test cluster with default values (same datacenter, 100 tokens)
+  6. Run `./script/dev-bucket.sh` to create a bucket named `eprouvette` and an API key that will be stored in `/tmp/garage.s3`
+  7. Run `source ./script/dev-env-aws.sh` to configure your CLI environment
+  8. You can use `garage` to manage the cluster. Try `garage --help`.
+  9. You can use the `awsgrg` alias to add, remove, and delete files. Try `awsgrg help`, `awsgrg cp /proc/cpuinfo s3://eprouvette/cpuinfo.txt`, or `awsgrg ls s3://eprouvette`. `awsgrg` is a wrapper on the `aws s3` command pre-configured with the previously generated API key (the one in `/tmp/garage.s3`) and localhost as the endpoint.
+
+Now you should be ready to start hacking on garage!
+
+## S3 compatibility
+
+Only a subset of S3 is supported: adding, listing, getting and deleting files in a bucket.
+Bucket management, ACL and other advanced features are not (yet?) handled through the S3 API but through the `garage` CLI.
+We primarily test `garage` against the `awscli` tool and `nextcloud`.
+
 ## Setting up Garage
 
 Use the `genkeys.sh` script to generate TLS keys for encrypting communications between Garage nodes.
@@ -67,7 +89,12 @@ node_key = "/path/to/garage/pki/garage.key"
 
 [s3_api]
 api_bind_addr = "[::1]:3900"	# the S3 API port, HTTP without TLS. Add a reverse proxy for the TLS part.
-region = "garage"				# set this to anything. S3 API calls will fail if they are not made against the region set here.
+s3_region = "garage"				# set this to anything. S3 API calls will fail if they are not made against the region set here.
+
+[s3_web]
+bind_addr = "[::1]:3902"
+root_domain = ".garage.tld"
+index = "index.html"
 ```
 
 Build Garage using `cargo build --release`.

config.dev.toml

@@ -0,0 +1,22 @@
+block_size = 1048576			# objects are split in blocks of maximum this number of bytes
+
+metadata_dir = "/tmp/garage-meta"
+data_dir = "/tmp/garage-data"
+
+rpc_bind_addr = "[::]:3901"		# the port other Garage nodes will use to talk to this node
+
+bootstrap_peers = []
+
+max_concurrent_rpc_requests = 12
+data_replication_factor = 3
+meta_replication_factor = 3
+meta_epidemic_fanout = 3
+
+[s3_api]
+api_bind_addr = "[::1]:3900"	# the S3 API port, HTTP without TLS. Add a reverse proxy for the TLS part.
+s3_region = "garage"				# set this to anything. S3 API calls will fail if they are not made against the region set here.
+
+[s3_web]
+bind_addr = "[::1]:3902"
+root_domain = ".garage.tld"
+index = "index.html"

doc/20201202_talk/.gitignore

@@ -0,0 +1,12 @@
+*
+
+!img
+
+!.gitignore
+!*.svg
+!*.png
+!*.jpg
+!*.tex
+!Makefile
+!.gitignore
+!talk.pdf

doc/20201202_talk/Makefile

@@ -0,0 +1,6 @@
+talk.pdf: talk.tex img/garage_distributed.pdf img/consistent_hashing_1.pdf img/consistent_hashing_2.pdf img/consistent_hashing_3.pdf img/consistent_hashing_4.pdf img/garage_tables.pdf
+	pdflatex talk.tex
+	
+img/%.pdf: img/%.svg
+	inkscape -D -z --file=$^ --export-pdf=$@
+

doc/20201202_talk/img/garage_distributed.svg

@@ -0,0 +1,404 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<svg
+   xmlns:dc="http://purl.org/dc/elements/1.1/"
+   xmlns:cc="http://creativecommons.org/ns#"
+   xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+   xmlns:svg="http://www.w3.org/2000/svg"
+   xmlns="http://www.w3.org/2000/svg"
+   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
+   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
+   width="480"
+   height="480"
+   viewBox="0 0 127 127"
+   version="1.1"
+   id="svg8"
+   inkscape:version="1.0.1 (3bc2e813f5, 2020-09-07)"
+   sodipodi:docname="garage_distributed.svg">
+  <defs
+     id="defs2" />
+  <sodipodi:namedview
+     id="base"
+     pagecolor="#ffffff"
+     bordercolor="#666666"
+     borderopacity="1.0"
+     inkscape:pageopacity="0.0"
+     inkscape:pageshadow="2"
+     inkscape:zoom="1.979899"
+     inkscape:cx="171.34852"
+     inkscape:cy="170.69443"
+     inkscape:document-units="mm"
+     inkscape:current-layer="layer1-3"
+     inkscape:document-rotation="0"
+     showgrid="false"
+     units="px"
+     inkscape:window-width="1404"
+     inkscape:window-height="1016"
+     inkscape:window-x="103"
+     inkscape:window-y="27"
+     inkscape:window-maximized="0" />
+  <metadata
+     id="metadata5">
+    <rdf:RDF>
+      <cc:Work
+         rdf:about="">
+        <dc:format>image/svg+xml</dc:format>
+        <dc:type
+           rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
+        <dc:title />
+      </cc:Work>
+    </rdf:RDF>
+  </metadata>
+  <g
+     inkscape:label="Layer 1"
+     inkscape:groupmode="layer"
+     id="layer1">
+    <g
+       inkscape:label="Layer 1"
+       id="layer1-3"
+       transform="matrix(0.42851498,0,0,0.42851498,24.079728,-24.925134)">
+      <path
+         style="fill:none;stroke:#000000;stroke-width:2.065;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
+         d="M 66.78016,80.71889 99.921832,61.598165 132.84481,80.509232 V 127.38418 H 66.701651 Z"
+         id="path124"
+         sodipodi:nodetypes="cccccc" />
+      <g
+         id="g1106-5"
+         transform="matrix(0,0.95201267,-0.95201267,0,194.01664,-57.627274)"
+         style="stroke-width:2.17959;stroke-miterlimit:4;stroke-dasharray:none">
+        <g
+           id="g1061-3"
+           style="stroke-width:2.17959;stroke-miterlimit:4;stroke-dasharray:none">
+          <circle
+             style="fill:none;stroke:#000000;stroke-width:2.17959;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
+             id="path956-5"
+             cx="168.8569"
+             cy="92.889587"
+             r="13.125794" />
+          <circle
+             style="fill:none;stroke:#000000;stroke-width:2.17959;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
+             id="path958-6"
+             cx="168.77444"
+             cy="92.702293"
+             r="3.0778286" />
+          <path
+             id="path960-2"
+             style="fill:none;stroke:#000000;stroke-width:2.17959;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
+             d="m 169.46072,82.84435 c 4.95795,0.336608 8.87296,4.341959 9.09638,9.306301"
+             sodipodi:nodetypes="cc" />
+        </g>
+        <path
+           style="fill:none;stroke:#000000;stroke-width:2.17959;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
+           d="m 154.67824,112.84018 11.89881,-13.038071 c 1.46407,-1.552664 3.79541,0.878511 2.81832,2.089181 l -10.57965,14.481 c -1.8851,2.02632 -6.10786,-1.06119 -4.13748,-3.53211 z"
+           id="path964-9"
+           sodipodi:nodetypes="ccccc" />
+        <g
+           id="g1071-1"
+           style="stroke-width:2.17959;stroke-miterlimit:4;stroke-dasharray:none" />
+        <g
+           id="g1065-3"
+           style="stroke-width:2.17959;stroke-miterlimit:4;stroke-dasharray:none">
+          <rect
+             style="fill:none;stroke:#000000;stroke-width:2.17959;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
+             id="rect949-6"
+             width="35.576611"
+             height="48.507355"
+             x="150.9623"
+             y="74.698929"
+             ry="2.7302756" />
+          <path
+             style="fill:none;stroke:#000000;stroke-width:2.17959;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
+             d="m 150.76919,106.16944 6.36181,-0.0223 c 2.53845,3.46232 6.29787,4.20243 10.1055,4.40362 l 0.0176,13.09251"
+             id="path1033-0"
+             sodipodi:nodetypes="cccc" />
+        </g>
+      </g>
+    </g>
+    <g
+       inkscape:label="Layer 1"
+       id="layer1-3-5"
+       transform="matrix(0.42851499,0,0,0.42851499,68.181495,12.180995)">
+      <path
+         style="fill:none;stroke:#000000;stroke-width:2.065;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
+         d="M 66.78016,73.340623 99.921832,54.219898 132.84481,73.130965 V 120.00591 H 66.701651 Z"
+         id="path124-6"
+         sodipodi:nodetypes="cccccc" />
+      <g
+         id="g1106-5-2"
+         transform="matrix(0,0.95201267,-0.95201267,0,194.01664,-65.058377)"
+         style="stroke-width:2.17959;stroke-miterlimit:4;stroke-dasharray:none">
+        <g
+           id="g1061-3-9"
+           style="stroke-width:2.17959;stroke-miterlimit:4;stroke-dasharray:none">
+          <circle
+             style="fill:none;stroke:#000000;stroke-width:2.17959;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
+             id="path956-5-1"
+             cx="168.8569"
+             cy="92.889587"
+             r="13.125794" />
+          <circle
+             style="fill:none;stroke:#000000;stroke-width:2.17959;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
+             id="path958-6-2"
+             cx="168.77444"
+             cy="92.702293"
+             r="3.0778286" />
+          <path
+             id="path960-2-7"
+             style="fill:none;stroke:#000000;stroke-width:2.17959;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
+             d="m 169.46072,82.84435 c 4.95795,0.336608 8.87296,4.341959 9.09638,9.306301"
+             sodipodi:nodetypes="cc" />
+        </g>
+        <path
+           style="fill:none;stroke:#000000;stroke-width:2.17959;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
+           d="m 154.67824,112.84018 11.89881,-13.038071 c 1.46407,-1.552664 3.79541,0.878511 2.81832,2.089181 l -10.57965,14.481 c -1.8851,2.02632 -6.10786,-1.06119 -4.13748,-3.53211 z"
+           id="path964-9-0"
+           sodipodi:nodetypes="ccccc" />
+        <g
+           id="g1071-1-9"
+           style="stroke-width:2.17959;stroke-miterlimit:4;stroke-dasharray:none" />
+        <g
+           id="g1065-3-3"
+           style="stroke-width:2.17959;stroke-miterlimit:4;stroke-dasharray:none">
+          <rect
+             style="fill:none;stroke:#000000;stroke-width:2.17959;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
+             id="rect949-6-6"
+             width="35.576611"
+             height="48.507355"
+             x="150.9623"
+             y="74.698929"
+             ry="2.7302756" />
+          <path
+             style="fill:none;stroke:#000000;stroke-width:2.17959;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
+             d="m 150.76919,106.16944 6.36181,-0.0223 c 2.53845,3.46232 6.29787,4.20243 10.1055,4.40362 l 0.0176,13.09251"
+             id="path1033-0-0"
+             sodipodi:nodetypes="cccc" />
+        </g>
+      </g>
+    </g>
+    <g
+       inkscape:label="Layer 1"
+       id="layer1-3-6"
+       transform="matrix(0.42851499,0,0,0.42851499,-20.953301,19.351613)">
+      <path
+         style="fill:none;stroke:#000000;stroke-width:2.065;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
+         d="M 66.78016,73.340623 99.921832,54.219898 132.84481,73.130965 V 120.00591 H 66.701651 Z"
+         id="path124-2"
+         sodipodi:nodetypes="cccccc" />
+      <g
+         id="g1106-5-6"
+         transform="matrix(0,0.95201267,-0.95201267,0,194.01664,-65.058377)"
+         style="stroke-width:2.17959;stroke-miterlimit:4;stroke-dasharray:none">
+        <g
+           id="g1061-3-1"
+           style="stroke-width:2.17959;stroke-miterlimit:4;stroke-dasharray:none">
+          <circle
+             style="fill:none;stroke:#000000;stroke-width:2.17959;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
+             id="path956-5-8"
+             cx="168.8569"
+             cy="92.889587"
+             r="13.125794" />
+          <circle
+             style="fill:none;stroke:#000000;stroke-width:2.17959;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
+             id="path958-6-7"
+             cx="168.77444"
+             cy="92.702293"
+             r="3.0778286" />
+          <path
+             id="path960-2-9"
+             style="fill:none;stroke:#000000;stroke-width:2.17959;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
+             d="m 169.46072,82.84435 c 4.95795,0.336608 8.87296,4.341959 9.09638,9.306301"
+             sodipodi:nodetypes="cc" />
+        </g>
+        <path
+           style="fill:none;stroke:#000000;stroke-width:2.17959;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
+           d="m 154.67824,112.84018 11.89881,-13.038071 c 1.46407,-1.552664 3.79541,0.878511 2.81832,2.089181 l -10.57965,14.481 c -1.8851,2.02632 -6.10786,-1.06119 -4.13748,-3.53211 z"
+           id="path964-9-2"
+           sodipodi:nodetypes="ccccc" />
+        <g
+           id="g1071-1-0"
+           style="stroke-width:2.17959;stroke-miterlimit:4;stroke-dasharray:none" />
+        <g
+           id="g1065-3-2"
+           style="stroke-width:2.17959;stroke-miterlimit:4;stroke-dasharray:none">
+          <rect
+             style="fill:none;stroke:#000000;stroke-width:2.17959;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
+             id="rect949-6-3"
+             width="35.576611"
+             height="48.507355"
+             x="150.9623"
+             y="74.698929"
+             ry="2.7302756" />
+          <path
+             style="fill:none;stroke:#000000;stroke-width:2.17959;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
+             d="m 150.76919,106.16944 6.36181,-0.0223 c 2.53845,3.46232 6.29787,4.20243 10.1055,4.40362 l 0.0176,13.09251"
+             id="path1033-0-7"
+             sodipodi:nodetypes="cccc" />
+        </g>
+      </g>
+    </g>
+    <g
+       inkscape:label="Layer 1"
+       id="layer1-3-59"
+       transform="matrix(0.42851499,0,0,0.42851499,51.949789,75.218277)">
+      <path
+         style="fill:none;stroke:#000000;stroke-width:2.065;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
+         d="M 66.78016,73.340623 99.921832,54.219898 132.84481,73.130965 V 120.00591 H 66.701651 Z"
+         id="path124-22"
+         sodipodi:nodetypes="cccccc" />
+      <g
+         id="g1106-5-8"
+         transform="matrix(0,0.95201267,-0.95201267,0,194.01664,-65.058377)"
+         style="stroke-width:2.17959;stroke-miterlimit:4;stroke-dasharray:none">
+        <g
+           id="g1061-3-97"
+           style="stroke-width:2.17959;stroke-miterlimit:4;stroke-dasharray:none">
+          <circle
+             style="fill:none;stroke:#000000;stroke-width:2.17959;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
+             id="path956-5-3"
+             cx="168.8569"
+             cy="92.889587"
+             r="13.125794" />
+          <circle
+             style="fill:none;stroke:#000000;stroke-width:2.17959;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
+             id="path958-6-6"
+             cx="168.77444"
+             cy="92.702293"
+             r="3.0778286" />
+          <path
+             id="path960-2-1"
+             style="fill:none;stroke:#000000;stroke-width:2.17959;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
+             d="m 169.46072,82.84435 c 4.95795,0.336608 8.87296,4.341959 9.09638,9.306301"
+             sodipodi:nodetypes="cc" />
+        </g>
+        <path
+           style="fill:none;stroke:#000000;stroke-width:2.17959;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
+           d="m 154.67824,112.84018 11.89881,-13.038071 c 1.46407,-1.552664 3.79541,0.878511 2.81832,2.089181 l -10.57965,14.481 c -1.8851,2.02632 -6.10786,-1.06119 -4.13748,-3.53211 z"
+           id="path964-9-29"
+           sodipodi:nodetypes="ccccc" />
+        <g
+           id="g1071-1-3"
+           style="stroke-width:2.17959;stroke-miterlimit:4;stroke-dasharray:none" />
+        <g
+           id="g1065-3-1"
+           style="stroke-width:2.17959;stroke-miterlimit:4;stroke-dasharray:none">
+          <rect
+             style="fill:none;stroke:#000000;stroke-width:2.17959;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
+             id="rect949-6-9"
+             width="35.576611"
+             height="48.507355"
+             x="150.9623"
+             y="74.698929"
+             ry="2.7302756" />
+          <path
+             style="fill:none;stroke:#000000;stroke-width:2.17959;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
+             d="m 150.76919,106.16944 6.36181,-0.0223 c 2.53845,3.46232 6.29787,4.20243 10.1055,4.40362 l 0.0176,13.09251"
+             id="path1033-0-4"
+             sodipodi:nodetypes="cccc" />
+        </g>
+      </g>
+    </g>
+    <g
+       inkscape:label="Layer 1"
+       id="layer1-3-7"
+       transform="matrix(0.42851499,0,0,0.42851499,-1.173447,75.150288)">
+      <path
+         style="fill:none;stroke:#000000;stroke-width:2.065;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
+         d="M 66.78016,73.340623 99.921832,54.219898 132.84481,73.130965 V 120.00591 H 66.701651 Z"
+         id="path124-8"
+         sodipodi:nodetypes="cccccc" />
+      <g
+         id="g1106-5-4"
+         transform="matrix(0,0.95201267,-0.95201267,0,194.01664,-65.058377)"
+         style="stroke-width:2.17959;stroke-miterlimit:4;stroke-dasharray:none">
+        <g
+           id="g1061-3-5"
+           style="stroke-width:2.17959;stroke-miterlimit:4;stroke-dasharray:none">
+          <circle
+             style="fill:none;stroke:#000000;stroke-width:2.17959;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
+             id="path956-5-0"
+             cx="168.8569"
+             cy="92.889587"
+             r="13.125794" />
+          <circle
+             style="fill:none;stroke:#000000;stroke-width:2.17959;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
+             id="path958-6-3"
+             cx="168.77444"
+             cy="92.702293"
+             r="3.0778286" />
+          <path
+             id="path960-2-6"
+             style="fill:none;stroke:#000000;stroke-width:2.17959;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
+             d="m 169.46072,82.84435 c 4.95795,0.336608 8.87296,4.341959 9.09638,9.306301"
+             sodipodi:nodetypes="cc" />
+        </g>
+        <path
+           style="fill:none;stroke:#000000;stroke-width:2.17959;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
+           d="m 154.67824,112.84018 11.89881,-13.038071 c 1.46407,-1.552664 3.79541,0.878511 2.81832,2.089181 l -10.57965,14.481 c -1.8851,2.02632 -6.10786,-1.06119 -4.13748,-3.53211 z"
+           id="path964-9-1"
+           sodipodi:nodetypes="ccccc" />
+        <g
+           id="g1071-1-06"
+           style="stroke-width:2.17959;stroke-miterlimit:4;stroke-dasharray:none" />
+        <g
+           id="g1065-3-32"
+           style="stroke-width:2.17959;stroke-miterlimit:4;stroke-dasharray:none">
+          <rect
+             style="fill:none;stroke:#000000;stroke-width:2.17959;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
+             id="rect949-6-0"
+             width="35.576611"
+             height="48.507355"
+             x="150.9623"
+             y="74.698929"
+             ry="2.7302756" />
+          <path
+             style="fill:none;stroke:#000000;stroke-width:2.17959;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
+             d="m 150.76919,106.16944 6.36181,-0.0223 c 2.53845,3.46232 6.29787,4.20243 10.1055,4.40362 l 0.0176,13.09251"
+             id="path1033-0-6"
+             sodipodi:nodetypes="cccc" />
+        </g>
+      </g>
+    </g>
+    <path
+       style="fill:none;stroke:#000000;stroke-width:1;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
+       d="M 35.21897,43.254452 46.803736,32.872178"
+       id="path1045" />
+    <path
+       style="fill:none;stroke:#000000;stroke-width:1;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
+       d="m 85.798392,29.613721 10.944185,7.688225"
+       id="path1047" />
+    <path
+       style="fill:none;stroke:#000000;stroke-width:1;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
+       d="m 107.59813,71.879386 -6.2564,22.552649"
+       id="path1049" />
+    <path
+       style="fill:none;stroke:#000000;stroke-width:1;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
+       d="M 75.866769,119.14997 61.529058,118.74136"
+       id="path1051"
+       sodipodi:nodetypes="cc" />
+    <path
+       style="fill:none;stroke:#000000;stroke-width:1;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
+       d="M 29.414211,98.256475 C 29.681482,96.462435 21.07721,77.446418 21.07721,77.446418"
+       id="path1053" />
+    <path
+       style="fill:none;stroke:#000000;stroke-width:1;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
+       d="M 39.447822,61.341585 90.641428,57.562618"
+       id="path1055" />
+    <path
+       style="fill:none;stroke:#000000;stroke-width:1;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
+       d="M 90.440176,64.423751 54.180736,100.02908"
+       id="path1057"
+       sodipodi:nodetypes="cc" />
+    <path
+       style="fill:none;stroke:#000000;stroke-width:1;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
+       d="M 47.163557,96.532205 61.535331,33.078667"
+       id="path1059"
+       sodipodi:nodetypes="cc" />
+    <path
+       style="fill:none;stroke:#000000;stroke-width:1;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
+       d="m 71.396211,33.058731 15.77285,60.595014"
+       id="path1061" />
+    <path
+       style="fill:none;stroke:#000000;stroke-width:1;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
+       d="M 79.384641,100.96895 41.150775,67.902625"
+       id="path1063" />
+  </g>
+</svg>

doc/20201202_talk/img/garage_tables.svg

@@ -0,0 +1,502 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<svg
+   xmlns:dc="http://purl.org/dc/elements/1.1/"
+   xmlns:cc="http://creativecommons.org/ns#"
+   xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+   xmlns:svg="http://www.w3.org/2000/svg"
+   xmlns="http://www.w3.org/2000/svg"
+   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
+   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
+   width="850"
+   height="480"
+   viewBox="0 0 224.89584 127"
+   version="1.1"
+   id="svg8"
+   inkscape:version="1.0.1 (3bc2e813f5, 2020-09-07)"
+   sodipodi:docname="garage_tables.svg">
+  <defs
+     id="defs2">
+    <marker
+       style="overflow:visible"
+       id="marker1262"
+       refX="0"
+       refY="0"
+       orient="auto"
+       inkscape:stockid="Arrow1Mend"
+       inkscape:isstock="true">
+      <path
+         transform="matrix(-0.4,0,0,-0.4,-4,0)"
+         style="fill:#000000;fill-opacity:1;fill-rule:evenodd;stroke:#000000;stroke-width:1pt;stroke-opacity:1"
+         d="M 0,0 5,-5 -12.5,0 5,5 Z"
+         id="path1260" />
+    </marker>
+    <marker
+       style="overflow:visible"
+       id="Arrow1Mend"
+       refX="0"
+       refY="0"
+       orient="auto"
+       inkscape:stockid="Arrow1Mend"
+       inkscape:isstock="true"
+       inkscape:collect="always">
+      <path
+         transform="matrix(-0.4,0,0,-0.4,-4,0)"
+         style="fill:#000000;fill-opacity:1;fill-rule:evenodd;stroke:#000000;stroke-width:1pt;stroke-opacity:1"
+         d="M 0,0 5,-5 -12.5,0 5,5 Z"
+         id="path965" />
+    </marker>
+    <marker
+       style="overflow:visible"
+       id="Arrow1Lend"
+       refX="0"
+       refY="0"
+       orient="auto"
+       inkscape:stockid="Arrow1Lend"
+       inkscape:isstock="true">
+      <path
+         transform="matrix(-0.8,0,0,-0.8,-10,0)"
+         style="fill:#000000;fill-opacity:1;fill-rule:evenodd;stroke:#000000;stroke-width:1pt;stroke-opacity:1"
+         d="M 0,0 5,-5 -12.5,0 5,5 Z"
+         id="path959" />
+    </marker>
+  </defs>
+  <sodipodi:namedview
+     id="base"
+     pagecolor="#ffffff"
+     bordercolor="#666666"
+     borderopacity="1.0"
+     inkscape:pageopacity="0.0"
+     inkscape:pageshadow="2"
+     inkscape:zoom="0.98994949"
+     inkscape:cx="381.09221"
+     inkscape:cy="219.5592"
+     inkscape:document-units="mm"
+     inkscape:current-layer="layer1"
+     inkscape:document-rotation="0"
+     showgrid="false"
+     units="px"
+     inkscape:window-width="1867"
+     inkscape:window-height="1016"
+     inkscape:window-x="53"
+     inkscape:window-y="27"
+     inkscape:window-maximized="1" />
+  <metadata
+     id="metadata5">
+    <rdf:RDF>
+      <cc:Work
+         rdf:about="">
+        <dc:format>image/svg+xml</dc:format>
+        <dc:type
+           rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
+        <dc:title></dc:title>
+      </cc:Work>
+    </rdf:RDF>
+  </metadata>
+  <g
+     inkscape:label="Layer 1"
+     inkscape:groupmode="layer"
+     id="layer1">
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:5.64444px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.264583"
+       x="39.570904"
+       y="38.452755"
+       id="text2025"><tspan
+         sodipodi:role="line"
+         id="tspan2023"
+         x="39.570904"
+         y="38.452755"
+         style="font-size:5.64444px;stroke-width:0.264583" /></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:10.5833px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.264583"
+       x="101.95796"
+       y="92.835831"
+       id="text2139"><tspan
+         sodipodi:role="line"
+         id="tspan2137"
+         x="101.95796"
+         y="92.835831"
+         style="stroke-width:0.264583"> </tspan></text>
+    <g
+       id="g2316"
+       transform="translate(-11.455511,1.5722486)">
+      <g
+         id="g2277">
+        <rect
+           style="fill:none;stroke:#000000;stroke-width:0.8;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1"
+           id="rect833"
+           width="47.419891"
+           height="95.353409"
+           x="18.534418"
+           y="24.42766" />
+        <rect
+           style="fill:none;stroke:#000000;stroke-width:0.799999;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1"
+           id="rect833-3"
+           width="47.419891"
+           height="86.973076"
+           x="18.534418"
+           y="32.807987" />
+        <text
+           xml:space="preserve"
+           style="font-style:normal;font-weight:normal;font-size:5.64444px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.264583"
+           x="32.250839"
+           y="29.894743"
+           id="text852"><tspan
+             sodipodi:role="line"
+             id="tspan850"
+             x="32.250839"
+             y="29.894743"
+             style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:5.64444px;font-family:'Liberation Mono';-inkscape-font-specification:'Liberation Mono Bold';stroke-width:0.264583">Object</tspan></text>
+      </g>
+      <g
+         id="g2066"
+         transform="translate(-2.1807817,-3.0621439)">
+        <g
+           id="g1969"
+           transform="matrix(0.12763631,0,0,0.12763631,0.7215051,24.717273)"
+           style="fill:#ff6600;fill-opacity:1;stroke:none;stroke-opacity:1">
+          <path
+             style="fill:#ff6600;fill-opacity:1;stroke:none;stroke-width:0.264583;stroke-opacity:1"
+             d="m 203.71837,154.80038 c -1.11451,3.75057 -2.45288,5.84095 -5.11132,7.98327 -2.2735,1.83211 -4.66721,2.65982 -8.09339,2.79857 -2.59227,0.10498 -2.92868,0.0577 -5.02863,-0.70611 -3.99215,-1.45212 -7.1627,-4.65496 -8.48408,-8.57046 -1.28374,-3.80398 -0.61478,-8.68216 1.64793,-12.01698 0.87317,-1.28689 3.15089,-3.48326 4.18771,-4.03815 l 0.53332,-28.51234 5.78454,-5.09197 6.95158,6.16704 -3.21112,3.49026 3.17616,3.45499 -3.17616,3.40822 2.98973,3.28645 -3.24843,3.3829 4.49203,4.58395 0.0516,5.69106 c 1.06874,0.64848 3.81974,3.24046 4.69548,4.56257 0.452,0.68241 1.06834,2.0197 1.36962,2.97176 0.62932,1.98864 0.88051,5.785 0.47342,7.15497 z m -10.0406,2.32604 c -0.88184,-3.17515 -4.92402,-3.78864 -6.75297,-1.02492 -0.58328,0.8814 -0.6898,1.28852 -0.58362,2.23056 0.26492,2.35041 2.45434,3.95262 4.60856,3.37255 1.19644,-0.32217 2.39435,-1.44872 2.72875,-2.56621 0.30682,-1.02529 0.30686,-0.9045 -7.9e-4,-2.01198 z"
+             id="path1971"
+             sodipodi:nodetypes="ssscsscccccccccccssscsssscc" />
+        </g>
+        <text
+           xml:space="preserve"
+           style="font-style:normal;font-weight:normal;font-size:5.64444px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.264583"
+           x="28.809687"
+           y="44.070885"
+           id="text852-9"><tspan
+             sodipodi:role="line"
+             id="tspan850-4"
+             x="28.809687"
+             y="44.070885"
+             style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:5.64444px;font-family:'Liberation Mono';-inkscape-font-specification:'Liberation Mono Bold';stroke-width:0.264583">bucket </tspan></text>
+      </g>
+      <g
+         id="g2066-7"
+         transform="translate(-2.1807817,6.2627616)">
+        <g
+           id="g1969-8"
+           transform="matrix(0.12763631,0,0,0.12763631,0.7215051,24.717273)"
+           style="fill:#ff6600;fill-opacity:1;stroke:none;stroke-opacity:1">
+          <path
+             style="fill:#ff6600;fill-opacity:1;stroke:none;stroke-width:0.264583;stroke-opacity:1"
+             d="m 203.71837,154.80038 c -1.11451,3.75057 -2.45288,5.84095 -5.11132,7.98327 -2.2735,1.83211 -4.66721,2.65982 -8.09339,2.79857 -2.59227,0.10498 -2.92868,0.0577 -5.02863,-0.70611 -3.99215,-1.45212 -7.1627,-4.65496 -8.48408,-8.57046 -1.28374,-3.80398 -0.61478,-8.68216 1.64793,-12.01698 0.87317,-1.28689 3.15089,-3.48326 4.18771,-4.03815 l 0.53332,-28.51234 5.78454,-5.09197 6.95158,6.16704 -3.21112,3.49026 3.17616,3.45499 -3.17616,3.40822 2.98973,3.28645 -3.24843,3.3829 4.49203,4.58395 0.0516,5.69106 c 1.06874,0.64848 3.81974,3.24046 4.69548,4.56257 0.452,0.68241 1.06834,2.0197 1.36962,2.97176 0.62932,1.98864 0.88051,5.785 0.47342,7.15497 z m -10.0406,2.32604 c -0.88184,-3.17515 -4.92402,-3.78864 -6.75297,-1.02492 -0.58328,0.8814 -0.6898,1.28852 -0.58362,2.23056 0.26492,2.35041 2.45434,3.95262 4.60856,3.37255 1.19644,-0.32217 2.39435,-1.44872 2.72875,-2.56621 0.30682,-1.02529 0.30686,-0.9045 -7.9e-4,-2.01198 z"
+             id="path1971-4"
+             sodipodi:nodetypes="ssscsscccccccccccssscsssscc" />
+        </g>
+        <text
+           xml:space="preserve"
+           style="font-style:normal;font-weight:normal;font-size:5.64444px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.264583"
+           x="28.809687"
+           y="44.070885"
+           id="text852-9-5"><tspan
+             sodipodi:role="line"
+             id="tspan850-4-0"
+             x="28.809687"
+             y="44.070885"
+             style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:5.64444px;font-family:'Liberation Mono';-inkscape-font-specification:'Liberation Mono Bold';stroke-width:0.264583">file path </tspan></text>
+      </g>
+      <g
+         id="g2161"
+         transform="translate(-62.264403,-59.333115)">
+        <g
+           id="g2271"
+           transform="translate(0,67.042823)">
+          <rect
+             style="fill:none;stroke:#000000;stroke-width:0.799999;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1"
+             id="rect833-6"
+             width="39.008453"
+             height="16.775949"
+             x="84.896881"
+             y="90.266838" />
+          <rect
+             style="fill:none;stroke:#000000;stroke-width:0.799999;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1"
+             id="rect833-3-1"
+             width="39.008453"
+             height="8.673645"
+             x="84.896881"
+             y="98.369141" />
+          <text
+             xml:space="preserve"
+             style="font-style:normal;font-weight:normal;font-size:5.64444px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.264583"
+             x="89.826942"
+             y="96.212921"
+             id="text852-0"><tspan
+               sodipodi:role="line"
+               id="tspan850-6"
+               x="89.826942"
+               y="96.212921"
+               style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:5.64444px;font-family:'Liberation Mono';-inkscape-font-specification:'Liberation Mono Bold';stroke-width:0.264583">Version 1</tspan></text>
+          <text
+             xml:space="preserve"
+             style="font-style:normal;font-weight:normal;font-size:5.64444px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.264583"
+             x="89.826942"
+             y="104.71013"
+             id="text852-0-3"><tspan
+               sodipodi:role="line"
+               id="tspan850-6-2"
+               x="89.826942"
+               y="104.71013"
+               style="font-style:italic;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:5.64444px;font-family:'Liberation Mono';-inkscape-font-specification:'Liberation Mono Bold';fill:#4d4d4d;stroke-width:0.264583">deleted</tspan></text>
+        </g>
+      </g>
+      <g
+         id="g2263"
+         transform="translate(0,-22.791204)">
+        <g
+           id="g2161-1"
+           transform="translate(-62.264403,-10.910843)">
+          <rect
+             style="fill:none;stroke:#000000;stroke-width:0.799999;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1"
+             id="rect833-6-5"
+             width="39.008453"
+             height="36.749603"
+             x="84.896881"
+             y="90.266838" />
+          <rect
+             style="fill:none;stroke:#000000;stroke-width:0.799999;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1"
+             id="rect833-3-1-5"
+             width="39.008453"
+             height="28.647301"
+             x="84.896881"
+             y="98.369141" />
+          <text
+             xml:space="preserve"
+             style="font-style:normal;font-weight:normal;font-size:5.64444px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.264583"
+             x="89.826942"
+             y="96.212921"
+             id="text852-0-4"><tspan
+               sodipodi:role="line"
+               id="tspan850-6-7"
+               x="89.826942"
+               y="96.212921"
+               style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:5.64444px;font-family:'Liberation Mono';-inkscape-font-specification:'Liberation Mono Bold';stroke-width:0.264583">Version 2</tspan></text>
+          <text
+             xml:space="preserve"
+             style="font-style:normal;font-weight:normal;font-size:5.64444px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.264583"
+             x="89.826942"
+             y="104.71013"
+             id="text852-0-3-6"><tspan
+               sodipodi:role="line"
+               id="tspan850-6-2-5"
+               x="89.826942"
+               y="104.71013"
+               style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:5.64444px;font-family:'Liberation Mono';-inkscape-font-specification:'Liberation Mono Bold';fill:#000000;stroke-width:0.264583">id</tspan></text>
+        </g>
+        <text
+           xml:space="preserve"
+           style="font-style:normal;font-weight:normal;font-size:5.64444px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.264583"
+           x="27.56254"
+           y="100.34132"
+           id="text852-0-3-6-6"><tspan
+             sodipodi:role="line"
+             id="tspan850-6-2-5-9"
+             x="27.56254"
+             y="100.34132"
+             style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:5.64444px;font-family:'Liberation Mono';-inkscape-font-specification:'Liberation Mono Bold';fill:#000000;stroke-width:0.264583">size</tspan></text>
+        <text
+           xml:space="preserve"
+           style="font-style:normal;font-weight:normal;font-size:5.64444px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.264583"
+           x="27.56254"
+           y="106.90263"
+           id="text852-0-3-6-6-3"><tspan
+             sodipodi:role="line"
+             id="tspan850-6-2-5-9-7"
+             x="27.56254"
+             y="106.90263"
+             style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:5.64444px;font-family:'Liberation Mono';-inkscape-font-specification:'Liberation Mono Bold';fill:#000000;stroke-width:0.264583">MIME type</tspan></text>
+        <text
+           xml:space="preserve"
+           style="font-style:normal;font-weight:normal;font-size:5.64444px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.264583"
+           x="27.56254"
+           y="111.92816"
+           id="text852-0-3-6-6-3-4"><tspan
+             sodipodi:role="line"
+             id="tspan850-6-2-5-9-7-5"
+             x="27.56254"
+             y="111.92816"
+             style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:5.64444px;font-family:'Liberation Mono';-inkscape-font-specification:'Liberation Mono Bold';fill:#000000;stroke-width:0.264583">...</tspan></text>
+      </g>
+    </g>
+    <g
+       id="g898"
+       transform="translate(-6.2484318,29.95006)">
+      <rect
+         style="fill:none;stroke:#000000;stroke-width:0.799999;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1"
+         id="rect833-7"
+         width="47.419891"
+         height="44.007515"
+         x="95.443573"
+         y="24.42766" />
+      <rect
+         style="fill:none;stroke:#000000;stroke-width:0.799999;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1"
+         id="rect833-3-4"
+         width="47.419891"
+         height="35.627186"
+         x="95.443573"
+         y="32.807987" />
+      <text
+         xml:space="preserve"
+         style="font-style:normal;font-weight:normal;font-size:5.64444px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.264583"
+         x="107.46638"
+         y="29.894743"
+         id="text852-4"><tspan
+           sodipodi:role="line"
+           id="tspan850-3"
+           x="107.46638"
+           y="29.894743"
+           style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:5.64444px;font-family:'Liberation Mono';-inkscape-font-specification:'Liberation Mono Bold';stroke-width:0.264583">Version</tspan></text>
+      <path
+         style="fill:#ff6600;fill-opacity:1;stroke:none;stroke-width:0.0337704;stroke-opacity:1"
+         d="m 102.90563,41.413279 c -0.14226,0.478709 -0.31308,0.745518 -0.65239,1.018956 -0.29019,0.233843 -0.59571,0.339489 -1.03301,0.357199 -0.33087,0.0134 -0.37381,0.0074 -0.64184,-0.09013 -0.50954,-0.185343 -0.914221,-0.594142 -1.082877,-1.093901 -0.163852,-0.485526 -0.07847,-1.108159 0.210335,-1.533803 0.111448,-0.164254 0.402172,-0.444591 0.534502,-0.515415 l 0.0681,-3.63921 0.73832,-0.64992 0.88727,0.787138 -0.40985,0.445484 0.40539,0.440982 -0.40539,0.435013 0.3816,0.41947 -0.41462,0.431781 0.57335,0.585078 0.007,0.726386 c 0.13641,0.08277 0.48753,0.413601 0.59931,0.58235 0.0577,0.0871 0.13636,0.257787 0.17481,0.379304 0.0803,0.253823 0.11239,0.738377 0.0604,0.913234 z m -1.28155,0.296888 c -0.11255,-0.405265 -0.62848,-0.483569 -0.86192,-0.130817 -0.0744,0.112498 -0.088,0.164461 -0.0745,0.2847 0.0338,0.299998 0.31326,0.504498 0.58822,0.43046 0.15271,-0.04112 0.3056,-0.184909 0.34828,-0.327542 0.0392,-0.130864 0.0392,-0.115447 -1e-4,-0.256801 z"
+         id="path1971-0"
+         sodipodi:nodetypes="ssscsscccccccccccssscsssscc" />
+      <text
+         xml:space="preserve"
+         style="font-style:normal;font-weight:normal;font-size:5.64444px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.264583"
+         x="104.99195"
+         y="41.008743"
+         id="text852-9-7"><tspan
+           sodipodi:role="line"
+           id="tspan850-4-8"
+           x="104.99195"
+           y="41.008743"
+           style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:5.64444px;font-family:'Liberation Mono';-inkscape-font-specification:'Liberation Mono Bold';stroke-width:0.264583">id </tspan></text>
+      <text
+         xml:space="preserve"
+         style="font-style:normal;font-weight:normal;font-size:5.64444px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.264583"
+         x="104.99195"
+         y="49.168018"
+         id="text852-9-7-6"><tspan
+           sodipodi:role="line"
+           id="tspan850-4-8-8"
+           x="104.99195"
+           y="49.168018"
+           style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:5.64444px;font-family:'Liberation Mono';-inkscape-font-specification:'Liberation Mono Bold';stroke-width:0.264583">h(block 1)</tspan></text>
+      <text
+         xml:space="preserve"
+         style="font-style:normal;font-weight:normal;font-size:5.64444px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.264583"
+         x="104.99195"
+         y="56.583336"
+         id="text852-9-7-6-8"><tspan
+           sodipodi:role="line"
+           id="tspan850-4-8-8-4"
+           x="104.99195"
+           y="56.583336"
+           style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:5.64444px;font-family:'Liberation Mono';-inkscape-font-specification:'Liberation Mono Bold';stroke-width:0.264583">h(block 2)</tspan></text>
+      <text
+         xml:space="preserve"
+         style="font-style:normal;font-weight:normal;font-size:5.64444px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.264583"
+         x="104.99195"
+         y="64.265732"
+         id="text852-9-7-6-3"><tspan
+           sodipodi:role="line"
+           id="tspan850-4-8-8-1"
+           x="104.99195"
+           y="64.265732"
+           style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:5.64444px;font-family:'Liberation Mono';-inkscape-font-specification:'Liberation Mono Bold';stroke-width:0.264583">...</tspan></text>
+    </g>
+    <g
+       id="g898-3"
+       transform="translate(75.777779,38.888663)">
+      <rect
+         style="fill:none;stroke:#000000;stroke-width:0.799999;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1"
+         id="rect833-7-6"
+         width="47.419891"
+         height="29.989157"
+         x="95.443573"
+         y="24.42766" />
+      <rect
+         style="fill:none;stroke:#000000;stroke-width:0.799999;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1"
+         id="rect833-3-4-7"
+         width="47.419891"
+         height="21.608831"
+         x="95.443573"
+         y="32.807987" />
+      <text
+         xml:space="preserve"
+         style="font-style:normal;font-weight:normal;font-size:5.64444px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.264583"
+         x="102.11134"
+         y="29.894743"
+         id="text852-4-5"><tspan
+           sodipodi:role="line"
+           id="tspan850-3-3"
+           x="102.11134"
+           y="29.894743"
+           style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:5.64444px;font-family:'Liberation Mono';-inkscape-font-specification:'Liberation Mono Bold';stroke-width:0.264583">Data block</tspan></text>
+      <path
+         style="fill:#ff6600;fill-opacity:1;stroke:none;stroke-width:0.0337704;stroke-opacity:1"
+         d="m 102.90563,41.413279 c -0.14226,0.478709 -0.31308,0.745518 -0.65239,1.018956 -0.29019,0.233843 -0.59571,0.339489 -1.03301,0.357199 -0.33087,0.0134 -0.37381,0.0074 -0.64184,-0.09013 -0.50954,-0.185343 -0.914221,-0.594142 -1.082877,-1.093901 -0.163852,-0.485526 -0.07847,-1.108159 0.210335,-1.533803 0.111448,-0.164254 0.402172,-0.444591 0.534502,-0.515415 l 0.0681,-3.63921 0.73832,-0.64992 0.88727,0.787138 -0.40985,0.445484 0.40539,0.440982 -0.40539,0.435013 0.3816,0.41947 -0.41462,0.431781 0.57335,0.585078 0.007,0.726386 c 0.13641,0.08277 0.48753,0.413601 0.59931,0.58235 0.0577,0.0871 0.13636,0.257787 0.17481,0.379304 0.0803,0.253823 0.11239,0.738377 0.0604,0.913234 z m -1.28155,0.296888 c -0.11255,-0.405265 -0.62848,-0.483569 -0.86192,-0.130817 -0.0744,0.112498 -0.088,0.164461 -0.0745,0.2847 0.0338,0.299998 0.31326,0.504498 0.58822,0.43046 0.15271,-0.04112 0.3056,-0.184909 0.34828,-0.327542 0.0392,-0.130864 0.0392,-0.115447 -1e-4,-0.256801 z"
+         id="path1971-0-5"
+         sodipodi:nodetypes="ssscsscccccccccccssscsssscc" />
+      <text
+         xml:space="preserve"
+         style="font-style:normal;font-weight:normal;font-size:5.64444px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.264583"
+         x="104.99195"
+         y="41.008743"
+         id="text852-9-7-62"><tspan
+           sodipodi:role="line"
+           id="tspan850-4-8-9"
+           x="104.99195"
+           y="41.008743"
+           style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:5.64444px;font-family:'Liberation Mono';-inkscape-font-specification:'Liberation Mono Bold';stroke-width:0.264583">hash </tspan></text>
+      <text
+         xml:space="preserve"
+         style="font-style:normal;font-weight:normal;font-size:5.64444px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.264583"
+         x="104.99195"
+         y="49.168018"
+         id="text852-9-7-6-1"><tspan
+           sodipodi:role="line"
+           id="tspan850-4-8-8-2"
+           x="104.99195"
+           y="49.168018"
+           style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:5.64444px;font-family:'Liberation Mono';-inkscape-font-specification:'Liberation Mono Bold';stroke-width:0.264583">data</tspan></text>
+    </g>
+    <path
+       style="fill:none;stroke:#000000;stroke-width:1;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1;marker-end:url(#Arrow1Mend)"
+       d="M 42.105292,69.455903 89.563703,69.317144"
+       id="path954"
+       sodipodi:nodetypes="cc" />
+    <path
+       style="fill:none;stroke:#000000;stroke-width:1;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1;marker-end:url(#marker1262)"
+       d="m 134.32612,77.363197 38.12618,0.260865"
+       id="path1258"
+       sodipodi:nodetypes="cc" />
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:5.64444px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.264583"
+       x="8.6727352"
+       y="16.687063"
+       id="text852-3"><tspan
+         sodipodi:role="line"
+         id="tspan850-67"
+         x="8.6727352"
+         y="16.687063"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:5.64444px;font-family:'Liberation Mono';-inkscape-font-specification:'Liberation Mono Bold';stroke-width:0.264583">Objects table </tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:5.64444px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.264583"
+       x="89.190445"
+       y="16.687063"
+       id="text852-3-5"><tspan
+         sodipodi:role="line"
+         id="tspan850-67-3"
+         x="89.190445"
+         y="16.687063"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:5.64444px;font-family:'Liberation Mono';-inkscape-font-specification:'Liberation Mono Bold';stroke-width:0.264583">Versions table </tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:5.64444px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.264583"
+       x="174.55702"
+       y="16.687063"
+       id="text852-3-56"><tspan
+         sodipodi:role="line"
+         id="tspan850-67-2"
+         x="174.55702"
+         y="16.687063"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:5.64444px;font-family:'Liberation Mono';-inkscape-font-specification:'Liberation Mono Bold';stroke-width:0.264583">Blocks table</tspan></text>
+  </g>
+</svg>

doc/20201202_talk/talk.tex

@@ -0,0 +1,247 @@
+%\nonstopmode
+\documentclass[aspectratio=169]{beamer}
+\usepackage[utf8]{inputenc}
+% \usepackage[frenchb]{babel}
+\usepackage{amsmath}
+\usepackage{mathtools}
+\usepackage{breqn}
+\usepackage{multirow}
+\usetheme{Luebeck}
+\usepackage{graphicx}
+%\useoutertheme[footline=authortitle,subsection=false]{miniframes}
+
+\beamertemplatenavigationsymbolsempty
+\setbeamertemplate{footline}
+{%
+  \leavevmode%
+  \hbox{\begin{beamercolorbox}[wd=.15\paperwidth,ht=2.5ex,dp=1.125ex,leftskip=.3cm,rightskip=.3cm plus1fill]{author in head/foot}%
+    \usebeamerfont{author in head/foot} \insertframenumber{} / \inserttotalframenumber
+  \end{beamercolorbox}%
+  \begin{beamercolorbox}[wd=.2\paperwidth,ht=2.5ex,dp=1.125ex,leftskip=.3cm plus1fill,rightskip=.3cm]{author in head/foot}%
+    \usebeamerfont{author in head/foot}\insertshortauthor
+  \end{beamercolorbox}%
+  \begin{beamercolorbox}[wd=.65\paperwidth,ht=2.5ex,dp=1.125ex,leftskip=.3cm,rightskip=.3cm plus1fil]{title in head/foot}%
+    \usebeamerfont{title in head/foot}\insertshorttitle~--~\insertshortdate
+  \end{beamercolorbox}}%
+  \vskip0pt%
+}
+
+\usepackage{tabu}
+\usepackage{multicol}
+\usepackage{vwcol}
+\usepackage{stmaryrd}
+\usepackage{graphicx}
+
+\usepackage[normalem]{ulem}
+
+\title[Garage : jouer dans la cour des grands quand on est un hébergeur associatif]{Garage : jouer dans la cour des grands \\quand on est un hébergeur associatif}
+\subtitle{(ou pourquoi on a décidé de réinventer la roue)}
+\author[Q. Dufour \& A. Auvolat]{Quentin Dufour \& Alex Auvolat}
+\date[02/12/2020]{Mercredi 2 décembre 2020}
+
+\begin{document}
+
+\begin{frame}
+	\titlepage
+\end{frame}
+
+\begin{frame}
+	\frametitle{La question qui tue}
+	
+	\begin{center}
+		\includegraphics[scale=3]{img/sync.png} \\
+		\Huge Pourquoi vous n'hébergez pas vos fichiers chez vous ? \\
+	\end{center}
+
+\end{frame}
+
+\begin{frame}[t]
+	\frametitle{La cour des grands}
+
+	\begin{columns}[t]
+	\begin{column}{0.5\textwidth}
+		{\huge Le modèle du cloud...}
+	 
+	\begin{center}
+		\includegraphics[scale=0.08]{img/cloud.png}
+	\end{center}
+	
+		+ \underline{intégrité} : plus de perte de données
+	
+		+ \underline{disponibilité} : tout le temps accessible
+		
+		+ \underline{service} : rien à gérer
+	
+		\vspace{0.15cm}
+		\textbf{changement des comportements}
+	\end{column}
+	\pause
+	\begin{column}{0.5\textwidth}
+		{\huge ...et son prix}
+	 
+	\begin{center}
+		\includegraphics[scale=0.07]{img/dc.jpg}
+	\end{center}
+	
+		- matériel couteux et polluant
+	
+		- logiciels secrets
+	
+		- gestion opaque
+	
+		\vspace{0.2cm}
+		\textbf{prisonnier de l'écosystème}
+	\end{column}
+	\end{columns}
+\end{frame}
+
+\begin{frame}[t]
+	\frametitle{Garage l'imposteur}
+
+	\begin{columns}[t]
+	\begin{column}{0.5\textwidth}
+		{\huge Ressemble à du cloud...}
+	 
+	\begin{center}
+		\includegraphics[scale=0.5]{img/shh.jpg}
+	\end{center}
+	
+	+ \underline{compatible} avec les apps existantes
+	
+	+ \underline{fonctionne} avec le mobile
+	
+	+ \underline{s'adapte} aux habitudes prises
+	
+	
+	\end{column}
+	
+	\pause
+	\begin{column}{0.5\textwidth}
+		{\huge ...fait du P2P}
+	
+	\begin{center}
+		\includegraphics[scale=1]{img/death.jpg}
+	\end{center}
+	
+	\vspace{0.4cm}
+	
+	+ \underline{contrôle} de l'infrastructure
+	
+	+ \underline{transparent} code libre 
+	
+	+ \underline{sobre} fonctionne avec de vieilles machines à la maison
+	\end{column}
+	\end{columns}
+
+\end{frame}
+
+
+\graphicspath{{img/}}
+
+\begin{frame}
+	\frametitle{Mais donc, c'est quoi Garage ?}
+
+	\begin{columns}[t]
+	\begin{column}{0.5\textwidth}
+		\centering
+		\textbf{Un système de stockage distribué}
+		\vspace{1em}
+
+			\includegraphics[width=.7\columnwidth]{img/garage_distributed.pdf}
+	\end{column}
+	\pause
+
+	\begin{column}{0.5\textwidth}
+		\centering
+		\textbf{qui implémente l'API S3}
+		\vspace{2em}
+
+		\includegraphics[width=.7\columnwidth]{img/Amazon-S3.jpg}
+	\end{column}
+	\end{columns}
+\end{frame}
+
+\begin{frame}
+	\frametitle{Consistent Hashing (DynamoDB)}
+	\textbf{Comment répartir les fichiers sur les différentes machines ?}
+	\vspace{1em}
+
+	\centering
+
+	\only<1>{\includegraphics[width=.55\columnwidth]{img/consistent_hashing_1.pdf}}%
+	\only<2>{\includegraphics[width=.55\columnwidth]{img/consistent_hashing_2.pdf}}%
+	\only<3>{\includegraphics[width=.55\columnwidth]{img/consistent_hashing_3.pdf}}%
+	\only<4>{\includegraphics[width=.55\columnwidth]{img/consistent_hashing_4.pdf}}%
+\end{frame}
+
+\begin{frame}
+	\frametitle{Garage Internals : 3 niveaux de consistent hashing}
+	\centering
+	\includegraphics[width=.85\columnwidth]{img/garage_tables.pdf}
+\end{frame}
+
+\begin{frame}
+	\frametitle{Modèles de cohérence}
+	Garage utilise un modèle de cohérence relativement faible :
+	\vspace{1em}
+
+	\begin{itemize}
+		\item Objets répliqués 3 fois, quorum de 2 pour les lectures et les écritures\\
+			$\to$ cohérence \textbf{``read your writes''}
+			\vspace{1em}
+		\item<2-> Types de donnée CRDT + mécanisme d'anti-entropie\\
+			$\to$ cohérence \textbf{à terme} (eventual consistency)
+			\vspace{1em}
+		\item<3-> Cela s'applique pour chaque fichier individuellement :\\
+			pas de linéarisabilté ou de cohérence causale entre les opérations\\
+			sur des fichiers différents
+			\vspace{1em}
+		\item<4-> \textbf{Avantage :} convient bien à un déploiement géodistribué (multi-datacenter)
+	\end{itemize}
+\end{frame}
+
+\begin{frame}
+	\frametitle{Rust : retour d'expérience}
+
+	\begin{columns}
+	\begin{column}{0.55\textwidth}
+		Garage est entièrement écrit en Rust !
+		\vspace{2em}
+
+		\textbf{Points forts :}
+		\vspace{.5em}
+		\begin{itemize}
+			\item Langage compilé, très rapide
+				\vspace{.5em}
+			\item Typage fort, beaucoup de sécurités
+				\vspace{.5em}
+			\item Le meilleur de plusieurs paradigmes:
+				fonctionnel, orienté objet, impératif
+				\vspace{.5em}
+			\item Un écosytème de librairies très complet:
+				serialisation, async/await, http, ...
+		\end{itemize}
+	\end{column}
+	
+	\begin{column}{0.45\textwidth}
+		\begin{centering}
+			\hspace{2em}\includegraphics[width=0.55\columnwidth]{img/rustacean-flat-happy.png}
+		\end{centering}
+
+		\vspace{2em}
+		\textbf{Points faibles :}
+		\vspace{.5em}
+		\begin{itemize}
+			\item Les temps de compilation...
+				\vspace{.5em}
+			\item Compliqué à apprendre
+		\end{itemize}
+		\vspace{2em}
+	\end{column}
+	\end{columns}
+
+\end{frame}
+
+\end{document}
+
+%% vim: set ts=4 sw=4 tw=0 noet spelllang=fr :

garage.svg

@@ -0,0 +1,119 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<svg
+   xmlns:dc="http://purl.org/dc/elements/1.1/"
+   xmlns:cc="http://creativecommons.org/ns#"
+   xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+   xmlns:svg="http://www.w3.org/2000/svg"
+   xmlns="http://www.w3.org/2000/svg"
+   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
+   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
+   width="108.2099mm"
+   height="108.00987mm"
+   viewBox="0 0 108.2099 108.00987"
+   version="1.1"
+   id="svg8"
+   inkscape:version="1.0.1 (3bc2e813f5, 2020-09-07)"
+   sodipodi:docname="garage.svg"
+   inkscape:export-filename="/home/lx/garage.png"
+   inkscape:export-xdpi="96"
+   inkscape:export-ydpi="96">
+  <defs
+     id="defs2" />
+  <sodipodi:namedview
+     id="base"
+     pagecolor="#ffffff"
+     bordercolor="#666666"
+     borderopacity="1.0"
+     inkscape:pageopacity="1"
+     inkscape:pageshadow="2"
+     inkscape:zoom="0.5"
+     inkscape:cx="-212.52783"
+     inkscape:cy="204.9547"
+     inkscape:document-units="mm"
+     inkscape:current-layer="layer1"
+     inkscape:document-rotation="0"
+     showgrid="false"
+     fit-margin-top="20"
+     fit-margin-left="20"
+     fit-margin-right="20"
+     fit-margin-bottom="20"
+     inkscape:window-width="1404"
+     inkscape:window-height="1016"
+     inkscape:window-x="103"
+     inkscape:window-y="27"
+     inkscape:window-maximized="0" />
+  <metadata
+     id="metadata5">
+    <rdf:RDF>
+      <cc:Work
+         rdf:about="">
+        <dc:format>image/svg+xml</dc:format>
+        <dc:type
+           rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
+        <dc:title></dc:title>
+      </cc:Work>
+    </rdf:RDF>
+  </metadata>
+  <g
+     inkscape:label="Layer 1"
+     inkscape:groupmode="layer"
+     id="layer1"
+     transform="translate(-45.667412,-33.028536)">
+    <path
+       style="fill:none;stroke:#000000;stroke-width:2.065;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
+       d="M 66.78016,73.340623 99.921832,54.219898 132.84481,73.130965 V 120.00591 H 66.701651 Z"
+       id="path124"
+       sodipodi:nodetypes="cccccc" />
+    <g
+       id="g1106-5"
+       transform="matrix(0,0.95201267,-0.95201267,0,194.01664,-65.058377)"
+       style="stroke-width:2.17959;stroke-miterlimit:4;stroke-dasharray:none">
+      <g
+         id="g1061-3"
+         style="stroke-width:2.17959;stroke-miterlimit:4;stroke-dasharray:none">
+        <circle
+           style="fill:none;stroke:#000000;stroke-width:2.17959;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
+           id="path956-5"
+           cx="168.8569"
+           cy="92.889587"
+           r="13.125794" />
+        <circle
+           style="fill:none;stroke:#000000;stroke-width:2.17959;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
+           id="path958-6"
+           cx="168.77444"
+           cy="92.702293"
+           r="3.0778286" />
+        <path
+           id="path960-2"
+           style="fill:none;stroke:#000000;stroke-width:2.17959;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
+           d="m 169.46072,82.84435 c 4.95795,0.336608 8.87296,4.341959 9.09638,9.306301"
+           sodipodi:nodetypes="cc" />
+      </g>
+      <path
+         style="fill:none;stroke:#000000;stroke-width:2.17959;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
+         d="m 154.67824,112.84018 11.89881,-13.038071 c 1.46407,-1.552664 3.79541,0.878511 2.81832,2.089181 l -10.57965,14.481 c -1.8851,2.02632 -6.10786,-1.06119 -4.13748,-3.53211 z"
+         id="path964-9"
+         sodipodi:nodetypes="ccccc" />
+      <g
+         id="g1071-1"
+         style="stroke-width:2.17959;stroke-miterlimit:4;stroke-dasharray:none" />
+      <g
+         id="g1065-3"
+         style="stroke-width:2.17959;stroke-miterlimit:4;stroke-dasharray:none">
+        <rect
+           style="fill:none;stroke:#000000;stroke-width:2.17959;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
+           id="rect949-6"
+           width="35.576611"
+           height="48.507355"
+           x="150.9623"
+           y="74.698929"
+           ry="2.7302756" />
+        <path
+           style="fill:none;stroke:#000000;stroke-width:2.17959;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
+           d="m 150.76919,106.16944 6.36181,-0.0223 c 2.53845,3.46232 6.29787,4.20243 10.1055,4.40362 l 0.0176,13.09251"
+           id="path1033-0"
+           sodipodi:nodetypes="cccc" />
+      </g>
+    </g>
+  </g>
+</svg>

script/dev-bucket.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+SCRIPT_FOLDER="`dirname \"$0\"`"
+REPO_FOLDER="${SCRIPT_FOLDER}/../"
+GARAGE_DEBUG="${REPO_FOLDER}/target/debug/"
+GARAGE_RELEASE="${REPO_FOLDER}/target/release/"
+PATH="${GARAGE_DEBUG}:${GARAGE_RELEASE}:$PATH"
+
+garage bucket create eprouvette
+KEY_INFO=`garage key new --name opérateur`
+ACCESS_KEY=`echo $KEY_INFO|grep -Po 'GK[a-f0-9]+'`
+SECRET_KEY=`echo $KEY_INFO|grep -Po 'secret_key: "[a-f0-9]+'|grep -Po '[a-f0-9]+$'`
+garage bucket allow eprouvette --read --write --key $ACCESS_KEY
+echo "$ACCESS_KEY $SECRET_KEY" > /tmp/garage.s3
+
+echo "Bucket s3://eprouvette created. Credentials stored in /tmp/garage.s3."

script/dev-clean.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+set -ex
+
+killall -9 garage || echo "garage is not running"
+rm -rf /tmp/garage*
+rm -rf /tmp/config.*.toml

script/dev-cluster.sh

@@ -0,0 +1,59 @@
+#!/bin/bash
+
+set -e
+
+SCRIPT_FOLDER="`dirname \"$0\"`"
+REPO_FOLDER="${SCRIPT_FOLDER}/../"
+GARAGE_DEBUG="${REPO_FOLDER}/target/debug/"
+GARAGE_RELEASE="${REPO_FOLDER}/target/release/"
+PATH="${GARAGE_DEBUG}:${GARAGE_RELEASE}:$PATH"
+FANCYCOLORS=("41m" "42m" "44m" "45m" "100m" "104m")
+
+export RUST_BACKTRACE=1 
+export RUST_LOG=garage=info,garage_api=debug
+MAIN_LABEL="\e[${FANCYCOLORS[0]}[main]\e[49m"
+
+WHICH_GARAGE=$(which garage || exit 1)
+echo -en "${MAIN_LABEL} Found garage at: ${WHICH_GARAGE}\n"
+
+for count in $(seq 1 3); do
+CONF_PATH="/tmp/config.$count.toml"
+LABEL="\e[${FANCYCOLORS[$count]}[$count]\e[49m"
+
+cat > $CONF_PATH <<EOF
+block_size = 1048576			# objects are split in blocks of maximum this number of bytes
+metadata_dir = "/tmp/garage-meta-$count"
+data_dir = "/tmp/garage-data-$count"
+rpc_bind_addr = "[::]:$((3900+$count))"		# the port other Garage nodes will use to talk to this node
+bootstrap_peers = [
+  "[::1]:3901",
+  "[::1]:3902",
+  "[::1]:3903"
+]
+max_concurrent_rpc_requests = 12
+data_replication_factor = 3
+meta_replication_factor = 3
+meta_epidemic_fanout = 3
+
+[s3_api]
+api_bind_addr = "[::]:$((3910+$count))"	# the S3 API port, HTTP without TLS. Add a reverse proxy for the TLS part.
+s3_region = "garage"				# set this to anything. S3 API calls will fail if they are not made against the region set here.
+
+[s3_web]
+bind_addr = "[::]:$((3920+$count))"
+root_domain = ".garage.tld"
+index = "index.html"
+EOF
+
+echo -en "$LABEL configuration written to $CONF_PATH\n"
+
+(garage server -c /tmp/config.$count.toml 2>&1|while read r; do echo -en "$LABEL $r\n"; done) &
+done
+
+until garage status 2>&1|grep -q Healthy ; do 
+	echo -en "${MAIN_LABEL} cluster starting...\n"
+	sleep 1
+done
+echo -en "${MAIN_LABEL} cluster started\n"
+
+wait

script/dev-configure.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+SCRIPT_FOLDER="`dirname \"$0\"`"
+REPO_FOLDER="${SCRIPT_FOLDER}/../"
+GARAGE_DEBUG="${REPO_FOLDER}/target/debug/"
+GARAGE_RELEASE="${REPO_FOLDER}/target/release/"
+PATH="${GARAGE_DEBUG}:${GARAGE_RELEASE}:$PATH"
+
+until garage status 2>&1|grep -q Healthy ; do 
+	echo "cluster starting..."
+	sleep 1
+done
+
+garage status \
+	| grep UNCONFIGURED \
+	| grep -Po '^[0-9a-f]+' \
+	| while read id; do 
+	  garage node configure -d dc1 -n 100 $id
+	done
+

script/dev-env-aws.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+SCRIPT_FOLDER="`dirname \"${BASH_SOURCE[0]}\"`"
+REPO_FOLDER="${SCRIPT_FOLDER}/../"
+GARAGE_DEBUG="${REPO_FOLDER}/target/debug/"
+GARAGE_RELEASE="${REPO_FOLDER}/target/release/"
+PATH="${GARAGE_DEBUG}:${GARAGE_RELEASE}:$PATH"
+
+export AWS_ACCESS_KEY_ID=`cat /tmp/garage.s3 |cut -d' ' -f1`
+export AWS_SECRET_ACCESS_KEY=`cat /tmp/garage.s3 |cut -d' ' -f2`
+export AWS_DEFAULT_REGION='garage'
+
+alias awsgrg="aws s3 \
+  --endpoint-url http://127.0.0.1:3911"

script/dev-env-s3cmd.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+SCRIPT_FOLDER="`dirname \"${BASH_SOURCE[0]}\"`"
+REPO_FOLDER="${SCRIPT_FOLDER}/../"
+GARAGE_DEBUG="${REPO_FOLDER}/target/debug/"
+GARAGE_RELEASE="${REPO_FOLDER}/target/release/"
+PATH="${GARAGE_DEBUG}:${GARAGE_RELEASE}:$PATH"
+
+ACCESS_KEY=`cat /tmp/garage.s3 |cut -d' ' -f1`
+SECRET_KEY=`cat /tmp/garage.s3 |cut -d' ' -f2`
+
+alias s3grg="s3cmd \
+  --host 127.0.0.1:3911 \
+  --host-bucket 127.0.0.1:3911 \
+  --access_key=$ACCESS_KEY \
+  --secret_key=$SECRET_KEY \
+  --region=garage \
+  --no-ssl"
+

script/test-smoke.sh

@@ -0,0 +1,76 @@
+#!/bin/bash
+
+set -ex
+shopt -s expand_aliases
+
+SCRIPT_FOLDER="`dirname \"$0\"`"
+REPO_FOLDER="${SCRIPT_FOLDER}/../"
+
+echo "setup"
+cargo build
+${SCRIPT_FOLDER}/dev-clean.sh
+${SCRIPT_FOLDER}/dev-cluster.sh > /tmp/garage.log 2>&1 &
+${SCRIPT_FOLDER}/dev-configure.sh
+${SCRIPT_FOLDER}/dev-bucket.sh
+source ${SCRIPT_FOLDER}/dev-env-aws.sh
+source ${SCRIPT_FOLDER}/dev-env-s3cmd.sh
+
+garage status
+garage key list
+garage bucket list
+
+dd if=/dev/urandom of=/tmp/garage.1.rnd bs=1k count=2 # < INLINE_THRESHOLD = 3072 bytes
+dd if=/dev/urandom of=/tmp/garage.2.rnd bs=1M count=5
+dd if=/dev/urandom of=/tmp/garage.3.rnd bs=1M count=10
+
+echo "s3 api testing..."
+for idx in $(seq 1 3); do
+  # AWS sends	
+  awsgrg cp /tmp/garage.$idx.rnd s3://eprouvette/garage.$idx.aws
+  
+  awsgrg ls s3://eprouvette
+  
+  awsgrg cp s3://eprouvette/garage.$idx.aws /tmp/garage.$idx.dl
+  diff /tmp/garage.$idx.rnd /tmp/garage.$idx.dl
+  rm /tmp/garage.$idx.dl
+  
+  s3grg get s3://eprouvette/garage.$idx.aws /tmp/garage.$idx.dl
+  diff /tmp/garage.$idx.rnd /tmp/garage.$idx.dl
+  rm /tmp/garage.$idx.dl
+  
+  awsgrg rm s3://eprouvette/garage.$idx.aws
+
+  # S3CMD sends
+  s3grg put /tmp/garage.$idx.rnd s3://eprouvette/garage.$idx.s3cmd
+
+  s3grg ls s3://eprouvette
+	
+  s3grg get s3://eprouvette/garage.$idx.s3cmd /tmp/garage.$idx.dl
+  diff /tmp/garage.$idx.rnd /tmp/garage.$idx.dl
+  rm /tmp/garage.$idx.dl
+  
+  awsgrg cp s3://eprouvette/garage.$idx.s3cmd /tmp/garage.$idx.dl
+  diff /tmp/garage.$idx.rnd /tmp/garage.$idx.dl
+  rm /tmp/garage.$idx.dl
+	
+  s3grg rm s3://eprouvette/garage.$idx.s3cmd
+done
+rm /tmp/garage.{1,2,3}.rnd
+
+echo "website testing"
+echo "<h1>hello world</h1>" > /tmp/garage-index.html
+awsgrg cp /tmp/garage-index.html s3://eprouvette/index.html
+[ `curl -s -o /dev/null -w "%{http_code}" --header "Host: eprouvette.garage.tld"  http://127.0.0.1:3923/ ` == 404 ]
+garage bucket website --allow eprouvette
+[ `curl -s -o /dev/null -w "%{http_code}" --header "Host: eprouvette.garage.tld"  http://127.0.0.1:3923/ ` == 200 ]
+garage bucket website --deny eprouvette
+[ `curl -s -o /dev/null -w "%{http_code}" --header "Host: eprouvette.garage.tld"  http://127.0.0.1:3923/ ` == 404 ]
+awsgrg rm s3://eprouvette/index.html
+rm /tmp/garage-index.html
+
+echo "teardown"
+garage bucket deny --read --write eprouvette --key $AWS_ACCESS_KEY_ID
+garage bucket delete --yes eprouvette
+garage key delete --yes $AWS_ACCESS_KEY_ID
+
+echo "success"

src/api/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "garage_api"
-version = "0.1.0"
+version = "0.1.1"
 authors = ["Alex Auvolat <alex@adnab.me>"]
 edition = "2018"
 license = "GPL-3.0"
@@ -13,24 +13,28 @@ path = "lib.rs"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-garage_util = { version = "0.1", path = "../util" }
-garage_table = { version = "0.1", path = "../table" }
-garage_model = { version = "0.1", path = "../model" }
+garage_util = { version = "0.1.1", path = "../util" }
+garage_table = { version = "0.1.1", path = "../table" }
+garage_model = { version = "0.1.1", path = "../model" }
 
+err-derive = "0.2.3"
 bytes = "0.4"
 hex = "0.3"
+base64 = "0.13"
 log = "0.4"
 chrono = "0.4"
+md-5 = "0.9.1"
 sha2 = "0.8"
 hmac = "0.7"
 crypto-mac = "0.7"
+rand = "0.7"
 
 futures = "0.3"
 futures-util = "0.3"
 tokio = { version = "0.2", default-features = false, features = ["rt-core", "rt-threaded", "io-driver", "net", "tcp", "time", "macros", "sync", "signal", "fs"] }
 
 http = "0.2"
-hyper = "0.13"
+hyper = "^0.13.6"
 url = "2.1"
 httpdate = "0.3"
 percent-encoding = "2.1.0"

src/api/api_server.rs

@@ -7,11 +7,11 @@ use hyper::server::conn::AddrStream;
 use hyper::service::{make_service_fn, service_fn};
 use hyper::{Body, Method, Request, Response, Server};
 
-use garage_util::error::Error;
+use garage_util::error::Error as GarageError;
 
 use garage_model::garage::Garage;
 
-use crate::http_util::*;
+use crate::error::*;
 use crate::signature::check_signature;
 
 use crate::s3_copy::*;
@@ -23,14 +23,14 @@ use crate::s3_put::*;
 pub async fn run_api_server(
 	garage: Arc<Garage>,
 	shutdown_signal: impl Future<Output = ()>,
-) -> Result<(), Error> {
+) -> Result<(), GarageError> {
 	let addr = &garage.config.s3_api.api_bind_addr;
 
 	let service = make_service_fn(|conn: &AddrStream| {
 		let garage = garage.clone();
 		let client_addr = conn.remote_addr();
 		async move {
-			Ok::<_, Error>(service_fn(move |req: Request<Body>| {
+			Ok::<_, GarageError>(service_fn(move |req: Request<Body>| {
 				let garage = garage.clone();
 				handler(garage, req, client_addr)
 			}))
@@ -50,7 +50,7 @@ async fn handler(
 	garage: Arc<Garage>,
 	req: Request<Body>,
 	addr: SocketAddr,
-) -> Result<Response<BodyType>, Error> {
+) -> Result<Response<Body>, GarageError> {
 	info!("{} {} {}", addr, req.method(), req.uri());
 	debug!("{:?}", req);
 	match handler_inner(garage, req).await {
@@ -59,7 +59,7 @@ async fn handler(
 			Ok(x)
 		}
 		Err(e) => {
-			let body: BodyType = Box::new(BytesBody::from(format!("{}\n", e)));
+			let body: Body = Body::from(format!("{}\n", e));
 			let mut http_error = Response::new(body);
 			*http_error.status_mut() = e.http_status_code();
 			warn!("Response: error {}, {}", e.http_status_code(), e);
@@ -68,21 +68,13 @@ async fn handler(
 	}
 }
 
-async fn handler_inner(
-	garage: Arc<Garage>,
-	req: Request<Body>,
-) -> Result<Response<BodyType>, Error> {
+async fn handler_inner(garage: Arc<Garage>, req: Request<Body>) -> Result<Response<Body>, Error> {
 	let path = req.uri().path().to_string();
 	let path = percent_encoding::percent_decode_str(&path).decode_utf8()?;
 
 	let (bucket, key) = parse_bucket_key(&path)?;
-	if bucket.len() == 0 {
-		return Err(Error::Forbidden(format!(
-			"Operations on buckets not allowed"
-		)));
-	}
 
-	let api_key = check_signature(&garage, &req).await?;
+	let (api_key, content_sha256) = check_signature(&garage, &req).await?;
 	let allowed = match req.method() {
 		&Method::HEAD | &Method::GET => api_key.allow_read(&bucket),
 		_ => api_key.allow_write(&bucket),
@@ -118,7 +110,16 @@ async fn handler_inner(
 					// UploadPart query
 					let part_number = params.get("partnumber").unwrap();
 					let upload_id = params.get("uploadid").unwrap();
-					Ok(handle_put_part(garage, req, &bucket, &key, part_number, upload_id).await?)
+					Ok(handle_put_part(
+						garage,
+						req,
+						&bucket,
+						&key,
+						part_number,
+						upload_id,
+						content_sha256,
+					)
+					.await?)
 				} else if req.headers().contains_key("x-amz-copy-source") {
 					// CopyObject query
 					let copy_source = req.headers().get("x-amz-copy-source").unwrap().to_str()?;
@@ -131,14 +132,11 @@ async fn handler_inner(
 							source_bucket
 						)));
 					}
-					let source_key = match source_key {
-						None => return Err(Error::BadRequest(format!("No source key specified"))),
-						Some(x) => x,
-					};
+					let source_key = source_key.ok_or_bad_request("No source key specified")?;
 					Ok(handle_copy(garage, &bucket, &key, &source_bucket, &source_key).await?)
 				} else {
 					// PutObject query
-					Ok(handle_put(garage, req, &bucket, &key).await?)
+					Ok(handle_put(garage, req, &bucket, &key, content_sha256).await?)
 				}
 			}
 			&Method::DELETE => {
@@ -175,12 +173,12 @@ async fn handler_inner(
 			&Method::PUT => {
 				// CreateBucket
 				// If we're here, the bucket already exists, so just answer ok
-				println!(
+				debug!(
 					"Body: {}",
 					std::str::from_utf8(&hyper::body::to_bytes(req.into_body()).await?)
 						.unwrap_or("<invalid utf8>")
 				);
-				let empty_body: BodyType = Box::new(BytesBody::from(vec![]));
+				let empty_body: Body = Body::from(vec![]);
 				let response = Response::builder()
 					.header("Location", format!("/{}", bucket))
 					.body(empty_body)
@@ -189,7 +187,7 @@ async fn handler_inner(
 			}
 			&Method::HEAD => {
 				// HeadBucket
-				let empty_body: BodyType = Box::new(BytesBody::from(vec![]));
+				let empty_body: Body = Body::from(vec![]);
 				let response = Response::builder().body(empty_body).unwrap();
 				Ok(response)
 			}
@@ -205,9 +203,8 @@ async fn handler_inner(
 				let max_keys = params
 					.get("max-keys")
 					.map(|x| {
-						x.parse::<usize>().map_err(|e| {
-							Error::BadRequest(format!("Invalid value for max-keys: {}", e))
-						})
+						x.parse::<usize>()
+							.ok_or_bad_request("Invalid value for max-keys")
 					})
 					.unwrap_or(Ok(1000))?;
 				let prefix = params.get("prefix").map(|x| x.as_str()).unwrap_or(&"");
@@ -232,7 +229,7 @@ async fn handler_inner(
 					// DeleteObjects
 					Ok(handle_delete_objects(garage, bucket, req).await?)
 				} else {
-					println!(
+					debug!(
 						"Body: {}",
 						std::str::from_utf8(&hyper::body::to_bytes(req.into_body()).await?)
 							.unwrap_or("<invalid utf8>")
@@ -245,18 +242,60 @@ async fn handler_inner(
 	}
 }
 
+/// Extract the bucket name and the key name from an HTTP path
+///
+/// S3 internally manages only buckets and keys. This function splits
+/// an HTTP path to get the corresponding bucket name and key.
 fn parse_bucket_key(path: &str) -> Result<(&str, Option<&str>), Error> {
 	let path = path.trim_start_matches('/');
 
-	match path.find('/') {
+	let (bucket, key) = match path.find('/') {
 		Some(i) => {
 			let key = &path[i + 1..];
 			if key.len() > 0 {
-				Ok((&path[..i], Some(key)))
+				(&path[..i], Some(key))
 			} else {
-				Ok((&path[..i], None))
+				(&path[..i], None)
 			}
 		}
-		None => Ok((path, None)),
+		None => (path, None),
+	};
+	if bucket.len() == 0 {
+		return Err(Error::BadRequest(format!("No bucket specified")));
+	}
+	Ok((bucket, key))
+}
+
+#[cfg(test)]
+mod tests {
+	use super::*;
+
+	#[test]
+	fn parse_bucket_containing_a_key() -> Result<(), Error> {
+		let (bucket, key) = parse_bucket_key("/my_bucket/a/super/file.jpg")?;
+		assert_eq!(bucket, "my_bucket");
+		assert_eq!(key.expect("key must be set"), "a/super/file.jpg");
+		Ok(())
+	}
+
+	#[test]
+	fn parse_bucket_containing_no_key() -> Result<(), Error> {
+		let (bucket, key) = parse_bucket_key("/my_bucket/")?;
+		assert_eq!(bucket, "my_bucket");
+		assert!(key.is_none());
+		let (bucket, key) = parse_bucket_key("/my_bucket")?;
+		assert_eq!(bucket, "my_bucket");
+		assert!(key.is_none());
+		Ok(())
+	}
+
+	#[test]
+	fn parse_bucket_containing_no_bucket() {
+		let parsed = parse_bucket_key("");
+		assert!(parsed.is_err());
+		let parsed = parse_bucket_key("/");
+		assert!(parsed.is_err());
+		let parsed = parse_bucket_key("////");
+		assert!(parsed.is_err());
 	}
 }

src/api/error.rs

@@ -0,0 +1,116 @@
+use err_derive::Error;
+use hyper::StatusCode;
+
+use garage_util::error::Error as GarageError;
+
+#[derive(Debug, Error)]
+pub enum Error {
+	// Category: internal error
+	#[error(display = "Internal error: {}", _0)]
+	InternalError(#[error(source)] GarageError),
+
+	#[error(display = "Internal error (Hyper error): {}", _0)]
+	Hyper(#[error(source)] hyper::Error),
+
+	#[error(display = "Internal error (HTTP error): {}", _0)]
+	HTTP(#[error(source)] http::Error),
+
+	// Category: cannot process
+	#[error(display = "Forbidden: {}", _0)]
+	Forbidden(String),
+
+	#[error(display = "Not found")]
+	NotFound,
+
+	// Category: bad request
+	#[error(display = "Invalid UTF-8: {}", _0)]
+	InvalidUTF8(#[error(source)] std::str::Utf8Error),
+
+	#[error(display = "Invalid XML: {}", _0)]
+	InvalidXML(#[error(source)] roxmltree::Error),
+
+	#[error(display = "Invalid header value: {}", _0)]
+	InvalidHeader(#[error(source)] hyper::header::ToStrError),
+
+	#[error(display = "Invalid HTTP range: {:?}", _0)]
+	InvalidRange(#[error(from)] http_range::HttpRangeParseError),
+
+	#[error(display = "Bad request: {}", _0)]
+	BadRequest(String),
+}
+
+impl Error {
+	pub fn http_status_code(&self) -> StatusCode {
+		match self {
+			Error::NotFound => StatusCode::NOT_FOUND,
+			Error::Forbidden(_) => StatusCode::FORBIDDEN,
+			Error::InternalError(GarageError::RPC(_)) => StatusCode::SERVICE_UNAVAILABLE,
+			Error::InternalError(_) | Error::Hyper(_) | Error::HTTP(_) => {
+				StatusCode::INTERNAL_SERVER_ERROR
+			}
+			_ => StatusCode::BAD_REQUEST,
+		}
+	}
+}
+
+pub trait OkOrBadRequest {
+	type S2;
+	fn ok_or_bad_request(self, reason: &'static str) -> Self::S2;
+}
+
+impl<T, E> OkOrBadRequest for Result<T, E>
+where
+	E: std::fmt::Display,
+{
+	type S2 = Result<T, Error>;
+	fn ok_or_bad_request(self, reason: &'static str) -> Result<T, Error> {
+		match self {
+			Ok(x) => Ok(x),
+			Err(e) => Err(Error::BadRequest(format!("{}: {}", reason, e))),
+		}
+	}
+}
+
+impl<T> OkOrBadRequest for Option<T> {
+	type S2 = Result<T, Error>;
+	fn ok_or_bad_request(self, reason: &'static str) -> Result<T, Error> {
+		match self {
+			Some(x) => Ok(x),
+			None => Err(Error::BadRequest(format!("{}", reason))),
+		}
+	}
+}
+
+pub trait OkOrInternalError {
+	type S2;
+	fn ok_or_internal_error(self, reason: &'static str) -> Self::S2;
+}
+
+impl<T, E> OkOrInternalError for Result<T, E>
+where
+	E: std::fmt::Display,
+{
+	type S2 = Result<T, Error>;
+	fn ok_or_internal_error(self, reason: &'static str) -> Result<T, Error> {
+		match self {
+			Ok(x) => Ok(x),
+			Err(e) => Err(Error::InternalError(GarageError::Message(format!(
+				"{}: {}",
+				reason, e
+			)))),
+		}
+	}
+}
+
+impl<T> OkOrInternalError for Option<T> {
+	type S2 = Result<T, Error>;
+	fn ok_or_internal_error(self, reason: &'static str) -> Result<T, Error> {
+		match self {
+			Some(x) => Ok(x),
+			None => Err(Error::InternalError(GarageError::Message(format!(
+				"{}",
+				reason
+			)))),
+		}
+	}
+}

src/api/http_util.rs

@@ -1,88 +0,0 @@
-use core::pin::Pin;
-use core::task::{Context, Poll};
-
-use futures::ready;
-use futures::stream::*;
-use hyper::body::{Bytes, HttpBody};
-
-use garage_util::error::Error;
-
-pub type BodyType = Box<dyn HttpBody<Data = Bytes, Error = Error> + Send + Unpin>;
-
-type StreamType = Pin<Box<dyn Stream<Item = Result<Bytes, Error>> + Send>>;
-
-pub struct StreamBody {
-	stream: StreamType,
-}
-
-impl StreamBody {
-	pub fn new(stream: StreamType) -> Self {
-		Self { stream }
-	}
-}
-
-impl HttpBody for StreamBody {
-	type Data = Bytes;
-	type Error = Error;
-
-	fn poll_data(
-		mut self: Pin<&mut Self>,
-		cx: &mut Context,
-	) -> Poll<Option<Result<Bytes, Self::Error>>> {
-		match ready!(self.stream.as_mut().poll_next(cx)) {
-			Some(res) => Poll::Ready(Some(res)),
-			None => Poll::Ready(None),
-		}
-	}
-
-	fn poll_trailers(
-		self: Pin<&mut Self>,
-		_cx: &mut Context,
-	) -> Poll<Result<Option<hyper::HeaderMap<hyper::header::HeaderValue>>, Self::Error>> {
-		Poll::Ready(Ok(None))
-	}
-}
-
-pub struct BytesBody {
-	bytes: Option<Bytes>,
-}
-
-impl BytesBody {
-	pub fn new(bytes: Bytes) -> Self {
-		Self { bytes: Some(bytes) }
-	}
-}
-
-impl HttpBody for BytesBody {
-	type Data = Bytes;
-	type Error = Error;
-
-	fn poll_data(
-		mut self: Pin<&mut Self>,
-		_cx: &mut Context,
-	) -> Poll<Option<Result<Bytes, Self::Error>>> {
-		Poll::Ready(self.bytes.take().map(Ok))
-	}
-
-	fn poll_trailers(
-		self: Pin<&mut Self>,
-		_cx: &mut Context,
-	) -> Poll<Result<Option<hyper::HeaderMap<hyper::header::HeaderValue>>, Self::Error>> {
-		Poll::Ready(Ok(None))
-	}
-}
-
-impl From<String> for BytesBody {
-	fn from(x: String) -> BytesBody {
-		Self::new(Bytes::from(x))
-	}
-}
-impl From<Vec<u8>> for BytesBody {
-	fn from(x: Vec<u8>) -> BytesBody {
-		Self::new(Bytes::from(x))
-	}
-}
-
-pub fn empty_body() -> BodyType {
-	Box::new(BytesBody::from(vec![]))
-}

src/api/lib.rs

@@ -1,8 +1,9 @@
 #[macro_use]
 extern crate log;
 
+pub mod error;
+
 pub mod encoding;
-pub mod http_util;
 
 pub mod api_server;
 pub mod signature;

src/api/s3_copy.rs

@@ -2,18 +2,17 @@ use std::fmt::Write;
 use std::sync::Arc;
 
 use chrono::{SecondsFormat, Utc};
-use hyper::Response;
+use hyper::{Body, Response};
 
 use garage_table::*;
 use garage_util::data::*;
-use garage_util::error::Error;
 
 use garage_model::block_ref_table::*;
 use garage_model::garage::Garage;
 use garage_model::object_table::*;
 use garage_model::version_table::*;
 
-use crate::http_util::*;
+use crate::error::*;
 
 pub async fn handle_copy(
 	garage: Arc<Garage>,
@@ -21,47 +20,38 @@ pub async fn handle_copy(
 	dest_key: &str,
 	source_bucket: &str,
 	source_key: &str,
-) -> Result<Response<BodyType>, Error> {
-	let source_object = match garage
+) -> Result<Response<Body>, Error> {
+	let source_object = garage
 		.object_table
 		.get(&source_bucket.to_string(), &source_key.to_string())
 		.await?
-	{
-		None => return Err(Error::NotFound),
-		Some(o) => o,
-	};
+		.ok_or(Error::NotFound)?;
 
-	let source_last_v = match source_object
+	let source_last_v = source_object
 		.versions()
 		.iter()
 		.rev()
 		.filter(|v| v.is_complete())
 		.next()
-	{
-		Some(v) => v,
-		None => return Err(Error::NotFound),
+		.ok_or(Error::NotFound)?;
+
+	let source_last_state = match &source_last_v.state {
+		ObjectVersionState::Complete(x) => x,
+		_ => unreachable!(),
 	};
 
 	let new_uuid = gen_uuid();
 	let dest_object_version = ObjectVersion {
 		uuid: new_uuid,
 		timestamp: now_msec(),
-		mime_type: source_last_v.mime_type.clone(),
-		size: source_last_v.size,
-		state: ObjectVersionState::Complete,
-		data: source_last_v.data.clone(),
+		state: ObjectVersionState::Complete(source_last_state.clone()),
 	};
 
-	match &source_last_v.data {
-		ObjectVersionData::Uploading => {
-			return Err(Error::Message(format!(
-				"Version is_complete() but data is stil Uploading (internal error)"
-			)));
-		}
+	match &source_last_state {
 		ObjectVersionData::DeleteMarker => {
 			return Err(Error::NotFound);
 		}
-		ObjectVersionData::Inline(_bytes) => {
+		ObjectVersionData::Inline(_meta, _bytes) => {
 			let dest_object = Object::new(
 				dest_bucket.to_string(),
 				dest_key.to_string(),
@@ -69,15 +59,12 @@ pub async fn handle_copy(
 			);
 			garage.object_table.insert(&dest_object).await?;
 		}
-		ObjectVersionData::FirstBlock(_first_block_hash) => {
+		ObjectVersionData::FirstBlock(_meta, _first_block_hash) => {
 			let source_version = garage
 				.version_table
 				.get(&source_last_v.uuid, &EmptyKey)
 				.await?;
-			let source_version = match source_version {
-				Some(v) => v,
-				None => return Err(Error::NotFound),
-			};
+			let source_version = source_version.ok_or(Error::NotFound)?;
 
 			let dest_version = Version::new(
 				new_uuid,
@@ -116,5 +103,5 @@ pub async fn handle_copy(
 	writeln!(&mut xml, "\t<LastModified>{}</LastModified>", last_modified).unwrap();
 	writeln!(&mut xml, "</CopyObjectResult>").unwrap();
 
-	Ok(Response::new(Box::new(BytesBody::from(xml.into_bytes()))))
+	Ok(Response::new(Body::from(xml.into_bytes())))
 }

src/api/s3_delete.rs

@@ -4,33 +4,28 @@ use std::sync::Arc;
 use hyper::{Body, Request, Response};
 
 use garage_util::data::*;
-use garage_util::error::Error;
 
 use garage_model::garage::Garage;
 use garage_model::object_table::*;
 
 use crate::encoding::*;
-use crate::http_util::*;
+use crate::error::*;
 
 async fn handle_delete_internal(
 	garage: &Garage,
 	bucket: &str,
 	key: &str,
 ) -> Result<(UUID, UUID), Error> {
-	let object = match garage
+	let object = garage
 		.object_table
 		.get(&bucket.to_string(), &key.to_string())
 		.await?
-	{
-		None => {
-			// No need to delete
-			return Err(Error::NotFound);
-		}
-		Some(o) => o,
-	};
+		.ok_or(Error::NotFound)?; // No need to delete
 
-	let interesting_versions = object.versions().iter().filter(|v| {
-		v.data != ObjectVersionData::DeleteMarker && v.state != ObjectVersionState::Aborted
+	let interesting_versions = object.versions().iter().filter(|v| match v.state {
+		ObjectVersionState::Aborted => false,
+		ObjectVersionState::Complete(ObjectVersionData::DeleteMarker) => false,
+		_ => true,
 	});
 
 	let mut must_delete = None;
@@ -42,10 +37,7 @@ async fn handle_delete_internal(
 		timestamp = std::cmp::max(timestamp, v.timestamp + 1);
 	}
 
-	let deleted_version = match must_delete {
-		None => return Err(Error::NotFound),
-		Some(v) => v,
-	};
+	let deleted_version = must_delete.ok_or(Error::NotFound)?;
 
 	let version_uuid = gen_uuid();
 
@@ -55,10 +47,7 @@ async fn handle_delete_internal(
 		vec![ObjectVersion {
 			uuid: version_uuid,
 			timestamp: now_msec(),
-			mime_type: "application/x-delete-marker".into(),
-			size: 0,
-			state: ObjectVersionState::Complete,
-			data: ObjectVersionData::DeleteMarker,
+			state: ObjectVersionState::Complete(ObjectVersionData::DeleteMarker),
 		}],
 	);
 
@@ -70,13 +59,13 @@ pub async fn handle_delete(
 	garage: Arc<Garage>,
 	bucket: &str,
 	key: &str,
-) -> Result<Response<BodyType>, Error> {
+) -> Result<Response<Body>, Error> {
 	let (_deleted_version, delete_marker_version) =
 		handle_delete_internal(&garage, bucket, key).await?;
 
 	Ok(Response::builder()
 		.header("x-amz-version-id", hex::encode(delete_marker_version))
-		.body(empty_body())
+		.body(Body::from(vec![]))
 		.unwrap())
 }
 
@@ -84,11 +73,10 @@ pub async fn handle_delete_objects(
 	garage: Arc<Garage>,
 	bucket: &str,
 	req: Request<Body>,
-) -> Result<Response<BodyType>, Error> {
+) -> Result<Response<Body>, Error> {
 	let body = hyper::body::to_bytes(req.into_body()).await?;
 	let cmd_xml = roxmltree::Document::parse(&std::str::from_utf8(&body)?)?;
-	let cmd = parse_delete_objects_xml(&cmd_xml)
-		.map_err(|e| Error::BadRequest(format!("Invald delete XML query: {}", e)))?;
+	let cmd = parse_delete_objects_xml(&cmd_xml).ok_or_bad_request("Invalid delete XML query")?;
 
 	let mut retxml = String::new();
 	writeln!(&mut retxml, r#"<?xml version="1.0" encoding="UTF-8"?>"#).unwrap();
@@ -130,9 +118,7 @@ pub async fn handle_delete_objects(
 
 	writeln!(&mut retxml, "</DeleteObjectsOutput>").unwrap();
 
-	Ok(Response::new(Box::new(BytesBody::from(
-		retxml.into_bytes(),
-	))))
+	Ok(Response::new(Body::from(retxml.into_bytes())))
 }
 
 struct DeleteRequest {
@@ -147,10 +133,8 @@ fn parse_delete_objects_xml(xml: &roxmltree::Document) -> Result<DeleteRequest,
 	let mut ret = DeleteRequest { objects: vec![] };
 
 	let root = xml.root();
-	let delete = match root.first_child() {
-		Some(del) => del,
-		None => return Err(format!("Delete tag not found")),
-	};
+	let delete = root.first_child().ok_or(format!("Delete tag not found"))?;
+
 	if !delete.has_tag_name("Delete") {
 		return Err(format!("Invalid root tag: {:?}", root));
 	}

src/api/s3_get.rs

@@ -5,53 +5,67 @@ use futures::stream::*;
 use hyper::body::Bytes;
 use hyper::{Body, Request, Response, StatusCode};
 
-use garage_util::error::Error;
-
 use garage_table::EmptyKey;
 
 use garage_model::garage::Garage;
 use garage_model::object_table::*;
 
-use crate::http_util::*;
+use crate::error::*;
 
-fn object_headers(version: &ObjectVersion) -> http::response::Builder {
+fn object_headers(
+	version: &ObjectVersion,
+	version_meta: &ObjectVersionMeta,
+) -> http::response::Builder {
 	let date = UNIX_EPOCH + Duration::from_millis(version.timestamp);
 	let date_str = httpdate::fmt_http_date(date);
 
-	Response::builder()
-		.header("Content-Type", version.mime_type.to_string())
-		.header("Content-Length", format!("{}", version.size))
+	let mut resp = Response::builder()
+		.header(
+			"Content-Type",
+			version_meta.headers.content_type.to_string(),
+		)
 		.header("Last-Modified", date_str)
-		.header("Accept-Ranges", format!("bytes"))
+		.header("Accept-Ranges", format!("bytes"));
+
+	if !version_meta.etag.is_empty() {
+		resp = resp.header("ETag", format!("\"{}\"", version_meta.etag));
+	}
+
+	for (k, v) in version_meta.headers.other.iter() {
+		resp = resp.header(k, v.to_string());
+	}
+
+	resp
 }
 
 pub async fn handle_head(
 	garage: Arc<Garage>,
 	bucket: &str,
 	key: &str,
-) -> Result<Response<BodyType>, Error> {
-	let object = match garage
+) -> Result<Response<Body>, Error> {
+	let object = garage
 		.object_table
 		.get(&bucket.to_string(), &key.to_string())
 		.await?
-	{
-		None => return Err(Error::NotFound),
-		Some(o) => o,
-	};
+		.ok_or(Error::NotFound)?;
 
-	let version = match object
+	let version = object
 		.versions()
 		.iter()
 		.rev()
 		.filter(|v| v.is_data())
 		.next()
-	{
-		Some(v) => v,
-		None => return Err(Error::NotFound),
+		.ok_or(Error::NotFound)?;
+
+	let version_meta = match &version.state {
+		ObjectVersionState::Complete(ObjectVersionData::Inline(meta, _)) => meta,
+		ObjectVersionState::Complete(ObjectVersionData::FirstBlock(meta, _)) => meta,
+		_ => unreachable!(),
 	};
 
-	let body: BodyType = Box::new(BytesBody::from(vec![]));
-	let response = object_headers(&version)
+	let body: Body = Body::from(vec![]);
+	let response = object_headers(&version, version_meta)
+		.header("Content-Length", format!("{}", version_meta.size))
 		.status(StatusCode::OK)
 		.body(body)
 		.unwrap();
@@ -63,34 +77,35 @@ pub async fn handle_get(
 	req: &Request<Body>,
 	bucket: &str,
 	key: &str,
-) -> Result<Response<BodyType>, Error> {
-	let object = match garage
+) -> Result<Response<Body>, Error> {
+	let object = garage
 		.object_table
 		.get(&bucket.to_string(), &key.to_string())
 		.await?
-	{
-		None => return Err(Error::NotFound),
-		Some(o) => o,
-	};
+		.ok_or(Error::NotFound)?;
 
-	let last_v = match object
+	let last_v = object
 		.versions()
 		.iter()
 		.rev()
 		.filter(|v| v.is_complete())
 		.next()
-	{
-		Some(v) => v,
-		None => return Err(Error::NotFound),
+		.ok_or(Error::NotFound)?;
+
+	let last_v_data = match &last_v.state {
+		ObjectVersionState::Complete(x) => x,
+		_ => unreachable!(),
+	};
+	let last_v_meta = match last_v_data {
+		ObjectVersionData::DeleteMarker => return Err(Error::NotFound),
+		ObjectVersionData::Inline(meta, _) => meta,
+		ObjectVersionData::FirstBlock(meta, _) => meta,
 	};
 
 	let range = match req.headers().get("range") {
 		Some(range) => {
-			let range_str = range
-				.to_str()
-				.map_err(|e| Error::BadRequest(format!("Invalid range header: {}", e)))?;
-			let mut ranges = http_range::HttpRange::parse(range_str, last_v.size)
-				.map_err(|_e| Error::BadRequest(format!("Invalid range")))?;
+			let range_str = range.to_str()?;
+			let mut ranges = http_range::HttpRange::parse(range_str, last_v_meta.size)?;
 			if ranges.len() > 1 {
 				return Err(Error::BadRequest(format!("Multiple ranges not supported")));
 			} else {
@@ -100,29 +115,33 @@ pub async fn handle_get(
 		None => None,
 	};
 	if let Some(range) = range {
-		return handle_get_range(garage, last_v, range.start, range.start + range.length).await;
+		return handle_get_range(
+			garage,
+			last_v,
+			last_v_data,
+			last_v_meta,
+			range.start,
+			range.start + range.length,
+		)
+		.await;
 	}
 
-	let resp_builder = object_headers(&last_v).status(StatusCode::OK);
+	let resp_builder = object_headers(&last_v, last_v_meta)
+		.header("Content-Length", format!("{}", last_v_meta.size))
+		.status(StatusCode::OK);
 
-	match &last_v.data {
-		ObjectVersionData::Uploading => Err(Error::Message(format!(
-			"Version is_complete() but data is stil Uploading (internal error)"
-		))),
-		ObjectVersionData::DeleteMarker => Err(Error::NotFound),
-		ObjectVersionData::Inline(bytes) => {
-			let body: BodyType = Box::new(BytesBody::from(bytes.to_vec()));
+	match &last_v_data {
+		ObjectVersionData::DeleteMarker => unreachable!(),
+		ObjectVersionData::Inline(_, bytes) => {
+			let body: Body = Body::from(bytes.to_vec());
 			Ok(resp_builder.body(body)?)
 		}
-		ObjectVersionData::FirstBlock(first_block_hash) => {
+		ObjectVersionData::FirstBlock(_, first_block_hash) => {
 			let read_first_block = garage.block_manager.rpc_get_block(&first_block_hash);
 			let get_next_blocks = garage.version_table.get(&last_v.uuid, &EmptyKey);
 
 			let (first_block, version) = futures::try_join!(read_first_block, get_next_blocks)?;
-			let version = match version {
-				Some(v) => v,
-				None => return Err(Error::NotFound),
-			};
+			let version = version.ok_or(Error::NotFound)?;
 
 			let mut blocks = version
 				.blocks()
@@ -147,7 +166,8 @@ pub async fn handle_get(
 					}
 				})
 				.buffered(2);
-			let body: BodyType = Box::new(StreamBody::new(Box::pin(body_stream)));
+
+			let body = hyper::body::Body::wrap_stream(body_stream);
 			Ok(resp_builder.body(body)?)
 		}
 	}
@@ -156,71 +176,86 @@ pub async fn handle_get(
 pub async fn handle_get_range(
 	garage: Arc<Garage>,
 	version: &ObjectVersion,
+	version_data: &ObjectVersionData,
+	version_meta: &ObjectVersionMeta,
 	begin: u64,
 	end: u64,
-) -> Result<Response<BodyType>, Error> {
-	if end > version.size {
+) -> Result<Response<Body>, Error> {
+	if end > version_meta.size {
 		return Err(Error::BadRequest(format!("Range not included in file")));
 	}
 
-	let resp_builder = object_headers(&version)
+	let resp_builder = object_headers(version, version_meta)
+		.header("Content-Length", format!("{}", end - begin))
 		.header(
 			"Content-Range",
-			format!("bytes {}-{}/{}", begin, end, version.size),
+			format!("bytes {}-{}/{}", begin, end - 1, version_meta.size),
 		)
 		.status(StatusCode::PARTIAL_CONTENT);
 
-	match &version.data {
-		ObjectVersionData::Uploading => Err(Error::Message(format!(
-			"Version is_complete() but data is stil Uploading (internal error)"
-		))),
-		ObjectVersionData::DeleteMarker => Err(Error::NotFound),
-		ObjectVersionData::Inline(bytes) => {
+	match &version_data {
+		ObjectVersionData::DeleteMarker => unreachable!(),
+		ObjectVersionData::Inline(_meta, bytes) => {
 			if end as usize <= bytes.len() {
-				let body: BodyType = Box::new(BytesBody::from(
-					bytes[begin as usize..end as usize].to_vec(),
-				));
+				let body: Body = Body::from(bytes[begin as usize..end as usize].to_vec());
 				Ok(resp_builder.body(body)?)
 			} else {
-				Err(Error::Message(format!("Internal error: requested range not present in inline bytes when it should have been")))
+				None.ok_or_internal_error(
+					"Requested range not present in inline bytes when it should have been",
+				)
 			}
 		}
-		ObjectVersionData::FirstBlock(_first_block_hash) => {
+		ObjectVersionData::FirstBlock(_meta, _first_block_hash) => {
 			let version = garage.version_table.get(&version.uuid, &EmptyKey).await?;
 			let version = match version {
 				Some(v) => v,
 				None => return Err(Error::NotFound),
 			};
 
-			let blocks = version
-				.blocks()
-				.iter()
-				.cloned()
-				.filter(|block| block.offset + block.size > begin && block.offset < end)
-				.collect::<Vec<_>>();
+			// We will store here the list of blocks that have an intersection with the requested
+			// range, as well as their "true offset", which is their actual offset in the complete
+			// file (whereas block.offset designates the offset of the block WITHIN THE PART
+			// block.part_number, which is not the same in the case of a multipart upload)
+			let mut blocks = Vec::with_capacity(std::cmp::min(
+				version.blocks().len(),
+				4 + ((end - begin) / std::cmp::max(version.blocks()[0].size as u64, 1024)) as usize,
+			));
+			let mut true_offset = 0;
+			for b in version.blocks().iter() {
+				if true_offset >= end {
+					break;
+				}
+				// Keep only blocks that have an intersection with the requested range
+				if true_offset < end && true_offset + b.size > begin {
+					blocks.push((b.clone(), true_offset));
+				}
+				true_offset += b.size;
+			}
 
 			let body_stream = futures::stream::iter(blocks)
-				.map(move |block| {
+				.map(move |(block, true_offset)| {
 					let garage = garage.clone();
 					async move {
 						let data = garage.block_manager.rpc_get_block(&block.hash).await?;
-						let start_in_block = if block.offset > begin {
+						let data = Bytes::from(data);
+						let start_in_block = if true_offset > begin {
 							0
 						} else {
-							begin - block.offset
+							begin - true_offset
 						};
-						let end_in_block = if block.offset + block.size < end {
+						let end_in_block = if true_offset + block.size < end {
 							block.size
 						} else {
-							end - block.offset
+							end - true_offset
 						};
-						Ok(Bytes::from(
-							data[start_in_block as usize..end_in_block as usize].to_vec(),
+						Result::<Bytes, Error>::Ok(Bytes::from(
+							data.slice(start_in_block as usize..end_in_block as usize),
 						))
 					}
 				})
 				.buffered(2);
-			let body: BodyType = Box::new(StreamBody::new(Box::pin(body_stream)));
+
+			let body = hyper::body::Body::wrap_stream(body_stream);
 			Ok(resp_builder.body(body)?)
 		}
 	}

src/api/s3_list.rs

@@ -3,19 +3,22 @@ use std::fmt::Write;
 use std::sync::Arc;
 
 use chrono::{DateTime, NaiveDateTime, SecondsFormat, Utc};
-use hyper::Response;
+use hyper::{Body, Response};
 
 use garage_util::error::Error;
 
 use garage_model::garage::Garage;
+use garage_model::object_table::*;
+
+use garage_table::DeletedFilter;
 
 use crate::encoding::*;
-use crate::http_util::*;
 
 #[derive(Debug)]
 struct ListResultInfo {
 	last_modified: u64,
 	size: u64,
+	etag: String,
 }
 
 pub async fn handle_list(
@@ -26,7 +29,7 @@ pub async fn handle_list(
 	prefix: &str,
 	marker: Option<&str>,
 	urlencode_resp: bool,
-) -> Result<Response<BodyType>, Error> {
+) -> Result<Response<Body>, Error> {
 	let mut result_keys = BTreeMap::<String, ListResultInfo>::new();
 	let mut result_common_prefixes = BTreeSet::<String>::new();
 
@@ -41,7 +44,7 @@ pub async fn handle_list(
 			.get_range(
 				&bucket.to_string(),
 				Some(next_chunk_start.clone()),
-				Some(()),
+				Some(DeletedFilter::NotDeleted),
 				max_keys + 1,
 			)
 			.await?;
@@ -54,30 +57,37 @@ pub async fn handle_list(
 
 		for object in objects.iter() {
 			if !object.key.starts_with(prefix) {
-				truncated = false;
+				truncated = None;
 				break 'query_loop;
 			}
 			if let Some(version) = object.versions().iter().find(|x| x.is_data()) {
 				if result_keys.len() + result_common_prefixes.len() >= max_keys {
-					truncated = true;
+					truncated = Some(object.key.to_string());
 					break 'query_loop;
 				}
 				let common_prefix = if delimiter.len() > 0 {
 					let relative_key = &object.key[prefix.len()..];
-					match relative_key.find(delimiter) {
-						Some(i) => Some(&object.key[..prefix.len() + i + delimiter.len()]),
-						None => None,
-					}
+					relative_key
+						.find(delimiter)
+						.map(|i| &object.key[..prefix.len() + i + delimiter.len()])
 				} else {
 					None
 				};
 				if let Some(pfx) = common_prefix {
 					result_common_prefixes.insert(pfx.to_string());
 				} else {
+					let meta = match &version.state {
+						ObjectVersionState::Complete(ObjectVersionData::Inline(meta, _)) => meta,
+						ObjectVersionState::Complete(ObjectVersionData::FirstBlock(meta, _)) => {
+							meta
+						}
+						_ => unreachable!(),
+					};
 					let info = match result_keys.get(&object.key) {
 						None => ListResultInfo {
 							last_modified: version.timestamp,
-							size: version.size,
+							size: meta.size,
+							etag: meta.etag.to_string(),
 						},
 						Some(_lri) => {
 							return Err(Error::Message(format!("Duplicate key?? {}", object.key)))
@@ -88,7 +98,7 @@ pub async fn handle_list(
 			}
 		}
 		if objects.len() < max_keys + 1 {
-			truncated = false;
+			truncated = None;
 			break 'query_loop;
 		}
 		if objects.len() > 0 {
@@ -103,11 +113,22 @@ pub async fn handle_list(
 		r#"<ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">"#
 	)
 	.unwrap();
-	writeln!(&mut xml, "\t<Bucket>{}</Bucket>", bucket).unwrap();
+	writeln!(&mut xml, "\t<Name>{}</Name>", bucket).unwrap();
 	writeln!(&mut xml, "\t<Prefix>{}</Prefix>", prefix).unwrap();
+	if let Some(mkr) = marker {
+		writeln!(&mut xml, "\t<Marker>{}</Marker>", mkr).unwrap();
+	}
 	writeln!(&mut xml, "\t<KeyCount>{}</KeyCount>", result_keys.len()).unwrap();
 	writeln!(&mut xml, "\t<MaxKeys>{}</MaxKeys>", max_keys).unwrap();
-	writeln!(&mut xml, "\t<IsTruncated>{}</IsTruncated>", truncated).unwrap();
+	writeln!(
+		&mut xml,
+		"\t<IsTruncated>{}</IsTruncated>",
+		truncated.is_some()
+	)
+	.unwrap();
+	if let Some(next_marker) = truncated {
+		writeln!(&mut xml, "\t<NextMarker>{}</NextMarker>", next_marker).unwrap();
+	}
 	for (key, info) in result_keys.iter() {
 		let last_modif = NaiveDateTime::from_timestamp(info.last_modified as i64 / 1000, 0);
 		let last_modif = DateTime::<Utc>::from_utc(last_modif, Utc);
@@ -122,12 +143,15 @@ pub async fn handle_list(
 		.unwrap();
 		writeln!(&mut xml, "\t\t<LastModified>{}</LastModified>", last_modif).unwrap();
 		writeln!(&mut xml, "\t\t<Size>{}</Size>", info.size).unwrap();
+		if !info.etag.is_empty() {
+			writeln!(&mut xml, "\t\t<ETag>\"{}\"</ETag>", info.etag).unwrap();
+		}
 		writeln!(&mut xml, "\t\t<StorageClass>STANDARD</StorageClass>").unwrap();
 		writeln!(&mut xml, "\t</Contents>").unwrap();
 	}
 	if result_common_prefixes.len() > 0 {
-		writeln!(&mut xml, "\t<CommonPrefixes>").unwrap();
 		for pfx in result_common_prefixes.iter() {
+			writeln!(&mut xml, "\t<CommonPrefixes>").unwrap();
 			writeln!(
 				&mut xml,
 				"\t\t<Prefix>{}</Prefix>",
@@ -135,11 +159,11 @@ pub async fn handle_list(
 				//xml_encode_key(pfx, urlencode_resp)
 			)
 			.unwrap();
+			writeln!(&mut xml, "\t</CommonPrefixes>").unwrap();
 		}
-		writeln!(&mut xml, "\t</CommonPrefixes>").unwrap();
 	}
 	writeln!(&mut xml, "</ListBucketResult>").unwrap();
-	println!("{}", xml);
+	debug!("{}", xml);
 
-	Ok(Response::new(Box::new(BytesBody::from(xml.into_bytes()))))
+	Ok(Response::new(Body::from(xml.into_bytes())))
 }

src/api/s3_put.rs

@@ -1,14 +1,17 @@
-use std::collections::VecDeque;
+use std::collections::{BTreeMap, VecDeque};
 use std::fmt::Write;
 use std::sync::Arc;
 
 use futures::stream::*;
 use hyper::{Body, Request, Response};
+use md5::{digest::generic_array::*, Digest as Md5Digest, Md5};
+use sha2::{Digest as Sha256Digest, Sha256};
 
 use garage_table::*;
 use garage_util::data::*;
-use garage_util::error::Error;
+use garage_util::error::Error as GarageError;
 
+use crate::error::*;
 use garage_model::block::INLINE_THRESHOLD;
 use garage_model::block_ref_table::*;
 use garage_model::garage::Garage;
@@ -16,50 +19,68 @@ use garage_model::object_table::*;
 use garage_model::version_table::*;
 
 use crate::encoding::*;
-use crate::http_util::*;
 
 pub async fn handle_put(
 	garage: Arc<Garage>,
 	req: Request<Body>,
 	bucket: &str,
 	key: &str,
-) -> Result<Response<BodyType>, Error> {
+	content_sha256: Option<Hash>,
+) -> Result<Response<Body>, Error> {
 	let version_uuid = gen_uuid();
-	let mime_type = get_mime_type(&req)?;
+	let headers = get_headers(&req)?;
+	let content_md5 = match req.headers().get("content-md5") {
+		Some(x) => Some(x.to_str()?.to_string()),
+		None => None,
+	};
+
 	let body = req.into_body();
 
 	let mut chunker = BodyChunker::new(body, garage.config.block_size);
-	let first_block = match chunker.next().await? {
-		Some(x) => x,
-		None => vec![],
-	};
+	let first_block = chunker.next().await?.unwrap_or(vec![]);
 
 	let mut object_version = ObjectVersion {
 		uuid: version_uuid,
 		timestamp: now_msec(),
-		mime_type,
-		size: first_block.len() as u64,
-		state: ObjectVersionState::Uploading,
-		data: ObjectVersionData::Uploading,
+		state: ObjectVersionState::Uploading(headers.clone()),
 	};
 
 	if first_block.len() < INLINE_THRESHOLD {
-		object_version.data = ObjectVersionData::Inline(first_block);
-		object_version.state = ObjectVersionState::Complete;
+		let mut md5sum = Md5::new();
+		md5sum.update(&first_block[..]);
+		let md5sum_arr = md5sum.finalize();
+		let md5sum_hex = hex::encode(md5sum_arr);
+
+		let sha256sum_hash = hash(&first_block[..]);
+
+		ensure_checksum_matches(
+			md5sum_arr.as_slice(),
+			sha256sum_hash,
+			content_md5.as_deref(),
+			content_sha256,
+		)?;
+
+		object_version.state = ObjectVersionState::Complete(ObjectVersionData::Inline(
+			ObjectVersionMeta {
+				headers,
+				size: first_block.len() as u64,
+				etag: md5sum_hex.clone(),
+			},
+			first_block,
+		));
 
 		let object = Object::new(bucket.into(), key.into(), vec![object_version]);
 		garage.object_table.insert(&object).await?;
-		return Ok(put_response(version_uuid));
+		return Ok(put_response(version_uuid, md5sum_hex));
 	}
 
 	let version = Version::new(version_uuid, bucket.into(), key.into(), false, vec![]);
 
 	let first_block_hash = hash(&first_block[..]);
-	object_version.data = ObjectVersionData::FirstBlock(first_block_hash);
 	let object = Object::new(bucket.into(), key.into(), vec![object_version.clone()]);
 	garage.object_table.insert(&object).await?;
 
-	let total_size = read_and_put_blocks(
+	let (total_size, md5sum_arr, sha256sum) = read_and_put_blocks(
 		&garage,
 		version,
 		1,
@@ -69,15 +90,57 @@ pub async fn handle_put(
 	)
 	.await?;
 
+	ensure_checksum_matches(
+		md5sum_arr.as_slice(),
+		sha256sum,
+		content_md5.as_deref(),
+		content_sha256,
+	)?;
+
 	// TODO: if at any step we have an error, we should undo everything we did
 
-	object_version.state = ObjectVersionState::Complete;
-	object_version.size = total_size;
+	let md5sum_hex = hex::encode(md5sum_arr);
+
+	object_version.state = ObjectVersionState::Complete(ObjectVersionData::FirstBlock(
+		ObjectVersionMeta {
+			headers,
+			size: total_size,
+			etag: md5sum_hex.clone(),
+		},
+		first_block_hash,
+	));
 
 	let object = Object::new(bucket.into(), key.into(), vec![object_version]);
 	garage.object_table.insert(&object).await?;
 
-	Ok(put_response(version_uuid))
+	Ok(put_response(version_uuid, md5sum_hex))
+}
+
+/// Validate MD5 sum against content-md5 header
+/// and sha256sum against signed content-sha256
+fn ensure_checksum_matches(
+	md5sum: &[u8],
+	sha256sum: garage_util::data::FixedBytes32,
+	content_md5: Option<&str>,
+	content_sha256: Option<garage_util::data::FixedBytes32>,
+) -> Result<(), Error> {
+	if let Some(expected_sha256) = content_sha256 {
+		if expected_sha256 != sha256sum {
+			return Err(Error::BadRequest(format!(
+				"Unable to validate x-amz-content-sha256"
+			)));
+		} else {
+			trace!("Successfully validated x-amz-content-sha256");
+		}
+	}
+	if let Some(expected_md5) = content_md5 {
+		if expected_md5.trim_matches('"') != base64::encode(md5sum) {
+			return Err(Error::BadRequest(format!("Unable to validate content-md5")));
+		} else {
+			trace!("Successfully validated content-md5");
+		}
+	}
+	Ok(())
 }
 
 async fn read_and_put_blocks(
@@ -87,7 +150,12 @@ async fn read_and_put_blocks(
 	first_block: Vec<u8>,
 	first_block_hash: Hash,
 	chunker: &mut BodyChunker,
-) -> Result<u64, Error> {
+) -> Result<(u64, GenericArray<u8, typenum::U16>, Hash), Error> {
+	let mut md5sum = Md5::new();
+	let mut sha256sum = Sha256::new();
+	md5sum.update(&first_block[..]);
+	sha256sum.input(&first_block[..]);
+
 	let mut next_offset = first_block.len();
 	let mut put_curr_version_block = put_block_meta(
 		garage.clone(),
@@ -105,6 +173,8 @@ async fn read_and_put_blocks(
 		let (_, _, next_block) =
 			futures::try_join!(put_curr_block, put_curr_version_block, chunker.next())?;
 		if let Some(block) = next_block {
+			md5sum.update(&block[..]);
+			sha256sum.input(&block[..]);
 			let block_hash = hash(&block[..]);
 			let block_len = block.len();
 			put_curr_version_block = put_block_meta(
@@ -122,7 +192,15 @@ async fn read_and_put_blocks(
 		}
 	}
 
-	Ok(next_offset as u64)
+	let total_size = next_offset as u64;
+	let md5sum_arr = md5sum.finalize();
+
+	let sha256sum = sha256sum.result();
+	let mut hash = [0u8; 32];
+	hash.copy_from_slice(&sha256sum[..]);
+	let sha256sum = Hash::from(hash);
+
+	Ok((total_size, md5sum_arr, sha256sum))
 }
 
 async fn put_block_meta(
@@ -132,7 +210,7 @@ async fn put_block_meta(
 	offset: u64,
 	hash: Hash,
 	size: u64,
-) -> Result<(), Error> {
+) -> Result<(), GarageError> {
 	// TODO: don't clone, restart from empty block list ??
 	let mut version = version.clone();
 	version
@@ -170,10 +248,10 @@ impl BodyChunker {
 			body,
 			read_all: false,
 			block_size,
-			buf: VecDeque::new(),
+			buf: VecDeque::with_capacity(2 * block_size),
 		}
 	}
-	async fn next(&mut self) -> Result<Option<Vec<u8>>, Error> {
+	async fn next(&mut self) -> Result<Option<Vec<u8>>, GarageError> {
 		while !self.read_all && self.buf.len() < self.block_size {
 			if let Some(block) = self.body.next().await {
 				let bytes = block?;
@@ -195,10 +273,11 @@ impl BodyChunker {
 	}
 }
 
-pub fn put_response(version_uuid: UUID) -> Response<BodyType> {
+pub fn put_response(version_uuid: UUID, md5sum_hex: String) -> Response<Body> {
 	Response::builder()
 		.header("x-amz-version-id", hex::encode(version_uuid))
-		.body(empty_body())
+		.header("ETag", format!("\"{}\"", md5sum_hex))
+		.body(Body::from(vec![]))
 		.unwrap()
 }
 
@@ -207,17 +286,14 @@ pub async fn handle_create_multipart_upload(
 	req: &Request<Body>,
 	bucket: &str,
 	key: &str,
-) -> Result<Response<BodyType>, Error> {
+) -> Result<Response<Body>, Error> {
 	let version_uuid = gen_uuid();
-	let mime_type = get_mime_type(req)?;
+	let headers = get_headers(req)?;
 
 	let object_version = ObjectVersion {
 		uuid: version_uuid,
 		timestamp: now_msec(),
-		mime_type,
-		size: 0,
-		state: ObjectVersionState::Uploading,
-		data: ObjectVersionData::Uploading,
+		state: ObjectVersionState::Uploading(headers),
 	};
 	let object = Object::new(bucket.to_string(), key.to_string(), vec![object_version]);
 	garage.object_table.insert(&object).await?;
@@ -239,7 +315,7 @@ pub async fn handle_create_multipart_upload(
 	.unwrap();
 	writeln!(&mut xml, "</InitiateMultipartUploadResult>").unwrap();
 
-	Ok(Response::new(Box::new(BytesBody::from(xml.into_bytes()))))
+	Ok(Response::new(Body::from(xml.into_bytes())))
 }
 
 pub async fn handle_put_part(
@@ -249,14 +325,19 @@ pub async fn handle_put_part(
 	key: &str,
 	part_number_str: &str,
 	upload_id: &str,
-) -> Result<Response<BodyType>, Error> {
+	content_sha256: Option<Hash>,
+) -> Result<Response<Body>, Error> {
 	// Check parameters
 	let part_number = part_number_str
 		.parse::<u64>()
-		.map_err(|e| Error::BadRequest(format!("Invalid part number: {}", e)))?;
+		.ok_or_bad_request("Invalid part number")?;
 
-	let version_uuid =
-		uuid_from_str(upload_id).map_err(|_| Error::BadRequest(format!("Invalid upload ID")))?;
+	let version_uuid = decode_upload_id(upload_id)?;
+
+	let content_md5 = match req.headers().get("content-md5") {
+		Some(x) => Some(x.to_str()?.to_string()),
+		None => None,
+	};
 
 	// Read first chuck, and at the same time try to get object to see if it exists
 	let mut chunker = BodyChunker::new(req.into_body(), garage.config.block_size);
@@ -268,28 +349,23 @@ pub async fn handle_put_part(
 	let (object, first_block) = futures::try_join!(get_object_fut, get_first_block_fut)?;
 
 	// Check object is valid and multipart block can be accepted
-	let first_block = match first_block {
-		None => return Err(Error::BadRequest(format!("Empty body"))),
-		Some(x) => x,
-	};
-	let object = match object {
-		None => return Err(Error::BadRequest(format!("Object not found"))),
-		Some(x) => x,
-	};
-	if !object.versions().iter().any(|v| {
-		v.uuid == version_uuid
-			&& v.state == ObjectVersionState::Uploading
-			&& v.data == ObjectVersionData::Uploading
-	}) {
+	let first_block = first_block.ok_or(Error::BadRequest(format!("Empty body")))?;
+	let object = object.ok_or(Error::BadRequest(format!("Object not found")))?;
+
+	if !object
+		.versions()
+		.iter()
+		.any(|v| v.uuid == version_uuid && v.is_uploading())
+	{
 		return Err(Error::BadRequest(format!(
 			"Multipart upload does not exist or is otherwise invalid"
 		)));
 	}
 
 	// Copy block to store
-	let version = Version::new(version_uuid, bucket.into(), key.into(), false, vec![]);
+	let version = Version::new(version_uuid, bucket, key, false, vec![]);
 	let first_block_hash = hash(&first_block[..]);
-	read_and_put_blocks(
+	let (_, md5sum_arr, sha256sum) = read_and_put_blocks(
 		&garage,
 		version,
 		part_number,
@@ -299,7 +375,18 @@ pub async fn handle_put_part(
 	)
 	.await?;
 
-	Ok(Response::new(Box::new(BytesBody::from(vec![]))))
+	ensure_checksum_matches(
+		md5sum_arr.as_slice(),
+		sha256sum,
+		content_md5.as_deref(),
+		content_sha256,
+	)?;
+
+	let response = Response::builder()
+		.header("ETag", format!("\"{}\"", hex::encode(md5sum_arr)))
+		.body(Body::from(vec![]))
+		.unwrap();
+	Ok(response)
 }
 
 pub async fn handle_complete_multipart_upload(
@@ -308,9 +395,8 @@ pub async fn handle_complete_multipart_upload(
 	bucket: &str,
 	key: &str,
 	upload_id: &str,
-) -> Result<Response<BodyType>, Error> {
-	let version_uuid =
-		uuid_from_str(upload_id).map_err(|_| Error::BadRequest(format!("Invalid upload ID")))?;
+) -> Result<Response<Body>, Error> {
+	let version_uuid = decode_upload_id(upload_id)?;
 
 	let bucket = bucket.to_string();
 	let key = key.to_string();
@@ -318,15 +404,12 @@ pub async fn handle_complete_multipart_upload(
 		garage.object_table.get(&bucket, &key),
 		garage.version_table.get(&version_uuid, &EmptyKey),
 	)?;
-	let object = match object {
-		None => return Err(Error::BadRequest(format!("Object not found"))),
-		Some(x) => x,
-	};
-	let object_version = object.versions().iter().find(|v| {
-		v.uuid == version_uuid
-			&& v.state == ObjectVersionState::Uploading
-			&& v.data == ObjectVersionData::Uploading
-	});
+	let object = object.ok_or(Error::BadRequest(format!("Object not found")))?;
+
+	let object_version = object
+		.versions()
+		.iter()
+		.find(|v| v.uuid == version_uuid && v.is_uploading());
 	let mut object_version = match object_version {
 		None => {
 			return Err(Error::BadRequest(format!(
@@ -335,15 +418,34 @@ pub async fn handle_complete_multipart_upload(
 		}
 		Some(x) => x.clone(),
 	};
-	let version = match version {
-		None => return Err(Error::BadRequest(format!("Version not found"))),
-		Some(x) => x,
-	};
+	let version = version.ok_or(Error::BadRequest(format!("Version not found")))?;
+
 	if version.blocks().len() == 0 {
 		return Err(Error::BadRequest(format!("No data was uploaded")));
 	}
+	let headers = match object_version.state {
+		ObjectVersionState::Uploading(headers) => headers.clone(),
+		_ => unreachable!(),
+	};
+
+	// ETag calculation: we produce ETags that have the same form as
+	// those of S3 multipart uploads, but we don't use their actual
+	// calculation for the first part (we use random bytes). This
+	// shouldn't impact compatibility as the S3 docs specify that
+	// the ETag is an opaque value in case of a multipart upload.
+	// See also: https://teppen.io/2018/06/23/aws_s3_etags/
+	let num_parts = version.blocks().last().unwrap().part_number
+		- version.blocks().first().unwrap().part_number
+		+ 1;
+	let etag = format!(
+		"{}-{}",
+		hex::encode(&rand::random::<[u8; 16]>()[..]),
+		num_parts
+	);
 
 	// TODO: check that all the parts that they pretend they gave us are indeed there
+	// TODO: when we read the XML from _req, remember to check the sha256 sum of the payload
+	//       against the signed x-amz-content-sha256
 	// TODO: check MD5 sum of all uploaded parts? but that would mean we have to store them somewhere...
 
 	let total_size = version
@@ -351,9 +453,15 @@ pub async fn handle_complete_multipart_upload(
 		.iter()
 		.map(|x| x.size)
 		.fold(0, |x, y| x + y);
-	object_version.size = total_size;
-	object_version.state = ObjectVersionState::Complete;
-	object_version.data = ObjectVersionData::FirstBlock(version.blocks()[0].hash);
+	object_version.state = ObjectVersionState::Complete(ObjectVersionData::FirstBlock(
+		ObjectVersionMeta {
+			headers,
+			size: total_size,
+			etag: etag,
+		},
+		version.blocks()[0].hash,
+	));
+
 	let final_object = Object::new(bucket.clone(), key.clone(), vec![object_version]);
 	garage.object_table.insert(&final_object).await?;
 
@@ -374,7 +482,7 @@ pub async fn handle_complete_multipart_upload(
 	writeln!(&mut xml, "\t<Key>{}</Key>", xml_escape(&key)).unwrap();
 	writeln!(&mut xml, "</CompleteMultipartUploadResult>").unwrap();
 
-	Ok(Response::new(Box::new(BytesBody::from(xml.into_bytes()))))
+	Ok(Response::new(Body::from(xml.into_bytes())))
 }
 
 pub async fn handle_abort_multipart_upload(
@@ -382,23 +490,19 @@ pub async fn handle_abort_multipart_upload(
 	bucket: &str,
 	key: &str,
 	upload_id: &str,
-) -> Result<Response<BodyType>, Error> {
-	let version_uuid =
-		uuid_from_str(upload_id).map_err(|_| Error::BadRequest(format!("Invalid upload ID")))?;
+) -> Result<Response<Body>, Error> {
+	let version_uuid = decode_upload_id(upload_id)?;
 
 	let object = garage
 		.object_table
 		.get(&bucket.to_string(), &key.to_string())
 		.await?;
-	let object = match object {
-		None => return Err(Error::BadRequest(format!("Object not found"))),
-		Some(x) => x,
-	};
-	let object_version = object.versions().iter().find(|v| {
-		v.uuid == version_uuid
-			&& v.state == ObjectVersionState::Uploading
-			&& v.data == ObjectVersionData::Uploading
-	});
+	let object = object.ok_or(Error::BadRequest(format!("Object not found")))?;
+
+	let object_version = object
+		.versions()
+		.iter()
+		.find(|v| v.uuid == version_uuid && v.is_uploading());
 	let mut object_version = match object_version {
 		None => {
 			return Err(Error::BadRequest(format!(
@@ -412,7 +516,7 @@ pub async fn handle_abort_multipart_upload(
 	let final_object = Object::new(bucket.to_string(), key.to_string(), vec![object_version]);
 	garage.object_table.insert(&final_object).await?;
 
-	Ok(Response::new(Box::new(BytesBody::from(vec![]))))
+	Ok(Response::new(Body::from(vec![])))
 }
 
 fn get_mime_type(req: &Request<Body>) -> Result<String, Error> {
@@ -424,10 +528,33 @@ fn get_mime_type(req: &Request<Body>) -> Result<String, Error> {
 		.to_string())
 }
 
-fn uuid_from_str(id: &str) -> Result<UUID, ()> {
-	let id_bin = hex::decode(id).map_err(|_| ())?;
+fn get_headers(req: &Request<Body>) -> Result<ObjectVersionHeaders, Error> {
+	let content_type = get_mime_type(req)?;
+	let other_headers = vec![
+		hyper::header::CACHE_CONTROL,
+		hyper::header::CONTENT_DISPOSITION,
+		hyper::header::CONTENT_ENCODING,
+		hyper::header::CONTENT_LANGUAGE,
+		hyper::header::EXPIRES,
+	];
+	let mut other = BTreeMap::new();
+	for h in other_headers.iter() {
+		if let Some(v) = req.headers().get(h) {
+			if let Ok(v_str) = v.to_str() {
+				other.insert(h.to_string(), v_str.to_string());
+			}
+		}
+	}
+	Ok(ObjectVersionHeaders {
+		content_type,
+		other: BTreeMap::new(),
+	})
+}
+
+fn decode_upload_id(id: &str) -> Result<UUID, Error> {
+	let id_bin = hex::decode(id).ok_or_bad_request("Invalid upload ID")?;
 	if id_bin.len() != 32 {
-		return Err(());
+		return None.ok_or_bad_request("Invalid upload ID");
 	}
 	let mut uuid = [0u8; 32];
 	uuid.copy_from_slice(&id_bin[..]);

src/api/signature.rs

@@ -6,19 +6,23 @@ use hyper::{Body, Method, Request};
 use sha2::{Digest, Sha256};
 
 use garage_table::*;
-use garage_util::error::Error;
+use garage_util::data::Hash;
 
 use garage_model::garage::Garage;
 use garage_model::key_table::*;
 
 use crate::encoding::uri_encode;
+use crate::error::*;
 
 const SHORT_DATE: &str = "%Y%m%d";
 const LONG_DATETIME: &str = "%Y%m%dT%H%M%SZ";
 
 type HmacSha256 = Hmac<Sha256>;
 
-pub async fn check_signature(garage: &Garage, request: &Request<Body>) -> Result<Key, Error> {
+pub async fn check_signature(
+	garage: &Garage,
+	request: &Request<Body>,
+) -> Result<(Key, Option<Hash>), Error> {
 	let mut headers = HashMap::new();
 	for (key, val) in request.headers() {
 		headers.insert(key.to_string(), val.to_str()?.to_string());
@@ -38,9 +42,9 @@ pub async fn check_signature(garage: &Garage, request: &Request<Body>) -> Result
 
 	let date = headers
 		.get("x-amz-date")
-		.ok_or(Error::BadRequest("Missing X-Amz-Date field".into()))?;
+		.ok_or_bad_request("Missing X-Amz-Date field")?;
 	let date: NaiveDateTime = NaiveDateTime::parse_from_str(date, LONG_DATETIME)
-		.map_err(|e| Error::BadRequest(format!("Invalid date: {}", e)))?
+		.ok_or_bad_request("Invalid date")?
 		.into();
 	let date: DateTime<Utc> = DateTime::from_utc(date, Utc);
 
@@ -64,7 +68,7 @@ pub async fn check_signature(garage: &Garage, request: &Request<Body>) -> Result
 		.key_table
 		.get(&EmptyKey, &authorization.key_id)
 		.await?
-		.filter(|k| !k.deleted)
+		.filter(|k| !k.deleted.get())
 		.ok_or(Error::Forbidden(format!(
 			"No such key: {}",
 			authorization.key_id
@@ -86,15 +90,31 @@ pub async fn check_signature(garage: &Garage, request: &Request<Body>) -> Result
 		&garage.config.s3_api.s3_region,
 		"s3",
 	)
-	.map_err(|e| Error::Message(format!("Unable to build signing HMAC: {}", e)))?;
+	.ok_or_internal_error("Unable to build signing HMAC")?;
 	hmac.input(string_to_sign.as_bytes());
 	let signature = hex::encode(hmac.result().code());
 
 	if authorization.signature != signature {
+		trace!("Canonical request: ``{}``", canonical_request);
+		trace!("String to sign: ``{}``", string_to_sign);
+		trace!("Expected: {}, got: {}", signature, authorization.signature);
 		return Err(Error::Forbidden(format!("Invalid signature")));
 	}
 
-	Ok(key)
+	let content_sha256 = if authorization.content_sha256 == "UNSIGNED-PAYLOAD" {
+		None
+	} else {
+		let bytes = hex::decode(authorization.content_sha256)
+			.ok_or_bad_request("Invalid content sha256 hash")?;
+		let mut hash = [0u8; 32];
+		if bytes.len() != 32 {
+			return Err(Error::BadRequest(format!("Invalid content sha256 hash")));
+		}
+		hash.copy_from_slice(&bytes[..]);
+		Some(Hash::from(hash))
+	};
+
+	Ok((key, content_sha256))
 }
 
 struct Authorization {
@@ -111,7 +131,7 @@ fn parse_authorization(
 ) -> Result<Authorization, Error> {
 	let first_space = authorization
 		.find(' ')
-		.ok_or(Error::BadRequest("Authorization field too short".into()))?;
+		.ok_or_bad_request("Authorization field to short")?;
 	let (auth_kind, rest) = authorization.split_at(first_space);
 
 	if auth_kind != "AWS4-HMAC-SHA256" {
@@ -121,41 +141,32 @@ fn parse_authorization(
 	let mut auth_params = HashMap::new();
 	for auth_part in rest.split(',') {
 		let auth_part = auth_part.trim();
-		let eq = auth_part.find('=').ok_or(Error::BadRequest(format!(
-			"Missing =value in authorization field {}",
-			auth_part
-		)))?;
+		let eq = auth_part
+			.find('=')
+			.ok_or_bad_request("Field without value in authorization header")?;
 		let (key, value) = auth_part.split_at(eq);
 		auth_params.insert(key.to_string(), value.trim_start_matches('=').to_string());
 	}
 
 	let cred = auth_params
 		.get("Credential")
-		.ok_or(Error::BadRequest(format!(
-			"Could not find Credential in Authorization field"
-		)))?;
+		.ok_or_bad_request("Could not find Credential in Authorization field")?;
 	let (key_id, scope) = parse_credential(cred)?;
 
 	let content_sha256 = headers
 		.get("x-amz-content-sha256")
-		.ok_or(Error::BadRequest(
-			"Missing X-Amz-Content-Sha256 field".into(),
-		))?;
+		.ok_or_bad_request("Missing X-Amz-Content-Sha256 field")?;
 
 	let auth = Authorization {
 		key_id,
 		scope,
 		signed_headers: auth_params
 			.get("SignedHeaders")
-			.ok_or(Error::BadRequest(format!(
-				"Could not find SignedHeaders in Authorization field"
-			)))?
+			.ok_or_bad_request("Could not find SignedHeaders in Authorization field")?
 			.to_string(),
 		signature: auth_params
 			.get("Signature")
-			.ok_or(Error::BadRequest(format!(
-				"Could not find Signature in Authorization field"
-			)))?
+			.ok_or_bad_request("Could not find Signature in Authorization field")?
 			.to_string(),
 		content_sha256: content_sha256.to_string(),
 	};
@@ -165,9 +176,7 @@ fn parse_authorization(
 fn parse_query_authorization(headers: &HashMap<String, String>) -> Result<Authorization, Error> {
 	let algo = headers
 		.get("x-amz-algorithm")
-		.ok_or(Error::BadRequest(format!(
-			"X-Amz-Algorithm not found in query parameters"
-		)))?;
+		.ok_or_bad_request("X-Amz-Algorithm not found in query parameters")?;
 	if algo != "AWS4-HMAC-SHA256" {
 		return Err(Error::BadRequest(format!(
 			"Unsupported authorization method"
@@ -176,34 +185,32 @@ fn parse_query_authorization(headers: &HashMap<String, String>) -> Result<Author
 
 	let cred = headers
 		.get("x-amz-credential")
-		.ok_or(Error::BadRequest(format!(
-			"X-Amz-Credential not found in query parameters"
-		)))?;
+		.ok_or_bad_request("X-Amz-Credential not found in query parameters")?;
 	let (key_id, scope) = parse_credential(cred)?;
 	let signed_headers = headers
 		.get("x-amz-signedheaders")
-		.ok_or(Error::BadRequest(format!(
-			"X-Amz-SignedHeaders not found in query parameters"
-		)))?;
+		.ok_or_bad_request("X-Amz-SignedHeaders not found in query parameters")?;
 	let signature = headers
 		.get("x-amz-signature")
-		.ok_or(Error::BadRequest(format!(
-			"X-Amz-Signature not found in query parameters"
-		)))?;
+		.ok_or_bad_request("X-Amz-Signature not found in query parameters")?;
+	let content_sha256 = headers
+		.get("x-amz-content-sha256")
+		.map(|x| x.as_str())
+		.unwrap_or("UNSIGNED-PAYLOAD");
 
 	Ok(Authorization {
 		key_id,
 		scope,
 		signed_headers: signed_headers.to_string(),
 		signature: signature.to_string(),
-		content_sha256: "UNSIGNED-PAYLOAD".to_string(),
+		content_sha256: content_sha256.to_string(),
 	})
 }
 
 fn parse_credential(cred: &str) -> Result<(String, String), Error> {
-	let first_slash = cred.find('/').ok_or(Error::BadRequest(format!(
-		"Credentials does not contain / in authorization field"
-	)))?;
+	let first_slash = cred
+		.find('/')
+		.ok_or_bad_request("Credentials does not contain / in authorization field")?;
 	let (key_id, scope) = cred.split_at(first_slash);
 	Ok((
 		key_id.to_string(),

src/garage/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "garage"
-version = "0.1.0"
+version = "0.1.1"
 authors = ["Alex Auvolat <alex@adnab.me>"]
 edition = "2018"
 license = "GPL-3.0"
@@ -14,11 +14,12 @@ path = "main.rs"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-garage_util = { version = "0.1", path = "../util" }
-garage_rpc = { version = "0.1", path = "../rpc" }
-garage_table = { version = "0.1", path = "../table" }
-garage_model = { version = "0.1", path = "../model" }
-garage_api = { version = "0.1", path = "../api" }
+garage_util = { version = "0.1.1", path = "../util" }
+garage_rpc = { version = "0.1.1", path = "../rpc" }
+garage_table = { version = "0.1.1", path = "../table" }
+garage_model = { version = "0.1.1", path = "../model" }
+garage_api = { version = "0.1.1", path = "../api" }
+garage_web = { version = "0.1.1", path = "../web" }
 
 bytes = "0.4"
 rand = "0.7"

src/garage/admin_rpc.rs

@@ -2,9 +2,9 @@ use std::sync::Arc;
 
 use serde::{Deserialize, Serialize};
 
-use garage_util::data::*;
 use garage_util::error::Error;
 
+use garage_table::crdt::CRDT;
 use garage_table::*;
 
 use garage_rpc::rpc_client::*;
@@ -55,7 +55,7 @@ impl AdminRpcHandler {
 					AdminRPC::BucketOperation(bo) => self2.handle_bucket_cmd(bo).await,
 					AdminRPC::KeyOperation(ko) => self2.handle_key_cmd(ko).await,
 					AdminRPC::LaunchRepair(opt) => self2.handle_launch_repair(opt).await,
-					_ => Err(Error::BadRequest(format!("Invalid RPC"))),
+					_ => Err(Error::BadRPC(format!("Invalid RPC"))),
 				}
 			}
 		});
@@ -67,7 +67,7 @@ impl AdminRpcHandler {
 				let bucket_names = self
 					.garage
 					.bucket_table
-					.get_range(&EmptyKey, None, Some(()), 10000)
+					.get_range(&EmptyKey, None, Some(DeletedFilter::NotDeleted), 10000)
 					.await?
 					.iter()
 					.map(|b| b.name.to_string())
@@ -79,61 +79,52 @@ impl AdminRpcHandler {
 				Ok(AdminRPC::BucketInfo(bucket))
 			}
 			BucketOperation::Create(query) => {
-				let bucket = self.garage.bucket_table.get(&EmptyKey, &query.name).await?;
-				if bucket.as_ref().filter(|b| !b.deleted).is_some() {
-					return Err(Error::BadRequest(format!(
-						"Bucket {} already exists",
-						query.name
-					)));
-				}
-				let new_time = match bucket {
-					Some(b) => std::cmp::max(b.timestamp + 1, now_msec()),
-					None => now_msec(),
+				let bucket = match self.garage.bucket_table.get(&EmptyKey, &query.name).await? {
+					Some(mut bucket) => {
+						if !bucket.is_deleted() {
+							return Err(Error::BadRPC(format!(
+								"Bucket {} already exists",
+								query.name
+							)));
+						}
+						bucket
+							.state
+							.update(BucketState::Present(BucketParams::new()));
+						bucket
+					}
+					None => Bucket::new(query.name.clone()),
 				};
-				self.garage
-					.bucket_table
-					.insert(&Bucket::new(query.name.clone(), new_time, false, vec![]))
-					.await?;
+				self.garage.bucket_table.insert(&bucket).await?;
 				Ok(AdminRPC::Ok(format!("Bucket {} was created.", query.name)))
 			}
 			BucketOperation::Delete(query) => {
-				let bucket = self.get_existing_bucket(&query.name).await?;
+				let mut bucket = self.get_existing_bucket(&query.name).await?;
 				let objects = self
 					.garage
 					.object_table
-					.get_range(&query.name, None, Some(()), 10)
+					.get_range(&query.name, None, Some(DeletedFilter::NotDeleted), 10)
 					.await?;
 				if !objects.is_empty() {
-					return Err(Error::BadRequest(format!(
-						"Bucket {} is not empty",
-						query.name
-					)));
+					return Err(Error::BadRPC(format!("Bucket {} is not empty", query.name)));
 				}
 				if !query.yes {
-					return Err(Error::BadRequest(format!(
+					return Err(Error::BadRPC(format!(
 						"Add --yes flag to really perform this operation"
 					)));
 				}
 				// --- done checking, now commit ---
-				for ak in bucket.authorized_keys() {
-					if let Some(key) = self.garage.key_table.get(&EmptyKey, &ak.key_id).await? {
-						if !key.deleted {
+				for (key_id, _, _) in bucket.authorized_keys() {
+					if let Some(key) = self.garage.key_table.get(&EmptyKey, key_id).await? {
+						if !key.deleted.get() {
 							self.update_key_bucket(key, &bucket.name, false, false)
 								.await?;
 						}
 					} else {
-						return Err(Error::Message(format!("Key not found: {}", ak.key_id)));
+						return Err(Error::Message(format!("Key not found: {}", key_id)));
 					}
 				}
-				self.garage
-					.bucket_table
-					.insert(&Bucket::new(
-						query.name.clone(),
-						std::cmp::max(bucket.timestamp + 1, now_msec()),
-						true,
-						vec![],
-					))
-					.await?;
+				bucket.state.update(BucketState::Deleted);
+				self.garage.bucket_table.insert(&bucket).await?;
 				Ok(AdminRPC::Ok(format!("Bucket {} was deleted.", query.name)))
 			}
 			BucketOperation::Allow(query) => {
@@ -164,6 +155,29 @@ impl AdminRpcHandler {
 					&query.key_id, &query.bucket, allow_read, allow_write
 				)))
 			}
+			BucketOperation::Website(query) => {
+				let mut bucket = self.get_existing_bucket(&query.bucket).await?;
+
+				if !(query.allow ^ query.deny) {
+					return Err(Error::Message(format!(
+						"You must specify exactly one flag, either --allow or --deny"
+					)));
+				}
+
+				if let BucketState::Present(state) = bucket.state.get_mut() {
+					state.website.update(query.allow);
+					self.garage.bucket_table.insert(&bucket).await?;
+					let msg = if query.allow {
+						format!("Website access allowed for {}", &query.bucket)
+					} else {
+						format!("Website access denied for {}", &query.bucket)
+					};
+
+					Ok(AdminRPC::Ok(msg.to_string()))
+				} else {
+					unreachable!();
+				}
+			}
 		}
 	}
 
@@ -173,10 +187,10 @@ impl AdminRpcHandler {
 				let key_ids = self
 					.garage
 					.key_table
-					.get_range(&EmptyKey, None, Some(()), 10000)
+					.get_range(&EmptyKey, None, Some(DeletedFilter::NotDeleted), 10000)
 					.await?
 					.iter()
-					.map(|k| (k.key_id.to_string(), k.name.to_string()))
+					.map(|k| (k.key_id.to_string(), k.name.get().clone()))
 					.collect::<Vec<_>>();
 				Ok(AdminRPC::KeyList(key_ids))
 			}
@@ -185,35 +199,32 @@ impl AdminRpcHandler {
 				Ok(AdminRPC::KeyInfo(key))
 			}
 			KeyOperation::New(query) => {
-				let key = Key::new(query.name, vec![]);
+				let key = Key::new(query.name);
 				self.garage.key_table.insert(&key).await?;
 				Ok(AdminRPC::KeyInfo(key))
 			}
 			KeyOperation::Rename(query) => {
 				let mut key = self.get_existing_key(&query.key_id).await?;
-				key.name_timestamp = std::cmp::max(key.name_timestamp + 1, now_msec());
-				key.name = query.new_name;
+				key.name.update(query.new_name);
 				self.garage.key_table.insert(&key).await?;
 				Ok(AdminRPC::KeyInfo(key))
 			}
 			KeyOperation::Delete(query) => {
 				let key = self.get_existing_key(&query.key_id).await?;
 				if !query.yes {
-					return Err(Error::BadRequest(format!(
+					return Err(Error::BadRPC(format!(
 						"Add --yes flag to really perform this operation"
 					)));
 				}
 				// --- done checking, now commit ---
-				for ab in key.authorized_buckets().iter() {
-					if let Some(bucket) =
-						self.garage.bucket_table.get(&EmptyKey, &ab.bucket).await?
-					{
-						if !bucket.deleted {
+				for (ab_name, _, _) in key.authorized_buckets.items().iter() {
+					if let Some(bucket) = self.garage.bucket_table.get(&EmptyKey, ab_name).await? {
+						if !bucket.is_deleted() {
 							self.update_bucket_key(bucket, &key.key_id, false, false)
 								.await?;
 						}
 					} else {
-						return Err(Error::Message(format!("Bucket not found: {}", ab.bucket)));
+						return Err(Error::Message(format!("Bucket not found: {}", ab_name)));
 					}
 				}
 				let del_key = Key::delete(key.key_id);
@@ -231,9 +242,9 @@ impl AdminRpcHandler {
 			.bucket_table
 			.get(&EmptyKey, bucket)
 			.await?
-			.filter(|b| !b.deleted)
+			.filter(|b| !b.is_deleted())
 			.map(Ok)
-			.unwrap_or(Err(Error::BadRequest(format!(
+			.unwrap_or(Err(Error::BadRPC(format!(
 				"Bucket {} does not exist",
 				bucket
 			))))
@@ -244,11 +255,12 @@ impl AdminRpcHandler {
 			.key_table
 			.get(&EmptyKey, id)
 			.await?
-			.filter(|k| !k.deleted)
+			.filter(|k| !k.deleted.get())
 			.map(Ok)
-			.unwrap_or(Err(Error::BadRequest(format!("Key {} does not exist", id))))
+			.unwrap_or(Err(Error::BadRPC(format!("Key {} does not exist", id))))
 	}
 
+	/// Update **bucket table** to inform of the new linked key
 	async fn update_bucket_key(
 		&self,
 		mut bucket: Bucket,
@@ -256,27 +268,26 @@ impl AdminRpcHandler {
 		allow_read: bool,
 		allow_write: bool,
 	) -> Result<(), Error> {
-		let timestamp = match bucket
-			.authorized_keys()
-			.iter()
-			.find(|x| x.key_id == *key_id)
-		{
-			None => now_msec(),
-			Some(ab) => std::cmp::max(ab.timestamp + 1, now_msec()),
-		};
-		bucket.clear_keys();
-		bucket
-			.add_key(AllowedKey {
-				key_id: key_id.clone(),
-				timestamp,
-				allow_read,
-				allow_write,
-			})
-			.unwrap();
+		if let BucketState::Present(params) = bucket.state.get_mut() {
+			let ak = &mut params.authorized_keys;
+			let old_ak = ak.take_and_clear();
+			ak.merge(&old_ak.update_mutator(
+				key_id.to_string(),
+				PermissionSet {
+					allow_read,
+					allow_write,
+				},
+			));
+		} else {
+			return Err(Error::Message(format!(
+				"Bucket is deleted in update_bucket_key"
+			)));
+		}
 		self.garage.bucket_table.insert(&bucket).await?;
 		Ok(())
 	}
 
+	/// Update **key table** to inform of the new linked bucket
 	async fn update_key_bucket(
 		&self,
 		mut key: Key,
@@ -284,29 +295,21 @@ impl AdminRpcHandler {
 		allow_read: bool,
 		allow_write: bool,
 	) -> Result<(), Error> {
-		let timestamp = match key
-			.authorized_buckets()
-			.iter()
-			.find(|x| x.bucket == *bucket)
-		{
-			None => now_msec(),
-			Some(ab) => std::cmp::max(ab.timestamp + 1, now_msec()),
-		};
-		key.clear_buckets();
-		key.add_bucket(AllowedBucket {
-			bucket: bucket.clone(),
-			timestamp,
-			allow_read,
-			allow_write,
-		})
-		.unwrap();
+		let old_map = key.authorized_buckets.take_and_clear();
+		key.authorized_buckets.merge(&old_map.update_mutator(
+			bucket.clone(),
+			PermissionSet {
+				allow_read,
+				allow_write,
+			},
+		));
 		self.garage.key_table.insert(&key).await?;
 		Ok(())
 	}
 
 	async fn handle_launch_repair(self: &Arc<Self>, opt: RepairOpt) -> Result<AdminRPC, Error> {
 		if !opt.yes {
-			return Err(Error::BadRequest(format!(
+			return Err(Error::BadRPC(format!(
 				"Please provide the --yes flag to initiate repair operations."
 			)));
 		}

src/garage/main.rs

@@ -141,6 +141,24 @@ pub enum BucketOperation {
 	/// Allow key to read or write to bucket
 	#[structopt(name = "deny")]
 	Deny(PermBucketOpt),
+
+	/// Expose as website or not
+	#[structopt(name = "website")]
+	Website(WebsiteOpt),
+}
+
+#[derive(Serialize, Deserialize, StructOpt, Debug)]
+pub struct WebsiteOpt {
+	/// Create
+	#[structopt(long = "allow")]
+	pub allow: bool,
+
+	/// Delete
+	#[structopt(long = "deny")]
+	pub deny: bool,
+
+	/// Bucket name
+	pub bucket: String,
 }
 
 #[derive(Serialize, Deserialize, StructOpt, Debug)]

src/garage/server.rs

@@ -11,6 +11,7 @@ use garage_util::error::Error;
 use garage_api::api_server;
 use garage_model::garage::Garage;
 use garage_rpc::rpc_server::RpcServer;
+use garage_web::web_server;
 
 use crate::admin_rpc::*;
 
@@ -56,6 +57,7 @@ pub async fn run_server(config_file: PathBuf) -> Result<(), Error> {
 	info!("Initializing RPC and API servers...");
 	let run_rpc_server = Arc::new(rpc_server).run(wait_from(watch_cancel.clone()));
 	let api_server = api_server::run_api_server(garage.clone(), wait_from(watch_cancel.clone()));
+	let web_server = web_server::run_web_server(garage.clone(), wait_from(watch_cancel.clone()));
 
 	futures::try_join!(
 		garage
@@ -78,6 +80,10 @@ pub async fn run_server(config_file: PathBuf) -> Result<(), Error> {
 			info!("API server exited");
 			rv
 		}),
+		web_server.map(|rv| {
+			info!("Web server exited");
+			rv
+		}),
 		background.run().map(|rv| {
 			info!("Background runner exited");
 			Ok(rv)

src/model/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "garage_model"
-version = "0.1.0"
+version = "0.1.1"
 authors = ["Alex Auvolat <alex@adnab.me>"]
 edition = "2018"
 license = "GPL-3.0"
@@ -13,9 +13,10 @@ path = "lib.rs"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-garage_util = { version = "0.1", path = "../util" }
-garage_rpc = { version = "0.1", path = "../rpc" }
-garage_table = { version = "0.1", path = "../table" }
+garage_util = { version = "0.1.1", path = "../util" }
+garage_rpc = { version = "0.1.1", path = "../rpc" }
+garage_table = { version = "0.1.1", path = "../table" }
+model010 = { package = "garage_model_010b", version = "0.0.1" }
 
 bytes = "0.4"
 rand = "0.7"

src/model/block.rs

@@ -20,7 +20,7 @@ use garage_rpc::rpc_client::*;
 use garage_rpc::rpc_server::*;
 
 use garage_table::table_sharded::TableShardedReplication;
-use garage_table::TableReplication;
+use garage_table::{DeletedFilter, TableReplication};
 
 use crate::block_ref_table::*;
 
@@ -123,7 +123,7 @@ impl BlockManager {
 			Message::PutBlock(m) => self.write_block(&m.hash, &m.data).await,
 			Message::GetBlock(h) => self.read_block(h).await,
 			Message::NeedBlockQuery(h) => self.need_block(h).await.map(Message::NeedBlockReply),
-			_ => Err(Error::BadRequest(format!("Unexpected RPC message"))),
+			_ => Err(Error::BadRPC(format!("Unexpected RPC message"))),
 		}
 	}
 
@@ -306,7 +306,7 @@ impl BlockManager {
 			let garage = self.garage.load_full().unwrap();
 			let active_refs = garage
 				.block_ref_table
-				.get_range(&hash, None, Some(()), 1)
+				.get_range(&hash, None, Some(DeletedFilter::NotDeleted), 1)
 				.await?;
 			let needed_by_others = !active_refs.is_empty();
 			if needed_by_others {

src/model/block_ref_table.rs

@@ -47,7 +47,7 @@ impl TableSchema for BlockRefTable {
 	type P = Hash;
 	type S = UUID;
 	type E = BlockRef;
-	type Filter = ();
+	type Filter = DeletedFilter;
 
 	async fn updated(&self, old: Option<Self::E>, new: Option<Self::E>) -> Result<(), Error> {
 		let block = &old.as_ref().or(new.as_ref()).unwrap().block;
@@ -62,7 +62,7 @@ impl TableSchema for BlockRefTable {
 		Ok(())
 	}
 
-	fn matches_filter(entry: &Self::E, _filter: &Self::Filter) -> bool {
-		!entry.deleted
+	fn matches_filter(entry: &Self::E, filter: &Self::Filter) -> bool {
+		filter.apply(entry.deleted)
 	}
 }

src/model/bucket_table.rs

@@ -1,69 +1,88 @@
 use async_trait::async_trait;
 use serde::{Deserialize, Serialize};
 
+use garage_table::crdt::CRDT;
 use garage_table::*;
+
 use garage_util::error::Error;
 
+use crate::key_table::PermissionSet;
+
+// We import the same file but in its version 0.1.0.
+// We can then access v0.1.0 data structures.
+// We use them to perform migrations.
+use model010::bucket_table as prev;
+
+/// A bucket is a collection of objects
+///
+/// Its parameters are not directly accessible as:
+///  - It must be possible to merge paramaters, hence the use of a LWW CRDT.
+///  - A bucket has 2 states, Present or Deleted and parameters make sense only if present.
 #[derive(PartialEq, Clone, Debug, Serialize, Deserialize)]
 pub struct Bucket {
 	// Primary key
 	pub name: String,
 
-	// Timestamp and deletion
-	// Upon version increment, all info is replaced
-	pub timestamp: u64,
-	pub deleted: bool,
-
-	// Authorized keys
-	authorized_keys: Vec<AllowedKey>,
+	pub state: crdt::LWW<BucketState>,
 }
 
-impl Bucket {
-	pub fn new(
-		name: String,
-		timestamp: u64,
-		deleted: bool,
-		authorized_keys: Vec<AllowedKey>,
-	) -> Self {
-		let mut ret = Bucket {
-			name,
-			timestamp,
-			deleted,
-			authorized_keys: vec![],
-		};
-		for key in authorized_keys {
-			ret.add_key(key)
-				.expect("Duplicate AllowedKey in Bucket constructor");
-		}
-		ret
-	}
-	/// Add a key only if it is not already present
-	pub fn add_key(&mut self, key: AllowedKey) -> Result<(), ()> {
-		match self
-			.authorized_keys
-			.binary_search_by(|k| k.key_id.cmp(&key.key_id))
-		{
-			Err(i) => {
-				self.authorized_keys.insert(i, key);
-				Ok(())
+#[derive(PartialEq, Clone, Debug, Serialize, Deserialize)]
+pub enum BucketState {
+	Deleted,
+	Present(BucketParams),
+}
+
+impl CRDT for BucketState {
+	fn merge(&mut self, o: &Self) {
+		match o {
+			BucketState::Deleted => *self = BucketState::Deleted,
+			BucketState::Present(other_params) => {
+				if let BucketState::Present(params) = self {
+					params.merge(other_params);
+				}
 			}
-			Ok(_) => Err(()),
 		}
 	}
-	pub fn authorized_keys(&self) -> &[AllowedKey] {
-		&self.authorized_keys[..]
-	}
-	pub fn clear_keys(&mut self) {
-		self.authorized_keys.clear();
-	}
 }
 
 #[derive(PartialEq, Clone, Debug, Serialize, Deserialize)]
-pub struct AllowedKey {
-	pub key_id: String,
-	pub timestamp: u64,
-	pub allow_read: bool,
-	pub allow_write: bool,
+pub struct BucketParams {
+	pub authorized_keys: crdt::LWWMap<String, PermissionSet>,
+	pub website: crdt::LWW<bool>,
+}
+
+impl CRDT for BucketParams {
+	fn merge(&mut self, o: &Self) {
+		self.authorized_keys.merge(&o.authorized_keys);
+		self.website.merge(&o.website);
+	}
+}
+
+impl BucketParams {
+	pub fn new() -> Self {
+		BucketParams {
+			authorized_keys: crdt::LWWMap::new(),
+			website: crdt::LWW::new(false),
+		}
+	}
+}
+
+impl Bucket {
+	pub fn new(name: String) -> Self {
+		Bucket {
+			name,
+			state: crdt::LWW::new(BucketState::Present(BucketParams::new())),
+		}
+	}
+	pub fn is_deleted(&self) -> bool {
+		*self.state.get() == BucketState::Deleted
+	}
+	pub fn authorized_keys(&self) -> &[(String, u64, PermissionSet)] {
+		match self.state.get() {
+			BucketState::Deleted => &[],
+			BucketState::Present(state) => state.authorized_keys.items(),
+		}
+	}
 }
 
 impl Entry<EmptyKey, String> for Bucket {
@@ -75,30 +94,7 @@ impl Entry<EmptyKey, String> for Bucket {
 	}
 
 	fn merge(&mut self, other: &Self) {
-		if other.timestamp < self.timestamp {
-			*self = other.clone();
-			return;
-		}
-		if self.timestamp > other.timestamp || self.deleted {
-			return;
-		}
-
-		for ak in other.authorized_keys.iter() {
-			match self
-				.authorized_keys
-				.binary_search_by(|our_ak| our_ak.key_id.cmp(&ak.key_id))
-			{
-				Ok(i) => {
-					let our_ak = &mut self.authorized_keys[i];
-					if ak.timestamp > our_ak.timestamp {
-						*our_ak = ak.clone();
-					}
-				}
-				Err(i) => {
-					self.authorized_keys.insert(i, ak.clone());
-				}
-			}
-		}
+		self.state.merge(&other.state);
 	}
 }
 
@@ -109,13 +105,48 @@ impl TableSchema for BucketTable {
 	type P = EmptyKey;
 	type S = String;
 	type E = Bucket;
-	type Filter = ();
+	type Filter = DeletedFilter;
 
 	async fn updated(&self, _old: Option<Self::E>, _new: Option<Self::E>) -> Result<(), Error> {
 		Ok(())
 	}
 
-	fn matches_filter(entry: &Self::E, _filter: &Self::Filter) -> bool {
-		!entry.deleted
+	fn matches_filter(entry: &Self::E, filter: &Self::Filter) -> bool {
+		filter.apply(entry.is_deleted())
+	}
+
+	fn try_migrate(bytes: &[u8]) -> Option<Self::E> {
+		let old = match rmp_serde::decode::from_read_ref::<_, prev::Bucket>(bytes) {
+			Ok(x) => x,
+			Err(_) => return None,
+		};
+		if old.deleted {
+			Some(Bucket {
+				name: old.name,
+				state: crdt::LWW::migrate_from_raw(old.timestamp, BucketState::Deleted),
+			})
+		} else {
+			let mut keys = crdt::LWWMap::new();
+			for ak in old.authorized_keys() {
+				keys.merge(&crdt::LWWMap::migrate_from_raw_item(
+					ak.key_id.clone(),
+					ak.timestamp,
+					PermissionSet {
+						allow_read: ak.allow_read,
+						allow_write: ak.allow_write,
+					},
+				));
+			}
+
+			let params = BucketParams {
+				authorized_keys: keys,
+				website: crdt::LWW::new(false),
+			};
+
+			Some(Bucket {
+				name: old.name,
+				state: crdt::LWW::migrate_from_raw(old.timestamp, BucketState::Present(params)),
+			})
+		}
 	}
 }

src/model/key_table.rs

@@ -1,10 +1,13 @@
 use async_trait::async_trait;
 use serde::{Deserialize, Serialize};
 
+use garage_table::crdt::CRDT;
 use garage_table::*;
-use garage_util::data::*;
+
 use garage_util::error::Error;
 
+use model010::key_table as prev;
+
 #[derive(PartialEq, Clone, Debug, Serialize, Deserialize)]
 pub struct Key {
 	// Primary key
@@ -14,83 +17,54 @@ pub struct Key {
 	pub secret_key: String,
 
 	// Name
-	pub name: String,
-	pub name_timestamp: u64,
+	pub name: crdt::LWW<String>,
 
 	// Deletion
-	pub deleted: bool,
+	pub deleted: crdt::Bool,
 
 	// Authorized keys
-	authorized_buckets: Vec<AllowedBucket>,
+	pub authorized_buckets: crdt::LWWMap<String, PermissionSet>,
+	// CRDT interaction: deleted implies authorized_buckets is empty
 }
 
 impl Key {
-	pub fn new(name: String, buckets: Vec<AllowedBucket>) -> Self {
+	pub fn new(name: String) -> Self {
 		let key_id = format!("GK{}", hex::encode(&rand::random::<[u8; 12]>()[..]));
 		let secret_key = hex::encode(&rand::random::<[u8; 32]>()[..]);
-		let mut ret = Self {
+		Self {
 			key_id,
 			secret_key,
-			name,
-			name_timestamp: now_msec(),
-			deleted: false,
-			authorized_buckets: vec![],
-		};
-		for b in buckets {
-			ret.add_bucket(b)
-				.expect("Duplicate AllowedBucket in Key constructor");
+			name: crdt::LWW::new(name),
+			deleted: crdt::Bool::new(false),
+			authorized_buckets: crdt::LWWMap::new(),
 		}
-		ret
 	}
 	pub fn delete(key_id: String) -> Self {
 		Self {
 			key_id,
 			secret_key: "".into(),
-			name: "".into(),
-			name_timestamp: now_msec(),
-			deleted: true,
-			authorized_buckets: vec![],
+			name: crdt::LWW::new("".to_string()),
+			deleted: crdt::Bool::new(true),
+			authorized_buckets: crdt::LWWMap::new(),
 		}
 	}
 	/// Add an authorized bucket, only if it wasn't there before
-	pub fn add_bucket(&mut self, new: AllowedBucket) -> Result<(), ()> {
-		match self
-			.authorized_buckets
-			.binary_search_by(|b| b.bucket.cmp(&new.bucket))
-		{
-			Err(i) => {
-				self.authorized_buckets.insert(i, new);
-				Ok(())
-			}
-			Ok(_) => Err(()),
-		}
-	}
-	pub fn authorized_buckets(&self) -> &[AllowedBucket] {
-		&self.authorized_buckets[..]
-	}
-	pub fn clear_buckets(&mut self) {
-		self.authorized_buckets.clear();
-	}
 	pub fn allow_read(&self, bucket: &str) -> bool {
 		self.authorized_buckets
-			.iter()
-			.find(|x| x.bucket.as_str() == bucket)
+			.get(&bucket.to_string())
 			.map(|x| x.allow_read)
 			.unwrap_or(false)
 	}
 	pub fn allow_write(&self, bucket: &str) -> bool {
 		self.authorized_buckets
-			.iter()
-			.find(|x| x.bucket.as_str() == bucket)
+			.get(&bucket.to_string())
 			.map(|x| x.allow_write)
 			.unwrap_or(false)
 	}
 }
 
-#[derive(PartialEq, Clone, Debug, Serialize, Deserialize)]
-pub struct AllowedBucket {
-	pub bucket: String,
-	pub timestamp: u64,
+#[derive(PartialOrd, Ord, PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
+pub struct PermissionSet {
 	pub allow_read: bool,
 	pub allow_write: bool,
 }
@@ -104,34 +78,15 @@ impl Entry<EmptyKey, String> for Key {
 	}
 
 	fn merge(&mut self, other: &Self) {
-		if other.deleted {
-			self.deleted = true;
-		}
-		if self.deleted {
+		self.name.merge(&other.name);
+		self.deleted.merge(&other.deleted);
+
+		if self.deleted.get() {
 			self.authorized_buckets.clear();
 			return;
 		}
-		if other.name_timestamp > self.name_timestamp {
-			self.name_timestamp = other.name_timestamp;
-			self.name = other.name.clone();
-		}
 
-		for ab in other.authorized_buckets.iter() {
-			match self
-				.authorized_buckets
-				.binary_search_by(|our_ab| our_ab.bucket.cmp(&ab.bucket))
-			{
-				Ok(i) => {
-					let our_ab = &mut self.authorized_buckets[i];
-					if ab.timestamp > our_ab.timestamp {
-						*our_ab = ab.clone();
-					}
-				}
-				Err(i) => {
-					self.authorized_buckets.insert(i, ab.clone());
-				}
-			}
-		}
+		self.authorized_buckets.merge(&other.authorized_buckets);
 	}
 }
 
@@ -142,13 +97,39 @@ impl TableSchema for KeyTable {
 	type P = EmptyKey;
 	type S = String;
 	type E = Key;
-	type Filter = ();
+	type Filter = DeletedFilter;
 
 	async fn updated(&self, _old: Option<Self::E>, _new: Option<Self::E>) -> Result<(), Error> {
 		Ok(())
 	}
 
-	fn matches_filter(entry: &Self::E, _filter: &Self::Filter) -> bool {
-		!entry.deleted
+	fn matches_filter(entry: &Self::E, filter: &Self::Filter) -> bool {
+		filter.apply(entry.deleted.get())
+	}
+
+	fn try_migrate(bytes: &[u8]) -> Option<Self::E> {
+		let old = match rmp_serde::decode::from_read_ref::<_, prev::Key>(bytes) {
+			Ok(x) => x,
+			Err(_) => return None,
+		};
+		let mut new = Self::E {
+			key_id: old.key_id.clone(),
+			secret_key: old.secret_key.clone(),
+			name: crdt::LWW::migrate_from_raw(old.name_timestamp, old.name.clone()),
+			deleted: crdt::Bool::new(old.deleted),
+			authorized_buckets: crdt::LWWMap::new(),
+		};
+		for ab in old.authorized_buckets() {
+			let it = crdt::LWWMap::migrate_from_raw_item(
+				ab.bucket.clone(),
+				ab.timestamp,
+				PermissionSet {
+					allow_read: ab.allow_read,
+					allow_write: ab.allow_write,
+				},
+			);
+			new.authorized_buckets.merge(&it);
+		}
+		Some(new)
 	}
 }

src/model/object_table.rs

@@ -1,5 +1,6 @@
 use async_trait::async_trait;
 use serde::{Deserialize, Serialize};
+use std::collections::BTreeMap;
 use std::sync::Arc;
 
 use garage_util::background::BackgroundRunner;
@@ -11,6 +12,8 @@ use garage_table::*;
 
 use crate::version_table::*;
 
+use model010::object_table as prev;
+
 #[derive(PartialEq, Clone, Debug, Serialize, Deserialize)]
 pub struct Object {
 	// Primary key
@@ -59,50 +62,90 @@ pub struct ObjectVersion {
 	pub uuid: UUID,
 	pub timestamp: u64,
 
-	pub mime_type: String,
-	pub size: u64,
 	pub state: ObjectVersionState,
-
-	pub data: ObjectVersionData,
 }
 
-#[derive(PartialEq, Clone, Copy, Debug, Serialize, Deserialize)]
+#[derive(PartialEq, Clone, Debug, Serialize, Deserialize)]
 pub enum ObjectVersionState {
-	Uploading,
-	Complete,
+	Uploading(ObjectVersionHeaders),
+	Complete(ObjectVersionData),
 	Aborted,
 }
 
 impl ObjectVersionState {
-	fn max(self, other: Self) -> Self {
+	fn merge(&mut self, other: &Self) {
 		use ObjectVersionState::*;
-		if self == Aborted || other == Aborted {
-			Aborted
-		} else if self == Complete || other == Complete {
-			Complete
-		} else {
-			Uploading
+		match other {
+			Aborted => {
+				*self = Aborted;
+			}
+			Complete(b) => match self {
+				Aborted => {}
+				Complete(a) => {
+					a.merge(b);
+				}
+				Uploading(_) => {
+					*self = Complete(b.clone());
+				}
+			},
+			Uploading(_) => {}
 		}
 	}
 }
 
 #[derive(PartialEq, Clone, Debug, Serialize, Deserialize)]
 pub enum ObjectVersionData {
-	Uploading,
 	DeleteMarker,
-	Inline(#[serde(with = "serde_bytes")] Vec<u8>),
-	FirstBlock(Hash),
+	Inline(ObjectVersionMeta, #[serde(with = "serde_bytes")] Vec<u8>),
+	FirstBlock(ObjectVersionMeta, Hash),
+}
+
+#[derive(PartialEq, Clone, Debug, Serialize, Deserialize)]
+pub struct ObjectVersionMeta {
+	pub headers: ObjectVersionHeaders,
+	pub size: u64,
+	pub etag: String,
+}
+
+#[derive(PartialEq, Clone, Debug, Serialize, Deserialize)]
+pub struct ObjectVersionHeaders {
+	pub content_type: String,
+	pub other: BTreeMap<String, String>,
+}
+
+impl ObjectVersionData {
+	fn merge(&mut self, b: &Self) {
+		if *self != *b {
+			warn!(
+				"Inconsistent object version data: {:?} (local) vs {:?} (remote)",
+				self, b
+			);
+		}
+	}
 }
 
 impl ObjectVersion {
 	fn cmp_key(&self) -> (u64, UUID) {
 		(self.timestamp, self.uuid)
 	}
+	pub fn is_uploading(&self) -> bool {
+		match self.state {
+			ObjectVersionState::Uploading(_) => true,
+			_ => false,
+		}
+	}
 	pub fn is_complete(&self) -> bool {
-		self.state == ObjectVersionState::Complete
+		match self.state {
+			ObjectVersionState::Complete(_) => true,
+			_ => false,
+		}
 	}
 	pub fn is_data(&self) -> bool {
-		self.state == ObjectVersionState::Complete && self.data != ObjectVersionData::DeleteMarker
+		match self.state {
+			ObjectVersionState::Complete(ObjectVersionData::DeleteMarker) => false,
+			ObjectVersionState::Complete(_) => true,
+			_ => false,
+		}
 	}
 }
 
@@ -121,14 +164,7 @@ impl Entry<String, String> for Object {
 				.binary_search_by(|v| v.cmp_key().cmp(&other_v.cmp_key()))
 			{
 				Ok(i) => {
-					let mut v = &mut self.versions[i];
-					if other_v.size > v.size {
-						v.size = other_v.size;
-					}
-					v.state = v.state.max(other_v.state);
-					if v.data == ObjectVersionData::Uploading {
-						v.data = other_v.data.clone();
-					}
+					self.versions[i].state.merge(&other_v.state);
 				}
 				Err(i) => {
 					self.versions.insert(i, other_v.clone());
@@ -160,7 +196,7 @@ impl TableSchema for ObjectTable {
 	type P = String;
 	type S = String;
 	type E = Object;
-	type Filter = ();
+	type Filter = DeletedFilter;
 
 	async fn updated(&self, old: Option<Self::E>, new: Option<Self::E>) -> Result<(), Error> {
 		let version_table = self.version_table.clone();
@@ -192,7 +228,59 @@ impl TableSchema for ObjectTable {
 		Ok(())
 	}
 
-	fn matches_filter(entry: &Self::E, _filter: &Self::Filter) -> bool {
-		entry.versions.iter().any(|v| v.is_data())
+	fn matches_filter(entry: &Self::E, filter: &Self::Filter) -> bool {
+		let deleted = !entry.versions.iter().any(|v| v.is_data());
+		filter.apply(deleted)
+	}
+
+	fn try_migrate(bytes: &[u8]) -> Option<Self::E> {
+		let old = match rmp_serde::decode::from_read_ref::<_, prev::Object>(bytes) {
+			Ok(x) => x,
+			Err(_) => return None,
+		};
+		let new_v = old
+			.versions()
+			.iter()
+			.map(migrate_version)
+			.collect::<Vec<_>>();
+		let new = Object::new(old.bucket.clone(), old.key.clone(), new_v);
+		Some(new)
+	}
+}
+
+fn migrate_version(old: &prev::ObjectVersion) -> ObjectVersion {
+	let headers = ObjectVersionHeaders {
+		content_type: old.mime_type.clone(),
+		other: BTreeMap::new(),
+	};
+	let meta = ObjectVersionMeta {
+		headers: headers.clone(),
+		size: old.size,
+		etag: "".to_string(),
+	};
+	let state = match old.state {
+		prev::ObjectVersionState::Uploading => ObjectVersionState::Uploading(headers),
+		prev::ObjectVersionState::Aborted => ObjectVersionState::Aborted,
+		prev::ObjectVersionState::Complete => match &old.data {
+			prev::ObjectVersionData::Uploading => ObjectVersionState::Uploading(headers),
+			prev::ObjectVersionData::DeleteMarker => {
+				ObjectVersionState::Complete(ObjectVersionData::DeleteMarker)
+			}
+			prev::ObjectVersionData::Inline(x) => {
+				ObjectVersionState::Complete(ObjectVersionData::Inline(meta, x.clone()))
+			}
+			prev::ObjectVersionData::FirstBlock(h) => {
+				let mut hash = [0u8; 32];
+				hash.copy_from_slice(h.as_ref());
+				ObjectVersionState::Complete(ObjectVersionData::FirstBlock(meta, Hash::from(hash)))
+			}
+		},
+	};
+	let mut uuid = [0u8; 32];
+	uuid.copy_from_slice(old.uuid.as_ref());
+	ObjectVersion {
+		uuid: UUID::from(uuid),
+		timestamp: old.timestamp,
+		state,
 	}
 }

src/model/version_table.rs

@@ -117,7 +117,7 @@ impl TableSchema for VersionTable {
 	type P = Hash;
 	type S = EmptyKey;
 	type E = Version;
-	type Filter = ();
+	type Filter = DeletedFilter;
 
 	async fn updated(&self, old: Option<Self::E>, new: Option<Self::E>) -> Result<(), Error> {
 		let block_ref_table = self.block_ref_table.clone();
@@ -139,7 +139,7 @@ impl TableSchema for VersionTable {
 		Ok(())
 	}
 
-	fn matches_filter(entry: &Self::E, _filter: &Self::Filter) -> bool {
-		!entry.deleted
+	fn matches_filter(entry: &Self::E, filter: &Self::Filter) -> bool {
+		filter.apply(entry.deleted)
 	}
 }

src/rpc/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "garage_rpc"
-version = "0.1.0"
+version = "0.1.1"
 authors = ["Alex Auvolat <alex@adnab.me>"]
 edition = "2018"
 license = "GPL-3.0"
@@ -13,7 +13,7 @@ path = "lib.rs"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-garage_util = { version = "0.1", path = "../util" }
+garage_util = { version = "0.1.1", path = "../util" }
 
 bytes = "0.4"
 rand = "0.7"

src/rpc/lib.rs

@@ -5,4 +5,4 @@ pub mod consul;
 pub mod membership;
 pub mod rpc_client;
 pub mod rpc_server;
-pub mod tls_util;
+pub(crate) mod tls_util;

src/rpc/membership.rs

@@ -46,13 +46,13 @@ impl RpcMessage for Message {}
 
 #[derive(Debug, Serialize, Deserialize)]
 pub struct PingMessage {
-	pub id: UUID,
-	pub rpc_port: u16,
+	id: UUID,
+	rpc_port: u16,
 
-	pub status_hash: Hash,
-	pub config_version: u64,
+	status_hash: Hash,
+	config_version: u64,
 
-	pub state_info: StateInfo,
+	state_info: StateInfo,
 }
 
 #[derive(Clone, Debug, Serialize, Deserialize)]
@@ -81,12 +81,13 @@ pub struct NetworkConfigEntry {
 
 pub struct System {
 	pub id: UUID,
-	pub data_dir: PathBuf,
-	pub rpc_local_port: u16,
 
-	pub state_info: StateInfo,
+	metadata_dir: PathBuf,
+	rpc_local_port: u16,
 
-	pub rpc_http_client: Arc<RpcHttpClient>,
+	state_info: StateInfo,
+
+	rpc_http_client: Arc<RpcHttpClient>,
 	rpc_client: Arc<RpcClient<Message>>,
 
 	pub status: watch::Receiver<Arc<Status>>,
@@ -296,15 +297,15 @@ fn read_network_config(metadata_dir: &PathBuf) -> Result<NetworkConfig, Error> {
 
 impl System {
 	pub fn new(
-		data_dir: PathBuf,
+		metadata_dir: PathBuf,
 		rpc_http_client: Arc<RpcHttpClient>,
 		background: Arc<BackgroundRunner>,
 		rpc_server: &mut RpcServer,
 	) -> Arc<Self> {
-		let id = gen_node_id(&data_dir).expect("Unable to read or generate node ID");
+		let id = gen_node_id(&metadata_dir).expect("Unable to read or generate node ID");
 		info!("Node ID: {}", hex::encode(&id));
 
-		let net_config = match read_network_config(&data_dir) {
+		let net_config = match read_network_config(&metadata_dir) {
 			Ok(x) => x,
 			Err(e) => {
 				info!(
@@ -347,7 +348,7 @@ impl System {
 
 		let sys = Arc::new(System {
 			id,
-			data_dir,
+			metadata_dir,
 			rpc_local_port: rpc_server.bind_addr.port(),
 			state_info,
 			rpc_http_client,
@@ -373,7 +374,7 @@ impl System {
 					Message::AdvertiseNodesUp(adv) => self2.handle_advertise_nodes_up(&adv).await,
 					Message::AdvertiseConfig(adv) => self2.handle_advertise_config(&adv).await,
 
-					_ => Err(Error::BadRequest(format!("Unexpected RPC message"))),
+					_ => Err(Error::BadRPC(format!("Unexpected RPC message"))),
 				}
 			}
 		});
@@ -388,7 +389,7 @@ impl System {
 	}
 
 	async fn save_network_config(self: Arc<Self>) -> Result<(), Error> {
-		let mut path = self.data_dir.clone();
+		let mut path = self.metadata_dir.clone();
 		path.push("network_config");
 
 		let ring = self.ring.borrow().clone();
@@ -399,7 +400,7 @@ impl System {
 		Ok(())
 	}
 
-	pub fn make_ping(&self) -> Message {
+	fn make_ping(&self) -> Message {
 		let status = self.status.borrow().clone();
 		let ring = self.ring.borrow().clone();
 		Message::Ping(PingMessage {
@@ -411,7 +412,7 @@ impl System {
 		})
 	}
 
-	pub async fn broadcast(self: Arc<Self>, msg: Message, timeout: Duration) {
+	async fn broadcast(self: Arc<Self>, msg: Message, timeout: Duration) {
 		let status = self.status.borrow().clone();
 		let to = status
 			.nodes
@@ -527,7 +528,7 @@ impl System {
 		}
 	}
 
-	pub async fn handle_ping(
+	async fn handle_ping(
 		self: Arc<Self>,
 		from: &SocketAddr,
 		ping: &PingMessage,
@@ -557,7 +558,7 @@ impl System {
 		Ok(self.make_ping())
 	}
 
-	pub fn handle_pull_status(&self) -> Result<Message, Error> {
+	fn handle_pull_status(&self) -> Result<Message, Error> {
 		let status = self.status.borrow().clone();
 		let mut mem = vec![];
 		for (node, status) in status.nodes.iter() {
@@ -577,12 +578,12 @@ impl System {
 		Ok(Message::AdvertiseNodesUp(mem))
 	}
 
-	pub fn handle_pull_config(&self) -> Result<Message, Error> {
+	fn handle_pull_config(&self) -> Result<Message, Error> {
 		let ring = self.ring.borrow().clone();
 		Ok(Message::AdvertiseConfig(ring.config.clone()))
 	}
 
-	pub async fn handle_advertise_nodes_up(
+	async fn handle_advertise_nodes_up(
 		self: Arc<Self>,
 		adv: &[AdvertisedNode],
 	) -> Result<Message, Error> {
@@ -635,7 +636,7 @@ impl System {
 		Ok(Message::Ok)
 	}
 
-	pub async fn handle_advertise_config(
+	async fn handle_advertise_config(
 		self: Arc<Self>,
 		adv: &NetworkConfig,
 	) -> Result<Message, Error> {
@@ -716,7 +717,7 @@ impl System {
 		}
 	}
 
-	pub fn pull_status(
+	fn pull_status(
 		self: Arc<Self>,
 		peer: UUID,
 	) -> impl futures::future::Future<Output = ()> + Send + 'static {
@@ -731,7 +732,7 @@ impl System {
 		}
 	}
 
-	pub async fn pull_config(self: Arc<Self>, peer: UUID) {
+	async fn pull_config(self: Arc<Self>, peer: UUID) {
 		let resp = self
 			.rpc_client
 			.call(peer, Message::PullConfig, PING_TIMEOUT)

src/rpc/rpc_client.rs

@@ -61,7 +61,7 @@ pub struct RpcClient<M: RpcMessage> {
 
 	local_handler: ArcSwapOption<(UUID, LocalHandlerFn<M>)>,
 
-	pub rpc_addr_client: RpcAddrClient<M>,
+	rpc_addr_client: RpcAddrClient<M>,
 }
 
 impl<M: RpcMessage + 'static> RpcClient<M> {
@@ -215,8 +215,8 @@ impl<M: RpcMessage + 'static> RpcClient<M> {
 pub struct RpcAddrClient<M: RpcMessage> {
 	phantom: PhantomData<M>,
 
-	pub http_client: Arc<RpcHttpClient>,
-	pub path: String,
+	http_client: Arc<RpcHttpClient>,
+	path: String,
 }
 
 impl<M: RpcMessage> RpcAddrClient<M> {

src/rpc/rpc_server.rs

@@ -61,7 +61,10 @@ where
 			let err_str = format!("{}", e);
 			let rep_bytes = rmp_to_vec_all_named::<Result<M, String>>(&Err(err_str))?;
 			let mut err_response = Response::new(Body::from(rep_bytes));
-			*err_response.status_mut() = e.http_status_code();
+			*err_response.status_mut() = match e {
+				Error::BadRPC(_) => StatusCode::BAD_REQUEST,
+				_ => StatusCode::INTERNAL_SERVER_ERROR,
+			};
 			warn!(
 				"RPC error ({}): {} ({} ms)",
 				name,

src/table/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "garage_table"
-version = "0.1.0"
+version = "0.1.1"
 authors = ["Alex Auvolat <alex@adnab.me>"]
 edition = "2018"
 license = "GPL-3.0"
@@ -13,14 +13,15 @@ path = "lib.rs"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-garage_util = { version = "0.1", path = "../util" }
-garage_rpc = { version = "0.1", path = "../rpc" }
+garage_util = { version = "0.1.1", path = "../util" }
+garage_rpc = { version = "0.1.1", path = "../rpc" }
 
 bytes = "0.4"
 rand = "0.7"
 hex = "0.3"
 arc-swap = "0.4"
 log = "0.4"
+hexdump = "0.1"
 
 sled = "0.31"
 

src/table/crdt.rs

@@ -0,0 +1,327 @@
+//! This package provides a simple implementation of conflict-free replicated data types (CRDTs)
+//!
+//! CRDTs are a type of data structures that do not require coordination.  In other words, we can
+//! edit them in parallel, we will always find a way to merge it.
+//!
+//! A general example is a counter. Its initial value is 0.  Alice and Bob get a copy of the
+//! counter.  Alice does +1 on her copy, she reads 1.  Bob does +3 on his copy, he reads 3.  Now,
+//! it is easy to merge their counters, order does not count: we always get 4.
+//!
+//! Learn more about CRDT [on Wikipedia](https://en.wikipedia.org/wiki/Conflict-free_replicated_data_type)
+
+use serde::{Deserialize, Serialize};
+
+use garage_util::data::*;
+
+/// Definition of a CRDT - all CRDT Rust types implement this.
+///
+/// A CRDT is defined as a merge operator that respects a certain set of axioms.
+///
+/// In particular, the merge operator must be commutative, associative,
+/// idempotent, and monotonic.
+/// In other words, if `a`, `b` and `c` are CRDTs, and `⊔` denotes the merge operator,
+/// the following axioms must apply:
+///
+/// ```text
+/// a ⊔ b = b ⊔ a                   (commutativity)
+/// (a ⊔ b) ⊔ c = a ⊔ (b ⊔ c)       (associativity)
+/// (a ⊔ b) ⊔ b = a ⊔ b             (idempotence)
+/// ```
+///
+/// Moreover, the relationship `≥` defined by `a ≥ b ⇔ ∃c. a = b ⊔ c` must be a partial order.
+/// This implies a few properties such as: if `a ⊔ b ≠ a`, then there is no `c` such that `(a ⊔ b) ⊔ c = a`,
+/// as this would imply a cycle in the partial order.
+pub trait CRDT {
+	/// Merge the two datastructures according to the CRDT rules.
+	/// `self` is modified to contain the merged CRDT value. `other` is not modified.
+	///
+	/// # Arguments
+	///
+	/// * `other` - the other CRDT we wish to merge with
+	fn merge(&mut self, other: &Self);
+}
+
+/// All types that implement `Ord` (a total order) also implement a trivial CRDT
+/// defined by the merge rule: `a ⊔ b = max(a, b)`.
+impl<T> CRDT for T
+where
+	T: Ord + Clone,
+{
+	fn merge(&mut self, other: &Self) {
+		if other > self {
+			*self = other.clone();
+		}
+	}
+}
+
+// ---- LWW Register ----
+
+/// Last Write Win (LWW)
+///
+/// An LWW CRDT associates a timestamp with a value, in order to implement a
+/// time-based reconciliation rule: the most recent write wins.
+/// For completeness, the LWW reconciliation rule must also be defined for two LWW CRDTs
+/// with the same timestamp but different values.
+///
+/// In our case, we add the constraint that the value that is wrapped inside the LWW CRDT must
+/// itself be a CRDT: in the case when the timestamp does not allow us to decide on which value to
+/// keep, the merge rule of the inner CRDT is applied on the wrapped values.  (Note that all types
+/// that implement the `Ord` trait get a default CRDT implemetnation that keeps the maximum value.
+/// This enables us to use LWW directly with primitive data types such as numbers or strings. It is
+/// generally desirable in this case to never explicitly produce LWW values with the same timestamp
+/// but different inner values, as the rule to keep the maximum value isn't generally the desired
+/// semantics.)
+///
+/// As multiple computers clocks are always desynchronized,
+/// when operations are close enough, it is equivalent to
+/// take one copy and drop the other one.
+///
+/// Given that clocks are not too desynchronized, this assumption
+/// is enough for most cases, as there is few chance that two humans
+/// coordonate themself faster than the time difference between two NTP servers.
+///
+/// As a more concret example, let's suppose you want to upload a file
+/// with the same key (path) in the same bucket at the very same time.
+/// For each request, the file will be timestamped by the receiving server
+/// and may differ from what you observed with your atomic clock!
+///
+/// This scheme is used by AWS S3 or Soundcloud and often without knowing
+/// in entreprise when reconciliating databases with ad-hoc scripts.
+#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
+pub struct LWW<T> {
+	ts: u64,
+	v: T,
+}
+
+impl<T> LWW<T>
+where
+	T: CRDT,
+{
+	/// Creates a new CRDT
+	///
+	/// CRDT's internal timestamp is set with current node's clock.
+	pub fn new(value: T) -> Self {
+		Self {
+			ts: now_msec(),
+			v: value,
+		}
+	}
+
+	/// Build a new CRDT from a previous non-compatible one
+	///
+	/// Compared to new, the CRDT's timestamp is not set to now
+	/// but must be set to the previous, non-compatible, CRDT's timestamp.
+	pub fn migrate_from_raw(ts: u64, value: T) -> Self {
+		Self { ts, v: value }
+	}
+
+	/// Update the LWW CRDT while keeping some causal ordering.
+	///
+	/// The timestamp of the LWW CRDT is updated to be the current node's clock
+	/// at time of update, or the previous timestamp + 1 if that's bigger,
+	/// so that the new timestamp is always strictly larger than the previous one.
+	/// This ensures that merging the update with the old value will result in keeping
+	/// the updated value.
+	pub fn update(&mut self, new_value: T) {
+		self.ts = std::cmp::max(self.ts + 1, now_msec());
+		self.v = new_value;
+	}
+
+	/// Get the CRDT value
+	pub fn get(&self) -> &T {
+		&self.v
+	}
+
+	/// Get a mutable reference to the CRDT's value
+	///
+	/// This is usefull to mutate the inside value without changing the LWW timestamp.
+	/// When such mutation is done, the merge between two LWW values is done using the inner
+	/// CRDT's merge operation. This is usefull in the case where the inner CRDT is a large
+	/// data type, such as a map, and we only want to change a single item in the map.
+	/// To do this, we can produce a "CRDT delta", i.e. a LWW that contains only the modification.
+	/// This delta consists in a LWW with the same timestamp, and the map
+	/// inside only contains the updated value.
+	/// The advantage of such a delta is that it is much smaller than the whole map.
+	///
+	/// Avoid using this if the inner data type is a primitive type such as a number or a string,
+	/// as you will then rely on the merge function defined on `Ord` types by keeping the maximum
+	/// of both values.
+	pub fn get_mut(&mut self) -> &mut T {
+		&mut self.v
+	}
+}
+
+impl<T> CRDT for LWW<T>
+where
+	T: Clone + CRDT,
+{
+	fn merge(&mut self, other: &Self) {
+		if other.ts > self.ts {
+			self.ts = other.ts;
+			self.v = other.v.clone();
+		} else if other.ts == self.ts {
+			self.v.merge(&other.v);
+		}
+	}
+}
+
+/// Boolean, where `true` is an absorbing state
+#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq)]
+pub struct Bool(bool);
+
+impl Bool {
+	/// Create a new boolean with the specified value
+	pub fn new(b: bool) -> Self {
+		Self(b)
+	}
+	/// Set the boolean to true
+	pub fn set(&mut self) {
+		self.0 = true;
+	}
+	/// Get the boolean value
+	pub fn get(&self) -> bool {
+		self.0
+	}
+}
+
+impl CRDT for Bool {
+	fn merge(&mut self, other: &Self) {
+		self.0 = self.0 || other.0;
+	}
+}
+
+/// Last Write Win Map
+///
+/// This types defines a CRDT for a map from keys to values.
+/// The values have an associated timestamp, such that the last written value
+/// takes precedence over previous ones. As for the simpler `LWW` type, the value
+/// type `V` is also required to implement the CRDT trait.
+/// We do not encourage mutating the values associated with a given key
+/// without updating the timestamp, in fact at the moment we do not provide a `.get_mut()`
+/// method that would allow that.
+///
+/// Internally, the map is stored as a vector of keys and values, sorted by ascending key order.
+/// This is why the key type `K` must implement `Ord` (and also to ensure a unique serialization,
+/// such that two values can be compared for equality based on their hashes). As a consequence,
+/// insertions take `O(n)` time. This means that LWWMap should be used for reasonably small maps.
+/// However, note that even if we were using a more efficient data structure such as a `BTreeMap`,
+/// the serialization cost `O(n)` would still have to be paid at each modification, so we are
+/// actually not losing anything here.
+#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
+pub struct LWWMap<K, V> {
+	vals: Vec<(K, u64, V)>,
+}
+
+impl<K, V> LWWMap<K, V>
+where
+	K: Ord,
+	V: CRDT,
+{
+	/// Create a new empty map CRDT
+	pub fn new() -> Self {
+		Self { vals: vec![] }
+	}
+	/// Used to migrate from a map defined in an incompatible format. This produces
+	/// a map that contains a single item with the specified timestamp (copied from
+	/// the incompatible format). Do this as many times as you have items to migrate,
+	/// and put them all together using the CRDT merge operator.
+	pub fn migrate_from_raw_item(k: K, ts: u64, v: V) -> Self {
+		Self {
+			vals: vec![(k, ts, v)],
+		}
+	}
+	/// Returns a map that contains a single mapping from the specified key to the specified value.
+	/// This map is a mutator, or a delta-CRDT, such that when it is merged with the original map,
+	/// the previous value will be replaced with the one specified here.
+	/// The timestamp in the provided mutator is set to the maximum of the current system's clock
+	/// and 1 + the previous value's timestamp (if there is one), so that the new value will always
+	/// take precedence (LWW rule).
+	///
+	/// Typically, to update the value associated to a key in the map, you would do the following:
+	///
+	/// ```ignore
+	/// let my_update = my_crdt.update_mutator(key_to_modify, new_value);
+	/// my_crdt.merge(&my_update);
+	/// ```
+	///
+	/// However extracting the mutator on its own and only sending that on the network is very
+	/// interesting as it is much smaller than the whole map.
+	pub fn update_mutator(&self, k: K, new_v: V) -> Self {
+		let new_vals = match self.vals.binary_search_by(|(k2, _, _)| k2.cmp(&k)) {
+			Ok(i) => {
+				let (_, old_ts, _) = self.vals[i];
+				let new_ts = std::cmp::max(old_ts + 1, now_msec());
+				vec![(k, new_ts, new_v)]
+			}
+			Err(_) => vec![(k, now_msec(), new_v)],
+		};
+		Self { vals: new_vals }
+	}
+	/// Takes all of the values of the map and returns them. The current map is reset to the
+	/// empty map. This is very usefull to produce in-place a new map that contains only a delta
+	/// that modifies a certain value:
+	///
+	/// ```ignore
+	/// let mut a = get_my_crdt_value();
+	/// let old_a = a.take_and_clear();
+	/// a.merge(&old_a.update_mutator(key_to_modify, new_value));
+	/// put_my_crdt_value(a);
+	/// ```
+	///
+	/// Of course in this simple example we could have written simply
+	/// `pyt_my_crdt_value(a.update_mutator(key_to_modify, new_value))`,
+	/// but in the case where the map is a field in a struct for instance (as is always the case),
+	/// this becomes very handy:
+	///
+	/// ```ignore
+	/// let mut a = get_my_crdt_value();
+	/// let old_a_map = a.map_field.take_and_clear();
+	/// a.map_field.merge(&old_a_map.update_mutator(key_to_modify, new_value));
+	/// put_my_crdt_value(a);
+	/// ```
+	pub fn take_and_clear(&mut self) -> Self {
+		let vals = std::mem::replace(&mut self.vals, vec![]);
+		Self { vals }
+	}
+	/// Removes all values from the map
+	pub fn clear(&mut self) {
+		self.vals.clear();
+	}
+	/// Get a reference to the value assigned to a key
+	pub fn get(&self, k: &K) -> Option<&V> {
+		match self.vals.binary_search_by(|(k2, _, _)| k2.cmp(&k)) {
+			Ok(i) => Some(&self.vals[i].2),
+			Err(_) => None,
+		}
+	}
+	/// Gets a reference to all of the items, as a slice. Usefull to iterate on all map values.
+	/// In most case you will want to ignore the timestamp (second item of the tuple).
+	pub fn items(&self) -> &[(K, u64, V)] {
+		&self.vals[..]
+	}
+}
+
+impl<K, V> CRDT for LWWMap<K, V>
+where
+	K: Clone + Ord,
+	V: Clone + CRDT,
+{
+	fn merge(&mut self, other: &Self) {
+		for (k, ts2, v2) in other.vals.iter() {
+			match self.vals.binary_search_by(|(k2, _, _)| k2.cmp(&k)) {
+				Ok(i) => {
+					let (_, ts1, _v1) = &self.vals[i];
+					if ts2 > ts1 {
+						self.vals[i].1 = *ts2;
+						self.vals[i].2 = v2.clone();
+					} else if ts1 == ts2 {
+						self.vals[i].2.merge(&v2);
+					}
+				}
+				Err(i) => {
+					self.vals.insert(i, (k.clone(), *ts2, v2.clone()));
+				}
+			}
+		}
+	}
+}

src/table/lib.rs

@@ -3,9 +3,15 @@
 #[macro_use]
 extern crate log;
 
+pub mod crdt;
+pub mod schema;
+pub mod util;
+
 pub mod table;
 pub mod table_fullcopy;
 pub mod table_sharded;
 pub mod table_sync;
 
+pub use schema::*;
 pub use table::*;
+pub use util::*;

src/table/schema.rs

@@ -0,0 +1,65 @@
+use async_trait::async_trait;
+use serde::{Deserialize, Serialize};
+
+use garage_util::data::*;
+use garage_util::error::Error;
+
+pub trait PartitionKey {
+	fn hash(&self) -> Hash;
+}
+
+impl PartitionKey for String {
+	fn hash(&self) -> Hash {
+		hash(self.as_bytes())
+	}
+}
+
+impl PartitionKey for Hash {
+	fn hash(&self) -> Hash {
+		self.clone()
+	}
+}
+
+pub trait SortKey {
+	fn sort_key(&self) -> &[u8];
+}
+
+impl SortKey for String {
+	fn sort_key(&self) -> &[u8] {
+		self.as_bytes()
+	}
+}
+
+impl SortKey for Hash {
+	fn sort_key(&self) -> &[u8] {
+		self.as_slice()
+	}
+}
+
+pub trait Entry<P: PartitionKey, S: SortKey>:
+	PartialEq + Clone + Serialize + for<'de> Deserialize<'de> + Send + Sync
+{
+	fn partition_key(&self) -> &P;
+	fn sort_key(&self) -> &S;
+
+	fn merge(&mut self, other: &Self);
+}
+
+#[async_trait]
+pub trait TableSchema: Send + Sync {
+	type P: PartitionKey + Clone + PartialEq + Serialize + for<'de> Deserialize<'de> + Send + Sync;
+	type S: SortKey + Clone + Serialize + for<'de> Deserialize<'de> + Send + Sync;
+	type E: Entry<Self::P, Self::S>;
+	type Filter: Clone + Serialize + for<'de> Deserialize<'de> + Send + Sync;
+
+	// Action to take if not able to decode current version:
+	// try loading from an older version
+	fn try_migrate(_bytes: &[u8]) -> Option<Self::E> {
+		None
+	}
+
+	async fn updated(&self, old: Option<Self::E>, new: Option<Self::E>) -> Result<(), Error>;
+	fn matches_filter(_entry: &Self::E, _filter: &Self::Filter) -> bool {
+		true
+	}
+}

src/table/table.rs

@@ -2,8 +2,9 @@ use std::collections::{BTreeMap, HashMap};
 use std::sync::Arc;
 use std::time::Duration;
 
+use log::warn;
+
 use arc_swap::ArcSwapOption;
-use async_trait::async_trait;
 use futures::stream::*;
 use serde::{Deserialize, Serialize};
 use serde_bytes::ByteBuf;
@@ -15,6 +16,7 @@ use garage_rpc::membership::{Ring, System};
 use garage_rpc::rpc_client::*;
 use garage_rpc::rpc_server::*;
 
+use crate::schema::*;
 use crate::table_sync::*;
 
 const TABLE_RPC_TIMEOUT: Duration = Duration::from_secs(10);
@@ -48,71 +50,6 @@ pub enum TableRPC<F: TableSchema> {
 
 impl<F: TableSchema> RpcMessage for TableRPC<F> {}
 
-pub trait PartitionKey {
-	fn hash(&self) -> Hash;
-}
-
-pub trait SortKey {
-	fn sort_key(&self) -> &[u8];
-}
-
-pub trait Entry<P: PartitionKey, S: SortKey>:
-	PartialEq + Clone + Serialize + for<'de> Deserialize<'de> + Send + Sync
-{
-	fn partition_key(&self) -> &P;
-	fn sort_key(&self) -> &S;
-
-	fn merge(&mut self, other: &Self);
-}
-
-#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)]
-pub struct EmptyKey;
-impl SortKey for EmptyKey {
-	fn sort_key(&self) -> &[u8] {
-		&[]
-	}
-}
-impl PartitionKey for EmptyKey {
-	fn hash(&self) -> Hash {
-		[0u8; 32].into()
-	}
-}
-
-impl PartitionKey for String {
-	fn hash(&self) -> Hash {
-		hash(self.as_bytes())
-	}
-}
-impl SortKey for String {
-	fn sort_key(&self) -> &[u8] {
-		self.as_bytes()
-	}
-}
-
-impl PartitionKey for Hash {
-	fn hash(&self) -> Hash {
-		self.clone()
-	}
-}
-impl SortKey for Hash {
-	fn sort_key(&self) -> &[u8] {
-		self.as_slice()
-	}
-}
-
-#[async_trait]
-pub trait TableSchema: Send + Sync {
-	type P: PartitionKey + Clone + PartialEq + Serialize + for<'de> Deserialize<'de> + Send + Sync;
-	type S: SortKey + Clone + Serialize + for<'de> Deserialize<'de> + Send + Sync;
-	type E: Entry<Self::P, Self::S>;
-	type Filter: Clone + Serialize + for<'de> Deserialize<'de> + Send + Sync;
-
-	async fn updated(&self, old: Option<Self::E>, new: Option<Self::E>) -> Result<(), Error>;
-	fn matches_filter(_entry: &Self::E, _filter: &Self::Filter) -> bool {
-		true
-	}
-}
-
 pub trait TableReplication: Send + Sync {
 	// See examples in table_sharded.rs and table_fullcopy.rs
 	// To understand various replication methods
@@ -250,7 +187,7 @@ where
 		for resp in resps {
 			if let TableRPC::ReadEntryResponse(value) = resp {
 				if let Some(v_bytes) = value {
-					let v = rmp_serde::decode::from_read_ref::<_, F::E>(v_bytes.as_slice())?;
+					let v = self.decode_entry(v_bytes.as_slice())?;
 					ret = match ret {
 						None => Some(v),
 						Some(mut x) => {
@@ -306,8 +243,7 @@ where
 		for resp in resps {
 			if let TableRPC::Update(entries) = resp {
 				for entry_bytes in entries.iter() {
-					let entry =
-						rmp_serde::decode::from_read_ref::<_, F::E>(entry_bytes.as_slice())?;
+					let entry = self.decode_entry(entry_bytes.as_slice())?;
 					let entry_key = self.tree_key(entry.partition_key(), entry.sort_key());
 					match ret.remove(&entry_key) {
 						None => {
@@ -395,7 +331,7 @@ where
 					.await?;
 				Ok(TableRPC::SyncRPC(response))
 			}
-			_ => Err(Error::BadRequest(format!("Unexpected table RPC"))),
+			_ => Err(Error::BadRPC(format!("Unexpected table RPC"))),
 		}
 	}
 
@@ -429,7 +365,7 @@ where
 			let keep = match filter {
 				None => true,
 				Some(f) => {
-					let entry = rmp_serde::decode::from_read_ref::<_, F::E>(value.as_ref())?;
+					let entry = self.decode_entry(value.as_ref())?;
 					F::matches_filter(&entry, f)
 				}
 			};
@@ -448,15 +384,15 @@ where
 		let mut epidemic_propagate = vec![];
 
 		for update_bytes in entries.iter() {
-			let update = rmp_serde::decode::from_read_ref::<_, F::E>(update_bytes.as_slice())?;
+			let update = self.decode_entry(update_bytes.as_slice())?;
 
 			let tree_key = self.tree_key(update.partition_key(), update.sort_key());
 
 			let (old_entry, new_entry) = self.store.transaction(|db| {
 				let (old_entry, new_entry) = match db.get(&tree_key)? {
 					Some(prev_bytes) => {
-						let old_entry = rmp_serde::decode::from_read_ref::<_, F::E>(&prev_bytes)
-							.map_err(Error::RMPDecode)
+						let old_entry = self
+							.decode_entry(&prev_bytes)
 							.map_err(sled::ConflictableTransactionError::Abort)?;
 						let mut new_entry = old_entry.clone();
 						new_entry.merge(&update);
@@ -504,7 +440,7 @@ where
 				break;
 			}
 			if let Some(old_val) = self.store.remove(&key)? {
-				let old_entry = rmp_serde::decode::from_read_ref::<_, F::E>(&old_val)?;
+				let old_entry = self.decode_entry(&old_val)?;
 				self.instance.updated(Some(old_entry), None).await?;
 				self.system
 					.background
@@ -521,4 +457,20 @@ where
 		ret.extend(s.sort_key());
 		ret
 	}
+
+	fn decode_entry(&self, bytes: &[u8]) -> Result<F::E, Error> {
+		match rmp_serde::decode::from_read_ref::<_, F::E>(bytes) {
+			Ok(x) => Ok(x),
+			Err(e) => match F::try_migrate(bytes) {
+				Some(x) => Ok(x),
+				None => {
+					warn!("Unable to decode entry of {}: {}", self.name, e);
+					for line in hexdump::hexdump_iter(bytes) {
+						debug!("{}", line);
+					}
+					Err(e.into())
+				}
+			},
+		}
+	}
 }

src/table/table_sync.rs

@@ -474,7 +474,7 @@ where
 				todo.push_back(root_ck);
 			}
 		} else {
-			return Err(Error::BadRequest(format!(
+			return Err(Error::Message(format!(
 				"Invalid respone to GetRootChecksumRange RPC: {}",
 				debug_serialize(root_cks_resp)
 			)));
@@ -530,7 +530,7 @@ where
 					self.send_items(who, items_to_send).await?;
 				}
 			} else {
-				return Err(Error::BadRequest(format!(
+				return Err(Error::Message(format!(
 					"Unexpected response to sync RPC checksums: {}",
 					debug_serialize(&rpc_resp)
 				)));

src/table/util.rs

@@ -0,0 +1,35 @@
+use serde::{Deserialize, Serialize};
+
+use garage_util::data::*;
+
+use crate::schema::*;
+
+#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)]
+pub struct EmptyKey;
+impl SortKey for EmptyKey {
+	fn sort_key(&self) -> &[u8] {
+		&[]
+	}
+}
+impl PartitionKey for EmptyKey {
+	fn hash(&self) -> Hash {
+		[0u8; 32].into()
+	}
+}
+
+#[derive(Clone, Copy, Debug, Serialize, Deserialize)]
+pub enum DeletedFilter {
+	All,
+	Deleted,
+	NotDeleted,
+}
+
+impl DeletedFilter {
+	pub fn apply(&self, deleted: bool) -> bool {
+		match self {
+			DeletedFilter::All => true,
+			DeletedFilter::Deleted => deleted,
+			DeletedFilter::NotDeleted => !deleted,
+		}
+	}
+}

src/util/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "garage_util"
-version = "0.1.0"
+version = "0.1.1"
 authors = ["Alex Auvolat <alex@adnab.me>"]
 edition = "2018"
 license = "GPL-3.0"

src/util/config.rs

@@ -35,6 +35,8 @@ pub struct Config {
 	pub rpc_tls: Option<TlsConfig>,
 
 	pub s3_api: ApiConfig,
+
+	pub s3_web: WebConfig,
 }
 
 #[derive(Deserialize, Debug, Clone)]
@@ -50,6 +52,13 @@ pub struct ApiConfig {
 	pub s3_region: String,
 }
 
+#[derive(Deserialize, Debug, Clone)]
+pub struct WebConfig {
+	pub bind_addr: SocketAddr,
+	pub root_domain: String,
+	pub index: String,
+}
+
 fn default_max_concurrent_rpc_requests() -> usize {
 	12
 }

src/util/error.rs

@@ -54,9 +54,6 @@ pub enum Error {
 	#[error(display = "TOML decode error: {}", _0)]
 	TomlDecode(#[error(source)] toml::de::Error),
 
-	#[error(display = "Timeout: {}", _0)]
-	RPCTimeout(#[error(source)] tokio::time::Elapsed),
-
 	#[error(display = "Tokio join error: {}", _0)]
 	TokioJoin(#[error(source)] tokio::task::JoinError),
 
@@ -66,14 +63,8 @@ pub enum Error {
 	#[error(display = "Remote error: {} (status code {})", _0, _1)]
 	RemoteError(String, StatusCode),
 
-	#[error(display = "Bad request: {}", _0)]
-	BadRequest(String),
-
-	#[error(display = "Forbidden: {}", _0)]
-	Forbidden(String),
-
-	#[error(display = "Not found")]
-	NotFound,
+	#[error(display = "Bad RPC: {}", _0)]
+	BadRPC(String),
 
 	#[error(display = "Corrupt data: does not match hash {:?}", _0)]
 	CorruptData(Hash),
@@ -82,18 +73,6 @@ pub enum Error {
 	Message(String),
 }
 
-impl Error {
-	pub fn http_status_code(&self) -> StatusCode {
-		match self {
-			Error::BadRequest(_) => StatusCode::BAD_REQUEST,
-			Error::NotFound => StatusCode::NOT_FOUND,
-			Error::Forbidden(_) => StatusCode::FORBIDDEN,
-			Error::RPC(_) => StatusCode::SERVICE_UNAVAILABLE,
-			_ => StatusCode::INTERNAL_SERVER_ERROR,
-		}
-	}
-}
-
 impl From<sled::TransactionError<Error>> for Error {
 	fn from(e: sled::TransactionError<Error>) -> Error {
 		match e {
@@ -114,15 +93,3 @@ impl<T> From<tokio::sync::mpsc::error::SendError<T>> for Error {
 		Error::Message(format!("MPSC send error"))
 	}
 }
-
-impl From<std::str::Utf8Error> for Error {
-	fn from(e: std::str::Utf8Error) -> Error {
-		Error::BadRequest(format!("Invalid UTF-8: {}", e))
-	}
-}
-
-impl From<roxmltree::Error> for Error {
-	fn from(e: roxmltree::Error) -> Error {
-		Error::BadRequest(format!("Invalid XML: {}", e))
-	}
-}

src/web/Cargo.toml

@@ -0,0 +1,28 @@
+[package]
+name = "garage_web"
+version = "0.1.1"
+authors = ["Alex Auvolat <alex@adnab.me>", "Quentin Dufour <quentin@dufour.io>"]
+edition = "2018"
+license = "GPL-3.0"
+description = "S3-like website endpoint crate for the Garage object store"
+repository = "https://git.deuxfleurs.fr/Deuxfleurs/garage"
+
+[lib]
+path = "lib.rs"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+garage_util = { version = "0.1.1", path = "../util" }
+garage_table = { version = "0.1.1", path = "../table" }
+garage_model = { version = "0.1.1", path = "../model" }
+garage_api = { version = "0.1.1", path = "../api" }
+
+err-derive = "0.2.3"
+log = "0.4"
+futures = "0.3"
+http = "0.2"
+hyper = "0.13"
+percent-encoding = "2.1.0"
+roxmltree = "0.11"
+idna = "0.2"

src/web/error.rs

@@ -0,0 +1,39 @@
+use err_derive::Error;
+use hyper::StatusCode;
+
+use garage_util::error::Error as GarageError;
+
+#[derive(Debug, Error)]
+pub enum Error {
+	#[error(display = "API error: {}", _0)]
+	ApiError(#[error(source)] garage_api::error::Error),
+
+	// Category: internal error
+	#[error(display = "Internal error: {}", _0)]
+	InternalError(#[error(source)] GarageError),
+
+	#[error(display = "Not found")]
+	NotFound,
+
+	// Category: bad request
+	#[error(display = "Invalid UTF-8: {}", _0)]
+	InvalidUTF8(#[error(source)] std::str::Utf8Error),
+
+	#[error(display = "Invalid header value: {}", _0)]
+	InvalidHeader(#[error(source)] hyper::header::ToStrError),
+
+	#[error(display = "Bad request: {}", _0)]
+	BadRequest(String),
+}
+
+impl Error {
+	pub fn http_status_code(&self) -> StatusCode {
+		match self {
+			Error::NotFound => StatusCode::NOT_FOUND,
+			Error::ApiError(e) => e.http_status_code(),
+			Error::InternalError(GarageError::RPC(_)) => StatusCode::SERVICE_UNAVAILABLE,
+			Error::InternalError(_) => StatusCode::INTERNAL_SERVER_ERROR,
+			_ => StatusCode::BAD_REQUEST,
+		}
+	}
+}

src/web/lib.rs

@@ -0,0 +1,6 @@
+#[macro_use]
+extern crate log;
+
+pub mod error;
+
+pub mod web_server;

src/web/web_server.rs

@@ -0,0 +1,260 @@
+use std::{borrow::Cow, convert::Infallible, net::SocketAddr, sync::Arc};
+
+use futures::future::Future;
+
+use hyper::{
+	header::HOST,
+	server::conn::AddrStream,
+	service::{make_service_fn, service_fn},
+	Body, Method, Request, Response, Server,
+};
+
+use idna::domain_to_unicode;
+
+use crate::error::*;
+use garage_api::s3_get::{handle_get, handle_head};
+use garage_model::bucket_table::*;
+use garage_model::garage::Garage;
+use garage_table::*;
+use garage_util::error::Error as GarageError;
+
+pub async fn run_web_server(
+	garage: Arc<Garage>,
+	shutdown_signal: impl Future<Output = ()>,
+) -> Result<(), GarageError> {
+	let addr = &garage.config.s3_web.bind_addr;
+
+	let service = make_service_fn(|conn: &AddrStream| {
+		let garage = garage.clone();
+		let client_addr = conn.remote_addr();
+		async move {
+			Ok::<_, Error>(service_fn(move |req: Request<Body>| {
+				let garage = garage.clone();
+				handle_request(garage, req, client_addr)
+			}))
+		}
+	});
+
+	let server = Server::bind(&addr).serve(service);
+	let graceful = server.with_graceful_shutdown(shutdown_signal);
+	info!("Web server listening on http://{}", addr);
+
+	graceful.await?;
+	Ok(())
+}
+
+async fn handle_request(
+	garage: Arc<Garage>,
+	req: Request<Body>,
+	addr: SocketAddr,
+) -> Result<Response<Body>, Infallible> {
+	info!("{} {} {}", addr, req.method(), req.uri());
+	let res = serve_file(garage, req).await;
+	match &res {
+		Ok(r) => debug!("{} {:?}", r.status(), r.headers()),
+		Err(e) => warn!("Response: error {}, {}", e.http_status_code(), e),
+	}
+
+	Ok(res.unwrap_or_else(error_to_res))
+}
+
+fn error_to_res(e: Error) -> Response<Body> {
+	let body: Body = Body::from(format!("{}\n", e));
+	let mut http_error = Response::new(body);
+	*http_error.status_mut() = e.http_status_code();
+	http_error
+}
+
+async fn serve_file(garage: Arc<Garage>, req: Request<Body>) -> Result<Response<Body>, Error> {
+	// Get http authority string (eg. [::1]:3902 or garage.tld:80)
+	let authority = req
+		.headers()
+		.get(HOST)
+		.ok_or(Error::BadRequest(format!("HOST header required")))?
+		.to_str()?;
+
+	// Get bucket
+	let (host, _) = domain_to_unicode(authority_to_host(authority)?);
+	let root = &garage.config.s3_web.root_domain;
+	let bucket = host_to_bucket(&host, root);
+
+	// Check bucket is exposed as a website
+	let bucket_desc = garage
+		.bucket_table
+		.get(&EmptyKey, &bucket.to_string())
+		.await?
+		.filter(|b| !b.is_deleted())
+		.ok_or(Error::NotFound)?;
+
+	match bucket_desc.state.get() {
+		BucketState::Present(params) if *params.website.get() => Ok(()),
+		_ => Err(Error::NotFound),
+	}?;
+
+	// Get path
+	let path = req.uri().path().to_string();
+	let index = &garage.config.s3_web.index;
+	let key = path_to_key(&path, &index)?;
+
+	info!("Selected bucket: \"{}\", selected key: \"{}\"", bucket, key);
+
+	let res = match req.method() {
+		&Method::HEAD => handle_head(garage, &bucket, &key).await?,
+		&Method::GET => handle_get(garage, &req, bucket, &key).await?,
+		_ => return Err(Error::BadRequest(format!("HTTP method not supported"))),
+	};
+
+	Ok(res)
+}
+
+/// Extract host from the authority section given by the HTTP host header
+///
+/// The HTTP host contains both a host and a port.
+/// Extracting the port is more complex than just finding the colon (:) symbol due to IPv6
+/// We do not use the collect pattern as there is no way in std rust to collect over a stack allocated value
+/// check here: https://docs.rs/collect_slice/1.2.0/collect_slice/
+fn authority_to_host(authority: &str) -> Result<&str, Error> {
+	let mut iter = authority.chars().enumerate();
+	let (_, first_char) = iter
+		.next()
+		.ok_or(Error::BadRequest(format!("Authority is empty")))?;
+
+	let split = match first_char {
+		'[' => {
+			let mut iter = iter.skip_while(|(_, c)| c != &']');
+			match iter.next() {
+				Some((_, ']')) => iter.next(),
+				_ => {
+					return Err(Error::BadRequest(format!(
+						"Authority {} has an illegal format",
+						authority
+					)))
+				}
+			}
+		}
+		_ => iter.skip_while(|(_, c)| c != &':').next(),
+	};
+
+	match split {
+		Some((i, ':')) => Ok(&authority[..i]),
+		None => Ok(authority),
+		Some((_, _)) => Err(Error::BadRequest(format!(
+			"Authority {} has an illegal format",
+			authority
+		))),
+	}
+}
+
+/// Host to bucket
+///
+/// Convert a host, like "bucket.garage-site.tld" or "john.doe.com"
+/// to the corresponding bucket, resp. "bucket" and "john.doe.com"
+/// considering that ".garage-site.tld" is the "root domain".
+/// This behavior has been chosen to follow AWS S3 semantic.
+fn host_to_bucket<'a>(host: &'a str, root: &str) -> &'a str {
+	if root.len() >= host.len() || !host.ends_with(root) {
+		return host;
+	}
+
+	let len_diff = host.len() - root.len();
+	let missing_starting_dot = root.chars().next() != Some('.');
+	let cursor = if missing_starting_dot {
+		len_diff - 1
+	} else {
+		len_diff
+	};
+	&host[..cursor]
+}
+
+/// Path to key
+///
+/// Convert the provided path to the internal key
+/// When a path ends with "/", we append the index name to match traditional web server behavior
+/// which is also AWS S3 behavior.
+fn path_to_key<'a>(path: &'a str, index: &str) -> Result<Cow<'a, str>, Error> {
+	let path_utf8 = percent_encoding::percent_decode_str(&path).decode_utf8()?;
+
+	if path_utf8.chars().next() != Some('/') {
+		return Err(Error::BadRequest(format!(
+			"Path must start with a / (slash)"
+		)));
+	}
+
+	match path_utf8.chars().last() {
+		None => unreachable!(),
+		Some('/') => {
+			let mut key = String::with_capacity(path_utf8.len() + index.len());
+			key.push_str(&path_utf8[1..]);
+			key.push_str(index);
+			Ok(key.into())
+		}
+		Some(_) => match path_utf8 {
+			Cow::Borrowed(pu8) => Ok((&pu8[1..]).into()),
+			Cow::Owned(pu8) => Ok((&pu8[1..]).to_string().into()),
+		},
+	}
+}
+
+#[cfg(test)]
+mod tests {
+	use super::*;
+
+	#[test]
+	fn authority_to_host_with_port() -> Result<(), Error> {
+		let domain = authority_to_host("[::1]:3902")?;
+		assert_eq!(domain, "[::1]");
+		let domain2 = authority_to_host("garage.tld:65200")?;
+		assert_eq!(domain2, "garage.tld");
+		let domain3 = authority_to_host("127.0.0.1:80")?;
+		assert_eq!(domain3, "127.0.0.1");
+		Ok(())
+	}
+
+	#[test]
+	fn authority_to_host_without_port() -> Result<(), Error> {
+		let domain = authority_to_host("[::1]")?;
+		assert_eq!(domain, "[::1]");
+		let domain2 = authority_to_host("garage.tld")?;
+		assert_eq!(domain2, "garage.tld");
+		let domain3 = authority_to_host("127.0.0.1")?;
+		assert_eq!(domain3, "127.0.0.1");
+		assert!(authority_to_host("[").is_err());
+		assert!(authority_to_host("[hello").is_err());
+		Ok(())
+	}
+
+	#[test]
+	fn host_to_bucket_test() {
+		assert_eq!(
+			host_to_bucket("john.doe.garage.tld", ".garage.tld"),
+			"john.doe"
+		);
+
+		assert_eq!(
+			host_to_bucket("john.doe.garage.tld", "garage.tld"),
+			"john.doe"
+		);
+
+		assert_eq!(host_to_bucket("john.doe.com", "garage.tld"), "john.doe.com");
+
+		assert_eq!(
+			host_to_bucket("john.doe.com", ".garage.tld"),
+			"john.doe.com"
+		);
+
+		assert_eq!(host_to_bucket("garage.tld", "garage.tld"), "garage.tld");
+
+		assert_eq!(host_to_bucket("garage.tld", ".garage.tld"), "garage.tld");
+	}
+
+	#[test]
+	fn path_to_key_test() -> Result<(), Error> {
+		assert_eq!(path_to_key("/file%20.jpg", "index.html")?, "file .jpg");
+		assert_eq!(path_to_key("/%20t/", "index.html")?, " t/index.html");
+		assert_eq!(path_to_key("/", "index.html")?, "index.html");
+		assert_eq!(path_to_key("/hello", "index.html")?, "hello");
+		assert!(path_to_key("", "index.html").is_err());
+		assert!(path_to_key("i/am/relative", "index.html").is_err());
+		Ok(())
+	}
+}

test_delete.sh

@@ -1,6 +0,0 @@
-#!/bin/bash
-
-for FILE in $(find target); do
-	curl localhost:3900/$FILE -X DELETE -H 'Host: garage'
-done
-

test_read.sh

@@ -1,13 +0,0 @@
-
-#!/bin/bash
-
-for FILE in $(find target/debug/deps); do 
-	SHA2=$(curl localhost:3900/$FILE -H 'Host: garage' 2>/dev/null | sha256sum | cut -d ' ' -f 1)
-	SHA2REF=$(sha256sum $FILE | cut -d ' ' -f 1)
-	if [ "$SHA2" = "$SHA2REF" ]; then
-		echo "OK $FILE"
-	else
-		echo "!!!! ERROR $FILE !!!!"
-	fi
-done
-

test_write.sh

@@ -1,7 +0,0 @@
-#!/bin/bash
-
-for FILE in $(find target/debug/deps); do 
-	echo -n "$FILE "
-	curl localhost:3900/$FILE -X PUT -H 'Host: garage' -H 'Content-Type: application/blob' --data-binary "@$FILE" || echo "ERROR"
-done
-