mirror of
https://github.com/deuxfleurs-org/garage.git
synced 2026-08-16 00:43:11 +00:00
Compare commits
1 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Alex Auvolat
|
c21317dc58 |
Generated
+225
-886
File diff suppressed because it is too large
Load Diff
+2
-3
@@ -4,9 +4,8 @@ members = [
|
|||||||
"src/rpc",
|
"src/rpc",
|
||||||
"src/table",
|
"src/table",
|
||||||
"src/model",
|
"src/model",
|
||||||
"src/api",
|
#"src/api",
|
||||||
"src/web",
|
#"src/garage",
|
||||||
"src/garage",
|
|
||||||
]
|
]
|
||||||
|
|
||||||
[profile.dev]
|
[profile.dev]
|
||||||
|
|||||||
@@ -3,12 +3,10 @@ DOCKER=lxpz/garage_amd64
|
|||||||
|
|
||||||
all:
|
all:
|
||||||
#cargo fmt || true
|
#cargo fmt || true
|
||||||
#RUSTFLAGS="-C link-arg=-fuse-ld=lld" cargo build
|
RUSTFLAGS="-C link-arg=-fuse-ld=lld" cargo build
|
||||||
cargo build
|
|
||||||
|
|
||||||
$(BIN):
|
$(BIN):
|
||||||
#RUSTFLAGS="-C link-arg=-fuse-ld=lld" cargo build --release
|
RUSTFLAGS="-C link-arg=-fuse-ld=lld" cargo build --release
|
||||||
cargo build --release
|
|
||||||
|
|
||||||
$(BIN).stripped: $(BIN)
|
$(BIN).stripped: $(BIN)
|
||||||
cp $^ $@
|
cp $^ $@
|
||||||
|
|||||||
@@ -16,28 +16,6 @@ Non-goals include:
|
|||||||
|
|
||||||
Our main use case is to provide a distributed storage layer for small-scale self hosted services such as [Deuxfleurs](https://deuxfleurs.fr).
|
Our main use case is to provide a distributed storage layer for small-scale self hosted services such as [Deuxfleurs](https://deuxfleurs.fr).
|
||||||
|
|
||||||
## Development
|
|
||||||
|
|
||||||
We propose the following quickstart to setup a full dev. environment as quickly as possible:
|
|
||||||
|
|
||||||
1. Setup a rust/cargo environment. eg. `dnf install rust cargo`
|
|
||||||
2. Install awscli v2 by following the guide [here](https://docs.aws.amazon.com/cli/latest/userguide/install-cliv2.html).
|
|
||||||
3. Run `cargo build` to build the project
|
|
||||||
4. Run `./script/dev-cluster.sh` to launch a test cluster (feel free to read the script)
|
|
||||||
5. Run `./script/dev-configure.sh` to configure your test cluster with default values (same datacenter, 100 tokens)
|
|
||||||
6. Run `./script/dev-bucket.sh` to create a bucket named `eprouvette` and an API key that will be stored in `/tmp/garage.s3`
|
|
||||||
7. Run `source ./script/dev-env-aws.sh` to configure your CLI environment
|
|
||||||
8. You can use `garage` to manage the cluster. Try `garage --help`.
|
|
||||||
9. You can use the `awsgrg` alias to add, remove, and delete files. Try `awsgrg help`, `awsgrg cp /proc/cpuinfo s3://eprouvette/cpuinfo.txt`, or `awsgrg ls s3://eprouvette`. `awsgrg` is a wrapper on the `aws s3` command pre-configured with the previously generated API key (the one in `/tmp/garage.s3`) and localhost as the endpoint.
|
|
||||||
|
|
||||||
Now you should be ready to start hacking on garage!
|
|
||||||
|
|
||||||
## S3 compatibility
|
|
||||||
|
|
||||||
Only a subset of S3 is supported: adding, listing, getting and deleting files in a bucket.
|
|
||||||
Bucket management, ACL and other advanced features are not (yet?) handled through the S3 API but through the `garage` CLI.
|
|
||||||
We primarily test `garage` against the `awscli` tool and `nextcloud`.
|
|
||||||
|
|
||||||
## Setting up Garage
|
## Setting up Garage
|
||||||
|
|
||||||
Use the `genkeys.sh` script to generate TLS keys for encrypting communications between Garage nodes.
|
Use the `genkeys.sh` script to generate TLS keys for encrypting communications between Garage nodes.
|
||||||
@@ -89,12 +67,7 @@ node_key = "/path/to/garage/pki/garage.key"
|
|||||||
|
|
||||||
[s3_api]
|
[s3_api]
|
||||||
api_bind_addr = "[::1]:3900" # the S3 API port, HTTP without TLS. Add a reverse proxy for the TLS part.
|
api_bind_addr = "[::1]:3900" # the S3 API port, HTTP without TLS. Add a reverse proxy for the TLS part.
|
||||||
s3_region = "garage" # set this to anything. S3 API calls will fail if they are not made against the region set here.
|
region = "garage" # set this to anything. S3 API calls will fail if they are not made against the region set here.
|
||||||
|
|
||||||
[s3_web]
|
|
||||||
bind_addr = "[::1]:3902"
|
|
||||||
root_domain = ".garage.tld"
|
|
||||||
index = "index.html"
|
|
||||||
```
|
```
|
||||||
|
|
||||||
Build Garage using `cargo build --release`.
|
Build Garage using `cargo build --release`.
|
||||||
|
|||||||
@@ -1,22 +0,0 @@
|
|||||||
block_size = 1048576 # objects are split in blocks of maximum this number of bytes
|
|
||||||
|
|
||||||
metadata_dir = "/tmp/garage-meta"
|
|
||||||
data_dir = "/tmp/garage-data"
|
|
||||||
|
|
||||||
rpc_bind_addr = "[::]:3901" # the port other Garage nodes will use to talk to this node
|
|
||||||
|
|
||||||
bootstrap_peers = []
|
|
||||||
|
|
||||||
max_concurrent_rpc_requests = 12
|
|
||||||
data_replication_factor = 3
|
|
||||||
meta_replication_factor = 3
|
|
||||||
meta_epidemic_fanout = 3
|
|
||||||
|
|
||||||
[s3_api]
|
|
||||||
api_bind_addr = "[::1]:3900" # the S3 API port, HTTP without TLS. Add a reverse proxy for the TLS part.
|
|
||||||
s3_region = "garage" # set this to anything. S3 API calls will fail if they are not made against the region set here.
|
|
||||||
|
|
||||||
[s3_web]
|
|
||||||
bind_addr = "[::1]:3902"
|
|
||||||
root_domain = ".garage.tld"
|
|
||||||
index = "index.html"
|
|
||||||
@@ -1,12 +0,0 @@
|
|||||||
*
|
|
||||||
|
|
||||||
!img
|
|
||||||
|
|
||||||
!.gitignore
|
|
||||||
!*.svg
|
|
||||||
!*.png
|
|
||||||
!*.jpg
|
|
||||||
!*.tex
|
|
||||||
!Makefile
|
|
||||||
!.gitignore
|
|
||||||
!talk.pdf
|
|
||||||
@@ -1,6 +0,0 @@
|
|||||||
talk.pdf: talk.tex img/garage_distributed.pdf img/consistent_hashing_1.pdf img/consistent_hashing_2.pdf img/consistent_hashing_3.pdf img/consistent_hashing_4.pdf img/garage_tables.pdf
|
|
||||||
pdflatex talk.tex
|
|
||||||
|
|
||||||
img/%.pdf: img/%.svg
|
|
||||||
inkscape -D -z --file=$^ --export-pdf=$@
|
|
||||||
|
|
||||||
Binary file not shown.
|
Before Width: | Height: | Size: 44 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 19 KiB |
File diff suppressed because one or more lines are too long
|
Before Width: | Height: | Size: 53 KiB |
File diff suppressed because one or more lines are too long
|
Before Width: | Height: | Size: 54 KiB |
File diff suppressed because one or more lines are too long
|
Before Width: | Height: | Size: 56 KiB |
File diff suppressed because one or more lines are too long
|
Before Width: | Height: | Size: 57 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 360 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 39 KiB |
@@ -1,404 +0,0 @@
|
|||||||
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
|
|
||||||
<svg
|
|
||||||
xmlns:dc="http://purl.org/dc/elements/1.1/"
|
|
||||||
xmlns:cc="http://creativecommons.org/ns#"
|
|
||||||
xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
|
|
||||||
xmlns:svg="http://www.w3.org/2000/svg"
|
|
||||||
xmlns="http://www.w3.org/2000/svg"
|
|
||||||
xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
|
|
||||||
xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
|
|
||||||
width="480"
|
|
||||||
height="480"
|
|
||||||
viewBox="0 0 127 127"
|
|
||||||
version="1.1"
|
|
||||||
id="svg8"
|
|
||||||
inkscape:version="1.0.1 (3bc2e813f5, 2020-09-07)"
|
|
||||||
sodipodi:docname="garage_distributed.svg">
|
|
||||||
<defs
|
|
||||||
id="defs2" />
|
|
||||||
<sodipodi:namedview
|
|
||||||
id="base"
|
|
||||||
pagecolor="#ffffff"
|
|
||||||
bordercolor="#666666"
|
|
||||||
borderopacity="1.0"
|
|
||||||
inkscape:pageopacity="0.0"
|
|
||||||
inkscape:pageshadow="2"
|
|
||||||
inkscape:zoom="1.979899"
|
|
||||||
inkscape:cx="171.34852"
|
|
||||||
inkscape:cy="170.69443"
|
|
||||||
inkscape:document-units="mm"
|
|
||||||
inkscape:current-layer="layer1-3"
|
|
||||||
inkscape:document-rotation="0"
|
|
||||||
showgrid="false"
|
|
||||||
units="px"
|
|
||||||
inkscape:window-width="1404"
|
|
||||||
inkscape:window-height="1016"
|
|
||||||
inkscape:window-x="103"
|
|
||||||
inkscape:window-y="27"
|
|
||||||
inkscape:window-maximized="0" />
|
|
||||||
<metadata
|
|
||||||
id="metadata5">
|
|
||||||
<rdf:RDF>
|
|
||||||
<cc:Work
|
|
||||||
rdf:about="">
|
|
||||||
<dc:format>image/svg+xml</dc:format>
|
|
||||||
<dc:type
|
|
||||||
rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
|
|
||||||
<dc:title />
|
|
||||||
</cc:Work>
|
|
||||||
</rdf:RDF>
|
|
||||||
</metadata>
|
|
||||||
<g
|
|
||||||
inkscape:label="Layer 1"
|
|
||||||
inkscape:groupmode="layer"
|
|
||||||
id="layer1">
|
|
||||||
<g
|
|
||||||
inkscape:label="Layer 1"
|
|
||||||
id="layer1-3"
|
|
||||||
transform="matrix(0.42851498,0,0,0.42851498,24.079728,-24.925134)">
|
|
||||||
<path
|
|
||||||
style="fill:none;stroke:#000000;stroke-width:2.065;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
|
|
||||||
d="M 66.78016,80.71889 99.921832,61.598165 132.84481,80.509232 V 127.38418 H 66.701651 Z"
|
|
||||||
id="path124"
|
|
||||||
sodipodi:nodetypes="cccccc" />
|
|
||||||
<g
|
|
||||||
id="g1106-5"
|
|
||||||
transform="matrix(0,0.95201267,-0.95201267,0,194.01664,-57.627274)"
|
|
||||||
style="stroke-width:2.17959;stroke-miterlimit:4;stroke-dasharray:none">
|
|
||||||
<g
|
|
||||||
id="g1061-3"
|
|
||||||
style="stroke-width:2.17959;stroke-miterlimit:4;stroke-dasharray:none">
|
|
||||||
<circle
|
|
||||||
style="fill:none;stroke:#000000;stroke-width:2.17959;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
|
|
||||||
id="path956-5"
|
|
||||||
cx="168.8569"
|
|
||||||
cy="92.889587"
|
|
||||||
r="13.125794" />
|
|
||||||
<circle
|
|
||||||
style="fill:none;stroke:#000000;stroke-width:2.17959;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
|
|
||||||
id="path958-6"
|
|
||||||
cx="168.77444"
|
|
||||||
cy="92.702293"
|
|
||||||
r="3.0778286" />
|
|
||||||
<path
|
|
||||||
id="path960-2"
|
|
||||||
style="fill:none;stroke:#000000;stroke-width:2.17959;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
|
|
||||||
d="m 169.46072,82.84435 c 4.95795,0.336608 8.87296,4.341959 9.09638,9.306301"
|
|
||||||
sodipodi:nodetypes="cc" />
|
|
||||||
</g>
|
|
||||||
<path
|
|
||||||
style="fill:none;stroke:#000000;stroke-width:2.17959;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
|
|
||||||
d="m 154.67824,112.84018 11.89881,-13.038071 c 1.46407,-1.552664 3.79541,0.878511 2.81832,2.089181 l -10.57965,14.481 c -1.8851,2.02632 -6.10786,-1.06119 -4.13748,-3.53211 z"
|
|
||||||
id="path964-9"
|
|
||||||
sodipodi:nodetypes="ccccc" />
|
|
||||||
<g
|
|
||||||
id="g1071-1"
|
|
||||||
style="stroke-width:2.17959;stroke-miterlimit:4;stroke-dasharray:none" />
|
|
||||||
<g
|
|
||||||
id="g1065-3"
|
|
||||||
style="stroke-width:2.17959;stroke-miterlimit:4;stroke-dasharray:none">
|
|
||||||
<rect
|
|
||||||
style="fill:none;stroke:#000000;stroke-width:2.17959;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
|
|
||||||
id="rect949-6"
|
|
||||||
width="35.576611"
|
|
||||||
height="48.507355"
|
|
||||||
x="150.9623"
|
|
||||||
y="74.698929"
|
|
||||||
ry="2.7302756" />
|
|
||||||
<path
|
|
||||||
style="fill:none;stroke:#000000;stroke-width:2.17959;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
|
|
||||||
d="m 150.76919,106.16944 6.36181,-0.0223 c 2.53845,3.46232 6.29787,4.20243 10.1055,4.40362 l 0.0176,13.09251"
|
|
||||||
id="path1033-0"
|
|
||||||
sodipodi:nodetypes="cccc" />
|
|
||||||
</g>
|
|
||||||
</g>
|
|
||||||
</g>
|
|
||||||
<g
|
|
||||||
inkscape:label="Layer 1"
|
|
||||||
id="layer1-3-5"
|
|
||||||
transform="matrix(0.42851499,0,0,0.42851499,68.181495,12.180995)">
|
|
||||||
<path
|
|
||||||
style="fill:none;stroke:#000000;stroke-width:2.065;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
|
|
||||||
d="M 66.78016,73.340623 99.921832,54.219898 132.84481,73.130965 V 120.00591 H 66.701651 Z"
|
|
||||||
id="path124-6"
|
|
||||||
sodipodi:nodetypes="cccccc" />
|
|
||||||
<g
|
|
||||||
id="g1106-5-2"
|
|
||||||
transform="matrix(0,0.95201267,-0.95201267,0,194.01664,-65.058377)"
|
|
||||||
style="stroke-width:2.17959;stroke-miterlimit:4;stroke-dasharray:none">
|
|
||||||
<g
|
|
||||||
id="g1061-3-9"
|
|
||||||
style="stroke-width:2.17959;stroke-miterlimit:4;stroke-dasharray:none">
|
|
||||||
<circle
|
|
||||||
style="fill:none;stroke:#000000;stroke-width:2.17959;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
|
|
||||||
id="path956-5-1"
|
|
||||||
cx="168.8569"
|
|
||||||
cy="92.889587"
|
|
||||||
r="13.125794" />
|
|
||||||
<circle
|
|
||||||
style="fill:none;stroke:#000000;stroke-width:2.17959;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
|
|
||||||
id="path958-6-2"
|
|
||||||
cx="168.77444"
|
|
||||||
cy="92.702293"
|
|
||||||
r="3.0778286" />
|
|
||||||
<path
|
|
||||||
id="path960-2-7"
|
|
||||||
style="fill:none;stroke:#000000;stroke-width:2.17959;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
|
|
||||||
d="m 169.46072,82.84435 c 4.95795,0.336608 8.87296,4.341959 9.09638,9.306301"
|
|
||||||
sodipodi:nodetypes="cc" />
|
|
||||||
</g>
|
|
||||||
<path
|
|
||||||
style="fill:none;stroke:#000000;stroke-width:2.17959;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
|
|
||||||
d="m 154.67824,112.84018 11.89881,-13.038071 c 1.46407,-1.552664 3.79541,0.878511 2.81832,2.089181 l -10.57965,14.481 c -1.8851,2.02632 -6.10786,-1.06119 -4.13748,-3.53211 z"
|
|
||||||
id="path964-9-0"
|
|
||||||
sodipodi:nodetypes="ccccc" />
|
|
||||||
<g
|
|
||||||
id="g1071-1-9"
|
|
||||||
style="stroke-width:2.17959;stroke-miterlimit:4;stroke-dasharray:none" />
|
|
||||||
<g
|
|
||||||
id="g1065-3-3"
|
|
||||||
style="stroke-width:2.17959;stroke-miterlimit:4;stroke-dasharray:none">
|
|
||||||
<rect
|
|
||||||
style="fill:none;stroke:#000000;stroke-width:2.17959;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
|
|
||||||
id="rect949-6-6"
|
|
||||||
width="35.576611"
|
|
||||||
height="48.507355"
|
|
||||||
x="150.9623"
|
|
||||||
y="74.698929"
|
|
||||||
ry="2.7302756" />
|
|
||||||
<path
|
|
||||||
style="fill:none;stroke:#000000;stroke-width:2.17959;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
|
|
||||||
d="m 150.76919,106.16944 6.36181,-0.0223 c 2.53845,3.46232 6.29787,4.20243 10.1055,4.40362 l 0.0176,13.09251"
|
|
||||||
id="path1033-0-0"
|
|
||||||
sodipodi:nodetypes="cccc" />
|
|
||||||
</g>
|
|
||||||
</g>
|
|
||||||
</g>
|
|
||||||
<g
|
|
||||||
inkscape:label="Layer 1"
|
|
||||||
id="layer1-3-6"
|
|
||||||
transform="matrix(0.42851499,0,0,0.42851499,-20.953301,19.351613)">
|
|
||||||
<path
|
|
||||||
style="fill:none;stroke:#000000;stroke-width:2.065;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
|
|
||||||
d="M 66.78016,73.340623 99.921832,54.219898 132.84481,73.130965 V 120.00591 H 66.701651 Z"
|
|
||||||
id="path124-2"
|
|
||||||
sodipodi:nodetypes="cccccc" />
|
|
||||||
<g
|
|
||||||
id="g1106-5-6"
|
|
||||||
transform="matrix(0,0.95201267,-0.95201267,0,194.01664,-65.058377)"
|
|
||||||
style="stroke-width:2.17959;stroke-miterlimit:4;stroke-dasharray:none">
|
|
||||||
<g
|
|
||||||
id="g1061-3-1"
|
|
||||||
style="stroke-width:2.17959;stroke-miterlimit:4;stroke-dasharray:none">
|
|
||||||
<circle
|
|
||||||
style="fill:none;stroke:#000000;stroke-width:2.17959;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
|
|
||||||
id="path956-5-8"
|
|
||||||
cx="168.8569"
|
|
||||||
cy="92.889587"
|
|
||||||
r="13.125794" />
|
|
||||||
<circle
|
|
||||||
style="fill:none;stroke:#000000;stroke-width:2.17959;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
|
|
||||||
id="path958-6-7"
|
|
||||||
cx="168.77444"
|
|
||||||
cy="92.702293"
|
|
||||||
r="3.0778286" />
|
|
||||||
<path
|
|
||||||
id="path960-2-9"
|
|
||||||
style="fill:none;stroke:#000000;stroke-width:2.17959;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
|
|
||||||
d="m 169.46072,82.84435 c 4.95795,0.336608 8.87296,4.341959 9.09638,9.306301"
|
|
||||||
sodipodi:nodetypes="cc" />
|
|
||||||
</g>
|
|
||||||
<path
|
|
||||||
style="fill:none;stroke:#000000;stroke-width:2.17959;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
|
|
||||||
d="m 154.67824,112.84018 11.89881,-13.038071 c 1.46407,-1.552664 3.79541,0.878511 2.81832,2.089181 l -10.57965,14.481 c -1.8851,2.02632 -6.10786,-1.06119 -4.13748,-3.53211 z"
|
|
||||||
id="path964-9-2"
|
|
||||||
sodipodi:nodetypes="ccccc" />
|
|
||||||
<g
|
|
||||||
id="g1071-1-0"
|
|
||||||
style="stroke-width:2.17959;stroke-miterlimit:4;stroke-dasharray:none" />
|
|
||||||
<g
|
|
||||||
id="g1065-3-2"
|
|
||||||
style="stroke-width:2.17959;stroke-miterlimit:4;stroke-dasharray:none">
|
|
||||||
<rect
|
|
||||||
style="fill:none;stroke:#000000;stroke-width:2.17959;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
|
|
||||||
id="rect949-6-3"
|
|
||||||
width="35.576611"
|
|
||||||
height="48.507355"
|
|
||||||
x="150.9623"
|
|
||||||
y="74.698929"
|
|
||||||
ry="2.7302756" />
|
|
||||||
<path
|
|
||||||
style="fill:none;stroke:#000000;stroke-width:2.17959;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
|
|
||||||
d="m 150.76919,106.16944 6.36181,-0.0223 c 2.53845,3.46232 6.29787,4.20243 10.1055,4.40362 l 0.0176,13.09251"
|
|
||||||
id="path1033-0-7"
|
|
||||||
sodipodi:nodetypes="cccc" />
|
|
||||||
</g>
|
|
||||||
</g>
|
|
||||||
</g>
|
|
||||||
<g
|
|
||||||
inkscape:label="Layer 1"
|
|
||||||
id="layer1-3-59"
|
|
||||||
transform="matrix(0.42851499,0,0,0.42851499,51.949789,75.218277)">
|
|
||||||
<path
|
|
||||||
style="fill:none;stroke:#000000;stroke-width:2.065;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
|
|
||||||
d="M 66.78016,73.340623 99.921832,54.219898 132.84481,73.130965 V 120.00591 H 66.701651 Z"
|
|
||||||
id="path124-22"
|
|
||||||
sodipodi:nodetypes="cccccc" />
|
|
||||||
<g
|
|
||||||
id="g1106-5-8"
|
|
||||||
transform="matrix(0,0.95201267,-0.95201267,0,194.01664,-65.058377)"
|
|
||||||
style="stroke-width:2.17959;stroke-miterlimit:4;stroke-dasharray:none">
|
|
||||||
<g
|
|
||||||
id="g1061-3-97"
|
|
||||||
style="stroke-width:2.17959;stroke-miterlimit:4;stroke-dasharray:none">
|
|
||||||
<circle
|
|
||||||
style="fill:none;stroke:#000000;stroke-width:2.17959;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
|
|
||||||
id="path956-5-3"
|
|
||||||
cx="168.8569"
|
|
||||||
cy="92.889587"
|
|
||||||
r="13.125794" />
|
|
||||||
<circle
|
|
||||||
style="fill:none;stroke:#000000;stroke-width:2.17959;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
|
|
||||||
id="path958-6-6"
|
|
||||||
cx="168.77444"
|
|
||||||
cy="92.702293"
|
|
||||||
r="3.0778286" />
|
|
||||||
<path
|
|
||||||
id="path960-2-1"
|
|
||||||
style="fill:none;stroke:#000000;stroke-width:2.17959;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
|
|
||||||
d="m 169.46072,82.84435 c 4.95795,0.336608 8.87296,4.341959 9.09638,9.306301"
|
|
||||||
sodipodi:nodetypes="cc" />
|
|
||||||
</g>
|
|
||||||
<path
|
|
||||||
style="fill:none;stroke:#000000;stroke-width:2.17959;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
|
|
||||||
d="m 154.67824,112.84018 11.89881,-13.038071 c 1.46407,-1.552664 3.79541,0.878511 2.81832,2.089181 l -10.57965,14.481 c -1.8851,2.02632 -6.10786,-1.06119 -4.13748,-3.53211 z"
|
|
||||||
id="path964-9-29"
|
|
||||||
sodipodi:nodetypes="ccccc" />
|
|
||||||
<g
|
|
||||||
id="g1071-1-3"
|
|
||||||
style="stroke-width:2.17959;stroke-miterlimit:4;stroke-dasharray:none" />
|
|
||||||
<g
|
|
||||||
id="g1065-3-1"
|
|
||||||
style="stroke-width:2.17959;stroke-miterlimit:4;stroke-dasharray:none">
|
|
||||||
<rect
|
|
||||||
style="fill:none;stroke:#000000;stroke-width:2.17959;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
|
|
||||||
id="rect949-6-9"
|
|
||||||
width="35.576611"
|
|
||||||
height="48.507355"
|
|
||||||
x="150.9623"
|
|
||||||
y="74.698929"
|
|
||||||
ry="2.7302756" />
|
|
||||||
<path
|
|
||||||
style="fill:none;stroke:#000000;stroke-width:2.17959;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
|
|
||||||
d="m 150.76919,106.16944 6.36181,-0.0223 c 2.53845,3.46232 6.29787,4.20243 10.1055,4.40362 l 0.0176,13.09251"
|
|
||||||
id="path1033-0-4"
|
|
||||||
sodipodi:nodetypes="cccc" />
|
|
||||||
</g>
|
|
||||||
</g>
|
|
||||||
</g>
|
|
||||||
<g
|
|
||||||
inkscape:label="Layer 1"
|
|
||||||
id="layer1-3-7"
|
|
||||||
transform="matrix(0.42851499,0,0,0.42851499,-1.173447,75.150288)">
|
|
||||||
<path
|
|
||||||
style="fill:none;stroke:#000000;stroke-width:2.065;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
|
|
||||||
d="M 66.78016,73.340623 99.921832,54.219898 132.84481,73.130965 V 120.00591 H 66.701651 Z"
|
|
||||||
id="path124-8"
|
|
||||||
sodipodi:nodetypes="cccccc" />
|
|
||||||
<g
|
|
||||||
id="g1106-5-4"
|
|
||||||
transform="matrix(0,0.95201267,-0.95201267,0,194.01664,-65.058377)"
|
|
||||||
style="stroke-width:2.17959;stroke-miterlimit:4;stroke-dasharray:none">
|
|
||||||
<g
|
|
||||||
id="g1061-3-5"
|
|
||||||
style="stroke-width:2.17959;stroke-miterlimit:4;stroke-dasharray:none">
|
|
||||||
<circle
|
|
||||||
style="fill:none;stroke:#000000;stroke-width:2.17959;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
|
|
||||||
id="path956-5-0"
|
|
||||||
cx="168.8569"
|
|
||||||
cy="92.889587"
|
|
||||||
r="13.125794" />
|
|
||||||
<circle
|
|
||||||
style="fill:none;stroke:#000000;stroke-width:2.17959;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
|
|
||||||
id="path958-6-3"
|
|
||||||
cx="168.77444"
|
|
||||||
cy="92.702293"
|
|
||||||
r="3.0778286" />
|
|
||||||
<path
|
|
||||||
id="path960-2-6"
|
|
||||||
style="fill:none;stroke:#000000;stroke-width:2.17959;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
|
|
||||||
d="m 169.46072,82.84435 c 4.95795,0.336608 8.87296,4.341959 9.09638,9.306301"
|
|
||||||
sodipodi:nodetypes="cc" />
|
|
||||||
</g>
|
|
||||||
<path
|
|
||||||
style="fill:none;stroke:#000000;stroke-width:2.17959;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
|
|
||||||
d="m 154.67824,112.84018 11.89881,-13.038071 c 1.46407,-1.552664 3.79541,0.878511 2.81832,2.089181 l -10.57965,14.481 c -1.8851,2.02632 -6.10786,-1.06119 -4.13748,-3.53211 z"
|
|
||||||
id="path964-9-1"
|
|
||||||
sodipodi:nodetypes="ccccc" />
|
|
||||||
<g
|
|
||||||
id="g1071-1-06"
|
|
||||||
style="stroke-width:2.17959;stroke-miterlimit:4;stroke-dasharray:none" />
|
|
||||||
<g
|
|
||||||
id="g1065-3-32"
|
|
||||||
style="stroke-width:2.17959;stroke-miterlimit:4;stroke-dasharray:none">
|
|
||||||
<rect
|
|
||||||
style="fill:none;stroke:#000000;stroke-width:2.17959;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
|
|
||||||
id="rect949-6-0"
|
|
||||||
width="35.576611"
|
|
||||||
height="48.507355"
|
|
||||||
x="150.9623"
|
|
||||||
y="74.698929"
|
|
||||||
ry="2.7302756" />
|
|
||||||
<path
|
|
||||||
style="fill:none;stroke:#000000;stroke-width:2.17959;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
|
|
||||||
d="m 150.76919,106.16944 6.36181,-0.0223 c 2.53845,3.46232 6.29787,4.20243 10.1055,4.40362 l 0.0176,13.09251"
|
|
||||||
id="path1033-0-6"
|
|
||||||
sodipodi:nodetypes="cccc" />
|
|
||||||
</g>
|
|
||||||
</g>
|
|
||||||
</g>
|
|
||||||
<path
|
|
||||||
style="fill:none;stroke:#000000;stroke-width:1;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
|
|
||||||
d="M 35.21897,43.254452 46.803736,32.872178"
|
|
||||||
id="path1045" />
|
|
||||||
<path
|
|
||||||
style="fill:none;stroke:#000000;stroke-width:1;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
|
|
||||||
d="m 85.798392,29.613721 10.944185,7.688225"
|
|
||||||
id="path1047" />
|
|
||||||
<path
|
|
||||||
style="fill:none;stroke:#000000;stroke-width:1;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
|
|
||||||
d="m 107.59813,71.879386 -6.2564,22.552649"
|
|
||||||
id="path1049" />
|
|
||||||
<path
|
|
||||||
style="fill:none;stroke:#000000;stroke-width:1;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
|
|
||||||
d="M 75.866769,119.14997 61.529058,118.74136"
|
|
||||||
id="path1051"
|
|
||||||
sodipodi:nodetypes="cc" />
|
|
||||||
<path
|
|
||||||
style="fill:none;stroke:#000000;stroke-width:1;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
|
|
||||||
d="M 29.414211,98.256475 C 29.681482,96.462435 21.07721,77.446418 21.07721,77.446418"
|
|
||||||
id="path1053" />
|
|
||||||
<path
|
|
||||||
style="fill:none;stroke:#000000;stroke-width:1;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
|
|
||||||
d="M 39.447822,61.341585 90.641428,57.562618"
|
|
||||||
id="path1055" />
|
|
||||||
<path
|
|
||||||
style="fill:none;stroke:#000000;stroke-width:1;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
|
|
||||||
d="M 90.440176,64.423751 54.180736,100.02908"
|
|
||||||
id="path1057"
|
|
||||||
sodipodi:nodetypes="cc" />
|
|
||||||
<path
|
|
||||||
style="fill:none;stroke:#000000;stroke-width:1;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
|
|
||||||
d="M 47.163557,96.532205 61.535331,33.078667"
|
|
||||||
id="path1059"
|
|
||||||
sodipodi:nodetypes="cc" />
|
|
||||||
<path
|
|
||||||
style="fill:none;stroke:#000000;stroke-width:1;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
|
|
||||||
d="m 71.396211,33.058731 15.77285,60.595014"
|
|
||||||
id="path1061" />
|
|
||||||
<path
|
|
||||||
style="fill:none;stroke:#000000;stroke-width:1;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
|
|
||||||
d="M 79.384641,100.96895 41.150775,67.902625"
|
|
||||||
id="path1063" />
|
|
||||||
</g>
|
|
||||||
</svg>
|
|
||||||
|
Before Width: | Height: | Size: 19 KiB |
@@ -1,502 +0,0 @@
|
|||||||
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
|
|
||||||
<svg
|
|
||||||
xmlns:dc="http://purl.org/dc/elements/1.1/"
|
|
||||||
xmlns:cc="http://creativecommons.org/ns#"
|
|
||||||
xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
|
|
||||||
xmlns:svg="http://www.w3.org/2000/svg"
|
|
||||||
xmlns="http://www.w3.org/2000/svg"
|
|
||||||
xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
|
|
||||||
xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
|
|
||||||
width="850"
|
|
||||||
height="480"
|
|
||||||
viewBox="0 0 224.89584 127"
|
|
||||||
version="1.1"
|
|
||||||
id="svg8"
|
|
||||||
inkscape:version="1.0.1 (3bc2e813f5, 2020-09-07)"
|
|
||||||
sodipodi:docname="garage_tables.svg">
|
|
||||||
<defs
|
|
||||||
id="defs2">
|
|
||||||
<marker
|
|
||||||
style="overflow:visible"
|
|
||||||
id="marker1262"
|
|
||||||
refX="0"
|
|
||||||
refY="0"
|
|
||||||
orient="auto"
|
|
||||||
inkscape:stockid="Arrow1Mend"
|
|
||||||
inkscape:isstock="true">
|
|
||||||
<path
|
|
||||||
transform="matrix(-0.4,0,0,-0.4,-4,0)"
|
|
||||||
style="fill:#000000;fill-opacity:1;fill-rule:evenodd;stroke:#000000;stroke-width:1pt;stroke-opacity:1"
|
|
||||||
d="M 0,0 5,-5 -12.5,0 5,5 Z"
|
|
||||||
id="path1260" />
|
|
||||||
</marker>
|
|
||||||
<marker
|
|
||||||
style="overflow:visible"
|
|
||||||
id="Arrow1Mend"
|
|
||||||
refX="0"
|
|
||||||
refY="0"
|
|
||||||
orient="auto"
|
|
||||||
inkscape:stockid="Arrow1Mend"
|
|
||||||
inkscape:isstock="true"
|
|
||||||
inkscape:collect="always">
|
|
||||||
<path
|
|
||||||
transform="matrix(-0.4,0,0,-0.4,-4,0)"
|
|
||||||
style="fill:#000000;fill-opacity:1;fill-rule:evenodd;stroke:#000000;stroke-width:1pt;stroke-opacity:1"
|
|
||||||
d="M 0,0 5,-5 -12.5,0 5,5 Z"
|
|
||||||
id="path965" />
|
|
||||||
</marker>
|
|
||||||
<marker
|
|
||||||
style="overflow:visible"
|
|
||||||
id="Arrow1Lend"
|
|
||||||
refX="0"
|
|
||||||
refY="0"
|
|
||||||
orient="auto"
|
|
||||||
inkscape:stockid="Arrow1Lend"
|
|
||||||
inkscape:isstock="true">
|
|
||||||
<path
|
|
||||||
transform="matrix(-0.8,0,0,-0.8,-10,0)"
|
|
||||||
style="fill:#000000;fill-opacity:1;fill-rule:evenodd;stroke:#000000;stroke-width:1pt;stroke-opacity:1"
|
|
||||||
d="M 0,0 5,-5 -12.5,0 5,5 Z"
|
|
||||||
id="path959" />
|
|
||||||
</marker>
|
|
||||||
</defs>
|
|
||||||
<sodipodi:namedview
|
|
||||||
id="base"
|
|
||||||
pagecolor="#ffffff"
|
|
||||||
bordercolor="#666666"
|
|
||||||
borderopacity="1.0"
|
|
||||||
inkscape:pageopacity="0.0"
|
|
||||||
inkscape:pageshadow="2"
|
|
||||||
inkscape:zoom="0.98994949"
|
|
||||||
inkscape:cx="381.09221"
|
|
||||||
inkscape:cy="219.5592"
|
|
||||||
inkscape:document-units="mm"
|
|
||||||
inkscape:current-layer="layer1"
|
|
||||||
inkscape:document-rotation="0"
|
|
||||||
showgrid="false"
|
|
||||||
units="px"
|
|
||||||
inkscape:window-width="1867"
|
|
||||||
inkscape:window-height="1016"
|
|
||||||
inkscape:window-x="53"
|
|
||||||
inkscape:window-y="27"
|
|
||||||
inkscape:window-maximized="1" />
|
|
||||||
<metadata
|
|
||||||
id="metadata5">
|
|
||||||
<rdf:RDF>
|
|
||||||
<cc:Work
|
|
||||||
rdf:about="">
|
|
||||||
<dc:format>image/svg+xml</dc:format>
|
|
||||||
<dc:type
|
|
||||||
rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
|
|
||||||
<dc:title></dc:title>
|
|
||||||
</cc:Work>
|
|
||||||
</rdf:RDF>
|
|
||||||
</metadata>
|
|
||||||
<g
|
|
||||||
inkscape:label="Layer 1"
|
|
||||||
inkscape:groupmode="layer"
|
|
||||||
id="layer1">
|
|
||||||
<text
|
|
||||||
xml:space="preserve"
|
|
||||||
style="font-style:normal;font-weight:normal;font-size:5.64444px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.264583"
|
|
||||||
x="39.570904"
|
|
||||||
y="38.452755"
|
|
||||||
id="text2025"><tspan
|
|
||||||
sodipodi:role="line"
|
|
||||||
id="tspan2023"
|
|
||||||
x="39.570904"
|
|
||||||
y="38.452755"
|
|
||||||
style="font-size:5.64444px;stroke-width:0.264583" /></text>
|
|
||||||
<text
|
|
||||||
xml:space="preserve"
|
|
||||||
style="font-style:normal;font-weight:normal;font-size:10.5833px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.264583"
|
|
||||||
x="101.95796"
|
|
||||||
y="92.835831"
|
|
||||||
id="text2139"><tspan
|
|
||||||
sodipodi:role="line"
|
|
||||||
id="tspan2137"
|
|
||||||
x="101.95796"
|
|
||||||
y="92.835831"
|
|
||||||
style="stroke-width:0.264583"> </tspan></text>
|
|
||||||
<g
|
|
||||||
id="g2316"
|
|
||||||
transform="translate(-11.455511,1.5722486)">
|
|
||||||
<g
|
|
||||||
id="g2277">
|
|
||||||
<rect
|
|
||||||
style="fill:none;stroke:#000000;stroke-width:0.8;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1"
|
|
||||||
id="rect833"
|
|
||||||
width="47.419891"
|
|
||||||
height="95.353409"
|
|
||||||
x="18.534418"
|
|
||||||
y="24.42766" />
|
|
||||||
<rect
|
|
||||||
style="fill:none;stroke:#000000;stroke-width:0.799999;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1"
|
|
||||||
id="rect833-3"
|
|
||||||
width="47.419891"
|
|
||||||
height="86.973076"
|
|
||||||
x="18.534418"
|
|
||||||
y="32.807987" />
|
|
||||||
<text
|
|
||||||
xml:space="preserve"
|
|
||||||
style="font-style:normal;font-weight:normal;font-size:5.64444px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.264583"
|
|
||||||
x="32.250839"
|
|
||||||
y="29.894743"
|
|
||||||
id="text852"><tspan
|
|
||||||
sodipodi:role="line"
|
|
||||||
id="tspan850"
|
|
||||||
x="32.250839"
|
|
||||||
y="29.894743"
|
|
||||||
style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:5.64444px;font-family:'Liberation Mono';-inkscape-font-specification:'Liberation Mono Bold';stroke-width:0.264583">Object</tspan></text>
|
|
||||||
</g>
|
|
||||||
<g
|
|
||||||
id="g2066"
|
|
||||||
transform="translate(-2.1807817,-3.0621439)">
|
|
||||||
<g
|
|
||||||
id="g1969"
|
|
||||||
transform="matrix(0.12763631,0,0,0.12763631,0.7215051,24.717273)"
|
|
||||||
style="fill:#ff6600;fill-opacity:1;stroke:none;stroke-opacity:1">
|
|
||||||
<path
|
|
||||||
style="fill:#ff6600;fill-opacity:1;stroke:none;stroke-width:0.264583;stroke-opacity:1"
|
|
||||||
d="m 203.71837,154.80038 c -1.11451,3.75057 -2.45288,5.84095 -5.11132,7.98327 -2.2735,1.83211 -4.66721,2.65982 -8.09339,2.79857 -2.59227,0.10498 -2.92868,0.0577 -5.02863,-0.70611 -3.99215,-1.45212 -7.1627,-4.65496 -8.48408,-8.57046 -1.28374,-3.80398 -0.61478,-8.68216 1.64793,-12.01698 0.87317,-1.28689 3.15089,-3.48326 4.18771,-4.03815 l 0.53332,-28.51234 5.78454,-5.09197 6.95158,6.16704 -3.21112,3.49026 3.17616,3.45499 -3.17616,3.40822 2.98973,3.28645 -3.24843,3.3829 4.49203,4.58395 0.0516,5.69106 c 1.06874,0.64848 3.81974,3.24046 4.69548,4.56257 0.452,0.68241 1.06834,2.0197 1.36962,2.97176 0.62932,1.98864 0.88051,5.785 0.47342,7.15497 z m -10.0406,2.32604 c -0.88184,-3.17515 -4.92402,-3.78864 -6.75297,-1.02492 -0.58328,0.8814 -0.6898,1.28852 -0.58362,2.23056 0.26492,2.35041 2.45434,3.95262 4.60856,3.37255 1.19644,-0.32217 2.39435,-1.44872 2.72875,-2.56621 0.30682,-1.02529 0.30686,-0.9045 -7.9e-4,-2.01198 z"
|
|
||||||
id="path1971"
|
|
||||||
sodipodi:nodetypes="ssscsscccccccccccssscsssscc" />
|
|
||||||
</g>
|
|
||||||
<text
|
|
||||||
xml:space="preserve"
|
|
||||||
style="font-style:normal;font-weight:normal;font-size:5.64444px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.264583"
|
|
||||||
x="28.809687"
|
|
||||||
y="44.070885"
|
|
||||||
id="text852-9"><tspan
|
|
||||||
sodipodi:role="line"
|
|
||||||
id="tspan850-4"
|
|
||||||
x="28.809687"
|
|
||||||
y="44.070885"
|
|
||||||
style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:5.64444px;font-family:'Liberation Mono';-inkscape-font-specification:'Liberation Mono Bold';stroke-width:0.264583">bucket </tspan></text>
|
|
||||||
</g>
|
|
||||||
<g
|
|
||||||
id="g2066-7"
|
|
||||||
transform="translate(-2.1807817,6.2627616)">
|
|
||||||
<g
|
|
||||||
id="g1969-8"
|
|
||||||
transform="matrix(0.12763631,0,0,0.12763631,0.7215051,24.717273)"
|
|
||||||
style="fill:#ff6600;fill-opacity:1;stroke:none;stroke-opacity:1">
|
|
||||||
<path
|
|
||||||
style="fill:#ff6600;fill-opacity:1;stroke:none;stroke-width:0.264583;stroke-opacity:1"
|
|
||||||
d="m 203.71837,154.80038 c -1.11451,3.75057 -2.45288,5.84095 -5.11132,7.98327 -2.2735,1.83211 -4.66721,2.65982 -8.09339,2.79857 -2.59227,0.10498 -2.92868,0.0577 -5.02863,-0.70611 -3.99215,-1.45212 -7.1627,-4.65496 -8.48408,-8.57046 -1.28374,-3.80398 -0.61478,-8.68216 1.64793,-12.01698 0.87317,-1.28689 3.15089,-3.48326 4.18771,-4.03815 l 0.53332,-28.51234 5.78454,-5.09197 6.95158,6.16704 -3.21112,3.49026 3.17616,3.45499 -3.17616,3.40822 2.98973,3.28645 -3.24843,3.3829 4.49203,4.58395 0.0516,5.69106 c 1.06874,0.64848 3.81974,3.24046 4.69548,4.56257 0.452,0.68241 1.06834,2.0197 1.36962,2.97176 0.62932,1.98864 0.88051,5.785 0.47342,7.15497 z m -10.0406,2.32604 c -0.88184,-3.17515 -4.92402,-3.78864 -6.75297,-1.02492 -0.58328,0.8814 -0.6898,1.28852 -0.58362,2.23056 0.26492,2.35041 2.45434,3.95262 4.60856,3.37255 1.19644,-0.32217 2.39435,-1.44872 2.72875,-2.56621 0.30682,-1.02529 0.30686,-0.9045 -7.9e-4,-2.01198 z"
|
|
||||||
id="path1971-4"
|
|
||||||
sodipodi:nodetypes="ssscsscccccccccccssscsssscc" />
|
|
||||||
</g>
|
|
||||||
<text
|
|
||||||
xml:space="preserve"
|
|
||||||
style="font-style:normal;font-weight:normal;font-size:5.64444px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.264583"
|
|
||||||
x="28.809687"
|
|
||||||
y="44.070885"
|
|
||||||
id="text852-9-5"><tspan
|
|
||||||
sodipodi:role="line"
|
|
||||||
id="tspan850-4-0"
|
|
||||||
x="28.809687"
|
|
||||||
y="44.070885"
|
|
||||||
style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:5.64444px;font-family:'Liberation Mono';-inkscape-font-specification:'Liberation Mono Bold';stroke-width:0.264583">file path </tspan></text>
|
|
||||||
</g>
|
|
||||||
<g
|
|
||||||
id="g2161"
|
|
||||||
transform="translate(-62.264403,-59.333115)">
|
|
||||||
<g
|
|
||||||
id="g2271"
|
|
||||||
transform="translate(0,67.042823)">
|
|
||||||
<rect
|
|
||||||
style="fill:none;stroke:#000000;stroke-width:0.799999;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1"
|
|
||||||
id="rect833-6"
|
|
||||||
width="39.008453"
|
|
||||||
height="16.775949"
|
|
||||||
x="84.896881"
|
|
||||||
y="90.266838" />
|
|
||||||
<rect
|
|
||||||
style="fill:none;stroke:#000000;stroke-width:0.799999;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1"
|
|
||||||
id="rect833-3-1"
|
|
||||||
width="39.008453"
|
|
||||||
height="8.673645"
|
|
||||||
x="84.896881"
|
|
||||||
y="98.369141" />
|
|
||||||
<text
|
|
||||||
xml:space="preserve"
|
|
||||||
style="font-style:normal;font-weight:normal;font-size:5.64444px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.264583"
|
|
||||||
x="89.826942"
|
|
||||||
y="96.212921"
|
|
||||||
id="text852-0"><tspan
|
|
||||||
sodipodi:role="line"
|
|
||||||
id="tspan850-6"
|
|
||||||
x="89.826942"
|
|
||||||
y="96.212921"
|
|
||||||
style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:5.64444px;font-family:'Liberation Mono';-inkscape-font-specification:'Liberation Mono Bold';stroke-width:0.264583">Version 1</tspan></text>
|
|
||||||
<text
|
|
||||||
xml:space="preserve"
|
|
||||||
style="font-style:normal;font-weight:normal;font-size:5.64444px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.264583"
|
|
||||||
x="89.826942"
|
|
||||||
y="104.71013"
|
|
||||||
id="text852-0-3"><tspan
|
|
||||||
sodipodi:role="line"
|
|
||||||
id="tspan850-6-2"
|
|
||||||
x="89.826942"
|
|
||||||
y="104.71013"
|
|
||||||
style="font-style:italic;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:5.64444px;font-family:'Liberation Mono';-inkscape-font-specification:'Liberation Mono Bold';fill:#4d4d4d;stroke-width:0.264583">deleted</tspan></text>
|
|
||||||
</g>
|
|
||||||
</g>
|
|
||||||
<g
|
|
||||||
id="g2263"
|
|
||||||
transform="translate(0,-22.791204)">
|
|
||||||
<g
|
|
||||||
id="g2161-1"
|
|
||||||
transform="translate(-62.264403,-10.910843)">
|
|
||||||
<rect
|
|
||||||
style="fill:none;stroke:#000000;stroke-width:0.799999;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1"
|
|
||||||
id="rect833-6-5"
|
|
||||||
width="39.008453"
|
|
||||||
height="36.749603"
|
|
||||||
x="84.896881"
|
|
||||||
y="90.266838" />
|
|
||||||
<rect
|
|
||||||
style="fill:none;stroke:#000000;stroke-width:0.799999;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1"
|
|
||||||
id="rect833-3-1-5"
|
|
||||||
width="39.008453"
|
|
||||||
height="28.647301"
|
|
||||||
x="84.896881"
|
|
||||||
y="98.369141" />
|
|
||||||
<text
|
|
||||||
xml:space="preserve"
|
|
||||||
style="font-style:normal;font-weight:normal;font-size:5.64444px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.264583"
|
|
||||||
x="89.826942"
|
|
||||||
y="96.212921"
|
|
||||||
id="text852-0-4"><tspan
|
|
||||||
sodipodi:role="line"
|
|
||||||
id="tspan850-6-7"
|
|
||||||
x="89.826942"
|
|
||||||
y="96.212921"
|
|
||||||
style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:5.64444px;font-family:'Liberation Mono';-inkscape-font-specification:'Liberation Mono Bold';stroke-width:0.264583">Version 2</tspan></text>
|
|
||||||
<text
|
|
||||||
xml:space="preserve"
|
|
||||||
style="font-style:normal;font-weight:normal;font-size:5.64444px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.264583"
|
|
||||||
x="89.826942"
|
|
||||||
y="104.71013"
|
|
||||||
id="text852-0-3-6"><tspan
|
|
||||||
sodipodi:role="line"
|
|
||||||
id="tspan850-6-2-5"
|
|
||||||
x="89.826942"
|
|
||||||
y="104.71013"
|
|
||||||
style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:5.64444px;font-family:'Liberation Mono';-inkscape-font-specification:'Liberation Mono Bold';fill:#000000;stroke-width:0.264583">id</tspan></text>
|
|
||||||
</g>
|
|
||||||
<text
|
|
||||||
xml:space="preserve"
|
|
||||||
style="font-style:normal;font-weight:normal;font-size:5.64444px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.264583"
|
|
||||||
x="27.56254"
|
|
||||||
y="100.34132"
|
|
||||||
id="text852-0-3-6-6"><tspan
|
|
||||||
sodipodi:role="line"
|
|
||||||
id="tspan850-6-2-5-9"
|
|
||||||
x="27.56254"
|
|
||||||
y="100.34132"
|
|
||||||
style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:5.64444px;font-family:'Liberation Mono';-inkscape-font-specification:'Liberation Mono Bold';fill:#000000;stroke-width:0.264583">size</tspan></text>
|
|
||||||
<text
|
|
||||||
xml:space="preserve"
|
|
||||||
style="font-style:normal;font-weight:normal;font-size:5.64444px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.264583"
|
|
||||||
x="27.56254"
|
|
||||||
y="106.90263"
|
|
||||||
id="text852-0-3-6-6-3"><tspan
|
|
||||||
sodipodi:role="line"
|
|
||||||
id="tspan850-6-2-5-9-7"
|
|
||||||
x="27.56254"
|
|
||||||
y="106.90263"
|
|
||||||
style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:5.64444px;font-family:'Liberation Mono';-inkscape-font-specification:'Liberation Mono Bold';fill:#000000;stroke-width:0.264583">MIME type</tspan></text>
|
|
||||||
<text
|
|
||||||
xml:space="preserve"
|
|
||||||
style="font-style:normal;font-weight:normal;font-size:5.64444px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.264583"
|
|
||||||
x="27.56254"
|
|
||||||
y="111.92816"
|
|
||||||
id="text852-0-3-6-6-3-4"><tspan
|
|
||||||
sodipodi:role="line"
|
|
||||||
id="tspan850-6-2-5-9-7-5"
|
|
||||||
x="27.56254"
|
|
||||||
y="111.92816"
|
|
||||||
style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:5.64444px;font-family:'Liberation Mono';-inkscape-font-specification:'Liberation Mono Bold';fill:#000000;stroke-width:0.264583">...</tspan></text>
|
|
||||||
</g>
|
|
||||||
</g>
|
|
||||||
<g
|
|
||||||
id="g898"
|
|
||||||
transform="translate(-6.2484318,29.95006)">
|
|
||||||
<rect
|
|
||||||
style="fill:none;stroke:#000000;stroke-width:0.799999;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1"
|
|
||||||
id="rect833-7"
|
|
||||||
width="47.419891"
|
|
||||||
height="44.007515"
|
|
||||||
x="95.443573"
|
|
||||||
y="24.42766" />
|
|
||||||
<rect
|
|
||||||
style="fill:none;stroke:#000000;stroke-width:0.799999;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1"
|
|
||||||
id="rect833-3-4"
|
|
||||||
width="47.419891"
|
|
||||||
height="35.627186"
|
|
||||||
x="95.443573"
|
|
||||||
y="32.807987" />
|
|
||||||
<text
|
|
||||||
xml:space="preserve"
|
|
||||||
style="font-style:normal;font-weight:normal;font-size:5.64444px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.264583"
|
|
||||||
x="107.46638"
|
|
||||||
y="29.894743"
|
|
||||||
id="text852-4"><tspan
|
|
||||||
sodipodi:role="line"
|
|
||||||
id="tspan850-3"
|
|
||||||
x="107.46638"
|
|
||||||
y="29.894743"
|
|
||||||
style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:5.64444px;font-family:'Liberation Mono';-inkscape-font-specification:'Liberation Mono Bold';stroke-width:0.264583">Version</tspan></text>
|
|
||||||
<path
|
|
||||||
style="fill:#ff6600;fill-opacity:1;stroke:none;stroke-width:0.0337704;stroke-opacity:1"
|
|
||||||
d="m 102.90563,41.413279 c -0.14226,0.478709 -0.31308,0.745518 -0.65239,1.018956 -0.29019,0.233843 -0.59571,0.339489 -1.03301,0.357199 -0.33087,0.0134 -0.37381,0.0074 -0.64184,-0.09013 -0.50954,-0.185343 -0.914221,-0.594142 -1.082877,-1.093901 -0.163852,-0.485526 -0.07847,-1.108159 0.210335,-1.533803 0.111448,-0.164254 0.402172,-0.444591 0.534502,-0.515415 l 0.0681,-3.63921 0.73832,-0.64992 0.88727,0.787138 -0.40985,0.445484 0.40539,0.440982 -0.40539,0.435013 0.3816,0.41947 -0.41462,0.431781 0.57335,0.585078 0.007,0.726386 c 0.13641,0.08277 0.48753,0.413601 0.59931,0.58235 0.0577,0.0871 0.13636,0.257787 0.17481,0.379304 0.0803,0.253823 0.11239,0.738377 0.0604,0.913234 z m -1.28155,0.296888 c -0.11255,-0.405265 -0.62848,-0.483569 -0.86192,-0.130817 -0.0744,0.112498 -0.088,0.164461 -0.0745,0.2847 0.0338,0.299998 0.31326,0.504498 0.58822,0.43046 0.15271,-0.04112 0.3056,-0.184909 0.34828,-0.327542 0.0392,-0.130864 0.0392,-0.115447 -1e-4,-0.256801 z"
|
|
||||||
id="path1971-0"
|
|
||||||
sodipodi:nodetypes="ssscsscccccccccccssscsssscc" />
|
|
||||||
<text
|
|
||||||
xml:space="preserve"
|
|
||||||
style="font-style:normal;font-weight:normal;font-size:5.64444px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.264583"
|
|
||||||
x="104.99195"
|
|
||||||
y="41.008743"
|
|
||||||
id="text852-9-7"><tspan
|
|
||||||
sodipodi:role="line"
|
|
||||||
id="tspan850-4-8"
|
|
||||||
x="104.99195"
|
|
||||||
y="41.008743"
|
|
||||||
style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:5.64444px;font-family:'Liberation Mono';-inkscape-font-specification:'Liberation Mono Bold';stroke-width:0.264583">id </tspan></text>
|
|
||||||
<text
|
|
||||||
xml:space="preserve"
|
|
||||||
style="font-style:normal;font-weight:normal;font-size:5.64444px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.264583"
|
|
||||||
x="104.99195"
|
|
||||||
y="49.168018"
|
|
||||||
id="text852-9-7-6"><tspan
|
|
||||||
sodipodi:role="line"
|
|
||||||
id="tspan850-4-8-8"
|
|
||||||
x="104.99195"
|
|
||||||
y="49.168018"
|
|
||||||
style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:5.64444px;font-family:'Liberation Mono';-inkscape-font-specification:'Liberation Mono Bold';stroke-width:0.264583">h(block 1)</tspan></text>
|
|
||||||
<text
|
|
||||||
xml:space="preserve"
|
|
||||||
style="font-style:normal;font-weight:normal;font-size:5.64444px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.264583"
|
|
||||||
x="104.99195"
|
|
||||||
y="56.583336"
|
|
||||||
id="text852-9-7-6-8"><tspan
|
|
||||||
sodipodi:role="line"
|
|
||||||
id="tspan850-4-8-8-4"
|
|
||||||
x="104.99195"
|
|
||||||
y="56.583336"
|
|
||||||
style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:5.64444px;font-family:'Liberation Mono';-inkscape-font-specification:'Liberation Mono Bold';stroke-width:0.264583">h(block 2)</tspan></text>
|
|
||||||
<text
|
|
||||||
xml:space="preserve"
|
|
||||||
style="font-style:normal;font-weight:normal;font-size:5.64444px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.264583"
|
|
||||||
x="104.99195"
|
|
||||||
y="64.265732"
|
|
||||||
id="text852-9-7-6-3"><tspan
|
|
||||||
sodipodi:role="line"
|
|
||||||
id="tspan850-4-8-8-1"
|
|
||||||
x="104.99195"
|
|
||||||
y="64.265732"
|
|
||||||
style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:5.64444px;font-family:'Liberation Mono';-inkscape-font-specification:'Liberation Mono Bold';stroke-width:0.264583">...</tspan></text>
|
|
||||||
</g>
|
|
||||||
<g
|
|
||||||
id="g898-3"
|
|
||||||
transform="translate(75.777779,38.888663)">
|
|
||||||
<rect
|
|
||||||
style="fill:none;stroke:#000000;stroke-width:0.799999;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1"
|
|
||||||
id="rect833-7-6"
|
|
||||||
width="47.419891"
|
|
||||||
height="29.989157"
|
|
||||||
x="95.443573"
|
|
||||||
y="24.42766" />
|
|
||||||
<rect
|
|
||||||
style="fill:none;stroke:#000000;stroke-width:0.799999;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1"
|
|
||||||
id="rect833-3-4-7"
|
|
||||||
width="47.419891"
|
|
||||||
height="21.608831"
|
|
||||||
x="95.443573"
|
|
||||||
y="32.807987" />
|
|
||||||
<text
|
|
||||||
xml:space="preserve"
|
|
||||||
style="font-style:normal;font-weight:normal;font-size:5.64444px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.264583"
|
|
||||||
x="102.11134"
|
|
||||||
y="29.894743"
|
|
||||||
id="text852-4-5"><tspan
|
|
||||||
sodipodi:role="line"
|
|
||||||
id="tspan850-3-3"
|
|
||||||
x="102.11134"
|
|
||||||
y="29.894743"
|
|
||||||
style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:5.64444px;font-family:'Liberation Mono';-inkscape-font-specification:'Liberation Mono Bold';stroke-width:0.264583">Data block</tspan></text>
|
|
||||||
<path
|
|
||||||
style="fill:#ff6600;fill-opacity:1;stroke:none;stroke-width:0.0337704;stroke-opacity:1"
|
|
||||||
d="m 102.90563,41.413279 c -0.14226,0.478709 -0.31308,0.745518 -0.65239,1.018956 -0.29019,0.233843 -0.59571,0.339489 -1.03301,0.357199 -0.33087,0.0134 -0.37381,0.0074 -0.64184,-0.09013 -0.50954,-0.185343 -0.914221,-0.594142 -1.082877,-1.093901 -0.163852,-0.485526 -0.07847,-1.108159 0.210335,-1.533803 0.111448,-0.164254 0.402172,-0.444591 0.534502,-0.515415 l 0.0681,-3.63921 0.73832,-0.64992 0.88727,0.787138 -0.40985,0.445484 0.40539,0.440982 -0.40539,0.435013 0.3816,0.41947 -0.41462,0.431781 0.57335,0.585078 0.007,0.726386 c 0.13641,0.08277 0.48753,0.413601 0.59931,0.58235 0.0577,0.0871 0.13636,0.257787 0.17481,0.379304 0.0803,0.253823 0.11239,0.738377 0.0604,0.913234 z m -1.28155,0.296888 c -0.11255,-0.405265 -0.62848,-0.483569 -0.86192,-0.130817 -0.0744,0.112498 -0.088,0.164461 -0.0745,0.2847 0.0338,0.299998 0.31326,0.504498 0.58822,0.43046 0.15271,-0.04112 0.3056,-0.184909 0.34828,-0.327542 0.0392,-0.130864 0.0392,-0.115447 -1e-4,-0.256801 z"
|
|
||||||
id="path1971-0-5"
|
|
||||||
sodipodi:nodetypes="ssscsscccccccccccssscsssscc" />
|
|
||||||
<text
|
|
||||||
xml:space="preserve"
|
|
||||||
style="font-style:normal;font-weight:normal;font-size:5.64444px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.264583"
|
|
||||||
x="104.99195"
|
|
||||||
y="41.008743"
|
|
||||||
id="text852-9-7-62"><tspan
|
|
||||||
sodipodi:role="line"
|
|
||||||
id="tspan850-4-8-9"
|
|
||||||
x="104.99195"
|
|
||||||
y="41.008743"
|
|
||||||
style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:5.64444px;font-family:'Liberation Mono';-inkscape-font-specification:'Liberation Mono Bold';stroke-width:0.264583">hash </tspan></text>
|
|
||||||
<text
|
|
||||||
xml:space="preserve"
|
|
||||||
style="font-style:normal;font-weight:normal;font-size:5.64444px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.264583"
|
|
||||||
x="104.99195"
|
|
||||||
y="49.168018"
|
|
||||||
id="text852-9-7-6-1"><tspan
|
|
||||||
sodipodi:role="line"
|
|
||||||
id="tspan850-4-8-8-2"
|
|
||||||
x="104.99195"
|
|
||||||
y="49.168018"
|
|
||||||
style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:5.64444px;font-family:'Liberation Mono';-inkscape-font-specification:'Liberation Mono Bold';stroke-width:0.264583">data</tspan></text>
|
|
||||||
</g>
|
|
||||||
<path
|
|
||||||
style="fill:none;stroke:#000000;stroke-width:1;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1;marker-end:url(#Arrow1Mend)"
|
|
||||||
d="M 42.105292,69.455903 89.563703,69.317144"
|
|
||||||
id="path954"
|
|
||||||
sodipodi:nodetypes="cc" />
|
|
||||||
<path
|
|
||||||
style="fill:none;stroke:#000000;stroke-width:1;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1;marker-end:url(#marker1262)"
|
|
||||||
d="m 134.32612,77.363197 38.12618,0.260865"
|
|
||||||
id="path1258"
|
|
||||||
sodipodi:nodetypes="cc" />
|
|
||||||
<text
|
|
||||||
xml:space="preserve"
|
|
||||||
style="font-style:normal;font-weight:normal;font-size:5.64444px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.264583"
|
|
||||||
x="8.6727352"
|
|
||||||
y="16.687063"
|
|
||||||
id="text852-3"><tspan
|
|
||||||
sodipodi:role="line"
|
|
||||||
id="tspan850-67"
|
|
||||||
x="8.6727352"
|
|
||||||
y="16.687063"
|
|
||||||
style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:5.64444px;font-family:'Liberation Mono';-inkscape-font-specification:'Liberation Mono Bold';stroke-width:0.264583">Objects table </tspan></text>
|
|
||||||
<text
|
|
||||||
xml:space="preserve"
|
|
||||||
style="font-style:normal;font-weight:normal;font-size:5.64444px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.264583"
|
|
||||||
x="89.190445"
|
|
||||||
y="16.687063"
|
|
||||||
id="text852-3-5"><tspan
|
|
||||||
sodipodi:role="line"
|
|
||||||
id="tspan850-67-3"
|
|
||||||
x="89.190445"
|
|
||||||
y="16.687063"
|
|
||||||
style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:5.64444px;font-family:'Liberation Mono';-inkscape-font-specification:'Liberation Mono Bold';stroke-width:0.264583">Versions table </tspan></text>
|
|
||||||
<text
|
|
||||||
xml:space="preserve"
|
|
||||||
style="font-style:normal;font-weight:normal;font-size:5.64444px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.264583"
|
|
||||||
x="174.55702"
|
|
||||||
y="16.687063"
|
|
||||||
id="text852-3-56"><tspan
|
|
||||||
sodipodi:role="line"
|
|
||||||
id="tspan850-67-2"
|
|
||||||
x="174.55702"
|
|
||||||
y="16.687063"
|
|
||||||
style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:5.64444px;font-family:'Liberation Mono';-inkscape-font-specification:'Liberation Mono Bold';stroke-width:0.264583">Blocks table</tspan></text>
|
|
||||||
</g>
|
|
||||||
</svg>
|
|
||||||
|
Before Width: | Height: | Size: 27 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 32 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 86 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 6.1 KiB |
Binary file not shown.
@@ -1,247 +0,0 @@
|
|||||||
%\nonstopmode
|
|
||||||
\documentclass[aspectratio=169]{beamer}
|
|
||||||
\usepackage[utf8]{inputenc}
|
|
||||||
% \usepackage[frenchb]{babel}
|
|
||||||
\usepackage{amsmath}
|
|
||||||
\usepackage{mathtools}
|
|
||||||
\usepackage{breqn}
|
|
||||||
\usepackage{multirow}
|
|
||||||
\usetheme{Luebeck}
|
|
||||||
\usepackage{graphicx}
|
|
||||||
%\useoutertheme[footline=authortitle,subsection=false]{miniframes}
|
|
||||||
|
|
||||||
\beamertemplatenavigationsymbolsempty
|
|
||||||
\setbeamertemplate{footline}
|
|
||||||
{%
|
|
||||||
\leavevmode%
|
|
||||||
\hbox{\begin{beamercolorbox}[wd=.15\paperwidth,ht=2.5ex,dp=1.125ex,leftskip=.3cm,rightskip=.3cm plus1fill]{author in head/foot}%
|
|
||||||
\usebeamerfont{author in head/foot} \insertframenumber{} / \inserttotalframenumber
|
|
||||||
\end{beamercolorbox}%
|
|
||||||
\begin{beamercolorbox}[wd=.2\paperwidth,ht=2.5ex,dp=1.125ex,leftskip=.3cm plus1fill,rightskip=.3cm]{author in head/foot}%
|
|
||||||
\usebeamerfont{author in head/foot}\insertshortauthor
|
|
||||||
\end{beamercolorbox}%
|
|
||||||
\begin{beamercolorbox}[wd=.65\paperwidth,ht=2.5ex,dp=1.125ex,leftskip=.3cm,rightskip=.3cm plus1fil]{title in head/foot}%
|
|
||||||
\usebeamerfont{title in head/foot}\insertshorttitle~--~\insertshortdate
|
|
||||||
\end{beamercolorbox}}%
|
|
||||||
\vskip0pt%
|
|
||||||
}
|
|
||||||
|
|
||||||
\usepackage{tabu}
|
|
||||||
\usepackage{multicol}
|
|
||||||
\usepackage{vwcol}
|
|
||||||
\usepackage{stmaryrd}
|
|
||||||
\usepackage{graphicx}
|
|
||||||
|
|
||||||
\usepackage[normalem]{ulem}
|
|
||||||
|
|
||||||
\title[Garage : jouer dans la cour des grands quand on est un hébergeur associatif]{Garage : jouer dans la cour des grands \\quand on est un hébergeur associatif}
|
|
||||||
\subtitle{(ou pourquoi on a décidé de réinventer la roue)}
|
|
||||||
\author[Q. Dufour \& A. Auvolat]{Quentin Dufour \& Alex Auvolat}
|
|
||||||
\date[02/12/2020]{Mercredi 2 décembre 2020}
|
|
||||||
|
|
||||||
\begin{document}
|
|
||||||
|
|
||||||
\begin{frame}
|
|
||||||
\titlepage
|
|
||||||
\end{frame}
|
|
||||||
|
|
||||||
\begin{frame}
|
|
||||||
\frametitle{La question qui tue}
|
|
||||||
|
|
||||||
\begin{center}
|
|
||||||
\includegraphics[scale=3]{img/sync.png} \\
|
|
||||||
\Huge Pourquoi vous n'hébergez pas vos fichiers chez vous ? \\
|
|
||||||
\end{center}
|
|
||||||
|
|
||||||
\end{frame}
|
|
||||||
|
|
||||||
\begin{frame}[t]
|
|
||||||
\frametitle{La cour des grands}
|
|
||||||
|
|
||||||
\begin{columns}[t]
|
|
||||||
\begin{column}{0.5\textwidth}
|
|
||||||
{\huge Le modèle du cloud...}
|
|
||||||
|
|
||||||
\begin{center}
|
|
||||||
\includegraphics[scale=0.08]{img/cloud.png}
|
|
||||||
\end{center}
|
|
||||||
|
|
||||||
+ \underline{intégrité} : plus de perte de données
|
|
||||||
|
|
||||||
+ \underline{disponibilité} : tout le temps accessible
|
|
||||||
|
|
||||||
+ \underline{service} : rien à gérer
|
|
||||||
|
|
||||||
\vspace{0.15cm}
|
|
||||||
\textbf{changement des comportements}
|
|
||||||
\end{column}
|
|
||||||
\pause
|
|
||||||
\begin{column}{0.5\textwidth}
|
|
||||||
{\huge ...et son prix}
|
|
||||||
|
|
||||||
\begin{center}
|
|
||||||
\includegraphics[scale=0.07]{img/dc.jpg}
|
|
||||||
\end{center}
|
|
||||||
|
|
||||||
- matériel couteux et polluant
|
|
||||||
|
|
||||||
- logiciels secrets
|
|
||||||
|
|
||||||
- gestion opaque
|
|
||||||
|
|
||||||
\vspace{0.2cm}
|
|
||||||
\textbf{prisonnier de l'écosystème}
|
|
||||||
\end{column}
|
|
||||||
\end{columns}
|
|
||||||
\end{frame}
|
|
||||||
|
|
||||||
\begin{frame}[t]
|
|
||||||
\frametitle{Garage l'imposteur}
|
|
||||||
|
|
||||||
\begin{columns}[t]
|
|
||||||
\begin{column}{0.5\textwidth}
|
|
||||||
{\huge Ressemble à du cloud...}
|
|
||||||
|
|
||||||
\begin{center}
|
|
||||||
\includegraphics[scale=0.5]{img/shh.jpg}
|
|
||||||
\end{center}
|
|
||||||
|
|
||||||
+ \underline{compatible} avec les apps existantes
|
|
||||||
|
|
||||||
+ \underline{fonctionne} avec le mobile
|
|
||||||
|
|
||||||
+ \underline{s'adapte} aux habitudes prises
|
|
||||||
|
|
||||||
|
|
||||||
\end{column}
|
|
||||||
|
|
||||||
\pause
|
|
||||||
\begin{column}{0.5\textwidth}
|
|
||||||
{\huge ...fait du P2P}
|
|
||||||
|
|
||||||
\begin{center}
|
|
||||||
\includegraphics[scale=1]{img/death.jpg}
|
|
||||||
\end{center}
|
|
||||||
|
|
||||||
\vspace{0.4cm}
|
|
||||||
|
|
||||||
+ \underline{contrôle} de l'infrastructure
|
|
||||||
|
|
||||||
+ \underline{transparent} code libre
|
|
||||||
|
|
||||||
+ \underline{sobre} fonctionne avec de vieilles machines à la maison
|
|
||||||
\end{column}
|
|
||||||
\end{columns}
|
|
||||||
|
|
||||||
\end{frame}
|
|
||||||
|
|
||||||
|
|
||||||
\graphicspath{{img/}}
|
|
||||||
|
|
||||||
\begin{frame}
|
|
||||||
\frametitle{Mais donc, c'est quoi Garage ?}
|
|
||||||
|
|
||||||
\begin{columns}[t]
|
|
||||||
\begin{column}{0.5\textwidth}
|
|
||||||
\centering
|
|
||||||
\textbf{Un système de stockage distribué}
|
|
||||||
\vspace{1em}
|
|
||||||
|
|
||||||
\includegraphics[width=.7\columnwidth]{img/garage_distributed.pdf}
|
|
||||||
\end{column}
|
|
||||||
\pause
|
|
||||||
|
|
||||||
\begin{column}{0.5\textwidth}
|
|
||||||
\centering
|
|
||||||
\textbf{qui implémente l'API S3}
|
|
||||||
\vspace{2em}
|
|
||||||
|
|
||||||
\includegraphics[width=.7\columnwidth]{img/Amazon-S3.jpg}
|
|
||||||
\end{column}
|
|
||||||
\end{columns}
|
|
||||||
\end{frame}
|
|
||||||
|
|
||||||
\begin{frame}
|
|
||||||
\frametitle{Consistent Hashing (DynamoDB)}
|
|
||||||
\textbf{Comment répartir les fichiers sur les différentes machines ?}
|
|
||||||
\vspace{1em}
|
|
||||||
|
|
||||||
\centering
|
|
||||||
|
|
||||||
\only<1>{\includegraphics[width=.55\columnwidth]{img/consistent_hashing_1.pdf}}%
|
|
||||||
\only<2>{\includegraphics[width=.55\columnwidth]{img/consistent_hashing_2.pdf}}%
|
|
||||||
\only<3>{\includegraphics[width=.55\columnwidth]{img/consistent_hashing_3.pdf}}%
|
|
||||||
\only<4>{\includegraphics[width=.55\columnwidth]{img/consistent_hashing_4.pdf}}%
|
|
||||||
\end{frame}
|
|
||||||
|
|
||||||
\begin{frame}
|
|
||||||
\frametitle{Garage Internals : 3 niveaux de consistent hashing}
|
|
||||||
\centering
|
|
||||||
\includegraphics[width=.85\columnwidth]{img/garage_tables.pdf}
|
|
||||||
\end{frame}
|
|
||||||
|
|
||||||
\begin{frame}
|
|
||||||
\frametitle{Modèles de cohérence}
|
|
||||||
Garage utilise un modèle de cohérence relativement faible :
|
|
||||||
\vspace{1em}
|
|
||||||
|
|
||||||
\begin{itemize}
|
|
||||||
\item Objets répliqués 3 fois, quorum de 2 pour les lectures et les écritures\\
|
|
||||||
$\to$ cohérence \textbf{``read your writes''}
|
|
||||||
\vspace{1em}
|
|
||||||
\item<2-> Types de donnée CRDT + mécanisme d'anti-entropie\\
|
|
||||||
$\to$ cohérence \textbf{à terme} (eventual consistency)
|
|
||||||
\vspace{1em}
|
|
||||||
\item<3-> Cela s'applique pour chaque fichier individuellement :\\
|
|
||||||
pas de linéarisabilté ou de cohérence causale entre les opérations\\
|
|
||||||
sur des fichiers différents
|
|
||||||
\vspace{1em}
|
|
||||||
\item<4-> \textbf{Avantage :} convient bien à un déploiement géodistribué (multi-datacenter)
|
|
||||||
\end{itemize}
|
|
||||||
\end{frame}
|
|
||||||
|
|
||||||
\begin{frame}
|
|
||||||
\frametitle{Rust : retour d'expérience}
|
|
||||||
|
|
||||||
\begin{columns}
|
|
||||||
\begin{column}{0.55\textwidth}
|
|
||||||
Garage est entièrement écrit en Rust !
|
|
||||||
\vspace{2em}
|
|
||||||
|
|
||||||
\textbf{Points forts :}
|
|
||||||
\vspace{.5em}
|
|
||||||
\begin{itemize}
|
|
||||||
\item Langage compilé, très rapide
|
|
||||||
\vspace{.5em}
|
|
||||||
\item Typage fort, beaucoup de sécurités
|
|
||||||
\vspace{.5em}
|
|
||||||
\item Le meilleur de plusieurs paradigmes:
|
|
||||||
fonctionnel, orienté objet, impératif
|
|
||||||
\vspace{.5em}
|
|
||||||
\item Un écosytème de librairies très complet:
|
|
||||||
serialisation, async/await, http, ...
|
|
||||||
\end{itemize}
|
|
||||||
\end{column}
|
|
||||||
|
|
||||||
\begin{column}{0.45\textwidth}
|
|
||||||
\begin{centering}
|
|
||||||
\hspace{2em}\includegraphics[width=0.55\columnwidth]{img/rustacean-flat-happy.png}
|
|
||||||
\end{centering}
|
|
||||||
|
|
||||||
\vspace{2em}
|
|
||||||
\textbf{Points faibles :}
|
|
||||||
\vspace{.5em}
|
|
||||||
\begin{itemize}
|
|
||||||
\item Les temps de compilation...
|
|
||||||
\vspace{.5em}
|
|
||||||
\item Compliqué à apprendre
|
|
||||||
\end{itemize}
|
|
||||||
\vspace{2em}
|
|
||||||
\end{column}
|
|
||||||
\end{columns}
|
|
||||||
|
|
||||||
\end{frame}
|
|
||||||
|
|
||||||
\end{document}
|
|
||||||
|
|
||||||
%% vim: set ts=4 sw=4 tw=0 noet spelllang=fr :
|
|
||||||
BIN
Binary file not shown.
|
Before Width: | Height: | Size: 15 KiB |
-119
@@ -1,119 +0,0 @@
|
|||||||
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
|
|
||||||
<svg
|
|
||||||
xmlns:dc="http://purl.org/dc/elements/1.1/"
|
|
||||||
xmlns:cc="http://creativecommons.org/ns#"
|
|
||||||
xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
|
|
||||||
xmlns:svg="http://www.w3.org/2000/svg"
|
|
||||||
xmlns="http://www.w3.org/2000/svg"
|
|
||||||
xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
|
|
||||||
xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
|
|
||||||
width="108.2099mm"
|
|
||||||
height="108.00987mm"
|
|
||||||
viewBox="0 0 108.2099 108.00987"
|
|
||||||
version="1.1"
|
|
||||||
id="svg8"
|
|
||||||
inkscape:version="1.0.1 (3bc2e813f5, 2020-09-07)"
|
|
||||||
sodipodi:docname="garage.svg"
|
|
||||||
inkscape:export-filename="/home/lx/garage.png"
|
|
||||||
inkscape:export-xdpi="96"
|
|
||||||
inkscape:export-ydpi="96">
|
|
||||||
<defs
|
|
||||||
id="defs2" />
|
|
||||||
<sodipodi:namedview
|
|
||||||
id="base"
|
|
||||||
pagecolor="#ffffff"
|
|
||||||
bordercolor="#666666"
|
|
||||||
borderopacity="1.0"
|
|
||||||
inkscape:pageopacity="1"
|
|
||||||
inkscape:pageshadow="2"
|
|
||||||
inkscape:zoom="0.5"
|
|
||||||
inkscape:cx="-212.52783"
|
|
||||||
inkscape:cy="204.9547"
|
|
||||||
inkscape:document-units="mm"
|
|
||||||
inkscape:current-layer="layer1"
|
|
||||||
inkscape:document-rotation="0"
|
|
||||||
showgrid="false"
|
|
||||||
fit-margin-top="20"
|
|
||||||
fit-margin-left="20"
|
|
||||||
fit-margin-right="20"
|
|
||||||
fit-margin-bottom="20"
|
|
||||||
inkscape:window-width="1404"
|
|
||||||
inkscape:window-height="1016"
|
|
||||||
inkscape:window-x="103"
|
|
||||||
inkscape:window-y="27"
|
|
||||||
inkscape:window-maximized="0" />
|
|
||||||
<metadata
|
|
||||||
id="metadata5">
|
|
||||||
<rdf:RDF>
|
|
||||||
<cc:Work
|
|
||||||
rdf:about="">
|
|
||||||
<dc:format>image/svg+xml</dc:format>
|
|
||||||
<dc:type
|
|
||||||
rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
|
|
||||||
<dc:title></dc:title>
|
|
||||||
</cc:Work>
|
|
||||||
</rdf:RDF>
|
|
||||||
</metadata>
|
|
||||||
<g
|
|
||||||
inkscape:label="Layer 1"
|
|
||||||
inkscape:groupmode="layer"
|
|
||||||
id="layer1"
|
|
||||||
transform="translate(-45.667412,-33.028536)">
|
|
||||||
<path
|
|
||||||
style="fill:none;stroke:#000000;stroke-width:2.065;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
|
|
||||||
d="M 66.78016,73.340623 99.921832,54.219898 132.84481,73.130965 V 120.00591 H 66.701651 Z"
|
|
||||||
id="path124"
|
|
||||||
sodipodi:nodetypes="cccccc" />
|
|
||||||
<g
|
|
||||||
id="g1106-5"
|
|
||||||
transform="matrix(0,0.95201267,-0.95201267,0,194.01664,-65.058377)"
|
|
||||||
style="stroke-width:2.17959;stroke-miterlimit:4;stroke-dasharray:none">
|
|
||||||
<g
|
|
||||||
id="g1061-3"
|
|
||||||
style="stroke-width:2.17959;stroke-miterlimit:4;stroke-dasharray:none">
|
|
||||||
<circle
|
|
||||||
style="fill:none;stroke:#000000;stroke-width:2.17959;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
|
|
||||||
id="path956-5"
|
|
||||||
cx="168.8569"
|
|
||||||
cy="92.889587"
|
|
||||||
r="13.125794" />
|
|
||||||
<circle
|
|
||||||
style="fill:none;stroke:#000000;stroke-width:2.17959;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
|
|
||||||
id="path958-6"
|
|
||||||
cx="168.77444"
|
|
||||||
cy="92.702293"
|
|
||||||
r="3.0778286" />
|
|
||||||
<path
|
|
||||||
id="path960-2"
|
|
||||||
style="fill:none;stroke:#000000;stroke-width:2.17959;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
|
|
||||||
d="m 169.46072,82.84435 c 4.95795,0.336608 8.87296,4.341959 9.09638,9.306301"
|
|
||||||
sodipodi:nodetypes="cc" />
|
|
||||||
</g>
|
|
||||||
<path
|
|
||||||
style="fill:none;stroke:#000000;stroke-width:2.17959;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
|
|
||||||
d="m 154.67824,112.84018 11.89881,-13.038071 c 1.46407,-1.552664 3.79541,0.878511 2.81832,2.089181 l -10.57965,14.481 c -1.8851,2.02632 -6.10786,-1.06119 -4.13748,-3.53211 z"
|
|
||||||
id="path964-9"
|
|
||||||
sodipodi:nodetypes="ccccc" />
|
|
||||||
<g
|
|
||||||
id="g1071-1"
|
|
||||||
style="stroke-width:2.17959;stroke-miterlimit:4;stroke-dasharray:none" />
|
|
||||||
<g
|
|
||||||
id="g1065-3"
|
|
||||||
style="stroke-width:2.17959;stroke-miterlimit:4;stroke-dasharray:none">
|
|
||||||
<rect
|
|
||||||
style="fill:none;stroke:#000000;stroke-width:2.17959;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
|
|
||||||
id="rect949-6"
|
|
||||||
width="35.576611"
|
|
||||||
height="48.507355"
|
|
||||||
x="150.9623"
|
|
||||||
y="74.698929"
|
|
||||||
ry="2.7302756" />
|
|
||||||
<path
|
|
||||||
style="fill:none;stroke:#000000;stroke-width:2.17959;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
|
|
||||||
d="m 150.76919,106.16944 6.36181,-0.0223 c 2.53845,3.46232 6.29787,4.20243 10.1055,4.40362 l 0.0176,13.09251"
|
|
||||||
id="path1033-0"
|
|
||||||
sodipodi:nodetypes="cccc" />
|
|
||||||
</g>
|
|
||||||
</g>
|
|
||||||
</g>
|
|
||||||
</svg>
|
|
||||||
|
Before Width: | Height: | Size: 4.5 KiB |
@@ -1,16 +0,0 @@
|
|||||||
#!/bin/bash
|
|
||||||
|
|
||||||
SCRIPT_FOLDER="`dirname \"$0\"`"
|
|
||||||
REPO_FOLDER="${SCRIPT_FOLDER}/../"
|
|
||||||
GARAGE_DEBUG="${REPO_FOLDER}/target/debug/"
|
|
||||||
GARAGE_RELEASE="${REPO_FOLDER}/target/release/"
|
|
||||||
PATH="${GARAGE_DEBUG}:${GARAGE_RELEASE}:$PATH"
|
|
||||||
|
|
||||||
garage bucket create eprouvette
|
|
||||||
KEY_INFO=`garage key new --name opérateur`
|
|
||||||
ACCESS_KEY=`echo $KEY_INFO|grep -Po 'GK[a-f0-9]+'`
|
|
||||||
SECRET_KEY=`echo $KEY_INFO|grep -Po 'secret_key: "[a-f0-9]+'|grep -Po '[a-f0-9]+$'`
|
|
||||||
garage bucket allow eprouvette --read --write --key $ACCESS_KEY
|
|
||||||
echo "$ACCESS_KEY $SECRET_KEY" > /tmp/garage.s3
|
|
||||||
|
|
||||||
echo "Bucket s3://eprouvette created. Credentials stored in /tmp/garage.s3."
|
|
||||||
@@ -1,7 +0,0 @@
|
|||||||
#!/bin/bash
|
|
||||||
|
|
||||||
set -ex
|
|
||||||
|
|
||||||
killall -9 garage || echo "garage is not running"
|
|
||||||
rm -rf /tmp/garage*
|
|
||||||
rm -rf /tmp/config.*.toml
|
|
||||||
@@ -1,59 +0,0 @@
|
|||||||
#!/bin/bash
|
|
||||||
|
|
||||||
set -e
|
|
||||||
|
|
||||||
SCRIPT_FOLDER="`dirname \"$0\"`"
|
|
||||||
REPO_FOLDER="${SCRIPT_FOLDER}/../"
|
|
||||||
GARAGE_DEBUG="${REPO_FOLDER}/target/debug/"
|
|
||||||
GARAGE_RELEASE="${REPO_FOLDER}/target/release/"
|
|
||||||
PATH="${GARAGE_DEBUG}:${GARAGE_RELEASE}:$PATH"
|
|
||||||
FANCYCOLORS=("41m" "42m" "44m" "45m" "100m" "104m")
|
|
||||||
|
|
||||||
export RUST_BACKTRACE=1
|
|
||||||
export RUST_LOG=garage=info,garage_api=debug
|
|
||||||
MAIN_LABEL="\e[${FANCYCOLORS[0]}[main]\e[49m"
|
|
||||||
|
|
||||||
WHICH_GARAGE=$(which garage || exit 1)
|
|
||||||
echo -en "${MAIN_LABEL} Found garage at: ${WHICH_GARAGE}\n"
|
|
||||||
|
|
||||||
for count in $(seq 1 3); do
|
|
||||||
CONF_PATH="/tmp/config.$count.toml"
|
|
||||||
LABEL="\e[${FANCYCOLORS[$count]}[$count]\e[49m"
|
|
||||||
|
|
||||||
cat > $CONF_PATH <<EOF
|
|
||||||
block_size = 1048576 # objects are split in blocks of maximum this number of bytes
|
|
||||||
metadata_dir = "/tmp/garage-meta-$count"
|
|
||||||
data_dir = "/tmp/garage-data-$count"
|
|
||||||
rpc_bind_addr = "[::]:$((3900+$count))" # the port other Garage nodes will use to talk to this node
|
|
||||||
bootstrap_peers = [
|
|
||||||
"[::1]:3901",
|
|
||||||
"[::1]:3902",
|
|
||||||
"[::1]:3903"
|
|
||||||
]
|
|
||||||
max_concurrent_rpc_requests = 12
|
|
||||||
data_replication_factor = 3
|
|
||||||
meta_replication_factor = 3
|
|
||||||
meta_epidemic_fanout = 3
|
|
||||||
|
|
||||||
[s3_api]
|
|
||||||
api_bind_addr = "[::]:$((3910+$count))" # the S3 API port, HTTP without TLS. Add a reverse proxy for the TLS part.
|
|
||||||
s3_region = "garage" # set this to anything. S3 API calls will fail if they are not made against the region set here.
|
|
||||||
|
|
||||||
[s3_web]
|
|
||||||
bind_addr = "[::]:$((3920+$count))"
|
|
||||||
root_domain = ".garage.tld"
|
|
||||||
index = "index.html"
|
|
||||||
EOF
|
|
||||||
|
|
||||||
echo -en "$LABEL configuration written to $CONF_PATH\n"
|
|
||||||
|
|
||||||
(garage server -c /tmp/config.$count.toml 2>&1|while read r; do echo -en "$LABEL $r\n"; done) &
|
|
||||||
done
|
|
||||||
|
|
||||||
until garage status 2>&1|grep -q Healthy ; do
|
|
||||||
echo -en "${MAIN_LABEL} cluster starting...\n"
|
|
||||||
sleep 1
|
|
||||||
done
|
|
||||||
echo -en "${MAIN_LABEL} cluster started\n"
|
|
||||||
|
|
||||||
wait
|
|
||||||
@@ -1,20 +0,0 @@
|
|||||||
#!/bin/bash
|
|
||||||
|
|
||||||
SCRIPT_FOLDER="`dirname \"$0\"`"
|
|
||||||
REPO_FOLDER="${SCRIPT_FOLDER}/../"
|
|
||||||
GARAGE_DEBUG="${REPO_FOLDER}/target/debug/"
|
|
||||||
GARAGE_RELEASE="${REPO_FOLDER}/target/release/"
|
|
||||||
PATH="${GARAGE_DEBUG}:${GARAGE_RELEASE}:$PATH"
|
|
||||||
|
|
||||||
until garage status 2>&1|grep -q Healthy ; do
|
|
||||||
echo "cluster starting..."
|
|
||||||
sleep 1
|
|
||||||
done
|
|
||||||
|
|
||||||
garage status \
|
|
||||||
| grep UNCONFIGURED \
|
|
||||||
| grep -Po '^[0-9a-f]+' \
|
|
||||||
| while read id; do
|
|
||||||
garage node configure -d dc1 -n 100 $id
|
|
||||||
done
|
|
||||||
|
|
||||||
@@ -1,14 +0,0 @@
|
|||||||
#!/bin/bash
|
|
||||||
|
|
||||||
SCRIPT_FOLDER="`dirname \"${BASH_SOURCE[0]}\"`"
|
|
||||||
REPO_FOLDER="${SCRIPT_FOLDER}/../"
|
|
||||||
GARAGE_DEBUG="${REPO_FOLDER}/target/debug/"
|
|
||||||
GARAGE_RELEASE="${REPO_FOLDER}/target/release/"
|
|
||||||
PATH="${GARAGE_DEBUG}:${GARAGE_RELEASE}:$PATH"
|
|
||||||
|
|
||||||
export AWS_ACCESS_KEY_ID=`cat /tmp/garage.s3 |cut -d' ' -f1`
|
|
||||||
export AWS_SECRET_ACCESS_KEY=`cat /tmp/garage.s3 |cut -d' ' -f2`
|
|
||||||
export AWS_DEFAULT_REGION='garage'
|
|
||||||
|
|
||||||
alias awsgrg="aws s3 \
|
|
||||||
--endpoint-url http://127.0.0.1:3911"
|
|
||||||
@@ -1,19 +0,0 @@
|
|||||||
#!/bin/bash
|
|
||||||
|
|
||||||
SCRIPT_FOLDER="`dirname \"${BASH_SOURCE[0]}\"`"
|
|
||||||
REPO_FOLDER="${SCRIPT_FOLDER}/../"
|
|
||||||
GARAGE_DEBUG="${REPO_FOLDER}/target/debug/"
|
|
||||||
GARAGE_RELEASE="${REPO_FOLDER}/target/release/"
|
|
||||||
PATH="${GARAGE_DEBUG}:${GARAGE_RELEASE}:$PATH"
|
|
||||||
|
|
||||||
ACCESS_KEY=`cat /tmp/garage.s3 |cut -d' ' -f1`
|
|
||||||
SECRET_KEY=`cat /tmp/garage.s3 |cut -d' ' -f2`
|
|
||||||
|
|
||||||
alias s3grg="s3cmd \
|
|
||||||
--host 127.0.0.1:3911 \
|
|
||||||
--host-bucket 127.0.0.1:3911 \
|
|
||||||
--access_key=$ACCESS_KEY \
|
|
||||||
--secret_key=$SECRET_KEY \
|
|
||||||
--region=garage \
|
|
||||||
--no-ssl"
|
|
||||||
|
|
||||||
@@ -1,76 +0,0 @@
|
|||||||
#!/bin/bash
|
|
||||||
|
|
||||||
set -ex
|
|
||||||
shopt -s expand_aliases
|
|
||||||
|
|
||||||
SCRIPT_FOLDER="`dirname \"$0\"`"
|
|
||||||
REPO_FOLDER="${SCRIPT_FOLDER}/../"
|
|
||||||
|
|
||||||
echo "setup"
|
|
||||||
cargo build
|
|
||||||
${SCRIPT_FOLDER}/dev-clean.sh
|
|
||||||
${SCRIPT_FOLDER}/dev-cluster.sh > /tmp/garage.log 2>&1 &
|
|
||||||
${SCRIPT_FOLDER}/dev-configure.sh
|
|
||||||
${SCRIPT_FOLDER}/dev-bucket.sh
|
|
||||||
source ${SCRIPT_FOLDER}/dev-env-aws.sh
|
|
||||||
source ${SCRIPT_FOLDER}/dev-env-s3cmd.sh
|
|
||||||
|
|
||||||
garage status
|
|
||||||
garage key list
|
|
||||||
garage bucket list
|
|
||||||
|
|
||||||
dd if=/dev/urandom of=/tmp/garage.1.rnd bs=1k count=2 # < INLINE_THRESHOLD = 3072 bytes
|
|
||||||
dd if=/dev/urandom of=/tmp/garage.2.rnd bs=1M count=5
|
|
||||||
dd if=/dev/urandom of=/tmp/garage.3.rnd bs=1M count=10
|
|
||||||
|
|
||||||
echo "s3 api testing..."
|
|
||||||
for idx in $(seq 1 3); do
|
|
||||||
# AWS sends
|
|
||||||
awsgrg cp /tmp/garage.$idx.rnd s3://eprouvette/garage.$idx.aws
|
|
||||||
|
|
||||||
awsgrg ls s3://eprouvette
|
|
||||||
|
|
||||||
awsgrg cp s3://eprouvette/garage.$idx.aws /tmp/garage.$idx.dl
|
|
||||||
diff /tmp/garage.$idx.rnd /tmp/garage.$idx.dl
|
|
||||||
rm /tmp/garage.$idx.dl
|
|
||||||
|
|
||||||
s3grg get s3://eprouvette/garage.$idx.aws /tmp/garage.$idx.dl
|
|
||||||
diff /tmp/garage.$idx.rnd /tmp/garage.$idx.dl
|
|
||||||
rm /tmp/garage.$idx.dl
|
|
||||||
|
|
||||||
awsgrg rm s3://eprouvette/garage.$idx.aws
|
|
||||||
|
|
||||||
# S3CMD sends
|
|
||||||
s3grg put /tmp/garage.$idx.rnd s3://eprouvette/garage.$idx.s3cmd
|
|
||||||
|
|
||||||
s3grg ls s3://eprouvette
|
|
||||||
|
|
||||||
s3grg get s3://eprouvette/garage.$idx.s3cmd /tmp/garage.$idx.dl
|
|
||||||
diff /tmp/garage.$idx.rnd /tmp/garage.$idx.dl
|
|
||||||
rm /tmp/garage.$idx.dl
|
|
||||||
|
|
||||||
awsgrg cp s3://eprouvette/garage.$idx.s3cmd /tmp/garage.$idx.dl
|
|
||||||
diff /tmp/garage.$idx.rnd /tmp/garage.$idx.dl
|
|
||||||
rm /tmp/garage.$idx.dl
|
|
||||||
|
|
||||||
s3grg rm s3://eprouvette/garage.$idx.s3cmd
|
|
||||||
done
|
|
||||||
rm /tmp/garage.{1,2,3}.rnd
|
|
||||||
|
|
||||||
echo "website testing"
|
|
||||||
echo "<h1>hello world</h1>" > /tmp/garage-index.html
|
|
||||||
awsgrg cp /tmp/garage-index.html s3://eprouvette/index.html
|
|
||||||
[ `curl -s -o /dev/null -w "%{http_code}" --header "Host: eprouvette.garage.tld" http://127.0.0.1:3923/ ` == 404 ]
|
|
||||||
garage bucket website --allow eprouvette
|
|
||||||
[ `curl -s -o /dev/null -w "%{http_code}" --header "Host: eprouvette.garage.tld" http://127.0.0.1:3923/ ` == 200 ]
|
|
||||||
garage bucket website --deny eprouvette
|
|
||||||
[ `curl -s -o /dev/null -w "%{http_code}" --header "Host: eprouvette.garage.tld" http://127.0.0.1:3923/ ` == 404 ]
|
|
||||||
awsgrg rm s3://eprouvette/index.html
|
|
||||||
rm /tmp/garage-index.html
|
|
||||||
|
|
||||||
echo "teardown"
|
|
||||||
garage bucket deny --read --write eprouvette --key $AWS_ACCESS_KEY_ID
|
|
||||||
garage bucket delete --yes eprouvette
|
|
||||||
garage key delete --yes $AWS_ACCESS_KEY_ID
|
|
||||||
|
|
||||||
echo "success"
|
|
||||||
+5
-9
@@ -1,6 +1,6 @@
|
|||||||
[package]
|
[package]
|
||||||
name = "garage_api"
|
name = "garage_api"
|
||||||
version = "0.1.1"
|
version = "0.1.0"
|
||||||
authors = ["Alex Auvolat <alex@adnab.me>"]
|
authors = ["Alex Auvolat <alex@adnab.me>"]
|
||||||
edition = "2018"
|
edition = "2018"
|
||||||
license = "GPL-3.0"
|
license = "GPL-3.0"
|
||||||
@@ -13,28 +13,24 @@ path = "lib.rs"
|
|||||||
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
|
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
|
||||||
|
|
||||||
[dependencies]
|
[dependencies]
|
||||||
garage_util = { version = "0.1.1", path = "../util" }
|
garage_util = { version = "0.1", path = "../util" }
|
||||||
garage_table = { version = "0.1.1", path = "../table" }
|
garage_table = { version = "0.1", path = "../table" }
|
||||||
garage_model = { version = "0.1.1", path = "../model" }
|
garage_model = { version = "0.1", path = "../model" }
|
||||||
|
|
||||||
err-derive = "0.2.3"
|
|
||||||
bytes = "0.4"
|
bytes = "0.4"
|
||||||
hex = "0.3"
|
hex = "0.3"
|
||||||
base64 = "0.13"
|
|
||||||
log = "0.4"
|
log = "0.4"
|
||||||
chrono = "0.4"
|
chrono = "0.4"
|
||||||
md-5 = "0.9.1"
|
|
||||||
sha2 = "0.8"
|
sha2 = "0.8"
|
||||||
hmac = "0.7"
|
hmac = "0.7"
|
||||||
crypto-mac = "0.7"
|
crypto-mac = "0.7"
|
||||||
rand = "0.7"
|
|
||||||
|
|
||||||
futures = "0.3"
|
futures = "0.3"
|
||||||
futures-util = "0.3"
|
futures-util = "0.3"
|
||||||
tokio = { version = "0.2", default-features = false, features = ["rt-core", "rt-threaded", "io-driver", "net", "tcp", "time", "macros", "sync", "signal", "fs"] }
|
tokio = { version = "0.2", default-features = false, features = ["rt-core", "rt-threaded", "io-driver", "net", "tcp", "time", "macros", "sync", "signal", "fs"] }
|
||||||
|
|
||||||
http = "0.2"
|
http = "0.2"
|
||||||
hyper = "^0.13.6"
|
hyper = "0.13"
|
||||||
url = "2.1"
|
url = "2.1"
|
||||||
httpdate = "0.3"
|
httpdate = "0.3"
|
||||||
percent-encoding = "2.1.0"
|
percent-encoding = "2.1.0"
|
||||||
|
|||||||
+33
-72
@@ -7,11 +7,11 @@ use hyper::server::conn::AddrStream;
|
|||||||
use hyper::service::{make_service_fn, service_fn};
|
use hyper::service::{make_service_fn, service_fn};
|
||||||
use hyper::{Body, Method, Request, Response, Server};
|
use hyper::{Body, Method, Request, Response, Server};
|
||||||
|
|
||||||
use garage_util::error::Error as GarageError;
|
use garage_util::error::Error;
|
||||||
|
|
||||||
use garage_model::garage::Garage;
|
use garage_model::garage::Garage;
|
||||||
|
|
||||||
use crate::error::*;
|
use crate::http_util::*;
|
||||||
use crate::signature::check_signature;
|
use crate::signature::check_signature;
|
||||||
|
|
||||||
use crate::s3_copy::*;
|
use crate::s3_copy::*;
|
||||||
@@ -23,14 +23,14 @@ use crate::s3_put::*;
|
|||||||
pub async fn run_api_server(
|
pub async fn run_api_server(
|
||||||
garage: Arc<Garage>,
|
garage: Arc<Garage>,
|
||||||
shutdown_signal: impl Future<Output = ()>,
|
shutdown_signal: impl Future<Output = ()>,
|
||||||
) -> Result<(), GarageError> {
|
) -> Result<(), Error> {
|
||||||
let addr = &garage.config.s3_api.api_bind_addr;
|
let addr = &garage.config.s3_api.api_bind_addr;
|
||||||
|
|
||||||
let service = make_service_fn(|conn: &AddrStream| {
|
let service = make_service_fn(|conn: &AddrStream| {
|
||||||
let garage = garage.clone();
|
let garage = garage.clone();
|
||||||
let client_addr = conn.remote_addr();
|
let client_addr = conn.remote_addr();
|
||||||
async move {
|
async move {
|
||||||
Ok::<_, GarageError>(service_fn(move |req: Request<Body>| {
|
Ok::<_, Error>(service_fn(move |req: Request<Body>| {
|
||||||
let garage = garage.clone();
|
let garage = garage.clone();
|
||||||
handler(garage, req, client_addr)
|
handler(garage, req, client_addr)
|
||||||
}))
|
}))
|
||||||
@@ -50,7 +50,7 @@ async fn handler(
|
|||||||
garage: Arc<Garage>,
|
garage: Arc<Garage>,
|
||||||
req: Request<Body>,
|
req: Request<Body>,
|
||||||
addr: SocketAddr,
|
addr: SocketAddr,
|
||||||
) -> Result<Response<Body>, GarageError> {
|
) -> Result<Response<BodyType>, Error> {
|
||||||
info!("{} {} {}", addr, req.method(), req.uri());
|
info!("{} {} {}", addr, req.method(), req.uri());
|
||||||
debug!("{:?}", req);
|
debug!("{:?}", req);
|
||||||
match handler_inner(garage, req).await {
|
match handler_inner(garage, req).await {
|
||||||
@@ -59,7 +59,7 @@ async fn handler(
|
|||||||
Ok(x)
|
Ok(x)
|
||||||
}
|
}
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
let body: Body = Body::from(format!("{}\n", e));
|
let body: BodyType = Box::new(BytesBody::from(format!("{}\n", e)));
|
||||||
let mut http_error = Response::new(body);
|
let mut http_error = Response::new(body);
|
||||||
*http_error.status_mut() = e.http_status_code();
|
*http_error.status_mut() = e.http_status_code();
|
||||||
warn!("Response: error {}, {}", e.http_status_code(), e);
|
warn!("Response: error {}, {}", e.http_status_code(), e);
|
||||||
@@ -68,13 +68,21 @@ async fn handler(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn handler_inner(garage: Arc<Garage>, req: Request<Body>) -> Result<Response<Body>, Error> {
|
async fn handler_inner(
|
||||||
|
garage: Arc<Garage>,
|
||||||
|
req: Request<Body>,
|
||||||
|
) -> Result<Response<BodyType>, Error> {
|
||||||
let path = req.uri().path().to_string();
|
let path = req.uri().path().to_string();
|
||||||
let path = percent_encoding::percent_decode_str(&path).decode_utf8()?;
|
let path = percent_encoding::percent_decode_str(&path).decode_utf8()?;
|
||||||
|
|
||||||
let (bucket, key) = parse_bucket_key(&path)?;
|
let (bucket, key) = parse_bucket_key(&path)?;
|
||||||
|
if bucket.len() == 0 {
|
||||||
|
return Err(Error::Forbidden(format!(
|
||||||
|
"Operations on buckets not allowed"
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
|
||||||
let (api_key, content_sha256) = check_signature(&garage, &req).await?;
|
let api_key = check_signature(&garage, &req).await?;
|
||||||
let allowed = match req.method() {
|
let allowed = match req.method() {
|
||||||
&Method::HEAD | &Method::GET => api_key.allow_read(&bucket),
|
&Method::HEAD | &Method::GET => api_key.allow_read(&bucket),
|
||||||
_ => api_key.allow_write(&bucket),
|
_ => api_key.allow_write(&bucket),
|
||||||
@@ -110,16 +118,7 @@ async fn handler_inner(garage: Arc<Garage>, req: Request<Body>) -> Result<Respon
|
|||||||
// UploadPart query
|
// UploadPart query
|
||||||
let part_number = params.get("partnumber").unwrap();
|
let part_number = params.get("partnumber").unwrap();
|
||||||
let upload_id = params.get("uploadid").unwrap();
|
let upload_id = params.get("uploadid").unwrap();
|
||||||
Ok(handle_put_part(
|
Ok(handle_put_part(garage, req, &bucket, &key, part_number, upload_id).await?)
|
||||||
garage,
|
|
||||||
req,
|
|
||||||
&bucket,
|
|
||||||
&key,
|
|
||||||
part_number,
|
|
||||||
upload_id,
|
|
||||||
content_sha256,
|
|
||||||
)
|
|
||||||
.await?)
|
|
||||||
} else if req.headers().contains_key("x-amz-copy-source") {
|
} else if req.headers().contains_key("x-amz-copy-source") {
|
||||||
// CopyObject query
|
// CopyObject query
|
||||||
let copy_source = req.headers().get("x-amz-copy-source").unwrap().to_str()?;
|
let copy_source = req.headers().get("x-amz-copy-source").unwrap().to_str()?;
|
||||||
@@ -132,11 +131,14 @@ async fn handler_inner(garage: Arc<Garage>, req: Request<Body>) -> Result<Respon
|
|||||||
source_bucket
|
source_bucket
|
||||||
)));
|
)));
|
||||||
}
|
}
|
||||||
let source_key = source_key.ok_or_bad_request("No source key specified")?;
|
let source_key = match source_key {
|
||||||
|
None => return Err(Error::BadRequest(format!("No source key specified"))),
|
||||||
|
Some(x) => x,
|
||||||
|
};
|
||||||
Ok(handle_copy(garage, &bucket, &key, &source_bucket, &source_key).await?)
|
Ok(handle_copy(garage, &bucket, &key, &source_bucket, &source_key).await?)
|
||||||
} else {
|
} else {
|
||||||
// PutObject query
|
// PutObject query
|
||||||
Ok(handle_put(garage, req, &bucket, &key, content_sha256).await?)
|
Ok(handle_put(garage, req, &bucket, &key).await?)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
&Method::DELETE => {
|
&Method::DELETE => {
|
||||||
@@ -173,12 +175,12 @@ async fn handler_inner(garage: Arc<Garage>, req: Request<Body>) -> Result<Respon
|
|||||||
&Method::PUT => {
|
&Method::PUT => {
|
||||||
// CreateBucket
|
// CreateBucket
|
||||||
// If we're here, the bucket already exists, so just answer ok
|
// If we're here, the bucket already exists, so just answer ok
|
||||||
debug!(
|
println!(
|
||||||
"Body: {}",
|
"Body: {}",
|
||||||
std::str::from_utf8(&hyper::body::to_bytes(req.into_body()).await?)
|
std::str::from_utf8(&hyper::body::to_bytes(req.into_body()).await?)
|
||||||
.unwrap_or("<invalid utf8>")
|
.unwrap_or("<invalid utf8>")
|
||||||
);
|
);
|
||||||
let empty_body: Body = Body::from(vec![]);
|
let empty_body: BodyType = Box::new(BytesBody::from(vec![]));
|
||||||
let response = Response::builder()
|
let response = Response::builder()
|
||||||
.header("Location", format!("/{}", bucket))
|
.header("Location", format!("/{}", bucket))
|
||||||
.body(empty_body)
|
.body(empty_body)
|
||||||
@@ -187,7 +189,7 @@ async fn handler_inner(garage: Arc<Garage>, req: Request<Body>) -> Result<Respon
|
|||||||
}
|
}
|
||||||
&Method::HEAD => {
|
&Method::HEAD => {
|
||||||
// HeadBucket
|
// HeadBucket
|
||||||
let empty_body: Body = Body::from(vec![]);
|
let empty_body: BodyType = Box::new(BytesBody::from(vec![]));
|
||||||
let response = Response::builder().body(empty_body).unwrap();
|
let response = Response::builder().body(empty_body).unwrap();
|
||||||
Ok(response)
|
Ok(response)
|
||||||
}
|
}
|
||||||
@@ -203,8 +205,9 @@ async fn handler_inner(garage: Arc<Garage>, req: Request<Body>) -> Result<Respon
|
|||||||
let max_keys = params
|
let max_keys = params
|
||||||
.get("max-keys")
|
.get("max-keys")
|
||||||
.map(|x| {
|
.map(|x| {
|
||||||
x.parse::<usize>()
|
x.parse::<usize>().map_err(|e| {
|
||||||
.ok_or_bad_request("Invalid value for max-keys")
|
Error::BadRequest(format!("Invalid value for max-keys: {}", e))
|
||||||
|
})
|
||||||
})
|
})
|
||||||
.unwrap_or(Ok(1000))?;
|
.unwrap_or(Ok(1000))?;
|
||||||
let prefix = params.get("prefix").map(|x| x.as_str()).unwrap_or(&"");
|
let prefix = params.get("prefix").map(|x| x.as_str()).unwrap_or(&"");
|
||||||
@@ -229,7 +232,7 @@ async fn handler_inner(garage: Arc<Garage>, req: Request<Body>) -> Result<Respon
|
|||||||
// DeleteObjects
|
// DeleteObjects
|
||||||
Ok(handle_delete_objects(garage, bucket, req).await?)
|
Ok(handle_delete_objects(garage, bucket, req).await?)
|
||||||
} else {
|
} else {
|
||||||
debug!(
|
println!(
|
||||||
"Body: {}",
|
"Body: {}",
|
||||||
std::str::from_utf8(&hyper::body::to_bytes(req.into_body()).await?)
|
std::str::from_utf8(&hyper::body::to_bytes(req.into_body()).await?)
|
||||||
.unwrap_or("<invalid utf8>")
|
.unwrap_or("<invalid utf8>")
|
||||||
@@ -242,60 +245,18 @@ async fn handler_inner(garage: Arc<Garage>, req: Request<Body>) -> Result<Respon
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Extract the bucket name and the key name from an HTTP path
|
|
||||||
///
|
|
||||||
/// S3 internally manages only buckets and keys. This function splits
|
|
||||||
/// an HTTP path to get the corresponding bucket name and key.
|
|
||||||
fn parse_bucket_key(path: &str) -> Result<(&str, Option<&str>), Error> {
|
fn parse_bucket_key(path: &str) -> Result<(&str, Option<&str>), Error> {
|
||||||
let path = path.trim_start_matches('/');
|
let path = path.trim_start_matches('/');
|
||||||
|
|
||||||
let (bucket, key) = match path.find('/') {
|
match path.find('/') {
|
||||||
Some(i) => {
|
Some(i) => {
|
||||||
let key = &path[i + 1..];
|
let key = &path[i + 1..];
|
||||||
if key.len() > 0 {
|
if key.len() > 0 {
|
||||||
(&path[..i], Some(key))
|
Ok((&path[..i], Some(key)))
|
||||||
} else {
|
} else {
|
||||||
(&path[..i], None)
|
Ok((&path[..i], None))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
None => (path, None),
|
None => Ok((path, None)),
|
||||||
};
|
|
||||||
if bucket.len() == 0 {
|
|
||||||
return Err(Error::BadRequest(format!("No bucket specified")));
|
|
||||||
}
|
|
||||||
Ok((bucket, key))
|
|
||||||
}
|
|
||||||
|
|
||||||
#[cfg(test)]
|
|
||||||
mod tests {
|
|
||||||
use super::*;
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn parse_bucket_containing_a_key() -> Result<(), Error> {
|
|
||||||
let (bucket, key) = parse_bucket_key("/my_bucket/a/super/file.jpg")?;
|
|
||||||
assert_eq!(bucket, "my_bucket");
|
|
||||||
assert_eq!(key.expect("key must be set"), "a/super/file.jpg");
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn parse_bucket_containing_no_key() -> Result<(), Error> {
|
|
||||||
let (bucket, key) = parse_bucket_key("/my_bucket/")?;
|
|
||||||
assert_eq!(bucket, "my_bucket");
|
|
||||||
assert!(key.is_none());
|
|
||||||
let (bucket, key) = parse_bucket_key("/my_bucket")?;
|
|
||||||
assert_eq!(bucket, "my_bucket");
|
|
||||||
assert!(key.is_none());
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn parse_bucket_containing_no_bucket() {
|
|
||||||
let parsed = parse_bucket_key("");
|
|
||||||
assert!(parsed.is_err());
|
|
||||||
let parsed = parse_bucket_key("/");
|
|
||||||
assert!(parsed.is_err());
|
|
||||||
let parsed = parse_bucket_key("////");
|
|
||||||
assert!(parsed.is_err());
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,116 +0,0 @@
|
|||||||
use err_derive::Error;
|
|
||||||
use hyper::StatusCode;
|
|
||||||
|
|
||||||
use garage_util::error::Error as GarageError;
|
|
||||||
|
|
||||||
#[derive(Debug, Error)]
|
|
||||||
pub enum Error {
|
|
||||||
// Category: internal error
|
|
||||||
#[error(display = "Internal error: {}", _0)]
|
|
||||||
InternalError(#[error(source)] GarageError),
|
|
||||||
|
|
||||||
#[error(display = "Internal error (Hyper error): {}", _0)]
|
|
||||||
Hyper(#[error(source)] hyper::Error),
|
|
||||||
|
|
||||||
#[error(display = "Internal error (HTTP error): {}", _0)]
|
|
||||||
HTTP(#[error(source)] http::Error),
|
|
||||||
|
|
||||||
// Category: cannot process
|
|
||||||
#[error(display = "Forbidden: {}", _0)]
|
|
||||||
Forbidden(String),
|
|
||||||
|
|
||||||
#[error(display = "Not found")]
|
|
||||||
NotFound,
|
|
||||||
|
|
||||||
// Category: bad request
|
|
||||||
#[error(display = "Invalid UTF-8: {}", _0)]
|
|
||||||
InvalidUTF8(#[error(source)] std::str::Utf8Error),
|
|
||||||
|
|
||||||
#[error(display = "Invalid XML: {}", _0)]
|
|
||||||
InvalidXML(#[error(source)] roxmltree::Error),
|
|
||||||
|
|
||||||
#[error(display = "Invalid header value: {}", _0)]
|
|
||||||
InvalidHeader(#[error(source)] hyper::header::ToStrError),
|
|
||||||
|
|
||||||
#[error(display = "Invalid HTTP range: {:?}", _0)]
|
|
||||||
InvalidRange(#[error(from)] http_range::HttpRangeParseError),
|
|
||||||
|
|
||||||
#[error(display = "Bad request: {}", _0)]
|
|
||||||
BadRequest(String),
|
|
||||||
}
|
|
||||||
|
|
||||||
impl Error {
|
|
||||||
pub fn http_status_code(&self) -> StatusCode {
|
|
||||||
match self {
|
|
||||||
Error::NotFound => StatusCode::NOT_FOUND,
|
|
||||||
Error::Forbidden(_) => StatusCode::FORBIDDEN,
|
|
||||||
Error::InternalError(GarageError::RPC(_)) => StatusCode::SERVICE_UNAVAILABLE,
|
|
||||||
Error::InternalError(_) | Error::Hyper(_) | Error::HTTP(_) => {
|
|
||||||
StatusCode::INTERNAL_SERVER_ERROR
|
|
||||||
}
|
|
||||||
_ => StatusCode::BAD_REQUEST,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
pub trait OkOrBadRequest {
|
|
||||||
type S2;
|
|
||||||
fn ok_or_bad_request(self, reason: &'static str) -> Self::S2;
|
|
||||||
}
|
|
||||||
|
|
||||||
impl<T, E> OkOrBadRequest for Result<T, E>
|
|
||||||
where
|
|
||||||
E: std::fmt::Display,
|
|
||||||
{
|
|
||||||
type S2 = Result<T, Error>;
|
|
||||||
fn ok_or_bad_request(self, reason: &'static str) -> Result<T, Error> {
|
|
||||||
match self {
|
|
||||||
Ok(x) => Ok(x),
|
|
||||||
Err(e) => Err(Error::BadRequest(format!("{}: {}", reason, e))),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl<T> OkOrBadRequest for Option<T> {
|
|
||||||
type S2 = Result<T, Error>;
|
|
||||||
fn ok_or_bad_request(self, reason: &'static str) -> Result<T, Error> {
|
|
||||||
match self {
|
|
||||||
Some(x) => Ok(x),
|
|
||||||
None => Err(Error::BadRequest(format!("{}", reason))),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
pub trait OkOrInternalError {
|
|
||||||
type S2;
|
|
||||||
fn ok_or_internal_error(self, reason: &'static str) -> Self::S2;
|
|
||||||
}
|
|
||||||
|
|
||||||
impl<T, E> OkOrInternalError for Result<T, E>
|
|
||||||
where
|
|
||||||
E: std::fmt::Display,
|
|
||||||
{
|
|
||||||
type S2 = Result<T, Error>;
|
|
||||||
fn ok_or_internal_error(self, reason: &'static str) -> Result<T, Error> {
|
|
||||||
match self {
|
|
||||||
Ok(x) => Ok(x),
|
|
||||||
Err(e) => Err(Error::InternalError(GarageError::Message(format!(
|
|
||||||
"{}: {}",
|
|
||||||
reason, e
|
|
||||||
)))),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl<T> OkOrInternalError for Option<T> {
|
|
||||||
type S2 = Result<T, Error>;
|
|
||||||
fn ok_or_internal_error(self, reason: &'static str) -> Result<T, Error> {
|
|
||||||
match self {
|
|
||||||
Some(x) => Ok(x),
|
|
||||||
None => Err(Error::InternalError(GarageError::Message(format!(
|
|
||||||
"{}",
|
|
||||||
reason
|
|
||||||
)))),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -0,0 +1,88 @@
|
|||||||
|
use core::pin::Pin;
|
||||||
|
use core::task::{Context, Poll};
|
||||||
|
|
||||||
|
use futures::ready;
|
||||||
|
use futures::stream::*;
|
||||||
|
use hyper::body::{Bytes, HttpBody};
|
||||||
|
|
||||||
|
use garage_util::error::Error;
|
||||||
|
|
||||||
|
pub type BodyType = Box<dyn HttpBody<Data = Bytes, Error = Error> + Send + Unpin>;
|
||||||
|
|
||||||
|
type StreamType = Pin<Box<dyn Stream<Item = Result<Bytes, Error>> + Send>>;
|
||||||
|
|
||||||
|
pub struct StreamBody {
|
||||||
|
stream: StreamType,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl StreamBody {
|
||||||
|
pub fn new(stream: StreamType) -> Self {
|
||||||
|
Self { stream }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl HttpBody for StreamBody {
|
||||||
|
type Data = Bytes;
|
||||||
|
type Error = Error;
|
||||||
|
|
||||||
|
fn poll_data(
|
||||||
|
mut self: Pin<&mut Self>,
|
||||||
|
cx: &mut Context,
|
||||||
|
) -> Poll<Option<Result<Bytes, Self::Error>>> {
|
||||||
|
match ready!(self.stream.as_mut().poll_next(cx)) {
|
||||||
|
Some(res) => Poll::Ready(Some(res)),
|
||||||
|
None => Poll::Ready(None),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn poll_trailers(
|
||||||
|
self: Pin<&mut Self>,
|
||||||
|
_cx: &mut Context,
|
||||||
|
) -> Poll<Result<Option<hyper::HeaderMap<hyper::header::HeaderValue>>, Self::Error>> {
|
||||||
|
Poll::Ready(Ok(None))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub struct BytesBody {
|
||||||
|
bytes: Option<Bytes>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl BytesBody {
|
||||||
|
pub fn new(bytes: Bytes) -> Self {
|
||||||
|
Self { bytes: Some(bytes) }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl HttpBody for BytesBody {
|
||||||
|
type Data = Bytes;
|
||||||
|
type Error = Error;
|
||||||
|
|
||||||
|
fn poll_data(
|
||||||
|
mut self: Pin<&mut Self>,
|
||||||
|
_cx: &mut Context,
|
||||||
|
) -> Poll<Option<Result<Bytes, Self::Error>>> {
|
||||||
|
Poll::Ready(self.bytes.take().map(Ok))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn poll_trailers(
|
||||||
|
self: Pin<&mut Self>,
|
||||||
|
_cx: &mut Context,
|
||||||
|
) -> Poll<Result<Option<hyper::HeaderMap<hyper::header::HeaderValue>>, Self::Error>> {
|
||||||
|
Poll::Ready(Ok(None))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl From<String> for BytesBody {
|
||||||
|
fn from(x: String) -> BytesBody {
|
||||||
|
Self::new(Bytes::from(x))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
impl From<Vec<u8>> for BytesBody {
|
||||||
|
fn from(x: Vec<u8>) -> BytesBody {
|
||||||
|
Self::new(Bytes::from(x))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn empty_body() -> BodyType {
|
||||||
|
Box::new(BytesBody::from(vec![]))
|
||||||
|
}
|
||||||
+1
-2
@@ -1,9 +1,8 @@
|
|||||||
#[macro_use]
|
#[macro_use]
|
||||||
extern crate log;
|
extern crate log;
|
||||||
|
|
||||||
pub mod error;
|
|
||||||
|
|
||||||
pub mod encoding;
|
pub mod encoding;
|
||||||
|
pub mod http_util;
|
||||||
|
|
||||||
pub mod api_server;
|
pub mod api_server;
|
||||||
pub mod signature;
|
pub mod signature;
|
||||||
|
|||||||
+30
-17
@@ -2,17 +2,18 @@ use std::fmt::Write;
|
|||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
|
|
||||||
use chrono::{SecondsFormat, Utc};
|
use chrono::{SecondsFormat, Utc};
|
||||||
use hyper::{Body, Response};
|
use hyper::Response;
|
||||||
|
|
||||||
use garage_table::*;
|
use garage_table::*;
|
||||||
use garage_util::data::*;
|
use garage_util::data::*;
|
||||||
|
use garage_util::error::Error;
|
||||||
|
|
||||||
use garage_model::block_ref_table::*;
|
use garage_model::block_ref_table::*;
|
||||||
use garage_model::garage::Garage;
|
use garage_model::garage::Garage;
|
||||||
use garage_model::object_table::*;
|
use garage_model::object_table::*;
|
||||||
use garage_model::version_table::*;
|
use garage_model::version_table::*;
|
||||||
|
|
||||||
use crate::error::*;
|
use crate::http_util::*;
|
||||||
|
|
||||||
pub async fn handle_copy(
|
pub async fn handle_copy(
|
||||||
garage: Arc<Garage>,
|
garage: Arc<Garage>,
|
||||||
@@ -20,38 +21,47 @@ pub async fn handle_copy(
|
|||||||
dest_key: &str,
|
dest_key: &str,
|
||||||
source_bucket: &str,
|
source_bucket: &str,
|
||||||
source_key: &str,
|
source_key: &str,
|
||||||
) -> Result<Response<Body>, Error> {
|
) -> Result<Response<BodyType>, Error> {
|
||||||
let source_object = garage
|
let source_object = match garage
|
||||||
.object_table
|
.object_table
|
||||||
.get(&source_bucket.to_string(), &source_key.to_string())
|
.get(&source_bucket.to_string(), &source_key.to_string())
|
||||||
.await?
|
.await?
|
||||||
.ok_or(Error::NotFound)?;
|
{
|
||||||
|
None => return Err(Error::NotFound),
|
||||||
|
Some(o) => o,
|
||||||
|
};
|
||||||
|
|
||||||
let source_last_v = source_object
|
let source_last_v = match source_object
|
||||||
.versions()
|
.versions()
|
||||||
.iter()
|
.iter()
|
||||||
.rev()
|
.rev()
|
||||||
.filter(|v| v.is_complete())
|
.filter(|v| v.is_complete())
|
||||||
.next()
|
.next()
|
||||||
.ok_or(Error::NotFound)?;
|
{
|
||||||
|
Some(v) => v,
|
||||||
let source_last_state = match &source_last_v.state {
|
None => return Err(Error::NotFound),
|
||||||
ObjectVersionState::Complete(x) => x,
|
|
||||||
_ => unreachable!(),
|
|
||||||
};
|
};
|
||||||
|
|
||||||
let new_uuid = gen_uuid();
|
let new_uuid = gen_uuid();
|
||||||
let dest_object_version = ObjectVersion {
|
let dest_object_version = ObjectVersion {
|
||||||
uuid: new_uuid,
|
uuid: new_uuid,
|
||||||
timestamp: now_msec(),
|
timestamp: now_msec(),
|
||||||
state: ObjectVersionState::Complete(source_last_state.clone()),
|
mime_type: source_last_v.mime_type.clone(),
|
||||||
|
size: source_last_v.size,
|
||||||
|
state: ObjectVersionState::Complete,
|
||||||
|
data: source_last_v.data.clone(),
|
||||||
};
|
};
|
||||||
|
|
||||||
match &source_last_state {
|
match &source_last_v.data {
|
||||||
|
ObjectVersionData::Uploading => {
|
||||||
|
return Err(Error::Message(format!(
|
||||||
|
"Version is_complete() but data is stil Uploading (internal error)"
|
||||||
|
)));
|
||||||
|
}
|
||||||
ObjectVersionData::DeleteMarker => {
|
ObjectVersionData::DeleteMarker => {
|
||||||
return Err(Error::NotFound);
|
return Err(Error::NotFound);
|
||||||
}
|
}
|
||||||
ObjectVersionData::Inline(_meta, _bytes) => {
|
ObjectVersionData::Inline(_bytes) => {
|
||||||
let dest_object = Object::new(
|
let dest_object = Object::new(
|
||||||
dest_bucket.to_string(),
|
dest_bucket.to_string(),
|
||||||
dest_key.to_string(),
|
dest_key.to_string(),
|
||||||
@@ -59,12 +69,15 @@ pub async fn handle_copy(
|
|||||||
);
|
);
|
||||||
garage.object_table.insert(&dest_object).await?;
|
garage.object_table.insert(&dest_object).await?;
|
||||||
}
|
}
|
||||||
ObjectVersionData::FirstBlock(_meta, _first_block_hash) => {
|
ObjectVersionData::FirstBlock(_first_block_hash) => {
|
||||||
let source_version = garage
|
let source_version = garage
|
||||||
.version_table
|
.version_table
|
||||||
.get(&source_last_v.uuid, &EmptyKey)
|
.get(&source_last_v.uuid, &EmptyKey)
|
||||||
.await?;
|
.await?;
|
||||||
let source_version = source_version.ok_or(Error::NotFound)?;
|
let source_version = match source_version {
|
||||||
|
Some(v) => v,
|
||||||
|
None => return Err(Error::NotFound),
|
||||||
|
};
|
||||||
|
|
||||||
let dest_version = Version::new(
|
let dest_version = Version::new(
|
||||||
new_uuid,
|
new_uuid,
|
||||||
@@ -103,5 +116,5 @@ pub async fn handle_copy(
|
|||||||
writeln!(&mut xml, "\t<LastModified>{}</LastModified>", last_modified).unwrap();
|
writeln!(&mut xml, "\t<LastModified>{}</LastModified>", last_modified).unwrap();
|
||||||
writeln!(&mut xml, "</CopyObjectResult>").unwrap();
|
writeln!(&mut xml, "</CopyObjectResult>").unwrap();
|
||||||
|
|
||||||
Ok(Response::new(Body::from(xml.into_bytes())))
|
Ok(Response::new(Box::new(BytesBody::from(xml.into_bytes()))))
|
||||||
}
|
}
|
||||||
|
|||||||
+32
-16
@@ -4,28 +4,33 @@ use std::sync::Arc;
|
|||||||
use hyper::{Body, Request, Response};
|
use hyper::{Body, Request, Response};
|
||||||
|
|
||||||
use garage_util::data::*;
|
use garage_util::data::*;
|
||||||
|
use garage_util::error::Error;
|
||||||
|
|
||||||
use garage_model::garage::Garage;
|
use garage_model::garage::Garage;
|
||||||
use garage_model::object_table::*;
|
use garage_model::object_table::*;
|
||||||
|
|
||||||
use crate::encoding::*;
|
use crate::encoding::*;
|
||||||
use crate::error::*;
|
use crate::http_util::*;
|
||||||
|
|
||||||
async fn handle_delete_internal(
|
async fn handle_delete_internal(
|
||||||
garage: &Garage,
|
garage: &Garage,
|
||||||
bucket: &str,
|
bucket: &str,
|
||||||
key: &str,
|
key: &str,
|
||||||
) -> Result<(UUID, UUID), Error> {
|
) -> Result<(UUID, UUID), Error> {
|
||||||
let object = garage
|
let object = match garage
|
||||||
.object_table
|
.object_table
|
||||||
.get(&bucket.to_string(), &key.to_string())
|
.get(&bucket.to_string(), &key.to_string())
|
||||||
.await?
|
.await?
|
||||||
.ok_or(Error::NotFound)?; // No need to delete
|
{
|
||||||
|
None => {
|
||||||
|
// No need to delete
|
||||||
|
return Err(Error::NotFound);
|
||||||
|
}
|
||||||
|
Some(o) => o,
|
||||||
|
};
|
||||||
|
|
||||||
let interesting_versions = object.versions().iter().filter(|v| match v.state {
|
let interesting_versions = object.versions().iter().filter(|v| {
|
||||||
ObjectVersionState::Aborted => false,
|
v.data != ObjectVersionData::DeleteMarker && v.state != ObjectVersionState::Aborted
|
||||||
ObjectVersionState::Complete(ObjectVersionData::DeleteMarker) => false,
|
|
||||||
_ => true,
|
|
||||||
});
|
});
|
||||||
|
|
||||||
let mut must_delete = None;
|
let mut must_delete = None;
|
||||||
@@ -37,7 +42,10 @@ async fn handle_delete_internal(
|
|||||||
timestamp = std::cmp::max(timestamp, v.timestamp + 1);
|
timestamp = std::cmp::max(timestamp, v.timestamp + 1);
|
||||||
}
|
}
|
||||||
|
|
||||||
let deleted_version = must_delete.ok_or(Error::NotFound)?;
|
let deleted_version = match must_delete {
|
||||||
|
None => return Err(Error::NotFound),
|
||||||
|
Some(v) => v,
|
||||||
|
};
|
||||||
|
|
||||||
let version_uuid = gen_uuid();
|
let version_uuid = gen_uuid();
|
||||||
|
|
||||||
@@ -47,7 +55,10 @@ async fn handle_delete_internal(
|
|||||||
vec![ObjectVersion {
|
vec![ObjectVersion {
|
||||||
uuid: version_uuid,
|
uuid: version_uuid,
|
||||||
timestamp: now_msec(),
|
timestamp: now_msec(),
|
||||||
state: ObjectVersionState::Complete(ObjectVersionData::DeleteMarker),
|
mime_type: "application/x-delete-marker".into(),
|
||||||
|
size: 0,
|
||||||
|
state: ObjectVersionState::Complete,
|
||||||
|
data: ObjectVersionData::DeleteMarker,
|
||||||
}],
|
}],
|
||||||
);
|
);
|
||||||
|
|
||||||
@@ -59,13 +70,13 @@ pub async fn handle_delete(
|
|||||||
garage: Arc<Garage>,
|
garage: Arc<Garage>,
|
||||||
bucket: &str,
|
bucket: &str,
|
||||||
key: &str,
|
key: &str,
|
||||||
) -> Result<Response<Body>, Error> {
|
) -> Result<Response<BodyType>, Error> {
|
||||||
let (_deleted_version, delete_marker_version) =
|
let (_deleted_version, delete_marker_version) =
|
||||||
handle_delete_internal(&garage, bucket, key).await?;
|
handle_delete_internal(&garage, bucket, key).await?;
|
||||||
|
|
||||||
Ok(Response::builder()
|
Ok(Response::builder()
|
||||||
.header("x-amz-version-id", hex::encode(delete_marker_version))
|
.header("x-amz-version-id", hex::encode(delete_marker_version))
|
||||||
.body(Body::from(vec![]))
|
.body(empty_body())
|
||||||
.unwrap())
|
.unwrap())
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -73,10 +84,11 @@ pub async fn handle_delete_objects(
|
|||||||
garage: Arc<Garage>,
|
garage: Arc<Garage>,
|
||||||
bucket: &str,
|
bucket: &str,
|
||||||
req: Request<Body>,
|
req: Request<Body>,
|
||||||
) -> Result<Response<Body>, Error> {
|
) -> Result<Response<BodyType>, Error> {
|
||||||
let body = hyper::body::to_bytes(req.into_body()).await?;
|
let body = hyper::body::to_bytes(req.into_body()).await?;
|
||||||
let cmd_xml = roxmltree::Document::parse(&std::str::from_utf8(&body)?)?;
|
let cmd_xml = roxmltree::Document::parse(&std::str::from_utf8(&body)?)?;
|
||||||
let cmd = parse_delete_objects_xml(&cmd_xml).ok_or_bad_request("Invalid delete XML query")?;
|
let cmd = parse_delete_objects_xml(&cmd_xml)
|
||||||
|
.map_err(|e| Error::BadRequest(format!("Invald delete XML query: {}", e)))?;
|
||||||
|
|
||||||
let mut retxml = String::new();
|
let mut retxml = String::new();
|
||||||
writeln!(&mut retxml, r#"<?xml version="1.0" encoding="UTF-8"?>"#).unwrap();
|
writeln!(&mut retxml, r#"<?xml version="1.0" encoding="UTF-8"?>"#).unwrap();
|
||||||
@@ -118,7 +130,9 @@ pub async fn handle_delete_objects(
|
|||||||
|
|
||||||
writeln!(&mut retxml, "</DeleteObjectsOutput>").unwrap();
|
writeln!(&mut retxml, "</DeleteObjectsOutput>").unwrap();
|
||||||
|
|
||||||
Ok(Response::new(Body::from(retxml.into_bytes())))
|
Ok(Response::new(Box::new(BytesBody::from(
|
||||||
|
retxml.into_bytes(),
|
||||||
|
))))
|
||||||
}
|
}
|
||||||
|
|
||||||
struct DeleteRequest {
|
struct DeleteRequest {
|
||||||
@@ -133,8 +147,10 @@ fn parse_delete_objects_xml(xml: &roxmltree::Document) -> Result<DeleteRequest,
|
|||||||
let mut ret = DeleteRequest { objects: vec![] };
|
let mut ret = DeleteRequest { objects: vec![] };
|
||||||
|
|
||||||
let root = xml.root();
|
let root = xml.root();
|
||||||
let delete = root.first_child().ok_or(format!("Delete tag not found"))?;
|
let delete = match root.first_child() {
|
||||||
|
Some(del) => del,
|
||||||
|
None => return Err(format!("Delete tag not found")),
|
||||||
|
};
|
||||||
if !delete.has_tag_name("Delete") {
|
if !delete.has_tag_name("Delete") {
|
||||||
return Err(format!("Invalid root tag: {:?}", root));
|
return Err(format!("Invalid root tag: {:?}", root));
|
||||||
}
|
}
|
||||||
|
|||||||
+79
-114
@@ -5,67 +5,53 @@ use futures::stream::*;
|
|||||||
use hyper::body::Bytes;
|
use hyper::body::Bytes;
|
||||||
use hyper::{Body, Request, Response, StatusCode};
|
use hyper::{Body, Request, Response, StatusCode};
|
||||||
|
|
||||||
|
use garage_util::error::Error;
|
||||||
|
|
||||||
use garage_table::EmptyKey;
|
use garage_table::EmptyKey;
|
||||||
|
|
||||||
use garage_model::garage::Garage;
|
use garage_model::garage::Garage;
|
||||||
use garage_model::object_table::*;
|
use garage_model::object_table::*;
|
||||||
|
|
||||||
use crate::error::*;
|
use crate::http_util::*;
|
||||||
|
|
||||||
fn object_headers(
|
fn object_headers(version: &ObjectVersion) -> http::response::Builder {
|
||||||
version: &ObjectVersion,
|
|
||||||
version_meta: &ObjectVersionMeta,
|
|
||||||
) -> http::response::Builder {
|
|
||||||
let date = UNIX_EPOCH + Duration::from_millis(version.timestamp);
|
let date = UNIX_EPOCH + Duration::from_millis(version.timestamp);
|
||||||
let date_str = httpdate::fmt_http_date(date);
|
let date_str = httpdate::fmt_http_date(date);
|
||||||
|
|
||||||
let mut resp = Response::builder()
|
Response::builder()
|
||||||
.header(
|
.header("Content-Type", version.mime_type.to_string())
|
||||||
"Content-Type",
|
.header("Content-Length", format!("{}", version.size))
|
||||||
version_meta.headers.content_type.to_string(),
|
|
||||||
)
|
|
||||||
.header("Last-Modified", date_str)
|
.header("Last-Modified", date_str)
|
||||||
.header("Accept-Ranges", format!("bytes"));
|
.header("Accept-Ranges", format!("bytes"))
|
||||||
|
|
||||||
if !version_meta.etag.is_empty() {
|
|
||||||
resp = resp.header("ETag", format!("\"{}\"", version_meta.etag));
|
|
||||||
}
|
|
||||||
|
|
||||||
for (k, v) in version_meta.headers.other.iter() {
|
|
||||||
resp = resp.header(k, v.to_string());
|
|
||||||
}
|
|
||||||
|
|
||||||
resp
|
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn handle_head(
|
pub async fn handle_head(
|
||||||
garage: Arc<Garage>,
|
garage: Arc<Garage>,
|
||||||
bucket: &str,
|
bucket: &str,
|
||||||
key: &str,
|
key: &str,
|
||||||
) -> Result<Response<Body>, Error> {
|
) -> Result<Response<BodyType>, Error> {
|
||||||
let object = garage
|
let object = match garage
|
||||||
.object_table
|
.object_table
|
||||||
.get(&bucket.to_string(), &key.to_string())
|
.get(&bucket.to_string(), &key.to_string())
|
||||||
.await?
|
.await?
|
||||||
.ok_or(Error::NotFound)?;
|
{
|
||||||
|
None => return Err(Error::NotFound),
|
||||||
|
Some(o) => o,
|
||||||
|
};
|
||||||
|
|
||||||
let version = object
|
let version = match object
|
||||||
.versions()
|
.versions()
|
||||||
.iter()
|
.iter()
|
||||||
.rev()
|
.rev()
|
||||||
.filter(|v| v.is_data())
|
.filter(|v| v.is_data())
|
||||||
.next()
|
.next()
|
||||||
.ok_or(Error::NotFound)?;
|
{
|
||||||
|
Some(v) => v,
|
||||||
let version_meta = match &version.state {
|
None => return Err(Error::NotFound),
|
||||||
ObjectVersionState::Complete(ObjectVersionData::Inline(meta, _)) => meta,
|
|
||||||
ObjectVersionState::Complete(ObjectVersionData::FirstBlock(meta, _)) => meta,
|
|
||||||
_ => unreachable!(),
|
|
||||||
};
|
};
|
||||||
|
|
||||||
let body: Body = Body::from(vec![]);
|
let body: BodyType = Box::new(BytesBody::from(vec![]));
|
||||||
let response = object_headers(&version, version_meta)
|
let response = object_headers(&version)
|
||||||
.header("Content-Length", format!("{}", version_meta.size))
|
|
||||||
.status(StatusCode::OK)
|
.status(StatusCode::OK)
|
||||||
.body(body)
|
.body(body)
|
||||||
.unwrap();
|
.unwrap();
|
||||||
@@ -77,35 +63,34 @@ pub async fn handle_get(
|
|||||||
req: &Request<Body>,
|
req: &Request<Body>,
|
||||||
bucket: &str,
|
bucket: &str,
|
||||||
key: &str,
|
key: &str,
|
||||||
) -> Result<Response<Body>, Error> {
|
) -> Result<Response<BodyType>, Error> {
|
||||||
let object = garage
|
let object = match garage
|
||||||
.object_table
|
.object_table
|
||||||
.get(&bucket.to_string(), &key.to_string())
|
.get(&bucket.to_string(), &key.to_string())
|
||||||
.await?
|
.await?
|
||||||
.ok_or(Error::NotFound)?;
|
{
|
||||||
|
None => return Err(Error::NotFound),
|
||||||
|
Some(o) => o,
|
||||||
|
};
|
||||||
|
|
||||||
let last_v = object
|
let last_v = match object
|
||||||
.versions()
|
.versions()
|
||||||
.iter()
|
.iter()
|
||||||
.rev()
|
.rev()
|
||||||
.filter(|v| v.is_complete())
|
.filter(|v| v.is_complete())
|
||||||
.next()
|
.next()
|
||||||
.ok_or(Error::NotFound)?;
|
{
|
||||||
|
Some(v) => v,
|
||||||
let last_v_data = match &last_v.state {
|
None => return Err(Error::NotFound),
|
||||||
ObjectVersionState::Complete(x) => x,
|
|
||||||
_ => unreachable!(),
|
|
||||||
};
|
|
||||||
let last_v_meta = match last_v_data {
|
|
||||||
ObjectVersionData::DeleteMarker => return Err(Error::NotFound),
|
|
||||||
ObjectVersionData::Inline(meta, _) => meta,
|
|
||||||
ObjectVersionData::FirstBlock(meta, _) => meta,
|
|
||||||
};
|
};
|
||||||
|
|
||||||
let range = match req.headers().get("range") {
|
let range = match req.headers().get("range") {
|
||||||
Some(range) => {
|
Some(range) => {
|
||||||
let range_str = range.to_str()?;
|
let range_str = range
|
||||||
let mut ranges = http_range::HttpRange::parse(range_str, last_v_meta.size)?;
|
.to_str()
|
||||||
|
.map_err(|e| Error::BadRequest(format!("Invalid range header: {}", e)))?;
|
||||||
|
let mut ranges = http_range::HttpRange::parse(range_str, last_v.size)
|
||||||
|
.map_err(|_e| Error::BadRequest(format!("Invalid range")))?;
|
||||||
if ranges.len() > 1 {
|
if ranges.len() > 1 {
|
||||||
return Err(Error::BadRequest(format!("Multiple ranges not supported")));
|
return Err(Error::BadRequest(format!("Multiple ranges not supported")));
|
||||||
} else {
|
} else {
|
||||||
@@ -115,33 +100,29 @@ pub async fn handle_get(
|
|||||||
None => None,
|
None => None,
|
||||||
};
|
};
|
||||||
if let Some(range) = range {
|
if let Some(range) = range {
|
||||||
return handle_get_range(
|
return handle_get_range(garage, last_v, range.start, range.start + range.length).await;
|
||||||
garage,
|
|
||||||
last_v,
|
|
||||||
last_v_data,
|
|
||||||
last_v_meta,
|
|
||||||
range.start,
|
|
||||||
range.start + range.length,
|
|
||||||
)
|
|
||||||
.await;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
let resp_builder = object_headers(&last_v, last_v_meta)
|
let resp_builder = object_headers(&last_v).status(StatusCode::OK);
|
||||||
.header("Content-Length", format!("{}", last_v_meta.size))
|
|
||||||
.status(StatusCode::OK);
|
|
||||||
|
|
||||||
match &last_v_data {
|
match &last_v.data {
|
||||||
ObjectVersionData::DeleteMarker => unreachable!(),
|
ObjectVersionData::Uploading => Err(Error::Message(format!(
|
||||||
ObjectVersionData::Inline(_, bytes) => {
|
"Version is_complete() but data is stil Uploading (internal error)"
|
||||||
let body: Body = Body::from(bytes.to_vec());
|
))),
|
||||||
|
ObjectVersionData::DeleteMarker => Err(Error::NotFound),
|
||||||
|
ObjectVersionData::Inline(bytes) => {
|
||||||
|
let body: BodyType = Box::new(BytesBody::from(bytes.to_vec()));
|
||||||
Ok(resp_builder.body(body)?)
|
Ok(resp_builder.body(body)?)
|
||||||
}
|
}
|
||||||
ObjectVersionData::FirstBlock(_, first_block_hash) => {
|
ObjectVersionData::FirstBlock(first_block_hash) => {
|
||||||
let read_first_block = garage.block_manager.rpc_get_block(&first_block_hash);
|
let read_first_block = garage.block_manager.rpc_get_block(&first_block_hash);
|
||||||
let get_next_blocks = garage.version_table.get(&last_v.uuid, &EmptyKey);
|
let get_next_blocks = garage.version_table.get(&last_v.uuid, &EmptyKey);
|
||||||
|
|
||||||
let (first_block, version) = futures::try_join!(read_first_block, get_next_blocks)?;
|
let (first_block, version) = futures::try_join!(read_first_block, get_next_blocks)?;
|
||||||
let version = version.ok_or(Error::NotFound)?;
|
let version = match version {
|
||||||
|
Some(v) => v,
|
||||||
|
None => return Err(Error::NotFound),
|
||||||
|
};
|
||||||
|
|
||||||
let mut blocks = version
|
let mut blocks = version
|
||||||
.blocks()
|
.blocks()
|
||||||
@@ -166,8 +147,7 @@ pub async fn handle_get(
|
|||||||
}
|
}
|
||||||
})
|
})
|
||||||
.buffered(2);
|
.buffered(2);
|
||||||
|
let body: BodyType = Box::new(StreamBody::new(Box::pin(body_stream)));
|
||||||
let body = hyper::body::Body::wrap_stream(body_stream);
|
|
||||||
Ok(resp_builder.body(body)?)
|
Ok(resp_builder.body(body)?)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -176,86 +156,71 @@ pub async fn handle_get(
|
|||||||
pub async fn handle_get_range(
|
pub async fn handle_get_range(
|
||||||
garage: Arc<Garage>,
|
garage: Arc<Garage>,
|
||||||
version: &ObjectVersion,
|
version: &ObjectVersion,
|
||||||
version_data: &ObjectVersionData,
|
|
||||||
version_meta: &ObjectVersionMeta,
|
|
||||||
begin: u64,
|
begin: u64,
|
||||||
end: u64,
|
end: u64,
|
||||||
) -> Result<Response<Body>, Error> {
|
) -> Result<Response<BodyType>, Error> {
|
||||||
if end > version_meta.size {
|
if end > version.size {
|
||||||
return Err(Error::BadRequest(format!("Range not included in file")));
|
return Err(Error::BadRequest(format!("Range not included in file")));
|
||||||
}
|
}
|
||||||
|
|
||||||
let resp_builder = object_headers(version, version_meta)
|
let resp_builder = object_headers(&version)
|
||||||
.header("Content-Length", format!("{}", end - begin))
|
|
||||||
.header(
|
.header(
|
||||||
"Content-Range",
|
"Content-Range",
|
||||||
format!("bytes {}-{}/{}", begin, end - 1, version_meta.size),
|
format!("bytes {}-{}/{}", begin, end, version.size),
|
||||||
)
|
)
|
||||||
.status(StatusCode::PARTIAL_CONTENT);
|
.status(StatusCode::PARTIAL_CONTENT);
|
||||||
|
|
||||||
match &version_data {
|
match &version.data {
|
||||||
ObjectVersionData::DeleteMarker => unreachable!(),
|
ObjectVersionData::Uploading => Err(Error::Message(format!(
|
||||||
ObjectVersionData::Inline(_meta, bytes) => {
|
"Version is_complete() but data is stil Uploading (internal error)"
|
||||||
|
))),
|
||||||
|
ObjectVersionData::DeleteMarker => Err(Error::NotFound),
|
||||||
|
ObjectVersionData::Inline(bytes) => {
|
||||||
if end as usize <= bytes.len() {
|
if end as usize <= bytes.len() {
|
||||||
let body: Body = Body::from(bytes[begin as usize..end as usize].to_vec());
|
let body: BodyType = Box::new(BytesBody::from(
|
||||||
|
bytes[begin as usize..end as usize].to_vec(),
|
||||||
|
));
|
||||||
Ok(resp_builder.body(body)?)
|
Ok(resp_builder.body(body)?)
|
||||||
} else {
|
} else {
|
||||||
None.ok_or_internal_error(
|
Err(Error::Message(format!("Internal error: requested range not present in inline bytes when it should have been")))
|
||||||
"Requested range not present in inline bytes when it should have been",
|
|
||||||
)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
ObjectVersionData::FirstBlock(_meta, _first_block_hash) => {
|
ObjectVersionData::FirstBlock(_first_block_hash) => {
|
||||||
let version = garage.version_table.get(&version.uuid, &EmptyKey).await?;
|
let version = garage.version_table.get(&version.uuid, &EmptyKey).await?;
|
||||||
let version = match version {
|
let version = match version {
|
||||||
Some(v) => v,
|
Some(v) => v,
|
||||||
None => return Err(Error::NotFound),
|
None => return Err(Error::NotFound),
|
||||||
};
|
};
|
||||||
|
|
||||||
// We will store here the list of blocks that have an intersection with the requested
|
let blocks = version
|
||||||
// range, as well as their "true offset", which is their actual offset in the complete
|
.blocks()
|
||||||
// file (whereas block.offset designates the offset of the block WITHIN THE PART
|
.iter()
|
||||||
// block.part_number, which is not the same in the case of a multipart upload)
|
.cloned()
|
||||||
let mut blocks = Vec::with_capacity(std::cmp::min(
|
.filter(|block| block.offset + block.size > begin && block.offset < end)
|
||||||
version.blocks().len(),
|
.collect::<Vec<_>>();
|
||||||
4 + ((end - begin) / std::cmp::max(version.blocks()[0].size as u64, 1024)) as usize,
|
|
||||||
));
|
|
||||||
let mut true_offset = 0;
|
|
||||||
for b in version.blocks().iter() {
|
|
||||||
if true_offset >= end {
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
// Keep only blocks that have an intersection with the requested range
|
|
||||||
if true_offset < end && true_offset + b.size > begin {
|
|
||||||
blocks.push((b.clone(), true_offset));
|
|
||||||
}
|
|
||||||
true_offset += b.size;
|
|
||||||
}
|
|
||||||
|
|
||||||
let body_stream = futures::stream::iter(blocks)
|
let body_stream = futures::stream::iter(blocks)
|
||||||
.map(move |(block, true_offset)| {
|
.map(move |block| {
|
||||||
let garage = garage.clone();
|
let garage = garage.clone();
|
||||||
async move {
|
async move {
|
||||||
let data = garage.block_manager.rpc_get_block(&block.hash).await?;
|
let data = garage.block_manager.rpc_get_block(&block.hash).await?;
|
||||||
let data = Bytes::from(data);
|
let start_in_block = if block.offset > begin {
|
||||||
let start_in_block = if true_offset > begin {
|
|
||||||
0
|
0
|
||||||
} else {
|
} else {
|
||||||
begin - true_offset
|
begin - block.offset
|
||||||
};
|
};
|
||||||
let end_in_block = if true_offset + block.size < end {
|
let end_in_block = if block.offset + block.size < end {
|
||||||
block.size
|
block.size
|
||||||
} else {
|
} else {
|
||||||
end - true_offset
|
end - block.offset
|
||||||
};
|
};
|
||||||
Result::<Bytes, Error>::Ok(Bytes::from(
|
Ok(Bytes::from(
|
||||||
data.slice(start_in_block as usize..end_in_block as usize),
|
data[start_in_block as usize..end_in_block as usize].to_vec(),
|
||||||
))
|
))
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
.buffered(2);
|
.buffered(2);
|
||||||
|
let body: BodyType = Box::new(StreamBody::new(Box::pin(body_stream)));
|
||||||
let body = hyper::body::Body::wrap_stream(body_stream);
|
|
||||||
Ok(resp_builder.body(body)?)
|
Ok(resp_builder.body(body)?)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
+18
-42
@@ -3,22 +3,19 @@ use std::fmt::Write;
|
|||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
|
|
||||||
use chrono::{DateTime, NaiveDateTime, SecondsFormat, Utc};
|
use chrono::{DateTime, NaiveDateTime, SecondsFormat, Utc};
|
||||||
use hyper::{Body, Response};
|
use hyper::Response;
|
||||||
|
|
||||||
use garage_util::error::Error;
|
use garage_util::error::Error;
|
||||||
|
|
||||||
use garage_model::garage::Garage;
|
use garage_model::garage::Garage;
|
||||||
use garage_model::object_table::*;
|
|
||||||
|
|
||||||
use garage_table::DeletedFilter;
|
|
||||||
|
|
||||||
use crate::encoding::*;
|
use crate::encoding::*;
|
||||||
|
use crate::http_util::*;
|
||||||
|
|
||||||
#[derive(Debug)]
|
#[derive(Debug)]
|
||||||
struct ListResultInfo {
|
struct ListResultInfo {
|
||||||
last_modified: u64,
|
last_modified: u64,
|
||||||
size: u64,
|
size: u64,
|
||||||
etag: String,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn handle_list(
|
pub async fn handle_list(
|
||||||
@@ -29,7 +26,7 @@ pub async fn handle_list(
|
|||||||
prefix: &str,
|
prefix: &str,
|
||||||
marker: Option<&str>,
|
marker: Option<&str>,
|
||||||
urlencode_resp: bool,
|
urlencode_resp: bool,
|
||||||
) -> Result<Response<Body>, Error> {
|
) -> Result<Response<BodyType>, Error> {
|
||||||
let mut result_keys = BTreeMap::<String, ListResultInfo>::new();
|
let mut result_keys = BTreeMap::<String, ListResultInfo>::new();
|
||||||
let mut result_common_prefixes = BTreeSet::<String>::new();
|
let mut result_common_prefixes = BTreeSet::<String>::new();
|
||||||
|
|
||||||
@@ -44,7 +41,7 @@ pub async fn handle_list(
|
|||||||
.get_range(
|
.get_range(
|
||||||
&bucket.to_string(),
|
&bucket.to_string(),
|
||||||
Some(next_chunk_start.clone()),
|
Some(next_chunk_start.clone()),
|
||||||
Some(DeletedFilter::NotDeleted),
|
Some(()),
|
||||||
max_keys + 1,
|
max_keys + 1,
|
||||||
)
|
)
|
||||||
.await?;
|
.await?;
|
||||||
@@ -57,37 +54,30 @@ pub async fn handle_list(
|
|||||||
|
|
||||||
for object in objects.iter() {
|
for object in objects.iter() {
|
||||||
if !object.key.starts_with(prefix) {
|
if !object.key.starts_with(prefix) {
|
||||||
truncated = None;
|
truncated = false;
|
||||||
break 'query_loop;
|
break 'query_loop;
|
||||||
}
|
}
|
||||||
if let Some(version) = object.versions().iter().find(|x| x.is_data()) {
|
if let Some(version) = object.versions().iter().find(|x| x.is_data()) {
|
||||||
if result_keys.len() + result_common_prefixes.len() >= max_keys {
|
if result_keys.len() + result_common_prefixes.len() >= max_keys {
|
||||||
truncated = Some(object.key.to_string());
|
truncated = true;
|
||||||
break 'query_loop;
|
break 'query_loop;
|
||||||
}
|
}
|
||||||
let common_prefix = if delimiter.len() > 0 {
|
let common_prefix = if delimiter.len() > 0 {
|
||||||
let relative_key = &object.key[prefix.len()..];
|
let relative_key = &object.key[prefix.len()..];
|
||||||
relative_key
|
match relative_key.find(delimiter) {
|
||||||
.find(delimiter)
|
Some(i) => Some(&object.key[..prefix.len() + i + delimiter.len()]),
|
||||||
.map(|i| &object.key[..prefix.len() + i + delimiter.len()])
|
None => None,
|
||||||
|
}
|
||||||
} else {
|
} else {
|
||||||
None
|
None
|
||||||
};
|
};
|
||||||
if let Some(pfx) = common_prefix {
|
if let Some(pfx) = common_prefix {
|
||||||
result_common_prefixes.insert(pfx.to_string());
|
result_common_prefixes.insert(pfx.to_string());
|
||||||
} else {
|
} else {
|
||||||
let meta = match &version.state {
|
|
||||||
ObjectVersionState::Complete(ObjectVersionData::Inline(meta, _)) => meta,
|
|
||||||
ObjectVersionState::Complete(ObjectVersionData::FirstBlock(meta, _)) => {
|
|
||||||
meta
|
|
||||||
}
|
|
||||||
_ => unreachable!(),
|
|
||||||
};
|
|
||||||
let info = match result_keys.get(&object.key) {
|
let info = match result_keys.get(&object.key) {
|
||||||
None => ListResultInfo {
|
None => ListResultInfo {
|
||||||
last_modified: version.timestamp,
|
last_modified: version.timestamp,
|
||||||
size: meta.size,
|
size: version.size,
|
||||||
etag: meta.etag.to_string(),
|
|
||||||
},
|
},
|
||||||
Some(_lri) => {
|
Some(_lri) => {
|
||||||
return Err(Error::Message(format!("Duplicate key?? {}", object.key)))
|
return Err(Error::Message(format!("Duplicate key?? {}", object.key)))
|
||||||
@@ -98,7 +88,7 @@ pub async fn handle_list(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
if objects.len() < max_keys + 1 {
|
if objects.len() < max_keys + 1 {
|
||||||
truncated = None;
|
truncated = false;
|
||||||
break 'query_loop;
|
break 'query_loop;
|
||||||
}
|
}
|
||||||
if objects.len() > 0 {
|
if objects.len() > 0 {
|
||||||
@@ -113,22 +103,11 @@ pub async fn handle_list(
|
|||||||
r#"<ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">"#
|
r#"<ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">"#
|
||||||
)
|
)
|
||||||
.unwrap();
|
.unwrap();
|
||||||
writeln!(&mut xml, "\t<Name>{}</Name>", bucket).unwrap();
|
writeln!(&mut xml, "\t<Bucket>{}</Bucket>", bucket).unwrap();
|
||||||
writeln!(&mut xml, "\t<Prefix>{}</Prefix>", prefix).unwrap();
|
writeln!(&mut xml, "\t<Prefix>{}</Prefix>", prefix).unwrap();
|
||||||
if let Some(mkr) = marker {
|
|
||||||
writeln!(&mut xml, "\t<Marker>{}</Marker>", mkr).unwrap();
|
|
||||||
}
|
|
||||||
writeln!(&mut xml, "\t<KeyCount>{}</KeyCount>", result_keys.len()).unwrap();
|
writeln!(&mut xml, "\t<KeyCount>{}</KeyCount>", result_keys.len()).unwrap();
|
||||||
writeln!(&mut xml, "\t<MaxKeys>{}</MaxKeys>", max_keys).unwrap();
|
writeln!(&mut xml, "\t<MaxKeys>{}</MaxKeys>", max_keys).unwrap();
|
||||||
writeln!(
|
writeln!(&mut xml, "\t<IsTruncated>{}</IsTruncated>", truncated).unwrap();
|
||||||
&mut xml,
|
|
||||||
"\t<IsTruncated>{}</IsTruncated>",
|
|
||||||
truncated.is_some()
|
|
||||||
)
|
|
||||||
.unwrap();
|
|
||||||
if let Some(next_marker) = truncated {
|
|
||||||
writeln!(&mut xml, "\t<NextMarker>{}</NextMarker>", next_marker).unwrap();
|
|
||||||
}
|
|
||||||
for (key, info) in result_keys.iter() {
|
for (key, info) in result_keys.iter() {
|
||||||
let last_modif = NaiveDateTime::from_timestamp(info.last_modified as i64 / 1000, 0);
|
let last_modif = NaiveDateTime::from_timestamp(info.last_modified as i64 / 1000, 0);
|
||||||
let last_modif = DateTime::<Utc>::from_utc(last_modif, Utc);
|
let last_modif = DateTime::<Utc>::from_utc(last_modif, Utc);
|
||||||
@@ -143,15 +122,12 @@ pub async fn handle_list(
|
|||||||
.unwrap();
|
.unwrap();
|
||||||
writeln!(&mut xml, "\t\t<LastModified>{}</LastModified>", last_modif).unwrap();
|
writeln!(&mut xml, "\t\t<LastModified>{}</LastModified>", last_modif).unwrap();
|
||||||
writeln!(&mut xml, "\t\t<Size>{}</Size>", info.size).unwrap();
|
writeln!(&mut xml, "\t\t<Size>{}</Size>", info.size).unwrap();
|
||||||
if !info.etag.is_empty() {
|
|
||||||
writeln!(&mut xml, "\t\t<ETag>\"{}\"</ETag>", info.etag).unwrap();
|
|
||||||
}
|
|
||||||
writeln!(&mut xml, "\t\t<StorageClass>STANDARD</StorageClass>").unwrap();
|
writeln!(&mut xml, "\t\t<StorageClass>STANDARD</StorageClass>").unwrap();
|
||||||
writeln!(&mut xml, "\t</Contents>").unwrap();
|
writeln!(&mut xml, "\t</Contents>").unwrap();
|
||||||
}
|
}
|
||||||
if result_common_prefixes.len() > 0 {
|
if result_common_prefixes.len() > 0 {
|
||||||
|
writeln!(&mut xml, "\t<CommonPrefixes>").unwrap();
|
||||||
for pfx in result_common_prefixes.iter() {
|
for pfx in result_common_prefixes.iter() {
|
||||||
writeln!(&mut xml, "\t<CommonPrefixes>").unwrap();
|
|
||||||
writeln!(
|
writeln!(
|
||||||
&mut xml,
|
&mut xml,
|
||||||
"\t\t<Prefix>{}</Prefix>",
|
"\t\t<Prefix>{}</Prefix>",
|
||||||
@@ -159,11 +135,11 @@ pub async fn handle_list(
|
|||||||
//xml_encode_key(pfx, urlencode_resp)
|
//xml_encode_key(pfx, urlencode_resp)
|
||||||
)
|
)
|
||||||
.unwrap();
|
.unwrap();
|
||||||
writeln!(&mut xml, "\t</CommonPrefixes>").unwrap();
|
|
||||||
}
|
}
|
||||||
|
writeln!(&mut xml, "\t</CommonPrefixes>").unwrap();
|
||||||
}
|
}
|
||||||
writeln!(&mut xml, "</ListBucketResult>").unwrap();
|
writeln!(&mut xml, "</ListBucketResult>").unwrap();
|
||||||
debug!("{}", xml);
|
println!("{}", xml);
|
||||||
|
|
||||||
Ok(Response::new(Body::from(xml.into_bytes())))
|
Ok(Response::new(Box::new(BytesBody::from(xml.into_bytes()))))
|
||||||
}
|
}
|
||||||
|
|||||||
+91
-218
@@ -1,17 +1,14 @@
|
|||||||
use std::collections::{BTreeMap, VecDeque};
|
use std::collections::VecDeque;
|
||||||
use std::fmt::Write;
|
use std::fmt::Write;
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
|
|
||||||
use futures::stream::*;
|
use futures::stream::*;
|
||||||
use hyper::{Body, Request, Response};
|
use hyper::{Body, Request, Response};
|
||||||
use md5::{digest::generic_array::*, Digest as Md5Digest, Md5};
|
|
||||||
use sha2::{Digest as Sha256Digest, Sha256};
|
|
||||||
|
|
||||||
use garage_table::*;
|
use garage_table::*;
|
||||||
use garage_util::data::*;
|
use garage_util::data::*;
|
||||||
use garage_util::error::Error as GarageError;
|
use garage_util::error::Error;
|
||||||
|
|
||||||
use crate::error::*;
|
|
||||||
use garage_model::block::INLINE_THRESHOLD;
|
use garage_model::block::INLINE_THRESHOLD;
|
||||||
use garage_model::block_ref_table::*;
|
use garage_model::block_ref_table::*;
|
||||||
use garage_model::garage::Garage;
|
use garage_model::garage::Garage;
|
||||||
@@ -19,68 +16,50 @@ use garage_model::object_table::*;
|
|||||||
use garage_model::version_table::*;
|
use garage_model::version_table::*;
|
||||||
|
|
||||||
use crate::encoding::*;
|
use crate::encoding::*;
|
||||||
|
use crate::http_util::*;
|
||||||
|
|
||||||
pub async fn handle_put(
|
pub async fn handle_put(
|
||||||
garage: Arc<Garage>,
|
garage: Arc<Garage>,
|
||||||
req: Request<Body>,
|
req: Request<Body>,
|
||||||
bucket: &str,
|
bucket: &str,
|
||||||
key: &str,
|
key: &str,
|
||||||
content_sha256: Option<Hash>,
|
) -> Result<Response<BodyType>, Error> {
|
||||||
) -> Result<Response<Body>, Error> {
|
|
||||||
let version_uuid = gen_uuid();
|
let version_uuid = gen_uuid();
|
||||||
let headers = get_headers(&req)?;
|
let mime_type = get_mime_type(&req)?;
|
||||||
let content_md5 = match req.headers().get("content-md5") {
|
|
||||||
Some(x) => Some(x.to_str()?.to_string()),
|
|
||||||
None => None,
|
|
||||||
};
|
|
||||||
|
|
||||||
let body = req.into_body();
|
let body = req.into_body();
|
||||||
|
|
||||||
let mut chunker = BodyChunker::new(body, garage.config.block_size);
|
let mut chunker = BodyChunker::new(body, garage.config.block_size);
|
||||||
let first_block = chunker.next().await?.unwrap_or(vec![]);
|
let first_block = match chunker.next().await? {
|
||||||
|
Some(x) => x,
|
||||||
|
None => vec![],
|
||||||
|
};
|
||||||
|
|
||||||
let mut object_version = ObjectVersion {
|
let mut object_version = ObjectVersion {
|
||||||
uuid: version_uuid,
|
uuid: version_uuid,
|
||||||
timestamp: now_msec(),
|
timestamp: now_msec(),
|
||||||
state: ObjectVersionState::Uploading(headers.clone()),
|
mime_type,
|
||||||
|
size: first_block.len() as u64,
|
||||||
|
state: ObjectVersionState::Uploading,
|
||||||
|
data: ObjectVersionData::Uploading,
|
||||||
};
|
};
|
||||||
|
|
||||||
if first_block.len() < INLINE_THRESHOLD {
|
if first_block.len() < INLINE_THRESHOLD {
|
||||||
let mut md5sum = Md5::new();
|
object_version.data = ObjectVersionData::Inline(first_block);
|
||||||
md5sum.update(&first_block[..]);
|
object_version.state = ObjectVersionState::Complete;
|
||||||
let md5sum_arr = md5sum.finalize();
|
|
||||||
let md5sum_hex = hex::encode(md5sum_arr);
|
|
||||||
|
|
||||||
let sha256sum_hash = hash(&first_block[..]);
|
|
||||||
|
|
||||||
ensure_checksum_matches(
|
|
||||||
md5sum_arr.as_slice(),
|
|
||||||
sha256sum_hash,
|
|
||||||
content_md5.as_deref(),
|
|
||||||
content_sha256,
|
|
||||||
)?;
|
|
||||||
|
|
||||||
object_version.state = ObjectVersionState::Complete(ObjectVersionData::Inline(
|
|
||||||
ObjectVersionMeta {
|
|
||||||
headers,
|
|
||||||
size: first_block.len() as u64,
|
|
||||||
etag: md5sum_hex.clone(),
|
|
||||||
},
|
|
||||||
first_block,
|
|
||||||
));
|
|
||||||
|
|
||||||
let object = Object::new(bucket.into(), key.into(), vec![object_version]);
|
let object = Object::new(bucket.into(), key.into(), vec![object_version]);
|
||||||
garage.object_table.insert(&object).await?;
|
garage.object_table.insert(&object).await?;
|
||||||
return Ok(put_response(version_uuid, md5sum_hex));
|
return Ok(put_response(version_uuid));
|
||||||
}
|
}
|
||||||
|
|
||||||
let version = Version::new(version_uuid, bucket.into(), key.into(), false, vec![]);
|
let version = Version::new(version_uuid, bucket.into(), key.into(), false, vec![]);
|
||||||
|
|
||||||
let first_block_hash = hash(&first_block[..]);
|
let first_block_hash = hash(&first_block[..]);
|
||||||
|
object_version.data = ObjectVersionData::FirstBlock(first_block_hash);
|
||||||
let object = Object::new(bucket.into(), key.into(), vec![object_version.clone()]);
|
let object = Object::new(bucket.into(), key.into(), vec![object_version.clone()]);
|
||||||
garage.object_table.insert(&object).await?;
|
garage.object_table.insert(&object).await?;
|
||||||
|
|
||||||
let (total_size, md5sum_arr, sha256sum) = read_and_put_blocks(
|
let total_size = read_and_put_blocks(
|
||||||
&garage,
|
&garage,
|
||||||
version,
|
version,
|
||||||
1,
|
1,
|
||||||
@@ -90,57 +69,15 @@ pub async fn handle_put(
|
|||||||
)
|
)
|
||||||
.await?;
|
.await?;
|
||||||
|
|
||||||
ensure_checksum_matches(
|
|
||||||
md5sum_arr.as_slice(),
|
|
||||||
sha256sum,
|
|
||||||
content_md5.as_deref(),
|
|
||||||
content_sha256,
|
|
||||||
)?;
|
|
||||||
|
|
||||||
// TODO: if at any step we have an error, we should undo everything we did
|
// TODO: if at any step we have an error, we should undo everything we did
|
||||||
|
|
||||||
let md5sum_hex = hex::encode(md5sum_arr);
|
object_version.state = ObjectVersionState::Complete;
|
||||||
|
object_version.size = total_size;
|
||||||
object_version.state = ObjectVersionState::Complete(ObjectVersionData::FirstBlock(
|
|
||||||
ObjectVersionMeta {
|
|
||||||
headers,
|
|
||||||
size: total_size,
|
|
||||||
etag: md5sum_hex.clone(),
|
|
||||||
},
|
|
||||||
first_block_hash,
|
|
||||||
));
|
|
||||||
|
|
||||||
let object = Object::new(bucket.into(), key.into(), vec![object_version]);
|
let object = Object::new(bucket.into(), key.into(), vec![object_version]);
|
||||||
garage.object_table.insert(&object).await?;
|
garage.object_table.insert(&object).await?;
|
||||||
|
|
||||||
Ok(put_response(version_uuid, md5sum_hex))
|
Ok(put_response(version_uuid))
|
||||||
}
|
|
||||||
|
|
||||||
/// Validate MD5 sum against content-md5 header
|
|
||||||
/// and sha256sum against signed content-sha256
|
|
||||||
fn ensure_checksum_matches(
|
|
||||||
md5sum: &[u8],
|
|
||||||
sha256sum: garage_util::data::FixedBytes32,
|
|
||||||
content_md5: Option<&str>,
|
|
||||||
content_sha256: Option<garage_util::data::FixedBytes32>,
|
|
||||||
) -> Result<(), Error> {
|
|
||||||
if let Some(expected_sha256) = content_sha256 {
|
|
||||||
if expected_sha256 != sha256sum {
|
|
||||||
return Err(Error::BadRequest(format!(
|
|
||||||
"Unable to validate x-amz-content-sha256"
|
|
||||||
)));
|
|
||||||
} else {
|
|
||||||
trace!("Successfully validated x-amz-content-sha256");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if let Some(expected_md5) = content_md5 {
|
|
||||||
if expected_md5.trim_matches('"') != base64::encode(md5sum) {
|
|
||||||
return Err(Error::BadRequest(format!("Unable to validate content-md5")));
|
|
||||||
} else {
|
|
||||||
trace!("Successfully validated content-md5");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Ok(())
|
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn read_and_put_blocks(
|
async fn read_and_put_blocks(
|
||||||
@@ -150,12 +87,7 @@ async fn read_and_put_blocks(
|
|||||||
first_block: Vec<u8>,
|
first_block: Vec<u8>,
|
||||||
first_block_hash: Hash,
|
first_block_hash: Hash,
|
||||||
chunker: &mut BodyChunker,
|
chunker: &mut BodyChunker,
|
||||||
) -> Result<(u64, GenericArray<u8, typenum::U16>, Hash), Error> {
|
) -> Result<u64, Error> {
|
||||||
let mut md5sum = Md5::new();
|
|
||||||
let mut sha256sum = Sha256::new();
|
|
||||||
md5sum.update(&first_block[..]);
|
|
||||||
sha256sum.input(&first_block[..]);
|
|
||||||
|
|
||||||
let mut next_offset = first_block.len();
|
let mut next_offset = first_block.len();
|
||||||
let mut put_curr_version_block = put_block_meta(
|
let mut put_curr_version_block = put_block_meta(
|
||||||
garage.clone(),
|
garage.clone(),
|
||||||
@@ -173,8 +105,6 @@ async fn read_and_put_blocks(
|
|||||||
let (_, _, next_block) =
|
let (_, _, next_block) =
|
||||||
futures::try_join!(put_curr_block, put_curr_version_block, chunker.next())?;
|
futures::try_join!(put_curr_block, put_curr_version_block, chunker.next())?;
|
||||||
if let Some(block) = next_block {
|
if let Some(block) = next_block {
|
||||||
md5sum.update(&block[..]);
|
|
||||||
sha256sum.input(&block[..]);
|
|
||||||
let block_hash = hash(&block[..]);
|
let block_hash = hash(&block[..]);
|
||||||
let block_len = block.len();
|
let block_len = block.len();
|
||||||
put_curr_version_block = put_block_meta(
|
put_curr_version_block = put_block_meta(
|
||||||
@@ -192,15 +122,7 @@ async fn read_and_put_blocks(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
let total_size = next_offset as u64;
|
Ok(next_offset as u64)
|
||||||
let md5sum_arr = md5sum.finalize();
|
|
||||||
|
|
||||||
let sha256sum = sha256sum.result();
|
|
||||||
let mut hash = [0u8; 32];
|
|
||||||
hash.copy_from_slice(&sha256sum[..]);
|
|
||||||
let sha256sum = Hash::from(hash);
|
|
||||||
|
|
||||||
Ok((total_size, md5sum_arr, sha256sum))
|
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn put_block_meta(
|
async fn put_block_meta(
|
||||||
@@ -210,7 +132,7 @@ async fn put_block_meta(
|
|||||||
offset: u64,
|
offset: u64,
|
||||||
hash: Hash,
|
hash: Hash,
|
||||||
size: u64,
|
size: u64,
|
||||||
) -> Result<(), GarageError> {
|
) -> Result<(), Error> {
|
||||||
// TODO: don't clone, restart from empty block list ??
|
// TODO: don't clone, restart from empty block list ??
|
||||||
let mut version = version.clone();
|
let mut version = version.clone();
|
||||||
version
|
version
|
||||||
@@ -248,10 +170,10 @@ impl BodyChunker {
|
|||||||
body,
|
body,
|
||||||
read_all: false,
|
read_all: false,
|
||||||
block_size,
|
block_size,
|
||||||
buf: VecDeque::with_capacity(2 * block_size),
|
buf: VecDeque::new(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
async fn next(&mut self) -> Result<Option<Vec<u8>>, GarageError> {
|
async fn next(&mut self) -> Result<Option<Vec<u8>>, Error> {
|
||||||
while !self.read_all && self.buf.len() < self.block_size {
|
while !self.read_all && self.buf.len() < self.block_size {
|
||||||
if let Some(block) = self.body.next().await {
|
if let Some(block) = self.body.next().await {
|
||||||
let bytes = block?;
|
let bytes = block?;
|
||||||
@@ -273,11 +195,10 @@ impl BodyChunker {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn put_response(version_uuid: UUID, md5sum_hex: String) -> Response<Body> {
|
pub fn put_response(version_uuid: UUID) -> Response<BodyType> {
|
||||||
Response::builder()
|
Response::builder()
|
||||||
.header("x-amz-version-id", hex::encode(version_uuid))
|
.header("x-amz-version-id", hex::encode(version_uuid))
|
||||||
.header("ETag", format!("\"{}\"", md5sum_hex))
|
.body(empty_body())
|
||||||
.body(Body::from(vec![]))
|
|
||||||
.unwrap()
|
.unwrap()
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -286,14 +207,17 @@ pub async fn handle_create_multipart_upload(
|
|||||||
req: &Request<Body>,
|
req: &Request<Body>,
|
||||||
bucket: &str,
|
bucket: &str,
|
||||||
key: &str,
|
key: &str,
|
||||||
) -> Result<Response<Body>, Error> {
|
) -> Result<Response<BodyType>, Error> {
|
||||||
let version_uuid = gen_uuid();
|
let version_uuid = gen_uuid();
|
||||||
let headers = get_headers(req)?;
|
let mime_type = get_mime_type(req)?;
|
||||||
|
|
||||||
let object_version = ObjectVersion {
|
let object_version = ObjectVersion {
|
||||||
uuid: version_uuid,
|
uuid: version_uuid,
|
||||||
timestamp: now_msec(),
|
timestamp: now_msec(),
|
||||||
state: ObjectVersionState::Uploading(headers),
|
mime_type,
|
||||||
|
size: 0,
|
||||||
|
state: ObjectVersionState::Uploading,
|
||||||
|
data: ObjectVersionData::Uploading,
|
||||||
};
|
};
|
||||||
let object = Object::new(bucket.to_string(), key.to_string(), vec![object_version]);
|
let object = Object::new(bucket.to_string(), key.to_string(), vec![object_version]);
|
||||||
garage.object_table.insert(&object).await?;
|
garage.object_table.insert(&object).await?;
|
||||||
@@ -315,7 +239,7 @@ pub async fn handle_create_multipart_upload(
|
|||||||
.unwrap();
|
.unwrap();
|
||||||
writeln!(&mut xml, "</InitiateMultipartUploadResult>").unwrap();
|
writeln!(&mut xml, "</InitiateMultipartUploadResult>").unwrap();
|
||||||
|
|
||||||
Ok(Response::new(Body::from(xml.into_bytes())))
|
Ok(Response::new(Box::new(BytesBody::from(xml.into_bytes()))))
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn handle_put_part(
|
pub async fn handle_put_part(
|
||||||
@@ -325,19 +249,14 @@ pub async fn handle_put_part(
|
|||||||
key: &str,
|
key: &str,
|
||||||
part_number_str: &str,
|
part_number_str: &str,
|
||||||
upload_id: &str,
|
upload_id: &str,
|
||||||
content_sha256: Option<Hash>,
|
) -> Result<Response<BodyType>, Error> {
|
||||||
) -> Result<Response<Body>, Error> {
|
|
||||||
// Check parameters
|
// Check parameters
|
||||||
let part_number = part_number_str
|
let part_number = part_number_str
|
||||||
.parse::<u64>()
|
.parse::<u64>()
|
||||||
.ok_or_bad_request("Invalid part number")?;
|
.map_err(|e| Error::BadRequest(format!("Invalid part number: {}", e)))?;
|
||||||
|
|
||||||
let version_uuid = decode_upload_id(upload_id)?;
|
let version_uuid =
|
||||||
|
uuid_from_str(upload_id).map_err(|_| Error::BadRequest(format!("Invalid upload ID")))?;
|
||||||
let content_md5 = match req.headers().get("content-md5") {
|
|
||||||
Some(x) => Some(x.to_str()?.to_string()),
|
|
||||||
None => None,
|
|
||||||
};
|
|
||||||
|
|
||||||
// Read first chuck, and at the same time try to get object to see if it exists
|
// Read first chuck, and at the same time try to get object to see if it exists
|
||||||
let mut chunker = BodyChunker::new(req.into_body(), garage.config.block_size);
|
let mut chunker = BodyChunker::new(req.into_body(), garage.config.block_size);
|
||||||
@@ -349,23 +268,28 @@ pub async fn handle_put_part(
|
|||||||
let (object, first_block) = futures::try_join!(get_object_fut, get_first_block_fut)?;
|
let (object, first_block) = futures::try_join!(get_object_fut, get_first_block_fut)?;
|
||||||
|
|
||||||
// Check object is valid and multipart block can be accepted
|
// Check object is valid and multipart block can be accepted
|
||||||
let first_block = first_block.ok_or(Error::BadRequest(format!("Empty body")))?;
|
let first_block = match first_block {
|
||||||
let object = object.ok_or(Error::BadRequest(format!("Object not found")))?;
|
None => return Err(Error::BadRequest(format!("Empty body"))),
|
||||||
|
Some(x) => x,
|
||||||
if !object
|
};
|
||||||
.versions()
|
let object = match object {
|
||||||
.iter()
|
None => return Err(Error::BadRequest(format!("Object not found"))),
|
||||||
.any(|v| v.uuid == version_uuid && v.is_uploading())
|
Some(x) => x,
|
||||||
{
|
};
|
||||||
|
if !object.versions().iter().any(|v| {
|
||||||
|
v.uuid == version_uuid
|
||||||
|
&& v.state == ObjectVersionState::Uploading
|
||||||
|
&& v.data == ObjectVersionData::Uploading
|
||||||
|
}) {
|
||||||
return Err(Error::BadRequest(format!(
|
return Err(Error::BadRequest(format!(
|
||||||
"Multipart upload does not exist or is otherwise invalid"
|
"Multipart upload does not exist or is otherwise invalid"
|
||||||
)));
|
)));
|
||||||
}
|
}
|
||||||
|
|
||||||
// Copy block to store
|
// Copy block to store
|
||||||
let version = Version::new(version_uuid, bucket, key, false, vec![]);
|
let version = Version::new(version_uuid, bucket.into(), key.into(), false, vec![]);
|
||||||
let first_block_hash = hash(&first_block[..]);
|
let first_block_hash = hash(&first_block[..]);
|
||||||
let (_, md5sum_arr, sha256sum) = read_and_put_blocks(
|
read_and_put_blocks(
|
||||||
&garage,
|
&garage,
|
||||||
version,
|
version,
|
||||||
part_number,
|
part_number,
|
||||||
@@ -375,18 +299,7 @@ pub async fn handle_put_part(
|
|||||||
)
|
)
|
||||||
.await?;
|
.await?;
|
||||||
|
|
||||||
ensure_checksum_matches(
|
Ok(Response::new(Box::new(BytesBody::from(vec![]))))
|
||||||
md5sum_arr.as_slice(),
|
|
||||||
sha256sum,
|
|
||||||
content_md5.as_deref(),
|
|
||||||
content_sha256,
|
|
||||||
)?;
|
|
||||||
|
|
||||||
let response = Response::builder()
|
|
||||||
.header("ETag", format!("\"{}\"", hex::encode(md5sum_arr)))
|
|
||||||
.body(Body::from(vec![]))
|
|
||||||
.unwrap();
|
|
||||||
Ok(response)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn handle_complete_multipart_upload(
|
pub async fn handle_complete_multipart_upload(
|
||||||
@@ -395,8 +308,9 @@ pub async fn handle_complete_multipart_upload(
|
|||||||
bucket: &str,
|
bucket: &str,
|
||||||
key: &str,
|
key: &str,
|
||||||
upload_id: &str,
|
upload_id: &str,
|
||||||
) -> Result<Response<Body>, Error> {
|
) -> Result<Response<BodyType>, Error> {
|
||||||
let version_uuid = decode_upload_id(upload_id)?;
|
let version_uuid =
|
||||||
|
uuid_from_str(upload_id).map_err(|_| Error::BadRequest(format!("Invalid upload ID")))?;
|
||||||
|
|
||||||
let bucket = bucket.to_string();
|
let bucket = bucket.to_string();
|
||||||
let key = key.to_string();
|
let key = key.to_string();
|
||||||
@@ -404,12 +318,15 @@ pub async fn handle_complete_multipart_upload(
|
|||||||
garage.object_table.get(&bucket, &key),
|
garage.object_table.get(&bucket, &key),
|
||||||
garage.version_table.get(&version_uuid, &EmptyKey),
|
garage.version_table.get(&version_uuid, &EmptyKey),
|
||||||
)?;
|
)?;
|
||||||
let object = object.ok_or(Error::BadRequest(format!("Object not found")))?;
|
let object = match object {
|
||||||
|
None => return Err(Error::BadRequest(format!("Object not found"))),
|
||||||
let object_version = object
|
Some(x) => x,
|
||||||
.versions()
|
};
|
||||||
.iter()
|
let object_version = object.versions().iter().find(|v| {
|
||||||
.find(|v| v.uuid == version_uuid && v.is_uploading());
|
v.uuid == version_uuid
|
||||||
|
&& v.state == ObjectVersionState::Uploading
|
||||||
|
&& v.data == ObjectVersionData::Uploading
|
||||||
|
});
|
||||||
let mut object_version = match object_version {
|
let mut object_version = match object_version {
|
||||||
None => {
|
None => {
|
||||||
return Err(Error::BadRequest(format!(
|
return Err(Error::BadRequest(format!(
|
||||||
@@ -418,34 +335,15 @@ pub async fn handle_complete_multipart_upload(
|
|||||||
}
|
}
|
||||||
Some(x) => x.clone(),
|
Some(x) => x.clone(),
|
||||||
};
|
};
|
||||||
let version = version.ok_or(Error::BadRequest(format!("Version not found")))?;
|
let version = match version {
|
||||||
|
None => return Err(Error::BadRequest(format!("Version not found"))),
|
||||||
|
Some(x) => x,
|
||||||
|
};
|
||||||
if version.blocks().len() == 0 {
|
if version.blocks().len() == 0 {
|
||||||
return Err(Error::BadRequest(format!("No data was uploaded")));
|
return Err(Error::BadRequest(format!("No data was uploaded")));
|
||||||
}
|
}
|
||||||
let headers = match object_version.state {
|
|
||||||
ObjectVersionState::Uploading(headers) => headers.clone(),
|
|
||||||
_ => unreachable!(),
|
|
||||||
};
|
|
||||||
|
|
||||||
// ETag calculation: we produce ETags that have the same form as
|
|
||||||
// those of S3 multipart uploads, but we don't use their actual
|
|
||||||
// calculation for the first part (we use random bytes). This
|
|
||||||
// shouldn't impact compatibility as the S3 docs specify that
|
|
||||||
// the ETag is an opaque value in case of a multipart upload.
|
|
||||||
// See also: https://teppen.io/2018/06/23/aws_s3_etags/
|
|
||||||
let num_parts = version.blocks().last().unwrap().part_number
|
|
||||||
- version.blocks().first().unwrap().part_number
|
|
||||||
+ 1;
|
|
||||||
let etag = format!(
|
|
||||||
"{}-{}",
|
|
||||||
hex::encode(&rand::random::<[u8; 16]>()[..]),
|
|
||||||
num_parts
|
|
||||||
);
|
|
||||||
|
|
||||||
// TODO: check that all the parts that they pretend they gave us are indeed there
|
// TODO: check that all the parts that they pretend they gave us are indeed there
|
||||||
// TODO: when we read the XML from _req, remember to check the sha256 sum of the payload
|
|
||||||
// against the signed x-amz-content-sha256
|
|
||||||
// TODO: check MD5 sum of all uploaded parts? but that would mean we have to store them somewhere...
|
// TODO: check MD5 sum of all uploaded parts? but that would mean we have to store them somewhere...
|
||||||
|
|
||||||
let total_size = version
|
let total_size = version
|
||||||
@@ -453,15 +351,9 @@ pub async fn handle_complete_multipart_upload(
|
|||||||
.iter()
|
.iter()
|
||||||
.map(|x| x.size)
|
.map(|x| x.size)
|
||||||
.fold(0, |x, y| x + y);
|
.fold(0, |x, y| x + y);
|
||||||
object_version.state = ObjectVersionState::Complete(ObjectVersionData::FirstBlock(
|
object_version.size = total_size;
|
||||||
ObjectVersionMeta {
|
object_version.state = ObjectVersionState::Complete;
|
||||||
headers,
|
object_version.data = ObjectVersionData::FirstBlock(version.blocks()[0].hash);
|
||||||
size: total_size,
|
|
||||||
etag: etag,
|
|
||||||
},
|
|
||||||
version.blocks()[0].hash,
|
|
||||||
));
|
|
||||||
|
|
||||||
let final_object = Object::new(bucket.clone(), key.clone(), vec![object_version]);
|
let final_object = Object::new(bucket.clone(), key.clone(), vec![object_version]);
|
||||||
garage.object_table.insert(&final_object).await?;
|
garage.object_table.insert(&final_object).await?;
|
||||||
|
|
||||||
@@ -482,7 +374,7 @@ pub async fn handle_complete_multipart_upload(
|
|||||||
writeln!(&mut xml, "\t<Key>{}</Key>", xml_escape(&key)).unwrap();
|
writeln!(&mut xml, "\t<Key>{}</Key>", xml_escape(&key)).unwrap();
|
||||||
writeln!(&mut xml, "</CompleteMultipartUploadResult>").unwrap();
|
writeln!(&mut xml, "</CompleteMultipartUploadResult>").unwrap();
|
||||||
|
|
||||||
Ok(Response::new(Body::from(xml.into_bytes())))
|
Ok(Response::new(Box::new(BytesBody::from(xml.into_bytes()))))
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn handle_abort_multipart_upload(
|
pub async fn handle_abort_multipart_upload(
|
||||||
@@ -490,19 +382,23 @@ pub async fn handle_abort_multipart_upload(
|
|||||||
bucket: &str,
|
bucket: &str,
|
||||||
key: &str,
|
key: &str,
|
||||||
upload_id: &str,
|
upload_id: &str,
|
||||||
) -> Result<Response<Body>, Error> {
|
) -> Result<Response<BodyType>, Error> {
|
||||||
let version_uuid = decode_upload_id(upload_id)?;
|
let version_uuid =
|
||||||
|
uuid_from_str(upload_id).map_err(|_| Error::BadRequest(format!("Invalid upload ID")))?;
|
||||||
|
|
||||||
let object = garage
|
let object = garage
|
||||||
.object_table
|
.object_table
|
||||||
.get(&bucket.to_string(), &key.to_string())
|
.get(&bucket.to_string(), &key.to_string())
|
||||||
.await?;
|
.await?;
|
||||||
let object = object.ok_or(Error::BadRequest(format!("Object not found")))?;
|
let object = match object {
|
||||||
|
None => return Err(Error::BadRequest(format!("Object not found"))),
|
||||||
let object_version = object
|
Some(x) => x,
|
||||||
.versions()
|
};
|
||||||
.iter()
|
let object_version = object.versions().iter().find(|v| {
|
||||||
.find(|v| v.uuid == version_uuid && v.is_uploading());
|
v.uuid == version_uuid
|
||||||
|
&& v.state == ObjectVersionState::Uploading
|
||||||
|
&& v.data == ObjectVersionData::Uploading
|
||||||
|
});
|
||||||
let mut object_version = match object_version {
|
let mut object_version = match object_version {
|
||||||
None => {
|
None => {
|
||||||
return Err(Error::BadRequest(format!(
|
return Err(Error::BadRequest(format!(
|
||||||
@@ -516,7 +412,7 @@ pub async fn handle_abort_multipart_upload(
|
|||||||
let final_object = Object::new(bucket.to_string(), key.to_string(), vec![object_version]);
|
let final_object = Object::new(bucket.to_string(), key.to_string(), vec![object_version]);
|
||||||
garage.object_table.insert(&final_object).await?;
|
garage.object_table.insert(&final_object).await?;
|
||||||
|
|
||||||
Ok(Response::new(Body::from(vec![])))
|
Ok(Response::new(Box::new(BytesBody::from(vec![]))))
|
||||||
}
|
}
|
||||||
|
|
||||||
fn get_mime_type(req: &Request<Body>) -> Result<String, Error> {
|
fn get_mime_type(req: &Request<Body>) -> Result<String, Error> {
|
||||||
@@ -528,33 +424,10 @@ fn get_mime_type(req: &Request<Body>) -> Result<String, Error> {
|
|||||||
.to_string())
|
.to_string())
|
||||||
}
|
}
|
||||||
|
|
||||||
fn get_headers(req: &Request<Body>) -> Result<ObjectVersionHeaders, Error> {
|
fn uuid_from_str(id: &str) -> Result<UUID, ()> {
|
||||||
let content_type = get_mime_type(req)?;
|
let id_bin = hex::decode(id).map_err(|_| ())?;
|
||||||
let other_headers = vec![
|
|
||||||
hyper::header::CACHE_CONTROL,
|
|
||||||
hyper::header::CONTENT_DISPOSITION,
|
|
||||||
hyper::header::CONTENT_ENCODING,
|
|
||||||
hyper::header::CONTENT_LANGUAGE,
|
|
||||||
hyper::header::EXPIRES,
|
|
||||||
];
|
|
||||||
let mut other = BTreeMap::new();
|
|
||||||
for h in other_headers.iter() {
|
|
||||||
if let Some(v) = req.headers().get(h) {
|
|
||||||
if let Ok(v_str) = v.to_str() {
|
|
||||||
other.insert(h.to_string(), v_str.to_string());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Ok(ObjectVersionHeaders {
|
|
||||||
content_type,
|
|
||||||
other: BTreeMap::new(),
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
fn decode_upload_id(id: &str) -> Result<UUID, Error> {
|
|
||||||
let id_bin = hex::decode(id).ok_or_bad_request("Invalid upload ID")?;
|
|
||||||
if id_bin.len() != 32 {
|
if id_bin.len() != 32 {
|
||||||
return None.ok_or_bad_request("Invalid upload ID");
|
return Err(());
|
||||||
}
|
}
|
||||||
let mut uuid = [0u8; 32];
|
let mut uuid = [0u8; 32];
|
||||||
uuid.copy_from_slice(&id_bin[..]);
|
uuid.copy_from_slice(&id_bin[..]);
|
||||||
|
|||||||
+40
-47
@@ -6,23 +6,19 @@ use hyper::{Body, Method, Request};
|
|||||||
use sha2::{Digest, Sha256};
|
use sha2::{Digest, Sha256};
|
||||||
|
|
||||||
use garage_table::*;
|
use garage_table::*;
|
||||||
use garage_util::data::Hash;
|
use garage_util::error::Error;
|
||||||
|
|
||||||
use garage_model::garage::Garage;
|
use garage_model::garage::Garage;
|
||||||
use garage_model::key_table::*;
|
use garage_model::key_table::*;
|
||||||
|
|
||||||
use crate::encoding::uri_encode;
|
use crate::encoding::uri_encode;
|
||||||
use crate::error::*;
|
|
||||||
|
|
||||||
const SHORT_DATE: &str = "%Y%m%d";
|
const SHORT_DATE: &str = "%Y%m%d";
|
||||||
const LONG_DATETIME: &str = "%Y%m%dT%H%M%SZ";
|
const LONG_DATETIME: &str = "%Y%m%dT%H%M%SZ";
|
||||||
|
|
||||||
type HmacSha256 = Hmac<Sha256>;
|
type HmacSha256 = Hmac<Sha256>;
|
||||||
|
|
||||||
pub async fn check_signature(
|
pub async fn check_signature(garage: &Garage, request: &Request<Body>) -> Result<Key, Error> {
|
||||||
garage: &Garage,
|
|
||||||
request: &Request<Body>,
|
|
||||||
) -> Result<(Key, Option<Hash>), Error> {
|
|
||||||
let mut headers = HashMap::new();
|
let mut headers = HashMap::new();
|
||||||
for (key, val) in request.headers() {
|
for (key, val) in request.headers() {
|
||||||
headers.insert(key.to_string(), val.to_str()?.to_string());
|
headers.insert(key.to_string(), val.to_str()?.to_string());
|
||||||
@@ -42,9 +38,9 @@ pub async fn check_signature(
|
|||||||
|
|
||||||
let date = headers
|
let date = headers
|
||||||
.get("x-amz-date")
|
.get("x-amz-date")
|
||||||
.ok_or_bad_request("Missing X-Amz-Date field")?;
|
.ok_or(Error::BadRequest("Missing X-Amz-Date field".into()))?;
|
||||||
let date: NaiveDateTime = NaiveDateTime::parse_from_str(date, LONG_DATETIME)
|
let date: NaiveDateTime = NaiveDateTime::parse_from_str(date, LONG_DATETIME)
|
||||||
.ok_or_bad_request("Invalid date")?
|
.map_err(|e| Error::BadRequest(format!("Invalid date: {}", e)))?
|
||||||
.into();
|
.into();
|
||||||
let date: DateTime<Utc> = DateTime::from_utc(date, Utc);
|
let date: DateTime<Utc> = DateTime::from_utc(date, Utc);
|
||||||
|
|
||||||
@@ -68,7 +64,7 @@ pub async fn check_signature(
|
|||||||
.key_table
|
.key_table
|
||||||
.get(&EmptyKey, &authorization.key_id)
|
.get(&EmptyKey, &authorization.key_id)
|
||||||
.await?
|
.await?
|
||||||
.filter(|k| !k.deleted.get())
|
.filter(|k| !k.deleted)
|
||||||
.ok_or(Error::Forbidden(format!(
|
.ok_or(Error::Forbidden(format!(
|
||||||
"No such key: {}",
|
"No such key: {}",
|
||||||
authorization.key_id
|
authorization.key_id
|
||||||
@@ -90,31 +86,15 @@ pub async fn check_signature(
|
|||||||
&garage.config.s3_api.s3_region,
|
&garage.config.s3_api.s3_region,
|
||||||
"s3",
|
"s3",
|
||||||
)
|
)
|
||||||
.ok_or_internal_error("Unable to build signing HMAC")?;
|
.map_err(|e| Error::Message(format!("Unable to build signing HMAC: {}", e)))?;
|
||||||
hmac.input(string_to_sign.as_bytes());
|
hmac.input(string_to_sign.as_bytes());
|
||||||
let signature = hex::encode(hmac.result().code());
|
let signature = hex::encode(hmac.result().code());
|
||||||
|
|
||||||
if authorization.signature != signature {
|
if authorization.signature != signature {
|
||||||
trace!("Canonical request: ``{}``", canonical_request);
|
|
||||||
trace!("String to sign: ``{}``", string_to_sign);
|
|
||||||
trace!("Expected: {}, got: {}", signature, authorization.signature);
|
|
||||||
return Err(Error::Forbidden(format!("Invalid signature")));
|
return Err(Error::Forbidden(format!("Invalid signature")));
|
||||||
}
|
}
|
||||||
|
|
||||||
let content_sha256 = if authorization.content_sha256 == "UNSIGNED-PAYLOAD" {
|
Ok(key)
|
||||||
None
|
|
||||||
} else {
|
|
||||||
let bytes = hex::decode(authorization.content_sha256)
|
|
||||||
.ok_or_bad_request("Invalid content sha256 hash")?;
|
|
||||||
let mut hash = [0u8; 32];
|
|
||||||
if bytes.len() != 32 {
|
|
||||||
return Err(Error::BadRequest(format!("Invalid content sha256 hash")));
|
|
||||||
}
|
|
||||||
hash.copy_from_slice(&bytes[..]);
|
|
||||||
Some(Hash::from(hash))
|
|
||||||
};
|
|
||||||
|
|
||||||
Ok((key, content_sha256))
|
|
||||||
}
|
}
|
||||||
|
|
||||||
struct Authorization {
|
struct Authorization {
|
||||||
@@ -131,7 +111,7 @@ fn parse_authorization(
|
|||||||
) -> Result<Authorization, Error> {
|
) -> Result<Authorization, Error> {
|
||||||
let first_space = authorization
|
let first_space = authorization
|
||||||
.find(' ')
|
.find(' ')
|
||||||
.ok_or_bad_request("Authorization field to short")?;
|
.ok_or(Error::BadRequest("Authorization field too short".into()))?;
|
||||||
let (auth_kind, rest) = authorization.split_at(first_space);
|
let (auth_kind, rest) = authorization.split_at(first_space);
|
||||||
|
|
||||||
if auth_kind != "AWS4-HMAC-SHA256" {
|
if auth_kind != "AWS4-HMAC-SHA256" {
|
||||||
@@ -141,32 +121,41 @@ fn parse_authorization(
|
|||||||
let mut auth_params = HashMap::new();
|
let mut auth_params = HashMap::new();
|
||||||
for auth_part in rest.split(',') {
|
for auth_part in rest.split(',') {
|
||||||
let auth_part = auth_part.trim();
|
let auth_part = auth_part.trim();
|
||||||
let eq = auth_part
|
let eq = auth_part.find('=').ok_or(Error::BadRequest(format!(
|
||||||
.find('=')
|
"Missing =value in authorization field {}",
|
||||||
.ok_or_bad_request("Field without value in authorization header")?;
|
auth_part
|
||||||
|
)))?;
|
||||||
let (key, value) = auth_part.split_at(eq);
|
let (key, value) = auth_part.split_at(eq);
|
||||||
auth_params.insert(key.to_string(), value.trim_start_matches('=').to_string());
|
auth_params.insert(key.to_string(), value.trim_start_matches('=').to_string());
|
||||||
}
|
}
|
||||||
|
|
||||||
let cred = auth_params
|
let cred = auth_params
|
||||||
.get("Credential")
|
.get("Credential")
|
||||||
.ok_or_bad_request("Could not find Credential in Authorization field")?;
|
.ok_or(Error::BadRequest(format!(
|
||||||
|
"Could not find Credential in Authorization field"
|
||||||
|
)))?;
|
||||||
let (key_id, scope) = parse_credential(cred)?;
|
let (key_id, scope) = parse_credential(cred)?;
|
||||||
|
|
||||||
let content_sha256 = headers
|
let content_sha256 = headers
|
||||||
.get("x-amz-content-sha256")
|
.get("x-amz-content-sha256")
|
||||||
.ok_or_bad_request("Missing X-Amz-Content-Sha256 field")?;
|
.ok_or(Error::BadRequest(
|
||||||
|
"Missing X-Amz-Content-Sha256 field".into(),
|
||||||
|
))?;
|
||||||
|
|
||||||
let auth = Authorization {
|
let auth = Authorization {
|
||||||
key_id,
|
key_id,
|
||||||
scope,
|
scope,
|
||||||
signed_headers: auth_params
|
signed_headers: auth_params
|
||||||
.get("SignedHeaders")
|
.get("SignedHeaders")
|
||||||
.ok_or_bad_request("Could not find SignedHeaders in Authorization field")?
|
.ok_or(Error::BadRequest(format!(
|
||||||
|
"Could not find SignedHeaders in Authorization field"
|
||||||
|
)))?
|
||||||
.to_string(),
|
.to_string(),
|
||||||
signature: auth_params
|
signature: auth_params
|
||||||
.get("Signature")
|
.get("Signature")
|
||||||
.ok_or_bad_request("Could not find Signature in Authorization field")?
|
.ok_or(Error::BadRequest(format!(
|
||||||
|
"Could not find Signature in Authorization field"
|
||||||
|
)))?
|
||||||
.to_string(),
|
.to_string(),
|
||||||
content_sha256: content_sha256.to_string(),
|
content_sha256: content_sha256.to_string(),
|
||||||
};
|
};
|
||||||
@@ -176,7 +165,9 @@ fn parse_authorization(
|
|||||||
fn parse_query_authorization(headers: &HashMap<String, String>) -> Result<Authorization, Error> {
|
fn parse_query_authorization(headers: &HashMap<String, String>) -> Result<Authorization, Error> {
|
||||||
let algo = headers
|
let algo = headers
|
||||||
.get("x-amz-algorithm")
|
.get("x-amz-algorithm")
|
||||||
.ok_or_bad_request("X-Amz-Algorithm not found in query parameters")?;
|
.ok_or(Error::BadRequest(format!(
|
||||||
|
"X-Amz-Algorithm not found in query parameters"
|
||||||
|
)))?;
|
||||||
if algo != "AWS4-HMAC-SHA256" {
|
if algo != "AWS4-HMAC-SHA256" {
|
||||||
return Err(Error::BadRequest(format!(
|
return Err(Error::BadRequest(format!(
|
||||||
"Unsupported authorization method"
|
"Unsupported authorization method"
|
||||||
@@ -185,32 +176,34 @@ fn parse_query_authorization(headers: &HashMap<String, String>) -> Result<Author
|
|||||||
|
|
||||||
let cred = headers
|
let cred = headers
|
||||||
.get("x-amz-credential")
|
.get("x-amz-credential")
|
||||||
.ok_or_bad_request("X-Amz-Credential not found in query parameters")?;
|
.ok_or(Error::BadRequest(format!(
|
||||||
|
"X-Amz-Credential not found in query parameters"
|
||||||
|
)))?;
|
||||||
let (key_id, scope) = parse_credential(cred)?;
|
let (key_id, scope) = parse_credential(cred)?;
|
||||||
let signed_headers = headers
|
let signed_headers = headers
|
||||||
.get("x-amz-signedheaders")
|
.get("x-amz-signedheaders")
|
||||||
.ok_or_bad_request("X-Amz-SignedHeaders not found in query parameters")?;
|
.ok_or(Error::BadRequest(format!(
|
||||||
|
"X-Amz-SignedHeaders not found in query parameters"
|
||||||
|
)))?;
|
||||||
let signature = headers
|
let signature = headers
|
||||||
.get("x-amz-signature")
|
.get("x-amz-signature")
|
||||||
.ok_or_bad_request("X-Amz-Signature not found in query parameters")?;
|
.ok_or(Error::BadRequest(format!(
|
||||||
let content_sha256 = headers
|
"X-Amz-Signature not found in query parameters"
|
||||||
.get("x-amz-content-sha256")
|
)))?;
|
||||||
.map(|x| x.as_str())
|
|
||||||
.unwrap_or("UNSIGNED-PAYLOAD");
|
|
||||||
|
|
||||||
Ok(Authorization {
|
Ok(Authorization {
|
||||||
key_id,
|
key_id,
|
||||||
scope,
|
scope,
|
||||||
signed_headers: signed_headers.to_string(),
|
signed_headers: signed_headers.to_string(),
|
||||||
signature: signature.to_string(),
|
signature: signature.to_string(),
|
||||||
content_sha256: content_sha256.to_string(),
|
content_sha256: "UNSIGNED-PAYLOAD".to_string(),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
fn parse_credential(cred: &str) -> Result<(String, String), Error> {
|
fn parse_credential(cred: &str) -> Result<(String, String), Error> {
|
||||||
let first_slash = cred
|
let first_slash = cred.find('/').ok_or(Error::BadRequest(format!(
|
||||||
.find('/')
|
"Credentials does not contain / in authorization field"
|
||||||
.ok_or_bad_request("Credentials does not contain / in authorization field")?;
|
)))?;
|
||||||
let (key_id, scope) = cred.split_at(first_slash);
|
let (key_id, scope) = cred.split_at(first_slash);
|
||||||
Ok((
|
Ok((
|
||||||
key_id.to_string(),
|
key_id.to_string(),
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
[package]
|
[package]
|
||||||
name = "garage"
|
name = "garage"
|
||||||
version = "0.1.1"
|
version = "0.1.0"
|
||||||
authors = ["Alex Auvolat <alex@adnab.me>"]
|
authors = ["Alex Auvolat <alex@adnab.me>"]
|
||||||
edition = "2018"
|
edition = "2018"
|
||||||
license = "GPL-3.0"
|
license = "GPL-3.0"
|
||||||
@@ -14,12 +14,11 @@ path = "main.rs"
|
|||||||
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
|
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
|
||||||
|
|
||||||
[dependencies]
|
[dependencies]
|
||||||
garage_util = { version = "0.1.1", path = "../util" }
|
garage_util = { version = "0.1", path = "../util" }
|
||||||
garage_rpc = { version = "0.1.1", path = "../rpc" }
|
garage_rpc = { version = "0.1", path = "../rpc" }
|
||||||
garage_table = { version = "0.1.1", path = "../table" }
|
garage_table = { version = "0.1", path = "../table" }
|
||||||
garage_model = { version = "0.1.1", path = "../model" }
|
garage_model = { version = "0.1", path = "../model" }
|
||||||
garage_api = { version = "0.1.1", path = "../api" }
|
garage_api = { version = "0.1", path = "../api" }
|
||||||
garage_web = { version = "0.1.1", path = "../web" }
|
|
||||||
|
|
||||||
bytes = "0.4"
|
bytes = "0.4"
|
||||||
rand = "0.7"
|
rand = "0.7"
|
||||||
|
|||||||
+87
-90
@@ -2,9 +2,9 @@ use std::sync::Arc;
|
|||||||
|
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
|
|
||||||
|
use garage_util::data::*;
|
||||||
use garage_util::error::Error;
|
use garage_util::error::Error;
|
||||||
|
|
||||||
use garage_table::crdt::CRDT;
|
|
||||||
use garage_table::*;
|
use garage_table::*;
|
||||||
|
|
||||||
use garage_rpc::rpc_client::*;
|
use garage_rpc::rpc_client::*;
|
||||||
@@ -55,7 +55,7 @@ impl AdminRpcHandler {
|
|||||||
AdminRPC::BucketOperation(bo) => self2.handle_bucket_cmd(bo).await,
|
AdminRPC::BucketOperation(bo) => self2.handle_bucket_cmd(bo).await,
|
||||||
AdminRPC::KeyOperation(ko) => self2.handle_key_cmd(ko).await,
|
AdminRPC::KeyOperation(ko) => self2.handle_key_cmd(ko).await,
|
||||||
AdminRPC::LaunchRepair(opt) => self2.handle_launch_repair(opt).await,
|
AdminRPC::LaunchRepair(opt) => self2.handle_launch_repair(opt).await,
|
||||||
_ => Err(Error::BadRPC(format!("Invalid RPC"))),
|
_ => Err(Error::BadRequest(format!("Invalid RPC"))),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
@@ -67,7 +67,7 @@ impl AdminRpcHandler {
|
|||||||
let bucket_names = self
|
let bucket_names = self
|
||||||
.garage
|
.garage
|
||||||
.bucket_table
|
.bucket_table
|
||||||
.get_range(&EmptyKey, None, Some(DeletedFilter::NotDeleted), 10000)
|
.get_range(&EmptyKey, None, Some(()), 10000)
|
||||||
.await?
|
.await?
|
||||||
.iter()
|
.iter()
|
||||||
.map(|b| b.name.to_string())
|
.map(|b| b.name.to_string())
|
||||||
@@ -79,52 +79,61 @@ impl AdminRpcHandler {
|
|||||||
Ok(AdminRPC::BucketInfo(bucket))
|
Ok(AdminRPC::BucketInfo(bucket))
|
||||||
}
|
}
|
||||||
BucketOperation::Create(query) => {
|
BucketOperation::Create(query) => {
|
||||||
let bucket = match self.garage.bucket_table.get(&EmptyKey, &query.name).await? {
|
let bucket = self.garage.bucket_table.get(&EmptyKey, &query.name).await?;
|
||||||
Some(mut bucket) => {
|
if bucket.as_ref().filter(|b| !b.deleted).is_some() {
|
||||||
if !bucket.is_deleted() {
|
return Err(Error::BadRequest(format!(
|
||||||
return Err(Error::BadRPC(format!(
|
"Bucket {} already exists",
|
||||||
"Bucket {} already exists",
|
query.name
|
||||||
query.name
|
)));
|
||||||
)));
|
}
|
||||||
}
|
let new_time = match bucket {
|
||||||
bucket
|
Some(b) => std::cmp::max(b.timestamp + 1, now_msec()),
|
||||||
.state
|
None => now_msec(),
|
||||||
.update(BucketState::Present(BucketParams::new()));
|
|
||||||
bucket
|
|
||||||
}
|
|
||||||
None => Bucket::new(query.name.clone()),
|
|
||||||
};
|
};
|
||||||
self.garage.bucket_table.insert(&bucket).await?;
|
self.garage
|
||||||
|
.bucket_table
|
||||||
|
.insert(&Bucket::new(query.name.clone(), new_time, false, vec![]))
|
||||||
|
.await?;
|
||||||
Ok(AdminRPC::Ok(format!("Bucket {} was created.", query.name)))
|
Ok(AdminRPC::Ok(format!("Bucket {} was created.", query.name)))
|
||||||
}
|
}
|
||||||
BucketOperation::Delete(query) => {
|
BucketOperation::Delete(query) => {
|
||||||
let mut bucket = self.get_existing_bucket(&query.name).await?;
|
let bucket = self.get_existing_bucket(&query.name).await?;
|
||||||
let objects = self
|
let objects = self
|
||||||
.garage
|
.garage
|
||||||
.object_table
|
.object_table
|
||||||
.get_range(&query.name, None, Some(DeletedFilter::NotDeleted), 10)
|
.get_range(&query.name, None, Some(()), 10)
|
||||||
.await?;
|
.await?;
|
||||||
if !objects.is_empty() {
|
if !objects.is_empty() {
|
||||||
return Err(Error::BadRPC(format!("Bucket {} is not empty", query.name)));
|
return Err(Error::BadRequest(format!(
|
||||||
|
"Bucket {} is not empty",
|
||||||
|
query.name
|
||||||
|
)));
|
||||||
}
|
}
|
||||||
if !query.yes {
|
if !query.yes {
|
||||||
return Err(Error::BadRPC(format!(
|
return Err(Error::BadRequest(format!(
|
||||||
"Add --yes flag to really perform this operation"
|
"Add --yes flag to really perform this operation"
|
||||||
)));
|
)));
|
||||||
}
|
}
|
||||||
// --- done checking, now commit ---
|
// --- done checking, now commit ---
|
||||||
for (key_id, _, _) in bucket.authorized_keys() {
|
for ak in bucket.authorized_keys() {
|
||||||
if let Some(key) = self.garage.key_table.get(&EmptyKey, key_id).await? {
|
if let Some(key) = self.garage.key_table.get(&EmptyKey, &ak.key_id).await? {
|
||||||
if !key.deleted.get() {
|
if !key.deleted {
|
||||||
self.update_key_bucket(key, &bucket.name, false, false)
|
self.update_key_bucket(key, &bucket.name, false, false)
|
||||||
.await?;
|
.await?;
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
return Err(Error::Message(format!("Key not found: {}", key_id)));
|
return Err(Error::Message(format!("Key not found: {}", ak.key_id)));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
bucket.state.update(BucketState::Deleted);
|
self.garage
|
||||||
self.garage.bucket_table.insert(&bucket).await?;
|
.bucket_table
|
||||||
|
.insert(&Bucket::new(
|
||||||
|
query.name.clone(),
|
||||||
|
std::cmp::max(bucket.timestamp + 1, now_msec()),
|
||||||
|
true,
|
||||||
|
vec![],
|
||||||
|
))
|
||||||
|
.await?;
|
||||||
Ok(AdminRPC::Ok(format!("Bucket {} was deleted.", query.name)))
|
Ok(AdminRPC::Ok(format!("Bucket {} was deleted.", query.name)))
|
||||||
}
|
}
|
||||||
BucketOperation::Allow(query) => {
|
BucketOperation::Allow(query) => {
|
||||||
@@ -155,29 +164,6 @@ impl AdminRpcHandler {
|
|||||||
&query.key_id, &query.bucket, allow_read, allow_write
|
&query.key_id, &query.bucket, allow_read, allow_write
|
||||||
)))
|
)))
|
||||||
}
|
}
|
||||||
BucketOperation::Website(query) => {
|
|
||||||
let mut bucket = self.get_existing_bucket(&query.bucket).await?;
|
|
||||||
|
|
||||||
if !(query.allow ^ query.deny) {
|
|
||||||
return Err(Error::Message(format!(
|
|
||||||
"You must specify exactly one flag, either --allow or --deny"
|
|
||||||
)));
|
|
||||||
}
|
|
||||||
|
|
||||||
if let BucketState::Present(state) = bucket.state.get_mut() {
|
|
||||||
state.website.update(query.allow);
|
|
||||||
self.garage.bucket_table.insert(&bucket).await?;
|
|
||||||
let msg = if query.allow {
|
|
||||||
format!("Website access allowed for {}", &query.bucket)
|
|
||||||
} else {
|
|
||||||
format!("Website access denied for {}", &query.bucket)
|
|
||||||
};
|
|
||||||
|
|
||||||
Ok(AdminRPC::Ok(msg.to_string()))
|
|
||||||
} else {
|
|
||||||
unreachable!();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -187,10 +173,10 @@ impl AdminRpcHandler {
|
|||||||
let key_ids = self
|
let key_ids = self
|
||||||
.garage
|
.garage
|
||||||
.key_table
|
.key_table
|
||||||
.get_range(&EmptyKey, None, Some(DeletedFilter::NotDeleted), 10000)
|
.get_range(&EmptyKey, None, Some(()), 10000)
|
||||||
.await?
|
.await?
|
||||||
.iter()
|
.iter()
|
||||||
.map(|k| (k.key_id.to_string(), k.name.get().clone()))
|
.map(|k| (k.key_id.to_string(), k.name.to_string()))
|
||||||
.collect::<Vec<_>>();
|
.collect::<Vec<_>>();
|
||||||
Ok(AdminRPC::KeyList(key_ids))
|
Ok(AdminRPC::KeyList(key_ids))
|
||||||
}
|
}
|
||||||
@@ -199,32 +185,35 @@ impl AdminRpcHandler {
|
|||||||
Ok(AdminRPC::KeyInfo(key))
|
Ok(AdminRPC::KeyInfo(key))
|
||||||
}
|
}
|
||||||
KeyOperation::New(query) => {
|
KeyOperation::New(query) => {
|
||||||
let key = Key::new(query.name);
|
let key = Key::new(query.name, vec![]);
|
||||||
self.garage.key_table.insert(&key).await?;
|
self.garage.key_table.insert(&key).await?;
|
||||||
Ok(AdminRPC::KeyInfo(key))
|
Ok(AdminRPC::KeyInfo(key))
|
||||||
}
|
}
|
||||||
KeyOperation::Rename(query) => {
|
KeyOperation::Rename(query) => {
|
||||||
let mut key = self.get_existing_key(&query.key_id).await?;
|
let mut key = self.get_existing_key(&query.key_id).await?;
|
||||||
key.name.update(query.new_name);
|
key.name_timestamp = std::cmp::max(key.name_timestamp + 1, now_msec());
|
||||||
|
key.name = query.new_name;
|
||||||
self.garage.key_table.insert(&key).await?;
|
self.garage.key_table.insert(&key).await?;
|
||||||
Ok(AdminRPC::KeyInfo(key))
|
Ok(AdminRPC::KeyInfo(key))
|
||||||
}
|
}
|
||||||
KeyOperation::Delete(query) => {
|
KeyOperation::Delete(query) => {
|
||||||
let key = self.get_existing_key(&query.key_id).await?;
|
let key = self.get_existing_key(&query.key_id).await?;
|
||||||
if !query.yes {
|
if !query.yes {
|
||||||
return Err(Error::BadRPC(format!(
|
return Err(Error::BadRequest(format!(
|
||||||
"Add --yes flag to really perform this operation"
|
"Add --yes flag to really perform this operation"
|
||||||
)));
|
)));
|
||||||
}
|
}
|
||||||
// --- done checking, now commit ---
|
// --- done checking, now commit ---
|
||||||
for (ab_name, _, _) in key.authorized_buckets.items().iter() {
|
for ab in key.authorized_buckets().iter() {
|
||||||
if let Some(bucket) = self.garage.bucket_table.get(&EmptyKey, ab_name).await? {
|
if let Some(bucket) =
|
||||||
if !bucket.is_deleted() {
|
self.garage.bucket_table.get(&EmptyKey, &ab.bucket).await?
|
||||||
|
{
|
||||||
|
if !bucket.deleted {
|
||||||
self.update_bucket_key(bucket, &key.key_id, false, false)
|
self.update_bucket_key(bucket, &key.key_id, false, false)
|
||||||
.await?;
|
.await?;
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
return Err(Error::Message(format!("Bucket not found: {}", ab_name)));
|
return Err(Error::Message(format!("Bucket not found: {}", ab.bucket)));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
let del_key = Key::delete(key.key_id);
|
let del_key = Key::delete(key.key_id);
|
||||||
@@ -242,9 +231,9 @@ impl AdminRpcHandler {
|
|||||||
.bucket_table
|
.bucket_table
|
||||||
.get(&EmptyKey, bucket)
|
.get(&EmptyKey, bucket)
|
||||||
.await?
|
.await?
|
||||||
.filter(|b| !b.is_deleted())
|
.filter(|b| !b.deleted)
|
||||||
.map(Ok)
|
.map(Ok)
|
||||||
.unwrap_or(Err(Error::BadRPC(format!(
|
.unwrap_or(Err(Error::BadRequest(format!(
|
||||||
"Bucket {} does not exist",
|
"Bucket {} does not exist",
|
||||||
bucket
|
bucket
|
||||||
))))
|
))))
|
||||||
@@ -255,12 +244,11 @@ impl AdminRpcHandler {
|
|||||||
.key_table
|
.key_table
|
||||||
.get(&EmptyKey, id)
|
.get(&EmptyKey, id)
|
||||||
.await?
|
.await?
|
||||||
.filter(|k| !k.deleted.get())
|
.filter(|k| !k.deleted)
|
||||||
.map(Ok)
|
.map(Ok)
|
||||||
.unwrap_or(Err(Error::BadRPC(format!("Key {} does not exist", id))))
|
.unwrap_or(Err(Error::BadRequest(format!("Key {} does not exist", id))))
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Update **bucket table** to inform of the new linked key
|
|
||||||
async fn update_bucket_key(
|
async fn update_bucket_key(
|
||||||
&self,
|
&self,
|
||||||
mut bucket: Bucket,
|
mut bucket: Bucket,
|
||||||
@@ -268,26 +256,27 @@ impl AdminRpcHandler {
|
|||||||
allow_read: bool,
|
allow_read: bool,
|
||||||
allow_write: bool,
|
allow_write: bool,
|
||||||
) -> Result<(), Error> {
|
) -> Result<(), Error> {
|
||||||
if let BucketState::Present(params) = bucket.state.get_mut() {
|
let timestamp = match bucket
|
||||||
let ak = &mut params.authorized_keys;
|
.authorized_keys()
|
||||||
let old_ak = ak.take_and_clear();
|
.iter()
|
||||||
ak.merge(&old_ak.update_mutator(
|
.find(|x| x.key_id == *key_id)
|
||||||
key_id.to_string(),
|
{
|
||||||
PermissionSet {
|
None => now_msec(),
|
||||||
allow_read,
|
Some(ab) => std::cmp::max(ab.timestamp + 1, now_msec()),
|
||||||
allow_write,
|
};
|
||||||
},
|
bucket.clear_keys();
|
||||||
));
|
bucket
|
||||||
} else {
|
.add_key(AllowedKey {
|
||||||
return Err(Error::Message(format!(
|
key_id: key_id.clone(),
|
||||||
"Bucket is deleted in update_bucket_key"
|
timestamp,
|
||||||
)));
|
allow_read,
|
||||||
}
|
allow_write,
|
||||||
|
})
|
||||||
|
.unwrap();
|
||||||
self.garage.bucket_table.insert(&bucket).await?;
|
self.garage.bucket_table.insert(&bucket).await?;
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Update **key table** to inform of the new linked bucket
|
|
||||||
async fn update_key_bucket(
|
async fn update_key_bucket(
|
||||||
&self,
|
&self,
|
||||||
mut key: Key,
|
mut key: Key,
|
||||||
@@ -295,21 +284,29 @@ impl AdminRpcHandler {
|
|||||||
allow_read: bool,
|
allow_read: bool,
|
||||||
allow_write: bool,
|
allow_write: bool,
|
||||||
) -> Result<(), Error> {
|
) -> Result<(), Error> {
|
||||||
let old_map = key.authorized_buckets.take_and_clear();
|
let timestamp = match key
|
||||||
key.authorized_buckets.merge(&old_map.update_mutator(
|
.authorized_buckets()
|
||||||
bucket.clone(),
|
.iter()
|
||||||
PermissionSet {
|
.find(|x| x.bucket == *bucket)
|
||||||
allow_read,
|
{
|
||||||
allow_write,
|
None => now_msec(),
|
||||||
},
|
Some(ab) => std::cmp::max(ab.timestamp + 1, now_msec()),
|
||||||
));
|
};
|
||||||
|
key.clear_buckets();
|
||||||
|
key.add_bucket(AllowedBucket {
|
||||||
|
bucket: bucket.clone(),
|
||||||
|
timestamp,
|
||||||
|
allow_read,
|
||||||
|
allow_write,
|
||||||
|
})
|
||||||
|
.unwrap();
|
||||||
self.garage.key_table.insert(&key).await?;
|
self.garage.key_table.insert(&key).await?;
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn handle_launch_repair(self: &Arc<Self>, opt: RepairOpt) -> Result<AdminRPC, Error> {
|
async fn handle_launch_repair(self: &Arc<Self>, opt: RepairOpt) -> Result<AdminRPC, Error> {
|
||||||
if !opt.yes {
|
if !opt.yes {
|
||||||
return Err(Error::BadRPC(format!(
|
return Err(Error::BadRequest(format!(
|
||||||
"Please provide the --yes flag to initiate repair operations."
|
"Please provide the --yes flag to initiate repair operations."
|
||||||
)));
|
)));
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -141,24 +141,6 @@ pub enum BucketOperation {
|
|||||||
/// Allow key to read or write to bucket
|
/// Allow key to read or write to bucket
|
||||||
#[structopt(name = "deny")]
|
#[structopt(name = "deny")]
|
||||||
Deny(PermBucketOpt),
|
Deny(PermBucketOpt),
|
||||||
|
|
||||||
/// Expose as website or not
|
|
||||||
#[structopt(name = "website")]
|
|
||||||
Website(WebsiteOpt),
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Serialize, Deserialize, StructOpt, Debug)]
|
|
||||||
pub struct WebsiteOpt {
|
|
||||||
/// Create
|
|
||||||
#[structopt(long = "allow")]
|
|
||||||
pub allow: bool,
|
|
||||||
|
|
||||||
/// Delete
|
|
||||||
#[structopt(long = "deny")]
|
|
||||||
pub deny: bool,
|
|
||||||
|
|
||||||
/// Bucket name
|
|
||||||
pub bucket: String,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Serialize, Deserialize, StructOpt, Debug)]
|
#[derive(Serialize, Deserialize, StructOpt, Debug)]
|
||||||
|
|||||||
@@ -11,7 +11,6 @@ use garage_util::error::Error;
|
|||||||
use garage_api::api_server;
|
use garage_api::api_server;
|
||||||
use garage_model::garage::Garage;
|
use garage_model::garage::Garage;
|
||||||
use garage_rpc::rpc_server::RpcServer;
|
use garage_rpc::rpc_server::RpcServer;
|
||||||
use garage_web::web_server;
|
|
||||||
|
|
||||||
use crate::admin_rpc::*;
|
use crate::admin_rpc::*;
|
||||||
|
|
||||||
@@ -57,7 +56,6 @@ pub async fn run_server(config_file: PathBuf) -> Result<(), Error> {
|
|||||||
info!("Initializing RPC and API servers...");
|
info!("Initializing RPC and API servers...");
|
||||||
let run_rpc_server = Arc::new(rpc_server).run(wait_from(watch_cancel.clone()));
|
let run_rpc_server = Arc::new(rpc_server).run(wait_from(watch_cancel.clone()));
|
||||||
let api_server = api_server::run_api_server(garage.clone(), wait_from(watch_cancel.clone()));
|
let api_server = api_server::run_api_server(garage.clone(), wait_from(watch_cancel.clone()));
|
||||||
let web_server = web_server::run_web_server(garage.clone(), wait_from(watch_cancel.clone()));
|
|
||||||
|
|
||||||
futures::try_join!(
|
futures::try_join!(
|
||||||
garage
|
garage
|
||||||
@@ -80,10 +78,6 @@ pub async fn run_server(config_file: PathBuf) -> Result<(), Error> {
|
|||||||
info!("API server exited");
|
info!("API server exited");
|
||||||
rv
|
rv
|
||||||
}),
|
}),
|
||||||
web_server.map(|rv| {
|
|
||||||
info!("Web server exited");
|
|
||||||
rv
|
|
||||||
}),
|
|
||||||
background.run().map(|rv| {
|
background.run().map(|rv| {
|
||||||
info!("Background runner exited");
|
info!("Background runner exited");
|
||||||
Ok(rv)
|
Ok(rv)
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
[package]
|
[package]
|
||||||
name = "garage_model"
|
name = "garage_model_010b"
|
||||||
version = "0.1.1"
|
version = "0.0.1"
|
||||||
authors = ["Alex Auvolat <alex@adnab.me>"]
|
authors = ["Alex Auvolat <alex@adnab.me>"]
|
||||||
edition = "2018"
|
edition = "2018"
|
||||||
license = "GPL-3.0"
|
license = "GPL-3.0"
|
||||||
@@ -13,10 +13,9 @@ path = "lib.rs"
|
|||||||
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
|
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
|
||||||
|
|
||||||
[dependencies]
|
[dependencies]
|
||||||
garage_util = { version = "0.1.1", path = "../util" }
|
garage_util = { version = "0.1", path = "../util" }
|
||||||
garage_rpc = { version = "0.1.1", path = "../rpc" }
|
garage_rpc = { version = "0.1", path = "../rpc" }
|
||||||
garage_table = { version = "0.1.1", path = "../table" }
|
garage_table = { version = "0.1", path = "../table" }
|
||||||
model010 = { package = "garage_model_010b", version = "0.0.1" }
|
|
||||||
|
|
||||||
bytes = "0.4"
|
bytes = "0.4"
|
||||||
rand = "0.7"
|
rand = "0.7"
|
||||||
|
|||||||
+2
-2
@@ -20,7 +20,7 @@ use garage_rpc::rpc_client::*;
|
|||||||
use garage_rpc::rpc_server::*;
|
use garage_rpc::rpc_server::*;
|
||||||
|
|
||||||
use garage_table::table_sharded::TableShardedReplication;
|
use garage_table::table_sharded::TableShardedReplication;
|
||||||
use garage_table::{DeletedFilter, TableReplication};
|
use garage_table::TableReplication;
|
||||||
|
|
||||||
use crate::block_ref_table::*;
|
use crate::block_ref_table::*;
|
||||||
|
|
||||||
@@ -306,7 +306,7 @@ impl BlockManager {
|
|||||||
let garage = self.garage.load_full().unwrap();
|
let garage = self.garage.load_full().unwrap();
|
||||||
let active_refs = garage
|
let active_refs = garage
|
||||||
.block_ref_table
|
.block_ref_table
|
||||||
.get_range(&hash, None, Some(DeletedFilter::NotDeleted), 1)
|
.get_range(&hash, None, Some(()), 1)
|
||||||
.await?;
|
.await?;
|
||||||
let needed_by_others = !active_refs.is_empty();
|
let needed_by_others = !active_refs.is_empty();
|
||||||
if needed_by_others {
|
if needed_by_others {
|
||||||
|
|||||||
@@ -47,7 +47,7 @@ impl TableSchema for BlockRefTable {
|
|||||||
type P = Hash;
|
type P = Hash;
|
||||||
type S = UUID;
|
type S = UUID;
|
||||||
type E = BlockRef;
|
type E = BlockRef;
|
||||||
type Filter = DeletedFilter;
|
type Filter = ();
|
||||||
|
|
||||||
async fn updated(&self, old: Option<Self::E>, new: Option<Self::E>) -> Result<(), Error> {
|
async fn updated(&self, old: Option<Self::E>, new: Option<Self::E>) -> Result<(), Error> {
|
||||||
let block = &old.as_ref().or(new.as_ref()).unwrap().block;
|
let block = &old.as_ref().or(new.as_ref()).unwrap().block;
|
||||||
@@ -62,7 +62,7 @@ impl TableSchema for BlockRefTable {
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
fn matches_filter(entry: &Self::E, filter: &Self::Filter) -> bool {
|
fn matches_filter(entry: &Self::E, _filter: &Self::Filter) -> bool {
|
||||||
filter.apply(entry.deleted)
|
!entry.deleted
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
+73
-104
@@ -1,90 +1,71 @@
|
|||||||
use async_trait::async_trait;
|
use async_trait::async_trait;
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
|
|
||||||
use garage_table::crdt::CRDT;
|
|
||||||
use garage_table::*;
|
use garage_table::*;
|
||||||
|
|
||||||
use garage_util::error::Error;
|
use garage_util::error::Error;
|
||||||
|
|
||||||
use crate::key_table::PermissionSet;
|
|
||||||
|
|
||||||
// We import the same file but in its version 0.1.0.
|
|
||||||
// We can then access v0.1.0 data structures.
|
|
||||||
// We use them to perform migrations.
|
|
||||||
use model010::bucket_table as prev;
|
|
||||||
|
|
||||||
/// A bucket is a collection of objects
|
|
||||||
///
|
|
||||||
/// Its parameters are not directly accessible as:
|
|
||||||
/// - It must be possible to merge paramaters, hence the use of a LWW CRDT.
|
|
||||||
/// - A bucket has 2 states, Present or Deleted and parameters make sense only if present.
|
|
||||||
#[derive(PartialEq, Clone, Debug, Serialize, Deserialize)]
|
#[derive(PartialEq, Clone, Debug, Serialize, Deserialize)]
|
||||||
pub struct Bucket {
|
pub struct Bucket {
|
||||||
// Primary key
|
// Primary key
|
||||||
pub name: String,
|
pub name: String,
|
||||||
|
|
||||||
pub state: crdt::LWW<BucketState>,
|
// Timestamp and deletion
|
||||||
}
|
// Upon version increment, all info is replaced
|
||||||
|
pub timestamp: u64,
|
||||||
|
pub deleted: bool,
|
||||||
|
|
||||||
#[derive(PartialEq, Clone, Debug, Serialize, Deserialize)]
|
// Authorized keys
|
||||||
pub enum BucketState {
|
authorized_keys: Vec<AllowedKey>,
|
||||||
Deleted,
|
|
||||||
Present(BucketParams),
|
|
||||||
}
|
|
||||||
|
|
||||||
impl CRDT for BucketState {
|
|
||||||
fn merge(&mut self, o: &Self) {
|
|
||||||
match o {
|
|
||||||
BucketState::Deleted => *self = BucketState::Deleted,
|
|
||||||
BucketState::Present(other_params) => {
|
|
||||||
if let BucketState::Present(params) = self {
|
|
||||||
params.merge(other_params);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(PartialEq, Clone, Debug, Serialize, Deserialize)]
|
|
||||||
pub struct BucketParams {
|
|
||||||
pub authorized_keys: crdt::LWWMap<String, PermissionSet>,
|
|
||||||
pub website: crdt::LWW<bool>,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl CRDT for BucketParams {
|
|
||||||
fn merge(&mut self, o: &Self) {
|
|
||||||
self.authorized_keys.merge(&o.authorized_keys);
|
|
||||||
self.website.merge(&o.website);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl BucketParams {
|
|
||||||
pub fn new() -> Self {
|
|
||||||
BucketParams {
|
|
||||||
authorized_keys: crdt::LWWMap::new(),
|
|
||||||
website: crdt::LWW::new(false),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Bucket {
|
impl Bucket {
|
||||||
pub fn new(name: String) -> Self {
|
pub fn new(
|
||||||
Bucket {
|
name: String,
|
||||||
|
timestamp: u64,
|
||||||
|
deleted: bool,
|
||||||
|
authorized_keys: Vec<AllowedKey>,
|
||||||
|
) -> Self {
|
||||||
|
let mut ret = Bucket {
|
||||||
name,
|
name,
|
||||||
state: crdt::LWW::new(BucketState::Present(BucketParams::new())),
|
timestamp,
|
||||||
|
deleted,
|
||||||
|
authorized_keys: vec![],
|
||||||
|
};
|
||||||
|
for key in authorized_keys {
|
||||||
|
ret.add_key(key)
|
||||||
|
.expect("Duplicate AllowedKey in Bucket constructor");
|
||||||
|
}
|
||||||
|
ret
|
||||||
|
}
|
||||||
|
/// Add a key only if it is not already present
|
||||||
|
pub fn add_key(&mut self, key: AllowedKey) -> Result<(), ()> {
|
||||||
|
match self
|
||||||
|
.authorized_keys
|
||||||
|
.binary_search_by(|k| k.key_id.cmp(&key.key_id))
|
||||||
|
{
|
||||||
|
Err(i) => {
|
||||||
|
self.authorized_keys.insert(i, key);
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
Ok(_) => Err(()),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
pub fn is_deleted(&self) -> bool {
|
pub fn authorized_keys(&self) -> &[AllowedKey] {
|
||||||
*self.state.get() == BucketState::Deleted
|
&self.authorized_keys[..]
|
||||||
}
|
}
|
||||||
pub fn authorized_keys(&self) -> &[(String, u64, PermissionSet)] {
|
pub fn clear_keys(&mut self) {
|
||||||
match self.state.get() {
|
self.authorized_keys.clear();
|
||||||
BucketState::Deleted => &[],
|
|
||||||
BucketState::Present(state) => state.authorized_keys.items(),
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(PartialEq, Clone, Debug, Serialize, Deserialize)]
|
||||||
|
pub struct AllowedKey {
|
||||||
|
pub key_id: String,
|
||||||
|
pub timestamp: u64,
|
||||||
|
pub allow_read: bool,
|
||||||
|
pub allow_write: bool,
|
||||||
|
}
|
||||||
|
|
||||||
impl Entry<EmptyKey, String> for Bucket {
|
impl Entry<EmptyKey, String> for Bucket {
|
||||||
fn partition_key(&self) -> &EmptyKey {
|
fn partition_key(&self) -> &EmptyKey {
|
||||||
&EmptyKey
|
&EmptyKey
|
||||||
@@ -94,7 +75,30 @@ impl Entry<EmptyKey, String> for Bucket {
|
|||||||
}
|
}
|
||||||
|
|
||||||
fn merge(&mut self, other: &Self) {
|
fn merge(&mut self, other: &Self) {
|
||||||
self.state.merge(&other.state);
|
if other.timestamp < self.timestamp {
|
||||||
|
*self = other.clone();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if self.timestamp > other.timestamp || self.deleted {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
for ak in other.authorized_keys.iter() {
|
||||||
|
match self
|
||||||
|
.authorized_keys
|
||||||
|
.binary_search_by(|our_ak| our_ak.key_id.cmp(&ak.key_id))
|
||||||
|
{
|
||||||
|
Ok(i) => {
|
||||||
|
let our_ak = &mut self.authorized_keys[i];
|
||||||
|
if ak.timestamp > our_ak.timestamp {
|
||||||
|
*our_ak = ak.clone();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Err(i) => {
|
||||||
|
self.authorized_keys.insert(i, ak.clone());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -105,48 +109,13 @@ impl TableSchema for BucketTable {
|
|||||||
type P = EmptyKey;
|
type P = EmptyKey;
|
||||||
type S = String;
|
type S = String;
|
||||||
type E = Bucket;
|
type E = Bucket;
|
||||||
type Filter = DeletedFilter;
|
type Filter = ();
|
||||||
|
|
||||||
async fn updated(&self, _old: Option<Self::E>, _new: Option<Self::E>) -> Result<(), Error> {
|
async fn updated(&self, _old: Option<Self::E>, _new: Option<Self::E>) -> Result<(), Error> {
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
fn matches_filter(entry: &Self::E, filter: &Self::Filter) -> bool {
|
fn matches_filter(entry: &Self::E, _filter: &Self::Filter) -> bool {
|
||||||
filter.apply(entry.is_deleted())
|
!entry.deleted
|
||||||
}
|
|
||||||
|
|
||||||
fn try_migrate(bytes: &[u8]) -> Option<Self::E> {
|
|
||||||
let old = match rmp_serde::decode::from_read_ref::<_, prev::Bucket>(bytes) {
|
|
||||||
Ok(x) => x,
|
|
||||||
Err(_) => return None,
|
|
||||||
};
|
|
||||||
if old.deleted {
|
|
||||||
Some(Bucket {
|
|
||||||
name: old.name,
|
|
||||||
state: crdt::LWW::migrate_from_raw(old.timestamp, BucketState::Deleted),
|
|
||||||
})
|
|
||||||
} else {
|
|
||||||
let mut keys = crdt::LWWMap::new();
|
|
||||||
for ak in old.authorized_keys() {
|
|
||||||
keys.merge(&crdt::LWWMap::migrate_from_raw_item(
|
|
||||||
ak.key_id.clone(),
|
|
||||||
ak.timestamp,
|
|
||||||
PermissionSet {
|
|
||||||
allow_read: ak.allow_read,
|
|
||||||
allow_write: ak.allow_write,
|
|
||||||
},
|
|
||||||
));
|
|
||||||
}
|
|
||||||
|
|
||||||
let params = BucketParams {
|
|
||||||
authorized_keys: keys,
|
|
||||||
website: crdt::LWW::new(false),
|
|
||||||
};
|
|
||||||
|
|
||||||
Some(Bucket {
|
|
||||||
name: old.name,
|
|
||||||
state: crdt::LWW::migrate_from_raw(old.timestamp, BucketState::Present(params)),
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
+73
-54
@@ -1,13 +1,10 @@
|
|||||||
use async_trait::async_trait;
|
use async_trait::async_trait;
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
|
|
||||||
use garage_table::crdt::CRDT;
|
|
||||||
use garage_table::*;
|
use garage_table::*;
|
||||||
|
use garage_util::data::*;
|
||||||
use garage_util::error::Error;
|
use garage_util::error::Error;
|
||||||
|
|
||||||
use model010::key_table as prev;
|
|
||||||
|
|
||||||
#[derive(PartialEq, Clone, Debug, Serialize, Deserialize)]
|
#[derive(PartialEq, Clone, Debug, Serialize, Deserialize)]
|
||||||
pub struct Key {
|
pub struct Key {
|
||||||
// Primary key
|
// Primary key
|
||||||
@@ -17,54 +14,83 @@ pub struct Key {
|
|||||||
pub secret_key: String,
|
pub secret_key: String,
|
||||||
|
|
||||||
// Name
|
// Name
|
||||||
pub name: crdt::LWW<String>,
|
pub name: String,
|
||||||
|
pub name_timestamp: u64,
|
||||||
|
|
||||||
// Deletion
|
// Deletion
|
||||||
pub deleted: crdt::Bool,
|
pub deleted: bool,
|
||||||
|
|
||||||
// Authorized keys
|
// Authorized keys
|
||||||
pub authorized_buckets: crdt::LWWMap<String, PermissionSet>,
|
authorized_buckets: Vec<AllowedBucket>,
|
||||||
// CRDT interaction: deleted implies authorized_buckets is empty
|
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Key {
|
impl Key {
|
||||||
pub fn new(name: String) -> Self {
|
pub fn new(name: String, buckets: Vec<AllowedBucket>) -> Self {
|
||||||
let key_id = format!("GK{}", hex::encode(&rand::random::<[u8; 12]>()[..]));
|
let key_id = format!("GK{}", hex::encode(&rand::random::<[u8; 12]>()[..]));
|
||||||
let secret_key = hex::encode(&rand::random::<[u8; 32]>()[..]);
|
let secret_key = hex::encode(&rand::random::<[u8; 32]>()[..]);
|
||||||
Self {
|
let mut ret = Self {
|
||||||
key_id,
|
key_id,
|
||||||
secret_key,
|
secret_key,
|
||||||
name: crdt::LWW::new(name),
|
name,
|
||||||
deleted: crdt::Bool::new(false),
|
name_timestamp: now_msec(),
|
||||||
authorized_buckets: crdt::LWWMap::new(),
|
deleted: false,
|
||||||
|
authorized_buckets: vec![],
|
||||||
|
};
|
||||||
|
for b in buckets {
|
||||||
|
ret.add_bucket(b)
|
||||||
|
.expect("Duplicate AllowedBucket in Key constructor");
|
||||||
}
|
}
|
||||||
|
ret
|
||||||
}
|
}
|
||||||
pub fn delete(key_id: String) -> Self {
|
pub fn delete(key_id: String) -> Self {
|
||||||
Self {
|
Self {
|
||||||
key_id,
|
key_id,
|
||||||
secret_key: "".into(),
|
secret_key: "".into(),
|
||||||
name: crdt::LWW::new("".to_string()),
|
name: "".into(),
|
||||||
deleted: crdt::Bool::new(true),
|
name_timestamp: now_msec(),
|
||||||
authorized_buckets: crdt::LWWMap::new(),
|
deleted: true,
|
||||||
|
authorized_buckets: vec![],
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
/// Add an authorized bucket, only if it wasn't there before
|
/// Add an authorized bucket, only if it wasn't there before
|
||||||
|
pub fn add_bucket(&mut self, new: AllowedBucket) -> Result<(), ()> {
|
||||||
|
match self
|
||||||
|
.authorized_buckets
|
||||||
|
.binary_search_by(|b| b.bucket.cmp(&new.bucket))
|
||||||
|
{
|
||||||
|
Err(i) => {
|
||||||
|
self.authorized_buckets.insert(i, new);
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
Ok(_) => Err(()),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
pub fn authorized_buckets(&self) -> &[AllowedBucket] {
|
||||||
|
&self.authorized_buckets[..]
|
||||||
|
}
|
||||||
|
pub fn clear_buckets(&mut self) {
|
||||||
|
self.authorized_buckets.clear();
|
||||||
|
}
|
||||||
pub fn allow_read(&self, bucket: &str) -> bool {
|
pub fn allow_read(&self, bucket: &str) -> bool {
|
||||||
self.authorized_buckets
|
self.authorized_buckets
|
||||||
.get(&bucket.to_string())
|
.iter()
|
||||||
|
.find(|x| x.bucket.as_str() == bucket)
|
||||||
.map(|x| x.allow_read)
|
.map(|x| x.allow_read)
|
||||||
.unwrap_or(false)
|
.unwrap_or(false)
|
||||||
}
|
}
|
||||||
pub fn allow_write(&self, bucket: &str) -> bool {
|
pub fn allow_write(&self, bucket: &str) -> bool {
|
||||||
self.authorized_buckets
|
self.authorized_buckets
|
||||||
.get(&bucket.to_string())
|
.iter()
|
||||||
|
.find(|x| x.bucket.as_str() == bucket)
|
||||||
.map(|x| x.allow_write)
|
.map(|x| x.allow_write)
|
||||||
.unwrap_or(false)
|
.unwrap_or(false)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(PartialOrd, Ord, PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
|
#[derive(PartialEq, Clone, Debug, Serialize, Deserialize)]
|
||||||
pub struct PermissionSet {
|
pub struct AllowedBucket {
|
||||||
|
pub bucket: String,
|
||||||
|
pub timestamp: u64,
|
||||||
pub allow_read: bool,
|
pub allow_read: bool,
|
||||||
pub allow_write: bool,
|
pub allow_write: bool,
|
||||||
}
|
}
|
||||||
@@ -78,15 +104,34 @@ impl Entry<EmptyKey, String> for Key {
|
|||||||
}
|
}
|
||||||
|
|
||||||
fn merge(&mut self, other: &Self) {
|
fn merge(&mut self, other: &Self) {
|
||||||
self.name.merge(&other.name);
|
if other.deleted {
|
||||||
self.deleted.merge(&other.deleted);
|
self.deleted = true;
|
||||||
|
}
|
||||||
if self.deleted.get() {
|
if self.deleted {
|
||||||
self.authorized_buckets.clear();
|
self.authorized_buckets.clear();
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
if other.name_timestamp > self.name_timestamp {
|
||||||
|
self.name_timestamp = other.name_timestamp;
|
||||||
|
self.name = other.name.clone();
|
||||||
|
}
|
||||||
|
|
||||||
self.authorized_buckets.merge(&other.authorized_buckets);
|
for ab in other.authorized_buckets.iter() {
|
||||||
|
match self
|
||||||
|
.authorized_buckets
|
||||||
|
.binary_search_by(|our_ab| our_ab.bucket.cmp(&ab.bucket))
|
||||||
|
{
|
||||||
|
Ok(i) => {
|
||||||
|
let our_ab = &mut self.authorized_buckets[i];
|
||||||
|
if ab.timestamp > our_ab.timestamp {
|
||||||
|
*our_ab = ab.clone();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Err(i) => {
|
||||||
|
self.authorized_buckets.insert(i, ab.clone());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -97,39 +142,13 @@ impl TableSchema for KeyTable {
|
|||||||
type P = EmptyKey;
|
type P = EmptyKey;
|
||||||
type S = String;
|
type S = String;
|
||||||
type E = Key;
|
type E = Key;
|
||||||
type Filter = DeletedFilter;
|
type Filter = ();
|
||||||
|
|
||||||
async fn updated(&self, _old: Option<Self::E>, _new: Option<Self::E>) -> Result<(), Error> {
|
async fn updated(&self, _old: Option<Self::E>, _new: Option<Self::E>) -> Result<(), Error> {
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
fn matches_filter(entry: &Self::E, filter: &Self::Filter) -> bool {
|
fn matches_filter(entry: &Self::E, _filter: &Self::Filter) -> bool {
|
||||||
filter.apply(entry.deleted.get())
|
!entry.deleted
|
||||||
}
|
|
||||||
|
|
||||||
fn try_migrate(bytes: &[u8]) -> Option<Self::E> {
|
|
||||||
let old = match rmp_serde::decode::from_read_ref::<_, prev::Key>(bytes) {
|
|
||||||
Ok(x) => x,
|
|
||||||
Err(_) => return None,
|
|
||||||
};
|
|
||||||
let mut new = Self::E {
|
|
||||||
key_id: old.key_id.clone(),
|
|
||||||
secret_key: old.secret_key.clone(),
|
|
||||||
name: crdt::LWW::migrate_from_raw(old.name_timestamp, old.name.clone()),
|
|
||||||
deleted: crdt::Bool::new(old.deleted),
|
|
||||||
authorized_buckets: crdt::LWWMap::new(),
|
|
||||||
};
|
|
||||||
for ab in old.authorized_buckets() {
|
|
||||||
let it = crdt::LWWMap::migrate_from_raw_item(
|
|
||||||
ab.bucket.clone(),
|
|
||||||
ab.timestamp,
|
|
||||||
PermissionSet {
|
|
||||||
allow_read: ab.allow_read,
|
|
||||||
allow_write: ab.allow_write,
|
|
||||||
},
|
|
||||||
);
|
|
||||||
new.authorized_buckets.merge(&it);
|
|
||||||
}
|
|
||||||
Some(new)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
+30
-118
@@ -1,6 +1,5 @@
|
|||||||
use async_trait::async_trait;
|
use async_trait::async_trait;
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
use std::collections::BTreeMap;
|
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
|
|
||||||
use garage_util::background::BackgroundRunner;
|
use garage_util::background::BackgroundRunner;
|
||||||
@@ -12,8 +11,6 @@ use garage_table::*;
|
|||||||
|
|
||||||
use crate::version_table::*;
|
use crate::version_table::*;
|
||||||
|
|
||||||
use model010::object_table as prev;
|
|
||||||
|
|
||||||
#[derive(PartialEq, Clone, Debug, Serialize, Deserialize)]
|
#[derive(PartialEq, Clone, Debug, Serialize, Deserialize)]
|
||||||
pub struct Object {
|
pub struct Object {
|
||||||
// Primary key
|
// Primary key
|
||||||
@@ -62,90 +59,50 @@ pub struct ObjectVersion {
|
|||||||
pub uuid: UUID,
|
pub uuid: UUID,
|
||||||
pub timestamp: u64,
|
pub timestamp: u64,
|
||||||
|
|
||||||
|
pub mime_type: String,
|
||||||
|
pub size: u64,
|
||||||
pub state: ObjectVersionState,
|
pub state: ObjectVersionState,
|
||||||
|
|
||||||
|
pub data: ObjectVersionData,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(PartialEq, Clone, Debug, Serialize, Deserialize)]
|
#[derive(PartialEq, Clone, Copy, Debug, Serialize, Deserialize)]
|
||||||
pub enum ObjectVersionState {
|
pub enum ObjectVersionState {
|
||||||
Uploading(ObjectVersionHeaders),
|
Uploading,
|
||||||
Complete(ObjectVersionData),
|
Complete,
|
||||||
Aborted,
|
Aborted,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl ObjectVersionState {
|
impl ObjectVersionState {
|
||||||
fn merge(&mut self, other: &Self) {
|
fn max(self, other: Self) -> Self {
|
||||||
use ObjectVersionState::*;
|
use ObjectVersionState::*;
|
||||||
match other {
|
if self == Aborted || other == Aborted {
|
||||||
Aborted => {
|
Aborted
|
||||||
*self = Aborted;
|
} else if self == Complete || other == Complete {
|
||||||
}
|
Complete
|
||||||
Complete(b) => match self {
|
} else {
|
||||||
Aborted => {}
|
Uploading
|
||||||
Complete(a) => {
|
|
||||||
a.merge(b);
|
|
||||||
}
|
|
||||||
Uploading(_) => {
|
|
||||||
*self = Complete(b.clone());
|
|
||||||
}
|
|
||||||
},
|
|
||||||
Uploading(_) => {}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(PartialEq, Clone, Debug, Serialize, Deserialize)]
|
#[derive(PartialEq, Clone, Debug, Serialize, Deserialize)]
|
||||||
pub enum ObjectVersionData {
|
pub enum ObjectVersionData {
|
||||||
|
Uploading,
|
||||||
DeleteMarker,
|
DeleteMarker,
|
||||||
Inline(ObjectVersionMeta, #[serde(with = "serde_bytes")] Vec<u8>),
|
Inline(#[serde(with = "serde_bytes")] Vec<u8>),
|
||||||
FirstBlock(ObjectVersionMeta, Hash),
|
FirstBlock(Hash),
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(PartialEq, Clone, Debug, Serialize, Deserialize)]
|
|
||||||
pub struct ObjectVersionMeta {
|
|
||||||
pub headers: ObjectVersionHeaders,
|
|
||||||
pub size: u64,
|
|
||||||
pub etag: String,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(PartialEq, Clone, Debug, Serialize, Deserialize)]
|
|
||||||
pub struct ObjectVersionHeaders {
|
|
||||||
pub content_type: String,
|
|
||||||
pub other: BTreeMap<String, String>,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl ObjectVersionData {
|
|
||||||
fn merge(&mut self, b: &Self) {
|
|
||||||
if *self != *b {
|
|
||||||
warn!(
|
|
||||||
"Inconsistent object version data: {:?} (local) vs {:?} (remote)",
|
|
||||||
self, b
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
impl ObjectVersion {
|
impl ObjectVersion {
|
||||||
fn cmp_key(&self) -> (u64, UUID) {
|
fn cmp_key(&self) -> (u64, UUID) {
|
||||||
(self.timestamp, self.uuid)
|
(self.timestamp, self.uuid)
|
||||||
}
|
}
|
||||||
pub fn is_uploading(&self) -> bool {
|
|
||||||
match self.state {
|
|
||||||
ObjectVersionState::Uploading(_) => true,
|
|
||||||
_ => false,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
pub fn is_complete(&self) -> bool {
|
pub fn is_complete(&self) -> bool {
|
||||||
match self.state {
|
self.state == ObjectVersionState::Complete
|
||||||
ObjectVersionState::Complete(_) => true,
|
|
||||||
_ => false,
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
pub fn is_data(&self) -> bool {
|
pub fn is_data(&self) -> bool {
|
||||||
match self.state {
|
self.state == ObjectVersionState::Complete && self.data != ObjectVersionData::DeleteMarker
|
||||||
ObjectVersionState::Complete(ObjectVersionData::DeleteMarker) => false,
|
|
||||||
ObjectVersionState::Complete(_) => true,
|
|
||||||
_ => false,
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -164,7 +121,14 @@ impl Entry<String, String> for Object {
|
|||||||
.binary_search_by(|v| v.cmp_key().cmp(&other_v.cmp_key()))
|
.binary_search_by(|v| v.cmp_key().cmp(&other_v.cmp_key()))
|
||||||
{
|
{
|
||||||
Ok(i) => {
|
Ok(i) => {
|
||||||
self.versions[i].state.merge(&other_v.state);
|
let mut v = &mut self.versions[i];
|
||||||
|
if other_v.size > v.size {
|
||||||
|
v.size = other_v.size;
|
||||||
|
}
|
||||||
|
v.state = v.state.max(other_v.state);
|
||||||
|
if v.data == ObjectVersionData::Uploading {
|
||||||
|
v.data = other_v.data.clone();
|
||||||
|
}
|
||||||
}
|
}
|
||||||
Err(i) => {
|
Err(i) => {
|
||||||
self.versions.insert(i, other_v.clone());
|
self.versions.insert(i, other_v.clone());
|
||||||
@@ -196,7 +160,7 @@ impl TableSchema for ObjectTable {
|
|||||||
type P = String;
|
type P = String;
|
||||||
type S = String;
|
type S = String;
|
||||||
type E = Object;
|
type E = Object;
|
||||||
type Filter = DeletedFilter;
|
type Filter = ();
|
||||||
|
|
||||||
async fn updated(&self, old: Option<Self::E>, new: Option<Self::E>) -> Result<(), Error> {
|
async fn updated(&self, old: Option<Self::E>, new: Option<Self::E>) -> Result<(), Error> {
|
||||||
let version_table = self.version_table.clone();
|
let version_table = self.version_table.clone();
|
||||||
@@ -228,59 +192,7 @@ impl TableSchema for ObjectTable {
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
fn matches_filter(entry: &Self::E, filter: &Self::Filter) -> bool {
|
fn matches_filter(entry: &Self::E, _filter: &Self::Filter) -> bool {
|
||||||
let deleted = !entry.versions.iter().any(|v| v.is_data());
|
entry.versions.iter().any(|v| v.is_data())
|
||||||
filter.apply(deleted)
|
|
||||||
}
|
|
||||||
|
|
||||||
fn try_migrate(bytes: &[u8]) -> Option<Self::E> {
|
|
||||||
let old = match rmp_serde::decode::from_read_ref::<_, prev::Object>(bytes) {
|
|
||||||
Ok(x) => x,
|
|
||||||
Err(_) => return None,
|
|
||||||
};
|
|
||||||
let new_v = old
|
|
||||||
.versions()
|
|
||||||
.iter()
|
|
||||||
.map(migrate_version)
|
|
||||||
.collect::<Vec<_>>();
|
|
||||||
let new = Object::new(old.bucket.clone(), old.key.clone(), new_v);
|
|
||||||
Some(new)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn migrate_version(old: &prev::ObjectVersion) -> ObjectVersion {
|
|
||||||
let headers = ObjectVersionHeaders {
|
|
||||||
content_type: old.mime_type.clone(),
|
|
||||||
other: BTreeMap::new(),
|
|
||||||
};
|
|
||||||
let meta = ObjectVersionMeta {
|
|
||||||
headers: headers.clone(),
|
|
||||||
size: old.size,
|
|
||||||
etag: "".to_string(),
|
|
||||||
};
|
|
||||||
let state = match old.state {
|
|
||||||
prev::ObjectVersionState::Uploading => ObjectVersionState::Uploading(headers),
|
|
||||||
prev::ObjectVersionState::Aborted => ObjectVersionState::Aborted,
|
|
||||||
prev::ObjectVersionState::Complete => match &old.data {
|
|
||||||
prev::ObjectVersionData::Uploading => ObjectVersionState::Uploading(headers),
|
|
||||||
prev::ObjectVersionData::DeleteMarker => {
|
|
||||||
ObjectVersionState::Complete(ObjectVersionData::DeleteMarker)
|
|
||||||
}
|
|
||||||
prev::ObjectVersionData::Inline(x) => {
|
|
||||||
ObjectVersionState::Complete(ObjectVersionData::Inline(meta, x.clone()))
|
|
||||||
}
|
|
||||||
prev::ObjectVersionData::FirstBlock(h) => {
|
|
||||||
let mut hash = [0u8; 32];
|
|
||||||
hash.copy_from_slice(h.as_ref());
|
|
||||||
ObjectVersionState::Complete(ObjectVersionData::FirstBlock(meta, Hash::from(hash)))
|
|
||||||
}
|
|
||||||
},
|
|
||||||
};
|
|
||||||
let mut uuid = [0u8; 32];
|
|
||||||
uuid.copy_from_slice(old.uuid.as_ref());
|
|
||||||
ObjectVersion {
|
|
||||||
uuid: UUID::from(uuid),
|
|
||||||
timestamp: old.timestamp,
|
|
||||||
state,
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -117,7 +117,7 @@ impl TableSchema for VersionTable {
|
|||||||
type P = Hash;
|
type P = Hash;
|
||||||
type S = EmptyKey;
|
type S = EmptyKey;
|
||||||
type E = Version;
|
type E = Version;
|
||||||
type Filter = DeletedFilter;
|
type Filter = ();
|
||||||
|
|
||||||
async fn updated(&self, old: Option<Self::E>, new: Option<Self::E>) -> Result<(), Error> {
|
async fn updated(&self, old: Option<Self::E>, new: Option<Self::E>) -> Result<(), Error> {
|
||||||
let block_ref_table = self.block_ref_table.clone();
|
let block_ref_table = self.block_ref_table.clone();
|
||||||
@@ -139,7 +139,7 @@ impl TableSchema for VersionTable {
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
fn matches_filter(entry: &Self::E, filter: &Self::Filter) -> bool {
|
fn matches_filter(entry: &Self::E, _filter: &Self::Filter) -> bool {
|
||||||
filter.apply(entry.deleted)
|
!entry.deleted
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
+2
-2
@@ -1,6 +1,6 @@
|
|||||||
[package]
|
[package]
|
||||||
name = "garage_rpc"
|
name = "garage_rpc"
|
||||||
version = "0.1.1"
|
version = "0.1.0"
|
||||||
authors = ["Alex Auvolat <alex@adnab.me>"]
|
authors = ["Alex Auvolat <alex@adnab.me>"]
|
||||||
edition = "2018"
|
edition = "2018"
|
||||||
license = "GPL-3.0"
|
license = "GPL-3.0"
|
||||||
@@ -13,7 +13,7 @@ path = "lib.rs"
|
|||||||
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
|
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
|
||||||
|
|
||||||
[dependencies]
|
[dependencies]
|
||||||
garage_util = { version = "0.1.1", path = "../util" }
|
garage_util = { version = "0.1", path = "../util" }
|
||||||
|
|
||||||
bytes = "0.4"
|
bytes = "0.4"
|
||||||
rand = "0.7"
|
rand = "0.7"
|
||||||
|
|||||||
+1
-1
@@ -5,4 +5,4 @@ pub mod consul;
|
|||||||
pub mod membership;
|
pub mod membership;
|
||||||
pub mod rpc_client;
|
pub mod rpc_client;
|
||||||
pub mod rpc_server;
|
pub mod rpc_server;
|
||||||
pub(crate) mod tls_util;
|
pub mod tls_util;
|
||||||
|
|||||||
+24
-25
@@ -46,13 +46,13 @@ impl RpcMessage for Message {}
|
|||||||
|
|
||||||
#[derive(Debug, Serialize, Deserialize)]
|
#[derive(Debug, Serialize, Deserialize)]
|
||||||
pub struct PingMessage {
|
pub struct PingMessage {
|
||||||
id: UUID,
|
pub id: UUID,
|
||||||
rpc_port: u16,
|
pub rpc_port: u16,
|
||||||
|
|
||||||
status_hash: Hash,
|
pub status_hash: Hash,
|
||||||
config_version: u64,
|
pub config_version: u64,
|
||||||
|
|
||||||
state_info: StateInfo,
|
pub state_info: StateInfo,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Clone, Debug, Serialize, Deserialize)]
|
#[derive(Clone, Debug, Serialize, Deserialize)]
|
||||||
@@ -81,13 +81,12 @@ pub struct NetworkConfigEntry {
|
|||||||
|
|
||||||
pub struct System {
|
pub struct System {
|
||||||
pub id: UUID,
|
pub id: UUID,
|
||||||
|
pub data_dir: PathBuf,
|
||||||
|
pub rpc_local_port: u16,
|
||||||
|
|
||||||
metadata_dir: PathBuf,
|
pub state_info: StateInfo,
|
||||||
rpc_local_port: u16,
|
|
||||||
|
|
||||||
state_info: StateInfo,
|
pub rpc_http_client: Arc<RpcHttpClient>,
|
||||||
|
|
||||||
rpc_http_client: Arc<RpcHttpClient>,
|
|
||||||
rpc_client: Arc<RpcClient<Message>>,
|
rpc_client: Arc<RpcClient<Message>>,
|
||||||
|
|
||||||
pub status: watch::Receiver<Arc<Status>>,
|
pub status: watch::Receiver<Arc<Status>>,
|
||||||
@@ -297,15 +296,15 @@ fn read_network_config(metadata_dir: &PathBuf) -> Result<NetworkConfig, Error> {
|
|||||||
|
|
||||||
impl System {
|
impl System {
|
||||||
pub fn new(
|
pub fn new(
|
||||||
metadata_dir: PathBuf,
|
data_dir: PathBuf,
|
||||||
rpc_http_client: Arc<RpcHttpClient>,
|
rpc_http_client: Arc<RpcHttpClient>,
|
||||||
background: Arc<BackgroundRunner>,
|
background: Arc<BackgroundRunner>,
|
||||||
rpc_server: &mut RpcServer,
|
rpc_server: &mut RpcServer,
|
||||||
) -> Arc<Self> {
|
) -> Arc<Self> {
|
||||||
let id = gen_node_id(&metadata_dir).expect("Unable to read or generate node ID");
|
let id = gen_node_id(&data_dir).expect("Unable to read or generate node ID");
|
||||||
info!("Node ID: {}", hex::encode(&id));
|
info!("Node ID: {}", hex::encode(&id));
|
||||||
|
|
||||||
let net_config = match read_network_config(&metadata_dir) {
|
let net_config = match read_network_config(&data_dir) {
|
||||||
Ok(x) => x,
|
Ok(x) => x,
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
info!(
|
info!(
|
||||||
@@ -348,7 +347,7 @@ impl System {
|
|||||||
|
|
||||||
let sys = Arc::new(System {
|
let sys = Arc::new(System {
|
||||||
id,
|
id,
|
||||||
metadata_dir,
|
data_dir,
|
||||||
rpc_local_port: rpc_server.bind_addr.port(),
|
rpc_local_port: rpc_server.bind_addr.port(),
|
||||||
state_info,
|
state_info,
|
||||||
rpc_http_client,
|
rpc_http_client,
|
||||||
@@ -374,7 +373,7 @@ impl System {
|
|||||||
Message::AdvertiseNodesUp(adv) => self2.handle_advertise_nodes_up(&adv).await,
|
Message::AdvertiseNodesUp(adv) => self2.handle_advertise_nodes_up(&adv).await,
|
||||||
Message::AdvertiseConfig(adv) => self2.handle_advertise_config(&adv).await,
|
Message::AdvertiseConfig(adv) => self2.handle_advertise_config(&adv).await,
|
||||||
|
|
||||||
_ => Err(Error::BadRPC(format!("Unexpected RPC message"))),
|
_ => Err(Error::BadRequest(format!("Unexpected RPC message"))),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
@@ -389,7 +388,7 @@ impl System {
|
|||||||
}
|
}
|
||||||
|
|
||||||
async fn save_network_config(self: Arc<Self>) -> Result<(), Error> {
|
async fn save_network_config(self: Arc<Self>) -> Result<(), Error> {
|
||||||
let mut path = self.metadata_dir.clone();
|
let mut path = self.data_dir.clone();
|
||||||
path.push("network_config");
|
path.push("network_config");
|
||||||
|
|
||||||
let ring = self.ring.borrow().clone();
|
let ring = self.ring.borrow().clone();
|
||||||
@@ -400,7 +399,7 @@ impl System {
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
fn make_ping(&self) -> Message {
|
pub fn make_ping(&self) -> Message {
|
||||||
let status = self.status.borrow().clone();
|
let status = self.status.borrow().clone();
|
||||||
let ring = self.ring.borrow().clone();
|
let ring = self.ring.borrow().clone();
|
||||||
Message::Ping(PingMessage {
|
Message::Ping(PingMessage {
|
||||||
@@ -412,7 +411,7 @@ impl System {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn broadcast(self: Arc<Self>, msg: Message, timeout: Duration) {
|
pub async fn broadcast(self: Arc<Self>, msg: Message, timeout: Duration) {
|
||||||
let status = self.status.borrow().clone();
|
let status = self.status.borrow().clone();
|
||||||
let to = status
|
let to = status
|
||||||
.nodes
|
.nodes
|
||||||
@@ -528,7 +527,7 @@ impl System {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn handle_ping(
|
pub async fn handle_ping(
|
||||||
self: Arc<Self>,
|
self: Arc<Self>,
|
||||||
from: &SocketAddr,
|
from: &SocketAddr,
|
||||||
ping: &PingMessage,
|
ping: &PingMessage,
|
||||||
@@ -558,7 +557,7 @@ impl System {
|
|||||||
Ok(self.make_ping())
|
Ok(self.make_ping())
|
||||||
}
|
}
|
||||||
|
|
||||||
fn handle_pull_status(&self) -> Result<Message, Error> {
|
pub fn handle_pull_status(&self) -> Result<Message, Error> {
|
||||||
let status = self.status.borrow().clone();
|
let status = self.status.borrow().clone();
|
||||||
let mut mem = vec![];
|
let mut mem = vec![];
|
||||||
for (node, status) in status.nodes.iter() {
|
for (node, status) in status.nodes.iter() {
|
||||||
@@ -578,12 +577,12 @@ impl System {
|
|||||||
Ok(Message::AdvertiseNodesUp(mem))
|
Ok(Message::AdvertiseNodesUp(mem))
|
||||||
}
|
}
|
||||||
|
|
||||||
fn handle_pull_config(&self) -> Result<Message, Error> {
|
pub fn handle_pull_config(&self) -> Result<Message, Error> {
|
||||||
let ring = self.ring.borrow().clone();
|
let ring = self.ring.borrow().clone();
|
||||||
Ok(Message::AdvertiseConfig(ring.config.clone()))
|
Ok(Message::AdvertiseConfig(ring.config.clone()))
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn handle_advertise_nodes_up(
|
pub async fn handle_advertise_nodes_up(
|
||||||
self: Arc<Self>,
|
self: Arc<Self>,
|
||||||
adv: &[AdvertisedNode],
|
adv: &[AdvertisedNode],
|
||||||
) -> Result<Message, Error> {
|
) -> Result<Message, Error> {
|
||||||
@@ -636,7 +635,7 @@ impl System {
|
|||||||
Ok(Message::Ok)
|
Ok(Message::Ok)
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn handle_advertise_config(
|
pub async fn handle_advertise_config(
|
||||||
self: Arc<Self>,
|
self: Arc<Self>,
|
||||||
adv: &NetworkConfig,
|
adv: &NetworkConfig,
|
||||||
) -> Result<Message, Error> {
|
) -> Result<Message, Error> {
|
||||||
@@ -717,7 +716,7 @@ impl System {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn pull_status(
|
pub fn pull_status(
|
||||||
self: Arc<Self>,
|
self: Arc<Self>,
|
||||||
peer: UUID,
|
peer: UUID,
|
||||||
) -> impl futures::future::Future<Output = ()> + Send + 'static {
|
) -> impl futures::future::Future<Output = ()> + Send + 'static {
|
||||||
@@ -732,7 +731,7 @@ impl System {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn pull_config(self: Arc<Self>, peer: UUID) {
|
pub async fn pull_config(self: Arc<Self>, peer: UUID) {
|
||||||
let resp = self
|
let resp = self
|
||||||
.rpc_client
|
.rpc_client
|
||||||
.call(peer, Message::PullConfig, PING_TIMEOUT)
|
.call(peer, Message::PullConfig, PING_TIMEOUT)
|
||||||
|
|||||||
@@ -61,7 +61,7 @@ pub struct RpcClient<M: RpcMessage> {
|
|||||||
|
|
||||||
local_handler: ArcSwapOption<(UUID, LocalHandlerFn<M>)>,
|
local_handler: ArcSwapOption<(UUID, LocalHandlerFn<M>)>,
|
||||||
|
|
||||||
rpc_addr_client: RpcAddrClient<M>,
|
pub rpc_addr_client: RpcAddrClient<M>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl<M: RpcMessage + 'static> RpcClient<M> {
|
impl<M: RpcMessage + 'static> RpcClient<M> {
|
||||||
@@ -215,8 +215,8 @@ impl<M: RpcMessage + 'static> RpcClient<M> {
|
|||||||
pub struct RpcAddrClient<M: RpcMessage> {
|
pub struct RpcAddrClient<M: RpcMessage> {
|
||||||
phantom: PhantomData<M>,
|
phantom: PhantomData<M>,
|
||||||
|
|
||||||
http_client: Arc<RpcHttpClient>,
|
pub http_client: Arc<RpcHttpClient>,
|
||||||
path: String,
|
pub path: String,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl<M: RpcMessage> RpcAddrClient<M> {
|
impl<M: RpcMessage> RpcAddrClient<M> {
|
||||||
|
|||||||
@@ -61,10 +61,7 @@ where
|
|||||||
let err_str = format!("{}", e);
|
let err_str = format!("{}", e);
|
||||||
let rep_bytes = rmp_to_vec_all_named::<Result<M, String>>(&Err(err_str))?;
|
let rep_bytes = rmp_to_vec_all_named::<Result<M, String>>(&Err(err_str))?;
|
||||||
let mut err_response = Response::new(Body::from(rep_bytes));
|
let mut err_response = Response::new(Body::from(rep_bytes));
|
||||||
*err_response.status_mut() = match e {
|
*err_response.status_mut() = e.http_status_code();
|
||||||
Error::BadRPC(_) => StatusCode::BAD_REQUEST,
|
|
||||||
_ => StatusCode::INTERNAL_SERVER_ERROR,
|
|
||||||
};
|
|
||||||
warn!(
|
warn!(
|
||||||
"RPC error ({}): {} ({} ms)",
|
"RPC error ({}): {} ({} ms)",
|
||||||
name,
|
name,
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
[package]
|
[package]
|
||||||
name = "garage_table"
|
name = "garage_table"
|
||||||
version = "0.1.1"
|
version = "0.1.0"
|
||||||
authors = ["Alex Auvolat <alex@adnab.me>"]
|
authors = ["Alex Auvolat <alex@adnab.me>"]
|
||||||
edition = "2018"
|
edition = "2018"
|
||||||
license = "GPL-3.0"
|
license = "GPL-3.0"
|
||||||
@@ -13,15 +13,14 @@ path = "lib.rs"
|
|||||||
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
|
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
|
||||||
|
|
||||||
[dependencies]
|
[dependencies]
|
||||||
garage_util = { version = "0.1.1", path = "../util" }
|
garage_util = { version = "0.1", path = "../util" }
|
||||||
garage_rpc = { version = "0.1.1", path = "../rpc" }
|
garage_rpc = { version = "0.1", path = "../rpc" }
|
||||||
|
|
||||||
bytes = "0.4"
|
bytes = "0.4"
|
||||||
rand = "0.7"
|
rand = "0.7"
|
||||||
hex = "0.3"
|
hex = "0.3"
|
||||||
arc-swap = "0.4"
|
arc-swap = "0.4"
|
||||||
log = "0.4"
|
log = "0.4"
|
||||||
hexdump = "0.1"
|
|
||||||
|
|
||||||
sled = "0.31"
|
sled = "0.31"
|
||||||
|
|
||||||
|
|||||||
@@ -1,327 +0,0 @@
|
|||||||
//! This package provides a simple implementation of conflict-free replicated data types (CRDTs)
|
|
||||||
//!
|
|
||||||
//! CRDTs are a type of data structures that do not require coordination. In other words, we can
|
|
||||||
//! edit them in parallel, we will always find a way to merge it.
|
|
||||||
//!
|
|
||||||
//! A general example is a counter. Its initial value is 0. Alice and Bob get a copy of the
|
|
||||||
//! counter. Alice does +1 on her copy, she reads 1. Bob does +3 on his copy, he reads 3. Now,
|
|
||||||
//! it is easy to merge their counters, order does not count: we always get 4.
|
|
||||||
//!
|
|
||||||
//! Learn more about CRDT [on Wikipedia](https://en.wikipedia.org/wiki/Conflict-free_replicated_data_type)
|
|
||||||
|
|
||||||
use serde::{Deserialize, Serialize};
|
|
||||||
|
|
||||||
use garage_util::data::*;
|
|
||||||
|
|
||||||
/// Definition of a CRDT - all CRDT Rust types implement this.
|
|
||||||
///
|
|
||||||
/// A CRDT is defined as a merge operator that respects a certain set of axioms.
|
|
||||||
///
|
|
||||||
/// In particular, the merge operator must be commutative, associative,
|
|
||||||
/// idempotent, and monotonic.
|
|
||||||
/// In other words, if `a`, `b` and `c` are CRDTs, and `⊔` denotes the merge operator,
|
|
||||||
/// the following axioms must apply:
|
|
||||||
///
|
|
||||||
/// ```text
|
|
||||||
/// a ⊔ b = b ⊔ a (commutativity)
|
|
||||||
/// (a ⊔ b) ⊔ c = a ⊔ (b ⊔ c) (associativity)
|
|
||||||
/// (a ⊔ b) ⊔ b = a ⊔ b (idempotence)
|
|
||||||
/// ```
|
|
||||||
///
|
|
||||||
/// Moreover, the relationship `≥` defined by `a ≥ b ⇔ ∃c. a = b ⊔ c` must be a partial order.
|
|
||||||
/// This implies a few properties such as: if `a ⊔ b ≠ a`, then there is no `c` such that `(a ⊔ b) ⊔ c = a`,
|
|
||||||
/// as this would imply a cycle in the partial order.
|
|
||||||
pub trait CRDT {
|
|
||||||
/// Merge the two datastructures according to the CRDT rules.
|
|
||||||
/// `self` is modified to contain the merged CRDT value. `other` is not modified.
|
|
||||||
///
|
|
||||||
/// # Arguments
|
|
||||||
///
|
|
||||||
/// * `other` - the other CRDT we wish to merge with
|
|
||||||
fn merge(&mut self, other: &Self);
|
|
||||||
}
|
|
||||||
|
|
||||||
/// All types that implement `Ord` (a total order) also implement a trivial CRDT
|
|
||||||
/// defined by the merge rule: `a ⊔ b = max(a, b)`.
|
|
||||||
impl<T> CRDT for T
|
|
||||||
where
|
|
||||||
T: Ord + Clone,
|
|
||||||
{
|
|
||||||
fn merge(&mut self, other: &Self) {
|
|
||||||
if other > self {
|
|
||||||
*self = other.clone();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// ---- LWW Register ----
|
|
||||||
|
|
||||||
/// Last Write Win (LWW)
|
|
||||||
///
|
|
||||||
/// An LWW CRDT associates a timestamp with a value, in order to implement a
|
|
||||||
/// time-based reconciliation rule: the most recent write wins.
|
|
||||||
/// For completeness, the LWW reconciliation rule must also be defined for two LWW CRDTs
|
|
||||||
/// with the same timestamp but different values.
|
|
||||||
///
|
|
||||||
/// In our case, we add the constraint that the value that is wrapped inside the LWW CRDT must
|
|
||||||
/// itself be a CRDT: in the case when the timestamp does not allow us to decide on which value to
|
|
||||||
/// keep, the merge rule of the inner CRDT is applied on the wrapped values. (Note that all types
|
|
||||||
/// that implement the `Ord` trait get a default CRDT implemetnation that keeps the maximum value.
|
|
||||||
/// This enables us to use LWW directly with primitive data types such as numbers or strings. It is
|
|
||||||
/// generally desirable in this case to never explicitly produce LWW values with the same timestamp
|
|
||||||
/// but different inner values, as the rule to keep the maximum value isn't generally the desired
|
|
||||||
/// semantics.)
|
|
||||||
///
|
|
||||||
/// As multiple computers clocks are always desynchronized,
|
|
||||||
/// when operations are close enough, it is equivalent to
|
|
||||||
/// take one copy and drop the other one.
|
|
||||||
///
|
|
||||||
/// Given that clocks are not too desynchronized, this assumption
|
|
||||||
/// is enough for most cases, as there is few chance that two humans
|
|
||||||
/// coordonate themself faster than the time difference between two NTP servers.
|
|
||||||
///
|
|
||||||
/// As a more concret example, let's suppose you want to upload a file
|
|
||||||
/// with the same key (path) in the same bucket at the very same time.
|
|
||||||
/// For each request, the file will be timestamped by the receiving server
|
|
||||||
/// and may differ from what you observed with your atomic clock!
|
|
||||||
///
|
|
||||||
/// This scheme is used by AWS S3 or Soundcloud and often without knowing
|
|
||||||
/// in entreprise when reconciliating databases with ad-hoc scripts.
|
|
||||||
#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
|
|
||||||
pub struct LWW<T> {
|
|
||||||
ts: u64,
|
|
||||||
v: T,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl<T> LWW<T>
|
|
||||||
where
|
|
||||||
T: CRDT,
|
|
||||||
{
|
|
||||||
/// Creates a new CRDT
|
|
||||||
///
|
|
||||||
/// CRDT's internal timestamp is set with current node's clock.
|
|
||||||
pub fn new(value: T) -> Self {
|
|
||||||
Self {
|
|
||||||
ts: now_msec(),
|
|
||||||
v: value,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Build a new CRDT from a previous non-compatible one
|
|
||||||
///
|
|
||||||
/// Compared to new, the CRDT's timestamp is not set to now
|
|
||||||
/// but must be set to the previous, non-compatible, CRDT's timestamp.
|
|
||||||
pub fn migrate_from_raw(ts: u64, value: T) -> Self {
|
|
||||||
Self { ts, v: value }
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Update the LWW CRDT while keeping some causal ordering.
|
|
||||||
///
|
|
||||||
/// The timestamp of the LWW CRDT is updated to be the current node's clock
|
|
||||||
/// at time of update, or the previous timestamp + 1 if that's bigger,
|
|
||||||
/// so that the new timestamp is always strictly larger than the previous one.
|
|
||||||
/// This ensures that merging the update with the old value will result in keeping
|
|
||||||
/// the updated value.
|
|
||||||
pub fn update(&mut self, new_value: T) {
|
|
||||||
self.ts = std::cmp::max(self.ts + 1, now_msec());
|
|
||||||
self.v = new_value;
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Get the CRDT value
|
|
||||||
pub fn get(&self) -> &T {
|
|
||||||
&self.v
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Get a mutable reference to the CRDT's value
|
|
||||||
///
|
|
||||||
/// This is usefull to mutate the inside value without changing the LWW timestamp.
|
|
||||||
/// When such mutation is done, the merge between two LWW values is done using the inner
|
|
||||||
/// CRDT's merge operation. This is usefull in the case where the inner CRDT is a large
|
|
||||||
/// data type, such as a map, and we only want to change a single item in the map.
|
|
||||||
/// To do this, we can produce a "CRDT delta", i.e. a LWW that contains only the modification.
|
|
||||||
/// This delta consists in a LWW with the same timestamp, and the map
|
|
||||||
/// inside only contains the updated value.
|
|
||||||
/// The advantage of such a delta is that it is much smaller than the whole map.
|
|
||||||
///
|
|
||||||
/// Avoid using this if the inner data type is a primitive type such as a number or a string,
|
|
||||||
/// as you will then rely on the merge function defined on `Ord` types by keeping the maximum
|
|
||||||
/// of both values.
|
|
||||||
pub fn get_mut(&mut self) -> &mut T {
|
|
||||||
&mut self.v
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl<T> CRDT for LWW<T>
|
|
||||||
where
|
|
||||||
T: Clone + CRDT,
|
|
||||||
{
|
|
||||||
fn merge(&mut self, other: &Self) {
|
|
||||||
if other.ts > self.ts {
|
|
||||||
self.ts = other.ts;
|
|
||||||
self.v = other.v.clone();
|
|
||||||
} else if other.ts == self.ts {
|
|
||||||
self.v.merge(&other.v);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Boolean, where `true` is an absorbing state
|
|
||||||
#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq)]
|
|
||||||
pub struct Bool(bool);
|
|
||||||
|
|
||||||
impl Bool {
|
|
||||||
/// Create a new boolean with the specified value
|
|
||||||
pub fn new(b: bool) -> Self {
|
|
||||||
Self(b)
|
|
||||||
}
|
|
||||||
/// Set the boolean to true
|
|
||||||
pub fn set(&mut self) {
|
|
||||||
self.0 = true;
|
|
||||||
}
|
|
||||||
/// Get the boolean value
|
|
||||||
pub fn get(&self) -> bool {
|
|
||||||
self.0
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl CRDT for Bool {
|
|
||||||
fn merge(&mut self, other: &Self) {
|
|
||||||
self.0 = self.0 || other.0;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Last Write Win Map
|
|
||||||
///
|
|
||||||
/// This types defines a CRDT for a map from keys to values.
|
|
||||||
/// The values have an associated timestamp, such that the last written value
|
|
||||||
/// takes precedence over previous ones. As for the simpler `LWW` type, the value
|
|
||||||
/// type `V` is also required to implement the CRDT trait.
|
|
||||||
/// We do not encourage mutating the values associated with a given key
|
|
||||||
/// without updating the timestamp, in fact at the moment we do not provide a `.get_mut()`
|
|
||||||
/// method that would allow that.
|
|
||||||
///
|
|
||||||
/// Internally, the map is stored as a vector of keys and values, sorted by ascending key order.
|
|
||||||
/// This is why the key type `K` must implement `Ord` (and also to ensure a unique serialization,
|
|
||||||
/// such that two values can be compared for equality based on their hashes). As a consequence,
|
|
||||||
/// insertions take `O(n)` time. This means that LWWMap should be used for reasonably small maps.
|
|
||||||
/// However, note that even if we were using a more efficient data structure such as a `BTreeMap`,
|
|
||||||
/// the serialization cost `O(n)` would still have to be paid at each modification, so we are
|
|
||||||
/// actually not losing anything here.
|
|
||||||
#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
|
|
||||||
pub struct LWWMap<K, V> {
|
|
||||||
vals: Vec<(K, u64, V)>,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl<K, V> LWWMap<K, V>
|
|
||||||
where
|
|
||||||
K: Ord,
|
|
||||||
V: CRDT,
|
|
||||||
{
|
|
||||||
/// Create a new empty map CRDT
|
|
||||||
pub fn new() -> Self {
|
|
||||||
Self { vals: vec![] }
|
|
||||||
}
|
|
||||||
/// Used to migrate from a map defined in an incompatible format. This produces
|
|
||||||
/// a map that contains a single item with the specified timestamp (copied from
|
|
||||||
/// the incompatible format). Do this as many times as you have items to migrate,
|
|
||||||
/// and put them all together using the CRDT merge operator.
|
|
||||||
pub fn migrate_from_raw_item(k: K, ts: u64, v: V) -> Self {
|
|
||||||
Self {
|
|
||||||
vals: vec![(k, ts, v)],
|
|
||||||
}
|
|
||||||
}
|
|
||||||
/// Returns a map that contains a single mapping from the specified key to the specified value.
|
|
||||||
/// This map is a mutator, or a delta-CRDT, such that when it is merged with the original map,
|
|
||||||
/// the previous value will be replaced with the one specified here.
|
|
||||||
/// The timestamp in the provided mutator is set to the maximum of the current system's clock
|
|
||||||
/// and 1 + the previous value's timestamp (if there is one), so that the new value will always
|
|
||||||
/// take precedence (LWW rule).
|
|
||||||
///
|
|
||||||
/// Typically, to update the value associated to a key in the map, you would do the following:
|
|
||||||
///
|
|
||||||
/// ```ignore
|
|
||||||
/// let my_update = my_crdt.update_mutator(key_to_modify, new_value);
|
|
||||||
/// my_crdt.merge(&my_update);
|
|
||||||
/// ```
|
|
||||||
///
|
|
||||||
/// However extracting the mutator on its own and only sending that on the network is very
|
|
||||||
/// interesting as it is much smaller than the whole map.
|
|
||||||
pub fn update_mutator(&self, k: K, new_v: V) -> Self {
|
|
||||||
let new_vals = match self.vals.binary_search_by(|(k2, _, _)| k2.cmp(&k)) {
|
|
||||||
Ok(i) => {
|
|
||||||
let (_, old_ts, _) = self.vals[i];
|
|
||||||
let new_ts = std::cmp::max(old_ts + 1, now_msec());
|
|
||||||
vec![(k, new_ts, new_v)]
|
|
||||||
}
|
|
||||||
Err(_) => vec![(k, now_msec(), new_v)],
|
|
||||||
};
|
|
||||||
Self { vals: new_vals }
|
|
||||||
}
|
|
||||||
/// Takes all of the values of the map and returns them. The current map is reset to the
|
|
||||||
/// empty map. This is very usefull to produce in-place a new map that contains only a delta
|
|
||||||
/// that modifies a certain value:
|
|
||||||
///
|
|
||||||
/// ```ignore
|
|
||||||
/// let mut a = get_my_crdt_value();
|
|
||||||
/// let old_a = a.take_and_clear();
|
|
||||||
/// a.merge(&old_a.update_mutator(key_to_modify, new_value));
|
|
||||||
/// put_my_crdt_value(a);
|
|
||||||
/// ```
|
|
||||||
///
|
|
||||||
/// Of course in this simple example we could have written simply
|
|
||||||
/// `pyt_my_crdt_value(a.update_mutator(key_to_modify, new_value))`,
|
|
||||||
/// but in the case where the map is a field in a struct for instance (as is always the case),
|
|
||||||
/// this becomes very handy:
|
|
||||||
///
|
|
||||||
/// ```ignore
|
|
||||||
/// let mut a = get_my_crdt_value();
|
|
||||||
/// let old_a_map = a.map_field.take_and_clear();
|
|
||||||
/// a.map_field.merge(&old_a_map.update_mutator(key_to_modify, new_value));
|
|
||||||
/// put_my_crdt_value(a);
|
|
||||||
/// ```
|
|
||||||
pub fn take_and_clear(&mut self) -> Self {
|
|
||||||
let vals = std::mem::replace(&mut self.vals, vec![]);
|
|
||||||
Self { vals }
|
|
||||||
}
|
|
||||||
/// Removes all values from the map
|
|
||||||
pub fn clear(&mut self) {
|
|
||||||
self.vals.clear();
|
|
||||||
}
|
|
||||||
/// Get a reference to the value assigned to a key
|
|
||||||
pub fn get(&self, k: &K) -> Option<&V> {
|
|
||||||
match self.vals.binary_search_by(|(k2, _, _)| k2.cmp(&k)) {
|
|
||||||
Ok(i) => Some(&self.vals[i].2),
|
|
||||||
Err(_) => None,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
/// Gets a reference to all of the items, as a slice. Usefull to iterate on all map values.
|
|
||||||
/// In most case you will want to ignore the timestamp (second item of the tuple).
|
|
||||||
pub fn items(&self) -> &[(K, u64, V)] {
|
|
||||||
&self.vals[..]
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl<K, V> CRDT for LWWMap<K, V>
|
|
||||||
where
|
|
||||||
K: Clone + Ord,
|
|
||||||
V: Clone + CRDT,
|
|
||||||
{
|
|
||||||
fn merge(&mut self, other: &Self) {
|
|
||||||
for (k, ts2, v2) in other.vals.iter() {
|
|
||||||
match self.vals.binary_search_by(|(k2, _, _)| k2.cmp(&k)) {
|
|
||||||
Ok(i) => {
|
|
||||||
let (_, ts1, _v1) = &self.vals[i];
|
|
||||||
if ts2 > ts1 {
|
|
||||||
self.vals[i].1 = *ts2;
|
|
||||||
self.vals[i].2 = v2.clone();
|
|
||||||
} else if ts1 == ts2 {
|
|
||||||
self.vals[i].2.merge(&v2);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Err(i) => {
|
|
||||||
self.vals.insert(i, (k.clone(), *ts2, v2.clone()));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -3,15 +3,9 @@
|
|||||||
#[macro_use]
|
#[macro_use]
|
||||||
extern crate log;
|
extern crate log;
|
||||||
|
|
||||||
pub mod crdt;
|
|
||||||
pub mod schema;
|
|
||||||
pub mod util;
|
|
||||||
|
|
||||||
pub mod table;
|
pub mod table;
|
||||||
pub mod table_fullcopy;
|
pub mod table_fullcopy;
|
||||||
pub mod table_sharded;
|
pub mod table_sharded;
|
||||||
pub mod table_sync;
|
pub mod table_sync;
|
||||||
|
|
||||||
pub use schema::*;
|
|
||||||
pub use table::*;
|
pub use table::*;
|
||||||
pub use util::*;
|
|
||||||
|
|||||||
@@ -1,65 +0,0 @@
|
|||||||
use async_trait::async_trait;
|
|
||||||
use serde::{Deserialize, Serialize};
|
|
||||||
|
|
||||||
use garage_util::data::*;
|
|
||||||
use garage_util::error::Error;
|
|
||||||
|
|
||||||
pub trait PartitionKey {
|
|
||||||
fn hash(&self) -> Hash;
|
|
||||||
}
|
|
||||||
|
|
||||||
impl PartitionKey for String {
|
|
||||||
fn hash(&self) -> Hash {
|
|
||||||
hash(self.as_bytes())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl PartitionKey for Hash {
|
|
||||||
fn hash(&self) -> Hash {
|
|
||||||
self.clone()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
pub trait SortKey {
|
|
||||||
fn sort_key(&self) -> &[u8];
|
|
||||||
}
|
|
||||||
|
|
||||||
impl SortKey for String {
|
|
||||||
fn sort_key(&self) -> &[u8] {
|
|
||||||
self.as_bytes()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl SortKey for Hash {
|
|
||||||
fn sort_key(&self) -> &[u8] {
|
|
||||||
self.as_slice()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
pub trait Entry<P: PartitionKey, S: SortKey>:
|
|
||||||
PartialEq + Clone + Serialize + for<'de> Deserialize<'de> + Send + Sync
|
|
||||||
{
|
|
||||||
fn partition_key(&self) -> &P;
|
|
||||||
fn sort_key(&self) -> &S;
|
|
||||||
|
|
||||||
fn merge(&mut self, other: &Self);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[async_trait]
|
|
||||||
pub trait TableSchema: Send + Sync {
|
|
||||||
type P: PartitionKey + Clone + PartialEq + Serialize + for<'de> Deserialize<'de> + Send + Sync;
|
|
||||||
type S: SortKey + Clone + Serialize + for<'de> Deserialize<'de> + Send + Sync;
|
|
||||||
type E: Entry<Self::P, Self::S>;
|
|
||||||
type Filter: Clone + Serialize + for<'de> Deserialize<'de> + Send + Sync;
|
|
||||||
|
|
||||||
// Action to take if not able to decode current version:
|
|
||||||
// try loading from an older version
|
|
||||||
fn try_migrate(_bytes: &[u8]) -> Option<Self::E> {
|
|
||||||
None
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn updated(&self, old: Option<Self::E>, new: Option<Self::E>) -> Result<(), Error>;
|
|
||||||
fn matches_filter(_entry: &Self::E, _filter: &Self::Filter) -> bool {
|
|
||||||
true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
+75
-27
@@ -2,9 +2,8 @@ use std::collections::{BTreeMap, HashMap};
|
|||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
use std::time::Duration;
|
use std::time::Duration;
|
||||||
|
|
||||||
use log::warn;
|
|
||||||
|
|
||||||
use arc_swap::ArcSwapOption;
|
use arc_swap::ArcSwapOption;
|
||||||
|
use async_trait::async_trait;
|
||||||
use futures::stream::*;
|
use futures::stream::*;
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
use serde_bytes::ByteBuf;
|
use serde_bytes::ByteBuf;
|
||||||
@@ -16,7 +15,6 @@ use garage_rpc::membership::{Ring, System};
|
|||||||
use garage_rpc::rpc_client::*;
|
use garage_rpc::rpc_client::*;
|
||||||
use garage_rpc::rpc_server::*;
|
use garage_rpc::rpc_server::*;
|
||||||
|
|
||||||
use crate::schema::*;
|
|
||||||
use crate::table_sync::*;
|
use crate::table_sync::*;
|
||||||
|
|
||||||
const TABLE_RPC_TIMEOUT: Duration = Duration::from_secs(10);
|
const TABLE_RPC_TIMEOUT: Duration = Duration::from_secs(10);
|
||||||
@@ -50,6 +48,71 @@ pub enum TableRPC<F: TableSchema> {
|
|||||||
|
|
||||||
impl<F: TableSchema> RpcMessage for TableRPC<F> {}
|
impl<F: TableSchema> RpcMessage for TableRPC<F> {}
|
||||||
|
|
||||||
|
pub trait PartitionKey {
|
||||||
|
fn hash(&self) -> Hash;
|
||||||
|
}
|
||||||
|
|
||||||
|
pub trait SortKey {
|
||||||
|
fn sort_key(&self) -> &[u8];
|
||||||
|
}
|
||||||
|
|
||||||
|
pub trait Entry<P: PartitionKey, S: SortKey>:
|
||||||
|
PartialEq + Clone + Serialize + for<'de> Deserialize<'de> + Send + Sync
|
||||||
|
{
|
||||||
|
fn partition_key(&self) -> &P;
|
||||||
|
fn sort_key(&self) -> &S;
|
||||||
|
|
||||||
|
fn merge(&mut self, other: &Self);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
|
pub struct EmptyKey;
|
||||||
|
impl SortKey for EmptyKey {
|
||||||
|
fn sort_key(&self) -> &[u8] {
|
||||||
|
&[]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
impl PartitionKey for EmptyKey {
|
||||||
|
fn hash(&self) -> Hash {
|
||||||
|
[0u8; 32].into()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl PartitionKey for String {
|
||||||
|
fn hash(&self) -> Hash {
|
||||||
|
hash(self.as_bytes())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
impl SortKey for String {
|
||||||
|
fn sort_key(&self) -> &[u8] {
|
||||||
|
self.as_bytes()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl PartitionKey for Hash {
|
||||||
|
fn hash(&self) -> Hash {
|
||||||
|
self.clone()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
impl SortKey for Hash {
|
||||||
|
fn sort_key(&self) -> &[u8] {
|
||||||
|
self.as_slice()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[async_trait]
|
||||||
|
pub trait TableSchema: Send + Sync {
|
||||||
|
type P: PartitionKey + Clone + PartialEq + Serialize + for<'de> Deserialize<'de> + Send + Sync;
|
||||||
|
type S: SortKey + Clone + Serialize + for<'de> Deserialize<'de> + Send + Sync;
|
||||||
|
type E: Entry<Self::P, Self::S>;
|
||||||
|
type Filter: Clone + Serialize + for<'de> Deserialize<'de> + Send + Sync;
|
||||||
|
|
||||||
|
async fn updated(&self, old: Option<Self::E>, new: Option<Self::E>) -> Result<(), Error>;
|
||||||
|
fn matches_filter(_entry: &Self::E, _filter: &Self::Filter) -> bool {
|
||||||
|
true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
pub trait TableReplication: Send + Sync {
|
pub trait TableReplication: Send + Sync {
|
||||||
// See examples in table_sharded.rs and table_fullcopy.rs
|
// See examples in table_sharded.rs and table_fullcopy.rs
|
||||||
// To understand various replication methods
|
// To understand various replication methods
|
||||||
@@ -187,7 +250,7 @@ where
|
|||||||
for resp in resps {
|
for resp in resps {
|
||||||
if let TableRPC::ReadEntryResponse(value) = resp {
|
if let TableRPC::ReadEntryResponse(value) = resp {
|
||||||
if let Some(v_bytes) = value {
|
if let Some(v_bytes) = value {
|
||||||
let v = self.decode_entry(v_bytes.as_slice())?;
|
let v = rmp_serde::decode::from_read_ref::<_, F::E>(v_bytes.as_slice())?;
|
||||||
ret = match ret {
|
ret = match ret {
|
||||||
None => Some(v),
|
None => Some(v),
|
||||||
Some(mut x) => {
|
Some(mut x) => {
|
||||||
@@ -243,7 +306,8 @@ where
|
|||||||
for resp in resps {
|
for resp in resps {
|
||||||
if let TableRPC::Update(entries) = resp {
|
if let TableRPC::Update(entries) = resp {
|
||||||
for entry_bytes in entries.iter() {
|
for entry_bytes in entries.iter() {
|
||||||
let entry = self.decode_entry(entry_bytes.as_slice())?;
|
let entry =
|
||||||
|
rmp_serde::decode::from_read_ref::<_, F::E>(entry_bytes.as_slice())?;
|
||||||
let entry_key = self.tree_key(entry.partition_key(), entry.sort_key());
|
let entry_key = self.tree_key(entry.partition_key(), entry.sort_key());
|
||||||
match ret.remove(&entry_key) {
|
match ret.remove(&entry_key) {
|
||||||
None => {
|
None => {
|
||||||
@@ -331,7 +395,7 @@ where
|
|||||||
.await?;
|
.await?;
|
||||||
Ok(TableRPC::SyncRPC(response))
|
Ok(TableRPC::SyncRPC(response))
|
||||||
}
|
}
|
||||||
_ => Err(Error::BadRPC(format!("Unexpected table RPC"))),
|
_ => Err(Error::BadRequest(format!("Unexpected table RPC"))),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -365,7 +429,7 @@ where
|
|||||||
let keep = match filter {
|
let keep = match filter {
|
||||||
None => true,
|
None => true,
|
||||||
Some(f) => {
|
Some(f) => {
|
||||||
let entry = self.decode_entry(value.as_ref())?;
|
let entry = rmp_serde::decode::from_read_ref::<_, F::E>(value.as_ref())?;
|
||||||
F::matches_filter(&entry, f)
|
F::matches_filter(&entry, f)
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
@@ -384,15 +448,15 @@ where
|
|||||||
let mut epidemic_propagate = vec![];
|
let mut epidemic_propagate = vec![];
|
||||||
|
|
||||||
for update_bytes in entries.iter() {
|
for update_bytes in entries.iter() {
|
||||||
let update = self.decode_entry(update_bytes.as_slice())?;
|
let update = rmp_serde::decode::from_read_ref::<_, F::E>(update_bytes.as_slice())?;
|
||||||
|
|
||||||
let tree_key = self.tree_key(update.partition_key(), update.sort_key());
|
let tree_key = self.tree_key(update.partition_key(), update.sort_key());
|
||||||
|
|
||||||
let (old_entry, new_entry) = self.store.transaction(|db| {
|
let (old_entry, new_entry) = self.store.transaction(|db| {
|
||||||
let (old_entry, new_entry) = match db.get(&tree_key)? {
|
let (old_entry, new_entry) = match db.get(&tree_key)? {
|
||||||
Some(prev_bytes) => {
|
Some(prev_bytes) => {
|
||||||
let old_entry = self
|
let old_entry = rmp_serde::decode::from_read_ref::<_, F::E>(&prev_bytes)
|
||||||
.decode_entry(&prev_bytes)
|
.map_err(Error::RMPDecode)
|
||||||
.map_err(sled::ConflictableTransactionError::Abort)?;
|
.map_err(sled::ConflictableTransactionError::Abort)?;
|
||||||
let mut new_entry = old_entry.clone();
|
let mut new_entry = old_entry.clone();
|
||||||
new_entry.merge(&update);
|
new_entry.merge(&update);
|
||||||
@@ -440,7 +504,7 @@ where
|
|||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
if let Some(old_val) = self.store.remove(&key)? {
|
if let Some(old_val) = self.store.remove(&key)? {
|
||||||
let old_entry = self.decode_entry(&old_val)?;
|
let old_entry = rmp_serde::decode::from_read_ref::<_, F::E>(&old_val)?;
|
||||||
self.instance.updated(Some(old_entry), None).await?;
|
self.instance.updated(Some(old_entry), None).await?;
|
||||||
self.system
|
self.system
|
||||||
.background
|
.background
|
||||||
@@ -457,20 +521,4 @@ where
|
|||||||
ret.extend(s.sort_key());
|
ret.extend(s.sort_key());
|
||||||
ret
|
ret
|
||||||
}
|
}
|
||||||
|
|
||||||
fn decode_entry(&self, bytes: &[u8]) -> Result<F::E, Error> {
|
|
||||||
match rmp_serde::decode::from_read_ref::<_, F::E>(bytes) {
|
|
||||||
Ok(x) => Ok(x),
|
|
||||||
Err(e) => match F::try_migrate(bytes) {
|
|
||||||
Some(x) => Ok(x),
|
|
||||||
None => {
|
|
||||||
warn!("Unable to decode entry of {}: {}", self.name, e);
|
|
||||||
for line in hexdump::hexdump_iter(bytes) {
|
|
||||||
debug!("{}", line);
|
|
||||||
}
|
|
||||||
Err(e.into())
|
|
||||||
}
|
|
||||||
},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -474,7 +474,7 @@ where
|
|||||||
todo.push_back(root_ck);
|
todo.push_back(root_ck);
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
return Err(Error::Message(format!(
|
return Err(Error::BadRequest(format!(
|
||||||
"Invalid respone to GetRootChecksumRange RPC: {}",
|
"Invalid respone to GetRootChecksumRange RPC: {}",
|
||||||
debug_serialize(root_cks_resp)
|
debug_serialize(root_cks_resp)
|
||||||
)));
|
)));
|
||||||
@@ -530,7 +530,7 @@ where
|
|||||||
self.send_items(who, items_to_send).await?;
|
self.send_items(who, items_to_send).await?;
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
return Err(Error::Message(format!(
|
return Err(Error::BadRequest(format!(
|
||||||
"Unexpected response to sync RPC checksums: {}",
|
"Unexpected response to sync RPC checksums: {}",
|
||||||
debug_serialize(&rpc_resp)
|
debug_serialize(&rpc_resp)
|
||||||
)));
|
)));
|
||||||
|
|||||||
@@ -1,35 +0,0 @@
|
|||||||
use serde::{Deserialize, Serialize};
|
|
||||||
|
|
||||||
use garage_util::data::*;
|
|
||||||
|
|
||||||
use crate::schema::*;
|
|
||||||
|
|
||||||
#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)]
|
|
||||||
pub struct EmptyKey;
|
|
||||||
impl SortKey for EmptyKey {
|
|
||||||
fn sort_key(&self) -> &[u8] {
|
|
||||||
&[]
|
|
||||||
}
|
|
||||||
}
|
|
||||||
impl PartitionKey for EmptyKey {
|
|
||||||
fn hash(&self) -> Hash {
|
|
||||||
[0u8; 32].into()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Clone, Copy, Debug, Serialize, Deserialize)]
|
|
||||||
pub enum DeletedFilter {
|
|
||||||
All,
|
|
||||||
Deleted,
|
|
||||||
NotDeleted,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl DeletedFilter {
|
|
||||||
pub fn apply(&self, deleted: bool) -> bool {
|
|
||||||
match self {
|
|
||||||
DeletedFilter::All => true,
|
|
||||||
DeletedFilter::Deleted => deleted,
|
|
||||||
DeletedFilter::NotDeleted => !deleted,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
+1
-1
@@ -1,6 +1,6 @@
|
|||||||
[package]
|
[package]
|
||||||
name = "garage_util"
|
name = "garage_util"
|
||||||
version = "0.1.1"
|
version = "0.1.0"
|
||||||
authors = ["Alex Auvolat <alex@adnab.me>"]
|
authors = ["Alex Auvolat <alex@adnab.me>"]
|
||||||
edition = "2018"
|
edition = "2018"
|
||||||
license = "GPL-3.0"
|
license = "GPL-3.0"
|
||||||
|
|||||||
@@ -35,8 +35,6 @@ pub struct Config {
|
|||||||
pub rpc_tls: Option<TlsConfig>,
|
pub rpc_tls: Option<TlsConfig>,
|
||||||
|
|
||||||
pub s3_api: ApiConfig,
|
pub s3_api: ApiConfig,
|
||||||
|
|
||||||
pub s3_web: WebConfig,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Deserialize, Debug, Clone)]
|
#[derive(Deserialize, Debug, Clone)]
|
||||||
@@ -52,13 +50,6 @@ pub struct ApiConfig {
|
|||||||
pub s3_region: String,
|
pub s3_region: String,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Deserialize, Debug, Clone)]
|
|
||||||
pub struct WebConfig {
|
|
||||||
pub bind_addr: SocketAddr,
|
|
||||||
pub root_domain: String,
|
|
||||||
pub index: String,
|
|
||||||
}
|
|
||||||
|
|
||||||
fn default_max_concurrent_rpc_requests() -> usize {
|
fn default_max_concurrent_rpc_requests() -> usize {
|
||||||
12
|
12
|
||||||
}
|
}
|
||||||
|
|||||||
+35
-2
@@ -54,6 +54,9 @@ pub enum Error {
|
|||||||
#[error(display = "TOML decode error: {}", _0)]
|
#[error(display = "TOML decode error: {}", _0)]
|
||||||
TomlDecode(#[error(source)] toml::de::Error),
|
TomlDecode(#[error(source)] toml::de::Error),
|
||||||
|
|
||||||
|
#[error(display = "Timeout: {}", _0)]
|
||||||
|
RPCTimeout(#[error(source)] tokio::time::Elapsed),
|
||||||
|
|
||||||
#[error(display = "Tokio join error: {}", _0)]
|
#[error(display = "Tokio join error: {}", _0)]
|
||||||
TokioJoin(#[error(source)] tokio::task::JoinError),
|
TokioJoin(#[error(source)] tokio::task::JoinError),
|
||||||
|
|
||||||
@@ -63,8 +66,14 @@ pub enum Error {
|
|||||||
#[error(display = "Remote error: {} (status code {})", _0, _1)]
|
#[error(display = "Remote error: {} (status code {})", _0, _1)]
|
||||||
RemoteError(String, StatusCode),
|
RemoteError(String, StatusCode),
|
||||||
|
|
||||||
#[error(display = "Bad RPC: {}", _0)]
|
#[error(display = "Bad request: {}", _0)]
|
||||||
BadRPC(String),
|
BadRequest(String),
|
||||||
|
|
||||||
|
#[error(display = "Forbidden: {}", _0)]
|
||||||
|
Forbidden(String),
|
||||||
|
|
||||||
|
#[error(display = "Not found")]
|
||||||
|
NotFound,
|
||||||
|
|
||||||
#[error(display = "Corrupt data: does not match hash {:?}", _0)]
|
#[error(display = "Corrupt data: does not match hash {:?}", _0)]
|
||||||
CorruptData(Hash),
|
CorruptData(Hash),
|
||||||
@@ -73,6 +82,18 @@ pub enum Error {
|
|||||||
Message(String),
|
Message(String),
|
||||||
}
|
}
|
||||||
|
|
||||||
|
impl Error {
|
||||||
|
pub fn http_status_code(&self) -> StatusCode {
|
||||||
|
match self {
|
||||||
|
Error::BadRequest(_) => StatusCode::BAD_REQUEST,
|
||||||
|
Error::NotFound => StatusCode::NOT_FOUND,
|
||||||
|
Error::Forbidden(_) => StatusCode::FORBIDDEN,
|
||||||
|
Error::RPC(_) => StatusCode::SERVICE_UNAVAILABLE,
|
||||||
|
_ => StatusCode::INTERNAL_SERVER_ERROR,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
impl From<sled::TransactionError<Error>> for Error {
|
impl From<sled::TransactionError<Error>> for Error {
|
||||||
fn from(e: sled::TransactionError<Error>) -> Error {
|
fn from(e: sled::TransactionError<Error>) -> Error {
|
||||||
match e {
|
match e {
|
||||||
@@ -93,3 +114,15 @@ impl<T> From<tokio::sync::mpsc::error::SendError<T>> for Error {
|
|||||||
Error::Message(format!("MPSC send error"))
|
Error::Message(format!("MPSC send error"))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
impl From<std::str::Utf8Error> for Error {
|
||||||
|
fn from(e: std::str::Utf8Error) -> Error {
|
||||||
|
Error::BadRequest(format!("Invalid UTF-8: {}", e))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl From<roxmltree::Error> for Error {
|
||||||
|
fn from(e: roxmltree::Error) -> Error {
|
||||||
|
Error::BadRequest(format!("Invalid XML: {}", e))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -1,28 +0,0 @@
|
|||||||
[package]
|
|
||||||
name = "garage_web"
|
|
||||||
version = "0.1.1"
|
|
||||||
authors = ["Alex Auvolat <alex@adnab.me>", "Quentin Dufour <quentin@dufour.io>"]
|
|
||||||
edition = "2018"
|
|
||||||
license = "GPL-3.0"
|
|
||||||
description = "S3-like website endpoint crate for the Garage object store"
|
|
||||||
repository = "https://git.deuxfleurs.fr/Deuxfleurs/garage"
|
|
||||||
|
|
||||||
[lib]
|
|
||||||
path = "lib.rs"
|
|
||||||
|
|
||||||
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
|
|
||||||
|
|
||||||
[dependencies]
|
|
||||||
garage_util = { version = "0.1.1", path = "../util" }
|
|
||||||
garage_table = { version = "0.1.1", path = "../table" }
|
|
||||||
garage_model = { version = "0.1.1", path = "../model" }
|
|
||||||
garage_api = { version = "0.1.1", path = "../api" }
|
|
||||||
|
|
||||||
err-derive = "0.2.3"
|
|
||||||
log = "0.4"
|
|
||||||
futures = "0.3"
|
|
||||||
http = "0.2"
|
|
||||||
hyper = "0.13"
|
|
||||||
percent-encoding = "2.1.0"
|
|
||||||
roxmltree = "0.11"
|
|
||||||
idna = "0.2"
|
|
||||||
@@ -1,39 +0,0 @@
|
|||||||
use err_derive::Error;
|
|
||||||
use hyper::StatusCode;
|
|
||||||
|
|
||||||
use garage_util::error::Error as GarageError;
|
|
||||||
|
|
||||||
#[derive(Debug, Error)]
|
|
||||||
pub enum Error {
|
|
||||||
#[error(display = "API error: {}", _0)]
|
|
||||||
ApiError(#[error(source)] garage_api::error::Error),
|
|
||||||
|
|
||||||
// Category: internal error
|
|
||||||
#[error(display = "Internal error: {}", _0)]
|
|
||||||
InternalError(#[error(source)] GarageError),
|
|
||||||
|
|
||||||
#[error(display = "Not found")]
|
|
||||||
NotFound,
|
|
||||||
|
|
||||||
// Category: bad request
|
|
||||||
#[error(display = "Invalid UTF-8: {}", _0)]
|
|
||||||
InvalidUTF8(#[error(source)] std::str::Utf8Error),
|
|
||||||
|
|
||||||
#[error(display = "Invalid header value: {}", _0)]
|
|
||||||
InvalidHeader(#[error(source)] hyper::header::ToStrError),
|
|
||||||
|
|
||||||
#[error(display = "Bad request: {}", _0)]
|
|
||||||
BadRequest(String),
|
|
||||||
}
|
|
||||||
|
|
||||||
impl Error {
|
|
||||||
pub fn http_status_code(&self) -> StatusCode {
|
|
||||||
match self {
|
|
||||||
Error::NotFound => StatusCode::NOT_FOUND,
|
|
||||||
Error::ApiError(e) => e.http_status_code(),
|
|
||||||
Error::InternalError(GarageError::RPC(_)) => StatusCode::SERVICE_UNAVAILABLE,
|
|
||||||
Error::InternalError(_) => StatusCode::INTERNAL_SERVER_ERROR,
|
|
||||||
_ => StatusCode::BAD_REQUEST,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,6 +0,0 @@
|
|||||||
#[macro_use]
|
|
||||||
extern crate log;
|
|
||||||
|
|
||||||
pub mod error;
|
|
||||||
|
|
||||||
pub mod web_server;
|
|
||||||
@@ -1,260 +0,0 @@
|
|||||||
use std::{borrow::Cow, convert::Infallible, net::SocketAddr, sync::Arc};
|
|
||||||
|
|
||||||
use futures::future::Future;
|
|
||||||
|
|
||||||
use hyper::{
|
|
||||||
header::HOST,
|
|
||||||
server::conn::AddrStream,
|
|
||||||
service::{make_service_fn, service_fn},
|
|
||||||
Body, Method, Request, Response, Server,
|
|
||||||
};
|
|
||||||
|
|
||||||
use idna::domain_to_unicode;
|
|
||||||
|
|
||||||
use crate::error::*;
|
|
||||||
use garage_api::s3_get::{handle_get, handle_head};
|
|
||||||
use garage_model::bucket_table::*;
|
|
||||||
use garage_model::garage::Garage;
|
|
||||||
use garage_table::*;
|
|
||||||
use garage_util::error::Error as GarageError;
|
|
||||||
|
|
||||||
pub async fn run_web_server(
|
|
||||||
garage: Arc<Garage>,
|
|
||||||
shutdown_signal: impl Future<Output = ()>,
|
|
||||||
) -> Result<(), GarageError> {
|
|
||||||
let addr = &garage.config.s3_web.bind_addr;
|
|
||||||
|
|
||||||
let service = make_service_fn(|conn: &AddrStream| {
|
|
||||||
let garage = garage.clone();
|
|
||||||
let client_addr = conn.remote_addr();
|
|
||||||
async move {
|
|
||||||
Ok::<_, Error>(service_fn(move |req: Request<Body>| {
|
|
||||||
let garage = garage.clone();
|
|
||||||
handle_request(garage, req, client_addr)
|
|
||||||
}))
|
|
||||||
}
|
|
||||||
});
|
|
||||||
|
|
||||||
let server = Server::bind(&addr).serve(service);
|
|
||||||
let graceful = server.with_graceful_shutdown(shutdown_signal);
|
|
||||||
info!("Web server listening on http://{}", addr);
|
|
||||||
|
|
||||||
graceful.await?;
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn handle_request(
|
|
||||||
garage: Arc<Garage>,
|
|
||||||
req: Request<Body>,
|
|
||||||
addr: SocketAddr,
|
|
||||||
) -> Result<Response<Body>, Infallible> {
|
|
||||||
info!("{} {} {}", addr, req.method(), req.uri());
|
|
||||||
let res = serve_file(garage, req).await;
|
|
||||||
match &res {
|
|
||||||
Ok(r) => debug!("{} {:?}", r.status(), r.headers()),
|
|
||||||
Err(e) => warn!("Response: error {}, {}", e.http_status_code(), e),
|
|
||||||
}
|
|
||||||
|
|
||||||
Ok(res.unwrap_or_else(error_to_res))
|
|
||||||
}
|
|
||||||
|
|
||||||
fn error_to_res(e: Error) -> Response<Body> {
|
|
||||||
let body: Body = Body::from(format!("{}\n", e));
|
|
||||||
let mut http_error = Response::new(body);
|
|
||||||
*http_error.status_mut() = e.http_status_code();
|
|
||||||
http_error
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn serve_file(garage: Arc<Garage>, req: Request<Body>) -> Result<Response<Body>, Error> {
|
|
||||||
// Get http authority string (eg. [::1]:3902 or garage.tld:80)
|
|
||||||
let authority = req
|
|
||||||
.headers()
|
|
||||||
.get(HOST)
|
|
||||||
.ok_or(Error::BadRequest(format!("HOST header required")))?
|
|
||||||
.to_str()?;
|
|
||||||
|
|
||||||
// Get bucket
|
|
||||||
let (host, _) = domain_to_unicode(authority_to_host(authority)?);
|
|
||||||
let root = &garage.config.s3_web.root_domain;
|
|
||||||
let bucket = host_to_bucket(&host, root);
|
|
||||||
|
|
||||||
// Check bucket is exposed as a website
|
|
||||||
let bucket_desc = garage
|
|
||||||
.bucket_table
|
|
||||||
.get(&EmptyKey, &bucket.to_string())
|
|
||||||
.await?
|
|
||||||
.filter(|b| !b.is_deleted())
|
|
||||||
.ok_or(Error::NotFound)?;
|
|
||||||
|
|
||||||
match bucket_desc.state.get() {
|
|
||||||
BucketState::Present(params) if *params.website.get() => Ok(()),
|
|
||||||
_ => Err(Error::NotFound),
|
|
||||||
}?;
|
|
||||||
|
|
||||||
// Get path
|
|
||||||
let path = req.uri().path().to_string();
|
|
||||||
let index = &garage.config.s3_web.index;
|
|
||||||
let key = path_to_key(&path, &index)?;
|
|
||||||
|
|
||||||
info!("Selected bucket: \"{}\", selected key: \"{}\"", bucket, key);
|
|
||||||
|
|
||||||
let res = match req.method() {
|
|
||||||
&Method::HEAD => handle_head(garage, &bucket, &key).await?,
|
|
||||||
&Method::GET => handle_get(garage, &req, bucket, &key).await?,
|
|
||||||
_ => return Err(Error::BadRequest(format!("HTTP method not supported"))),
|
|
||||||
};
|
|
||||||
|
|
||||||
Ok(res)
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Extract host from the authority section given by the HTTP host header
|
|
||||||
///
|
|
||||||
/// The HTTP host contains both a host and a port.
|
|
||||||
/// Extracting the port is more complex than just finding the colon (:) symbol due to IPv6
|
|
||||||
/// We do not use the collect pattern as there is no way in std rust to collect over a stack allocated value
|
|
||||||
/// check here: https://docs.rs/collect_slice/1.2.0/collect_slice/
|
|
||||||
fn authority_to_host(authority: &str) -> Result<&str, Error> {
|
|
||||||
let mut iter = authority.chars().enumerate();
|
|
||||||
let (_, first_char) = iter
|
|
||||||
.next()
|
|
||||||
.ok_or(Error::BadRequest(format!("Authority is empty")))?;
|
|
||||||
|
|
||||||
let split = match first_char {
|
|
||||||
'[' => {
|
|
||||||
let mut iter = iter.skip_while(|(_, c)| c != &']');
|
|
||||||
match iter.next() {
|
|
||||||
Some((_, ']')) => iter.next(),
|
|
||||||
_ => {
|
|
||||||
return Err(Error::BadRequest(format!(
|
|
||||||
"Authority {} has an illegal format",
|
|
||||||
authority
|
|
||||||
)))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
_ => iter.skip_while(|(_, c)| c != &':').next(),
|
|
||||||
};
|
|
||||||
|
|
||||||
match split {
|
|
||||||
Some((i, ':')) => Ok(&authority[..i]),
|
|
||||||
None => Ok(authority),
|
|
||||||
Some((_, _)) => Err(Error::BadRequest(format!(
|
|
||||||
"Authority {} has an illegal format",
|
|
||||||
authority
|
|
||||||
))),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Host to bucket
|
|
||||||
///
|
|
||||||
/// Convert a host, like "bucket.garage-site.tld" or "john.doe.com"
|
|
||||||
/// to the corresponding bucket, resp. "bucket" and "john.doe.com"
|
|
||||||
/// considering that ".garage-site.tld" is the "root domain".
|
|
||||||
/// This behavior has been chosen to follow AWS S3 semantic.
|
|
||||||
fn host_to_bucket<'a>(host: &'a str, root: &str) -> &'a str {
|
|
||||||
if root.len() >= host.len() || !host.ends_with(root) {
|
|
||||||
return host;
|
|
||||||
}
|
|
||||||
|
|
||||||
let len_diff = host.len() - root.len();
|
|
||||||
let missing_starting_dot = root.chars().next() != Some('.');
|
|
||||||
let cursor = if missing_starting_dot {
|
|
||||||
len_diff - 1
|
|
||||||
} else {
|
|
||||||
len_diff
|
|
||||||
};
|
|
||||||
&host[..cursor]
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Path to key
|
|
||||||
///
|
|
||||||
/// Convert the provided path to the internal key
|
|
||||||
/// When a path ends with "/", we append the index name to match traditional web server behavior
|
|
||||||
/// which is also AWS S3 behavior.
|
|
||||||
fn path_to_key<'a>(path: &'a str, index: &str) -> Result<Cow<'a, str>, Error> {
|
|
||||||
let path_utf8 = percent_encoding::percent_decode_str(&path).decode_utf8()?;
|
|
||||||
|
|
||||||
if path_utf8.chars().next() != Some('/') {
|
|
||||||
return Err(Error::BadRequest(format!(
|
|
||||||
"Path must start with a / (slash)"
|
|
||||||
)));
|
|
||||||
}
|
|
||||||
|
|
||||||
match path_utf8.chars().last() {
|
|
||||||
None => unreachable!(),
|
|
||||||
Some('/') => {
|
|
||||||
let mut key = String::with_capacity(path_utf8.len() + index.len());
|
|
||||||
key.push_str(&path_utf8[1..]);
|
|
||||||
key.push_str(index);
|
|
||||||
Ok(key.into())
|
|
||||||
}
|
|
||||||
Some(_) => match path_utf8 {
|
|
||||||
Cow::Borrowed(pu8) => Ok((&pu8[1..]).into()),
|
|
||||||
Cow::Owned(pu8) => Ok((&pu8[1..]).to_string().into()),
|
|
||||||
},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[cfg(test)]
|
|
||||||
mod tests {
|
|
||||||
use super::*;
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn authority_to_host_with_port() -> Result<(), Error> {
|
|
||||||
let domain = authority_to_host("[::1]:3902")?;
|
|
||||||
assert_eq!(domain, "[::1]");
|
|
||||||
let domain2 = authority_to_host("garage.tld:65200")?;
|
|
||||||
assert_eq!(domain2, "garage.tld");
|
|
||||||
let domain3 = authority_to_host("127.0.0.1:80")?;
|
|
||||||
assert_eq!(domain3, "127.0.0.1");
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn authority_to_host_without_port() -> Result<(), Error> {
|
|
||||||
let domain = authority_to_host("[::1]")?;
|
|
||||||
assert_eq!(domain, "[::1]");
|
|
||||||
let domain2 = authority_to_host("garage.tld")?;
|
|
||||||
assert_eq!(domain2, "garage.tld");
|
|
||||||
let domain3 = authority_to_host("127.0.0.1")?;
|
|
||||||
assert_eq!(domain3, "127.0.0.1");
|
|
||||||
assert!(authority_to_host("[").is_err());
|
|
||||||
assert!(authority_to_host("[hello").is_err());
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn host_to_bucket_test() {
|
|
||||||
assert_eq!(
|
|
||||||
host_to_bucket("john.doe.garage.tld", ".garage.tld"),
|
|
||||||
"john.doe"
|
|
||||||
);
|
|
||||||
|
|
||||||
assert_eq!(
|
|
||||||
host_to_bucket("john.doe.garage.tld", "garage.tld"),
|
|
||||||
"john.doe"
|
|
||||||
);
|
|
||||||
|
|
||||||
assert_eq!(host_to_bucket("john.doe.com", "garage.tld"), "john.doe.com");
|
|
||||||
|
|
||||||
assert_eq!(
|
|
||||||
host_to_bucket("john.doe.com", ".garage.tld"),
|
|
||||||
"john.doe.com"
|
|
||||||
);
|
|
||||||
|
|
||||||
assert_eq!(host_to_bucket("garage.tld", "garage.tld"), "garage.tld");
|
|
||||||
|
|
||||||
assert_eq!(host_to_bucket("garage.tld", ".garage.tld"), "garage.tld");
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn path_to_key_test() -> Result<(), Error> {
|
|
||||||
assert_eq!(path_to_key("/file%20.jpg", "index.html")?, "file .jpg");
|
|
||||||
assert_eq!(path_to_key("/%20t/", "index.html")?, " t/index.html");
|
|
||||||
assert_eq!(path_to_key("/", "index.html")?, "index.html");
|
|
||||||
assert_eq!(path_to_key("/hello", "index.html")?, "hello");
|
|
||||||
assert!(path_to_key("", "index.html").is_err());
|
|
||||||
assert!(path_to_key("i/am/relative", "index.html").is_err());
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Executable
+6
@@ -0,0 +1,6 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
for FILE in $(find target); do
|
||||||
|
curl localhost:3900/$FILE -X DELETE -H 'Host: garage'
|
||||||
|
done
|
||||||
|
|
||||||
Executable
+13
@@ -0,0 +1,13 @@
|
|||||||
|
|
||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
for FILE in $(find target/debug/deps); do
|
||||||
|
SHA2=$(curl localhost:3900/$FILE -H 'Host: garage' 2>/dev/null | sha256sum | cut -d ' ' -f 1)
|
||||||
|
SHA2REF=$(sha256sum $FILE | cut -d ' ' -f 1)
|
||||||
|
if [ "$SHA2" = "$SHA2REF" ]; then
|
||||||
|
echo "OK $FILE"
|
||||||
|
else
|
||||||
|
echo "!!!! ERROR $FILE !!!!"
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|
||||||
Executable
+7
@@ -0,0 +1,7 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
for FILE in $(find target/debug/deps); do
|
||||||
|
echo -n "$FILE "
|
||||||
|
curl localhost:3900/$FILE -X PUT -H 'Host: garage' -H 'Content-Type: application/blob' --data-binary "@$FILE" || echo "ERROR"
|
||||||
|
done
|
||||||
|
|
||||||
Reference in New Issue
Block a user