mirror of
https://github.com/deuxfleurs-org/garage.git
synced 2026-08-13 15:46:53 +00:00
Compare commits
10 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Alex Auvolat
|
e8a52192d0 | ||
|
Alex Auvolat
|
8394f32d16 | ||
|
Quentin Dufour
|
cd5fa90c68 | ||
|
Quentin Dufour
|
d47af7b173 | ||
|
Quentin Dufour
|
1cc0de40f0 | ||
|
Quentin Dufour
|
40a140bd20 | ||
|
Quentin Dufour
|
71d4cf42f1 | ||
|
Quentin Dufour
|
ac98769009 | ||
|
Quentin Dufour
|
4279ca95eb | ||
|
Quentin Dufour
|
90a57fbc7e |
+1
-1
@@ -269,6 +269,6 @@ trigger:
|
||||
|
||||
---
|
||||
kind: signature
|
||||
hmac: 362639b4c9541ad9bd06ff7f72b5235b2b0216bcb16eececd25285b6fe94ba6f
|
||||
hmac: fa1f98f327abf88486c0c54984287285a4b951efa3776af9dd33b4d782b50815
|
||||
|
||||
...
|
||||
|
||||
Generated
+310
-672
File diff suppressed because it is too large
Load Diff
@@ -1,7 +1,5 @@
|
||||
[workspace]
|
||||
resolver = "2"
|
||||
members = [
|
||||
"src/db",
|
||||
"src/util",
|
||||
"src/rpc",
|
||||
"src/table",
|
||||
|
||||
@@ -1,27 +1,13 @@
|
||||
.PHONY: doc all release shell run1 run2 run3
|
||||
.PHONY: doc all release shell
|
||||
|
||||
all:
|
||||
clear; cargo build
|
||||
clear; cargo build --all-features
|
||||
|
||||
doc:
|
||||
cd doc/book; mdbook build
|
||||
|
||||
release:
|
||||
nix-build --arg release true
|
||||
|
||||
shell:
|
||||
nix-shell
|
||||
|
||||
# ----
|
||||
|
||||
run1:
|
||||
RUST_LOG=garage=debug ./target/debug/garage -c tmp/config1.toml server
|
||||
run1rel:
|
||||
RUST_LOG=garage=debug ./target/release/garage -c tmp/config1.toml server
|
||||
|
||||
run2:
|
||||
RUST_LOG=garage=debug ./target/debug/garage -c tmp/config2.toml server
|
||||
run2rel:
|
||||
RUST_LOG=garage=debug ./target/release/garage -c tmp/config2.toml server
|
||||
|
||||
run3:
|
||||
RUST_LOG=garage=debug ./target/debug/garage -c tmp/config3.toml server
|
||||
run3rel:
|
||||
RUST_LOG=garage=debug ./target/release/garage -c tmp/config3.toml server
|
||||
|
||||
@@ -15,24 +15,18 @@ Garage [,
|
||||
an experimental small-scale self hosted service provider,
|
||||
which has been using it in production since its first release in 2020.
|
||||
Non-goals include:
|
||||
|
||||
Learn more on our dedicated documentation pages:
|
||||
- Extremely high performance
|
||||
- Complete implementation of the S3 API
|
||||
- Erasure coding (our replication model is simply to copy the data as is on several nodes, in different datacenters if possible)
|
||||
|
||||
- [Goals and use cases](https://garagehq.deuxfleurs.fr/documentation/design/goals/)
|
||||
- [Features](https://garagehq.deuxfleurs.fr/documentation/reference-manual/features/)
|
||||
- [Quick start](https://garagehq.deuxfleurs.fr/documentation/quick-start/)
|
||||
|
||||
Garage is entirely free software released under the terms of the AGPLv3.
|
||||
Our main use case is to provide a distributed storage layer for small-scale self hosted services such as [Deuxfleurs](https://deuxfleurs.fr).
|
||||
|
||||
@@ -3,7 +3,7 @@ title = "Websites (Hugo, Jekyll, Publii...)"
|
||||
weight = 10
|
||||
+++
|
||||
|
||||
Garage is also suitable [to host static websites](@/documentation/cookbook/exposing-websites.md).
|
||||
Garage is also suitable to host static websites.
|
||||
While they can be deployed with traditional CLI tools, some static website generators have integrated options to ease your workflow.
|
||||
|
||||
| Name | Status | Note |
|
||||
|
||||
@@ -5,14 +5,12 @@ weight = 25
|
||||
|
||||
## Configuring a bucket for website access
|
||||
|
||||
There are three methods to expose buckets as website:
|
||||
There are two methods to expose buckets as website:
|
||||
|
||||
1. using the PutBucketWebsite S3 API call, which is allowed for access keys that have the owner permission bit set
|
||||
|
||||
2. from the Garage CLI, by an adminstrator of the cluster
|
||||
|
||||
3. using the Garage administration API
|
||||
|
||||
The `PutBucketWebsite` API endpoint [is documented](https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutBucketWebsite.html) in the official AWS docs.
|
||||
This endpoint can also be called [using `aws s3api`](https://docs.aws.amazon.com/cli/latest/reference/s3api/put-bucket-website.html) on the command line.
|
||||
The website configuration supported by Garage is only a subset of the possibilities on Amazon S3: redirections are not supported, only the index document and error document can be specified.
|
||||
|
||||
@@ -20,76 +20,40 @@ sudo apt-get update
|
||||
sudo apt-get install build-essential
|
||||
```
|
||||
|
||||
## Building from source from the Gitea repository
|
||||
## Using source from `crates.io`
|
||||
|
||||
Garage's source code is published on `crates.io`, Rust's official package repository.
|
||||
This means you can simply ask `cargo` to download and build this source code for you:
|
||||
|
||||
```bash
|
||||
cargo install garage
|
||||
```
|
||||
|
||||
That's all, `garage` should be in `$HOME/.cargo/bin`.
|
||||
|
||||
You can add this folder to your `$PATH` or copy the binary somewhere else on your system.
|
||||
For instance:
|
||||
|
||||
```bash
|
||||
sudo cp $HOME/.cargo/bin/garage /usr/local/bin/garage
|
||||
```
|
||||
|
||||
|
||||
## Using source from the Gitea repository
|
||||
|
||||
The primary location for Garage's source code is the
|
||||
[Gitea repository](https://git.deuxfleurs.fr/Deuxfleurs/garage),
|
||||
which contains all of the released versions as well as the code
|
||||
for the developpement of the next version.
|
||||
[Gitea repository](https://git.deuxfleurs.fr/Deuxfleurs/garage).
|
||||
|
||||
Clone the repository and enter it as follows:
|
||||
Clone the repository and build Garage with the following commands:
|
||||
|
||||
```bash
|
||||
git clone https://git.deuxfleurs.fr/Deuxfleurs/garage.git
|
||||
cd garage
|
||||
cargo build
|
||||
```
|
||||
|
||||
If you wish to build a specific version of Garage, check out the corresponding tag. For instance:
|
||||
Be careful, as this will make a debug build of Garage, which will be extremely slow!
|
||||
To make a release build, invoke `cargo build --release` (this takes much longer).
|
||||
|
||||
```bash
|
||||
git tag # List available tags
|
||||
git checkout v0.8.0 # Change v0.8.0 with the version you wish to build
|
||||
```
|
||||
The binaries built this way are found in `target/{debug,release}/garage`.
|
||||
|
||||
Otherwise you will be building a developpement build from the `main` branch
|
||||
that includes all of the changes to be released in the next version.
|
||||
Be careful that such a build might be unstable or contain bugs,
|
||||
and could be incompatible with nodes that run stable versions of Garage.
|
||||
|
||||
Finally, build Garage with the following command:
|
||||
|
||||
```bash
|
||||
cargo build --release
|
||||
```
|
||||
|
||||
The binary built this way can now be found in `target/release/garage`.
|
||||
You may simply copy this binary to somewhere in your `$PATH` in order to
|
||||
have the `garage` command available in your shell, for instance:
|
||||
|
||||
```bash
|
||||
sudo cp target/release/garage /usr/local/bin/garage
|
||||
```
|
||||
|
||||
If you are planning to develop Garage,
|
||||
you might be interested in producing debug builds, which compile faster but run slower:
|
||||
this can be done by removing the `--release` flag, and the resulting build can then
|
||||
be found in `target/debug/garage`.
|
||||
|
||||
## List of available Cargo feature flags
|
||||
|
||||
Garage supports a number of compilation options in the form of Cargo feature flags,
|
||||
which can be used to provide builds adapted to your system and your use case.
|
||||
To produce a build with a given set of features, invoke the `cargo build` command
|
||||
as follows:
|
||||
|
||||
```bash
|
||||
# This will build the default feature set plus feature1, feature2 and feature3
|
||||
cargo build --release --features feature1,feature2,feature3
|
||||
# This will build ONLY feature1, feature2 and feature3
|
||||
cargo build --release --no-default-features \
|
||||
--features feature1,feature2,feature3
|
||||
```
|
||||
|
||||
The following feature flags are available in v0.8.0:
|
||||
|
||||
| Feature flag | Enabled | Description |
|
||||
| ------------ | ------- | ----------- |
|
||||
| `bundled-libs` | *by default* | Use bundled version of sqlite3, zstd, lmdb and libsodium |
|
||||
| `system-libs` | optional | Use system version of sqlite3, zstd, lmdb and libsodium<br>if available (exclusive with `bundled-libs`, build using<br>`cargo build --no-default-features --features system-libs`) |
|
||||
| `k2v` | optional | Enable the experimental K2V API (if used, all nodes on your<br>Garage cluster must have it enabled as well) |
|
||||
| `kubernetes-discovery` | optional | Enable automatic registration and discovery<br>of cluster nodes through the Kubernetes API |
|
||||
| `metrics` | *by default* | Enable collection of metrics in Prometheus format on the admin API |
|
||||
| `telemetry-otlp` | optional | Enable collection of execution traces using OpenTelemetry |
|
||||
| `sled` | *by default* | Enable using Sled to store Garage's metadata |
|
||||
| `lmdb` | optional | Enable using LMDB to store Garage's metadata |
|
||||
| `sqlite` | optional | Enable using Sqlite3 to store Garage's metadata |
|
||||
|
||||
@@ -100,7 +100,7 @@ server {
|
||||
}
|
||||
```
|
||||
|
||||
### Exposing the web endpoint
|
||||
## Exposing the web endpoint
|
||||
|
||||
To better understand the logic involved, you can refer to the [Exposing buckets as websites](/cookbook/exposing_websites.html) section.
|
||||
Otherwise, the configuration is very similar to the S3 endpoint.
|
||||
@@ -140,165 +140,6 @@ server {
|
||||
|
||||
@TODO
|
||||
|
||||
## Traefik v2
|
||||
|
||||
We will see in this part how to set up a reverse proxy with [Traefik](https://docs.traefik.io/).
|
||||
|
||||
Here is [a basic configuration file](https://doc.traefik.io/traefik/https/acme/#configuration-examples):
|
||||
|
||||
```toml
|
||||
[entryPoints]
|
||||
[entryPoints.web]
|
||||
address = ":80"
|
||||
|
||||
[entryPoints.websecure]
|
||||
address = ":443"
|
||||
|
||||
[certificatesResolvers.myresolver.acme]
|
||||
email = "your-email@example.com"
|
||||
storage = "acme.json"
|
||||
[certificatesResolvers.myresolver.acme.httpChallenge]
|
||||
# used during the challenge
|
||||
entryPoint = "web"
|
||||
```
|
||||
|
||||
### Add Garage service
|
||||
|
||||
To add Garage on Traefik you should declare a new service using its IP address (or hostname) and port:
|
||||
|
||||
```toml
|
||||
[http.services]
|
||||
[http.services.my_garage_service.loadBalancer]
|
||||
[[http.services.my_garage_service.loadBalancer.servers]]
|
||||
url = "http://xxx.xxx.xxx.xxx"
|
||||
port = 3900
|
||||
```
|
||||
|
||||
It's possible to declare multiple Garage servers as back-ends:
|
||||
|
||||
```toml
|
||||
[http.services]
|
||||
[[http.services.my_garage_service.loadBalancer.servers]]
|
||||
url = "http://xxx.xxx.xxx.xxx"
|
||||
port = 3900
|
||||
[[http.services.my_garage_service.loadBalancer.servers]]
|
||||
url = "http://yyy.yyy.yyy.yyy"
|
||||
port = 3900
|
||||
[[http.services.my_garage_service.loadBalancer.servers]]
|
||||
url = "http://zzz.zzz.zzz.zzz"
|
||||
port = 3900
|
||||
```
|
||||
|
||||
Traefik can remove unhealthy servers automatically with [a health check configuration](https://doc.traefik.io/traefik/routing/services/#health-check):
|
||||
|
||||
```
|
||||
[http.services]
|
||||
[http.services.my_garage_service.loadBalancer]
|
||||
[http.services.my_garage_service.loadBalancer.healthCheck]
|
||||
path = "/"
|
||||
interval = "60s"
|
||||
timeout = "5s"
|
||||
```
|
||||
|
||||
### Adding a website
|
||||
|
||||
To add a new website, add the following declaration to your Traefik configuration file:
|
||||
|
||||
```toml
|
||||
[http.routers]
|
||||
[http.routers.my_website]
|
||||
rule = "Host(`yoururl.example.org`)"
|
||||
service = "my_garage_service"
|
||||
entryPoints = ["web"]
|
||||
```
|
||||
|
||||
Enable HTTPS access to your website with the following configuration section ([documentation](https://doc.traefik.io/traefik/https/overview/)):
|
||||
|
||||
```toml
|
||||
...
|
||||
entryPoints = ["websecure"]
|
||||
[http.routers.my_website.tls]
|
||||
certResolver = "myresolver"
|
||||
...
|
||||
```
|
||||
|
||||
### Adding gzip compression
|
||||
|
||||
Add the following configuration section [to compress response](https://doc.traefik.io/traefik/middlewares/http/compress/) using [gzip](https://developer.mozilla.org/en-US/docs/Glossary/GZip_compression) before sending them to the client:
|
||||
|
||||
```toml
|
||||
[http.routers]
|
||||
[http.routers.my_website]
|
||||
...
|
||||
middlewares = ["gzip_compress"]
|
||||
...
|
||||
[http.middlewares]
|
||||
[http.middlewares.gzip_compress.compress]
|
||||
```
|
||||
|
||||
### Add caching response
|
||||
|
||||
Traefik's caching middleware is only available on [entreprise version](https://doc.traefik.io/traefik-enterprise/middlewares/http-cache/), however the freely-available [Souin plugin](https://github.com/darkweak/souin#tr%C3%A6fik-container) can also do the job. (section to be completed)
|
||||
|
||||
### Complete example
|
||||
|
||||
```toml
|
||||
[entryPoints]
|
||||
[entryPoints.web]
|
||||
address = ":80"
|
||||
|
||||
[entryPoints.websecure]
|
||||
address = ":443"
|
||||
|
||||
[certificatesResolvers.myresolver.acme]
|
||||
email = "your-email@example.com"
|
||||
storage = "acme.json"
|
||||
[certificatesResolvers.myresolver.acme.httpChallenge]
|
||||
# used during the challenge
|
||||
entryPoint = "web"
|
||||
|
||||
[http.routers]
|
||||
[http.routers.my_website]
|
||||
rule = "Host(`yoururl.example.org`)"
|
||||
service = "my_garage_service"
|
||||
middlewares = ["gzip_compress"]
|
||||
entryPoints = ["websecure"]
|
||||
|
||||
[http.services]
|
||||
[http.services.my_garage_service.loadBalancer]
|
||||
[http.services.my_garage_service.loadBalancer.healthCheck]
|
||||
path = "/"
|
||||
interval = "60s"
|
||||
timeout = "5s"
|
||||
[[http.services.my_garage_service.loadBalancer.servers]]
|
||||
url = "http://xxx.xxx.xxx.xxx"
|
||||
[[http.services.my_garage_service.loadBalancer.servers]]
|
||||
url = "http://yyy.yyy.yyy.yyy"
|
||||
[[http.services.my_garage_service.loadBalancer.servers]]
|
||||
url = "http://zzz.zzz.zzz.zzz"
|
||||
|
||||
[http.middlewares]
|
||||
[http.middlewares.gzip_compress.compress]
|
||||
```
|
||||
|
||||
## Caddy
|
||||
|
||||
Your Caddy configuration can be as simple as:
|
||||
|
||||
```caddy
|
||||
s3.garage.tld, *.s3.garage.tld {
|
||||
reverse_proxy localhost:3900 192.168.1.2:3900 example.tld:3900
|
||||
}
|
||||
|
||||
*.web.garage.tld {
|
||||
reverse_proxy localhost:3902 192.168.1.2:3900 example.tld:3900
|
||||
}
|
||||
|
||||
admin.garage.tld {
|
||||
reverse_proxy localhost:3903
|
||||
}
|
||||
```
|
||||
|
||||
But at the same time, the `reverse_proxy` is very flexible.
|
||||
For a production deployment, you should [read its documentation](https://caddyserver.com/docs/caddyfile/directives/reverse_proxy) as it supports features like DNS discovery of upstreams, load balancing with checks, streaming parameters, etc.
|
||||
## Traefik
|
||||
|
||||
@TODO
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
+++
|
||||
title = "Benchmarks"
|
||||
weight = 40
|
||||
weight = 10
|
||||
+++
|
||||
|
||||
With Garage, we wanted to build a software defined storage service that follow the [KISS principle](https://en.wikipedia.org/wiki/KISS_principle),
|
||||
|
||||
@@ -1,23 +1,23 @@
|
||||
+++
|
||||
title = "Goals and use cases"
|
||||
weight = 10
|
||||
weight = 5
|
||||
+++
|
||||
|
||||
## Goals and non-goals
|
||||
|
||||
Garage is a lightweight geo-distributed data store that implements the
|
||||
[Amazon S3](https://docs.aws.amazon.com/AmazonS3/latest/API/Welcome.html)
|
||||
object storage protocol. It enables applications to store large blobs such
|
||||
object storage protocole. It enables applications to store large blobs such
|
||||
as pictures, video, images, documents, etc., in a redundant multi-node
|
||||
setting. S3 is versatile enough to also be used to publish a static
|
||||
website.
|
||||
|
||||
Garage is an opinionated object storage solutoin, we focus on the following **desirable properties**:
|
||||
|
||||
- **Internet enabled**: made for multi-sites (eg. datacenters, offices, households, etc.) interconnected through regular Internet connections.
|
||||
- **Self-contained & lightweight**: works everywhere and integrates well in existing environments to target [hyperconverged infrastructures](https://en.wikipedia.org/wiki/Hyper-converged_infrastructure).
|
||||
- **Highly resilient**: highly resilient to network failures, network latency, disk failures, sysadmin failures.
|
||||
- **Simple**: simple to understand, simple to operate, simple to debug.
|
||||
- **Internet enabled**: made for multi-sites (eg. datacenters, offices, households, etc.) interconnected through regular Internet connections.
|
||||
|
||||
We also noted that the pursuit of some other goals are detrimental to our initial goals.
|
||||
The following has been identified as **non-goals** (if these points matter to you, you should not use Garage):
|
||||
|
||||
@@ -20,49 +20,6 @@ In the meantime, you can find some information at the following links:
|
||||
- [an old design draft](@/documentation/working-documents/design-draft.md)
|
||||
|
||||
|
||||
## Request routing logic
|
||||
|
||||
Data retrieval requests to Garage endpoints (S3 API and websites) are resolved
|
||||
to an individual object in a bucket. Since objects are replicated to multiple nodes
|
||||
Garage must ensure consistency before answering the request.
|
||||
|
||||
### Using quorum to ensure consistency
|
||||
|
||||
Garage ensures consistency by attempting to establish a quorum with the
|
||||
data nodes responsible for the object. When a majority of the data nodes
|
||||
have provided metadata on a object Garage can then answer the request.
|
||||
|
||||
When a request arrives Garage will, assuming the recommended 3 replicas, perform the following actions:
|
||||
|
||||
- Make a request to the two preferred nodes for object metadata
|
||||
- Try the third node if one of the two initial requests fail
|
||||
- Check that the metadata from at least 2 nodes match
|
||||
- Check that the object hasn't been marked deleted
|
||||
- Answer the request with inline data from metadata if object is small enough
|
||||
- Or get data blocks from the preferred nodes and answer using the assembled object
|
||||
|
||||
Garage dynamically determines which nodes to query based on health, preference, and
|
||||
which nodes actually host a given data. Garage has no concept of "primary" so any
|
||||
healthy node with the data can be used as long as a quorum is reached for the metadata.
|
||||
|
||||
### Node health
|
||||
|
||||
Garage keeps a TCP session open to each node in the cluster and periodically pings them. If a connection
|
||||
cannot be established, or a node fails to answer a number of pings, the target node is marked as failed.
|
||||
Failed nodes are not used for quorum or other internal requests.
|
||||
|
||||
### Node preference
|
||||
|
||||
Garage prioritizes which nodes to query according to a few criteria:
|
||||
|
||||
- A node always prefers itself if it can answer the request
|
||||
- Then the node prioritizes nodes in the same zone
|
||||
- Finally the nodes with the lowest latency are prioritized
|
||||
|
||||
|
||||
For further reading on the cluster structure look at the [gateway](@/documentation/cookbook/gateways.md)
|
||||
and [cluster layout management](@/documentation/reference-manual/layout.md) pages.
|
||||
|
||||
## Garbage collection
|
||||
|
||||
A faulty garbage collection procedure has been the cause of
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
+++
|
||||
title = "Related work"
|
||||
weight = 50
|
||||
weight = 15
|
||||
+++
|
||||
|
||||
## Context
|
||||
|
||||
@@ -9,15 +9,6 @@ Let's start your Garage journey!
|
||||
In this chapter, we explain how to deploy Garage as a single-node server
|
||||
and how to interact with it.
|
||||
|
||||
## What is Garage?
|
||||
|
||||
Before jumping in, you might be interested in reading the following pages:
|
||||
|
||||
- [Goals and use cases](@/documentation/design/goals.md)
|
||||
- [List of features](@/documentation/reference-manual/features.md)
|
||||
|
||||
## Scope of this tutorial
|
||||
|
||||
Our goal is to introduce you to Garage's workflows.
|
||||
Following this guide is recommended before moving on to
|
||||
[configuring a multi-node cluster](@/documentation/cookbook/real-world.md).
|
||||
@@ -258,6 +249,16 @@ mc alias set \
|
||||
--api S3v4
|
||||
```
|
||||
|
||||
You must also add an environment variable to your configuration to
|
||||
inform MinIO of our region (`garage` by default, corresponding to the `s3_region` parameter
|
||||
in the configuration file).
|
||||
The best way is to add the following snippet to your `$HOME/.bash_profile`
|
||||
or `$HOME/.bashrc` file:
|
||||
|
||||
```bash
|
||||
export MC_REGION=garage
|
||||
```
|
||||
|
||||
### Use `mc`
|
||||
|
||||
You can not list buckets from `mc` currently.
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
+++
|
||||
title = "Administration API"
|
||||
weight = 60
|
||||
weight = 16
|
||||
+++
|
||||
|
||||
The Garage administration API is accessible through a dedicated server whose
|
||||
@@ -134,8 +134,8 @@ Example request body:
|
||||
|
||||
```json
|
||||
[
|
||||
"ec79480e0ce52ae26fd00c9da684e4fa56658d9c64cdcecb094e936de0bfe71f@10.0.0.11:3901",
|
||||
"4a6ae5a1d0d33bf895f5bb4f0a418b7dc94c47c0dd2eb108d1158f3c8f60b0ff@10.0.0.12:3901"
|
||||
"ec79480e0ce52ae26fd00c9da684e4fa56658d9c64cdcecb094e936de0bfe71f@10.0.0.11:3901",
|
||||
"4a6ae5a1d0d33bf895f5bb4f0a418b7dc94c47c0dd2eb108d1158f3c8f60b0ff@10.0.0.12:3901"
|
||||
]
|
||||
```
|
||||
|
||||
@@ -145,14 +145,14 @@ Example response:
|
||||
|
||||
```json
|
||||
[
|
||||
{
|
||||
"success": true,
|
||||
"error": null
|
||||
},
|
||||
{
|
||||
"success": false,
|
||||
"error": "Handshake error"
|
||||
}
|
||||
{
|
||||
"success": true,
|
||||
"error": null
|
||||
},
|
||||
{
|
||||
"success": false,
|
||||
"error": "Handshake error"
|
||||
}
|
||||
]
|
||||
```
|
||||
|
||||
@@ -301,7 +301,7 @@ Request body format:
|
||||
|
||||
```json
|
||||
{
|
||||
"name": "NameOfMyKey"
|
||||
"name": "NameOfMyKey"
|
||||
}
|
||||
```
|
||||
|
||||
@@ -313,9 +313,9 @@ Request body format:
|
||||
|
||||
```json
|
||||
{
|
||||
"accessKeyId": "GK31c2f218a2e44f485b94239e",
|
||||
"secretAccessKey": "b892c0665f0ada8a4755dae98baa3b133590e11dae3bcc1f9d769d67f16c3835",
|
||||
"name": "NameOfMyKey"
|
||||
"accessKeyId": "GK31c2f218a2e44f485b94239e",
|
||||
"secretAccessKey": "b892c0665f0ada8a4755dae98baa3b133590e11dae3bcc1f9d769d67f16c3835",
|
||||
"name": "NameOfMyKey"
|
||||
}
|
||||
```
|
||||
|
||||
@@ -403,11 +403,11 @@ Request body format:
|
||||
|
||||
```json
|
||||
{
|
||||
"name": "NameOfMyKey",
|
||||
"allow": {
|
||||
"createBucket": true,
|
||||
},
|
||||
"deny": {}
|
||||
"name": "NameOfMyKey",
|
||||
"allow": {
|
||||
"createBucket": true,
|
||||
},
|
||||
"deny": {}
|
||||
}
|
||||
```
|
||||
|
||||
@@ -473,31 +473,24 @@ Example response:
|
||||
|
||||
```json
|
||||
{
|
||||
"id": "afa8f0a22b40b1247ccd0affb869b0af5cff980924a20e4b5e0720a44deb8d39",
|
||||
"globalAliases": [],
|
||||
"websiteAccess": false,
|
||||
"websiteConfig": null,
|
||||
"keys": [
|
||||
{
|
||||
"accessKeyId": "GK31c2f218a2e44f485b94239e",
|
||||
"name": "Imported key",
|
||||
"permissions": {
|
||||
"read": true,
|
||||
"write": true,
|
||||
"owner": true
|
||||
},
|
||||
"bucketLocalAliases": [
|
||||
"debug"
|
||||
]
|
||||
}
|
||||
],
|
||||
"objects": 14827,
|
||||
"bytes": 13189855625,
|
||||
"unfinshedUploads": 0,
|
||||
"quotas": {
|
||||
"maxSize": null,
|
||||
"maxObjects": null
|
||||
}
|
||||
"id": "e6a14cd6a27f48684579ec6b381c078ab11697e6bc8513b72b2f5307e25fff9b",
|
||||
"globalAliases": [
|
||||
"alex"
|
||||
],
|
||||
"keys": [
|
||||
{
|
||||
"accessKeyId": "GK31c2f218a2e44f485b94239e",
|
||||
"name": "alex",
|
||||
"permissions": {
|
||||
"read": true,
|
||||
"write": true,
|
||||
"owner": true
|
||||
},
|
||||
"bucketLocalAliases": [
|
||||
"test"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
@@ -509,7 +502,7 @@ Request body format:
|
||||
|
||||
```json
|
||||
{
|
||||
"globalAlias": "NameOfMyBucket"
|
||||
"globalAlias": "NameOfMyBucket"
|
||||
}
|
||||
```
|
||||
|
||||
@@ -517,15 +510,15 @@ OR
|
||||
|
||||
```json
|
||||
{
|
||||
"localAlias": {
|
||||
"accessKeyId": "GK31c2f218a2e44f485b94239e",
|
||||
"alias": "NameOfMyBucket",
|
||||
"allow": {
|
||||
"read": true,
|
||||
"write": true,
|
||||
"owner": false
|
||||
}
|
||||
}
|
||||
"localAlias": {
|
||||
"accessKeyId": "GK31c2f218a2e44f485b94239e",
|
||||
"alias": "NameOfMyBucket",
|
||||
"allow": {
|
||||
"read": true,
|
||||
"write": true,
|
||||
"owner": false
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
@@ -547,37 +540,26 @@ Deletes a storage bucket. A bucket cannot be deleted if it is not empty.
|
||||
|
||||
Warning: this will delete all aliases associated with the bucket!
|
||||
|
||||
#### UpdateBucket `PUT /v0/bucket?id=<bucket id>`
|
||||
#### PutBucketWebsite `PUT /v0/bucket/website?id=<bucket id>`
|
||||
|
||||
Updates configuration of the given bucket.
|
||||
Sets the website configuration for a bucket (this also enables website access for this bucket).
|
||||
|
||||
Request body format:
|
||||
|
||||
```json
|
||||
{
|
||||
"websiteAccess": {
|
||||
"enabled": true,
|
||||
"indexDocument": "index.html",
|
||||
"errorDocument": "404.html"
|
||||
},
|
||||
"quotas": {
|
||||
"maxSize": 19029801,
|
||||
"maxObjects": null,
|
||||
}
|
||||
"indexDocument": "index.html",
|
||||
"errorDocument": "404.html"
|
||||
}
|
||||
```
|
||||
|
||||
All fields (`websiteAccess` and `quotas`) are optionnal.
|
||||
If they are present, the corresponding modifications are applied to the bucket, otherwise nothing is changed.
|
||||
The field `errorDocument` is optional, if no error document is set a generic error message is displayed when errors happen.
|
||||
|
||||
In `websiteAccess`: if `enabled` is `true`, `indexDocument` must be specified.
|
||||
The field `errorDocument` is optional, if no error document is set a generic
|
||||
error message is displayed when errors happen. Conversely, if `enabled` is
|
||||
`false`, neither `indexDocument` nor `errorDocument` must be specified.
|
||||
|
||||
In `quotas`: new values of `maxSize` and `maxObjects` must both be specified, or set to `null`
|
||||
to remove the quotas. An absent value will be considered the same as a `null`. It is not possible
|
||||
to change only one of the two quotas.
|
||||
#### DeleteBucketWebsite `DELETE /v0/bucket/website?id=<bucket id>`
|
||||
|
||||
Deletes the website configuration for a bucket (disables website access for this bucket).
|
||||
|
||||
|
||||
### Operations on permissions for keys on buckets
|
||||
|
||||
@@ -589,13 +571,13 @@ Request body format:
|
||||
|
||||
```json
|
||||
{
|
||||
"bucketId": "e6a14cd6a27f48684579ec6b381c078ab11697e6bc8513b72b2f5307e25fff9b",
|
||||
"accessKeyId": "GK31c2f218a2e44f485b94239e",
|
||||
"permissions": {
|
||||
"read": true,
|
||||
"write": true,
|
||||
"owner": true
|
||||
},
|
||||
"bucketId": "e6a14cd6a27f48684579ec6b381c078ab11697e6bc8513b72b2f5307e25fff9b",
|
||||
"accessKeyId": "GK31c2f218a2e44f485b94239e",
|
||||
"permissions": {
|
||||
"read": true,
|
||||
"write": true,
|
||||
"owner": true
|
||||
},
|
||||
}
|
||||
```
|
||||
|
||||
@@ -610,13 +592,13 @@ Request body format:
|
||||
|
||||
```json
|
||||
{
|
||||
"bucketId": "e6a14cd6a27f48684579ec6b381c078ab11697e6bc8513b72b2f5307e25fff9b",
|
||||
"accessKeyId": "GK31c2f218a2e44f485b94239e",
|
||||
"permissions": {
|
||||
"read": false,
|
||||
"write": false,
|
||||
"owner": true
|
||||
},
|
||||
"bucketId": "e6a14cd6a27f48684579ec6b381c078ab11697e6bc8513b72b2f5307e25fff9b",
|
||||
"accessKeyId": "GK31c2f218a2e44f485b94239e",
|
||||
"permissions": {
|
||||
"read": false,
|
||||
"write": false,
|
||||
"owner": true
|
||||
},
|
||||
}
|
||||
```
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
+++
|
||||
title = "Garage CLI"
|
||||
weight = 30
|
||||
weight = 15
|
||||
+++
|
||||
|
||||
The Garage CLI is mostly self-documented. Make use of the `help` subcommand
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
+++
|
||||
title = "Configuration file format"
|
||||
weight = 20
|
||||
weight = 5
|
||||
+++
|
||||
|
||||
Here is an example `garage.toml` configuration file that illustrates all of the possible options:
|
||||
@@ -10,6 +10,7 @@ metadata_dir = "/var/lib/garage/meta"
|
||||
data_dir = "/var/lib/garage/data"
|
||||
|
||||
block_size = 1048576
|
||||
block_manager_background_tranquility = 2
|
||||
|
||||
replication_mode = "3"
|
||||
|
||||
@@ -86,6 +87,17 @@ files will remain available. This however means that chunks from existing files
|
||||
will not be deduplicated with chunks from newly uploaded files, meaning you
|
||||
might use more storage space that is optimally possible.
|
||||
|
||||
### `block_manager_background_tranquility`
|
||||
|
||||
This parameter tunes the activity of the background worker responsible for
|
||||
resyncing data blocks between nodes. The higher the tranquility value is set,
|
||||
the more the background worker will wait between iterations, meaning the load
|
||||
on the system (including network usage between nodes) will be reduced. The
|
||||
minimal value for this parameter is `0`, where the background worker will
|
||||
allways work at maximal throughput to resynchronize blocks. The default value
|
||||
is `2`, where the background worker will try to spend at most 1/3 of its time
|
||||
working, and 2/3 sleeping in order to reduce system load.
|
||||
|
||||
### `replication_mode`
|
||||
|
||||
Garage supports the following replication modes:
|
||||
|
||||
@@ -1,125 +0,0 @@
|
||||
+++
|
||||
title = "List of Garage features"
|
||||
weight = 10
|
||||
+++
|
||||
|
||||
|
||||
### S3 API
|
||||
|
||||
The main goal of Garage is to provide an object storage service that is compatible with the
|
||||
[S3 API](https://docs.aws.amazon.com/AmazonS3/latest/API/Welcome.html) from Amazon Web Services.
|
||||
We try to adhere as strictly as possible to the semantics of the API as implemented by Amazon
|
||||
and other vendors such as Minio or CEPH.
|
||||
|
||||
Of course Garage does not implement the full span of API endpoints that AWS S3 does;
|
||||
the exact list of S3 features implemented by Garage can be found [on our S3 compatibility page](@/documentation/reference-manual/s3-compatibility.md).
|
||||
|
||||
### Geo-distribution
|
||||
|
||||
Garage allows you to store copies of your data in multiple geographical locations in order to maximize resilience
|
||||
to adverse events, such as network/power outages or hardware failures.
|
||||
This allows Garage to run very well even at home, using consumer-grade Internet connectivity
|
||||
(such as FTTH) and power, as long as cluster nodes can be spawned at several physical locations.
|
||||
Garage exploits knowledge of the capacity and physical location of each storage node to design
|
||||
a storage plan that best exploits the available storage capacity while satisfying the geo-distributed replication constraint.
|
||||
|
||||
To learn more about geo-distributed Garage clusters,
|
||||
read our documentation on [setting up a real-world deployment](@/documentation/cookbook/real-world.md).
|
||||
|
||||
### Standalone/self-contained
|
||||
|
||||
Garage is extremely simple to deploy, and does not depend on any external service to run.
|
||||
This makes setting up and administering storage clusters, we hope, as easy as it could be.
|
||||
|
||||
### Flexible topology
|
||||
|
||||
A Garage cluster can very easily evolve over time, as storage nodes are added or removed.
|
||||
Garage will automatically rebalance data between nodes as needed to ensure the desired number of copies.
|
||||
Read about cluster layout management [here](@/documentation/reference-manual/layout.md).
|
||||
|
||||
### No RAFT slowing you down
|
||||
|
||||
It might seem strange to tout the absence of something as a desirable feature,
|
||||
but this is in fact a very important point! Garage does not use RAFT or another
|
||||
consensus algorithm internally to order incoming requests: this means that all requests
|
||||
directed to a Garage cluster can be handled independently of one another instead
|
||||
of going through a central bottleneck (the leader node).
|
||||
As a consequence, requests can be handled much faster, even in cases where latency
|
||||
between cluster nodes is important (see our [benchmarks](@/documentation/design/benchmarks/index.md) for data on this).
|
||||
This is particularly usefull when nodes are far from one another and talk to one other through standard Internet connections.
|
||||
|
||||
### Several replication modes
|
||||
|
||||
Garage supports a variety of replication modes, with 1 copy, 2 copies or 3 copies of your data,
|
||||
and with various levels of consistency, in order to adapt to a variety of usage scenarios.
|
||||
Read our reference page on [supported replication modes](@/documentation/reference-manual/configuration.md#replication-mode)
|
||||
to select the replication mode best suited to your use case (hint: in most cases, `replication_mode = "3"` is what you want).
|
||||
|
||||
### Web server for static websites
|
||||
|
||||
A storage bucket can easily be configured to be served directly by Garage as a static web site.
|
||||
Domain names for multiple websites directly map to bucket names, making it easy to build
|
||||
a platform for your users to autonomously build and host their websites over Garage.
|
||||
Surprisingly, none of the other alternative S3 implementations we surveyed (such as Minio
|
||||
or CEPH) support publishing static websites from S3 buckets, a feature that is however
|
||||
directly inherited from S3 on AWS.
|
||||
Read more on our [dedicated documentation page](@/documentation/cookbook/exposing-websites.md).
|
||||
|
||||
### Bucket names as aliases
|
||||
|
||||
In Garage, a bucket may have several names, known as aliases.
|
||||
Aliases can easily be added and removed on demand:
|
||||
this allows to easily rename buckets if needed
|
||||
without having to copy all of their content, something that cannot be done on AWS.
|
||||
For buckets served as static websites, having multiple aliases for a bucket can allow
|
||||
exposing the same content under different domain names.
|
||||
|
||||
Garage also supports bucket aliases which are local to a single user:
|
||||
this allows different users to have different buckets with the same name, thus avoiding naming collisions.
|
||||
This can be helpfull for instance if you want to write an application that creates per-user buckets with always the same name.
|
||||
|
||||
This feature is totally invisible to S3 clients and does not break compatibility with AWS.
|
||||
|
||||
### Cluster administration API
|
||||
|
||||
Garage provides a fully-fledged REST API to administer your cluster programatically.
|
||||
Functionnality included in the admin API include: setting up and monitoring
|
||||
cluster nodes, managing access credentials, and managing storage buckets and bucket aliases.
|
||||
A full reference of the administration API is available [here](@/documentation/reference-manual/admin-api.md).
|
||||
|
||||
### Metrics and traces
|
||||
|
||||
Garage makes some internal metrics available in the Prometheus data format,
|
||||
which allows you to build interactive dashboards to visualize the load and internal state of your storage cluster.
|
||||
|
||||
For developpers and performance-savvy administrators,
|
||||
Garage also supports exporting traces of what it does internally in OpenTelemetry format.
|
||||
This allows to monitor the time spent at various steps of the processing of requests,
|
||||
in order to detect potential performance bottlenecks.
|
||||
|
||||
### Kubernetes and Nomad integrations
|
||||
|
||||
Garage can automatically discover other nodes in the cluster thanks to integration
|
||||
with orchestrators such as Kubernetes and Nomad (when used with Consul).
|
||||
This eases the configuration of your cluster as it removes one step where nodes need
|
||||
to be manually connected to one another.
|
||||
|
||||
### Support for changing IP addresses
|
||||
|
||||
As long as all of your nodes don't thange their IP address at the same time,
|
||||
Garage should be able to tolerate nodes with changing/dynamic IP addresses,
|
||||
as nodes will regularly exchange the IP addresses of their peers and try to
|
||||
reconnect using newer addresses when existing connections are broken.
|
||||
|
||||
### K2V API (experimental)
|
||||
|
||||
As part of an ongoing research project, Garage can expose an experimental key/value storage API called K2V.
|
||||
K2V is made for the storage and retrieval of many small key/value pairs that need to be processed in bulk.
|
||||
This completes the S3 API with an alternative that can be used to easily store and access metadata
|
||||
related to objects stored in an S3 bucket.
|
||||
|
||||
In the context of our research project, [Aérogramme](https://aerogramme.deuxfleurs.fr),
|
||||
K2V is used to provide metadata and log storage for operations on encrypted e-mail storage.
|
||||
|
||||
Learn more on the specification of K2V [here](https://git.deuxfleurs.fr/Deuxfleurs/garage/src/branch/k2v/doc/drafts/k2v-spec.md)
|
||||
and on how to enable it in Garage [here](@/documentation/reference-manual/k2v.md).
|
||||
@@ -1,6 +1,6 @@
|
||||
+++
|
||||
title = "K2V"
|
||||
weight = 70
|
||||
weight = 30
|
||||
+++
|
||||
|
||||
Starting with version 0.7.2, Garage introduces an optionnal feature, K2V,
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
+++
|
||||
title = "Cluster layout management"
|
||||
weight = 50
|
||||
weight = 10
|
||||
+++
|
||||
|
||||
The cluster layout in Garage is a table that assigns to each node a role in
|
||||
|
||||
@@ -0,0 +1,45 @@
|
||||
+++
|
||||
title = "Request routing logic"
|
||||
weight = 10
|
||||
+++
|
||||
|
||||
Data retrieval requests to Garage endpoints (S3 API and websites) are resolved
|
||||
to an individual object in a bucket. Since objects are replicated to multiple nodes
|
||||
Garage must ensure consistency before answering the request.
|
||||
|
||||
## Using quorum to ensure consistency
|
||||
|
||||
Garage ensures consistency by attempting to establish a quorum with the
|
||||
data nodes responsible for the object. When a majority of the data nodes
|
||||
have provided metadata on a object Garage can then answer the request.
|
||||
|
||||
When a request arrives Garage will, assuming the recommended 3 replicas, perform the following actions:
|
||||
|
||||
- Make a request to the two preferred nodes for object metadata
|
||||
- Try the third node if one of the two initial requests fail
|
||||
- Check that the metadata from at least 2 nodes match
|
||||
- Check that the object hasn't been marked deleted
|
||||
- Answer the request with inline data from metadata if object is small enough
|
||||
- Or get data blocks from the preferred nodes and answer using the assembled object
|
||||
|
||||
Garage dynamically determines which nodes to query based on health, preference, and
|
||||
which nodes actually host a given data. Garage has no concept of "primary" so any
|
||||
healthy node with the data can be used as long as a quorum is reached for the metadata.
|
||||
|
||||
## Node health
|
||||
|
||||
Garage keeps a TCP session open to each node in the cluster and periodically pings them. If a connection
|
||||
cannot be established, or a node fails to answer a number of pings, the target node is marked as failed.
|
||||
Failed nodes are not used for quorum or other internal requests.
|
||||
|
||||
## Node preference
|
||||
|
||||
Garage prioritizes which nodes to query according to a few criteria:
|
||||
|
||||
- A node always prefers itself if it can answer the request
|
||||
- Then the node prioritizes nodes in the same zone
|
||||
- Finally the nodes with the lowest latency are prioritized
|
||||
|
||||
|
||||
For further reading on the cluster structure look at the [gateway](@/documentation/cookbook/gateways.md)
|
||||
and [cluster layout management](@/documentation/reference-manual/layout.md) pages.
|
||||
@@ -1,6 +1,6 @@
|
||||
+++
|
||||
title = "S3 Compatibility status"
|
||||
weight = 40
|
||||
weight = 20
|
||||
+++
|
||||
|
||||
## DISCLAIMER
|
||||
@@ -148,7 +148,7 @@ Please open an issue if you have a use case for replication.
|
||||
| [PutBucketReplication](https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutBucketReplication.html) | ❌ Missing | ❌| ⚠ | ❌| ❌|
|
||||
|
||||
*Note: Ceph documentation briefly says that Ceph supports
|
||||
[replication through the S3 API](https://docs.ceph.com/en/latest/radosgw/multisite-sync-policy/#s3-replication-api)
|
||||
[replication though the S3 API](https://docs.ceph.com/en/latest/radosgw/multisite-sync-policy/#s3-replication-api)
|
||||
but with some limitations.
|
||||
Additionaly, replication endpoints are not documented in the S3 compatibility page so I don't know what kind of support we can expect.*
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
+++
|
||||
title = "Design draft (obsolete)"
|
||||
weight = 50
|
||||
title = "Design draft"
|
||||
weight = 25
|
||||
+++
|
||||
|
||||
**WARNING: this documentation is a design draft which was written before Garage's actual implementation.
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
+++
|
||||
title = "Load balancing data (obsolete)"
|
||||
weight = 60
|
||||
title = "Load balancing data"
|
||||
weight = 10
|
||||
+++
|
||||
|
||||
**This is being yet improved in release 0.5. The working document has not been updated yet, it still only applies to Garage 0.2 through 0.4.**
|
||||
|
||||
@@ -1,14 +0,0 @@
|
||||
*
|
||||
|
||||
!assets
|
||||
|
||||
!.gitignore
|
||||
!*.svg
|
||||
!*.png
|
||||
!*.jpg
|
||||
!*.tex
|
||||
!Makefile
|
||||
!.gitignore
|
||||
!assets/*.drawio.pdf
|
||||
|
||||
!talk.pdf
|
||||
@@ -1,5 +0,0 @@
|
||||
talk.pdf: talk.tex assets/consistent_hashing_1.pdf assets/consistent_hashing_2.pdf assets/consistent_hashing_3.pdf assets/consistent_hashing_4.pdf assets/garage_tables.pdf assets/deuxfleurs.pdf
|
||||
pdflatex talk.tex
|
||||
|
||||
assets/%.pdf: assets/%.svg
|
||||
inkscape -D -z --file=$^ --export-pdf=$@
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 32 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 115 KiB |
File diff suppressed because one or more lines are too long
|
Before Width: | Height: | Size: 184 KiB |
@@ -1,3 +0,0 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:9198d0cfc0e04a56f84a353dd660b294ecf0dc68fe429892c8e9e604de016748
|
||||
size 31966
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 26 KiB |
@@ -1,3 +0,0 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:14dbf4a45545889babb2fdc6580399811366f6cc786cb28e3467cbcedbfe9482
|
||||
size 31688
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 27 KiB |
@@ -1,3 +0,0 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:21a3bb004ad35c8b47f14195c720048f8db31f47bd6346c1d747000570149c67
|
||||
size 31073
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 8.9 KiB |
@@ -1,3 +0,0 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:c4e109078a784c3999ca2ccd1adda8ff5793d1b230a9e27b4ad2e419afa0d37d
|
||||
size 25145
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 18 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 4.8 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 263 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 82 KiB |
File diff suppressed because one or more lines are too long
|
Before Width: | Height: | Size: 53 KiB |
File diff suppressed because one or more lines are too long
|
Before Width: | Height: | Size: 54 KiB |
File diff suppressed because one or more lines are too long
|
Before Width: | Height: | Size: 56 KiB |
File diff suppressed because one or more lines are too long
|
Before Width: | Height: | Size: 57 KiB |
@@ -1,91 +0,0 @@
|
||||
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
|
||||
<svg
|
||||
viewBox="0 0 70.424515 70.300102"
|
||||
version="1.1"
|
||||
id="svg8"
|
||||
sodipodi:docname="logo.svg"
|
||||
inkscape:version="1.1 (c68e22c387, 2021-05-23)"
|
||||
inkscape:export-filename="/home/quentin/Documents/dev/deuxfleurs/site/src/img/logo.png"
|
||||
inkscape:export-xdpi="699.30194"
|
||||
inkscape:export-ydpi="699.30194"
|
||||
width="70.424515"
|
||||
height="70.300102"
|
||||
xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
|
||||
xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
|
||||
xmlns="http://www.w3.org/2000/svg"
|
||||
xmlns:svg="http://www.w3.org/2000/svg">
|
||||
<defs
|
||||
id="defs12" />
|
||||
<sodipodi:namedview
|
||||
id="namedview10"
|
||||
pagecolor="#ffffff"
|
||||
bordercolor="#666666"
|
||||
borderopacity="1.0"
|
||||
inkscape:pageshadow="2"
|
||||
inkscape:pageopacity="0.0"
|
||||
inkscape:pagecheckerboard="0"
|
||||
showgrid="false"
|
||||
inkscape:zoom="12.125"
|
||||
inkscape:cx="43.092783"
|
||||
inkscape:cy="48.082474"
|
||||
inkscape:window-width="3072"
|
||||
inkscape:window-height="1659"
|
||||
inkscape:window-x="0"
|
||||
inkscape:window-y="0"
|
||||
inkscape:window-maximized="1"
|
||||
inkscape:current-layer="svg8" />
|
||||
<g
|
||||
id="g79969"
|
||||
transform="translate(-0.827,34.992103)">
|
||||
<path
|
||||
fill="#ffffff"
|
||||
d="m 15.632,34.661 c -0.799,-0.597 -1.498,-1.484 -2.035,-2.592 l -0.228,-0.47 -0.46,0.249 c -0.975,0.528 -1.913,0.858 -2.744,0.969 L 9.963,29.061 6.327,30.029 C 6.17,29.175 6.202,28.142 6.423,27.007 L 6.526,26.482 5.994,26.416 C 4.752,26.262 3.688,25.891 2.89,25.336 L 4.411,22.419 1.423,20.896 C 1.742,19.952 2.371,19.014 3.257,18.161 L 3.634,17.798 3.255,17.438 C 2.452,16.674 1.847,15.884 1.485,15.127 L 4.995,13.774 2.95,10.615 C 3.69,10.213 4.643,9.929 5.739,9.783 L 6.258,9.715 6.167,9.201 C 5.952,7.99 5.995,6.863 6.291,5.913 l 3.308,0.523 0.524,-3.308 c 0.988,0.013 2.08,0.326 3.164,0.907 L 13.749,4.283 13.975,3.81 C 14.454,2.807 15.019,1.986 15.628,1.406 L 18,4.326 20.372,1.406 c 0.609,0.58 1.175,1.401 1.653,2.404 l 0.226,0.473 0.462,-0.247 C 23.798,3.455 24.891,3.142 25.877,3.13 L 26.4,6.438 29.71,5.913 c 0.296,0.951 0.34,2.078 0.124,3.288 l -0.092,0.515 0.518,0.069 c 1.095,0.145 2.048,0.43 2.788,0.832 l -2.046,3.156 3.511,1.355 c -0.361,0.757 -0.966,1.547 -1.77,2.311 l -0.379,0.36 0.377,0.363 c 0.888,0.854 1.516,1.793 1.835,2.736 l -2.984,1.52 1.521,2.984 c -0.812,0.574 -1.871,0.964 -3.094,1.134 l -0.518,0.072 0.096,0.514 c 0.201,1.089 0.226,2.083 0.073,2.909 l -3.634,-0.97 -0.204,3.757 c -0.83,-0.11 -1.768,-0.44 -2.742,-0.968 l -0.459,-0.249 -0.228,0.47 c -0.539,1.107 -1.237,1.994 -2.036,2.591 L 18,32.293 Z"
|
||||
id="path2" />
|
||||
<path
|
||||
d="M 7.092,10.678 C 6.562,9.189 6.394,7.708 6.66,6.478 l 2.368,0.375 0.987,0.156 0.157,-0.988 0.375,-2.368 C 11.808,3.78 13.16,4.396 14.409,5.359 14.527,5.022 14.653,4.696 14.791,4.392 13.24,3.257 11.568,2.629 10.061,2.629 9.938,2.629 9.816,2.633 9.695,2.642 L 9.184,5.865 5.96,5.354 C 5.36,6.841 5.395,8.769 6.045,10.747 6.38,10.71 6.729,10.686 7.092,10.678 Z M 21.593,5.359 c 1.248,-0.962 2.6,-1.578 3.86,-1.705 l 0.376,2.368 0.156,0.988 0.987,-0.157 2.369,-0.376 c 0.266,1.23 0.098,2.71 -0.432,4.2 0.361,0.009 0.711,0.032 1.046,0.07 C 30.606,8.769 30.64,6.841 30.04,5.353 L 26.815,5.865 26.304,2.641 c -0.12,-0.008 -0.242,-0.012 -0.365,-0.012 -1.507,0 -3.179,0.628 -4.73,1.762 0.14,0.306 0.266,0.631 0.384,0.968 z M 7.368,27 h 0.035 c 0.067,0 0.157,-0.604 0.26,-0.947 -0.098,0.004 -0.197,0.046 -0.294,0.046 -1.496,0 -2.826,-0.303 -3.83,-0.89 L 4.628,23.081 5.082,22.194 4.191,21.742 2.055,20.654 C 2.563,19.503 3.57,18.404 4.873,17.511 4.586,17.292 4.312,17.07 4.063,16.842 2.376,18.059 1.217,19.597 0.828,21.152 l 2.908,1.483 -1.482,2.843 C 3.475,26.501 5.303,27 7.368,27 Z m 27.806,-5.846 c -0.39,-1.555 -1.548,-3.093 -3.234,-4.311 -0.25,0.228 -0.523,0.451 -0.81,0.669 1.304,0.893 2.31,1.992 2.817,3.145 l -2.136,1.088 -0.891,0.453 0.454,0.892 1.089,2.137 c -1.004,0.587 -2.332,0.904 -3.828,0.904 -0.099,0 -0.199,-0.01 -0.299,-0.013 0.103,0.344 0.192,0.683 0.26,1.011 l 0.039,0.002 c 2.066,0 3.892,-0.563 5.112,-1.587 l -1.482,-2.908 z m -12.653,9.182 c -0.447,1.517 -1.181,2.812 -2.119,3.651 L 18.707,32.293 18,31.586 l -0.707,0.707 -1.695,1.694 c -0.938,-0.839 -1.673,-2.136 -2.12,-3.652 -0.296,0.206 -0.593,0.397 -0.886,0.563 0.636,1.98 1.741,3.559 3.1,4.409 L 18,33 l 2.308,2.308 c 1.358,-0.851 2.464,-2.428 3.101,-4.408 -0.295,-0.168 -0.591,-0.359 -0.888,-0.564 z"
|
||||
fill="#ea596e"
|
||||
id="path4" />
|
||||
<path
|
||||
fill="#ea596e"
|
||||
d="m 20.118,5.683 c 0.426,1.146 0.748,2.596 0.841,4.284 l 0.2,3.683 3.564,-0.946 c 1.32,-0.351 2.655,-0.536 3.86,-0.536 0.16,0 0.318,0.003 0.474,0.01 l -1.827,2.819 3.139,1.211 c -0.958,0.759 -2.237,1.514 -3.814,2.123 l -3.441,1.328 2.001,3.099 c 0.918,1.42 1.509,2.782 1.838,3.96 L 23.709,25.853 23.527,29.21 C 22.508,28.533 21.395,27.55 20.329,26.237 L 18,23.374 15.672,26.236 c -1.066,1.312 -2.179,2.295 -3.198,2.972 l -0.18,-3.354 -3.248,0.864 c 0.329,-1.178 0.921,-2.54 1.839,-3.961 L 12.889,19.658 9.447,18.33 C 7.87,17.721 6.591,16.967 5.633,16.208 L 8.768,15 6.941,12.177 c 0.155,-0.006 0.313,-0.01 0.473,-0.01 1.206,0 2.541,0.185 3.861,0.536 l 3.564,0.947 0.202,-3.683 c 0.092,-1.688 0.415,-3.138 0.84,-4.284 L 18,8.292 20.118,5.683 M 20.308,0.692 18,3.533 15.692,0.692 C 13.703,2.224 12.271,5.684 12.046,9.804 10.429,9.374 8.854,9.167 7.414,9.167 c -2.11,0 -3.929,0.445 -5.161,1.289 l 1.989,3.073 -3.415,1.316 c 0.842,2.366 3.69,4.797 7.54,6.283 -2.241,3.465 -3.116,7.106 -2.407,9.516 l 3.537,-0.941 0.196,3.654 c 2.512,-0.07 5.703,-2.027 8.307,-5.228 2.603,3.201 5.796,5.158 8.306,5.228 l 0.198,-3.655 3.535,0.943 c 0.71,-2.411 -0.165,-6.05 -2.404,-9.517 3.849,-1.485 6.696,-3.918 7.538,-6.283 l -3.415,-1.318 1.99,-3.07 c -1.233,-0.844 -3.053,-1.29 -5.164,-1.29 -1.438,0 -3.013,0.207 -4.63,0.636 C 23.729,5.684 22.297,2.224 20.308,0.692 Z"
|
||||
id="path6" />
|
||||
</g>
|
||||
<g
|
||||
id="g79964"
|
||||
transform="translate(-1.043816,35.993714)">
|
||||
<path
|
||||
fill="#ffffff"
|
||||
d="m 51.92633,-2.0247139 c -0.799,-0.597 -1.498,-1.484 -2.035,-2.592 l -0.228,-0.47 -0.46,0.249 c -0.975,0.528 -1.913,0.858 -2.744,0.969 l -0.202,-3.7560001 -3.636,0.968 c -0.157,-0.854 -0.125,-1.887 0.096,-3.022 l 0.103,-0.525 -0.532,-0.066 c -1.242,-0.154 -2.306,-0.525 -3.104,-1.08 l 1.521,-2.917 -2.988,-1.523 c 0.319,-0.944 0.948,-1.882 1.834,-2.735 l 0.377,-0.363 -0.379,-0.36 c -0.803,-0.764 -1.408,-1.554 -1.77,-2.311 l 3.51,-1.353 -2.045,-3.159 c 0.74,-0.402 1.693,-0.686 2.789,-0.832 l 0.519,-0.068 -0.091,-0.514 c -0.215,-1.211 -0.172,-2.338 0.124,-3.288 l 3.308,0.523 0.524,-3.308 c 0.988,0.013 2.08,0.326 3.164,0.907 l 0.462,0.248 0.226,-0.473 c 0.479,-1.003 1.044,-1.824 1.653,-2.404 l 2.372,2.92 2.372,-2.92 c 0.609,0.58 1.175,1.401 1.653,2.404 l 0.226,0.473 0.462,-0.247 c 1.085,-0.581 2.178,-0.894 3.164,-0.906 l 0.523,3.308 3.31,-0.525 c 0.296,0.951 0.34,2.078 0.124,3.288 l -0.092,0.515 0.518,0.069 c 1.095,0.145 2.048,0.43 2.788,0.832 l -2.046,3.156 3.511,1.355 c -0.361,0.757 -0.966,1.547 -1.77,2.311 l -0.379,0.36 0.377,0.363 c 0.888,0.854 1.516,1.793 1.835,2.736 l -2.984,1.52 1.521,2.984 c -0.812,0.574 -1.871,0.964 -3.094,1.134 l -0.518,0.072 0.096,0.514 c 0.201,1.089 0.226,2.083 0.073,2.909 l -3.634,-0.97 -0.204,3.7570001 c -0.83,-0.11 -1.768,-0.44 -2.742,-0.968 l -0.459,-0.249 -0.228,0.47 c -0.539,1.107 -1.237,1.994 -2.036,2.591 l -2.367,-2.369 z"
|
||||
id="path2-9" />
|
||||
<path
|
||||
d="m 43.38633,-26.007714 c -0.53,-1.489 -0.698,-2.97 -0.432,-4.2 l 2.368,0.375 0.987,0.156 0.157,-0.988 0.375,-2.368 c 1.261,0.127 2.613,0.743 3.862,1.706 0.118,-0.337 0.244,-0.663 0.382,-0.967 -1.551,-1.135 -3.223,-1.763 -4.73,-1.763 -0.123,0 -0.245,0.004 -0.366,0.013 l -0.511,3.223 -3.224,-0.511 c -0.6,1.487 -0.565,3.415 0.085,5.393 0.335,-0.037 0.684,-0.061 1.047,-0.069 z m 14.501,-5.319 c 1.248,-0.962 2.6,-1.578 3.86,-1.705 l 0.376,2.368 0.156,0.988 0.987,-0.157 2.369,-0.376 c 0.266,1.23 0.098,2.71 -0.432,4.2 0.361,0.009 0.711,0.032 1.046,0.07 0.651,-1.978 0.685,-3.906 0.085,-5.394 l -3.225,0.512 -0.511,-3.224 c -0.12,-0.008 -0.242,-0.012 -0.365,-0.012 -1.507,0 -3.179,0.628 -4.73,1.762 0.14,0.306 0.266,0.631 0.384,0.968 z m -14.225,21.641 h 0.035 c 0.067,0 0.157,-0.604 0.26,-0.947 -0.098,0.004 -0.197,0.046 -0.294,0.046 -1.496,0 -2.826,-0.303 -3.83,-0.89 l 1.089,-2.128 0.454,-0.887 -0.891,-0.452 -2.136,-1.088 c 0.508,-1.151 1.515,-2.25 2.818,-3.143 -0.287,-0.219 -0.561,-0.441 -0.81,-0.669 -1.687,1.217 -2.846,2.755 -3.235,4.31 l 2.908,1.483 -1.482,2.843 c 1.221,1.023 3.049,1.522 5.114,1.522 z m 27.806,-5.846 c -0.39,-1.555 -1.548,-3.093 -3.234,-4.311 -0.25,0.228 -0.523,0.451 -0.81,0.669 1.304,0.893 2.31,1.992 2.817,3.145 l -2.136,1.088 -0.891,0.453 0.454,0.892 1.089,2.137 c -1.004,0.587 -2.332,0.904 -3.828,0.904 -0.099,0 -0.199,-0.01 -0.299,-0.013 0.103,0.344 0.192,0.683 0.26,1.011 l 0.039,0.002 c 2.066,0 3.892,-0.563 5.112,-1.587 l -1.482,-2.908 z m -12.653,9.182 c -0.447,1.5170001 -1.181,2.8120001 -2.119,3.6510001 l -1.695,-1.694 -0.707,-0.707 -0.707,0.707 -1.695,1.694 c -0.938,-0.839 -1.673,-2.136 -2.12,-3.6520001 -0.296,0.2060001 -0.593,0.3970001 -0.886,0.5630001 0.636,1.98 1.741,3.559 3.1,4.409 l 2.308,-2.307 2.308,2.308 c 1.358,-0.851 2.464,-2.428 3.101,-4.408 -0.295,-0.168 -0.591,-0.359 -0.888,-0.5640001 z"
|
||||
fill="#ea596e"
|
||||
id="path4-3" />
|
||||
<path
|
||||
fill="#ea596e"
|
||||
d="m 56.41233,-31.002714 c 0.426,1.146 0.748,2.596 0.841,4.284 l 0.2,3.683 3.564,-0.946 c 1.32,-0.351 2.655,-0.536 3.86,-0.536 0.16,0 0.318,0.003 0.474,0.01 l -1.827,2.819 3.139,1.211 c -0.958,0.759 -2.237,1.514 -3.814,2.123 l -3.441,1.328 2.001,3.099 c 0.918,1.42 1.509,2.782 1.838,3.96 l -3.244,-0.865 -0.182,3.357 c -1.019,-0.677 -2.132,-1.66 -3.198,-2.973 l -2.329,-2.863 -2.328,2.862 c -1.066,1.312 -2.179,2.295 -3.198,2.972 l -0.18,-3.354 -3.248,0.864 c 0.329,-1.178 0.921,-2.54 1.839,-3.961 l 2.004,-3.099 -3.442,-1.328 c -1.577,-0.609 -2.856,-1.363 -3.814,-2.122 l 3.135,-1.208 -1.827,-2.823 c 0.155,-0.006 0.313,-0.01 0.473,-0.01 1.206,0 2.541,0.185 3.861,0.536 l 3.564,0.947 0.202,-3.683 c 0.092,-1.688 0.415,-3.138 0.84,-4.284 l 2.119,2.609 2.118,-2.609 m 0.19,-4.991 -2.308,2.841 -2.308,-2.841 c -1.989,1.532 -3.421,4.992 -3.646,9.112 -1.617,-0.43 -3.192,-0.637 -4.632,-0.637 -2.11,0 -3.929,0.445 -5.161,1.289 l 1.989,3.073 -3.415,1.316 c 0.842,2.366 3.69,4.797 7.54,6.283 -2.241,3.465 -3.116,7.106 -2.407,9.5160001 l 3.537,-0.9410001 0.196,3.6540001 c 2.512,-0.07 5.703,-2.027 8.307,-5.2280001 2.603,3.2010001 5.796,5.1580001 8.306,5.2280001 l 0.198,-3.6550001 3.535,0.9430001 c 0.71,-2.4110001 -0.165,-6.0500001 -2.404,-9.5170001 3.849,-1.485 6.696,-3.918 7.538,-6.283 l -3.415,-1.318 1.99,-3.07 c -1.233,-0.844 -3.053,-1.29 -5.164,-1.29 -1.438,0 -3.013,0.207 -4.63,0.636 -0.225,-4.119 -1.657,-7.579 -3.646,-9.111 z"
|
||||
id="path6-6" />
|
||||
</g>
|
||||
<text
|
||||
xml:space="preserve"
|
||||
style="font-style:normal;font-weight:normal;font-size:42.6667px;line-height:1.25;font-family:sans-serif;fill:#ea596e;fill-opacity:1;stroke:none"
|
||||
x="2.2188232"
|
||||
y="31.430677"
|
||||
id="text46212"><tspan
|
||||
sodipodi:role="line"
|
||||
id="tspan46210"
|
||||
x="2.2188232"
|
||||
y="31.430677"
|
||||
style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:42.6667px;font-family:'TeX Gyre Termes';-inkscape-font-specification:'TeX Gyre Termes'">D</tspan></text>
|
||||
<text
|
||||
xml:space="preserve"
|
||||
style="font-style:normal;font-weight:normal;font-size:42.6667px;line-height:1.25;font-family:sans-serif;fill:#ea596e;fill-opacity:1;stroke:none"
|
||||
x="41.347008"
|
||||
y="67.114784"
|
||||
id="text46212-1"><tspan
|
||||
sodipodi:role="line"
|
||||
id="tspan46210-5"
|
||||
x="41.347008"
|
||||
y="67.114784"
|
||||
style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:42.6667px;font-family:'TeX Gyre Termes';-inkscape-font-specification:'TeX Gyre Termes'">F</tspan></text>
|
||||
</svg>
|
||||
|
Before Width: | Height: | Size: 12 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 129 KiB |
@@ -1,3 +0,0 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:0d9ca1a04f943664e14d0e1edbde596789e4798f8ad5e1cff68bc9dd0cc1334f
|
||||
size 26098
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 13 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 88 KiB |
@@ -1,3 +0,0 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:bffa5d4a72ad25e0b18b43022258a2e4471ce94ff5397aab892fe43ec9d4d7d5
|
||||
size 33911
|
||||
@@ -1,3 +0,0 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:9c4913b52e84b5af3f8effbcb7e2060845a8c6f43d5b38072b9ee33c0300d49f
|
||||
size 31051
|
||||
@@ -1,537 +0,0 @@
|
||||
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
|
||||
<svg
|
||||
width="850"
|
||||
height="480"
|
||||
viewBox="0 0 224.89584 127"
|
||||
version="1.1"
|
||||
id="svg8"
|
||||
inkscape:version="1.2 (dc2aedaf03, 2022-05-15)"
|
||||
sodipodi:docname="garage_tables.svg"
|
||||
xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
|
||||
xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
|
||||
xmlns="http://www.w3.org/2000/svg"
|
||||
xmlns:svg="http://www.w3.org/2000/svg"
|
||||
xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
|
||||
xmlns:cc="http://creativecommons.org/ns#"
|
||||
xmlns:dc="http://purl.org/dc/elements/1.1/">
|
||||
<defs
|
||||
id="defs2">
|
||||
<marker
|
||||
style="overflow:visible"
|
||||
id="marker1262"
|
||||
refX="0"
|
||||
refY="0"
|
||||
orient="auto"
|
||||
inkscape:stockid="Arrow1Mend"
|
||||
inkscape:isstock="true">
|
||||
<path
|
||||
transform="matrix(-0.4,0,0,-0.4,-4,0)"
|
||||
style="fill:#000000;fill-opacity:1;fill-rule:evenodd;stroke:#000000;stroke-width:1pt;stroke-opacity:1"
|
||||
d="M 0,0 5,-5 -12.5,0 5,5 Z"
|
||||
id="path1260" />
|
||||
</marker>
|
||||
<marker
|
||||
style="overflow:visible"
|
||||
id="Arrow1Mend"
|
||||
refX="0"
|
||||
refY="0"
|
||||
orient="auto"
|
||||
inkscape:stockid="Arrow1Mend"
|
||||
inkscape:isstock="true"
|
||||
inkscape:collect="always">
|
||||
<path
|
||||
transform="matrix(-0.4,0,0,-0.4,-4,0)"
|
||||
style="fill:#000000;fill-opacity:1;fill-rule:evenodd;stroke:#000000;stroke-width:1pt;stroke-opacity:1"
|
||||
d="M 0,0 5,-5 -12.5,0 5,5 Z"
|
||||
id="path965" />
|
||||
</marker>
|
||||
<marker
|
||||
style="overflow:visible"
|
||||
id="Arrow1Lend"
|
||||
refX="0"
|
||||
refY="0"
|
||||
orient="auto"
|
||||
inkscape:stockid="Arrow1Lend"
|
||||
inkscape:isstock="true">
|
||||
<path
|
||||
transform="matrix(-0.8,0,0,-0.8,-10,0)"
|
||||
style="fill:#000000;fill-opacity:1;fill-rule:evenodd;stroke:#000000;stroke-width:1pt;stroke-opacity:1"
|
||||
d="M 0,0 5,-5 -12.5,0 5,5 Z"
|
||||
id="path959" />
|
||||
</marker>
|
||||
</defs>
|
||||
<sodipodi:namedview
|
||||
id="base"
|
||||
pagecolor="#ffffff"
|
||||
bordercolor="#666666"
|
||||
borderopacity="1.0"
|
||||
inkscape:pageopacity="0.0"
|
||||
inkscape:pageshadow="2"
|
||||
inkscape:zoom="0.98994949"
|
||||
inkscape:cx="429.31483"
|
||||
inkscape:cy="289.40871"
|
||||
inkscape:document-units="mm"
|
||||
inkscape:current-layer="layer1"
|
||||
inkscape:document-rotation="0"
|
||||
showgrid="false"
|
||||
units="px"
|
||||
inkscape:window-width="1678"
|
||||
inkscape:window-height="993"
|
||||
inkscape:window-x="0"
|
||||
inkscape:window-y="0"
|
||||
inkscape:window-maximized="1"
|
||||
inkscape:showpageshadow="2"
|
||||
inkscape:pagecheckerboard="0"
|
||||
inkscape:deskcolor="#d1d1d1" />
|
||||
<metadata
|
||||
id="metadata5">
|
||||
<rdf:RDF>
|
||||
<cc:Work
|
||||
rdf:about="">
|
||||
<dc:format>image/svg+xml</dc:format>
|
||||
<dc:type
|
||||
rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
|
||||
<dc:title />
|
||||
</cc:Work>
|
||||
</rdf:RDF>
|
||||
</metadata>
|
||||
<g
|
||||
inkscape:label="Layer 1"
|
||||
inkscape:groupmode="layer"
|
||||
id="layer1">
|
||||
<text
|
||||
xml:space="preserve"
|
||||
style="font-style:normal;font-weight:normal;font-size:5.64444px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.264583"
|
||||
x="39.570904"
|
||||
y="38.452755"
|
||||
id="text2025"><tspan
|
||||
sodipodi:role="line"
|
||||
id="tspan2023"
|
||||
x="39.570904"
|
||||
y="38.452755"
|
||||
style="font-size:5.64444px;stroke-width:0.264583" /></text>
|
||||
<text
|
||||
xml:space="preserve"
|
||||
style="font-style:normal;font-weight:normal;font-size:10.5833px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.264583"
|
||||
x="101.95796"
|
||||
y="92.835831"
|
||||
id="text2139"><tspan
|
||||
sodipodi:role="line"
|
||||
id="tspan2137"
|
||||
x="101.95796"
|
||||
y="92.835831"
|
||||
style="stroke-width:0.264583"> </tspan></text>
|
||||
<g
|
||||
id="g2316"
|
||||
transform="translate(-11.455511,1.5722486)">
|
||||
<g
|
||||
id="g2277">
|
||||
<rect
|
||||
style="fill:none;stroke:#000000;stroke-width:0.8;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1"
|
||||
id="rect833"
|
||||
width="47.419891"
|
||||
height="95.353409"
|
||||
x="18.534418"
|
||||
y="24.42766" />
|
||||
<rect
|
||||
style="fill:none;stroke:#000000;stroke-width:0.799999;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1"
|
||||
id="rect833-3"
|
||||
width="47.419891"
|
||||
height="86.973076"
|
||||
x="18.534418"
|
||||
y="32.807987" />
|
||||
<text
|
||||
xml:space="preserve"
|
||||
style="font-style:normal;font-weight:normal;font-size:5.64444px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.264583"
|
||||
x="32.250839"
|
||||
y="29.894743"
|
||||
id="text852"><tspan
|
||||
sodipodi:role="line"
|
||||
id="tspan850"
|
||||
x="32.250839"
|
||||
y="29.894743"
|
||||
style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:5.64444px;font-family:'Liberation Mono';-inkscape-font-specification:'Liberation Mono Bold';stroke-width:0.264583">Object</tspan></text>
|
||||
</g>
|
||||
<g
|
||||
id="g2066"
|
||||
transform="translate(-2.1807817,-3.0621439)">
|
||||
<g
|
||||
id="g1969"
|
||||
transform="matrix(0.12763631,0,0,0.12763631,0.7215051,24.717273)"
|
||||
style="fill:#ff6600;fill-opacity:1;stroke:none;stroke-opacity:1">
|
||||
<path
|
||||
style="fill:#ff6600;fill-opacity:1;stroke:none;stroke-width:0.264583;stroke-opacity:1"
|
||||
d="m 203.71837,154.80038 c -1.11451,3.75057 -2.45288,5.84095 -5.11132,7.98327 -2.2735,1.83211 -4.66721,2.65982 -8.09339,2.79857 -2.59227,0.10498 -2.92868,0.0577 -5.02863,-0.70611 -3.99215,-1.45212 -7.1627,-4.65496 -8.48408,-8.57046 -1.28374,-3.80398 -0.61478,-8.68216 1.64793,-12.01698 0.87317,-1.28689 3.15089,-3.48326 4.18771,-4.03815 l 0.53332,-28.51234 5.78454,-5.09197 6.95158,6.16704 -3.21112,3.49026 3.17616,3.45499 -3.17616,3.40822 2.98973,3.28645 -3.24843,3.3829 4.49203,4.58395 0.0516,5.69106 c 1.06874,0.64848 3.81974,3.24046 4.69548,4.56257 0.452,0.68241 1.06834,2.0197 1.36962,2.97176 0.62932,1.98864 0.88051,5.785 0.47342,7.15497 z m -10.0406,2.32604 c -0.88184,-3.17515 -4.92402,-3.78864 -6.75297,-1.02492 -0.58328,0.8814 -0.6898,1.28852 -0.58362,2.23056 0.26492,2.35041 2.45434,3.95262 4.60856,3.37255 1.19644,-0.32217 2.39435,-1.44872 2.72875,-2.56621 0.30682,-1.02529 0.30686,-0.9045 -7.9e-4,-2.01198 z"
|
||||
id="path1971"
|
||||
sodipodi:nodetypes="ssscsscccccccccccssscsssscc" />
|
||||
</g>
|
||||
<text
|
||||
xml:space="preserve"
|
||||
style="font-style:normal;font-weight:normal;font-size:5.64444px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.264583"
|
||||
x="28.809687"
|
||||
y="44.070885"
|
||||
id="text852-9"><tspan
|
||||
sodipodi:role="line"
|
||||
id="tspan850-4"
|
||||
x="28.809687"
|
||||
y="44.070885"
|
||||
style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:5.64444px;font-family:'Liberation Mono';-inkscape-font-specification:'Liberation Mono Bold';stroke-width:0.264583">bucket </tspan></text>
|
||||
</g>
|
||||
<g
|
||||
id="g2066-7"
|
||||
transform="translate(-2.1807817,6.2627616)">
|
||||
<g
|
||||
id="g1969-8"
|
||||
transform="matrix(0.12763631,0,0,0.12763631,0.7215051,24.717273)"
|
||||
style="fill:#ff6600;fill-opacity:1;stroke:none;stroke-opacity:1">
|
||||
<path
|
||||
style="fill:#4040ff;fill-opacity:1;stroke:none;stroke-width:0.264583;stroke-opacity:1"
|
||||
d="m 203.71837,154.80038 c -1.11451,3.75057 -2.45288,5.84095 -5.11132,7.98327 -2.2735,1.83211 -4.66721,2.65982 -8.09339,2.79857 -2.59227,0.10498 -2.92868,0.0577 -5.02863,-0.70611 -3.99215,-1.45212 -7.1627,-4.65496 -8.48408,-8.57046 -1.28374,-3.80398 -0.61478,-8.68216 1.64793,-12.01698 0.87317,-1.28689 3.15089,-3.48326 4.18771,-4.03815 l 0.53332,-28.51234 5.78454,-5.09197 6.95158,6.16704 -3.21112,3.49026 3.17616,3.45499 -3.17616,3.40822 2.98973,3.28645 -3.24843,3.3829 4.49203,4.58395 0.0516,5.69106 c 1.06874,0.64848 3.81974,3.24046 4.69548,4.56257 0.452,0.68241 1.06834,2.0197 1.36962,2.97176 0.62932,1.98864 0.88051,5.785 0.47342,7.15497 z m -10.0406,2.32604 c -0.88184,-3.17515 -4.92402,-3.78864 -6.75297,-1.02492 -0.58328,0.8814 -0.6898,1.28852 -0.58362,2.23056 0.26492,2.35041 2.45434,3.95262 4.60856,3.37255 1.19644,-0.32217 2.39435,-1.44872 2.72875,-2.56621 0.30682,-1.02529 0.30686,-0.9045 -7.9e-4,-2.01198 z"
|
||||
id="path1971-4"
|
||||
sodipodi:nodetypes="ssscsscccccccccccssscsssscc" />
|
||||
</g>
|
||||
<text
|
||||
xml:space="preserve"
|
||||
style="font-style:normal;font-weight:normal;font-size:5.64444px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.264583"
|
||||
x="28.809687"
|
||||
y="44.070885"
|
||||
id="text852-9-5"><tspan
|
||||
sodipodi:role="line"
|
||||
id="tspan850-4-0"
|
||||
x="28.809687"
|
||||
y="44.070885"
|
||||
style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:5.64444px;font-family:'Liberation Mono';-inkscape-font-specification:'Liberation Mono Bold';stroke-width:0.264583">file path </tspan></text>
|
||||
<path
|
||||
style="fill:#ff6600;fill-opacity:1;stroke:none;stroke-width:0.0337704;stroke-opacity:1"
|
||||
d="m 174.20027,104.45585 c -0.14225,0.47871 -0.31308,0.74552 -0.65239,1.01896 -0.29018,0.23384 -0.5957,0.33949 -1.03301,0.3572 -0.33087,0.0134 -0.37381,0.007 -0.64184,-0.0901 -0.50954,-0.18534 -0.91422,-0.59414 -1.08287,-1.0939 -0.16385,-0.48552 -0.0785,-1.10816 0.21033,-1.5338 0.11145,-0.16426 0.40217,-0.44459 0.53451,-0.51542 l 0.0681,-3.639207 0.73832,-0.64992 0.88727,0.787138 -0.40986,0.445484 0.4054,0.440982 -0.4054,0.435013 0.3816,0.41947 -0.41461,0.43178 0.57334,0.58508 0.007,0.72639 c 0.13641,0.0828 0.48753,0.4136 0.59931,0.58235 0.0577,0.0871 0.13636,0.25778 0.17481,0.3793 0.0803,0.25382 0.11239,0.73838 0.0604,0.91323 z m -1.28154,0.29689 c -0.11256,-0.40526 -0.62849,-0.48357 -0.86193,-0.13082 -0.0745,0.1125 -0.088,0.16447 -0.0745,0.2847 0.0338,0.3 0.31326,0.5045 0.58822,0.43046 0.15271,-0.0411 0.30561,-0.1849 0.34829,-0.32754 0.0392,-0.13086 0.0392,-0.11544 -1e-4,-0.2568 z"
|
||||
id="path1971-3"
|
||||
sodipodi:nodetypes="ssscsscccccccccccssscsssscc" />
|
||||
<text
|
||||
xml:space="preserve"
|
||||
style="font-style:normal;font-weight:normal;font-size:5.64444px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.264583"
|
||||
x="177.8474"
|
||||
y="104.05132"
|
||||
id="text852-9-6"><tspan
|
||||
sodipodi:role="line"
|
||||
id="tspan850-4-7"
|
||||
x="177.8474"
|
||||
y="104.05132"
|
||||
style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:5.64444px;font-family:'Liberation Mono';-inkscape-font-specification:'Liberation Mono Bold';stroke-width:0.264583">= partition key </tspan></text>
|
||||
<path
|
||||
style="fill:#4040ff;fill-opacity:1;stroke:none;stroke-width:0.0337704;stroke-opacity:1"
|
||||
d="m 174.20027,113.78076 c -0.14225,0.47871 -0.31308,0.74552 -0.65239,1.01895 -0.29018,0.23385 -0.5957,0.33949 -1.03301,0.3572 -0.33087,0.0134 -0.37381,0.007 -0.64184,-0.0901 -0.50954,-0.18534 -0.91422,-0.59414 -1.08287,-1.0939 -0.16385,-0.48553 -0.0785,-1.10816 0.21033,-1.53381 0.11145,-0.16425 0.40217,-0.44459 0.53451,-0.51541 l 0.0681,-3.63921 0.73832,-0.64992 0.88727,0.78714 -0.40986,0.44548 0.4054,0.44098 -0.4054,0.43502 0.3816,0.41947 -0.41461,0.43178 0.57334,0.58508 0.007,0.72638 c 0.13641,0.0828 0.48753,0.4136 0.59931,0.58235 0.0577,0.0871 0.13636,0.25779 0.17481,0.37931 0.0803,0.25382 0.11239,0.73837 0.0604,0.91323 z m -1.28154,0.29689 c -0.11256,-0.40527 -0.62849,-0.48357 -0.86193,-0.13082 -0.0745,0.1125 -0.088,0.16446 -0.0745,0.2847 0.0338,0.3 0.31326,0.5045 0.58822,0.43046 0.15271,-0.0411 0.30561,-0.18491 0.34829,-0.32754 0.0392,-0.13087 0.0392,-0.11545 -1e-4,-0.2568 z"
|
||||
id="path1971-4-5"
|
||||
sodipodi:nodetypes="ssscsscccccccccccssscsssscc" />
|
||||
<text
|
||||
xml:space="preserve"
|
||||
style="font-style:normal;font-weight:normal;font-size:5.64444px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.264583"
|
||||
x="177.8474"
|
||||
y="113.37622"
|
||||
id="text852-9-5-3"><tspan
|
||||
sodipodi:role="line"
|
||||
id="tspan850-4-0-5"
|
||||
x="177.8474"
|
||||
y="113.37622"
|
||||
style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:5.64444px;font-family:'Liberation Mono';-inkscape-font-specification:'Liberation Mono Bold';stroke-width:0.264583">= sort key </tspan></text>
|
||||
</g>
|
||||
<g
|
||||
id="g2161"
|
||||
transform="translate(-62.264403,-59.333115)">
|
||||
<g
|
||||
id="g2271"
|
||||
transform="translate(0,67.042823)">
|
||||
<rect
|
||||
style="fill:none;stroke:#000000;stroke-width:0.799999;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1"
|
||||
id="rect833-6"
|
||||
width="39.008453"
|
||||
height="16.775949"
|
||||
x="84.896881"
|
||||
y="90.266838" />
|
||||
<rect
|
||||
style="fill:none;stroke:#000000;stroke-width:0.799999;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1"
|
||||
id="rect833-3-1"
|
||||
width="39.008453"
|
||||
height="8.673645"
|
||||
x="84.896881"
|
||||
y="98.369141" />
|
||||
<text
|
||||
xml:space="preserve"
|
||||
style="font-style:normal;font-weight:normal;font-size:5.64444px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.264583"
|
||||
x="89.826942"
|
||||
y="96.212921"
|
||||
id="text852-0"><tspan
|
||||
sodipodi:role="line"
|
||||
id="tspan850-6"
|
||||
x="89.826942"
|
||||
y="96.212921"
|
||||
style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:5.64444px;font-family:'Liberation Mono';-inkscape-font-specification:'Liberation Mono Bold';stroke-width:0.264583">Version 1</tspan></text>
|
||||
<text
|
||||
xml:space="preserve"
|
||||
style="font-style:normal;font-weight:normal;font-size:5.64444px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.264583"
|
||||
x="89.826942"
|
||||
y="104.71013"
|
||||
id="text852-0-3"><tspan
|
||||
sodipodi:role="line"
|
||||
id="tspan850-6-2"
|
||||
x="89.826942"
|
||||
y="104.71013"
|
||||
style="font-style:italic;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:5.64444px;font-family:'Liberation Mono';-inkscape-font-specification:'Liberation Mono Bold';fill:#4d4d4d;stroke-width:0.264583">deleted</tspan></text>
|
||||
</g>
|
||||
</g>
|
||||
<g
|
||||
id="g2263"
|
||||
transform="translate(0,-22.791204)">
|
||||
<g
|
||||
id="g2161-1"
|
||||
transform="translate(-62.264403,-10.910843)">
|
||||
<rect
|
||||
style="fill:none;stroke:#000000;stroke-width:0.799999;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1"
|
||||
id="rect833-6-5"
|
||||
width="39.008453"
|
||||
height="36.749603"
|
||||
x="84.896881"
|
||||
y="90.266838" />
|
||||
<rect
|
||||
style="fill:none;stroke:#000000;stroke-width:0.799999;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1"
|
||||
id="rect833-3-1-5"
|
||||
width="39.008453"
|
||||
height="28.647301"
|
||||
x="84.896881"
|
||||
y="98.369141" />
|
||||
<text
|
||||
xml:space="preserve"
|
||||
style="font-style:normal;font-weight:normal;font-size:5.64444px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.264583"
|
||||
x="89.826942"
|
||||
y="96.212921"
|
||||
id="text852-0-4"><tspan
|
||||
sodipodi:role="line"
|
||||
id="tspan850-6-7"
|
||||
x="89.826942"
|
||||
y="96.212921"
|
||||
style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:5.64444px;font-family:'Liberation Mono';-inkscape-font-specification:'Liberation Mono Bold';stroke-width:0.264583">Version 2</tspan></text>
|
||||
<text
|
||||
xml:space="preserve"
|
||||
style="font-style:normal;font-weight:normal;font-size:5.64444px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.264583"
|
||||
x="89.826942"
|
||||
y="104.71013"
|
||||
id="text852-0-3-6"><tspan
|
||||
sodipodi:role="line"
|
||||
id="tspan850-6-2-5"
|
||||
x="89.826942"
|
||||
y="104.71013"
|
||||
style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:5.64444px;font-family:'Liberation Mono';-inkscape-font-specification:'Liberation Mono Bold';fill:#000000;stroke-width:0.264583">id</tspan></text>
|
||||
</g>
|
||||
<text
|
||||
xml:space="preserve"
|
||||
style="font-style:normal;font-weight:normal;font-size:5.64444px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.264583"
|
||||
x="27.56254"
|
||||
y="100.34132"
|
||||
id="text852-0-3-6-6"><tspan
|
||||
sodipodi:role="line"
|
||||
id="tspan850-6-2-5-9"
|
||||
x="27.56254"
|
||||
y="100.34132"
|
||||
style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:5.64444px;font-family:'Liberation Mono';-inkscape-font-specification:'Liberation Mono Bold';fill:#000000;stroke-width:0.264583">size</tspan></text>
|
||||
<text
|
||||
xml:space="preserve"
|
||||
style="font-style:normal;font-weight:normal;font-size:5.64444px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.264583"
|
||||
x="27.56254"
|
||||
y="106.90263"
|
||||
id="text852-0-3-6-6-3"><tspan
|
||||
sodipodi:role="line"
|
||||
id="tspan850-6-2-5-9-7"
|
||||
x="27.56254"
|
||||
y="106.90263"
|
||||
style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:5.64444px;font-family:'Liberation Mono';-inkscape-font-specification:'Liberation Mono Bold';fill:#000000;stroke-width:0.264583">MIME type</tspan></text>
|
||||
<text
|
||||
xml:space="preserve"
|
||||
style="font-style:normal;font-weight:normal;font-size:5.64444px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.264583"
|
||||
x="27.56254"
|
||||
y="111.92816"
|
||||
id="text852-0-3-6-6-3-4"><tspan
|
||||
sodipodi:role="line"
|
||||
id="tspan850-6-2-5-9-7-5"
|
||||
x="27.56254"
|
||||
y="111.92816"
|
||||
style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:5.64444px;font-family:'Liberation Mono';-inkscape-font-specification:'Liberation Mono Bold';fill:#000000;stroke-width:0.264583">...</tspan></text>
|
||||
</g>
|
||||
</g>
|
||||
<g
|
||||
id="g898"
|
||||
transform="translate(-6.2484318,29.95006)">
|
||||
<rect
|
||||
style="fill:none;stroke:#000000;stroke-width:0.799999;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1"
|
||||
id="rect833-7"
|
||||
width="47.419891"
|
||||
height="44.007515"
|
||||
x="95.443573"
|
||||
y="24.42766" />
|
||||
<rect
|
||||
style="fill:none;stroke:#000000;stroke-width:0.799999;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1"
|
||||
id="rect833-3-4"
|
||||
width="47.419891"
|
||||
height="35.627186"
|
||||
x="95.443573"
|
||||
y="32.807987" />
|
||||
<text
|
||||
xml:space="preserve"
|
||||
style="font-style:normal;font-weight:normal;font-size:5.64444px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.264583"
|
||||
x="107.46638"
|
||||
y="29.894743"
|
||||
id="text852-4"><tspan
|
||||
sodipodi:role="line"
|
||||
id="tspan850-3"
|
||||
x="107.46638"
|
||||
y="29.894743"
|
||||
style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:5.64444px;font-family:'Liberation Mono';-inkscape-font-specification:'Liberation Mono Bold';stroke-width:0.264583">Version</tspan></text>
|
||||
<path
|
||||
style="fill:#ff6600;fill-opacity:1;stroke:none;stroke-width:0.0337704;stroke-opacity:1"
|
||||
d="m 102.90563,41.413279 c -0.14226,0.478709 -0.31308,0.745518 -0.65239,1.018956 -0.29019,0.233843 -0.59571,0.339489 -1.03301,0.357199 -0.33087,0.0134 -0.37381,0.0074 -0.64184,-0.09013 -0.50954,-0.185343 -0.914221,-0.594142 -1.082877,-1.093901 -0.163852,-0.485526 -0.07847,-1.108159 0.210335,-1.533803 0.111448,-0.164254 0.402172,-0.444591 0.534502,-0.515415 l 0.0681,-3.63921 0.73832,-0.64992 0.88727,0.787138 -0.40985,0.445484 0.40539,0.440982 -0.40539,0.435013 0.3816,0.41947 -0.41462,0.431781 0.57335,0.585078 0.007,0.726386 c 0.13641,0.08277 0.48753,0.413601 0.59931,0.58235 0.0577,0.0871 0.13636,0.257787 0.17481,0.379304 0.0803,0.253823 0.11239,0.738377 0.0604,0.913234 z m -1.28155,0.296888 c -0.11255,-0.405265 -0.62848,-0.483569 -0.86192,-0.130817 -0.0744,0.112498 -0.088,0.164461 -0.0745,0.2847 0.0338,0.299998 0.31326,0.504498 0.58822,0.43046 0.15271,-0.04112 0.3056,-0.184909 0.34828,-0.327542 0.0392,-0.130864 0.0392,-0.115447 -1e-4,-0.256801 z"
|
||||
id="path1971-0"
|
||||
sodipodi:nodetypes="ssscsscccccccccccssscsssscc" />
|
||||
<text
|
||||
xml:space="preserve"
|
||||
style="font-style:normal;font-weight:normal;font-size:5.64444px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.264583"
|
||||
x="104.99195"
|
||||
y="41.008743"
|
||||
id="text852-9-7"><tspan
|
||||
sodipodi:role="line"
|
||||
id="tspan850-4-8"
|
||||
x="104.99195"
|
||||
y="41.008743"
|
||||
style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:5.64444px;font-family:'Liberation Mono';-inkscape-font-specification:'Liberation Mono Bold';stroke-width:0.264583">id </tspan></text>
|
||||
<text
|
||||
xml:space="preserve"
|
||||
style="font-style:normal;font-weight:normal;font-size:5.64444px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.264583"
|
||||
x="104.99195"
|
||||
y="49.168018"
|
||||
id="text852-9-7-6"><tspan
|
||||
sodipodi:role="line"
|
||||
id="tspan850-4-8-8"
|
||||
x="104.99195"
|
||||
y="49.168018"
|
||||
style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:5.64444px;font-family:'Liberation Mono';-inkscape-font-specification:'Liberation Mono Bold';stroke-width:0.264583">h(block 1)</tspan></text>
|
||||
<text
|
||||
xml:space="preserve"
|
||||
style="font-style:normal;font-weight:normal;font-size:5.64444px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.264583"
|
||||
x="104.99195"
|
||||
y="56.583336"
|
||||
id="text852-9-7-6-8"><tspan
|
||||
sodipodi:role="line"
|
||||
id="tspan850-4-8-8-4"
|
||||
x="104.99195"
|
||||
y="56.583336"
|
||||
style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:5.64444px;font-family:'Liberation Mono';-inkscape-font-specification:'Liberation Mono Bold';stroke-width:0.264583">h(block 2)</tspan></text>
|
||||
<text
|
||||
xml:space="preserve"
|
||||
style="font-style:normal;font-weight:normal;font-size:5.64444px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.264583"
|
||||
x="104.99195"
|
||||
y="64.265732"
|
||||
id="text852-9-7-6-3"><tspan
|
||||
sodipodi:role="line"
|
||||
id="tspan850-4-8-8-1"
|
||||
x="104.99195"
|
||||
y="64.265732"
|
||||
style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:5.64444px;font-family:'Liberation Mono';-inkscape-font-specification:'Liberation Mono Bold';stroke-width:0.264583">...</tspan></text>
|
||||
</g>
|
||||
<g
|
||||
id="g898-3"
|
||||
transform="translate(75.777779,38.888663)">
|
||||
<rect
|
||||
style="fill:none;stroke:#000000;stroke-width:0.799999;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1"
|
||||
id="rect833-7-6"
|
||||
width="47.419891"
|
||||
height="29.989157"
|
||||
x="95.443573"
|
||||
y="24.42766" />
|
||||
<rect
|
||||
style="fill:none;stroke:#000000;stroke-width:0.799999;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1"
|
||||
id="rect833-3-4-7"
|
||||
width="47.419891"
|
||||
height="21.608831"
|
||||
x="95.443573"
|
||||
y="32.807987" />
|
||||
<text
|
||||
xml:space="preserve"
|
||||
style="font-style:normal;font-weight:normal;font-size:5.64444px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.264583"
|
||||
x="102.11134"
|
||||
y="29.894743"
|
||||
id="text852-4-5"><tspan
|
||||
sodipodi:role="line"
|
||||
id="tspan850-3-3"
|
||||
x="102.11134"
|
||||
y="29.894743"
|
||||
style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:5.64444px;font-family:'Liberation Mono';-inkscape-font-specification:'Liberation Mono Bold';stroke-width:0.264583">Data block</tspan></text>
|
||||
<path
|
||||
style="fill:#ff6600;fill-opacity:1;stroke:none;stroke-width:0.0337704;stroke-opacity:1"
|
||||
d="m 102.90563,41.413279 c -0.14226,0.478709 -0.31308,0.745518 -0.65239,1.018956 -0.29019,0.233843 -0.59571,0.339489 -1.03301,0.357199 -0.33087,0.0134 -0.37381,0.0074 -0.64184,-0.09013 -0.50954,-0.185343 -0.914221,-0.594142 -1.082877,-1.093901 -0.163852,-0.485526 -0.07847,-1.108159 0.210335,-1.533803 0.111448,-0.164254 0.402172,-0.444591 0.534502,-0.515415 l 0.0681,-3.63921 0.73832,-0.64992 0.88727,0.787138 -0.40985,0.445484 0.40539,0.440982 -0.40539,0.435013 0.3816,0.41947 -0.41462,0.431781 0.57335,0.585078 0.007,0.726386 c 0.13641,0.08277 0.48753,0.413601 0.59931,0.58235 0.0577,0.0871 0.13636,0.257787 0.17481,0.379304 0.0803,0.253823 0.11239,0.738377 0.0604,0.913234 z m -1.28155,0.296888 c -0.11255,-0.405265 -0.62848,-0.483569 -0.86192,-0.130817 -0.0744,0.112498 -0.088,0.164461 -0.0745,0.2847 0.0338,0.299998 0.31326,0.504498 0.58822,0.43046 0.15271,-0.04112 0.3056,-0.184909 0.34828,-0.327542 0.0392,-0.130864 0.0392,-0.115447 -1e-4,-0.256801 z"
|
||||
id="path1971-0-5"
|
||||
sodipodi:nodetypes="ssscsscccccccccccssscsssscc" />
|
||||
<text
|
||||
xml:space="preserve"
|
||||
style="font-style:normal;font-weight:normal;font-size:5.64444px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.264583"
|
||||
x="104.99195"
|
||||
y="41.008743"
|
||||
id="text852-9-7-62"><tspan
|
||||
sodipodi:role="line"
|
||||
id="tspan850-4-8-9"
|
||||
x="104.99195"
|
||||
y="41.008743"
|
||||
style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:5.64444px;font-family:'Liberation Mono';-inkscape-font-specification:'Liberation Mono Bold';stroke-width:0.264583">hash </tspan></text>
|
||||
<text
|
||||
xml:space="preserve"
|
||||
style="font-style:normal;font-weight:normal;font-size:5.64444px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.264583"
|
||||
x="104.99195"
|
||||
y="49.168018"
|
||||
id="text852-9-7-6-1"><tspan
|
||||
sodipodi:role="line"
|
||||
id="tspan850-4-8-8-2"
|
||||
x="104.99195"
|
||||
y="49.168018"
|
||||
style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:5.64444px;font-family:'Liberation Mono';-inkscape-font-specification:'Liberation Mono Bold';stroke-width:0.264583">data</tspan></text>
|
||||
</g>
|
||||
<path
|
||||
style="fill:none;stroke:#000000;stroke-width:1;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1;marker-end:url(#Arrow1Mend)"
|
||||
d="M 42.105292,69.455903 89.563703,69.317144"
|
||||
id="path954"
|
||||
sodipodi:nodetypes="cc" />
|
||||
<path
|
||||
style="fill:none;stroke:#000000;stroke-width:1;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1;marker-end:url(#marker1262)"
|
||||
d="m 134.32612,77.363197 38.12618,0.260865"
|
||||
id="path1258"
|
||||
sodipodi:nodetypes="cc" />
|
||||
<text
|
||||
xml:space="preserve"
|
||||
style="font-style:normal;font-weight:normal;font-size:5.64444px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.264583"
|
||||
x="8.6727352"
|
||||
y="16.687063"
|
||||
id="text852-3"><tspan
|
||||
sodipodi:role="line"
|
||||
id="tspan850-67"
|
||||
x="8.6727352"
|
||||
y="16.687063"
|
||||
style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:5.64444px;font-family:'Liberation Mono';-inkscape-font-specification:'Liberation Mono Bold';stroke-width:0.264583">Objects table </tspan></text>
|
||||
<text
|
||||
xml:space="preserve"
|
||||
style="font-style:normal;font-weight:normal;font-size:5.64444px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.264583"
|
||||
x="89.190445"
|
||||
y="16.687063"
|
||||
id="text852-3-5"><tspan
|
||||
sodipodi:role="line"
|
||||
id="tspan850-67-3"
|
||||
x="89.190445"
|
||||
y="16.687063"
|
||||
style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:5.64444px;font-family:'Liberation Mono';-inkscape-font-specification:'Liberation Mono Bold';stroke-width:0.264583">Versions table </tspan></text>
|
||||
<text
|
||||
xml:space="preserve"
|
||||
style="font-style:normal;font-weight:normal;font-size:5.64444px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.264583"
|
||||
x="174.55702"
|
||||
y="16.687063"
|
||||
id="text852-3-56"><tspan
|
||||
sodipodi:role="line"
|
||||
id="tspan850-67-2"
|
||||
x="174.55702"
|
||||
y="16.687063"
|
||||
style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:5.64444px;font-family:'Liberation Mono';-inkscape-font-specification:'Liberation Mono Bold';stroke-width:0.264583">Blocks table</tspan></text>
|
||||
</g>
|
||||
</svg>
|
||||
|
Before Width: | Height: | Size: 30 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 52 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 37 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 97 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 199 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 145 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 13 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 174 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 38 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 14 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 87 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 81 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 124 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 84 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 81 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 81 KiB |
File diff suppressed because it is too large
Load Diff
|
Before Width: | Height: | Size: 315 KiB |
File diff suppressed because one or more lines are too long
|
Before Width: | Height: | Size: 286 KiB |
@@ -1,3 +0,0 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:f245bb017eb85a227d96e576ae821f165ef478d46de02535c196638aa5fc84b0
|
||||
size 2575953
|
||||
@@ -1,480 +0,0 @@
|
||||
%\nonstopmode
|
||||
\documentclass[aspectratio=169]{beamer}
|
||||
\usepackage[utf8]{inputenc}
|
||||
% \usepackage[frenchb]{babel}
|
||||
\usepackage{amsmath}
|
||||
\usepackage{mathtools}
|
||||
\usepackage{breqn}
|
||||
\usepackage{multirow}
|
||||
\usetheme{boxes}
|
||||
\usepackage{graphicx}
|
||||
\usepackage{adjustbox}
|
||||
%\useoutertheme[footline=authortitle,subsection=false]{miniframes}
|
||||
%\useoutertheme[footline=authorinstitute,subsection=false]{miniframes}
|
||||
\useoutertheme{infolines}
|
||||
\setbeamertemplate{headline}{}
|
||||
|
||||
\beamertemplatenavigationsymbolsempty
|
||||
|
||||
\definecolor{TitleOrange}{RGB}{255,137,0}
|
||||
\setbeamercolor{title}{fg=TitleOrange}
|
||||
\setbeamercolor{frametitle}{fg=TitleOrange}
|
||||
|
||||
\definecolor{ListOrange}{RGB}{255,145,5}
|
||||
\setbeamertemplate{itemize item}{\color{ListOrange}$\blacktriangleright$}
|
||||
|
||||
\definecolor{verygrey}{RGB}{70,70,70}
|
||||
\setbeamercolor{normal text}{fg=verygrey}
|
||||
|
||||
|
||||
\usepackage{tabu}
|
||||
\usepackage{multicol}
|
||||
\usepackage{vwcol}
|
||||
\usepackage{stmaryrd}
|
||||
\usepackage{graphicx}
|
||||
|
||||
\usepackage[normalem]{ulem}
|
||||
|
||||
\title{Introducing Garage}
|
||||
\subtitle{a new storage platform for self-hosted geo-distributed clusters}
|
||||
\author{Deuxfleurs Association}
|
||||
\date{IMT Atlantique, 2022-06-23}
|
||||
|
||||
\begin{document}
|
||||
|
||||
\begin{frame}
|
||||
\centering
|
||||
\includegraphics[width=.3\linewidth]{../../sticker/Garage.pdf}
|
||||
\vspace{1em}
|
||||
|
||||
{\large\bf Deuxfleurs Association}
|
||||
\vspace{1em}
|
||||
|
||||
\url{https://garagehq.deuxfleurs.fr/}
|
||||
|
||||
Matrix channel: \texttt{\#garage:deuxfleurs.fr}
|
||||
\end{frame}
|
||||
|
||||
\begin{frame}
|
||||
\frametitle{Who we are}
|
||||
\begin{columns}[t]
|
||||
\begin{column}{.2\textwidth}
|
||||
\centering
|
||||
\adjincludegraphics[width=.4\linewidth, valign=t]{assets/alex.jpg}
|
||||
\end{column}
|
||||
\begin{column}{.6\textwidth}
|
||||
\textbf{Alex Auvolat}\\
|
||||
PhD at Inria, team WIDE; co-founder of Deuxfleurs
|
||||
\end{column}
|
||||
\begin{column}{.2\textwidth}
|
||||
~
|
||||
\end{column}
|
||||
\end{columns}
|
||||
\vspace{1em}
|
||||
|
||||
\begin{columns}[t]
|
||||
\begin{column}{.2\textwidth}
|
||||
~
|
||||
\end{column}
|
||||
\begin{column}{.6\textwidth}
|
||||
\textbf{Quentin Dufour}\\
|
||||
PhD at Inria, team WIDE; co-founder of Deuxfleurs
|
||||
\end{column}
|
||||
\begin{column}{.2\textwidth}
|
||||
\centering
|
||||
\adjincludegraphics[width=.5\linewidth, valign=t]{assets/quentin.jpg}
|
||||
\end{column}
|
||||
\end{columns}
|
||||
\vspace{2em}
|
||||
|
||||
\begin{columns}[t]
|
||||
\begin{column}{.2\textwidth}
|
||||
\centering
|
||||
\adjincludegraphics[width=.5\linewidth, valign=t]{assets/deuxfleurs.pdf}
|
||||
\end{column}
|
||||
\begin{column}{.6\textwidth}
|
||||
\textbf{Deuxfleurs}\\
|
||||
A non-profit self-hosting collective,\\
|
||||
member of the CHATONS network
|
||||
\end{column}
|
||||
\begin{column}{.2\textwidth}
|
||||
\centering
|
||||
\adjincludegraphics[width=.7\linewidth, valign=t]{assets/logo_chatons.png}
|
||||
\end{column}
|
||||
\end{columns}
|
||||
|
||||
\end{frame}
|
||||
|
||||
\begin{frame}
|
||||
\frametitle{Our objective at Deuxfleurs}
|
||||
|
||||
\begin{center}
|
||||
\textbf{Promote self-hosting and small-scale hosting\\
|
||||
as an alternative to large cloud providers}
|
||||
\end{center}
|
||||
\vspace{2em}
|
||||
\visible<2->{
|
||||
Why is it hard?
|
||||
}
|
||||
\visible<3->{
|
||||
\vspace{2em}
|
||||
\begin{center}
|
||||
\textbf{\underline{Resilience}}\\
|
||||
{\footnotesize (we want good uptime/availability with low supervision)}
|
||||
\end{center}
|
||||
}
|
||||
\end{frame}
|
||||
|
||||
\begin{frame}
|
||||
\frametitle{How to make a \underline{stable} system}
|
||||
|
||||
Enterprise-grade systems typically employ:
|
||||
\vspace{1em}
|
||||
\begin{itemize}
|
||||
\item RAID
|
||||
\item Redundant power grid + UPS
|
||||
\item Redundant Internet connections
|
||||
\item Low-latency links
|
||||
\item ...
|
||||
\end{itemize}
|
||||
\vspace{1em}
|
||||
$\to$ it's costly and only worth it at DC scale
|
||||
\end{frame}
|
||||
|
||||
\begin{frame}
|
||||
\frametitle{How to make a \underline{resilient} system}
|
||||
|
||||
\only<1,4-5>{
|
||||
Instead, we use:
|
||||
\vspace{1em}
|
||||
\begin{itemize}
|
||||
\item \textcolor<2->{gray}{Commodity hardware (e.g. old desktop PCs)}
|
||||
\vspace{.5em}
|
||||
\item<4-> \textcolor<5->{gray}{Commodity Internet (e.g. FTTB, FTTH) and power grid}
|
||||
\vspace{.5em}
|
||||
\item<5-> \textcolor<6->{gray}{\textbf{Geographical redundancy} (multi-site replication)}
|
||||
\end{itemize}
|
||||
}
|
||||
\only<2>{
|
||||
\begin{center}
|
||||
\includegraphics[width=.8\linewidth]{assets/atuin.jpg}
|
||||
\end{center}
|
||||
}
|
||||
\only<3>{
|
||||
\begin{center}
|
||||
\includegraphics[width=.8\linewidth]{assets/neptune.jpg}
|
||||
\end{center}
|
||||
}
|
||||
\only<6>{
|
||||
\begin{center}
|
||||
\includegraphics[width=.5\linewidth]{assets/inframap.jpg}
|
||||
\end{center}
|
||||
}
|
||||
\end{frame}
|
||||
|
||||
\begin{frame}
|
||||
\frametitle{How to make this happen}
|
||||
\begin{center}
|
||||
\only<1>{\includegraphics[width=.8\linewidth]{assets/slide1.png}}%
|
||||
\only<2>{\includegraphics[width=.8\linewidth]{assets/slide2.png}}%
|
||||
\only<3>{\includegraphics[width=.8\linewidth]{assets/slide3.png}}%
|
||||
\end{center}
|
||||
\end{frame}
|
||||
|
||||
\begin{frame}
|
||||
\frametitle{Distributed file systems are slow}
|
||||
File systems are complex, for example:
|
||||
\vspace{1em}
|
||||
\begin{itemize}
|
||||
\item Concurrent modification by several processes
|
||||
\vspace{1em}
|
||||
\item Folder hierarchies
|
||||
\vspace{1em}
|
||||
\item Other requirements of the POSIX spec
|
||||
\end{itemize}
|
||||
\vspace{1em}
|
||||
Coordination in a distributed system is costly
|
||||
|
||||
\vspace{1em}
|
||||
Costs explode with commodity hardware / Internet connections\\
|
||||
{\small (we experienced this!)}
|
||||
\end{frame}
|
||||
|
||||
\begin{frame}
|
||||
\frametitle{A simpler solution: object storage}
|
||||
Only two operations:
|
||||
\vspace{1em}
|
||||
\begin{itemize}
|
||||
\item Put an object at a key
|
||||
\vspace{1em}
|
||||
\item Retrieve an object from its key
|
||||
\end{itemize}
|
||||
\vspace{1em}
|
||||
{\footnotesize (and a few others)}
|
||||
|
||||
\vspace{1em}
|
||||
Sufficient for many applications!
|
||||
\end{frame}
|
||||
|
||||
\begin{frame}
|
||||
\frametitle{A simpler solution: object storage}
|
||||
\begin{center}
|
||||
\includegraphics[width=.2\linewidth]{../2020-12-02_wide-team/img/Amazon-S3.jpg}
|
||||
\hspace{5em}
|
||||
\includegraphics[width=.2\linewidth]{assets/minio.png}
|
||||
\end{center}
|
||||
\vspace{1em}
|
||||
S3: a de-facto standard, many compatible applications
|
||||
|
||||
\vspace{1em}
|
||||
|
||||
MinIO is self-hostable but not suited for geo-distributed deployments
|
||||
\end{frame}
|
||||
|
||||
|
||||
\begin{frame}
|
||||
\frametitle{But what is Garage, exactly?}
|
||||
\textbf{Garage is a self-hosted drop-in replacement for the Amazon S3 object store}\\
|
||||
\vspace{.5em}
|
||||
that implements resilience through geographical redundancy on commodity hardware
|
||||
\begin{center}
|
||||
\includegraphics[width=.8\linewidth]{assets/garageuses.png}
|
||||
\end{center}
|
||||
\end{frame}
|
||||
|
||||
\begin{frame}
|
||||
\frametitle{Overview}
|
||||
\begin{center}
|
||||
\only<1>{\includegraphics[width=.45\linewidth]{assets/garage2a.drawio.pdf}}%
|
||||
\only<2>{\includegraphics[width=.45\linewidth]{assets/garage2b.drawio.pdf}}%
|
||||
\end{center}
|
||||
\end{frame}
|
||||
|
||||
\begin{frame}
|
||||
\frametitle{Garage is \emph{location-aware}}
|
||||
\begin{center}
|
||||
\includegraphics[width=\linewidth]{assets/location-aware.png}
|
||||
\end{center}
|
||||
\vspace{2em}
|
||||
Garage replicates data on different zones when possible
|
||||
\end{frame}
|
||||
|
||||
\begin{frame}
|
||||
\frametitle{Garage is \emph{location-aware}}
|
||||
\begin{center}
|
||||
\includegraphics[width=.8\linewidth]{assets/map.png}
|
||||
\end{center}
|
||||
\end{frame}
|
||||
|
||||
\begin{frame}
|
||||
\frametitle{How to spread files over different cluster nodes?}
|
||||
\textbf{Consistent hashing (DynamoDB):}
|
||||
\vspace{1em}
|
||||
|
||||
\begin{center}
|
||||
\only<1>{\includegraphics[width=.45\columnwidth]{assets/consistent_hashing_1.pdf}}%
|
||||
\only<2>{\includegraphics[width=.45\columnwidth]{assets/consistent_hashing_2.pdf}}%
|
||||
\only<3>{\includegraphics[width=.45\columnwidth]{assets/consistent_hashing_3.pdf}}%
|
||||
\only<4>{\includegraphics[width=.45\columnwidth]{assets/consistent_hashing_4.pdf}}%
|
||||
\end{center}
|
||||
\end{frame}
|
||||
|
||||
\begin{frame}
|
||||
\frametitle{How to spread files over different cluster nodes?}
|
||||
\textbf{Issues with consistent hashing:}
|
||||
\vspace{1em}
|
||||
\begin{itemize}
|
||||
\item Doesn't dispatch data based on geographical location of nodes
|
||||
\vspace{1em}
|
||||
\item<2-> Geographically aware adaptation, try 1:\\
|
||||
data quantities not well balanced between nodes
|
||||
\vspace{1em}
|
||||
\item<3-> Geographically aware adaptation, try 2:\\
|
||||
too many reshuffles when adding/removing nodes
|
||||
\end{itemize}
|
||||
\end{frame}
|
||||
|
||||
\begin{frame}
|
||||
\frametitle{How to spread files over different cluster nodes?}
|
||||
\textbf{Garage's method: build an index table}
|
||||
\vspace{1em}
|
||||
|
||||
Realization: we can actually precompute an optimal solution
|
||||
\vspace{1em}
|
||||
|
||||
\visible<2->{
|
||||
\begin{center}
|
||||
\begin{tabular}{|l|l|l|l|}
|
||||
\hline
|
||||
\textbf{Partition} & \textbf{Node 1} & \textbf{Node 2} & \textbf{Node 3} \\
|
||||
\hline
|
||||
\hline
|
||||
Partition 0 & Io (jupiter) & Drosera (atuin) & Courgette (neptune) \\
|
||||
\hline
|
||||
Partition 1 & Datura (atuin) & Courgette (neptune) & Io (jupiter) \\
|
||||
\hline
|
||||
Partition 2 & Io(jupiter) & Celeri (neptune) & Drosera (atuin) \\
|
||||
\hline
|
||||
\hspace{1em}$\vdots$ & \hspace{1em}$\vdots$ & \hspace{1em}$\vdots$ & \hspace{1em}$\vdots$ \\
|
||||
\hline
|
||||
Partition 255 & Concombre (neptune) & Io (jupiter) & Drosera (atuin) \\
|
||||
\hline
|
||||
\end{tabular}
|
||||
\end{center}
|
||||
}
|
||||
\vspace{1em}
|
||||
\visible<3->{
|
||||
The index table is built centrally using an optimal* algorithm,\\
|
||||
then propagated to all nodes\\
|
||||
\hfill\footnotesize *not yet optimal but will be soon
|
||||
}
|
||||
\end{frame}
|
||||
|
||||
\begin{frame}
|
||||
\frametitle{Garage's internal data structures}
|
||||
\centering
|
||||
\includegraphics[width=.75\columnwidth]{assets/garage_tables.pdf}
|
||||
\end{frame}
|
||||
|
||||
%\begin{frame}
|
||||
% \frametitle{Garage's architecture}
|
||||
% \begin{center}
|
||||
% \includegraphics[width=.35\linewidth]{assets/garage.drawio.pdf}
|
||||
% \end{center}
|
||||
%\end{frame}
|
||||
|
||||
\begin{frame}
|
||||
\frametitle{Garage is \emph{coordination-free}:}
|
||||
\begin{itemize}
|
||||
\item No Raft or Paxos
|
||||
\vspace{1em}
|
||||
\item Internal data types are CRDTs
|
||||
\vspace{1em}
|
||||
\item All nodes are equivalent (no master/leader/index node)
|
||||
\end{itemize}
|
||||
\vspace{2em}
|
||||
$\to$ less sensitive to higher latencies between nodes
|
||||
\end{frame}
|
||||
|
||||
\begin{frame}
|
||||
\frametitle{Consistency model}
|
||||
\begin{itemize}
|
||||
\item Not ACID (not required by S3 spec) / not linearizable
|
||||
\vspace{1em}
|
||||
\item \textbf{Read-after-write consistency}\\
|
||||
{\footnotesize (stronger than eventual consistency)}
|
||||
\end{itemize}
|
||||
\end{frame}
|
||||
|
||||
\begin{frame}
|
||||
\frametitle{Impact on performances}
|
||||
\begin{center}
|
||||
\includegraphics[width=.8\linewidth]{assets/endpoint-latency-dc.png}
|
||||
\end{center}
|
||||
\end{frame}
|
||||
|
||||
|
||||
\begin{frame}
|
||||
\frametitle{An ever-increasing compatibility list}
|
||||
\begin{center}
|
||||
\includegraphics[width=.7\linewidth]{assets/compatibility.png}
|
||||
\end{center}
|
||||
\end{frame}
|
||||
|
||||
\begin{frame}
|
||||
\frametitle{Further plans for Garage}
|
||||
\begin{center}
|
||||
\only<1>{\includegraphics[width=.8\linewidth]{assets/slideB1.png}}%
|
||||
\only<2>{\includegraphics[width=.8\linewidth]{assets/slideB2.png}}%
|
||||
\only<3>{\includegraphics[width=.8\linewidth]{assets/slideB3.png}}%
|
||||
\end{center}
|
||||
\end{frame}
|
||||
|
||||
\begin{frame}
|
||||
\frametitle{K2V Design}
|
||||
\begin{itemize}
|
||||
\item A new, custom, minimal API
|
||||
\vspace{1em}
|
||||
\item<2-> Exposes the partitoning mechanism of Garage\\
|
||||
K2V = partition key / sort key / value (like Dynamo)
|
||||
\vspace{1em}
|
||||
\item<3-> Coordination-free, CRDT-friendly (inspired by Riak)\\
|
||||
\vspace{1em}
|
||||
\item<4-> Cryptography-friendly: values are binary blobs
|
||||
\end{itemize}
|
||||
\end{frame}
|
||||
|
||||
\begin{frame}
|
||||
\frametitle{Application: an e-mail storage server}
|
||||
\begin{center}
|
||||
\only<1>{\includegraphics[width=.9\linewidth]{assets/aerogramme.png}}%
|
||||
\end{center}
|
||||
\end{frame}
|
||||
|
||||
\begin{frame}
|
||||
\frametitle{Aerogramme data model}
|
||||
\begin{center}
|
||||
\only<1>{\includegraphics[width=.4\linewidth]{assets/aerogramme_datatype.drawio.pdf}}%
|
||||
\only<2->{\includegraphics[width=.9\linewidth]{assets/aerogramme_keys.drawio.pdf}\vspace{1em}}%
|
||||
\end{center}
|
||||
\visible<3->{Aerogramme encrypts all stored values for privacy\\
|
||||
(Garage server administrators can't read your mail)}
|
||||
\end{frame}
|
||||
|
||||
\begin{frame}
|
||||
\frametitle{Different deployment scenarios}
|
||||
\begin{center}
|
||||
\only<1>{\includegraphics[width=.9\linewidth]{assets/aerogramme_components1.drawio.pdf}}%
|
||||
\only<2>{\includegraphics[width=.9\linewidth]{assets/aerogramme_components2.drawio.pdf}}%
|
||||
\end{center}
|
||||
\end{frame}
|
||||
|
||||
\begin{frame}
|
||||
\frametitle{A new model for building resilient software}
|
||||
\begin{itemize}
|
||||
\item Design a data model suited to K2V\\
|
||||
{\footnotesize (see Cassandra docs on porting SQL data models to Cassandra)}
|
||||
\vspace{1em}
|
||||
\begin{itemize}
|
||||
\item Use CRDTs or other eventually consistent data types (see e.g. Bayou)
|
||||
\vspace{1em}
|
||||
\item Store opaque binary blobs to provide End-to-End Encryption\\
|
||||
\end{itemize}
|
||||
\vspace{1em}
|
||||
\item Store big blobs (files) in S3
|
||||
\vspace{1em}
|
||||
\item Let Garage manage sharding, replication, failover, etc.
|
||||
\end{itemize}
|
||||
\end{frame}
|
||||
|
||||
\begin{frame}
|
||||
\frametitle{Research perspectives}
|
||||
\begin{itemize}
|
||||
\item Write about Garage's global architecture \emph{(paper in progress)}
|
||||
\vspace{1em}
|
||||
\item Measure and improve Garage's performances
|
||||
\vspace{1em}
|
||||
\item Discuss the optimal layout algorithm, provide proofs
|
||||
\vspace{1em}
|
||||
\item Write about our proposed architecture for (E2EE) apps over K2V+S3
|
||||
\end{itemize}
|
||||
\end{frame}
|
||||
|
||||
\begin{frame}
|
||||
\frametitle{Where to find us}
|
||||
\begin{center}
|
||||
\includegraphics[width=.25\linewidth]{../../logo/garage_hires.png}\\
|
||||
\vspace{-1em}
|
||||
\url{https://garagehq.deuxfleurs.fr/}\\
|
||||
\url{mailto:garagehq@deuxfleurs.fr}\\
|
||||
\texttt{\#garage:deuxfleurs.fr} on Matrix
|
||||
|
||||
\vspace{1.5em}
|
||||
\includegraphics[width=.06\linewidth]{assets/rust_logo.png}
|
||||
\includegraphics[width=.13\linewidth]{assets/AGPLv3_Logo.png}
|
||||
\end{center}
|
||||
\end{frame}
|
||||
|
||||
\end{document}
|
||||
|
||||
%% vim: set ts=4 sw=4 tw=0 noet spelllang=en :
|
||||
+4
-4
@@ -8,10 +8,10 @@ rec {
|
||||
sha256 = "1xy9zpypqfxs5gcq5dcla4bfkhxmh5nzn9dyqkr03lqycm9wg5cr";
|
||||
};
|
||||
cargo2nixSrc = fetchGit {
|
||||
# As of 2022-08-29, stacking two patches: superboum@dedup_propagate and Alexis211@fix_fetchcrategit
|
||||
url = "https://github.com/Alexis211/cargo2nix";
|
||||
ref = "fix_fetchcrategit";
|
||||
rev = "4b31c0cc05b6394916d46e9289f51263d81973b9";
|
||||
# As of 2022-03-17
|
||||
url = "https://github.com/superboum/cargo2nix";
|
||||
ref = "dedup_propagate";
|
||||
rev = "486675c67249e735dd7eb68e1b9feac9db102be7";
|
||||
};
|
||||
|
||||
/*
|
||||
|
||||
+14
-32
@@ -117,34 +117,22 @@ let
|
||||
It speeds up the compilation (when the feature is not required) and released crates have less dependency by default (less attack surface, disk space, etc.).
|
||||
But we want to ship these additional features when we release Garage.
|
||||
In the end, we chose to exclude all features from debug builds while putting (all of) them in the release builds.
|
||||
|
||||
[5] We don't want libsodium-sys and zstd-sys to try to use pkgconfig to build against a system library.
|
||||
However the features to do so get activated for some reason (due to a bug in cargo2nix?),
|
||||
so disable them manually here.
|
||||
Currently, the only feature of Garage is kubernetes-discovery from the garage_rpc crate.
|
||||
*/
|
||||
(pkgs.rustBuilder.rustLib.makeOverride {
|
||||
name = "garage";
|
||||
overrideAttrs = drv:
|
||||
(if git_version != null then {
|
||||
/* [3] */ preConfigure = ''
|
||||
${drv.preConfigure or ""}
|
||||
export GIT_VERSION="${git_version}"
|
||||
'';
|
||||
} else {})
|
||||
//
|
||||
{
|
||||
overrideAttrs = drv: {
|
||||
/* [1] */ setBuildEnv = (buildEnv drv);
|
||||
/* [2] */ hardeningDisable = [ "pie" ];
|
||||
};
|
||||
overrideArgs = old: {
|
||||
/* [4] */ features = [ "bundled-libs" "sled" "metrics" ]
|
||||
++ (if release then [ "kubernetes-discovery" "telemetry-otlp" "lmdb" "sqlite" ] else []);
|
||||
};
|
||||
})
|
||||
|
||||
(pkgs.rustBuilder.rustLib.makeOverride {
|
||||
name = "garage_rpc";
|
||||
overrideAttrs = drv: { /* [1] */ setBuildEnv = (buildEnv drv); };
|
||||
overrideArgs = old: {
|
||||
/* [4] */ features = if release then [ "kubernetes-discovery" ] else [];
|
||||
};
|
||||
})
|
||||
|
||||
(pkgs.rustBuilder.rustLib.makeOverride {
|
||||
@@ -154,7 +142,15 @@ let
|
||||
|
||||
(pkgs.rustBuilder.rustLib.makeOverride {
|
||||
name = "garage_util";
|
||||
overrideAttrs = drv: { /* [1] */ setBuildEnv = (buildEnv drv); };
|
||||
overrideAttrs = drv:
|
||||
(if git_version != null then {
|
||||
/* [3] */ preConfigure = ''
|
||||
${drv.preConfigure or ""}
|
||||
export GIT_VERSION="${git_version}"
|
||||
'';
|
||||
} else {})
|
||||
//
|
||||
{ /* [1] */ setBuildEnv = (buildEnv drv); };
|
||||
})
|
||||
|
||||
(pkgs.rustBuilder.rustLib.makeOverride {
|
||||
@@ -186,20 +182,6 @@ let
|
||||
name = "k2v-client";
|
||||
overrideAttrs = drv: { /* [1] */ setBuildEnv = (buildEnv drv); };
|
||||
})
|
||||
|
||||
(pkgs.rustBuilder.rustLib.makeOverride {
|
||||
name = "libsodium-sys";
|
||||
overrideArgs = old: {
|
||||
features = [ ]; /* [5] */
|
||||
};
|
||||
})
|
||||
|
||||
(pkgs.rustBuilder.rustLib.makeOverride {
|
||||
name = "zstd-sys";
|
||||
overrideArgs = old: {
|
||||
features = [ ]; /* [5] */
|
||||
};
|
||||
})
|
||||
];
|
||||
|
||||
packageFun = import ../Cargo.nix;
|
||||
|
||||
@@ -11,7 +11,7 @@ PATH="${GARAGE_DEBUG}:${GARAGE_RELEASE}:${NIX_RELEASE}:$PATH"
|
||||
FANCYCOLORS=("41m" "42m" "44m" "45m" "100m" "104m")
|
||||
|
||||
export RUST_BACKTRACE=1
|
||||
export RUST_LOG=garage=info,garage_api=debug,netapp=trace
|
||||
export RUST_LOG=garage=info,garage_api=debug
|
||||
MAIN_LABEL="\e[${FANCYCOLORS[0]}[main]\e[49m"
|
||||
|
||||
WHICH_GARAGE=$(which garage || exit 1)
|
||||
|
||||
+13
-14
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "garage_api"
|
||||
version = "0.8.0"
|
||||
version = "0.7.3"
|
||||
authors = ["Alex Auvolat <alex@adnab.me>"]
|
||||
edition = "2018"
|
||||
license = "AGPL-3.0"
|
||||
@@ -14,31 +14,30 @@ path = "lib.rs"
|
||||
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
|
||||
|
||||
[dependencies]
|
||||
garage_model = { version = "0.8.0", path = "../model" }
|
||||
garage_table = { version = "0.8.0", path = "../table" }
|
||||
garage_block = { version = "0.8.0", path = "../block" }
|
||||
garage_util = { version = "0.8.0", path = "../util" }
|
||||
garage_rpc = { version = "0.8.0", path = "../rpc" }
|
||||
garage_model = { version = "0.7.3", path = "../model" }
|
||||
garage_table = { version = "0.7.3", path = "../table" }
|
||||
garage_block = { version = "0.7.3", path = "../block" }
|
||||
garage_util = { version = "0.7.3", path = "../util" }
|
||||
garage_rpc = { version = "0.7.3", path = "../rpc" }
|
||||
|
||||
async-trait = "0.1.7"
|
||||
base64 = "0.13"
|
||||
bytes = "1.0"
|
||||
chrono = "0.4"
|
||||
crypto-common = "0.1"
|
||||
crypto-mac = "0.10"
|
||||
err-derive = "0.3"
|
||||
hex = "0.4"
|
||||
hmac = "0.12"
|
||||
hmac = "0.10"
|
||||
idna = "0.2"
|
||||
tracing = "0.1.30"
|
||||
md-5 = "0.10"
|
||||
md-5 = "0.9"
|
||||
nom = "7.1"
|
||||
sha2 = "0.10"
|
||||
sha2 = "0.9"
|
||||
|
||||
futures = "0.3"
|
||||
futures-util = "0.3"
|
||||
pin-project = "1.0"
|
||||
tokio = { version = "1.0", default-features = false, features = ["rt", "rt-multi-thread", "io-util", "net", "time", "macros", "sync", "signal", "fs"] }
|
||||
tokio-stream = "0.1"
|
||||
|
||||
form_urlencoded = "1.0.0"
|
||||
http = "0.2"
|
||||
@@ -55,9 +54,9 @@ quick-xml = { version = "0.21", features = [ "serialize" ] }
|
||||
url = "2.1"
|
||||
|
||||
opentelemetry = "0.17"
|
||||
opentelemetry-prometheus = { version = "0.10", optional = true }
|
||||
prometheus = { version = "0.13", optional = true }
|
||||
opentelemetry-prometheus = "0.10"
|
||||
opentelemetry-otlp = "0.10"
|
||||
prometheus = "0.13"
|
||||
|
||||
[features]
|
||||
k2v = [ "garage_util/k2v", "garage_model/k2v" ]
|
||||
metrics = [ "opentelemetry-prometheus", "prometheus" ]
|
||||
|
||||
+34
-44
@@ -1,17 +1,15 @@
|
||||
use std::net::SocketAddr;
|
||||
use std::sync::Arc;
|
||||
|
||||
use async_trait::async_trait;
|
||||
|
||||
use futures::future::Future;
|
||||
use http::header::{ACCESS_CONTROL_ALLOW_METHODS, ACCESS_CONTROL_ALLOW_ORIGIN, ALLOW};
|
||||
use http::header::{
|
||||
ACCESS_CONTROL_ALLOW_METHODS, ACCESS_CONTROL_ALLOW_ORIGIN, ALLOW, CONTENT_TYPE,
|
||||
};
|
||||
use hyper::{Body, Request, Response};
|
||||
|
||||
use opentelemetry::trace::SpanRef;
|
||||
|
||||
#[cfg(feature = "metrics")]
|
||||
use opentelemetry::trace::{SpanRef, Tracer};
|
||||
use opentelemetry_prometheus::PrometheusExporter;
|
||||
#[cfg(feature = "metrics")]
|
||||
use prometheus::{Encoder, TextEncoder};
|
||||
|
||||
use garage_model::garage::Garage;
|
||||
@@ -27,17 +25,14 @@ use crate::admin::router::{Authorization, Endpoint};
|
||||
|
||||
pub struct AdminApiServer {
|
||||
garage: Arc<Garage>,
|
||||
#[cfg(feature = "metrics")]
|
||||
exporter: PrometheusExporter,
|
||||
metrics_token: Option<String>,
|
||||
admin_token: Option<String>,
|
||||
}
|
||||
|
||||
impl AdminApiServer {
|
||||
pub fn new(
|
||||
garage: Arc<Garage>,
|
||||
#[cfg(feature = "metrics")] exporter: PrometheusExporter,
|
||||
) -> Self {
|
||||
pub fn new(garage: Arc<Garage>) -> Self {
|
||||
let exporter = opentelemetry_prometheus::exporter().init();
|
||||
let cfg = &garage.config.admin;
|
||||
let metrics_token = cfg
|
||||
.metrics_token
|
||||
@@ -49,22 +44,21 @@ impl AdminApiServer {
|
||||
.map(|tok| format!("Bearer {}", tok));
|
||||
Self {
|
||||
garage,
|
||||
#[cfg(feature = "metrics")]
|
||||
exporter,
|
||||
metrics_token,
|
||||
admin_token,
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn run(
|
||||
self,
|
||||
bind_addr: SocketAddr,
|
||||
shutdown_signal: impl Future<Output = ()>,
|
||||
) -> Result<(), GarageError> {
|
||||
let region = self.garage.config.s3_api.s3_region.clone();
|
||||
ApiServer::new(region, self)
|
||||
.run_server(bind_addr, shutdown_signal)
|
||||
.await
|
||||
pub async fn run(self, shutdown_signal: impl Future<Output = ()>) -> Result<(), GarageError> {
|
||||
if let Some(bind_addr) = self.garage.config.admin.api_bind_addr {
|
||||
let region = self.garage.config.s3_api.s3_region.clone();
|
||||
ApiServer::new(region, self)
|
||||
.run_server(bind_addr, shutdown_signal)
|
||||
.await
|
||||
} else {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
fn handle_options(&self, _req: &Request<Body>) -> Result<Response<Body>, Error> {
|
||||
@@ -77,31 +71,22 @@ impl AdminApiServer {
|
||||
}
|
||||
|
||||
fn handle_metrics(&self) -> Result<Response<Body>, Error> {
|
||||
#[cfg(feature = "metrics")]
|
||||
{
|
||||
use opentelemetry::trace::Tracer;
|
||||
let mut buffer = vec![];
|
||||
let encoder = TextEncoder::new();
|
||||
|
||||
let mut buffer = vec![];
|
||||
let encoder = TextEncoder::new();
|
||||
let tracer = opentelemetry::global::tracer("garage");
|
||||
let metric_families = tracer.in_span("admin/gather_metrics", |_| {
|
||||
self.exporter.registry().gather()
|
||||
});
|
||||
|
||||
let tracer = opentelemetry::global::tracer("garage");
|
||||
let metric_families = tracer.in_span("admin/gather_metrics", |_| {
|
||||
self.exporter.registry().gather()
|
||||
});
|
||||
encoder
|
||||
.encode(&metric_families, &mut buffer)
|
||||
.ok_or_internal_error("Could not serialize metrics")?;
|
||||
|
||||
encoder
|
||||
.encode(&metric_families, &mut buffer)
|
||||
.ok_or_internal_error("Could not serialize metrics")?;
|
||||
|
||||
Ok(Response::builder()
|
||||
.status(200)
|
||||
.header(http::header::CONTENT_TYPE, encoder.format_type())
|
||||
.body(Body::from(buffer))?)
|
||||
}
|
||||
#[cfg(not(feature = "metrics"))]
|
||||
Err(Error::bad_request(
|
||||
"Garage was built without the metrics feature".to_string(),
|
||||
))
|
||||
Ok(Response::builder()
|
||||
.status(200)
|
||||
.header(CONTENT_TYPE, encoder.format_type())
|
||||
.body(Body::from(buffer))?)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -171,7 +156,12 @@ impl ApiHandler for AdminApiServer {
|
||||
}
|
||||
Endpoint::CreateBucket => handle_create_bucket(&self.garage, req).await,
|
||||
Endpoint::DeleteBucket { id } => handle_delete_bucket(&self.garage, id).await,
|
||||
Endpoint::UpdateBucket { id } => handle_update_bucket(&self.garage, id, req).await,
|
||||
Endpoint::PutBucketWebsite { id } => {
|
||||
handle_put_bucket_website(&self.garage, id, req).await
|
||||
}
|
||||
Endpoint::DeleteBucketWebsite { id } => {
|
||||
handle_delete_bucket_website(&self.garage, id).await
|
||||
}
|
||||
// Bucket-key permissions
|
||||
Endpoint::BucketAllowKey => {
|
||||
handle_bucket_change_key_perm(&self.garage, req, true).await
|
||||
|
||||
+29
-67
@@ -14,7 +14,6 @@ use garage_model::bucket_alias_table::*;
|
||||
use garage_model::bucket_table::*;
|
||||
use garage_model::garage::Garage;
|
||||
use garage_model::permission::*;
|
||||
use garage_model::s3::object_table::*;
|
||||
|
||||
use crate::admin::error::*;
|
||||
use crate::admin::key::ApiBucketKeyPerm;
|
||||
@@ -78,13 +77,6 @@ struct BucketLocalAlias {
|
||||
alias: String,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct ApiBucketQuotas {
|
||||
max_size: Option<u64>,
|
||||
max_objects: Option<u64>,
|
||||
}
|
||||
|
||||
pub async fn handle_get_bucket_info(
|
||||
garage: &Arc<Garage>,
|
||||
id: Option<String>,
|
||||
@@ -116,14 +108,6 @@ async fn bucket_info_results(
|
||||
.get_existing_bucket(bucket_id)
|
||||
.await?;
|
||||
|
||||
let counters = garage
|
||||
.object_counter_table
|
||||
.table
|
||||
.get(&bucket_id, &EmptyKey)
|
||||
.await?
|
||||
.map(|x| x.filtered_values(&garage.system.ring.borrow()))
|
||||
.unwrap_or_default();
|
||||
|
||||
let mut relevant_keys = HashMap::new();
|
||||
for (k, _) in bucket
|
||||
.state
|
||||
@@ -164,7 +148,6 @@ async fn bucket_info_results(
|
||||
|
||||
let state = bucket.state.as_option().unwrap();
|
||||
|
||||
let quotas = state.quotas.get();
|
||||
let res =
|
||||
GetBucketInfoResult {
|
||||
id: hex::encode(&bucket.id),
|
||||
@@ -208,16 +191,6 @@ async fn bucket_info_results(
|
||||
}
|
||||
})
|
||||
.collect::<Vec<_>>(),
|
||||
objects: counters.get(OBJECTS).cloned().unwrap_or_default(),
|
||||
bytes: counters.get(BYTES).cloned().unwrap_or_default(),
|
||||
unfinshed_uploads: counters
|
||||
.get(UNFINISHED_UPLOADS)
|
||||
.cloned()
|
||||
.unwrap_or_default(),
|
||||
quotas: ApiBucketQuotas {
|
||||
max_size: quotas.max_size,
|
||||
max_objects: quotas.max_objects,
|
||||
},
|
||||
};
|
||||
|
||||
Ok(json_ok_response(&res)?)
|
||||
@@ -232,10 +205,6 @@ struct GetBucketInfoResult {
|
||||
#[serde(default)]
|
||||
website_config: Option<GetBucketInfoWebsiteResult>,
|
||||
keys: Vec<GetBucketInfoKey>,
|
||||
objects: i64,
|
||||
bytes: i64,
|
||||
unfinshed_uploads: i64,
|
||||
quotas: ApiBucketQuotas,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
@@ -394,12 +363,14 @@ pub async fn handle_delete_bucket(
|
||||
.body(Body::empty())?)
|
||||
}
|
||||
|
||||
pub async fn handle_update_bucket(
|
||||
// ---- BUCKET WEBSITE CONFIGURATION ----
|
||||
|
||||
pub async fn handle_put_bucket_website(
|
||||
garage: &Arc<Garage>,
|
||||
id: String,
|
||||
req: Request<Body>,
|
||||
) -> Result<Response<Body>, Error> {
|
||||
let req = parse_json_body::<UpdateBucketRequest>(req).await?;
|
||||
let req = parse_json_body::<PutBucketWebsiteRequest>(req).await?;
|
||||
let bucket_id = parse_bucket_id(&id)?;
|
||||
|
||||
let mut bucket = garage
|
||||
@@ -408,31 +379,10 @@ pub async fn handle_update_bucket(
|
||||
.await?;
|
||||
|
||||
let state = bucket.state.as_option_mut().unwrap();
|
||||
|
||||
if let Some(wa) = req.website_access {
|
||||
if wa.enabled {
|
||||
state.website_config.update(Some(WebsiteConfig {
|
||||
index_document: wa.index_document.ok_or_bad_request(
|
||||
"Please specify indexDocument when enabling website access.",
|
||||
)?,
|
||||
error_document: wa.error_document,
|
||||
}));
|
||||
} else {
|
||||
if wa.index_document.is_some() || wa.error_document.is_some() {
|
||||
return Err(Error::bad_request(
|
||||
"Cannot specify indexDocument or errorDocument when disabling website access.",
|
||||
));
|
||||
}
|
||||
state.website_config.update(None);
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(q) = req.quotas {
|
||||
state.quotas.update(BucketQuotas {
|
||||
max_size: q.max_size,
|
||||
max_objects: q.max_objects,
|
||||
});
|
||||
}
|
||||
state.website_config.update(Some(WebsiteConfig {
|
||||
index_document: req.index_document,
|
||||
error_document: req.error_document,
|
||||
}));
|
||||
|
||||
garage.bucket_table.insert(&bucket).await?;
|
||||
|
||||
@@ -441,17 +391,29 @@ pub async fn handle_update_bucket(
|
||||
|
||||
#[derive(Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct UpdateBucketRequest {
|
||||
website_access: Option<UpdateBucketWebsiteAccess>,
|
||||
quotas: Option<ApiBucketQuotas>,
|
||||
struct PutBucketWebsiteRequest {
|
||||
index_document: String,
|
||||
#[serde(default)]
|
||||
error_document: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct UpdateBucketWebsiteAccess {
|
||||
enabled: bool,
|
||||
index_document: Option<String>,
|
||||
error_document: Option<String>,
|
||||
pub async fn handle_delete_bucket_website(
|
||||
garage: &Arc<Garage>,
|
||||
id: String,
|
||||
) -> Result<Response<Body>, Error> {
|
||||
let bucket_id = parse_bucket_id(&id)?;
|
||||
|
||||
let mut bucket = garage
|
||||
.bucket_helper()
|
||||
.get_existing_bucket(bucket_id)
|
||||
.await?;
|
||||
|
||||
let state = bucket.state.as_option_mut().unwrap();
|
||||
state.website_config.update(None);
|
||||
|
||||
garage.bucket_table.insert(&bucket).await?;
|
||||
|
||||
bucket_info_results(garage, bucket_id).await
|
||||
}
|
||||
|
||||
// ---- BUCKET/KEY PERMISSIONS ----
|
||||
|
||||
@@ -18,9 +18,7 @@ use crate::helpers::{json_ok_response, parse_json_body};
|
||||
pub async fn handle_get_cluster_status(garage: &Arc<Garage>) -> Result<Response<Body>, Error> {
|
||||
let res = GetClusterStatusResponse {
|
||||
node: hex::encode(garage.system.id),
|
||||
garage_version: garage_util::version::garage_version(),
|
||||
garage_features: garage_util::version::garage_features(),
|
||||
db_engine: garage.db.engine(),
|
||||
garage_version: garage.system.garage_version(),
|
||||
known_nodes: garage
|
||||
.system
|
||||
.get_known_nodes()
|
||||
@@ -100,8 +98,6 @@ fn get_cluster_layout(garage: &Arc<Garage>) -> GetClusterLayoutResponse {
|
||||
struct GetClusterStatusResponse {
|
||||
node: String,
|
||||
garage_version: &'static str,
|
||||
garage_features: Option<&'static [&'static str]>,
|
||||
db_engine: String,
|
||||
known_nodes: HashMap<String, KnownNodeResp>,
|
||||
layout: GetClusterLayoutResponse,
|
||||
}
|
||||
|
||||
@@ -48,7 +48,10 @@ pub enum Endpoint {
|
||||
DeleteBucket {
|
||||
id: String,
|
||||
},
|
||||
UpdateBucket {
|
||||
PutBucketWebsite {
|
||||
id: String,
|
||||
},
|
||||
DeleteBucketWebsite {
|
||||
id: String,
|
||||
},
|
||||
// Bucket-Key Permissions
|
||||
@@ -110,7 +113,8 @@ impl Endpoint {
|
||||
GET "/v0/bucket" => ListBuckets,
|
||||
POST "/v0/bucket" => CreateBucket,
|
||||
DELETE "/v0/bucket" if id => DeleteBucket (query::id),
|
||||
PUT "/v0/bucket" if id => UpdateBucket (query::id),
|
||||
PUT "/v0/bucket/website" if id => PutBucketWebsite (query::id),
|
||||
DELETE "/v0/bucket/website" if id => DeleteBucketWebsite (query::id),
|
||||
// Bucket-key permissions
|
||||
POST "/v0/bucket/allow" => BucketAllowKey,
|
||||
POST "/v0/bucket/deny" => BucketDenyKey,
|
||||
|
||||
@@ -1,4 +1,3 @@
|
||||
use std::net::SocketAddr;
|
||||
use std::sync::Arc;
|
||||
|
||||
use async_trait::async_trait;
|
||||
@@ -37,13 +36,20 @@ pub(crate) struct K2VApiEndpoint {
|
||||
impl K2VApiServer {
|
||||
pub async fn run(
|
||||
garage: Arc<Garage>,
|
||||
bind_addr: SocketAddr,
|
||||
s3_region: String,
|
||||
shutdown_signal: impl Future<Output = ()>,
|
||||
) -> Result<(), GarageError> {
|
||||
ApiServer::new(s3_region, K2VApiServer { garage })
|
||||
if let Some(cfg) = &garage.config.k2v_api {
|
||||
let bind_addr = cfg.api_bind_addr;
|
||||
|
||||
ApiServer::new(
|
||||
garage.config.s3_api.s3_region.clone(),
|
||||
K2VApiServer { garage },
|
||||
)
|
||||
.run_server(bind_addr, shutdown_signal)
|
||||
.await
|
||||
} else {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -10,7 +10,7 @@ use garage_rpc::ring::Ring;
|
||||
use garage_table::util::*;
|
||||
|
||||
use garage_model::garage::Garage;
|
||||
use garage_model::k2v::item_table::{BYTES, CONFLICTS, ENTRIES, VALUES};
|
||||
use garage_model::k2v::counter_table::{BYTES, CONFLICTS, ENTRIES, VALUES};
|
||||
|
||||
use crate::k2v::error::*;
|
||||
use crate::k2v::range::read_range;
|
||||
|
||||
+19
-10
@@ -1,11 +1,10 @@
|
||||
use std::net::SocketAddr;
|
||||
use std::sync::Arc;
|
||||
|
||||
use async_trait::async_trait;
|
||||
|
||||
use futures::future::Future;
|
||||
use hyper::header;
|
||||
use hyper::{Body, Request, Response};
|
||||
use hyper::{Body, Method, Request, Response};
|
||||
|
||||
use opentelemetry::{trace::SpanRef, KeyValue};
|
||||
|
||||
@@ -44,13 +43,16 @@ pub(crate) struct S3ApiEndpoint {
|
||||
impl S3ApiServer {
|
||||
pub async fn run(
|
||||
garage: Arc<Garage>,
|
||||
addr: SocketAddr,
|
||||
s3_region: String,
|
||||
shutdown_signal: impl Future<Output = ()>,
|
||||
) -> Result<(), GarageError> {
|
||||
ApiServer::new(s3_region, S3ApiServer { garage })
|
||||
.run_server(addr, shutdown_signal)
|
||||
.await
|
||||
let addr = garage.config.s3_api.api_bind_addr;
|
||||
|
||||
ApiServer::new(
|
||||
garage.config.s3_api.s3_region.clone(),
|
||||
S3ApiServer { garage },
|
||||
)
|
||||
.run_server(addr, shutdown_signal)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn handle_request_without_bucket(
|
||||
@@ -165,7 +167,14 @@ impl ApiHandler for S3ApiServer {
|
||||
return Err(Error::forbidden("Operation is not allowed for this key."));
|
||||
}
|
||||
|
||||
let matching_cors_rule = find_matching_cors_rule(&bucket, &req)?;
|
||||
// Look up what CORS rule might apply to response.
|
||||
// Requests for methods different than GET, HEAD or POST
|
||||
// are always preflighted, i.e. the browser should make
|
||||
// an OPTIONS call before to check it is allowed
|
||||
let matching_cors_rule = match *req.method() {
|
||||
Method::GET | Method::HEAD | Method::POST => find_matching_cors_rule(&bucket, &req)?,
|
||||
_ => None,
|
||||
};
|
||||
|
||||
let resp = match endpoint {
|
||||
Endpoint::HeadObject {
|
||||
@@ -210,7 +219,7 @@ impl ApiHandler for S3ApiServer {
|
||||
.await
|
||||
}
|
||||
Endpoint::PutObject { key } => {
|
||||
handle_put(garage, req, &bucket, &key, content_sha256).await
|
||||
handle_put(garage, req, bucket_id, &key, content_sha256).await
|
||||
}
|
||||
Endpoint::AbortMultipartUpload { key, upload_id } => {
|
||||
handle_abort_multipart_upload(garage, bucket_id, &key, &upload_id).await
|
||||
@@ -224,7 +233,7 @@ impl ApiHandler for S3ApiServer {
|
||||
garage,
|
||||
req,
|
||||
&bucket_name,
|
||||
&bucket,
|
||||
bucket_id,
|
||||
&key,
|
||||
&upload_id,
|
||||
content_sha256,
|
||||
|
||||
@@ -295,6 +295,7 @@ fn parse_create_bucket_xml(xml_bytes: &[u8]) -> Option<Option<String>> {
|
||||
|
||||
let mut ret = None;
|
||||
for item in cbc.children() {
|
||||
println!("{:?}", item);
|
||||
if item.has_tag_name("LocationConstraint") {
|
||||
if ret != None {
|
||||
return None;
|
||||
|
||||
+11
-18
@@ -5,12 +5,9 @@ use std::time::{Duration, SystemTime, UNIX_EPOCH};
|
||||
use futures::{stream, stream::Stream, StreamExt, TryFutureExt};
|
||||
use md5::{Digest as Md5Digest, Md5};
|
||||
|
||||
use bytes::Bytes;
|
||||
use hyper::{Body, Request, Response};
|
||||
use serde::Serialize;
|
||||
|
||||
use garage_rpc::netapp::bytes_buf::BytesBuf;
|
||||
use garage_rpc::rpc_helper::OrderTag;
|
||||
use garage_table::*;
|
||||
use garage_util::data::*;
|
||||
use garage_util::time::*;
|
||||
@@ -308,18 +305,13 @@ pub async fn handle_upload_part_copy(
|
||||
// if and only if the block returned is a block that already existed
|
||||
// in the Garage data store (thus we don't need to save it again).
|
||||
let garage2 = garage.clone();
|
||||
let order_stream = OrderTag::stream();
|
||||
let source_blocks = stream::iter(blocks_to_copy)
|
||||
.enumerate()
|
||||
.flat_map(|(i, (block_hash, range_to_copy))| {
|
||||
.flat_map(|(block_hash, range_to_copy)| {
|
||||
let garage3 = garage2.clone();
|
||||
stream::once(async move {
|
||||
let data = garage3
|
||||
.block_manager
|
||||
.rpc_get_block(&block_hash, Some(order_stream.order(i as u64)))
|
||||
.await?;
|
||||
let data = garage3.block_manager.rpc_get_block(&block_hash).await?;
|
||||
match range_to_copy {
|
||||
Some(r) => Ok((data.slice(r), None)),
|
||||
Some(r) => Ok((data[r].to_vec(), None)),
|
||||
None => Ok((data, Some(block_hash))),
|
||||
}
|
||||
})
|
||||
@@ -561,13 +553,13 @@ impl CopyPreconditionHeaders {
|
||||
}
|
||||
}
|
||||
|
||||
type BlockStreamItemOk = (Bytes, Option<Hash>);
|
||||
type BlockStreamItemOk = (Vec<u8>, Option<Hash>);
|
||||
type BlockStreamItem = Result<BlockStreamItemOk, garage_util::error::Error>;
|
||||
|
||||
struct Defragmenter<S: Stream<Item = BlockStreamItem>> {
|
||||
block_size: usize,
|
||||
block_stream: Pin<Box<stream::Peekable<S>>>,
|
||||
buffer: BytesBuf,
|
||||
buffer: Vec<u8>,
|
||||
hash: Option<Hash>,
|
||||
}
|
||||
|
||||
@@ -576,7 +568,7 @@ impl<S: Stream<Item = BlockStreamItem>> Defragmenter<S> {
|
||||
Self {
|
||||
block_size,
|
||||
block_stream,
|
||||
buffer: BytesBuf::new(),
|
||||
buffer: vec![],
|
||||
hash: None,
|
||||
}
|
||||
}
|
||||
@@ -594,7 +586,7 @@ impl<S: Stream<Item = BlockStreamItem>> Defragmenter<S> {
|
||||
|
||||
if self.buffer.is_empty() {
|
||||
let (next_block, next_block_hash) = self.block_stream.next().await.unwrap()?;
|
||||
self.buffer.extend(next_block);
|
||||
self.buffer = next_block;
|
||||
self.hash = next_block_hash;
|
||||
} else if self.buffer.len() + peeked_next_block.len() > self.block_size {
|
||||
break;
|
||||
@@ -605,11 +597,11 @@ impl<S: Stream<Item = BlockStreamItem>> Defragmenter<S> {
|
||||
}
|
||||
}
|
||||
|
||||
Ok((self.buffer.take_all(), self.hash.take()))
|
||||
Ok((std::mem::take(&mut self.buffer), self.hash.take()))
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, PartialEq, Eq)]
|
||||
#[derive(Debug, Serialize, PartialEq)]
|
||||
pub struct CopyObjectResult {
|
||||
#[serde(rename = "LastModified")]
|
||||
pub last_modified: s3_xml::Value,
|
||||
@@ -617,7 +609,7 @@ pub struct CopyObjectResult {
|
||||
pub etag: s3_xml::Value,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, PartialEq, Eq)]
|
||||
#[derive(Debug, Serialize, PartialEq)]
|
||||
pub struct CopyPartResult {
|
||||
#[serde(serialize_with = "xmlns_tag")]
|
||||
pub xmlns: (),
|
||||
@@ -662,6 +654,7 @@ mod tests {
|
||||
last_modified: s3_xml::Value("2011-04-11T20:34:56.000Z".into()),
|
||||
etag: s3_xml::Value("\"9b2cf535f27731c974343645a3985328\"".into()),
|
||||
};
|
||||
println!("{}", to_xml_with_header(&v)?);
|
||||
|
||||
assert_eq!(to_xml_with_header(&v)?, expected_retval);
|
||||
|
||||
|
||||
@@ -64,13 +64,14 @@ pub async fn handle_delete(
|
||||
bucket_id: Uuid,
|
||||
key: &str,
|
||||
) -> Result<Response<Body>, Error> {
|
||||
match handle_delete_internal(&garage, bucket_id, key).await {
|
||||
Ok(_) | Err(Error::NoSuchKey) => Ok(Response::builder()
|
||||
.status(StatusCode::NO_CONTENT)
|
||||
.body(Body::from(vec![]))
|
||||
.unwrap()),
|
||||
Err(e) => Err(e),
|
||||
}
|
||||
let (_deleted_version, delete_marker_version) =
|
||||
handle_delete_internal(&garage, bucket_id, key).await?;
|
||||
|
||||
Ok(Response::builder()
|
||||
.header("x-amz-version-id", hex::encode(delete_marker_version))
|
||||
.status(StatusCode::NO_CONTENT)
|
||||
.body(Body::from(vec![]))
|
||||
.unwrap())
|
||||
}
|
||||
|
||||
pub async fn handle_delete_objects(
|
||||
|
||||
+50
-112
@@ -2,19 +2,16 @@
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, UNIX_EPOCH};
|
||||
|
||||
use futures::future;
|
||||
use futures::stream::{self, StreamExt};
|
||||
use futures::stream::*;
|
||||
use http::header::{
|
||||
ACCEPT_RANGES, CONTENT_LENGTH, CONTENT_RANGE, CONTENT_TYPE, ETAG, IF_MODIFIED_SINCE,
|
||||
IF_NONE_MATCH, LAST_MODIFIED, RANGE,
|
||||
};
|
||||
use hyper::body::Bytes;
|
||||
use hyper::{Body, Request, Response, StatusCode};
|
||||
use tokio::sync::mpsc;
|
||||
|
||||
use garage_rpc::rpc_helper::{netapp::stream::ByteStream, OrderTag};
|
||||
use garage_table::EmptyKey;
|
||||
use garage_util::data::*;
|
||||
use garage_util::error::OkOrMessage;
|
||||
|
||||
use garage_model::garage::Garage;
|
||||
use garage_model::s3::object_table::*;
|
||||
@@ -245,56 +242,36 @@ pub async fn handle_get(
|
||||
Ok(resp_builder.body(body)?)
|
||||
}
|
||||
ObjectVersionData::FirstBlock(_, first_block_hash) => {
|
||||
let (tx, rx) = mpsc::channel(2);
|
||||
let read_first_block = garage.block_manager.rpc_get_block(first_block_hash);
|
||||
let get_next_blocks = garage.version_table.get(&last_v.uuid, &EmptyKey);
|
||||
|
||||
let order_stream = OrderTag::stream();
|
||||
let first_block_hash = *first_block_hash;
|
||||
let version_uuid = last_v.uuid;
|
||||
let (first_block, version) = futures::try_join!(read_first_block, get_next_blocks)?;
|
||||
let version = version.ok_or(Error::NoSuchKey)?;
|
||||
|
||||
tokio::spawn(async move {
|
||||
match async {
|
||||
let garage2 = garage.clone();
|
||||
let version_fut = tokio::spawn(async move {
|
||||
garage2.version_table.get(&version_uuid, &EmptyKey).await
|
||||
});
|
||||
let mut blocks = version
|
||||
.blocks
|
||||
.items()
|
||||
.iter()
|
||||
.map(|(_, vb)| (vb.hash, None))
|
||||
.collect::<Vec<_>>();
|
||||
blocks[0].1 = Some(first_block);
|
||||
|
||||
let stream_block_0 = garage
|
||||
.block_manager
|
||||
.rpc_get_block_streaming(&first_block_hash, Some(order_stream.order(0)))
|
||||
.await?;
|
||||
tx.send(stream_block_0)
|
||||
.await
|
||||
.ok_or_message("channel closed")?;
|
||||
|
||||
let version = version_fut.await.unwrap()?.ok_or(Error::NoSuchKey)?;
|
||||
for (i, (_, vb)) in version.blocks.items().iter().enumerate().skip(1) {
|
||||
let stream_block_i = garage
|
||||
.block_manager
|
||||
.rpc_get_block_streaming(&vb.hash, Some(order_stream.order(i as u64)))
|
||||
.await?;
|
||||
tx.send(stream_block_i)
|
||||
.await
|
||||
.ok_or_message("channel closed")?;
|
||||
let body_stream = futures::stream::iter(blocks)
|
||||
.map(move |(hash, data_opt)| {
|
||||
let garage = garage.clone();
|
||||
async move {
|
||||
if let Some(data) = data_opt {
|
||||
Ok(Bytes::from(data))
|
||||
} else {
|
||||
garage
|
||||
.block_manager
|
||||
.rpc_get_block(&hash)
|
||||
.await
|
||||
.map(Bytes::from)
|
||||
}
|
||||
}
|
||||
|
||||
Ok::<(), Error>(())
|
||||
}
|
||||
.await
|
||||
{
|
||||
Ok(()) => (),
|
||||
Err(e) => {
|
||||
let err = std::io::Error::new(
|
||||
std::io::ErrorKind::Other,
|
||||
format!("Error while getting object data: {}", e),
|
||||
);
|
||||
let _ = tx
|
||||
.send(Box::pin(stream::once(future::ready(Err(err)))))
|
||||
.await;
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
let body_stream = tokio_stream::wrappers::ReceiverStream::new(rx).flatten();
|
||||
})
|
||||
.buffered(2);
|
||||
|
||||
let body = hyper::body::Body::wrap_stream(body_stream);
|
||||
Ok(resp_builder.body(body)?)
|
||||
@@ -445,79 +422,40 @@ fn body_from_blocks_range(
|
||||
all_blocks.len(),
|
||||
4 + ((end - begin) / std::cmp::max(all_blocks[0].1.size as u64, 1024)) as usize,
|
||||
));
|
||||
let mut block_offset: u64 = 0;
|
||||
let mut true_offset = 0;
|
||||
for (_, b) in all_blocks.iter() {
|
||||
if block_offset >= end {
|
||||
if true_offset >= end {
|
||||
break;
|
||||
}
|
||||
// Keep only blocks that have an intersection with the requested range
|
||||
if block_offset < end && block_offset + b.size > begin {
|
||||
blocks.push((*b, block_offset));
|
||||
if true_offset < end && true_offset + b.size > begin {
|
||||
blocks.push((*b, true_offset));
|
||||
}
|
||||
block_offset += b.size as u64;
|
||||
true_offset += b.size;
|
||||
}
|
||||
|
||||
let order_stream = OrderTag::stream();
|
||||
let body_stream = futures::stream::iter(blocks)
|
||||
.enumerate()
|
||||
.map(move |(i, (block, block_offset))| {
|
||||
.map(move |(block, true_offset)| {
|
||||
let garage = garage.clone();
|
||||
async move {
|
||||
garage
|
||||
.block_manager
|
||||
.rpc_get_block_streaming(&block.hash, Some(order_stream.order(i as u64)))
|
||||
.await
|
||||
.unwrap_or_else(|e| error_stream(i, e))
|
||||
.scan(block_offset, move |chunk_offset, chunk| {
|
||||
let r = match chunk {
|
||||
Ok(chunk_bytes) => {
|
||||
let chunk_len = chunk_bytes.len() as u64;
|
||||
let r = if *chunk_offset >= end {
|
||||
// The current chunk is after the part we want to read.
|
||||
// Returning None here will stop the scan, the rest of the
|
||||
// stream will be ignored
|
||||
None
|
||||
} else if *chunk_offset + chunk_len <= begin {
|
||||
// The current chunk is before the part we want to read.
|
||||
// We return a None that will be removed by the filter_map
|
||||
// below.
|
||||
Some(None)
|
||||
} else {
|
||||
// The chunk has an intersection with the requested range
|
||||
let start_in_chunk = if *chunk_offset > begin {
|
||||
0
|
||||
} else {
|
||||
begin - *chunk_offset
|
||||
};
|
||||
let end_in_chunk = if *chunk_offset + chunk_len < end {
|
||||
chunk_len
|
||||
} else {
|
||||
end - *chunk_offset
|
||||
};
|
||||
Some(Some(Ok(chunk_bytes
|
||||
.slice(start_in_chunk as usize..end_in_chunk as usize))))
|
||||
};
|
||||
*chunk_offset += chunk_bytes.len() as u64;
|
||||
r
|
||||
}
|
||||
Err(e) => Some(Some(Err(e))),
|
||||
};
|
||||
futures::future::ready(r)
|
||||
})
|
||||
.filter_map(futures::future::ready)
|
||||
let data = garage.block_manager.rpc_get_block(&block.hash).await?;
|
||||
let data = Bytes::from(data);
|
||||
let start_in_block = if true_offset > begin {
|
||||
0
|
||||
} else {
|
||||
begin - true_offset
|
||||
};
|
||||
let end_in_block = if true_offset + block.size < end {
|
||||
block.size
|
||||
} else {
|
||||
end - true_offset
|
||||
};
|
||||
Result::<Bytes, Error>::Ok(
|
||||
data.slice(start_in_block as usize..end_in_block as usize),
|
||||
)
|
||||
}
|
||||
})
|
||||
.buffered(2)
|
||||
.flatten();
|
||||
.buffered(2);
|
||||
|
||||
hyper::body::Body::wrap_stream(body_stream)
|
||||
}
|
||||
|
||||
fn error_stream(i: usize, e: garage_util::error::Error) -> ByteStream {
|
||||
Box::pin(futures::stream::once(async move {
|
||||
Err(std::io::Error::new(
|
||||
std::io::ErrorKind::Other,
|
||||
format!("Could not get block {}: {}", i, e),
|
||||
))
|
||||
}))
|
||||
}
|
||||
|
||||
@@ -22,7 +22,7 @@ use crate::signature::payload::{parse_date, verify_v4};
|
||||
pub async fn handle_post_object(
|
||||
garage: Arc<Garage>,
|
||||
req: Request<Body>,
|
||||
bucket_name: String,
|
||||
bucket: String,
|
||||
) -> Result<Response<Body>, Error> {
|
||||
let boundary = req
|
||||
.headers()
|
||||
@@ -126,18 +126,13 @@ pub async fn handle_post_object(
|
||||
|
||||
let bucket_id = garage
|
||||
.bucket_helper()
|
||||
.resolve_bucket(&bucket_name, &api_key)
|
||||
.resolve_bucket(&bucket, &api_key)
|
||||
.await?;
|
||||
|
||||
if !api_key.allow_write(&bucket_id) {
|
||||
return Err(Error::forbidden("Operation is not allowed for this key."));
|
||||
}
|
||||
|
||||
let bucket = garage
|
||||
.bucket_helper()
|
||||
.get_existing_bucket(bucket_id)
|
||||
.await?;
|
||||
|
||||
let decoded_policy = base64::decode(&policy).ok_or_bad_request("Invalid policy")?;
|
||||
let decoded_policy: Policy =
|
||||
serde_json::from_slice(&decoded_policy).ok_or_bad_request("Invalid policy")?;
|
||||
@@ -232,7 +227,7 @@ pub async fn handle_post_object(
|
||||
garage,
|
||||
headers,
|
||||
StreamLimiter::new(stream, conditions.content_length),
|
||||
&bucket,
|
||||
bucket_id,
|
||||
&key,
|
||||
None,
|
||||
None,
|
||||
@@ -249,7 +244,7 @@ pub async fn handle_post_object(
|
||||
{
|
||||
target
|
||||
.query_pairs_mut()
|
||||
.append_pair("bucket", &bucket_name)
|
||||
.append_pair("bucket", &bucket)
|
||||
.append_pair("key", &key)
|
||||
.append_pair("etag", &etag);
|
||||
let target = target.to_string();
|
||||
@@ -294,7 +289,7 @@ pub async fn handle_post_object(
|
||||
let xml = s3_xml::PostObject {
|
||||
xmlns: (),
|
||||
location: s3_xml::Value(location),
|
||||
bucket: s3_xml::Value(bucket_name),
|
||||
bucket: s3_xml::Value(bucket),
|
||||
key: s3_xml::Value(key),
|
||||
etag: s3_xml::Value(etag),
|
||||
};
|
||||
|
||||
+47
-144
@@ -1,4 +1,4 @@
|
||||
use std::collections::{BTreeMap, BTreeSet, HashMap};
|
||||
use std::collections::{BTreeMap, BTreeSet, VecDeque};
|
||||
use std::sync::Arc;
|
||||
|
||||
use futures::prelude::*;
|
||||
@@ -8,22 +8,13 @@ use hyper::{Request, Response};
|
||||
use md5::{digest::generic_array::*, Digest as Md5Digest, Md5};
|
||||
use sha2::Sha256;
|
||||
|
||||
use opentelemetry::{
|
||||
trace::{FutureExt as OtelFutureExt, TraceContextExt, Tracer},
|
||||
Context,
|
||||
};
|
||||
|
||||
use garage_rpc::netapp::bytes_buf::BytesBuf;
|
||||
use garage_table::*;
|
||||
use garage_util::async_hash::*;
|
||||
use garage_util::data::*;
|
||||
use garage_util::error::Error as GarageError;
|
||||
use garage_util::time::*;
|
||||
|
||||
use garage_block::manager::INLINE_THRESHOLD;
|
||||
use garage_model::bucket_table::Bucket;
|
||||
use garage_model::garage::Garage;
|
||||
use garage_model::index_counter::CountedItem;
|
||||
use garage_model::s3::block_ref_table::*;
|
||||
use garage_model::s3::object_table::*;
|
||||
use garage_model::s3::version_table::*;
|
||||
@@ -35,7 +26,7 @@ use crate::signature::verify_signed_content;
|
||||
pub async fn handle_put(
|
||||
garage: Arc<Garage>,
|
||||
req: Request<Body>,
|
||||
bucket: &Bucket,
|
||||
bucket_id: Uuid,
|
||||
key: &str,
|
||||
content_sha256: Option<Hash>,
|
||||
) -> Result<Response<Body>, Error> {
|
||||
@@ -55,7 +46,7 @@ pub async fn handle_put(
|
||||
garage,
|
||||
headers,
|
||||
body,
|
||||
bucket,
|
||||
bucket_id,
|
||||
key,
|
||||
content_md5,
|
||||
content_sha256,
|
||||
@@ -68,7 +59,7 @@ pub(crate) async fn save_stream<S: Stream<Item = Result<Bytes, Error>> + Unpin>(
|
||||
garage: Arc<Garage>,
|
||||
headers: ObjectVersionHeaders,
|
||||
body: S,
|
||||
bucket: &Bucket,
|
||||
bucket_id: Uuid,
|
||||
key: &str,
|
||||
content_md5: Option<String>,
|
||||
content_sha256: Option<FixedBytes32>,
|
||||
@@ -89,7 +80,6 @@ pub(crate) async fn save_stream<S: Stream<Item = Result<Bytes, Error>> + Unpin>(
|
||||
let data_md5sum_hex = hex::encode(data_md5sum);
|
||||
|
||||
let data_sha256sum = sha256sum(&first_block[..]);
|
||||
let size = first_block.len() as u64;
|
||||
|
||||
ensure_checksum_matches(
|
||||
data_md5sum.as_slice(),
|
||||
@@ -98,22 +88,20 @@ pub(crate) async fn save_stream<S: Stream<Item = Result<Bytes, Error>> + Unpin>(
|
||||
content_sha256,
|
||||
)?;
|
||||
|
||||
check_quotas(&garage, bucket, key, size).await?;
|
||||
|
||||
let object_version = ObjectVersion {
|
||||
uuid: version_uuid,
|
||||
timestamp: version_timestamp,
|
||||
state: ObjectVersionState::Complete(ObjectVersionData::Inline(
|
||||
ObjectVersionMeta {
|
||||
headers,
|
||||
size,
|
||||
size: first_block.len() as u64,
|
||||
etag: data_md5sum_hex.clone(),
|
||||
},
|
||||
first_block.to_vec(),
|
||||
first_block,
|
||||
)),
|
||||
};
|
||||
|
||||
let object = Object::new(bucket.id, key.into(), vec![object_version]);
|
||||
let object = Object::new(bucket_id, key.into(), vec![object_version]);
|
||||
garage.object_table.insert(&object).await?;
|
||||
|
||||
return Ok((version_uuid, data_md5sum_hex));
|
||||
@@ -126,42 +114,36 @@ pub(crate) async fn save_stream<S: Stream<Item = Result<Bytes, Error>> + Unpin>(
|
||||
timestamp: version_timestamp,
|
||||
state: ObjectVersionState::Uploading(headers.clone()),
|
||||
};
|
||||
let object = Object::new(bucket.id, key.into(), vec![object_version.clone()]);
|
||||
let object = Object::new(bucket_id, key.into(), vec![object_version.clone()]);
|
||||
garage.object_table.insert(&object).await?;
|
||||
|
||||
// Initialize corresponding entry in version table
|
||||
// Write this entry now, even with empty block list,
|
||||
// to prevent block_ref entries from being deleted (they can be deleted
|
||||
// if the reference a version that isn't found in the version table)
|
||||
let version = Version::new(version_uuid, bucket.id, key.into(), false);
|
||||
let version = Version::new(version_uuid, bucket_id, key.into(), false);
|
||||
garage.version_table.insert(&version).await?;
|
||||
|
||||
// Transfer data and verify checksum
|
||||
let first_block_hash = async_blake2sum(first_block.clone()).await;
|
||||
|
||||
let tx_result = (|| async {
|
||||
let (total_size, data_md5sum, data_sha256sum) = read_and_put_blocks(
|
||||
&garage,
|
||||
&version,
|
||||
1,
|
||||
first_block,
|
||||
first_block_hash,
|
||||
&mut chunker,
|
||||
)
|
||||
.await?;
|
||||
|
||||
let first_block_hash = blake2sum(&first_block[..]);
|
||||
let tx_result = read_and_put_blocks(
|
||||
&garage,
|
||||
&version,
|
||||
1,
|
||||
first_block,
|
||||
first_block_hash,
|
||||
&mut chunker,
|
||||
)
|
||||
.await
|
||||
.and_then(|(total_size, data_md5sum, data_sha256sum)| {
|
||||
ensure_checksum_matches(
|
||||
data_md5sum.as_slice(),
|
||||
data_sha256sum,
|
||||
content_md5.as_deref(),
|
||||
content_sha256,
|
||||
)?;
|
||||
|
||||
check_quotas(&garage, bucket, key, total_size).await?;
|
||||
|
||||
Ok((total_size, data_md5sum))
|
||||
})()
|
||||
.await;
|
||||
)
|
||||
.map(|()| (total_size, data_md5sum))
|
||||
});
|
||||
|
||||
// If something went wrong, clean up
|
||||
let (total_size, md5sum_arr) = match tx_result {
|
||||
@@ -169,7 +151,7 @@ pub(crate) async fn save_stream<S: Stream<Item = Result<Bytes, Error>> + Unpin>(
|
||||
Err(e) => {
|
||||
// Mark object as aborted, this will free the blocks further down
|
||||
object_version.state = ObjectVersionState::Aborted;
|
||||
let object = Object::new(bucket.id, key.into(), vec![object_version.clone()]);
|
||||
let object = Object::new(bucket_id, key.into(), vec![object_version.clone()]);
|
||||
garage.object_table.insert(&object).await?;
|
||||
return Err(e);
|
||||
}
|
||||
@@ -185,7 +167,7 @@ pub(crate) async fn save_stream<S: Stream<Item = Result<Bytes, Error>> + Unpin>(
|
||||
},
|
||||
first_block_hash,
|
||||
));
|
||||
let object = Object::new(bucket.id, key.into(), vec![object_version]);
|
||||
let object = Object::new(bucket_id, key.into(), vec![object_version]);
|
||||
garage.object_table.insert(&object).await?;
|
||||
|
||||
Ok((version_uuid, md5sum_hex))
|
||||
@@ -218,85 +200,18 @@ fn ensure_checksum_matches(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Check that inserting this object with this size doesn't exceed bucket quotas
|
||||
async fn check_quotas(
|
||||
garage: &Arc<Garage>,
|
||||
bucket: &Bucket,
|
||||
key: &str,
|
||||
size: u64,
|
||||
) -> Result<(), Error> {
|
||||
let quotas = bucket.state.as_option().unwrap().quotas.get();
|
||||
if quotas.max_objects.is_none() && quotas.max_size.is_none() {
|
||||
return Ok(());
|
||||
};
|
||||
|
||||
let key = key.to_string();
|
||||
let (prev_object, counters) = futures::try_join!(
|
||||
garage.object_table.get(&bucket.id, &key),
|
||||
garage.object_counter_table.table.get(&bucket.id, &EmptyKey),
|
||||
)?;
|
||||
|
||||
let counters = counters
|
||||
.map(|x| x.filtered_values(&garage.system.ring.borrow()))
|
||||
.unwrap_or_default();
|
||||
|
||||
let (prev_cnt_obj, prev_cnt_size) = match prev_object {
|
||||
Some(o) => {
|
||||
let prev_cnt = o.counts().into_iter().collect::<HashMap<_, _>>();
|
||||
(
|
||||
prev_cnt.get(OBJECTS).cloned().unwrap_or_default(),
|
||||
prev_cnt.get(BYTES).cloned().unwrap_or_default(),
|
||||
)
|
||||
}
|
||||
None => (0, 0),
|
||||
};
|
||||
let cnt_obj_diff = 1 - prev_cnt_obj;
|
||||
let cnt_size_diff = size as i64 - prev_cnt_size;
|
||||
|
||||
if let Some(mo) = quotas.max_objects {
|
||||
let current_objects = counters.get(OBJECTS).cloned().unwrap_or_default();
|
||||
if cnt_obj_diff > 0 && current_objects + cnt_obj_diff > mo as i64 {
|
||||
return Err(Error::forbidden(format!(
|
||||
"Object quota is reached, maximum objects for this bucket: {}",
|
||||
mo
|
||||
)));
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(ms) = quotas.max_size {
|
||||
let current_size = counters.get(BYTES).cloned().unwrap_or_default();
|
||||
if cnt_size_diff > 0 && current_size + cnt_size_diff > ms as i64 {
|
||||
return Err(Error::forbidden(format!(
|
||||
"Bucket size quota is reached, maximum total size of objects for this bucket: {}. The bucket is already {} bytes, and this object would add {} bytes.",
|
||||
ms, current_size, size
|
||||
)));
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn read_and_put_blocks<S: Stream<Item = Result<Bytes, Error>> + Unpin>(
|
||||
garage: &Garage,
|
||||
version: &Version,
|
||||
part_number: u64,
|
||||
first_block: Bytes,
|
||||
first_block: Vec<u8>,
|
||||
first_block_hash: Hash,
|
||||
chunker: &mut StreamChunker<S>,
|
||||
) -> Result<(u64, GenericArray<u8, typenum::U16>, Hash), Error> {
|
||||
let tracer = opentelemetry::global::tracer("garage");
|
||||
|
||||
let md5hasher = AsyncHasher::<Md5>::new();
|
||||
let sha256hasher = AsyncHasher::<Sha256>::new();
|
||||
|
||||
futures::future::join(
|
||||
md5hasher.update(first_block.clone()),
|
||||
sha256hasher.update(first_block.clone()),
|
||||
)
|
||||
.with_context(Context::current_with_span(
|
||||
tracer.start("Hash first block (md5, sha256)"),
|
||||
))
|
||||
.await;
|
||||
let mut md5hasher = Md5::new();
|
||||
let mut sha256hasher = Sha256::new();
|
||||
md5hasher.update(&first_block[..]);
|
||||
sha256hasher.update(&first_block[..]);
|
||||
|
||||
let mut next_offset = first_block.len();
|
||||
let mut put_curr_version_block = put_block_meta(
|
||||
@@ -318,15 +233,9 @@ async fn read_and_put_blocks<S: Stream<Item = Result<Bytes, Error>> + Unpin>(
|
||||
chunker.next(),
|
||||
)?;
|
||||
if let Some(block) = next_block {
|
||||
let (_, _, block_hash) = futures::future::join3(
|
||||
md5hasher.update(block.clone()),
|
||||
sha256hasher.update(block.clone()),
|
||||
async_blake2sum(block.clone()),
|
||||
)
|
||||
.with_context(Context::current_with_span(
|
||||
tracer.start("Hash block (md5, sha256, blake2)"),
|
||||
))
|
||||
.await;
|
||||
md5hasher.update(&block[..]);
|
||||
sha256hasher.update(&block[..]);
|
||||
let block_hash = blake2sum(&block[..]);
|
||||
let block_len = block.len();
|
||||
put_curr_version_block = put_block_meta(
|
||||
garage,
|
||||
@@ -344,9 +253,9 @@ async fn read_and_put_blocks<S: Stream<Item = Result<Bytes, Error>> + Unpin>(
|
||||
}
|
||||
|
||||
let total_size = next_offset as u64;
|
||||
let data_md5sum = md5hasher.finalize().await;
|
||||
let data_md5sum = md5hasher.finalize();
|
||||
|
||||
let data_sha256sum = sha256hasher.finalize().await;
|
||||
let data_sha256sum = sha256hasher.finalize();
|
||||
let data_sha256sum = Hash::try_from(&data_sha256sum[..]).unwrap();
|
||||
|
||||
Ok((total_size, data_md5sum, data_sha256sum))
|
||||
@@ -386,7 +295,7 @@ struct StreamChunker<S: Stream<Item = Result<Bytes, Error>>> {
|
||||
stream: S,
|
||||
read_all: bool,
|
||||
block_size: usize,
|
||||
buf: BytesBuf,
|
||||
buf: VecDeque<u8>,
|
||||
}
|
||||
|
||||
impl<S: Stream<Item = Result<Bytes, Error>> + Unpin> StreamChunker<S> {
|
||||
@@ -395,11 +304,11 @@ impl<S: Stream<Item = Result<Bytes, Error>> + Unpin> StreamChunker<S> {
|
||||
stream,
|
||||
read_all: false,
|
||||
block_size,
|
||||
buf: BytesBuf::new(),
|
||||
buf: VecDeque::with_capacity(2 * block_size),
|
||||
}
|
||||
}
|
||||
|
||||
async fn next(&mut self) -> Result<Option<Bytes>, Error> {
|
||||
async fn next(&mut self) -> Result<Option<Vec<u8>>, Error> {
|
||||
while !self.read_all && self.buf.len() < self.block_size {
|
||||
if let Some(block) = self.stream.next().await {
|
||||
let bytes = block?;
|
||||
@@ -412,8 +321,12 @@ impl<S: Stream<Item = Result<Bytes, Error>> + Unpin> StreamChunker<S> {
|
||||
|
||||
if self.buf.is_empty() {
|
||||
Ok(None)
|
||||
} else if self.buf.len() <= self.block_size {
|
||||
let block = self.buf.drain(..).collect::<Vec<u8>>();
|
||||
Ok(Some(block))
|
||||
} else {
|
||||
Ok(Some(self.buf.take_max(self.block_size)))
|
||||
let block = self.buf.drain(..self.block_size).collect::<Vec<u8>>();
|
||||
Ok(Some(block))
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -522,9 +435,7 @@ pub async fn handle_put_part(
|
||||
|
||||
// Copy block to store
|
||||
let version = Version::new(version_uuid, bucket_id, key, false);
|
||||
|
||||
let first_block_hash = async_blake2sum(first_block.clone()).await;
|
||||
|
||||
let first_block_hash = blake2sum(&first_block[..]);
|
||||
let (_, data_md5sum, data_sha256sum) = read_and_put_blocks(
|
||||
&garage,
|
||||
&version,
|
||||
@@ -562,7 +473,7 @@ pub async fn handle_complete_multipart_upload(
|
||||
garage: Arc<Garage>,
|
||||
req: Request<Body>,
|
||||
bucket_name: &str,
|
||||
bucket: &Bucket,
|
||||
bucket_id: Uuid,
|
||||
key: &str,
|
||||
upload_id: &str,
|
||||
content_sha256: Option<Hash>,
|
||||
@@ -586,7 +497,7 @@ pub async fn handle_complete_multipart_upload(
|
||||
// Get object and version
|
||||
let key = key.to_string();
|
||||
let (object, version) = futures::try_join!(
|
||||
garage.object_table.get(&bucket.id, &key),
|
||||
garage.object_table.get(&bucket_id, &key),
|
||||
garage.version_table.get(&version_uuid, &EmptyKey),
|
||||
)?;
|
||||
|
||||
@@ -679,14 +590,6 @@ pub async fn handle_complete_multipart_upload(
|
||||
// Calculate total size of final object
|
||||
let total_size = version.blocks.items().iter().map(|x| x.1.size).sum();
|
||||
|
||||
if let Err(e) = check_quotas(&garage, bucket, &key, total_size).await {
|
||||
object_version.state = ObjectVersionState::Aborted;
|
||||
let final_object = Object::new(bucket.id, key.clone(), vec![object_version]);
|
||||
garage.object_table.insert(&final_object).await?;
|
||||
|
||||
return Err(e);
|
||||
}
|
||||
|
||||
// Write final object version
|
||||
object_version.state = ObjectVersionState::Complete(ObjectVersionData::FirstBlock(
|
||||
ObjectVersionMeta {
|
||||
@@ -697,7 +600,7 @@ pub async fn handle_complete_multipart_upload(
|
||||
version.blocks.items()[0].1.hash,
|
||||
));
|
||||
|
||||
let final_object = Object::new(bucket.id, key.clone(), vec![object_version]);
|
||||
let final_object = Object::new(bucket_id, key.clone(), vec![object_version]);
|
||||
garage.object_table.insert(&final_object).await?;
|
||||
|
||||
// Send response saying ok we're done
|
||||
|
||||
+21
-21
@@ -25,7 +25,7 @@ impl From<&str> for Value {
|
||||
#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)]
|
||||
pub struct IntValue(#[serde(rename = "$value")] pub i64);
|
||||
|
||||
#[derive(Debug, Serialize, PartialEq, Eq)]
|
||||
#[derive(Debug, Serialize, PartialEq)]
|
||||
pub struct Bucket {
|
||||
#[serde(rename = "CreationDate")]
|
||||
pub creation_date: Value,
|
||||
@@ -33,7 +33,7 @@ pub struct Bucket {
|
||||
pub name: Value,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, PartialEq, Eq)]
|
||||
#[derive(Debug, Serialize, PartialEq)]
|
||||
pub struct Owner {
|
||||
#[serde(rename = "DisplayName")]
|
||||
pub display_name: Value,
|
||||
@@ -41,13 +41,13 @@ pub struct Owner {
|
||||
pub id: Value,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, PartialEq, Eq)]
|
||||
#[derive(Debug, Serialize, PartialEq)]
|
||||
pub struct BucketList {
|
||||
#[serde(rename = "Bucket")]
|
||||
pub entries: Vec<Bucket>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, PartialEq, Eq)]
|
||||
#[derive(Debug, Serialize, PartialEq)]
|
||||
pub struct ListAllMyBucketsResult {
|
||||
#[serde(rename = "Buckets")]
|
||||
pub buckets: BucketList,
|
||||
@@ -55,7 +55,7 @@ pub struct ListAllMyBucketsResult {
|
||||
pub owner: Owner,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, PartialEq, Eq)]
|
||||
#[derive(Debug, Serialize, PartialEq)]
|
||||
pub struct LocationConstraint {
|
||||
#[serde(serialize_with = "xmlns_tag")]
|
||||
pub xmlns: (),
|
||||
@@ -63,7 +63,7 @@ pub struct LocationConstraint {
|
||||
pub region: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, PartialEq, Eq)]
|
||||
#[derive(Debug, Serialize, PartialEq)]
|
||||
pub struct Deleted {
|
||||
#[serde(rename = "Key")]
|
||||
pub key: Value,
|
||||
@@ -73,7 +73,7 @@ pub struct Deleted {
|
||||
pub delete_marker_version_id: Value,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, PartialEq, Eq)]
|
||||
#[derive(Debug, Serialize, PartialEq)]
|
||||
pub struct Error {
|
||||
#[serde(rename = "Code")]
|
||||
pub code: Value,
|
||||
@@ -85,7 +85,7 @@ pub struct Error {
|
||||
pub region: Option<Value>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, PartialEq, Eq)]
|
||||
#[derive(Debug, Serialize, PartialEq)]
|
||||
pub struct DeleteError {
|
||||
#[serde(rename = "Code")]
|
||||
pub code: Value,
|
||||
@@ -97,7 +97,7 @@ pub struct DeleteError {
|
||||
pub version_id: Option<Value>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, PartialEq, Eq)]
|
||||
#[derive(Debug, Serialize, PartialEq)]
|
||||
pub struct DeleteResult {
|
||||
#[serde(serialize_with = "xmlns_tag")]
|
||||
pub xmlns: (),
|
||||
@@ -107,7 +107,7 @@ pub struct DeleteResult {
|
||||
pub errors: Vec<DeleteError>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, PartialEq, Eq)]
|
||||
#[derive(Debug, Serialize, PartialEq)]
|
||||
pub struct InitiateMultipartUploadResult {
|
||||
#[serde(serialize_with = "xmlns_tag")]
|
||||
pub xmlns: (),
|
||||
@@ -119,7 +119,7 @@ pub struct InitiateMultipartUploadResult {
|
||||
pub upload_id: Value,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, PartialEq, Eq)]
|
||||
#[derive(Debug, Serialize, PartialEq)]
|
||||
pub struct CompleteMultipartUploadResult {
|
||||
#[serde(serialize_with = "xmlns_tag")]
|
||||
pub xmlns: (),
|
||||
@@ -133,7 +133,7 @@ pub struct CompleteMultipartUploadResult {
|
||||
pub etag: Value,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, PartialEq, Eq)]
|
||||
#[derive(Debug, Serialize, PartialEq)]
|
||||
pub struct Initiator {
|
||||
#[serde(rename = "DisplayName")]
|
||||
pub display_name: Value,
|
||||
@@ -141,7 +141,7 @@ pub struct Initiator {
|
||||
pub id: Value,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, PartialEq, Eq)]
|
||||
#[derive(Debug, Serialize, PartialEq)]
|
||||
pub struct ListMultipartItem {
|
||||
#[serde(rename = "Initiated")]
|
||||
pub initiated: Value,
|
||||
@@ -157,7 +157,7 @@ pub struct ListMultipartItem {
|
||||
pub storage_class: Value,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, PartialEq, Eq)]
|
||||
#[derive(Debug, Serialize, PartialEq)]
|
||||
pub struct ListMultipartUploadsResult {
|
||||
#[serde(serialize_with = "xmlns_tag")]
|
||||
pub xmlns: (),
|
||||
@@ -187,7 +187,7 @@ pub struct ListMultipartUploadsResult {
|
||||
pub encoding_type: Option<Value>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, PartialEq, Eq)]
|
||||
#[derive(Debug, Serialize, PartialEq)]
|
||||
pub struct PartItem {
|
||||
#[serde(rename = "ETag")]
|
||||
pub etag: Value,
|
||||
@@ -199,7 +199,7 @@ pub struct PartItem {
|
||||
pub size: IntValue,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, PartialEq, Eq)]
|
||||
#[derive(Debug, Serialize, PartialEq)]
|
||||
pub struct ListPartsResult {
|
||||
#[serde(serialize_with = "xmlns_tag")]
|
||||
pub xmlns: (),
|
||||
@@ -227,7 +227,7 @@ pub struct ListPartsResult {
|
||||
pub storage_class: Value,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, PartialEq, Eq)]
|
||||
#[derive(Debug, Serialize, PartialEq)]
|
||||
pub struct ListBucketItem {
|
||||
#[serde(rename = "Key")]
|
||||
pub key: Value,
|
||||
@@ -241,13 +241,13 @@ pub struct ListBucketItem {
|
||||
pub storage_class: Value,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, PartialEq, Eq)]
|
||||
#[derive(Debug, Serialize, PartialEq)]
|
||||
pub struct CommonPrefix {
|
||||
#[serde(rename = "Prefix")]
|
||||
pub prefix: Value,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, PartialEq, Eq)]
|
||||
#[derive(Debug, Serialize, PartialEq)]
|
||||
pub struct ListBucketResult {
|
||||
#[serde(serialize_with = "xmlns_tag")]
|
||||
pub xmlns: (),
|
||||
@@ -281,7 +281,7 @@ pub struct ListBucketResult {
|
||||
pub common_prefixes: Vec<CommonPrefix>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, PartialEq, Eq)]
|
||||
#[derive(Debug, Serialize, PartialEq)]
|
||||
pub struct VersioningConfiguration {
|
||||
#[serde(serialize_with = "xmlns_tag")]
|
||||
pub xmlns: (),
|
||||
@@ -289,7 +289,7 @@ pub struct VersioningConfiguration {
|
||||
pub status: Option<Value>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, PartialEq, Eq)]
|
||||
#[derive(Debug, Serialize, PartialEq)]
|
||||
pub struct PostObject {
|
||||
#[serde(serialize_with = "xmlns_tag")]
|
||||
pub xmlns: (),
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
use chrono::{DateTime, Utc};
|
||||
use hmac::{Hmac, Mac};
|
||||
use hmac::{Hmac, Mac, NewMac};
|
||||
use sha2::Sha256;
|
||||
|
||||
use garage_util::data::{sha256sum, Hash};
|
||||
@@ -29,17 +29,17 @@ pub fn signing_hmac(
|
||||
secret_key: &str,
|
||||
region: &str,
|
||||
service: &str,
|
||||
) -> Result<HmacSha256, crypto_common::InvalidLength> {
|
||||
) -> Result<HmacSha256, crypto_mac::InvalidKeyLength> {
|
||||
let secret = String::from("AWS4") + secret_key;
|
||||
let mut date_hmac = HmacSha256::new_from_slice(secret.as_bytes())?;
|
||||
let mut date_hmac = HmacSha256::new_varkey(secret.as_bytes())?;
|
||||
date_hmac.update(datetime.format(SHORT_DATE).to_string().as_bytes());
|
||||
let mut region_hmac = HmacSha256::new_from_slice(&date_hmac.finalize().into_bytes())?;
|
||||
let mut region_hmac = HmacSha256::new_varkey(&date_hmac.finalize().into_bytes())?;
|
||||
region_hmac.update(region.as_bytes());
|
||||
let mut service_hmac = HmacSha256::new_from_slice(®ion_hmac.finalize().into_bytes())?;
|
||||
let mut service_hmac = HmacSha256::new_varkey(®ion_hmac.finalize().into_bytes())?;
|
||||
service_hmac.update(service.as_bytes());
|
||||
let mut signing_hmac = HmacSha256::new_from_slice(&service_hmac.finalize().into_bytes())?;
|
||||
let mut signing_hmac = HmacSha256::new_varkey(&service_hmac.finalize().into_bytes())?;
|
||||
signing_hmac.update(b"aws4_request");
|
||||
let hmac = HmacSha256::new_from_slice(&signing_hmac.finalize().into_bytes())?;
|
||||
let hmac = HmacSha256::new_varkey(&signing_hmac.finalize().into_bytes())?;
|
||||
Ok(hmac)
|
||||
}
|
||||
|
||||
|
||||
+6
-12
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "garage_block"
|
||||
version = "0.8.0"
|
||||
version = "0.7.3"
|
||||
authors = ["Alex Auvolat <alex@adnab.me>"]
|
||||
edition = "2018"
|
||||
license = "AGPL-3.0"
|
||||
@@ -14,23 +14,21 @@ path = "lib.rs"
|
||||
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
|
||||
|
||||
[dependencies]
|
||||
garage_db = { version = "0.8.0", path = "../db" }
|
||||
garage_rpc = { version = "0.8.0", path = "../rpc" }
|
||||
garage_util = { version = "0.8.0", path = "../util" }
|
||||
garage_table = { version = "0.8.0", path = "../table" }
|
||||
garage_rpc = { version = "0.7.3", path = "../rpc" }
|
||||
garage_util = { version = "0.7.3", path = "../util" }
|
||||
garage_table = { version = "0.7.3", path = "../table" }
|
||||
|
||||
opentelemetry = "0.17"
|
||||
|
||||
arc-swap = "1.5"
|
||||
async-trait = "0.1.7"
|
||||
bytes = "1.0"
|
||||
hex = "0.4"
|
||||
tracing = "0.1.30"
|
||||
rand = "0.8"
|
||||
|
||||
async-compression = { version = "0.3", features = ["tokio", "zstd"] }
|
||||
zstd = { version = "0.9", default-features = false }
|
||||
|
||||
sled = "0.34"
|
||||
|
||||
rmp-serde = "0.15"
|
||||
serde = { version = "1.0", default-features = false, features = ["derive", "rc"] }
|
||||
serde_bytes = "0.11"
|
||||
@@ -38,7 +36,3 @@ serde_bytes = "0.11"
|
||||
futures = "0.3"
|
||||
futures-util = "0.3"
|
||||
tokio = { version = "1.0", default-features = false, features = ["rt", "rt-multi-thread", "io-util", "net", "time", "macros", "sync", "signal", "fs"] }
|
||||
tokio-util = { version = "0.6", features = ["io"] }
|
||||
|
||||
[features]
|
||||
system-libs = [ "zstd/pkg-config" ]
|
||||
|
||||
+12
-36
@@ -1,22 +1,16 @@
|
||||
use bytes::Bytes;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use zstd::stream::{decode_all as zstd_decode, Encoder};
|
||||
|
||||
use garage_util::data::*;
|
||||
use garage_util::error::*;
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize, Copy, Clone)]
|
||||
pub enum DataBlockHeader {
|
||||
Plain,
|
||||
Compressed,
|
||||
}
|
||||
|
||||
/// A possibly compressed block of data
|
||||
#[derive(Debug, Serialize, Deserialize)]
|
||||
pub enum DataBlock {
|
||||
/// Uncompressed data
|
||||
Plain(Bytes),
|
||||
Plain(#[serde(with = "serde_bytes")] Vec<u8>),
|
||||
/// Data compressed with zstd
|
||||
Compressed(Bytes),
|
||||
Compressed(#[serde(with = "serde_bytes")] Vec<u8>),
|
||||
}
|
||||
|
||||
impl DataBlock {
|
||||
@@ -36,7 +30,7 @@ impl DataBlock {
|
||||
/// Get the buffer, possibly decompressing it, and verify it's integrity.
|
||||
/// For Plain block, data is compared to hash, for Compressed block, zstd checksumming system
|
||||
/// is used instead.
|
||||
pub fn verify_get(self, hash: Hash) -> Result<Bytes, Error> {
|
||||
pub fn verify_get(self, hash: Hash) -> Result<Vec<u8>, Error> {
|
||||
match self {
|
||||
DataBlock::Plain(data) => {
|
||||
if blake2sum(&data) == hash {
|
||||
@@ -45,9 +39,9 @@ impl DataBlock {
|
||||
Err(Error::CorruptData(hash))
|
||||
}
|
||||
}
|
||||
DataBlock::Compressed(data) => zstd_decode(&data[..])
|
||||
.map_err(|_| Error::CorruptData(hash))
|
||||
.map(Bytes::from),
|
||||
DataBlock::Compressed(data) => {
|
||||
zstd_decode(&data[..]).map_err(|_| Error::CorruptData(hash))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -67,31 +61,13 @@ impl DataBlock {
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn from_buffer(data: Bytes, level: Option<i32>) -> DataBlock {
|
||||
tokio::task::spawn_blocking(move || {
|
||||
if let Some(level) = level {
|
||||
if let Ok(data) = zstd_encode(&data[..], level) {
|
||||
return DataBlock::Compressed(data.into());
|
||||
}
|
||||
pub fn from_buffer(data: Vec<u8>, level: Option<i32>) -> DataBlock {
|
||||
if let Some(level) = level {
|
||||
if let Ok(data) = zstd_encode(&data[..], level) {
|
||||
return DataBlock::Compressed(data);
|
||||
}
|
||||
DataBlock::Plain(data)
|
||||
})
|
||||
.await
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
pub fn into_parts(self) -> (DataBlockHeader, Bytes) {
|
||||
match self {
|
||||
DataBlock::Plain(data) => (DataBlockHeader::Plain, data),
|
||||
DataBlock::Compressed(data) => (DataBlockHeader::Compressed, data),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn from_parts(h: DataBlockHeader, bytes: Bytes) -> Self {
|
||||
match h {
|
||||
DataBlockHeader::Plain => DataBlock::Plain(bytes),
|
||||
DataBlockHeader::Compressed => DataBlock::Compressed(bytes),
|
||||
}
|
||||
DataBlock::Plain(data)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -2,8 +2,6 @@
|
||||
extern crate tracing;
|
||||
|
||||
pub mod manager;
|
||||
pub mod repair;
|
||||
pub mod resync;
|
||||
|
||||
mod block;
|
||||
mod metrics;
|
||||
|
||||
+632
-313
File diff suppressed because it is too large
Load Diff
@@ -1,6 +1,6 @@
|
||||
use opentelemetry::{global, metrics::*};
|
||||
|
||||
use garage_db::counted_tree_hack::CountedTree;
|
||||
use garage_util::sled_counter::SledCountedTree;
|
||||
|
||||
/// TableMetrics reference all counter used for metrics
|
||||
pub struct BlockManagerMetrics {
|
||||
@@ -23,7 +23,7 @@ pub struct BlockManagerMetrics {
|
||||
}
|
||||
|
||||
impl BlockManagerMetrics {
|
||||
pub fn new(resync_queue: CountedTree, resync_errors: CountedTree) -> Self {
|
||||
pub fn new(resync_queue: SledCountedTree, resync_errors: SledCountedTree) -> Self {
|
||||
let meter = global::meter("garage_model/block");
|
||||
Self {
|
||||
_resync_queue_len: meter
|
||||
|
||||
+17
-32
@@ -1,7 +1,5 @@
|
||||
use std::convert::TryInto;
|
||||
|
||||
use garage_db as db;
|
||||
|
||||
use garage_util::data::*;
|
||||
use garage_util::error::*;
|
||||
use garage_util::time::*;
|
||||
@@ -9,41 +7,31 @@ use garage_util::time::*;
|
||||
use crate::manager::BLOCK_GC_DELAY;
|
||||
|
||||
pub struct BlockRc {
|
||||
pub(crate) rc: db::Tree,
|
||||
pub(crate) rc: sled::Tree,
|
||||
}
|
||||
|
||||
impl BlockRc {
|
||||
pub(crate) fn new(rc: db::Tree) -> Self {
|
||||
pub(crate) fn new(rc: sled::Tree) -> Self {
|
||||
Self { rc }
|
||||
}
|
||||
|
||||
/// Increment the reference counter associated to a hash.
|
||||
/// Returns true if the RC goes from zero to nonzero.
|
||||
pub(crate) fn block_incref(
|
||||
&self,
|
||||
tx: &mut db::Transaction,
|
||||
hash: &Hash,
|
||||
) -> db::TxOpResult<bool> {
|
||||
let old_rc = RcEntry::parse_opt(tx.get(&self.rc, &hash)?);
|
||||
match old_rc.increment().serialize() {
|
||||
Some(x) => tx.insert(&self.rc, &hash, x)?,
|
||||
None => unreachable!(),
|
||||
};
|
||||
pub(crate) fn block_incref(&self, hash: &Hash) -> Result<bool, Error> {
|
||||
let old_rc = self
|
||||
.rc
|
||||
.fetch_and_update(&hash, |old| RcEntry::parse_opt(old).increment().serialize())?;
|
||||
let old_rc = RcEntry::parse_opt(old_rc);
|
||||
Ok(old_rc.is_zero())
|
||||
}
|
||||
|
||||
/// Decrement the reference counter associated to a hash.
|
||||
/// Returns true if the RC is now zero.
|
||||
pub(crate) fn block_decref(
|
||||
&self,
|
||||
tx: &mut db::Transaction,
|
||||
hash: &Hash,
|
||||
) -> db::TxOpResult<bool> {
|
||||
let new_rc = RcEntry::parse_opt(tx.get(&self.rc, &hash)?).decrement();
|
||||
match new_rc.serialize() {
|
||||
Some(x) => tx.insert(&self.rc, &hash, x)?,
|
||||
None => tx.remove(&self.rc, &hash)?,
|
||||
};
|
||||
pub(crate) fn block_decref(&self, hash: &Hash) -> Result<bool, Error> {
|
||||
let new_rc = self
|
||||
.rc
|
||||
.update_and_fetch(&hash, |old| RcEntry::parse_opt(old).decrement().serialize())?;
|
||||
let new_rc = RcEntry::parse_opt(new_rc);
|
||||
Ok(matches!(new_rc, RcEntry::Deletable { .. }))
|
||||
}
|
||||
|
||||
@@ -56,15 +44,12 @@ impl BlockRc {
|
||||
/// deletion time has passed
|
||||
pub(crate) fn clear_deleted_block_rc(&self, hash: &Hash) -> Result<(), Error> {
|
||||
let now = now_msec();
|
||||
self.rc.db().transaction(|mut tx| {
|
||||
let rcval = RcEntry::parse_opt(tx.get(&self.rc, &hash)?);
|
||||
match rcval {
|
||||
RcEntry::Deletable { at_time } if now > at_time => {
|
||||
tx.remove(&self.rc, &hash)?;
|
||||
}
|
||||
_ => (),
|
||||
self.rc.update_and_fetch(&hash, |rcval| {
|
||||
let updated = match RcEntry::parse_opt(rcval) {
|
||||
RcEntry::Deletable { at_time } if now > at_time => RcEntry::Absent,
|
||||
v => v,
|
||||
};
|
||||
tx.commit(())
|
||||
updated.serialize()
|
||||
})?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -1,466 +0,0 @@
|
||||
use core::ops::Bound;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use async_trait::async_trait;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use tokio::fs;
|
||||
use tokio::select;
|
||||
use tokio::sync::mpsc;
|
||||
use tokio::sync::watch;
|
||||
|
||||
use garage_util::background::*;
|
||||
use garage_util::data::*;
|
||||
use garage_util::error::*;
|
||||
use garage_util::persister::Persister;
|
||||
use garage_util::time::*;
|
||||
use garage_util::tranquilizer::Tranquilizer;
|
||||
|
||||
use crate::manager::*;
|
||||
|
||||
// Full scrub every 30 days
|
||||
const SCRUB_INTERVAL: Duration = Duration::from_secs(3600 * 24 * 30);
|
||||
// Scrub tranquility is initially set to 4, but can be changed in the CLI
|
||||
// and the updated version is persisted over Garage restarts
|
||||
const INITIAL_SCRUB_TRANQUILITY: u32 = 4;
|
||||
|
||||
// ---- ---- ----
|
||||
// FIRST KIND OF REPAIR: FINDING MISSING BLOCKS/USELESS BLOCKS
|
||||
// This is a one-shot repair operation that can be launched,
|
||||
// checks everything, and then exits.
|
||||
// ---- ---- ----
|
||||
|
||||
pub struct RepairWorker {
|
||||
manager: Arc<BlockManager>,
|
||||
next_start: Option<Hash>,
|
||||
block_iter: Option<BlockStoreIterator>,
|
||||
}
|
||||
|
||||
impl RepairWorker {
|
||||
pub fn new(manager: Arc<BlockManager>) -> Self {
|
||||
Self {
|
||||
manager,
|
||||
next_start: None,
|
||||
block_iter: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl Worker for RepairWorker {
|
||||
fn name(&self) -> String {
|
||||
"Block repair worker".into()
|
||||
}
|
||||
|
||||
fn info(&self) -> Option<String> {
|
||||
match self.block_iter.as_ref() {
|
||||
None => {
|
||||
let idx_bytes = self
|
||||
.next_start
|
||||
.as_ref()
|
||||
.map(|x| x.as_slice())
|
||||
.unwrap_or(&[]);
|
||||
let idx_bytes = if idx_bytes.len() > 4 {
|
||||
&idx_bytes[..4]
|
||||
} else {
|
||||
idx_bytes
|
||||
};
|
||||
Some(format!("Phase 1: {}", hex::encode(idx_bytes)))
|
||||
}
|
||||
Some(bi) => Some(format!("Phase 2: {:.2}% done", bi.progress() * 100.)),
|
||||
}
|
||||
}
|
||||
|
||||
async fn work(&mut self, _must_exit: &mut watch::Receiver<bool>) -> Result<WorkerState, Error> {
|
||||
match self.block_iter.as_mut() {
|
||||
None => {
|
||||
// Phase 1: Repair blocks from RC table.
|
||||
|
||||
// We have to do this complicated two-step process where we first read a bunch
|
||||
// of hashes from the RC table, and then insert them in the to-resync queue,
|
||||
// because of SQLite. Basically, as long as we have an iterator on a DB table,
|
||||
// we can't do anything else on the DB. The naive approach (which we had previously)
|
||||
// of just iterating on the RC table and inserting items one to one in the resync
|
||||
// queue can't work here, it would just provoke a deadlock in the SQLite adapter code.
|
||||
// This is mostly because the Rust bindings for SQLite assume a worst-case scenario
|
||||
// where SQLite is not compiled in thread-safe mode, so we have to wrap everything
|
||||
// in a mutex (see db/sqlite_adapter.rs and discussion in PR #322).
|
||||
// TODO: maybe do this with tokio::task::spawn_blocking ?
|
||||
let mut batch_of_hashes = vec![];
|
||||
let start_bound = match self.next_start.as_ref() {
|
||||
None => Bound::Unbounded,
|
||||
Some(x) => Bound::Excluded(x.as_slice()),
|
||||
};
|
||||
for entry in self
|
||||
.manager
|
||||
.rc
|
||||
.rc
|
||||
.range::<&[u8], _>((start_bound, Bound::Unbounded))?
|
||||
{
|
||||
let (hash, _) = entry?;
|
||||
let hash = Hash::try_from(&hash[..]).unwrap();
|
||||
batch_of_hashes.push(hash);
|
||||
if batch_of_hashes.len() >= 1000 {
|
||||
break;
|
||||
}
|
||||
}
|
||||
if batch_of_hashes.is_empty() {
|
||||
// move on to phase 2
|
||||
self.block_iter = Some(BlockStoreIterator::new(&self.manager));
|
||||
return Ok(WorkerState::Busy);
|
||||
}
|
||||
|
||||
for hash in batch_of_hashes.into_iter() {
|
||||
self.manager
|
||||
.resync
|
||||
.put_to_resync(&hash, Duration::from_secs(0))?;
|
||||
self.next_start = Some(hash)
|
||||
}
|
||||
|
||||
Ok(WorkerState::Busy)
|
||||
}
|
||||
Some(bi) => {
|
||||
// Phase 2: Repair blocks actually on disk
|
||||
// Lists all blocks on disk and adds them to the resync queue.
|
||||
// This allows us to find blocks we are storing but don't actually need,
|
||||
// so that we can offload them if necessary and then delete them locally.
|
||||
if let Some(hash) = bi.next().await? {
|
||||
self.manager
|
||||
.resync
|
||||
.put_to_resync(&hash, Duration::from_secs(0))?;
|
||||
Ok(WorkerState::Busy)
|
||||
} else {
|
||||
Ok(WorkerState::Done)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn wait_for_work(&mut self, _must_exit: &watch::Receiver<bool>) -> WorkerState {
|
||||
unreachable!()
|
||||
}
|
||||
}
|
||||
|
||||
// ---- ---- ----
|
||||
// SECOND KIND OF REPAIR: SCRUBBING THE DATASTORE
|
||||
// This is significantly more complex than the process above,
|
||||
// as it is a continuously-running task that triggers automatically
|
||||
// every SCRUB_INTERVAL, but can also be triggered manually
|
||||
// and whose parameter (esp. speed) can be controlled at runtime.
|
||||
// ---- ---- ----
|
||||
|
||||
pub struct ScrubWorker {
|
||||
manager: Arc<BlockManager>,
|
||||
rx_cmd: mpsc::Receiver<ScrubWorkerCommand>,
|
||||
|
||||
work: ScrubWorkerState,
|
||||
tranquilizer: Tranquilizer,
|
||||
|
||||
persister: Persister<ScrubWorkerPersisted>,
|
||||
persisted: ScrubWorkerPersisted,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize)]
|
||||
struct ScrubWorkerPersisted {
|
||||
tranquility: u32,
|
||||
time_last_complete_scrub: u64,
|
||||
corruptions_detected: u64,
|
||||
}
|
||||
|
||||
enum ScrubWorkerState {
|
||||
Running(BlockStoreIterator),
|
||||
Paused(BlockStoreIterator, u64), // u64 = time when to resume scrub
|
||||
Finished,
|
||||
}
|
||||
|
||||
impl Default for ScrubWorkerState {
|
||||
fn default() -> Self {
|
||||
ScrubWorkerState::Finished
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub enum ScrubWorkerCommand {
|
||||
Start,
|
||||
Pause(Duration),
|
||||
Resume,
|
||||
Cancel,
|
||||
SetTranquility(u32),
|
||||
}
|
||||
|
||||
impl ScrubWorker {
|
||||
pub fn new(manager: Arc<BlockManager>, rx_cmd: mpsc::Receiver<ScrubWorkerCommand>) -> Self {
|
||||
let persister = Persister::new(&manager.system.metadata_dir, "scrub_info");
|
||||
let persisted = match persister.load() {
|
||||
Ok(v) => v,
|
||||
Err(_) => ScrubWorkerPersisted {
|
||||
time_last_complete_scrub: 0,
|
||||
tranquility: INITIAL_SCRUB_TRANQUILITY,
|
||||
corruptions_detected: 0,
|
||||
},
|
||||
};
|
||||
Self {
|
||||
manager,
|
||||
rx_cmd,
|
||||
work: ScrubWorkerState::Finished,
|
||||
tranquilizer: Tranquilizer::new(30),
|
||||
persister,
|
||||
persisted,
|
||||
}
|
||||
}
|
||||
|
||||
async fn handle_cmd(&mut self, cmd: ScrubWorkerCommand) {
|
||||
match cmd {
|
||||
ScrubWorkerCommand::Start => {
|
||||
self.work = match std::mem::take(&mut self.work) {
|
||||
ScrubWorkerState::Finished => {
|
||||
let iterator = BlockStoreIterator::new(&self.manager);
|
||||
ScrubWorkerState::Running(iterator)
|
||||
}
|
||||
work => {
|
||||
error!("Cannot start scrub worker: already running!");
|
||||
work
|
||||
}
|
||||
};
|
||||
}
|
||||
ScrubWorkerCommand::Pause(dur) => {
|
||||
self.work = match std::mem::take(&mut self.work) {
|
||||
ScrubWorkerState::Running(it) | ScrubWorkerState::Paused(it, _) => {
|
||||
ScrubWorkerState::Paused(it, now_msec() + dur.as_millis() as u64)
|
||||
}
|
||||
work => {
|
||||
error!("Cannot pause scrub worker: not running!");
|
||||
work
|
||||
}
|
||||
};
|
||||
}
|
||||
ScrubWorkerCommand::Resume => {
|
||||
self.work = match std::mem::take(&mut self.work) {
|
||||
ScrubWorkerState::Paused(it, _) => ScrubWorkerState::Running(it),
|
||||
work => {
|
||||
error!("Cannot resume scrub worker: not paused!");
|
||||
work
|
||||
}
|
||||
};
|
||||
}
|
||||
ScrubWorkerCommand::Cancel => {
|
||||
self.work = match std::mem::take(&mut self.work) {
|
||||
ScrubWorkerState::Running(_) | ScrubWorkerState::Paused(_, _) => {
|
||||
ScrubWorkerState::Finished
|
||||
}
|
||||
work => {
|
||||
error!("Cannot cancel scrub worker: not running!");
|
||||
work
|
||||
}
|
||||
}
|
||||
}
|
||||
ScrubWorkerCommand::SetTranquility(t) => {
|
||||
self.persisted.tranquility = t;
|
||||
if let Err(e) = self.persister.save_async(&self.persisted).await {
|
||||
error!("Could not save new tranquilitiy value: {}", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl Worker for ScrubWorker {
|
||||
fn name(&self) -> String {
|
||||
"Block scrub worker".into()
|
||||
}
|
||||
|
||||
fn info(&self) -> Option<String> {
|
||||
let s = match &self.work {
|
||||
ScrubWorkerState::Running(bsi) => format!(
|
||||
"{:.2}% done (tranquility = {})",
|
||||
bsi.progress() * 100.,
|
||||
self.persisted.tranquility
|
||||
),
|
||||
ScrubWorkerState::Paused(bsi, rt) => {
|
||||
format!(
|
||||
"Paused, {:.2}% done, resumes at {}",
|
||||
bsi.progress() * 100.,
|
||||
msec_to_rfc3339(*rt)
|
||||
)
|
||||
}
|
||||
ScrubWorkerState::Finished => format!(
|
||||
"Last completed scrub: {}",
|
||||
msec_to_rfc3339(self.persisted.time_last_complete_scrub)
|
||||
),
|
||||
};
|
||||
Some(format!(
|
||||
"{} ; corruptions detected: {}",
|
||||
s, self.persisted.corruptions_detected
|
||||
))
|
||||
}
|
||||
|
||||
async fn work(&mut self, _must_exit: &mut watch::Receiver<bool>) -> Result<WorkerState, Error> {
|
||||
match self.rx_cmd.try_recv() {
|
||||
Ok(cmd) => self.handle_cmd(cmd).await,
|
||||
Err(mpsc::error::TryRecvError::Disconnected) => return Ok(WorkerState::Done),
|
||||
Err(mpsc::error::TryRecvError::Empty) => (),
|
||||
};
|
||||
|
||||
match &mut self.work {
|
||||
ScrubWorkerState::Running(bsi) => {
|
||||
self.tranquilizer.reset();
|
||||
if let Some(hash) = bsi.next().await? {
|
||||
match self.manager.read_block(&hash).await {
|
||||
Err(Error::CorruptData(_)) => {
|
||||
error!("Found corrupt data block during scrub: {:?}", hash);
|
||||
self.persisted.corruptions_detected += 1;
|
||||
self.persister.save_async(&self.persisted).await?;
|
||||
}
|
||||
Err(e) => return Err(e),
|
||||
_ => (),
|
||||
};
|
||||
Ok(self
|
||||
.tranquilizer
|
||||
.tranquilize_worker(self.persisted.tranquility))
|
||||
} else {
|
||||
self.persisted.time_last_complete_scrub = now_msec();
|
||||
self.persister.save_async(&self.persisted).await?;
|
||||
self.work = ScrubWorkerState::Finished;
|
||||
self.tranquilizer.clear();
|
||||
Ok(WorkerState::Idle)
|
||||
}
|
||||
}
|
||||
_ => Ok(WorkerState::Idle),
|
||||
}
|
||||
}
|
||||
|
||||
async fn wait_for_work(&mut self, _must_exit: &watch::Receiver<bool>) -> WorkerState {
|
||||
let (wait_until, command) = match &self.work {
|
||||
ScrubWorkerState::Running(_) => return WorkerState::Busy,
|
||||
ScrubWorkerState::Paused(_, resume_time) => (*resume_time, ScrubWorkerCommand::Resume),
|
||||
ScrubWorkerState::Finished => (
|
||||
self.persisted.time_last_complete_scrub + SCRUB_INTERVAL.as_millis() as u64,
|
||||
ScrubWorkerCommand::Start,
|
||||
),
|
||||
};
|
||||
|
||||
let now = now_msec();
|
||||
if now >= wait_until {
|
||||
self.handle_cmd(command).await;
|
||||
return WorkerState::Busy;
|
||||
}
|
||||
let delay = Duration::from_millis(wait_until - now);
|
||||
select! {
|
||||
_ = tokio::time::sleep(delay) => self.handle_cmd(command).await,
|
||||
cmd = self.rx_cmd.recv() => if let Some(cmd) = cmd {
|
||||
self.handle_cmd(cmd).await;
|
||||
} else {
|
||||
return WorkerState::Done;
|
||||
}
|
||||
}
|
||||
|
||||
match &self.work {
|
||||
ScrubWorkerState::Running(_) => WorkerState::Busy,
|
||||
_ => WorkerState::Idle,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ---- ---- ----
|
||||
// UTILITY FOR ENUMERATING THE BLOCK STORE
|
||||
// ---- ---- ----
|
||||
|
||||
struct BlockStoreIterator {
|
||||
path: Vec<ReadingDir>,
|
||||
}
|
||||
|
||||
enum ReadingDir {
|
||||
Pending(PathBuf),
|
||||
Read {
|
||||
subpaths: Vec<fs::DirEntry>,
|
||||
pos: usize,
|
||||
},
|
||||
}
|
||||
|
||||
impl BlockStoreIterator {
|
||||
fn new(manager: &BlockManager) -> Self {
|
||||
let root_dir = manager.data_dir.clone();
|
||||
Self {
|
||||
path: vec![ReadingDir::Pending(root_dir)],
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns progress done, between 0 and 1
|
||||
fn progress(&self) -> f32 {
|
||||
if self.path.is_empty() {
|
||||
1.0
|
||||
} else {
|
||||
let mut ret = 0.0;
|
||||
let mut next_div = 1;
|
||||
for p in self.path.iter() {
|
||||
match p {
|
||||
ReadingDir::Pending(_) => break,
|
||||
ReadingDir::Read { subpaths, pos } => {
|
||||
next_div *= subpaths.len();
|
||||
ret += ((*pos - 1) as f32) / (next_div as f32);
|
||||
}
|
||||
}
|
||||
}
|
||||
ret
|
||||
}
|
||||
}
|
||||
|
||||
async fn next(&mut self) -> Result<Option<Hash>, Error> {
|
||||
loop {
|
||||
let last_path = match self.path.last_mut() {
|
||||
None => return Ok(None),
|
||||
Some(lp) => lp,
|
||||
};
|
||||
|
||||
if let ReadingDir::Pending(path) = last_path {
|
||||
let mut reader = fs::read_dir(&path).await?;
|
||||
let mut subpaths = vec![];
|
||||
while let Some(ent) = reader.next_entry().await? {
|
||||
subpaths.push(ent);
|
||||
}
|
||||
*last_path = ReadingDir::Read { subpaths, pos: 0 };
|
||||
}
|
||||
|
||||
let (subpaths, pos) = match *last_path {
|
||||
ReadingDir::Read {
|
||||
ref subpaths,
|
||||
ref mut pos,
|
||||
} => (subpaths, pos),
|
||||
ReadingDir::Pending(_) => unreachable!(),
|
||||
};
|
||||
|
||||
let data_dir_ent = match subpaths.get(*pos) {
|
||||
None => {
|
||||
self.path.pop();
|
||||
continue;
|
||||
}
|
||||
Some(ent) => {
|
||||
*pos += 1;
|
||||
ent
|
||||
}
|
||||
};
|
||||
|
||||
let name = data_dir_ent.file_name();
|
||||
let name = if let Ok(n) = name.into_string() {
|
||||
n
|
||||
} else {
|
||||
continue;
|
||||
};
|
||||
let ent_type = data_dir_ent.file_type().await?;
|
||||
|
||||
let name = name.strip_suffix(".zst").unwrap_or(&name);
|
||||
if name.len() == 2 && hex::decode(&name).is_ok() && ent_type.is_dir() {
|
||||
let path = data_dir_ent.path();
|
||||
self.path.push(ReadingDir::Pending(path));
|
||||
} else if name.len() == 64 {
|
||||
if let Ok(h) = hex::decode(&name) {
|
||||
let mut hash = [0u8; 32];
|
||||
hash.copy_from_slice(&h);
|
||||
return Ok(Some(hash.into()));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,589 +0,0 @@
|
||||
use std::collections::HashSet;
|
||||
use std::convert::TryInto;
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::time::Duration;
|
||||
|
||||
use arc_swap::ArcSwap;
|
||||
use async_trait::async_trait;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use tokio::select;
|
||||
use tokio::sync::{watch, Notify};
|
||||
|
||||
use opentelemetry::{
|
||||
trace::{FutureExt as OtelFutureExt, TraceContextExt, Tracer},
|
||||
Context, KeyValue,
|
||||
};
|
||||
|
||||
use garage_db as db;
|
||||
use garage_db::counted_tree_hack::CountedTree;
|
||||
|
||||
use garage_util::background::*;
|
||||
use garage_util::data::*;
|
||||
use garage_util::error::*;
|
||||
use garage_util::metrics::RecordDuration;
|
||||
use garage_util::persister::Persister;
|
||||
use garage_util::time::*;
|
||||
use garage_util::tranquilizer::Tranquilizer;
|
||||
|
||||
use garage_rpc::system::System;
|
||||
use garage_rpc::*;
|
||||
|
||||
use garage_table::replication::TableReplication;
|
||||
|
||||
use crate::manager::*;
|
||||
|
||||
// The delay between the time where a resync operation fails
|
||||
// and the time when it is retried, with exponential backoff
|
||||
// (multiplied by 2, 4, 8, 16, etc. for every consecutive failure).
|
||||
pub(crate) const RESYNC_RETRY_DELAY: Duration = Duration::from_secs(60);
|
||||
// The minimum retry delay is 60 seconds = 1 minute
|
||||
// The maximum retry delay is 60 seconds * 2^6 = 60 seconds << 6 = 64 minutes (~1 hour)
|
||||
pub(crate) const RESYNC_RETRY_DELAY_MAX_BACKOFF_POWER: u64 = 6;
|
||||
|
||||
// No more than 4 resync workers can be running in the system
|
||||
pub(crate) const MAX_RESYNC_WORKERS: usize = 4;
|
||||
// Resync tranquility is initially set to 2, but can be changed in the CLI
|
||||
// and the updated version is persisted over Garage restarts
|
||||
const INITIAL_RESYNC_TRANQUILITY: u32 = 2;
|
||||
|
||||
pub struct BlockResyncManager {
|
||||
pub(crate) queue: CountedTree,
|
||||
pub(crate) notify: Notify,
|
||||
pub(crate) errors: CountedTree,
|
||||
|
||||
busy_set: BusySet,
|
||||
|
||||
persister: Persister<ResyncPersistedConfig>,
|
||||
persisted: ArcSwap<ResyncPersistedConfig>,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Clone, Copy)]
|
||||
struct ResyncPersistedConfig {
|
||||
n_workers: usize,
|
||||
tranquility: u32,
|
||||
}
|
||||
|
||||
enum ResyncIterResult {
|
||||
BusyDidSomething,
|
||||
BusyDidNothing,
|
||||
IdleFor(Duration),
|
||||
}
|
||||
|
||||
type BusySet = Arc<Mutex<HashSet<Vec<u8>>>>;
|
||||
|
||||
struct BusyBlock {
|
||||
time_bytes: Vec<u8>,
|
||||
hash_bytes: Vec<u8>,
|
||||
busy_set: BusySet,
|
||||
}
|
||||
|
||||
impl BlockResyncManager {
|
||||
pub(crate) fn new(db: &db::Db, system: &System) -> Self {
|
||||
let queue = db
|
||||
.open_tree("block_local_resync_queue")
|
||||
.expect("Unable to open block_local_resync_queue tree");
|
||||
let queue = CountedTree::new(queue).expect("Could not count block_local_resync_queue");
|
||||
|
||||
let errors = db
|
||||
.open_tree("block_local_resync_errors")
|
||||
.expect("Unable to open block_local_resync_errors tree");
|
||||
let errors = CountedTree::new(errors).expect("Could not count block_local_resync_errors");
|
||||
|
||||
let persister = Persister::new(&system.metadata_dir, "resync_cfg");
|
||||
let persisted = match persister.load() {
|
||||
Ok(v) => v,
|
||||
Err(_) => ResyncPersistedConfig {
|
||||
n_workers: 1,
|
||||
tranquility: INITIAL_RESYNC_TRANQUILITY,
|
||||
},
|
||||
};
|
||||
|
||||
Self {
|
||||
queue,
|
||||
notify: Notify::new(),
|
||||
errors,
|
||||
busy_set: Arc::new(Mutex::new(HashSet::new())),
|
||||
persister,
|
||||
persisted: ArcSwap::new(Arc::new(persisted)),
|
||||
}
|
||||
}
|
||||
|
||||
/// Get lenght of resync queue
|
||||
pub fn queue_len(&self) -> Result<usize, Error> {
|
||||
// This currently can't return an error because the CountedTree hack
|
||||
// doesn't error on .len(), but this will change when we remove the hack
|
||||
// (hopefully someday!)
|
||||
Ok(self.queue.len())
|
||||
}
|
||||
|
||||
/// Get number of blocks that have an error
|
||||
pub fn errors_len(&self) -> Result<usize, Error> {
|
||||
// (see queue_len comment)
|
||||
Ok(self.errors.len())
|
||||
}
|
||||
|
||||
// ---- Resync loop ----
|
||||
|
||||
// This part manages a queue of blocks that need to be
|
||||
// "resynchronized", i.e. that need to have a check that
|
||||
// they are at present if we need them, or that they are
|
||||
// deleted once the garbage collection delay has passed.
|
||||
//
|
||||
// Here are some explanations on how the resync queue works.
|
||||
// There are two Sled trees that are used to have information
|
||||
// about the status of blocks that need to be resynchronized:
|
||||
//
|
||||
// - resync.queue: a tree that is ordered first by a timestamp
|
||||
// (in milliseconds since Unix epoch) that is the time at which
|
||||
// the resync must be done, and second by block hash.
|
||||
// The key in this tree is just:
|
||||
// concat(timestamp (8 bytes), hash (32 bytes))
|
||||
// The value is the same 32-byte hash.
|
||||
//
|
||||
// - resync.errors: a tree that indicates for each block
|
||||
// if the last resync resulted in an error, and if so,
|
||||
// the following two informations (see the ErrorCounter struct):
|
||||
// - how many consecutive resync errors for this block?
|
||||
// - when was the last try?
|
||||
// These two informations are used to implement an
|
||||
// exponential backoff retry strategy.
|
||||
// The key in this tree is the 32-byte hash of the block,
|
||||
// and the value is the encoded ErrorCounter value.
|
||||
//
|
||||
// We need to have these two trees, because the resync queue
|
||||
// is not just a queue of items to process, but a set of items
|
||||
// that are waiting a specific delay until we can process them
|
||||
// (the delay being necessary both internally for the exponential
|
||||
// backoff strategy, and exposed as a parameter when adding items
|
||||
// to the queue, e.g. to wait until the GC delay has passed).
|
||||
// This is why we need one tree ordered by time, and one
|
||||
// ordered by identifier of item to be processed (block hash).
|
||||
//
|
||||
// When the worker wants to process an item it takes from
|
||||
// resync.queue, it checks in resync.errors that if there is an
|
||||
// exponential back-off delay to await, it has passed before we
|
||||
// process the item. If not, the item in the queue is skipped
|
||||
// (but added back for later processing after the time of the
|
||||
// delay).
|
||||
//
|
||||
// An alternative that would have seemed natural is to
|
||||
// only add items to resync.queue with a processing time that is
|
||||
// after the delay, but there are several issues with this:
|
||||
// - This requires to synchronize updates to resync.queue and
|
||||
// resync.errors (with the current model, there is only one thread,
|
||||
// the worker thread, that accesses resync.errors,
|
||||
// so no need to synchronize) by putting them both in a lock.
|
||||
// This would mean that block_incref might need to take a lock
|
||||
// before doing its thing, meaning it has much more chances of
|
||||
// not completing successfully if something bad happens to Garage.
|
||||
// Currently Garage is not able to recover from block_incref that
|
||||
// doesn't complete successfully, because it is necessary to ensure
|
||||
// the consistency between the state of the block manager and
|
||||
// information in the BlockRef table.
|
||||
// - If a resync fails, we put that block in the resync.errors table,
|
||||
// and also add it back to resync.queue to be processed after
|
||||
// the exponential back-off delay,
|
||||
// but maybe the block is already scheduled to be resynced again
|
||||
// at another time that is before the exponential back-off delay,
|
||||
// and we have no way to check that easily. This means that
|
||||
// in all cases, we need to check the resync.errors table
|
||||
// in the resync loop at the time when a block is popped from
|
||||
// the resync.queue.
|
||||
// Overall, the current design is therefore simpler and more robust
|
||||
// because it tolerates inconsistencies between the resync.queue
|
||||
// and resync.errors table (items being scheduled in resync.queue
|
||||
// for times that are earlier than the exponential back-off delay
|
||||
// is a natural condition that is handled properly).
|
||||
|
||||
pub(crate) fn put_to_resync(&self, hash: &Hash, delay: Duration) -> db::Result<()> {
|
||||
let when = now_msec() + delay.as_millis() as u64;
|
||||
self.put_to_resync_at(hash, when)
|
||||
}
|
||||
|
||||
pub(crate) fn put_to_resync_at(&self, hash: &Hash, when: u64) -> db::Result<()> {
|
||||
trace!("Put resync_queue: {} {:?}", when, hash);
|
||||
let mut key = u64::to_be_bytes(when).to_vec();
|
||||
key.extend(hash.as_ref());
|
||||
self.queue.insert(key, hash.as_ref())?;
|
||||
self.notify.notify_waiters();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn resync_iter(&self, manager: &BlockManager) -> Result<ResyncIterResult, db::Error> {
|
||||
if let Some(block) = self.get_block_to_resync()? {
|
||||
let time_msec = u64::from_be_bytes(block.time_bytes[0..8].try_into().unwrap());
|
||||
let now = now_msec();
|
||||
|
||||
if now >= time_msec {
|
||||
let hash = Hash::try_from(&block.hash_bytes[..]).unwrap();
|
||||
|
||||
if let Some(ec) = self.errors.get(hash.as_slice())? {
|
||||
let ec = ErrorCounter::decode(&ec);
|
||||
if now < ec.next_try() {
|
||||
// if next retry after an error is not yet,
|
||||
// don't do resync and return early, but still
|
||||
// make sure the item is still in queue at expected time
|
||||
self.put_to_resync_at(&hash, ec.next_try())?;
|
||||
// ec.next_try() > now >= time_msec, so this remove
|
||||
// is not removing the one we added just above
|
||||
// (we want to do the remove after the insert to ensure
|
||||
// that the item is not lost if we crash in-between)
|
||||
self.queue.remove(&block.time_bytes)?;
|
||||
return Ok(ResyncIterResult::BusyDidNothing);
|
||||
}
|
||||
}
|
||||
|
||||
let tracer = opentelemetry::global::tracer("garage");
|
||||
let trace_id = gen_uuid();
|
||||
let span = tracer
|
||||
.span_builder("Resync block")
|
||||
.with_trace_id(
|
||||
opentelemetry::trace::TraceId::from_hex(&hex::encode(
|
||||
&trace_id.as_slice()[..16],
|
||||
))
|
||||
.unwrap(),
|
||||
)
|
||||
.with_attributes(vec![KeyValue::new("block", format!("{:?}", hash))])
|
||||
.start(&tracer);
|
||||
|
||||
let res = self
|
||||
.resync_block(manager, &hash)
|
||||
.with_context(Context::current_with_span(span))
|
||||
.bound_record_duration(&manager.metrics.resync_duration)
|
||||
.await;
|
||||
|
||||
manager.metrics.resync_counter.add(1);
|
||||
|
||||
if let Err(e) = &res {
|
||||
manager.metrics.resync_error_counter.add(1);
|
||||
warn!("Error when resyncing {:?}: {}", hash, e);
|
||||
|
||||
let err_counter = match self.errors.get(hash.as_slice())? {
|
||||
Some(ec) => ErrorCounter::decode(&ec).add1(now + 1),
|
||||
None => ErrorCounter::new(now + 1),
|
||||
};
|
||||
|
||||
self.errors.insert(hash.as_slice(), err_counter.encode())?;
|
||||
|
||||
self.put_to_resync_at(&hash, err_counter.next_try())?;
|
||||
// err_counter.next_try() >= now + 1 > now,
|
||||
// the entry we remove from the queue is not
|
||||
// the entry we inserted with put_to_resync_at
|
||||
self.queue.remove(&block.time_bytes)?;
|
||||
} else {
|
||||
self.errors.remove(hash.as_slice())?;
|
||||
self.queue.remove(&block.time_bytes)?;
|
||||
}
|
||||
|
||||
Ok(ResyncIterResult::BusyDidSomething)
|
||||
} else {
|
||||
Ok(ResyncIterResult::IdleFor(Duration::from_millis(
|
||||
time_msec - now,
|
||||
)))
|
||||
}
|
||||
} else {
|
||||
// Here we wait either for a notification that an item has been
|
||||
// added to the queue, or for a constant delay of 10 secs to expire.
|
||||
// The delay avoids a race condition where the notification happens
|
||||
// between the time we checked the queue and the first poll
|
||||
// to resync_notify.notified(): if that happens, we'll just loop
|
||||
// back 10 seconds later, which is fine.
|
||||
Ok(ResyncIterResult::IdleFor(Duration::from_secs(10)))
|
||||
}
|
||||
}
|
||||
|
||||
fn get_block_to_resync(&self) -> Result<Option<BusyBlock>, db::Error> {
|
||||
let mut busy = self.busy_set.lock().unwrap();
|
||||
for it in self.queue.iter()? {
|
||||
let (time_bytes, hash_bytes) = it?;
|
||||
if !busy.contains(&time_bytes) {
|
||||
busy.insert(time_bytes.clone());
|
||||
return Ok(Some(BusyBlock {
|
||||
time_bytes,
|
||||
hash_bytes,
|
||||
busy_set: self.busy_set.clone(),
|
||||
}));
|
||||
}
|
||||
}
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
async fn resync_block(&self, manager: &BlockManager, hash: &Hash) -> Result<(), Error> {
|
||||
let BlockStatus { exists, needed } = manager.check_block_status(hash).await?;
|
||||
|
||||
if exists != needed.is_needed() || exists != needed.is_nonzero() {
|
||||
debug!(
|
||||
"Resync block {:?}: exists {}, nonzero rc {}, deletable {}",
|
||||
hash,
|
||||
exists,
|
||||
needed.is_nonzero(),
|
||||
needed.is_deletable(),
|
||||
);
|
||||
}
|
||||
|
||||
if exists && needed.is_deletable() {
|
||||
info!("Resync block {:?}: offloading and deleting", hash);
|
||||
|
||||
let mut who = manager.replication.write_nodes(hash);
|
||||
if who.len() < manager.replication.write_quorum() {
|
||||
return Err(Error::Message("Not trying to offload block because we don't have a quorum of nodes to write to".to_string()));
|
||||
}
|
||||
who.retain(|id| *id != manager.system.id);
|
||||
|
||||
let who_needs_resps = manager
|
||||
.system
|
||||
.rpc
|
||||
.call_many(
|
||||
&manager.endpoint,
|
||||
&who,
|
||||
BlockRpc::NeedBlockQuery(*hash),
|
||||
RequestStrategy::with_priority(PRIO_BACKGROUND),
|
||||
)
|
||||
.await?;
|
||||
|
||||
let mut need_nodes = vec![];
|
||||
for (node, needed) in who_needs_resps {
|
||||
match needed.err_context("NeedBlockQuery RPC")? {
|
||||
BlockRpc::NeedBlockReply(needed) => {
|
||||
if needed {
|
||||
need_nodes.push(node);
|
||||
}
|
||||
}
|
||||
m => {
|
||||
return Err(Error::unexpected_rpc_message(m));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if !need_nodes.is_empty() {
|
||||
trace!(
|
||||
"Block {:?} needed by {} nodes, sending",
|
||||
hash,
|
||||
need_nodes.len()
|
||||
);
|
||||
|
||||
for node in need_nodes.iter() {
|
||||
manager
|
||||
.metrics
|
||||
.resync_send_counter
|
||||
.add(1, &[KeyValue::new("to", format!("{:?}", node))]);
|
||||
}
|
||||
|
||||
let block = manager.read_block(hash).await?;
|
||||
let (header, bytes) = block.into_parts();
|
||||
let put_block_message = Req::new(BlockRpc::PutBlock {
|
||||
hash: *hash,
|
||||
header,
|
||||
})?
|
||||
.with_stream_from_buffer(bytes);
|
||||
manager
|
||||
.system
|
||||
.rpc
|
||||
.try_call_many(
|
||||
&manager.endpoint,
|
||||
&need_nodes[..],
|
||||
put_block_message,
|
||||
RequestStrategy::with_priority(PRIO_BACKGROUND)
|
||||
.with_quorum(need_nodes.len()),
|
||||
)
|
||||
.await
|
||||
.err_context("PutBlock RPC")?;
|
||||
}
|
||||
info!(
|
||||
"Deleting unneeded block {:?}, offload finished ({} / {})",
|
||||
hash,
|
||||
need_nodes.len(),
|
||||
who.len()
|
||||
);
|
||||
|
||||
manager.delete_if_unneeded(hash).await?;
|
||||
|
||||
manager.rc.clear_deleted_block_rc(hash)?;
|
||||
}
|
||||
|
||||
if needed.is_nonzero() && !exists {
|
||||
info!(
|
||||
"Resync block {:?}: fetching absent but needed block (refcount > 0)",
|
||||
hash
|
||||
);
|
||||
|
||||
let block_data = manager.rpc_get_raw_block(hash, None).await?;
|
||||
|
||||
manager.metrics.resync_recv_counter.add(1);
|
||||
|
||||
manager.write_block(hash, &block_data).await?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn update_persisted(
|
||||
&self,
|
||||
update: impl Fn(&mut ResyncPersistedConfig),
|
||||
) -> Result<(), Error> {
|
||||
let mut cfg: ResyncPersistedConfig = *self.persisted.load().as_ref();
|
||||
update(&mut cfg);
|
||||
self.persister.save_async(&cfg).await?;
|
||||
self.persisted.store(Arc::new(cfg));
|
||||
self.notify.notify_waiters();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn set_n_workers(&self, n_workers: usize) -> Result<(), Error> {
|
||||
if !(1..=MAX_RESYNC_WORKERS).contains(&n_workers) {
|
||||
return Err(Error::Message(format!(
|
||||
"Invalid number of resync workers, must be between 1 and {}",
|
||||
MAX_RESYNC_WORKERS
|
||||
)));
|
||||
}
|
||||
self.update_persisted(|cfg| cfg.n_workers = n_workers).await
|
||||
}
|
||||
|
||||
pub async fn set_tranquility(&self, tranquility: u32) -> Result<(), Error> {
|
||||
self.update_persisted(|cfg| cfg.tranquility = tranquility)
|
||||
.await
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for BusyBlock {
|
||||
fn drop(&mut self) {
|
||||
let mut busy = self.busy_set.lock().unwrap();
|
||||
busy.remove(&self.time_bytes);
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) struct ResyncWorker {
|
||||
index: usize,
|
||||
manager: Arc<BlockManager>,
|
||||
tranquilizer: Tranquilizer,
|
||||
next_delay: Duration,
|
||||
}
|
||||
|
||||
impl ResyncWorker {
|
||||
pub(crate) fn new(index: usize, manager: Arc<BlockManager>) -> Self {
|
||||
Self {
|
||||
index,
|
||||
manager,
|
||||
tranquilizer: Tranquilizer::new(30),
|
||||
next_delay: Duration::from_secs(10),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl Worker for ResyncWorker {
|
||||
fn name(&self) -> String {
|
||||
format!("Block resync worker #{}", self.index + 1)
|
||||
}
|
||||
|
||||
fn info(&self) -> Option<String> {
|
||||
let persisted = self.manager.resync.persisted.load();
|
||||
|
||||
if self.index >= persisted.n_workers {
|
||||
return Some("(unused)".into());
|
||||
}
|
||||
|
||||
let mut ret = vec![];
|
||||
ret.push(format!("tranquility = {}", persisted.tranquility));
|
||||
|
||||
let qlen = self.manager.resync.queue_len().unwrap_or(0);
|
||||
if qlen > 0 {
|
||||
ret.push(format!("{} blocks in queue", qlen));
|
||||
}
|
||||
|
||||
let elen = self.manager.resync.errors_len().unwrap_or(0);
|
||||
if elen > 0 {
|
||||
ret.push(format!("{} blocks in error state", elen));
|
||||
}
|
||||
|
||||
Some(ret.join(", "))
|
||||
}
|
||||
|
||||
async fn work(&mut self, _must_exit: &mut watch::Receiver<bool>) -> Result<WorkerState, Error> {
|
||||
if self.index >= self.manager.resync.persisted.load().n_workers {
|
||||
return Ok(WorkerState::Idle);
|
||||
}
|
||||
|
||||
self.tranquilizer.reset();
|
||||
match self.manager.resync.resync_iter(&self.manager).await {
|
||||
Ok(ResyncIterResult::BusyDidSomething) => Ok(self
|
||||
.tranquilizer
|
||||
.tranquilize_worker(self.manager.resync.persisted.load().tranquility)),
|
||||
Ok(ResyncIterResult::BusyDidNothing) => Ok(WorkerState::Busy),
|
||||
Ok(ResyncIterResult::IdleFor(delay)) => {
|
||||
self.next_delay = delay;
|
||||
Ok(WorkerState::Idle)
|
||||
}
|
||||
Err(e) => {
|
||||
// The errors that we have here are only Sled errors
|
||||
// We don't really know how to handle them so just ¯\_(ツ)_/¯
|
||||
// (there is kind of an assumption that Sled won't error on us,
|
||||
// if it does there is not much we can do -- TODO should we just panic?)
|
||||
// Here we just give the error to the worker manager,
|
||||
// it will print it to the logs and increment a counter
|
||||
Err(e.into())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn wait_for_work(&mut self, _must_exit: &watch::Receiver<bool>) -> WorkerState {
|
||||
while self.index >= self.manager.resync.persisted.load().n_workers {
|
||||
self.manager.resync.notify.notified().await
|
||||
}
|
||||
|
||||
select! {
|
||||
_ = tokio::time::sleep(self.next_delay) => (),
|
||||
_ = self.manager.resync.notify.notified() => (),
|
||||
};
|
||||
|
||||
WorkerState::Busy
|
||||
}
|
||||
}
|
||||
|
||||
/// Counts the number of errors when resyncing a block,
|
||||
/// and the time of the last try.
|
||||
/// Used to implement exponential backoff.
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
struct ErrorCounter {
|
||||
errors: u64,
|
||||
last_try: u64,
|
||||
}
|
||||
|
||||
impl ErrorCounter {
|
||||
fn new(now: u64) -> Self {
|
||||
Self {
|
||||
errors: 1,
|
||||
last_try: now,
|
||||
}
|
||||
}
|
||||
|
||||
fn decode(data: &[u8]) -> Self {
|
||||
Self {
|
||||
errors: u64::from_be_bytes(data[0..8].try_into().unwrap()),
|
||||
last_try: u64::from_be_bytes(data[8..16].try_into().unwrap()),
|
||||
}
|
||||
}
|
||||
fn encode(&self) -> Vec<u8> {
|
||||
[
|
||||
u64::to_be_bytes(self.errors),
|
||||
u64::to_be_bytes(self.last_try),
|
||||
]
|
||||
.concat()
|
||||
}
|
||||
|
||||
fn add1(self, now: u64) -> Self {
|
||||
Self {
|
||||
errors: self.errors + 1,
|
||||
last_try: now,
|
||||
}
|
||||
}
|
||||
|
||||
fn delay_msec(&self) -> u64 {
|
||||
(RESYNC_RETRY_DELAY.as_millis() as u64)
|
||||
<< std::cmp::min(self.errors - 1, RESYNC_RETRY_DELAY_MAX_BACKOFF_POWER)
|
||||
}
|
||||
fn next_try(&self) -> u64 {
|
||||
self.last_try + self.delay_msec()
|
||||
}
|
||||
}
|
||||
@@ -1,39 +0,0 @@
|
||||
[package]
|
||||
name = "garage_db"
|
||||
version = "0.8.0"
|
||||
authors = ["Alex Auvolat <alex@adnab.me>"]
|
||||
edition = "2018"
|
||||
license = "AGPL-3.0"
|
||||
description = "Abstraction over multiple key/value storage engines that supports transactions"
|
||||
repository = "https://git.deuxfleurs.fr/Deuxfleurs/garage"
|
||||
readme = "../../README.md"
|
||||
|
||||
[lib]
|
||||
path = "lib.rs"
|
||||
|
||||
[[bin]]
|
||||
name = "convert"
|
||||
path = "bin/convert.rs"
|
||||
required-features = ["cli"]
|
||||
|
||||
[dependencies]
|
||||
err-derive = "0.3"
|
||||
hexdump = "0.1"
|
||||
tracing = "0.1.30"
|
||||
|
||||
heed = { version = "0.11", default-features = false, features = ["lmdb"], optional = true }
|
||||
rusqlite = { version = "0.27", optional = true }
|
||||
sled = { version = "0.34", optional = true }
|
||||
|
||||
# cli deps
|
||||
clap = { version = "3.1.18", optional = true, features = ["derive", "env"] }
|
||||
pretty_env_logger = { version = "0.4", optional = true }
|
||||
|
||||
[dev-dependencies]
|
||||
mktemp = "0.4"
|
||||
|
||||
[features]
|
||||
bundled-libs = [ "rusqlite/bundled" ]
|
||||
cli = ["clap", "pretty_env_logger"]
|
||||
lmdb = [ "heed" ]
|
||||
sqlite = [ "rusqlite" ]
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user