Compare commits

..

20 Commits

Author SHA1 Message Date
Alex Auvolat 935670690f Probably fix test-smoke 2022-02-02 17:34:19 +01:00
Alex Auvolat ae2f32baf1 Hide deleted key in bucket info (fix #211) 2022-02-02 17:12:48 +01:00
Quentin Dufour f67029ce2a Improve testing conf + test CORS 2022-02-01 17:55:14 +01:00
Alex Auvolat 2760f1cb17 Add advice about --fast-list 2022-01-31 16:51:39 +01:00
Alex Auvolat 26849ed066 Add step to 0.6.0 migration guide 2022-01-27 14:31:25 +01:00
Alex Auvolat c99f55c420 Add restriction on part ordering in CompleteMultipartUpload 2022-01-25 12:45:00 +01:00
Alex Auvolat acdf893362 Fix partnumber 2022-01-25 12:25:23 +01:00
Alex Auvolat 338b1b83ee Implement part_number for GetObject 2022-01-24 21:04:42 +01:00
Alex Auvolat 6dab836f3a Multipart improvements
- support part_number for HeadObject
- add checks in complete_multipart_upload
2022-01-24 21:04:40 +01:00
Alex Auvolat 513a6b15f9 Handle OPTIONS on website endpoint 2022-01-24 12:32:28 +01:00
Alex Auvolat ea7fb901eb Implement {Put,Get,Delete}BucketCors and CORS in general
- OPTIONS request against API endpoint
- Returning corresponding CORS headers on API calls
- Returning corresponding CORS headers on website GET's
2022-01-24 11:58:00 +01:00
Trinity Pointard 820924534a use clamp instead of min(max()) 2022-01-24 11:56:59 +01:00
Quentin Dufour 94f0e7c135 Test ListParts endpoint with awscli 2022-01-21 10:42:35 +01:00
Quentin Dufour 440374524b Implement ListParts 2022-01-21 10:42:30 +01:00
Quentin Dufour fe003d6fbc Add ListPartsResult structure 2022-01-20 16:38:55 +01:00
trinity-1686a e55fa38c99 Add date verification to presigned urls (#196)
fix #96
fix #162 by returning Forbidden instead Bad Request

Co-authored-by: Trinity Pointard <trinity.pointard@gmail.com>
Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/196
Co-authored-by: trinity-1686a <trinity.pointard@gmail.com>
Co-committed-by: trinity-1686a <trinity.pointard@gmail.com>
2022-01-18 12:22:31 +01:00
trinity-1686a 178e35f868 refactor s3_router and api_server to make unused Endpoint parameters more obvious 2022-01-17 15:50:24 +01:00
Alex Auvolat 7c049f1c94 Fix extreme value to be less extreme so that integration test works on 32bits 2022-01-17 12:56:29 +01:00
Jill fdcddbe168 Fix Multipart Upload with WinSCP (#164) (#193)
Closes #164.

Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/193
Co-authored-by: Jill <kokakiwi@deuxfleurs.fr>
Co-committed-by: Jill <kokakiwi@deuxfleurs.fr>
2022-01-17 11:18:40 +01:00
Jill b45dcc1925 Support STREAMING-AWS4-HMAC-SHA256-PAYLOAD (#64) (#156)
Closes #64.

Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/156
Co-authored-by: Jill <kokakiwi@deuxfleurs.fr>
Co-committed-by: Jill <kokakiwi@deuxfleurs.fr>
2022-01-17 10:55:31 +01:00
38 changed files with 2105 additions and 934 deletions
Generated
+39
View File
@@ -433,7 +433,9 @@ dependencies = [
"idna",
"log",
"md-5",
"nom",
"percent-encoding",
"pin-project",
"quick-xml",
"roxmltree",
"serde",
@@ -967,6 +969,12 @@ dependencies = [
"autocfg",
]
[[package]]
name = "minimal-lexical"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a"
[[package]]
name = "mio"
version = "0.7.13"
@@ -1011,6 +1019,17 @@ dependencies = [
"tokio-util",
]
[[package]]
name = "nom"
version = "7.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1b1d11e1ef389c76fe5b81bcaf2ea32cf88b62bc494e19f493d0b30e7a930109"
dependencies = [
"memchr",
"minimal-lexical",
"version_check",
]
[[package]]
name = "ntapi"
version = "0.3.6"
@@ -1092,6 +1111,26 @@ version = "2.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e"
[[package]]
name = "pin-project"
version = "1.0.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "576bc800220cc65dac09e99e97b08b358cfab6e17078de8dc5fee223bd2d0c08"
dependencies = [
"pin-project-internal",
]
[[package]]
name = "pin-project-internal"
version = "1.0.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6e8fe8163d14ce7f0cdac2e040116f22eac817edabff0be91e8aff7e9accf389"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "pin-project-lite"
version = "0.2.7"
+53
View File
@@ -671,7 +671,9 @@ in
idna = rustPackages."registry+https://github.com/rust-lang/crates.io-index".idna."0.2.3" { inherit profileName; };
log = rustPackages."registry+https://github.com/rust-lang/crates.io-index".log."0.4.14" { inherit profileName; };
md5 = rustPackages."registry+https://github.com/rust-lang/crates.io-index".md-5."0.9.1" { inherit profileName; };
nom = rustPackages."registry+https://github.com/rust-lang/crates.io-index".nom."7.1.0" { inherit profileName; };
percent_encoding = rustPackages."registry+https://github.com/rust-lang/crates.io-index".percent-encoding."2.1.0" { inherit profileName; };
pin_project = rustPackages."registry+https://github.com/rust-lang/crates.io-index".pin-project."1.0.8" { inherit profileName; };
quick_xml = rustPackages."registry+https://github.com/rust-lang/crates.io-index".quick-xml."0.21.0" { inherit profileName; };
roxmltree = rustPackages."registry+https://github.com/rust-lang/crates.io-index".roxmltree."0.14.1" { inherit profileName; };
serde = rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde."1.0.130" { inherit profileName; };
@@ -1321,6 +1323,16 @@ in
};
});
"registry+https://github.com/rust-lang/crates.io-index".minimal-lexical."0.2.1" = overridableMkRustCrate (profileName: rec {
name = "minimal-lexical";
version = "0.2.1";
registry = "registry+https://github.com/rust-lang/crates.io-index";
src = fetchCratesIo { inherit name version; sha256 = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a"; };
features = builtins.concatLists [
[ "std" ]
];
});
"registry+https://github.com/rust-lang/crates.io-index".mio."0.7.13" = overridableMkRustCrate (profileName: rec {
name = "mio";
version = "0.7.13";
@@ -1381,6 +1393,25 @@ in
};
});
"registry+https://github.com/rust-lang/crates.io-index".nom."7.1.0" = overridableMkRustCrate (profileName: rec {
name = "nom";
version = "7.1.0";
registry = "registry+https://github.com/rust-lang/crates.io-index";
src = fetchCratesIo { inherit name version; sha256 = "1b1d11e1ef389c76fe5b81bcaf2ea32cf88b62bc494e19f493d0b30e7a930109"; };
features = builtins.concatLists [
[ "alloc" ]
[ "default" ]
[ "std" ]
];
dependencies = {
memchr = rustPackages."registry+https://github.com/rust-lang/crates.io-index".memchr."2.4.1" { inherit profileName; };
minimal_lexical = rustPackages."registry+https://github.com/rust-lang/crates.io-index".minimal-lexical."0.2.1" { inherit profileName; };
};
buildDependencies = {
version_check = buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".version_check."0.9.3" { profileName = "__noProfile"; };
};
});
"registry+https://github.com/rust-lang/crates.io-index".ntapi."0.3.6" = overridableMkRustCrate (profileName: rec {
name = "ntapi";
version = "0.3.6";
@@ -1490,6 +1521,28 @@ in
src = fetchCratesIo { inherit name version; sha256 = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e"; };
});
"registry+https://github.com/rust-lang/crates.io-index".pin-project."1.0.8" = overridableMkRustCrate (profileName: rec {
name = "pin-project";
version = "1.0.8";
registry = "registry+https://github.com/rust-lang/crates.io-index";
src = fetchCratesIo { inherit name version; sha256 = "576bc800220cc65dac09e99e97b08b358cfab6e17078de8dc5fee223bd2d0c08"; };
dependencies = {
pin_project_internal = buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".pin-project-internal."1.0.8" { profileName = "__noProfile"; };
};
});
"registry+https://github.com/rust-lang/crates.io-index".pin-project-internal."1.0.8" = overridableMkRustCrate (profileName: rec {
name = "pin-project-internal";
version = "1.0.8";
registry = "registry+https://github.com/rust-lang/crates.io-index";
src = fetchCratesIo { inherit name version; sha256 = "6e8fe8163d14ce7f0cdac2e040116f22eac817edabff0be91e8aff7e9accf389"; };
dependencies = {
proc_macro2 = rustPackages."registry+https://github.com/rust-lang/crates.io-index".proc-macro2."1.0.30" { inherit profileName; };
quote = rustPackages."registry+https://github.com/rust-lang/crates.io-index".quote."1.0.10" { inherit profileName; };
syn = rustPackages."registry+https://github.com/rust-lang/crates.io-index".syn."1.0.80" { inherit profileName; };
};
});
"registry+https://github.com/rust-lang/crates.io-index".pin-project-lite."0.2.7" = overridableMkRustCrate (profileName: rec {
name = "pin-project-lite";
version = "0.2.7";
+1 -1
View File
@@ -76,7 +76,7 @@ in let
*/
''^(src|tests)'' # fixed default
''.*\.(rs|toml)$'' # fixed default
''^(crdt|replication|cli|helper)'' # our crate submodules
''^(crdt|replication|cli|helper|signature)'' # our crate submodules
];
};
+1 -1
View File
@@ -14,10 +14,10 @@
- [Recovering from failures](./cookbook/recovering.md)
- [Integrations](./connect/index.md)
- [Browsing tools (awscli, mc...)](./connect/cli.md)
- [Apps (Nextcloud, Peertube...)](./connect/apps.md)
- [Websites (Hugo, Jekyll, Publii...)](./connect/websites.md)
- [Repositories (Docker, Nix, Git...)](./connect/repositories.md)
- [CLI tools (rclone, awscli, mc...)](./connect/cli.md)
- [Backups (restic, duplicity...)](./connect/backup.md)
- [Your code (PHP, JS, Go...)](./connect/code.md)
- [FUSE (s3fs, goofys, s3backer...)](./connect/fs.md)
+139 -48
View File
@@ -1,20 +1,6 @@
# Apps (Nextcloud, Peertube...)
In this section, we cover the following web applications:
| Name | Status | Note |
|------|--------|------|
| [Nextcloud](#nextcloud) | ✅ | Both Primary Storage and External Storage are supported |
| [Peertube](#peertube) | ✅ | Must be configured with the website endpoint |
| [Mastodon](#mastodon) | ❓ | Not yet tested |
| [Matrix](#matrix) | ✅ | Tested with `synapse-s3-storage-provider` |
| [Pixelfed](#pixelfed) | ❓ | Not yet tested |
| [Pleroma](#pleroma) | ❓ | Not yet tested |
| [Lemmy](#lemmy) | ❓ | Not yet tested |
| [Funkwhale](#funkwhale) | ❓ | Not yet tested |
| [Misskey](#misskey) | ❓ | Not yet tested |
| [Prismo](#prismo) | ❓ | Not yet tested |
| [Owncloud OCIS](#owncloud-infinite-scale-ocis) | ❓| Not yet tested |
In this section, we cover the following software: [Nextcloud](#nextcloud), [Peertube](#peertube), [Mastodon](#mastodon), [Matrix](#matrix)
## Nextcloud
@@ -122,8 +108,109 @@ Do not change the `use_path_style` and `legacy_auth` entries, other configuratio
Peertube proposes a clever integration of S3 by directly exposing its endpoint instead of proxifying requests through the application.
In other words, Peertube is only responsible of the "control plane" and offload the "data plane" to Garage.
In return, this system is a bit harder to configure.
We show how it is still possible to configure Garage with Peertube, allowing you to spread the load and the bandwidth usage on the Garage cluster.
In return, this system is a bit harder to configure, especially with Garage that supports less feature than other older S3 backends.
We show that it is still possible to configure Garage with Peertube, allowing you to spread the load and the bandwidth usage on the Garage cluster.
### Enable path-style access by patching Peertube
First, you will need to apply a small patch on Peertube ([#4510](https://github.com/Chocobozzz/PeerTube/pull/4510)):
```diff
From e3b4c641bdf67e07d406a1d49d6aa6b1fbce2ab4 Mon Sep 17 00:00:00 2001
From: Martin Honermeyer <maze@strahlungsfrei.de>
Date: Sun, 31 Oct 2021 12:34:04 +0100
Subject: [PATCH] Allow setting path-style access for object storage
---
config/default.yaml | 4 ++++
config/production.yaml.example | 4 ++++
server/initializers/config.ts | 1 +
server/lib/object-storage/shared/client.ts | 3 ++-
.../production/config/custom-environment-variables.yaml | 2 ++
5 files changed, 13 insertions(+), 1 deletion(-)
diff --git a/config/default.yaml b/config/default.yaml
index cf9d69a6211..4efd56fb804 100644
--- a/config/default.yaml
+++ b/config/default.yaml
@@ -123,6 +123,10 @@ object_storage:
# You can also use AWS_SECRET_ACCESS_KEY env variable
secret_access_key: ''
+ # Reference buckets via path rather than subdomain
+ # (i.e. "my-endpoint.com/bucket" instead of "bucket.my-endpoint.com")
+ force_path_style: false
+
# Maximum amount to upload in one request to object storage
max_upload_part: 2GB
diff --git a/config/production.yaml.example b/config/production.yaml.example
index 70993bf57a3..9ca2de5f4c9 100644
--- a/config/production.yaml.example
+++ b/config/production.yaml.example
@@ -121,6 +121,10 @@ object_storage:
# You can also use AWS_SECRET_ACCESS_KEY env variable
secret_access_key: ''
+ # Reference buckets via path rather than subdomain
+ # (i.e. "my-endpoint.com/bucket" instead of "bucket.my-endpoint.com")
+ force_path_style: false
+
# Maximum amount to upload in one request to object storage
max_upload_part: 2GB
diff --git a/server/initializers/config.ts b/server/initializers/config.ts
index 8375bf4304c..d726c59a4b6 100644
--- a/server/initializers/config.ts
+++ b/server/initializers/config.ts
@@ -91,6 +91,7 @@ const CONFIG = {
ACCESS_KEY_ID: config.get<string>('object_storage.credentials.access_key_id'),
SECRET_ACCESS_KEY: config.get<string>('object_storage.credentials.secret_access_key')
},
+ FORCE_PATH_STYLE: config.get<boolean>('object_storage.force_path_style'),
VIDEOS: {
BUCKET_NAME: config.get<string>('object_storage.videos.bucket_name'),
PREFIX: config.get<string>('object_storage.videos.prefix'),
diff --git a/server/lib/object-storage/shared/client.ts b/server/lib/object-storage/shared/client.ts
index c9a61459336..eadad02f93f 100644
--- a/server/lib/object-storage/shared/client.ts
+++ b/server/lib/object-storage/shared/client.ts
@@ -26,7 +26,8 @@ function getClient () {
accessKeyId: OBJECT_STORAGE.CREDENTIALS.ACCESS_KEY_ID,
secretAccessKey: OBJECT_STORAGE.CREDENTIALS.SECRET_ACCESS_KEY
}
- : undefined
+ : undefined,
+ forcePathStyle: CONFIG.OBJECT_STORAGE.FORCE_PATH_STYLE
})
logger.info('Initialized S3 client %s with region %s.', getEndpoint(), OBJECT_STORAGE.REGION, lTags())
diff --git a/support/docker/production/config/custom-environment-variables.yaml b/support/docker/production/config/custom-environment-variables.yaml
index c7cd28e6521..a960bab0bc9 100644
--- a/support/docker/production/config/custom-environment-variables.yaml
+++ b/support/docker/production/config/custom-environment-variables.yaml
@@ -54,6 +54,8 @@ object_storage:
region: "PEERTUBE_OBJECT_STORAGE_REGION"
+ force_path_style: "PEERTUBE_OBJECT_STORAGE_FORCE_PATH_STYLE"
+
max_upload_part:
__name: "PEERTUBE_OBJECT_STORAGE_MAX_UPLOAD_PART"
__format: "json"
```
You can then recompile it with:
```
npm run build
```
And it can be started with:
```
NODE_ENV=production NODE_CONFIG_DIR=/srv/peertube/config node dist/server.js
```
### Create resources in Garage
@@ -145,32 +232,31 @@ garage bucket create peertube-playlist
Now we allow our key to read and write on these buckets:
```
garage bucket allow peertube-playlists --read --write --owner --key peertube-key
garage bucket allow peertube-videos --read --write --owner --key peertube-key
garage bucket allow peertube-playlist --read --write --key peertube-key
garage bucket allow peertube-video --read --write --key peertube-key
```
We also need to expose these buckets publicly to serve their content to users:
Finally, we need to expose these buckets publicly to serve their content to users:
```bash
garage bucket website --allow peertube-playlists
garage bucket website --allow peertube-videos
```
Finally, we must allow Cross-Origin Resource Sharing (CORS).
CORS are required by your browser to allow requests triggered from the peertube website (eg. peertube.tld) to your bucket's domain (eg. peertube-videos.web.garage.tld)
```bash
export CORS='{"CORSRules":[{"AllowedHeaders":["*"],"AllowedMethods":["GET"],"AllowedOrigins":["*"]}]}'
aws --endpoint http://s3.garage.localhost s3api put-bucket-cors --bucket peertube-playlists --cors-configuration $CORS
aws --endpoint http://s3.garage.localhost s3api put-bucket-cors --bucket peertube-videos --cors-configuration $CORS
garage bucket website --allow peertube-playlist
garage bucket website --allow peertube-video
```
These buckets are now accessible on the web port (by default 3902) with the following URL: `http://<bucket><root_domain>:<web_port>` where the root domain is defined in your configuration file (by default `.web.garage`). So we have currently the following URLs:
* http://peertube-playlists.web.garage:3902
* http://peertube-videos.web.garage:3902
* http://peertube-playlist.web.garage:3902
* http://peertube-video.web.garage:3902
Make sure you (will) have a corresponding DNS entry for them.
### Configure a Reverse Proxy to serve CORS
Now we will configure a reverse proxy in front of Garage.
This is required as we have no other way to serve CORS headers yet.
Check the [Configuring a reverse proxy](/cookbook/reverse_proxy.html) section to know how.
Now make sure that your 2 dns entries are pointing to your reverse proxy.
### Configure Peertube
You must edit the file named `config/production.yaml`, we are only modifying the root key named `object_storage`:
@@ -182,6 +268,9 @@ object_storage:
# Put localhost only if you have a garage instance running on that node
endpoint: 'http://localhost:3900' # or "garage.example.com" if you have TLS on port 443
# This entry has been added by our patch, must be set to true
force_path_style: true
# Garage supports only one region for now, named garage
region: 'garage'
@@ -198,23 +287,28 @@ object_storage:
prefix: ''
# You must fill this field to make Peertube use our reverse proxy/website logic
base_url: 'http://peertube-playlists.web.garage.localhost' # Example: 'https://mirror.example.com'
base_url: 'http://peertube-playlist.web.garage' # Example: 'https://mirror.example.com'
# Same settings but for webtorrent videos
videos:
bucket_name: 'peertube-video'
prefix: ''
# You must fill this field to make Peertube use our reverse proxy/website logic
base_url: 'http://peertube-videos.web.garage.localhost'
base_url: 'http://peertube-video.web.garage'
```
### That's all
Everything must be configured now, simply restart Peertube and try to upload a video.
You must see in your browser console that data are fetched directly from our bucket (through the reverse proxy).
Peertube will start by serving the video from its own domain while it is encoding.
Once the encoding is done, the video is uploaded to Garage.
You can now reload the page and see in your browser console that data are fetched directly from your bucket.
### Miscellaneous
*Known bug:* The playback does not start and some 400 Bad Request Errors appear in your browser console and on Garage.
If the description of the error contains HTTP Invalid Range: InvalidRange, the error is due to a buggy ffmpeg version.
You must avoid the 4.4.0 and use either a newer or older version.
*Associated issues:* [#137](https://git.deuxfleurs.fr/Deuxfleurs/garage/issues/137), [#138](https://git.deuxfleurs.fr/Deuxfleurs/garage/issues/138), [#140](https://git.deuxfleurs.fr/Deuxfleurs/garage/issues/140). These issues are non blocking.
*External link:* [Peertube Documentation > Remote Storage](https://docs.joinpeertube.org/admin-remote-storage)
@@ -335,34 +429,31 @@ And add a new line. For example, to run it every 10 minutes:
## Pixelfed
[Pixelfed Technical Documentation > Configuration](https://docs.pixelfed.org/technical-documentation/env.html#filesystem)
https://docs.pixelfed.org/technical-documentation/env.html#filesystem
## Pleroma
[Pleroma Documentation > Pleroma.Uploaders.S3](https://docs-develop.pleroma.social/backend/configuration/cheatsheet/#pleromauploaderss3)
https://docs-develop.pleroma.social/backend/configuration/cheatsheet/#pleromauploaderss3
## Lemmy
Lemmy uses pict-rs that [supports S3 backends](https://git.asonix.dog/asonix/pict-rs/commit/f9f4fc63d670f357c93f24147c2ee3e1278e2d97)
via pict-rs
https://git.asonix.dog/asonix/pict-rs/commit/f9f4fc63d670f357c93f24147c2ee3e1278e2d97
## Funkwhale
[Funkwhale Documentation > S3 Storage](https://docs.funkwhale.audio/admin/configuration.html#s3-storage)
https://docs.funkwhale.audio/admin/configuration.html#s3-storage
## Misskey
[Misskey Github > commit 9d94424](https://github.com/misskey-dev/misskey/commit/9d944243a3a59e8880a360cbfe30fd5a3ec8d52d)
https://github.com/misskey-dev/misskey/commit/9d944243a3a59e8880a360cbfe30fd5a3ec8d52d
## Prismo
[Prismo Gitlab > .env.production.sample](https://gitlab.com/prismosuite/prismo/-/blob/dev/.env.production.sample#L26-33)
https://gitlab.com/prismosuite/prismo/-/blob/dev/.env.production.sample#L26-33
## Owncloud Infinite Scale (ocis)
OCIS could be compatible with S3:
- [Deploying OCIS with S3](https://owncloud.dev/ocis/deployment/ocis_s3/)
- [OCIS 1.7 release note](https://central.owncloud.org/t/owncloud-infinite-scale-tech-preview-1-7-enables-s3-storage/32514/3)
## Unsupported
- Mobilizon: No S3 integration
+9 -115
View File
@@ -1,19 +1,9 @@
# Browsing tools
# CLI tools
Browsing tools allow you to query the S3 API without too many abstractions.
CLI tools allow you to query the S3 API without too many abstractions.
These tools are particularly suitable for debug, backups, website deployments or any scripted task that need to handle data.
| Name | Status | Note |
|------|--------|------|
| [Minio client](#minio-client-recommended) | ✅ | Recommended |
| [AWS CLI](#aws-cli) | ✅ | Recommended |
| [rclone](#rclone) | ✅ | |
| [s3cmd](#s3cmd) | ✅ | |
| [(Cyber)duck](#cyberduck--duck) | ✅ | |
| [WinSCP (libs3)](#winscp) | ✅ | No instructions yet |
## Minio client
## Minio client (recommended)
Use the following command to set an "alias", i.e. define a new S3 server to be
used by the Minio client:
@@ -139,6 +129,11 @@ rclone copy garage:quentin.divers/hello.txt .
rclone help
```
**Advice with rclone:** use the `--fast-list` option when accessing buckets with large amounts of objects.
This will tremendously accelerate operations such as `rclone sync` or `rclone ncdu` by reducing the number
of ListObjects calls that are made.
## `s3cmd`
Here is a template for the `s3cmd.cfg` file to talk with Garage:
@@ -171,107 +166,6 @@ s3cmd get s3://my-bucket/hello.txt hello.txt
## Cyberduck & duck
Both Cyberduck (the GUI) and duck (the CLI) have a concept of "Connection Profiles" that contain some presets for a specific provider.
We wrote the following connection profile for Garage:
TODO
```xml
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
<key>Protocol</key>
<string>s3</string>
<key>Vendor</key>
<string>garage</string>
<key>Scheme</key>
<string>https</string>
<key>Description</key>
<string>GarageS3</string>
<key>Default Hostname</key>
<string>127.0.0.1</string>
<key>Default Port</key>
<string>4443</string>
<key>Hostname Configurable</key>
<false/>
<key>Port Configurable</key>
<false/>
<key>Username Configurable</key>
<true/>
<key>Username Placeholder</key>
<string>Access Key ID (GK...)</string>
<key>Password Placeholder</key>
<string>Secret Key</string>
<key>Properties</key>
<array>
<string>s3service.disable-dns-buckets=true</string>
</array>
<key>Region</key>
<string>garage</string>
<key>Regions</key>
<array>
<string>garage</string>
</array>
</dict>
</plist>
```
*Note: If your garage instance is configured with vhost access style, you can remove `s3service.disable-dns-buckets=true`.*
### Instructions for the GUI
Copy the connection profile, and save it anywhere as `garage.cyberduckprofile`.
Then find this file with your file explorer and double click on it: Cyberduck will open a connection wizard for this profile.
Simply follow the wizard and you should be done!
### Instuctions for the CLI
To configure duck (Cyberduck's CLI tool), start by creating its folder hierarchy:
```
mkdir -p ~/.duck/profiles/
```
Then, save the connection profile for Garage in `~/.duck/profiles/garage.cyberduckprofile`.
To set your credentials in `~/.duck/credentials`, use the following commands to generate the appropriate string:
```bash
export AWS_ACCESS_KEY_ID="GK..."
export AWS_SECRET_ACCESS_KEY="..."
export HOST="s3.garage.localhost"
export PORT="4443"
export PROTOCOL="https"
cat > ~/.duck/credentials <<EOF
$PROTOCOL\://$AWS_ACCESS_KEY_ID@$HOST\:$PORT=$AWS_SECRET_ACCESS_KEY
EOF
```
And finally, I recommend appending a small wrapper to your `~/.bashrc` to avoid setting the username on each command (do not forget to replace `GK...` by your access key):
```bash
function duck { command duck --username GK... $@ ; }
```
Finally, you can then use `duck` as follow:
```bash
# List buckets
duck --list garage:/
# List objects in a bucket
duck --list garage:/my-files/
# Download an object
duck --download garage:/my-files/an-object.txt /tmp/object.txt
# Upload an object
duck --upload /tmp/object.txt garage:/my-files/another-object.txt
# Delete an object
duck --delete garage:/my-files/an-object.txt
```
## WinSCP (libs3)
*No instruction yet. You can find ones in french [in our wiki](https://wiki.deuxfleurs.fr/fr/Guide/Garage/WinSCP).*
+8 -9
View File
@@ -4,12 +4,11 @@ Garage implements the Amazon S3 protocol, which makes it compatible with many ex
In particular, you will find here instructions to connect it with:
- [Browsing tools](./cli.md)
- [Applications](./apps.md)
- [Website hosting](./websites.md)
- [Software repositories](./repositories.md)
- [Your own code](./code.md)
- [FUSE](./fs.md)
- [web applications](./apps.md)
- [website hosting](./websites.md)
- [software repositories](./repositories.md)
- [CLI tools](./cli.md)
- [your own code](./code.md)
### Generic instructions
@@ -31,9 +30,9 @@ you will need the following parameters:
Most S3 clients can be configured easily with these parameters,
provided that you follow the following guidelines:
- **Be careful to DNS-style/path-style access:** Garage supports both DNS-style buckets, which are now by default
on Amazon S3, and legacy path-style buckets. If you use a reverse proxy in front of Garage,
make sure that you configured it to support the access-style required by the software you want to use.
- **Force path style:** Garage does not support DNS-style buckets, which are now by default
on Amazon S3. Instead, Garage uses the legacy path-style bucket addressing.
Remember to configure your client to acknowledge this fact.
- **Configuring the S3 region:** Garage requires your client to talk to the correct "S3 region",
which is set in the configuration file. This is often set just to `garage`.
Binary file not shown.

Before

Width:  |  Height:  |  Size: 134 KiB

+10 -49
View File
@@ -3,15 +3,6 @@
Whether you need to store and serve binary packages or source code, you may want to deploy a tool referred as a repository or registry.
Garage can also help you serve this content.
| Name | Status | Note |
|------|--------|------|
| [Gitea](#gitea) | ✅ | |
| [Docker](#generic-static-site-generator) | ✅ | Requires garage >= v0.6.0 |
| [Nix](#generic-static-site-generator) | ✅ | |
| [Gitlab](#gitlab) | ❓ | Not yet tested |
## Gitea
You can use Garage with Gitea to store your [git LFS](https://git-lfs.github.com/) data, your users' avatar, and their attachements.
@@ -61,42 +52,18 @@ $ aws s3 ls s3://gitea/avatars/
*External link:* [Gitea Documentation > Configuration Cheat Sheet](https://docs.gitea.io/en-us/config-cheat-sheet/)
## Gitlab
*External link:* [Gitlab Documentation > Object storage](https://docs.gitlab.com/ee/administration/object_storage.html)
## Private NPM Registry (Verdacio)
*External link:* [Verdaccio Github Repository > aws-storage plugin](https://github.com/verdaccio/verdaccio/tree/master/packages/plugins/aws-storage)
## Docker
Create a bucket and a key for your docker registry, then create `config.yml` with the following content:
```yml
version: 0.1
http:
addr: 0.0.0.0:5000
secret: asecretforlocaldevelopment
debug:
addr: localhost:5001
storage:
s3:
accesskey: GKxxxx
secretkey: yyyyy
region: garage
regionendpoint: http://localhost:3900
bucket: docker
secure: false
v4auth: true
rootdirectory: /
```
Replace the `accesskey`, `secretkey`, `bucket`, `regionendpoint` and `secure` values by the one fitting your deployment.
Then simply run the docker registry:
```bash
docker run \
--net=host \
-v `pwd`/config.yml:/etc/docker/registry/config.yml \
registry:2
```
*We started a plain text registry but docker clients require encrypted registries. You must either [setup TLS](https://docs.docker.com/registry/deploying/#run-an-externally-accessible-registry) on your registry or add `--insecure-registry=localhost:5000` to your docker daemon parameters.*
Not yet compatible, follow [#103](https://git.deuxfleurs.fr/Deuxfleurs/garage/issues/103).
*External link:* [Docker Documentation > Registry storage drivers > S3 storage driver](https://docs.docker.com/registry/storage-drivers/s3/)
@@ -200,9 +167,3 @@ on the binary cache, the client will download the result from the cache instead
Channels additionnaly serve Nix definitions, ie. a `.nix` file referencing
all the derivations you want to serve.
## Gitlab
*External link:* [Gitlab Documentation > Object storage](https://docs.gitlab.com/ee/administration/object_storage.html)
+17 -22
View File
@@ -3,12 +3,6 @@
Garage is also suitable to host static websites.
While they can be deployed with traditional CLI tools, some static website generators have integrated options to ease your workflow.
| Name | Status | Note |
|------|--------|------|
| [Hugo](#hugo) | ✅ | Publishing logic is integrated in the tool |
| [Publii](#publii) | ✅ | Require a correctly configured s3 vhost endpoint |
| [Generic Static Site Generator](#generic-static-site-generator) | ✅ | Works for Jekyll, Zola, Gatsby, Pelican, etc. |
## Hugo
Add to your `config.toml` the following section:
@@ -45,38 +39,39 @@ hugo deploy
## Publii
[![A screenshot of Publii's GUI](./publii.png)](./publii.png)
It would require a patch either on Garage or on Publii to make both systems work.
Deploying a website to Garage from Publii is natively supported.
First, make sure that your Garage administrator allowed and configured Garage to support vhost access style.
We also suppose that your bucket ("my-bucket") and key is already created and configured.
Currently, the proposed workaround is to deploy your website manually:
- On the left menu, click on Server, choose Manual Deployment (the logo looks like a compressed file)
- Set your website URL, keep Output type as "Non-compressed catalog"
- Click on Save changes
- Click on Sync your website (bottom left of the app)
- On the new page, click again on Sync your website
- Click on Get website files
- You need to synchronize the output folder you see in your file explorer, we will use minio client.
Then, from the left menu, click on server. Choose "S3" as the protocol.
In the configuration window, enter:
- Your finale website URL (eg. "http://my-bucket.web.garage.localhost:3902")
- Tick "Use a custom S3 provider"
- Set the S3 endpoint, (eg. "http://s3.garage.localhost:3900")
- Then put your access key (eg. "GK..."), your secret key, and your bucket (eg. "my-bucket")
- And hit the button "Save settings"
Be sure that you [configured minio client](cli.html#minio-client-recommended).
Now, each time you want to publish your website from Publii, just hit the bottom left button "Sync your website"!
Then copy this output folder
```bash
mc mirror --overwrite output garage/my-site
```
## Generic Static Site Generator
## Generic (eg. Jekyll)
Some tools do not support sending to a S3 backend but output a compiled folder on your system.
We can then use any CLI tool to upload this content to our S3 target.
First, start by [configuring minio client](cli.html#minio-client-recommended).
Then build your website (example for jekyll):
Then build your website:
```bash
jekyll build
```
And copy its output folder (`_site` for Jekyll) on S3:
And copy jekyll's output folder on S3:
```bash
mc mirror --overwrite _site garage/my-site
+59 -36
View File
@@ -1,6 +1,6 @@
# Configuring a reverse proxy
The main reason to add a reverse proxy in front of Garage is to provide TLS to your users and serve multiple web services on port 443.
The main reason to add a reverse proxy in front of Garage is to provide TLS to your users.
In production you will likely need your certificates signed by a certificate authority.
The most automated way is to use a provider supporting the [ACME protocol](https://datatracker.ietf.org/doc/html/rfc8555)
@@ -55,15 +55,16 @@ If you directly put the instructions in the root `nginx.conf`, keep in mind that
And do not forget to reload nginx with `systemctl reload nginx` or `nginx -s reload`.
### Exposing the S3 endpoints
### Defining backends
First, we need to tell to nginx how to access our Garage cluster.
Because we have multiple nodes, we want to leverage all of them by spreading the load.
In nginx, we can do that with the `upstream` directive.
Then in a `server` directive, we define the vhosts, the TLS certificates and the proxy rule.
In nginx, we can do that with the upstream directive.
Because we have 2 endpoints: one for the S3 API and one to serve websites,
we create 2 backends named respectively `s3_backend` and `web_backend`.
A possible configuration:
A documented example for the `s3_backend` assuming you chose port 3900:
```nginx
upstream s3_backend {
@@ -77,34 +78,9 @@ upstream s3_backend {
# that are more powerful than others
server garage2.example.com:3900 weight=2;
}
server {
listen [::]:443 http2 ssl;
ssl_certificate /tmp/garage.crt;
ssl_certificate_key /tmp/garage.key;
# You need multiple server names here:
# - s3.garage.tld is used for path-based s3 requests
# - *.s3.garage.tld is used for vhost-based s3 requests
server_name s3.garage.tld *.s3.garage.tld;
location / {
proxy_pass http://s3_backend;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header Host $host;
}
}
```
## Exposing the web endpoint
To better understand the logic involved, you can refer to the [Exposing buckets as websites](/cookbook/exposing_websites.html) section.
Otherwise, the configuration is very similar to the S3 endpoint.
You must only adapt `upstream` with the web port instead of the s3 port and change the `server_name` and `proxy_pass` entry
A possible configuration:
A similar example for the `web_backend` assuming you chose port 3902:
```nginx
upstream web_backend {
@@ -113,19 +89,65 @@ upstream web_backend {
server garage1.example.com:3902;
server garage2.example.com:3902 weight=2;
}
```
### Exposing the S3 API
The configuration section for the S3 API is simple as we only support path-access style yet.
We simply configure the TLS parameters and forward all the requests to the backend:
```nginx
server {
listen [::]:443 http2 ssl;
ssl_certificate /tmp/garage.crt;
ssl_certificate_key /tmp/garage.key;
# You need multiple server names here:
# - *.web.garage.tld is used for your users wanting a website without reserving a domain name
# - example.com, my-site.tld, etc. are reserved domain name by your users that chose to host their website as a garage's bucket
server_name *.web.garage.tld example.com my-site.tld;
# should be the endpoint you want
# aws uses s3.amazonaws.com for example
server_name garage.example.com;
location / {
proxy_pass http://s3_backend;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header Host $host;
}
}
```
### Exposing the web endpoint
The web endpoint is a bit more complicated to configure as it listens on many different `Host` fields.
To better understand the logic involved, you can refer to the [Exposing buckets as websites](/cookbook/exposing_websites.html) section.
Also, for some applications, you may need to serve CORS headers: Garage can not serve them directly but we show how we can use nginx to serve them.
You can use the following example as your starting point:
```nginx
server {
listen [::]:443 http2 ssl;
ssl_certificate /tmp/garage.crt;
ssl_certificate_key /tmp/garage.key;
# We list all the Hosts fields that can access our buckets
server_name *.web.garage
example.com
my-site.tld
;
location / {
# Add these headers only if you want to allow CORS requests
# For production use, more specific rules would be better for your security
add_header Access-Control-Allow-Origin *;
add_header Access-Control-Max-Age 3600;
add_header Access-Control-Expose-Headers Content-Length;
add_header Access-Control-Allow-Headers Range;
# We do not forward OPTIONS requests to Garage
# as it does not support them but they are needed for CORS.
if ($request_method = OPTIONS) {
return 200;
}
proxy_pass http://web_backend;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header Host $host;
@@ -133,6 +155,7 @@ server {
}
```
## Apache httpd
@TODO
+2 -2
View File
@@ -49,11 +49,11 @@ bootstrap_peers = []
[s3_api]
s3_region = "garage"
api_bind_addr = "[::]:3900"
root_domain = ".s3.garage.localhost"
root_domain = ".s3.garage"
[s3_web]
bind_addr = "[::]:3902"
root_domain = ".web.garage.localhost"
root_domain = ".web.garage"
index = "index.html"
```
@@ -47,7 +47,7 @@ All APIs that are not mentionned are not implemented and will return a 501 Not I
| ListObjects | Implemented, bugs? (see below) |
| ListObjectsV2 | Implemented |
| ListMultipartUpload | Implemented |
| ListParts | Missing |
| ListParts | Implemented |
| PutObject | Implemented |
| PutBucketCors | Implemented |
| PutBucketWebsite | Partially implemented (see below)|
@@ -24,8 +24,8 @@ your motivations for doing so in the PR message.
| | CompleteMultipartUpload |
| | AbortMultipartUpload |
| | UploadPart |
| | ListMultipartUploads |
| | ListParts |
| | [*ListMultipartUploads*](https://git.deuxfleurs.fr/Deuxfleurs/garage/issues/103) |
| | [*ListParts*](https://git.deuxfleurs.fr/Deuxfleurs/garage/issues/103) |
| **A-tier** | |
| | GetBucketCors |
| | PutBucketCors |
@@ -34,7 +34,6 @@ your motivations for doing so in the PR message.
| | GetBucketWebsite |
| | PutBucketWebsite |
| | DeleteBucketWebsite |
| | [PostObject](https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectPOST.html) |
| ~~~~~~~~~~~~~~~~~~~~~~~~~~ | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ |
| **B-tier** | |
| | GetBucketAcl |
@@ -38,9 +38,13 @@ The migration steps are as follows:
the buckets that existed previously. This will also give access to API keys
as it was before.
9. Check that all your buckets indeed appear in `garage bucket list`, and that
keys have the proper access flags set. If that is not the case, revert
everything and file a bug!
9. Do `garage repair -a --yes tables` and `garage repair -a --yes blocks`,
check the logs and check that all data seems to be synced correctly between
nodes.
10. Your upgraded cluster should be in a working state. Re-enable API and Web
10. Check that all your buckets indeed appear in `garage bucket list`, and that
keys have the proper access flags set. If that is not the case, revert
everything and file a bug!
11. Your upgraded cluster should be in a working state. Re-enable API and Web
access and check that everything went well.
+1 -1
View File
@@ -13,7 +13,7 @@ garage -c /tmp/config.1.toml bucket create eprouvette
KEY_INFO=$(garage -c /tmp/config.1.toml key new --name opérateur)
ACCESS_KEY=`echo $KEY_INFO|grep -Po 'GK[a-f0-9]+'`
SECRET_KEY=`echo $KEY_INFO|grep -Po 'Secret key: [a-f0-9]+'|grep -Po '[a-f0-9]+$'`
garage -c /tmp/config.1.toml bucket allow eprouvette --owner --read --write --key $ACCESS_KEY
garage -c /tmp/config.1.toml bucket allow eprouvette --read --write --owner --key $ACCESS_KEY
echo "$ACCESS_KEY $SECRET_KEY" > /tmp/garage.s3
echo "Bucket s3://eprouvette created. Credentials stored in /tmp/garage.s3."
+1 -1
View File
@@ -8,7 +8,7 @@ cat > /tmp/garage.mc/config.json <<EOF
"version": "10",
"aliases": {
"garage": {
"url": "https://localhost:4443",
"url": "http://127.0.0.1:3911",
"accessKey": "$ACCESS_KEY",
"secretKey": "$SECRET_KEY",
"api": "S3v4",
+61 -9
View File
@@ -137,7 +137,7 @@ if [ -z "$SKIP_AWS" ]; then
aws s3api list-objects-v2 --bucket eprouvette --page-size 0 >$CMDOUT
[ $(jq '.Contents | length' $CMDOUT) == 8 ]
[ $(jq '.CommonPrefixes | length' $CMDOUT) == 0 ]
aws s3api list-objects-v2 --bucket eprouvette --page-size 999999999999999 >$CMDOUT
aws s3api list-objects-v2 --bucket eprouvette --page-size 999999999 >$CMDOUT
[ $(jq '.Contents | length' $CMDOUT) == 8 ]
[ $(jq '.CommonPrefixes | length' $CMDOUT) == 0 ]
aws s3api list-objects-v2 --bucket eprouvette --page-size 1 >$CMDOUT
@@ -252,7 +252,58 @@ if [ -z "$SKIP_AWS" ]; then
echo "Deleted ${key}:${uid}"
done
# Test for UploadPartCopy
echo "Test for ListParts"
UPLOAD_ID=$(aws s3api create-multipart-upload --bucket eprouvette --key list-parts | jq -r .UploadId)
aws s3api list-parts --bucket eprouvette --key list-parts --upload-id $UPLOAD_ID >$CMDOUT
[ $(jq '.Parts | length' $CMDOUT) == 0 ]
[ $(jq -r '.StorageClass' $CMDOUT) == 'STANDARD' ] # check that the result is not empty
ETAG1=$(aws s3api upload-part --bucket eprouvette --key list-parts --upload-id $UPLOAD_ID --part-number 1 --body /tmp/garage.2.rnd | jq .ETag)
aws s3api list-parts --bucket eprouvette --key list-parts --upload-id $UPLOAD_ID >$CMDOUT
[ $(jq '.Parts | length' $CMDOUT) == 1 ]
[ $(jq '.Parts[0].PartNumber' $CMDOUT) == 1 ]
[ $(jq '.Parts[0].Size' $CMDOUT) == 5242880 ]
[ $(jq '.Parts[0].ETag' $CMDOUT) == $ETAG1 ]
ETAG2=$(aws s3api upload-part --bucket eprouvette --key list-parts --upload-id $UPLOAD_ID --part-number 3 --body /tmp/garage.3.rnd | jq .ETag)
ETAG3=$(aws s3api upload-part --bucket eprouvette --key list-parts --upload-id $UPLOAD_ID --part-number 2 --body /tmp/garage.2.rnd | jq .ETag)
aws s3api list-parts --bucket eprouvette --key list-parts --upload-id $UPLOAD_ID >$CMDOUT
[ $(jq '.Parts | length' $CMDOUT) == 3 ]
[ $(jq '.Parts[1].ETag' $CMDOUT) == $ETAG3 ]
aws s3api list-parts --bucket eprouvette --key list-parts --upload-id $UPLOAD_ID --page-size 1 >$CMDOUT
[ $(jq '.Parts | length' $CMDOUT) == 3 ]
[ $(jq '.Parts[1].ETag' $CMDOUT) == $ETAG3 ]
cat >/tmp/garage.multipart_struct <<EOF
{
"Parts": [
{
"ETag": $ETAG1,
"PartNumber": 1
},
{
"ETag": $ETAG3,
"PartNumber": 2
},
{
"ETag": $ETAG2,
"PartNumber": 3
}
]
}
EOF
aws s3api complete-multipart-upload \
--bucket eprouvette --key list-parts --upload-id $UPLOAD_ID \
--multipart-upload file:///tmp/garage.multipart_struct
! aws s3api list-parts --bucket eprouvette --key list-parts --upload-id $UPLOAD_ID >$CMDOUT
aws s3 rm "s3://eprouvette/list-parts"
# @FIXME We do not write tests with --starting-token due to a bug with awscli
# See here: https://github.com/aws/aws-cli/issues/6666
echo "Test for UploadPartCopy"
aws s3 cp "/tmp/garage.3.rnd" "s3://eprouvette/copy_part_source"
UPLOAD_ID=$(aws s3api create-multipart-upload --bucket eprouvette --key test_multipart | jq -r .UploadId)
PART1=$(aws s3api upload-part \
@@ -303,22 +354,23 @@ EOF
rm /tmp/garage.test_multipart_reference
rm /tmp/garage.test_multipart_diff
echo "Test CORS endpoints"
# @FIXME remove bucket allow if/when testing on s3 endpoint
garage -c /tmp/config.1.toml bucket website --allow eprouvette
aws s3api put-object --bucket eprouvette --key index.html
CORS='{"CORSRules":[{"AllowedHeaders":["*"],"AllowedMethods":["GET","PUT"],"AllowedOrigins":["*"]}]}'
aws s3api put-bucket-cors --bucket eprouvette --cors-configuration $CORS
[ `aws s3api get-bucket-cors --bucket eprouvette | jq -c` == $CORS ]
# @FIXME should we really return these CORS on the WEB endpoint and not on the S3 endpoint?
curl -s -i -H 'Origin: http://example.com' http://eprouvette.web.garage.localhost:3921 | grep access-control-allow-origin
curl -s -i -X OPTIONS -H 'Access-Control-Request-Method: PUT' -H 'Origin: http://example.com' http://eprouvette.web.garage.localhost:3921|grep access-control-allow-methods
curl -s -i -X OPTIONS -H 'Access-Control-Request-Method: DELETE' -H 'Origin: http://example.com' http://eprouvette.web.garage.localhost:3921 |grep '403 Forbidden'
curl -s -i -H 'Origin: http://example.com' --header "Host: eprouvette.web.garage.localhost" http://127.0.0.1:3921/ | grep access-control-allow-origin
curl -s -i -X OPTIONS -H 'Access-Control-Request-Method: PUT' -H 'Origin: http://example.com' --header "Host: eprouvette.web.garage.localhost" http://127.0.0.1:3921/ | grep access-control-allow-methods
curl -s -i -X OPTIONS -H 'Access-Control-Request-Method: DELETE' -H 'Origin: http://example.com' --header "Host: eprouvette.web.garage.localhost" http://127.0.0.1:3921/ | grep '403 Forbidden'
aws s3api delete-bucket-cors --bucket eprouvette
#@TODO we may want to test the S3 endpoint but we need to handle authentication, which is way more complex.
aws s3api delete-bucket-cors --bucket eprouvette
! [ -s `aws s3api get-bucket-cors --bucket eprouvette` ]
curl -s -i -X OPTIONS -H 'Access-Control-Request-Method: PUT' -H 'Origin: http://example.com' http://eprouvette.web.garage.localhost:3921|grep '403 Forbidden'
curl -s -i -X OPTIONS -H 'Access-Control-Request-Method: PUT' -H 'Origin: http://example.com' --header "Host: eprouvette.web.garage.localhost" http://127.0.0.1:3921/ | grep '403 Forbidden'
aws s3api delete-object --bucket eprouvette --key index.html
garage -c /tmp/config.1.toml bucket website --deny eprouvette
fi
+2
View File
@@ -28,10 +28,12 @@ hmac = "0.10"
idna = "0.2"
log = "0.4"
md-5 = "0.9"
nom = "7.1"
sha2 = "0.9"
futures = "0.3"
futures-util = "0.3"
pin-project = "1.0"
tokio = { version = "1.0", default-features = false, features = ["rt", "rt-multi-thread", "io-util", "net", "time", "macros", "sync", "signal", "fs"] }
http = "0.2"
+97 -57
View File
@@ -1,4 +1,3 @@
use std::cmp::{max, min};
use std::net::SocketAddr;
use std::sync::Arc;
@@ -6,7 +5,7 @@ use futures::future::Future;
use hyper::header;
use hyper::server::conn::AddrStream;
use hyper::service::{make_service_fn, service_fn};
use hyper::{Body, Request, Response, Server};
use hyper::{Body, Method, Request, Response, Server};
use garage_util::data::*;
use garage_util::error::Error as GarageError;
@@ -14,8 +13,10 @@ use garage_util::error::Error as GarageError;
use garage_model::garage::Garage;
use garage_model::key_table::Key;
use garage_table::util::*;
use crate::error::*;
use crate::signature::check_signature;
use crate::signature::payload::check_payload_signature;
use crate::helpers::*;
use crate::s3_bucket::*;
@@ -91,7 +92,10 @@ async fn handler(
}
async fn handler_inner(garage: Arc<Garage>, req: Request<Body>) -> Result<Response<Body>, Error> {
let (api_key, content_sha256) = check_signature(&garage, &req).await?;
let (api_key, content_sha256) = check_payload_signature(&garage, &req).await?;
let api_key = api_key.ok_or_else(|| {
Error::Forbidden("Garage does not support anonymous access yet".to_string())
})?;
let authority = req
.headers()
@@ -101,32 +105,38 @@ async fn handler_inner(garage: Arc<Garage>, req: Request<Body>) -> Result<Respon
let host = authority_to_host(authority)?;
let bucket = garage
let bucket_name = garage
.config
.s3_api
.root_domain
.as_ref()
.and_then(|root_domain| host_to_bucket(&host, root_domain));
let endpoint = Endpoint::from_request(&req, bucket.map(ToOwned::to_owned))?;
let (endpoint, bucket_name) = Endpoint::from_request(&req, bucket_name.map(ToOwned::to_owned))?;
debug!("Endpoint: {:?}", endpoint);
// Special code path for CreateBucket API endpoint
if let Endpoint::CreateBucket { bucket } = endpoint {
return handle_create_bucket(&garage, req, content_sha256, api_key, bucket).await;
}
let bucket_name = match endpoint.get_bucket() {
let bucket_name = match bucket_name {
None => return handle_request_without_bucket(garage, req, api_key, endpoint).await,
Some(bucket) => bucket.to_string(),
};
// Special code path for CreateBucket API endpoint
if let Endpoint::CreateBucket {} = endpoint {
return handle_create_bucket(&garage, req, content_sha256, api_key, bucket_name).await;
}
let bucket_id = resolve_bucket(&garage, &bucket_name, &api_key).await?;
let bucket = garage
.bucket_table
.get(&EmptyKey, &bucket_id)
.await?
.filter(|b| !b.state.is_deleted())
.ok_or(Error::NoSuchBucket)?;
let allowed = match endpoint.authorization_type() {
Authorization::Read(_) => api_key.allow_read(&bucket_id),
Authorization::Write(_) => api_key.allow_write(&bucket_id),
Authorization::Owner(_) => api_key.allow_owner(&bucket_id),
Authorization::Read => api_key.allow_read(&bucket_id),
Authorization::Write => api_key.allow_write(&bucket_id),
Authorization::Owner => api_key.allow_owner(&bucket_id),
_ => unreachable!(),
};
@@ -136,14 +146,27 @@ async fn handler_inner(garage: Arc<Garage>, req: Request<Body>) -> Result<Respon
));
}
match endpoint {
Endpoint::HeadObject { key, .. } => handle_head(garage, &req, bucket_id, &key).await,
Endpoint::GetObject { key, .. } => handle_get(garage, &req, bucket_id, &key).await,
// Look up what CORS rule might apply to response.
// Requests for methods different than GET, HEAD or POST
// are always preflighted, i.e. the browser should make
// an OPTIONS call before to check it is allowed
let matching_cors_rule = match *req.method() {
Method::GET | Method::HEAD | Method::POST => find_matching_cors_rule(&bucket, &req)?,
_ => None,
};
let resp = match endpoint {
Endpoint::Options => handle_options(&req, &bucket).await,
Endpoint::HeadObject {
key, part_number, ..
} => handle_head(garage, &req, bucket_id, &key, part_number).await,
Endpoint::GetObject {
key, part_number, ..
} => handle_get(garage, &req, bucket_id, &key, part_number).await,
Endpoint::UploadPart {
key,
part_number,
upload_id,
..
} => {
handle_put_part(
garage,
@@ -156,14 +179,11 @@ async fn handler_inner(garage: Arc<Garage>, req: Request<Body>) -> Result<Respon
)
.await
}
Endpoint::CopyObject { key, .. } => {
handle_copy(garage, &api_key, &req, bucket_id, &key).await
}
Endpoint::CopyObject { key } => handle_copy(garage, &api_key, &req, bucket_id, &key).await,
Endpoint::UploadPartCopy {
key,
part_number,
upload_id,
..
} => {
handle_upload_part_copy(
garage,
@@ -176,25 +196,21 @@ async fn handler_inner(garage: Arc<Garage>, req: Request<Body>) -> Result<Respon
)
.await
}
Endpoint::PutObject { key, .. } => {
handle_put(garage, req, bucket_id, &key, content_sha256).await
Endpoint::PutObject { key } => {
handle_put(garage, req, bucket_id, &key, &api_key, content_sha256).await
}
Endpoint::AbortMultipartUpload { key, upload_id, .. } => {
Endpoint::AbortMultipartUpload { key, upload_id } => {
handle_abort_multipart_upload(garage, bucket_id, &key, &upload_id).await
}
Endpoint::DeleteObject { key, .. } => handle_delete(garage, bucket_id, &key).await,
Endpoint::CreateMultipartUpload { bucket, key } => {
handle_create_multipart_upload(garage, &req, &bucket, bucket_id, &key).await
Endpoint::CreateMultipartUpload { key } => {
handle_create_multipart_upload(garage, &req, &bucket_name, bucket_id, &key).await
}
Endpoint::CompleteMultipartUpload {
bucket,
key,
upload_id,
} => {
Endpoint::CompleteMultipartUpload { key, upload_id } => {
handle_complete_multipart_upload(
garage,
req,
&bucket,
&bucket_name,
bucket_id,
&key,
&upload_id,
@@ -202,19 +218,18 @@ async fn handler_inner(garage: Arc<Garage>, req: Request<Body>) -> Result<Respon
)
.await
}
Endpoint::CreateBucket { .. } => unreachable!(),
Endpoint::HeadBucket { .. } => {
Endpoint::CreateBucket {} => unreachable!(),
Endpoint::HeadBucket {} => {
let empty_body: Body = Body::from(vec![]);
let response = Response::builder().body(empty_body).unwrap();
Ok(response)
}
Endpoint::DeleteBucket { .. } => {
Endpoint::DeleteBucket {} => {
handle_delete_bucket(&garage, bucket_id, bucket_name, api_key).await
}
Endpoint::GetBucketLocation { .. } => handle_get_bucket_location(garage),
Endpoint::GetBucketVersioning { .. } => handle_get_bucket_versioning(),
Endpoint::GetBucketLocation {} => handle_get_bucket_location(garage),
Endpoint::GetBucketVersioning {} => handle_get_bucket_versioning(),
Endpoint::ListObjects {
bucket,
delimiter,
encoding_type,
marker,
@@ -225,10 +240,10 @@ async fn handler_inner(garage: Arc<Garage>, req: Request<Body>) -> Result<Respon
garage,
&ListObjectsQuery {
common: ListQueryCommon {
bucket_name: bucket,
bucket_name,
bucket_id,
delimiter: delimiter.map(|d| d.to_string()),
page_size: max_keys.map(|p| min(1000, max(1, p))).unwrap_or(1000),
page_size: max_keys.map(|p| p.clamp(1, 1000)).unwrap_or(1000),
prefix: prefix.unwrap_or_default(),
urlencode_resp: encoding_type.map(|e| e == "url").unwrap_or(false),
},
@@ -241,7 +256,6 @@ async fn handler_inner(garage: Arc<Garage>, req: Request<Body>) -> Result<Respon
.await
}
Endpoint::ListObjectsV2 {
bucket,
delimiter,
encoding_type,
max_keys,
@@ -256,10 +270,10 @@ async fn handler_inner(garage: Arc<Garage>, req: Request<Body>) -> Result<Respon
garage,
&ListObjectsQuery {
common: ListQueryCommon {
bucket_name: bucket,
bucket_name,
bucket_id,
delimiter: delimiter.map(|d| d.to_string()),
page_size: max_keys.map(|p| min(1000, max(1, p))).unwrap_or(1000),
page_size: max_keys.map(|p| p.clamp(1, 1000)).unwrap_or(1000),
urlencode_resp: encoding_type.map(|e| e == "url").unwrap_or(false),
prefix: prefix.unwrap_or_default(),
},
@@ -278,7 +292,6 @@ async fn handler_inner(garage: Arc<Garage>, req: Request<Body>) -> Result<Respon
}
}
Endpoint::ListMultipartUploads {
bucket,
delimiter,
encoding_type,
key_marker,
@@ -290,10 +303,10 @@ async fn handler_inner(garage: Arc<Garage>, req: Request<Body>) -> Result<Respon
garage,
&ListMultipartUploadsQuery {
common: ListQueryCommon {
bucket_name: bucket,
bucket_name,
bucket_id,
delimiter: delimiter.map(|d| d.to_string()),
page_size: max_uploads.map(|p| min(1000, max(1, p))).unwrap_or(1000),
page_size: max_uploads.map(|p| p.clamp(1, 1000)).unwrap_or(1000),
prefix: prefix.unwrap_or_default(),
urlencode_resp: encoding_type.map(|e| e == "url").unwrap_or(false),
},
@@ -303,21 +316,48 @@ async fn handler_inner(garage: Arc<Garage>, req: Request<Body>) -> Result<Respon
)
.await
}
Endpoint::DeleteObjects { .. } => {
Endpoint::ListParts {
key,
max_parts,
part_number_marker,
upload_id,
} => {
handle_list_parts(
garage,
&ListPartsQuery {
bucket_name,
bucket_id,
key,
upload_id,
part_number_marker: part_number_marker.map(|p| p.clamp(1, 10000)),
max_parts: max_parts.map(|p| p.clamp(1, 1000)).unwrap_or(1000),
},
)
.await
}
Endpoint::DeleteObjects {} => {
handle_delete_objects(garage, bucket_id, req, content_sha256).await
}
Endpoint::GetBucketWebsite { .. } => handle_get_website(garage, bucket_id).await,
Endpoint::PutBucketWebsite { .. } => {
Endpoint::GetBucketWebsite {} => handle_get_website(&bucket).await,
Endpoint::PutBucketWebsite {} => {
handle_put_website(garage, bucket_id, req, content_sha256).await
}
Endpoint::DeleteBucketWebsite { .. } => handle_delete_website(garage, bucket_id).await,
Endpoint::GetBucketCors { .. } => handle_get_cors(garage, bucket_id).await,
Endpoint::PutBucketCors { .. } => {
handle_put_cors(garage, bucket_id, req, content_sha256).await
}
Endpoint::DeleteBucketCors { .. } => handle_delete_cors(garage, bucket_id).await,
Endpoint::DeleteBucketWebsite {} => handle_delete_website(garage, bucket_id).await,
Endpoint::GetBucketCors {} => handle_get_cors(&bucket).await,
Endpoint::PutBucketCors {} => handle_put_cors(garage, bucket_id, req, content_sha256).await,
Endpoint::DeleteBucketCors {} => handle_delete_cors(garage, bucket_id).await,
endpoint => Err(Error::NotImplemented(endpoint.name().to_owned())),
};
// If request was a success and we have a CORS rule that applies to it,
// add the corresponding CORS headers to the response
let mut resp_ok = resp?;
if let Some(rule) = matching_cors_rule {
add_cors_headers(&mut resp_ok, rule)
.ok_or_internal_error("Invalid bucket CORS configuration")?;
}
Ok(resp_ok)
}
async fn handle_request_without_bucket(
+17 -5
View File
@@ -58,6 +58,19 @@ pub enum Error {
#[error(display = "At least one of the preconditions you specified did not hold")]
PreconditionFailed,
/// Parts specified in CMU request do not match parts actually uploaded
#[error(display = "Parts given to CompleteMultipartUpload do not match uploaded parts")]
InvalidPart,
/// Parts given to CompleteMultipartUpload were not in ascending order
#[error(display = "Parts given to CompleteMultipartUpload were not in ascending order")]
InvalidPartOrder,
/// In CompleteMultipartUpload: not enough data
/// (here we are more lenient than AWS S3)
#[error(display = "Proposed upload is smaller than the minimum allowed object size")]
EntityTooSmall,
// Category: bad request
/// The request contained an invalid UTF-8 sequence in its path or in other parameters
#[error(display = "Invalid UTF-8: {}", _0)]
@@ -143,6 +156,9 @@ impl Error {
Error::BucketAlreadyExists => "BucketAlreadyExists",
Error::BucketNotEmpty => "BucketNotEmpty",
Error::PreconditionFailed => "PreconditionFailed",
Error::InvalidPart => "InvalidPart",
Error::InvalidPartOrder => "InvalidPartOrder",
Error::EntityTooSmall => "EntityTooSmall",
Error::Forbidden(_) => "AccessDenied",
Error::AuthorizationHeaderMalformed(_) => "AuthorizationHeaderMalformed",
Error::NotImplemented(_) => "NotImplemented",
@@ -206,11 +222,7 @@ where
fn ok_or_bad_request<M: AsRef<str>>(self, reason: M) -> Result<T, Error> {
match self {
Ok(x) => Ok(x),
Err(e) => Err(Error::BadRequest(format!(
"{}: {}",
reason.as_ref(),
e.to_string()
))),
Err(e) => Err(Error::BadRequest(format!("{}: {}", reason.as_ref(), e))),
}
}
}
+8 -5
View File
@@ -120,7 +120,10 @@ pub async fn handle_create_bucket(
bucket_name: String,
) -> Result<Response<Body>, Error> {
let body = hyper::body::to_bytes(req.into_body()).await?;
verify_signed_content(content_sha256, &body[..])?;
if let Some(content_sha256) = content_sha256 {
verify_signed_content(content_sha256, &body[..])?;
}
let cmd =
parse_create_bucket_xml(&body[..]).ok_or_bad_request("Invalid create bucket XML query")?;
@@ -320,7 +323,7 @@ mod tests {
assert_eq!(
parse_create_bucket_xml(
br#"
<CreateBucketConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
<CreateBucketConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
</CreateBucketConfiguration >
"#
),
@@ -329,8 +332,8 @@ mod tests {
assert_eq!(
parse_create_bucket_xml(
br#"
<CreateBucketConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
<LocationConstraint>Europe</LocationConstraint>
<CreateBucketConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
<LocationConstraint>Europe</LocationConstraint>
</CreateBucketConfiguration >
"#
),
@@ -339,7 +342,7 @@ mod tests {
assert_eq!(
parse_create_bucket_xml(
br#"
<CreateBucketConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
<CreateBucketConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
</Crea >
"#
),
+2 -2
View File
@@ -487,7 +487,7 @@ impl CopyPreconditionHeaders {
.get("x-amz-copy-source-if-modified-since")
.map(|x| x.to_str())
.transpose()?
.map(|x| httpdate::parse_http_date(x))
.map(httpdate::parse_http_date)
.transpose()
.ok_or_bad_request("Invalid date in x-amz-copy-source-if-modified-since")?,
copy_source_if_none_match: req
@@ -505,7 +505,7 @@ impl CopyPreconditionHeaders {
.get("x-amz-copy-source-if-unmodified-since")
.map(|x| x.to_str())
.transpose()?
.map(|x| httpdate::parse_http_date(x))
.map(httpdate::parse_http_date)
.transpose()
.ok_or_bad_request("Invalid date in x-amz-copy-source-if-unmodified-since")?,
})
+108 -56
View File
@@ -3,7 +3,7 @@ use std::sync::Arc;
use http::header::{
ACCESS_CONTROL_ALLOW_HEADERS, ACCESS_CONTROL_ALLOW_METHODS, ACCESS_CONTROL_ALLOW_ORIGIN,
ACCESS_CONTROL_EXPOSE_HEADERS,
ACCESS_CONTROL_EXPOSE_HEADERS, ACCESS_CONTROL_REQUEST_HEADERS, ACCESS_CONTROL_REQUEST_METHOD,
};
use hyper::{header::HeaderName, Body, Method, Request, Response, StatusCode};
@@ -13,21 +13,12 @@ use crate::error::*;
use crate::s3_xml::{to_xml_with_header, xmlns_tag, IntValue, Value};
use crate::signature::verify_signed_content;
use garage_model::bucket_table::CorsRule as GarageCorsRule;
use garage_model::bucket_table::{Bucket, CorsRule as GarageCorsRule};
use garage_model::garage::Garage;
use garage_table::*;
use garage_util::data::*;
pub async fn handle_get_cors(
garage: Arc<Garage>,
bucket_id: Uuid,
) -> Result<Response<Body>, Error> {
let bucket = garage
.bucket_table
.get(&EmptyKey, &bucket_id)
.await?
.ok_or(Error::NoSuchBucket)?;
pub async fn handle_get_cors(bucket: &Bucket) -> Result<Response<Body>, Error> {
let param = bucket
.params()
.ok_or_internal_error("Bucket should not be deleted at this point")?;
@@ -81,7 +72,10 @@ pub async fn handle_put_cors(
content_sha256: Option<Hash>,
) -> Result<Response<Body>, Error> {
let body = hyper::body::to_bytes(req.into_body()).await?;
verify_signed_content(content_sha256, &body[..])?;
if let Some(content_sha256) = content_sha256 {
verify_signed_content(content_sha256, &body[..])?;
}
let mut bucket = garage
.bucket_table
@@ -106,6 +100,107 @@ pub async fn handle_put_cors(
.body(Body::empty())?)
}
pub async fn handle_options(req: &Request<Body>, bucket: &Bucket) -> Result<Response<Body>, Error> {
let origin = req
.headers()
.get("Origin")
.ok_or_bad_request("Missing Origin header")?
.to_str()?;
let request_method = req
.headers()
.get(ACCESS_CONTROL_REQUEST_METHOD)
.ok_or_bad_request("Missing Access-Control-Request-Method header")?
.to_str()?;
let request_headers = match req.headers().get(ACCESS_CONTROL_REQUEST_HEADERS) {
Some(h) => h.to_str()?.split(',').map(|h| h.trim()).collect::<Vec<_>>(),
None => vec![],
};
if let Some(cors_config) = bucket.params().unwrap().cors_config.get() {
let matching_rule = cors_config
.iter()
.find(|rule| cors_rule_matches(rule, origin, request_method, request_headers.iter()));
if let Some(rule) = matching_rule {
let mut resp = Response::builder()
.status(StatusCode::OK)
.body(Body::empty())?;
add_cors_headers(&mut resp, rule).ok_or_internal_error("Invalid CORS configuration")?;
return Ok(resp);
}
}
Err(Error::Forbidden("This CORS request is not allowed.".into()))
}
pub fn find_matching_cors_rule<'a>(
bucket: &'a Bucket,
req: &Request<Body>,
) -> Result<Option<&'a GarageCorsRule>, Error> {
if let Some(cors_config) = bucket.params().unwrap().cors_config.get() {
if let Some(origin) = req.headers().get("Origin") {
let origin = origin.to_str()?;
let request_headers = match req.headers().get(ACCESS_CONTROL_REQUEST_HEADERS) {
Some(h) => h.to_str()?.split(',').map(|h| h.trim()).collect::<Vec<_>>(),
None => vec![],
};
return Ok(cors_config.iter().find(|rule| {
cors_rule_matches(
rule,
origin,
&req.method().to_string(),
request_headers.iter(),
)
}));
}
}
Ok(None)
}
fn cors_rule_matches<'a, HI, S>(
rule: &GarageCorsRule,
origin: &'a str,
method: &'a str,
mut request_headers: HI,
) -> bool
where
HI: Iterator<Item = S>,
S: AsRef<str>,
{
rule.allow_origins.iter().any(|x| x == "*" || x == origin)
&& rule.allow_methods.iter().any(|x| x == "*" || x == method)
&& request_headers.all(|h| {
rule.allow_headers
.iter()
.any(|x| x == "*" || x == h.as_ref())
})
}
pub fn add_cors_headers(
resp: &mut Response<Body>,
rule: &GarageCorsRule,
) -> Result<(), http::header::InvalidHeaderValue> {
let h = resp.headers_mut();
h.insert(
ACCESS_CONTROL_ALLOW_ORIGIN,
rule.allow_origins.join(", ").parse()?,
);
h.insert(
ACCESS_CONTROL_ALLOW_METHODS,
rule.allow_methods.join(", ").parse()?,
);
h.insert(
ACCESS_CONTROL_ALLOW_HEADERS,
rule.allow_headers.join(", ").parse()?,
);
h.insert(
ACCESS_CONTROL_EXPOSE_HEADERS,
rule.expose_headers.join(", ").parse()?,
);
Ok(())
}
// ---- SERIALIZATION AND DESERIALIZATION TO/FROM S3 XML ----
#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)]
#[serde(rename = "CORSConfiguration")]
pub struct CorsConfiguration {
@@ -217,49 +312,6 @@ impl CorsRule {
}
}
pub fn cors_rule_matches<'a, HI, S>(
rule: &GarageCorsRule,
origin: &'a str,
method: &'a str,
mut request_headers: HI,
) -> bool
where
HI: Iterator<Item = S>,
S: AsRef<str>,
{
rule.allow_origins.iter().any(|x| x == "*" || x == origin)
&& rule.allow_methods.iter().any(|x| x == "*" || x == method)
&& request_headers.all(|h| {
rule.allow_headers
.iter()
.any(|x| x == "*" || x == h.as_ref())
})
}
pub fn add_cors_headers(
resp: &mut Response<Body>,
rule: &GarageCorsRule,
) -> Result<(), http::header::InvalidHeaderValue> {
let h = resp.headers_mut();
h.insert(
ACCESS_CONTROL_ALLOW_ORIGIN,
rule.allow_origins.join(", ").parse()?,
);
h.insert(
ACCESS_CONTROL_ALLOW_METHODS,
rule.allow_methods.join(", ").parse()?,
);
h.insert(
ACCESS_CONTROL_ALLOW_HEADERS,
rule.allow_headers.join(", ").parse()?,
);
h.insert(
ACCESS_CONTROL_EXPOSE_HEADERS,
rule.expose_headers.join(", ").parse()?,
);
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
+4 -1
View File
@@ -81,7 +81,10 @@ pub async fn handle_delete_objects(
content_sha256: Option<Hash>,
) -> Result<Response<Body>, Error> {
let body = hyper::body::to_bytes(req.into_body()).await?;
verify_signed_content(content_sha256, &body[..])?;
if let Some(content_sha256) = content_sha256 {
verify_signed_content(content_sha256, &body[..])?;
}
let cmd_xml = roxmltree::Document::parse(std::str::from_utf8(&body)?)?;
let cmd = parse_delete_objects_xml(&cmd_xml).ok_or_bad_request("Invalid delete XML query")?;
+244 -98
View File
@@ -3,6 +3,10 @@ use std::sync::Arc;
use std::time::{Duration, UNIX_EPOCH};
use futures::stream::*;
use http::header::{
ACCEPT_RANGES, CONTENT_LENGTH, CONTENT_RANGE, CONTENT_TYPE, ETAG, IF_MODIFIED_SINCE,
IF_NONE_MATCH, LAST_MODIFIED, RANGE,
};
use hyper::body::Bytes;
use hyper::{Body, Request, Response, StatusCode};
@@ -11,9 +15,12 @@ use garage_util::data::*;
use garage_model::garage::Garage;
use garage_model::object_table::*;
use garage_model::version_table::*;
use crate::error::*;
const X_AMZ_MP_PARTS_COUNT: &str = "x-amz-mp-parts-count";
fn object_headers(
version: &ObjectVersion,
version_meta: &ObjectVersionMeta,
@@ -24,15 +31,12 @@ fn object_headers(
let date_str = httpdate::fmt_http_date(date);
let mut resp = Response::builder()
.header(
"Content-Type",
version_meta.headers.content_type.to_string(),
)
.header("Last-Modified", date_str)
.header("Accept-Ranges", "bytes".to_string());
.header(CONTENT_TYPE, version_meta.headers.content_type.to_string())
.header(LAST_MODIFIED, date_str)
.header(ACCEPT_RANGES, "bytes".to_string());
if !version_meta.etag.is_empty() {
resp = resp.header("ETag", format!("\"{}\"", version_meta.etag));
resp = resp.header(ETAG, format!("\"{}\"", version_meta.etag));
}
for (k, v) in version_meta.headers.other.iter() {
@@ -52,7 +56,7 @@ fn try_answer_cached(
// precedence and If-Modified-Since is ignored (as per 6.Precedence from rfc7232). The rational
// being that etag based matching is more accurate, it has no issue with sub-second precision
// for instance (in case of very fast updates)
let cached = if let Some(none_match) = req.headers().get(http::header::IF_NONE_MATCH) {
let cached = if let Some(none_match) = req.headers().get(IF_NONE_MATCH) {
let none_match = none_match.to_str().ok()?;
let expected = format!("\"{}\"", version_meta.etag);
let found = none_match
@@ -60,7 +64,7 @@ fn try_answer_cached(
.map(str::trim)
.any(|etag| etag == expected || etag == "\"*\"");
found
} else if let Some(modified_since) = req.headers().get(http::header::IF_MODIFIED_SINCE) {
} else if let Some(modified_since) = req.headers().get(IF_MODIFIED_SINCE) {
let modified_since = modified_since.to_str().ok()?;
let client_date = httpdate::parse_http_date(modified_since).ok()?;
let server_date = UNIX_EPOCH + Duration::from_millis(version.timestamp);
@@ -87,6 +91,7 @@ pub async fn handle_head(
req: &Request<Body>,
bucket_id: Uuid,
key: &str,
part_number: Option<u64>,
) -> Result<Response<Body>, Error> {
let object = garage
.object_table
@@ -94,30 +99,78 @@ pub async fn handle_head(
.await?
.ok_or(Error::NoSuchKey)?;
let version = object
let object_version = object
.versions()
.iter()
.rev()
.find(|v| v.is_data())
.ok_or(Error::NoSuchKey)?;
let version_meta = match &version.state {
ObjectVersionState::Complete(ObjectVersionData::Inline(meta, _)) => meta,
ObjectVersionState::Complete(ObjectVersionData::FirstBlock(meta, _)) => meta,
let version_data = match &object_version.state {
ObjectVersionState::Complete(c) => c,
_ => unreachable!(),
};
if let Some(cached) = try_answer_cached(version, version_meta, req) {
let version_meta = match version_data {
ObjectVersionData::Inline(meta, _) => meta,
ObjectVersionData::FirstBlock(meta, _) => meta,
_ => unreachable!(),
};
if let Some(cached) = try_answer_cached(object_version, version_meta, req) {
return Ok(cached);
}
let body: Body = Body::empty();
let response = object_headers(version, version_meta)
.header("Content-Length", format!("{}", version_meta.size))
.status(StatusCode::OK)
.body(body)
.unwrap();
Ok(response)
if let Some(pn) = part_number {
match version_data {
ObjectVersionData::Inline(_, bytes) => {
if pn != 1 {
return Err(Error::InvalidPart);
}
Ok(object_headers(object_version, version_meta)
.header(CONTENT_LENGTH, format!("{}", bytes.len()))
.header(
CONTENT_RANGE,
format!("bytes 0-{}/{}", bytes.len() - 1, bytes.len()),
)
.header(X_AMZ_MP_PARTS_COUNT, "1")
.status(StatusCode::PARTIAL_CONTENT)
.body(Body::empty())?)
}
ObjectVersionData::FirstBlock(_, _) => {
let version = garage
.version_table
.get(&object_version.uuid, &EmptyKey)
.await?
.ok_or(Error::NoSuchKey)?;
let (part_offset, part_end) =
calculate_part_bounds(&version, pn).ok_or(Error::InvalidPart)?;
let n_parts = version.parts_etags.items().len();
Ok(object_headers(object_version, version_meta)
.header(CONTENT_LENGTH, format!("{}", part_end - part_offset))
.header(
CONTENT_RANGE,
format!(
"bytes {}-{}/{}",
part_offset,
part_end - 1,
version_meta.size
),
)
.header(X_AMZ_MP_PARTS_COUNT, format!("{}", n_parts))
.status(StatusCode::PARTIAL_CONTENT)
.body(Body::empty())?)
}
_ => unreachable!(),
}
} else {
Ok(object_headers(object_version, version_meta)
.header(CONTENT_LENGTH, format!("{}", version_meta.size))
.status(StatusCode::OK)
.body(Body::empty())?)
}
}
/// Handle GET request
@@ -126,6 +179,7 @@ pub async fn handle_get(
req: &Request<Body>,
bucket_id: Uuid,
key: &str,
part_number: Option<u64>,
) -> Result<Response<Body>, Error> {
let object = garage
.object_table
@@ -154,35 +208,31 @@ pub async fn handle_get(
return Ok(cached);
}
let range = match req.headers().get("range") {
Some(range) => {
let range_str = range.to_str()?;
let mut ranges = http_range::HttpRange::parse(range_str, last_v_meta.size)
.map_err(|e| (e, last_v_meta.size))?;
if ranges.len() > 1 {
// garage does not support multi-range requests yet, so we respond with the entire
// object when multiple ranges are requested
None
} else {
ranges.pop()
}
match (part_number, parse_range_header(req, last_v_meta.size)?) {
(Some(_), Some(_)) => {
return Err(Error::BadRequest(
"Cannot specify both partNumber and Range header".into(),
));
}
None => None,
};
if let Some(range) = range {
return handle_get_range(
garage,
last_v,
last_v_data,
last_v_meta,
range.start,
range.start + range.length,
)
.await;
(Some(pn), None) => {
return handle_get_part(garage, last_v, last_v_data, last_v_meta, pn).await;
}
(None, Some(range)) => {
return handle_get_range(
garage,
last_v,
last_v_data,
last_v_meta,
range.start,
range.start + range.length,
)
.await;
}
(None, None) => (),
}
let resp_builder = object_headers(last_v, last_v_meta)
.header("Content-Length", format!("{}", last_v_meta.size))
.header(CONTENT_LENGTH, format!("{}", last_v_meta.size))
.status(StatusCode::OK);
match &last_v_data {
@@ -238,9 +288,9 @@ async fn handle_get_range(
end: u64,
) -> Result<Response<Body>, Error> {
let resp_builder = object_headers(version, version_meta)
.header("Content-Length", format!("{}", end - begin))
.header(CONTENT_LENGTH, format!("{}", end - begin))
.header(
"Content-Range",
CONTENT_RANGE,
format!("bytes {}-{}/{}", begin, end - 1, version_meta.size),
)
.status(StatusCode::PARTIAL_CONTENT);
@@ -258,58 +308,154 @@ async fn handle_get_range(
}
}
ObjectVersionData::FirstBlock(_meta, _first_block_hash) => {
let version = garage.version_table.get(&version.uuid, &EmptyKey).await?;
let version = match version {
Some(v) => v,
None => return Err(Error::NoSuchKey),
};
let version = garage
.version_table
.get(&version.uuid, &EmptyKey)
.await?
.ok_or(Error::NoSuchKey)?;
// We will store here the list of blocks that have an intersection with the requested
// range, as well as their "true offset", which is their actual offset in the complete
// file (whereas block.offset designates the offset of the block WITHIN THE PART
// block.part_number, which is not the same in the case of a multipart upload)
let mut blocks = Vec::with_capacity(std::cmp::min(
version.blocks.len(),
4 + ((end - begin) / std::cmp::max(version.blocks.items()[0].1.size as u64, 1024))
as usize,
));
let mut true_offset = 0;
for (_, b) in version.blocks.items().iter() {
if true_offset >= end {
break;
}
// Keep only blocks that have an intersection with the requested range
if true_offset < end && true_offset + b.size > begin {
blocks.push((*b, true_offset));
}
true_offset += b.size;
}
let body_stream = futures::stream::iter(blocks)
.map(move |(block, true_offset)| {
let garage = garage.clone();
async move {
let data = garage.block_manager.rpc_get_block(&block.hash).await?;
let data = Bytes::from(data);
let start_in_block = if true_offset > begin {
0
} else {
begin - true_offset
};
let end_in_block = if true_offset + block.size < end {
block.size
} else {
end - true_offset
};
Result::<Bytes, Error>::Ok(
data.slice(start_in_block as usize..end_in_block as usize),
)
}
})
.buffered(2);
let body = hyper::body::Body::wrap_stream(body_stream);
let body = body_from_blocks_range(garage, version.blocks.items(), begin, end);
Ok(resp_builder.body(body)?)
}
}
}
async fn handle_get_part(
garage: Arc<Garage>,
object_version: &ObjectVersion,
version_data: &ObjectVersionData,
version_meta: &ObjectVersionMeta,
part_number: u64,
) -> Result<Response<Body>, Error> {
let resp_builder =
object_headers(object_version, version_meta).status(StatusCode::PARTIAL_CONTENT);
match version_data {
ObjectVersionData::Inline(_, bytes) => {
if part_number != 1 {
return Err(Error::InvalidPart);
}
Ok(resp_builder
.header(CONTENT_LENGTH, format!("{}", bytes.len()))
.header(
CONTENT_RANGE,
format!("bytes {}-{}/{}", 0, bytes.len() - 1, bytes.len()),
)
.header(X_AMZ_MP_PARTS_COUNT, "1")
.body(Body::from(bytes.to_vec()))?)
}
ObjectVersionData::FirstBlock(_, _) => {
let version = garage
.version_table
.get(&object_version.uuid, &EmptyKey)
.await?
.ok_or(Error::NoSuchKey)?;
let (begin, end) =
calculate_part_bounds(&version, part_number).ok_or(Error::InvalidPart)?;
let n_parts = version.parts_etags.items().len();
let body = body_from_blocks_range(garage, version.blocks.items(), begin, end);
Ok(resp_builder
.header(CONTENT_LENGTH, format!("{}", end - begin))
.header(
CONTENT_RANGE,
format!("bytes {}-{}/{}", begin, end - 1, version_meta.size),
)
.header(X_AMZ_MP_PARTS_COUNT, format!("{}", n_parts))
.body(body)?)
}
_ => unreachable!(),
}
}
fn parse_range_header(
req: &Request<Body>,
total_size: u64,
) -> Result<Option<http_range::HttpRange>, Error> {
let range = match req.headers().get(RANGE) {
Some(range) => {
let range_str = range.to_str()?;
let mut ranges =
http_range::HttpRange::parse(range_str, total_size).map_err(|e| (e, total_size))?;
if ranges.len() > 1 {
// garage does not support multi-range requests yet, so we respond with the entire
// object when multiple ranges are requested
None
} else {
ranges.pop()
}
}
None => None,
};
Ok(range)
}
fn calculate_part_bounds(v: &Version, part_number: u64) -> Option<(u64, u64)> {
let mut offset = 0;
for (i, (bk, bv)) in v.blocks.items().iter().enumerate() {
if bk.part_number == part_number {
let size: u64 = v.blocks.items()[i..]
.iter()
.take_while(|(k, _)| k.part_number == part_number)
.map(|(_, v)| v.size)
.sum();
return Some((offset, offset + size));
}
offset += bv.size;
}
None
}
fn body_from_blocks_range(
garage: Arc<Garage>,
all_blocks: &[(VersionBlockKey, VersionBlock)],
begin: u64,
end: u64,
) -> Body {
// We will store here the list of blocks that have an intersection with the requested
// range, as well as their "true offset", which is their actual offset in the complete
// file (whereas block.offset designates the offset of the block WITHIN THE PART
// block.part_number, which is not the same in the case of a multipart upload)
let mut blocks: Vec<(VersionBlock, u64)> = Vec::with_capacity(std::cmp::min(
all_blocks.len(),
4 + ((end - begin) / std::cmp::max(all_blocks[0].1.size as u64, 1024)) as usize,
));
let mut true_offset = 0;
for (_, b) in all_blocks.iter() {
if true_offset >= end {
break;
}
// Keep only blocks that have an intersection with the requested range
if true_offset < end && true_offset + b.size > begin {
blocks.push((*b, true_offset));
}
true_offset += b.size;
}
let body_stream = futures::stream::iter(blocks)
.map(move |(block, true_offset)| {
let garage = garage.clone();
async move {
let data = garage.block_manager.rpc_get_block(&block.hash).await?;
let data = Bytes::from(data);
let start_in_block = if true_offset > begin {
0
} else {
begin - true_offset
};
let end_in_block = if true_offset + block.size < end {
block.size
} else {
end - true_offset
};
Result::<Bytes, Error>::Ok(
data.slice(start_in_block as usize..end_in_block as usize),
)
}
})
.buffered(2);
hyper::body::Body::wrap_stream(body_stream)
}
+371 -5
View File
@@ -1,3 +1,4 @@
use std::cmp::Ordering;
use std::collections::{BTreeMap, BTreeSet};
use std::iter::{Iterator, Peekable};
use std::sync::Arc;
@@ -10,12 +11,18 @@ use garage_util::time::*;
use garage_model::garage::Garage;
use garage_model::object_table::*;
use garage_model::version_table::Version;
use garage_table::EmptyKey;
use crate::encoding::*;
use crate::error::*;
use crate::s3_put;
use crate::s3_xml;
const DUMMY_NAME: &str = "Dummy Key";
const DUMMY_KEY: &str = "GKDummyKey";
#[derive(Debug)]
pub struct ListQueryCommon {
pub bucket_name: String,
@@ -42,6 +49,16 @@ pub struct ListMultipartUploadsQuery {
pub common: ListQueryCommon,
}
#[derive(Debug)]
pub struct ListPartsQuery {
pub bucket_name: String,
pub bucket_id: Uuid,
pub key: String,
pub upload_id: String,
pub part_number_marker: Option<u64>,
pub max_parts: u64,
}
pub async fn handle_list(
garage: Arc<Garage>,
query: &ListObjectsQuery,
@@ -54,6 +71,7 @@ pub async fn handle_list(
}
};
debug!("ListObjects {:?}", query);
let mut acc = query.build_accumulator();
let pagination = fetch_list_entries(&query.common, query.begin()?, &mut acc, &io).await?;
@@ -121,7 +139,7 @@ pub async fn handle_list(
key: uriencode_maybe(key, query.common.urlencode_resp),
last_modified: s3_xml::Value(msec_to_rfc3339(info.last_modified)),
size: s3_xml::IntValue(info.size as i64),
etag: s3_xml::Value(format!("\"{}\"", info.etag.to_string())),
etag: s3_xml::Value(format!("\"{}\"", info.etag)),
storage_class: s3_xml::Value("STANDARD".to_string()),
})
.collect(),
@@ -152,6 +170,7 @@ pub async fn handle_list_multipart_upload(
}
};
debug!("ListMultipartUploads {:?}", query);
let mut acc = query.build_accumulator();
let pagination = fetch_list_entries(&query.common, query.begin()?, &mut acc, &io).await?;
@@ -208,12 +227,12 @@ pub async fn handle_list_multipart_upload(
upload_id: s3_xml::Value(hex::encode(uuid)),
storage_class: s3_xml::Value("STANDARD".to_string()),
initiator: s3_xml::Initiator {
display_name: s3_xml::Value("Dummy Key".to_string()),
id: s3_xml::Value("GKDummyKey".to_string()),
display_name: s3_xml::Value(DUMMY_NAME.to_string()),
id: s3_xml::Value(DUMMY_KEY.to_string()),
},
owner: s3_xml::Owner {
display_name: s3_xml::Value("Dummy Key".to_string()),
id: s3_xml::Value("GKDummyKey".to_string()),
display_name: s3_xml::Value(DUMMY_NAME.to_string()),
id: s3_xml::Value(DUMMY_KEY.to_string()),
},
})
.collect(),
@@ -233,6 +252,57 @@ pub async fn handle_list_multipart_upload(
.body(Body::from(xml.into_bytes()))?)
}
pub async fn handle_list_parts(
garage: Arc<Garage>,
query: &ListPartsQuery,
) -> Result<Response<Body>, Error> {
debug!("ListParts {:?}", query);
let upload_id = s3_put::decode_upload_id(&query.upload_id)?;
let (object, version) = futures::try_join!(
garage.object_table.get(&query.bucket_id, &query.key),
garage.version_table.get(&upload_id, &EmptyKey),
)?;
let (info, next) = fetch_part_info(query, object, version, upload_id)?;
let result = s3_xml::ListPartsResult {
xmlns: (),
bucket: s3_xml::Value(query.bucket_name.to_string()),
key: s3_xml::Value(query.key.to_string()),
upload_id: s3_xml::Value(query.upload_id.to_string()),
part_number_marker: query.part_number_marker.map(|e| s3_xml::IntValue(e as i64)),
next_part_number_marker: next.map(|e| s3_xml::IntValue(e as i64)),
max_parts: s3_xml::IntValue(query.max_parts as i64),
is_truncated: s3_xml::Value(next.map(|_| "true").unwrap_or("false").to_string()),
parts: info
.iter()
.map(|part| s3_xml::PartItem {
etag: s3_xml::Value(format!("\"{}\"", part.etag)),
last_modified: s3_xml::Value(msec_to_rfc3339(part.timestamp)),
part_number: s3_xml::IntValue(part.part_number as i64),
size: s3_xml::IntValue(part.size as i64),
})
.collect(),
initiator: s3_xml::Initiator {
display_name: s3_xml::Value(DUMMY_NAME.to_string()),
id: s3_xml::Value(DUMMY_KEY.to_string()),
},
owner: s3_xml::Owner {
display_name: s3_xml::Value(DUMMY_NAME.to_string()),
id: s3_xml::Value(DUMMY_KEY.to_string()),
},
storage_class: s3_xml::Value("STANDARD".to_string()),
};
let xml = s3_xml::to_xml_with_header(&result)?;
Ok(Response::builder()
.header("Content-Type", "application/xml")
.body(Body::from(xml.into_bytes()))?)
}
/*
* Private enums and structs
*/
@@ -250,6 +320,14 @@ struct UploadInfo {
timestamp: u64,
}
#[derive(Debug, PartialEq)]
struct PartInfo {
etag: String,
timestamp: u64,
part_number: u64,
size: u64,
}
enum ExtractionResult {
NoMore,
Filled,
@@ -364,6 +442,109 @@ where
}
}
fn fetch_part_info(
query: &ListPartsQuery,
object: Option<Object>,
version: Option<Version>,
upload_id: Uuid,
) -> Result<(Vec<PartInfo>, Option<u64>), Error> {
// Check results
let object = object.ok_or(Error::NoSuchKey)?;
let obj_version = object
.versions()
.iter()
.find(|v| v.uuid == upload_id && v.is_uploading())
.ok_or(Error::NoSuchUpload)?;
let version = version.ok_or(Error::NoSuchKey)?;
// Cut the beginning of our 2 vectors if required
let (etags, blocks) = match &query.part_number_marker {
Some(marker) => {
let next = marker + 1;
let part_idx = into_ok_or_err(
version
.parts_etags
.items()
.binary_search_by(|(part_num, _)| part_num.cmp(&next)),
);
let parts = &version.parts_etags.items()[part_idx..];
let block_idx = into_ok_or_err(
version
.blocks
.items()
.binary_search_by(|(vkey, _)| vkey.part_number.cmp(&next)),
);
let blocks = &version.blocks.items()[block_idx..];
(parts, blocks)
}
None => (version.parts_etags.items(), version.blocks.items()),
};
// Use the block vector to compute a (part_number, size) vector
let mut size = Vec::<(u64, u64)>::new();
blocks.iter().for_each(|(key, val)| {
let mut new_size = val.size;
match size.pop() {
Some((part_number, size)) if part_number == key.part_number => new_size += size,
Some(v) => size.push(v),
None => (),
}
size.push((key.part_number, new_size))
});
// Merge the etag vector and size vector to build a PartInfo vector
let max_parts = query.max_parts as usize;
let (mut etag_iter, mut size_iter) = (etags.iter().peekable(), size.iter().peekable());
let mut info = Vec::<PartInfo>::with_capacity(max_parts);
while info.len() < max_parts {
match (etag_iter.peek(), size_iter.peek()) {
(Some((ep, etag)), Some((sp, size))) => match ep.cmp(sp) {
Ordering::Less => {
debug!("ETag information ignored due to missing corresponding block information. Query: {:?}", query);
etag_iter.next();
}
Ordering::Equal => {
info.push(PartInfo {
etag: etag.to_string(),
timestamp: obj_version.timestamp,
part_number: *ep,
size: *size,
});
etag_iter.next();
size_iter.next();
}
Ordering::Greater => {
debug!("Block information ignored due to missing corresponding ETag information. Query: {:?}", query);
size_iter.next();
}
},
(None, None) => return Ok((info, None)),
_ => {
debug!(
"Additional block or ETag information ignored. Query: {:?}",
query
);
return Ok((info, None));
}
}
}
match info.last() {
Some(part_info) => {
let pagination = Some(part_info.part_number);
Ok((info, pagination))
}
None => Ok((info, None)),
}
}
/*
* ListQuery logic
*/
@@ -715,6 +896,14 @@ impl ExtractAccumulator for UploadAccumulator {
* Utility functions
*/
/// This is a stub for Result::into_ok_or_err that is not yet in Rust stable
fn into_ok_or_err<T>(r: Result<T, T>) -> T {
match r {
Ok(r) => r,
Err(r) => r,
}
}
/// Returns the common prefix of the object given the query prefix and delimiter
fn common_prefix<'a>(object: &'a Object, query: &ListQueryCommon) -> Option<&'a str> {
match &query.delimiter {
@@ -766,6 +955,8 @@ fn key_after_prefix(pfx: &str) -> Option<String> {
#[cfg(test)]
mod tests {
use super::*;
use garage_model::version_table::*;
use garage_util::*;
use std::iter::FromIterator;
const TS: u64 = 1641394898314;
@@ -1014,4 +1205,179 @@ mod tests {
Ok(())
}
fn version() -> Version {
let uuid = Uuid::from([0x08; 32]);
let blocks = vec![
(
VersionBlockKey {
part_number: 1,
offset: 1,
},
VersionBlock {
hash: uuid,
size: 3,
},
),
(
VersionBlockKey {
part_number: 1,
offset: 2,
},
VersionBlock {
hash: uuid,
size: 2,
},
),
(
VersionBlockKey {
part_number: 2,
offset: 1,
},
VersionBlock {
hash: uuid,
size: 8,
},
),
(
VersionBlockKey {
part_number: 5,
offset: 1,
},
VersionBlock {
hash: uuid,
size: 7,
},
),
(
VersionBlockKey {
part_number: 8,
offset: 1,
},
VersionBlock {
hash: uuid,
size: 5,
},
),
];
let etags = vec![
(1, "etag1".to_string()),
(3, "etag2".to_string()),
(5, "etag3".to_string()),
(8, "etag4".to_string()),
(9, "etag5".to_string()),
];
Version {
bucket_id: uuid,
key: "a".to_string(),
uuid: uuid,
deleted: false.into(),
blocks: crdt::Map::<VersionBlockKey, VersionBlock>::from_iter(blocks),
parts_etags: crdt::Map::<u64, String>::from_iter(etags),
}
}
fn obj() -> Object {
Object::new(bucket(), "d".to_string(), vec![objup_version([0x08; 32])])
}
#[test]
fn test_fetch_part_info() -> Result<(), Error> {
let uuid = Uuid::from([0x08; 32]);
let mut query = ListPartsQuery {
bucket_name: "a".to_string(),
bucket_id: uuid,
key: "a".to_string(),
upload_id: "xx".to_string(),
part_number_marker: None,
max_parts: 2,
};
assert!(
fetch_part_info(&query, None, None, uuid).is_err(),
"No object and version should fail"
);
assert!(
fetch_part_info(&query, Some(obj()), None, uuid).is_err(),
"No version should faild"
);
assert!(
fetch_part_info(&query, None, Some(version()), uuid).is_err(),
"No object should fail"
);
// Start from the beginning but with limited size to trigger pagination
let (info, pagination) = fetch_part_info(&query, Some(obj()), Some(version()), uuid)?;
assert_eq!(pagination.unwrap(), 5);
assert_eq!(
info,
vec![
PartInfo {
etag: "etag1".to_string(),
timestamp: TS,
part_number: 1,
size: 5
},
PartInfo {
etag: "etag3".to_string(),
timestamp: TS,
part_number: 5,
size: 7
},
]
);
// Use previous pagination to make a new request
query.part_number_marker = Some(pagination.unwrap());
let (info, pagination) = fetch_part_info(&query, Some(obj()), Some(version()), uuid)?;
assert!(pagination.is_none());
assert_eq!(
info,
vec![PartInfo {
etag: "etag4".to_string(),
timestamp: TS,
part_number: 8,
size: 5
},]
);
// Trying to access a part that is way larger than registered ones
query.part_number_marker = Some(9999);
let (info, pagination) = fetch_part_info(&query, Some(obj()), Some(version()), uuid)?;
assert!(pagination.is_none());
assert_eq!(info, vec![]);
// Try without any limitation
query.max_parts = 1000;
query.part_number_marker = None;
let (info, pagination) = fetch_part_info(&query, Some(obj()), Some(version()), uuid)?;
assert!(pagination.is_none());
assert_eq!(
info,
vec![
PartInfo {
etag: "etag1".to_string(),
timestamp: TS,
part_number: 1,
size: 5
},
PartInfo {
etag: "etag3".to_string(),
timestamp: TS,
part_number: 5,
size: 7
},
PartInfo {
etag: "etag4".to_string(),
timestamp: TS,
part_number: 8,
size: 5
},
]
);
Ok(())
}
}
+138 -30
View File
@@ -1,8 +1,10 @@
use std::collections::{BTreeMap, VecDeque};
use std::collections::{BTreeMap, BTreeSet, VecDeque};
use std::sync::Arc;
use futures::stream::*;
use hyper::{Body, Request, Response};
use chrono::{DateTime, NaiveDateTime, Utc};
use futures::{prelude::*, TryFutureExt};
use hyper::body::{Body, Bytes};
use hyper::{Request, Response};
use md5::{digest::generic_array::*, Digest as Md5Digest, Md5};
use sha2::Sha256;
@@ -14,19 +16,23 @@ use garage_util::time::*;
use garage_model::block::INLINE_THRESHOLD;
use garage_model::block_ref_table::*;
use garage_model::garage::Garage;
use garage_model::key_table::Key;
use garage_model::object_table::*;
use garage_model::version_table::*;
use crate::error::*;
use crate::s3_xml;
use crate::signature::verify_signed_content;
use crate::signature::streaming::SignedPayloadStream;
use crate::signature::LONG_DATETIME;
use crate::signature::{compute_scope, verify_signed_content};
pub async fn handle_put(
garage: Arc<Garage>,
req: Request<Body>,
bucket_id: Uuid,
key: &str,
content_sha256: Option<Hash>,
api_key: &Key,
mut content_sha256: Option<Hash>,
) -> Result<Response<Body>, Error> {
// Generate identity of new version
let version_uuid = gen_uuid();
@@ -40,11 +46,53 @@ pub async fn handle_put(
Some(x) => Some(x.to_str()?.to_string()),
None => None,
};
let payload_seed_signature = match req.headers().get("x-amz-content-sha256") {
Some(header) if header == "STREAMING-AWS4-HMAC-SHA256-PAYLOAD" => {
let content_sha256 = content_sha256
.take()
.ok_or_bad_request("No signature provided")?;
Some(content_sha256)
}
_ => None,
};
// Parse body of uploaded file
let body = req.into_body();
let (head, body) = req.into_parts();
let body = body.map_err(Error::from);
let mut chunker = BodyChunker::new(body, garage.config.block_size);
let body = if let Some(signature) = payload_seed_signature {
let secret_key = &api_key
.state
.as_option()
.ok_or_internal_error("Deleted key state")?
.secret_key;
let date = head
.headers
.get("x-amz-date")
.ok_or_bad_request("Missing X-Amz-Date field")?
.to_str()?;
let date: NaiveDateTime =
NaiveDateTime::parse_from_str(date, LONG_DATETIME).ok_or_bad_request("Invalid date")?;
let date: DateTime<Utc> = DateTime::from_utc(date, Utc);
let scope = compute_scope(&date, &garage.config.s3_api.s3_region);
let signing_hmac = crate::signature::signing_hmac(
&date,
secret_key,
&garage.config.s3_api.s3_region,
"s3",
)
.ok_or_internal_error("Unable to build signing HMAC")?;
SignedPayloadStream::new(body, signing_hmac, date, &scope, signature)?
.map_err(Error::from)
.boxed()
} else {
body.boxed()
};
let mut chunker = StreamChunker::new(body, garage.config.block_size);
let first_block = chunker.next().await?.unwrap_or_default();
// If body is small enough, store it directly in the object table
@@ -178,13 +226,13 @@ fn ensure_checksum_matches(
Ok(())
}
async fn read_and_put_blocks(
async fn read_and_put_blocks<S: Stream<Item = Result<Bytes, Error>> + Unpin>(
garage: &Garage,
version: &Version,
part_number: u64,
first_block: Vec<u8>,
first_block_hash: Hash,
chunker: &mut BodyChunker,
chunker: &mut StreamChunker<S>,
) -> Result<(u64, GenericArray<u8, typenum::U16>, Hash), Error> {
let mut md5hasher = Md5::new();
let mut sha256hasher = Sha256::new();
@@ -205,8 +253,11 @@ async fn read_and_put_blocks(
.rpc_put_block(first_block_hash, first_block);
loop {
let (_, _, next_block) =
futures::try_join!(put_curr_block, put_curr_version_block, chunker.next())?;
let (_, _, next_block) = futures::try_join!(
put_curr_block.map_err(Error::from),
put_curr_version_block.map_err(Error::from),
chunker.next(),
)?;
if let Some(block) = next_block {
md5hasher.update(&block[..]);
sha256hasher.update(&block[..]);
@@ -266,32 +317,34 @@ async fn put_block_meta(
Ok(())
}
struct BodyChunker {
body: Body,
struct StreamChunker<S: Stream<Item = Result<Bytes, Error>>> {
stream: S,
read_all: bool,
block_size: usize,
buf: VecDeque<u8>,
}
impl BodyChunker {
fn new(body: Body, block_size: usize) -> Self {
impl<S: Stream<Item = Result<Bytes, Error>> + Unpin> StreamChunker<S> {
fn new(stream: S, block_size: usize) -> Self {
Self {
body,
stream,
read_all: false,
block_size,
buf: VecDeque::with_capacity(2 * block_size),
}
}
async fn next(&mut self) -> Result<Option<Vec<u8>>, GarageError> {
async fn next(&mut self) -> Result<Option<Vec<u8>>, Error> {
while !self.read_all && self.buf.len() < self.block_size {
if let Some(block) = self.body.next().await {
if let Some(block) = self.stream.next().await {
let bytes = block?;
trace!("Body next: {} bytes", bytes.len());
self.buf.extend(&bytes[..]);
self.buf.extend(bytes);
} else {
self.read_all = true;
}
}
if self.buf.is_empty() {
Ok(None)
} else if self.buf.len() <= self.block_size {
@@ -368,12 +421,20 @@ pub async fn handle_put_part(
// Read first chuck, and at the same time try to get object to see if it exists
let key = key.to_string();
let mut chunker = BodyChunker::new(req.into_body(), garage.config.block_size);
let body = req.into_body().map_err(Error::from);
let mut chunker = StreamChunker::new(body, garage.config.block_size);
let (object, version, first_block) = futures::try_join!(
garage.object_table.get(&bucket_id, &key),
garage.version_table.get(&version_uuid, &EmptyKey),
chunker.next()
garage
.object_table
.get(&bucket_id, &key)
.map_err(Error::from),
garage
.version_table
.get(&version_uuid, &EmptyKey)
.map_err(Error::from),
chunker.next(),
)?;
// Check object is valid and multipart block can be accepted
@@ -444,10 +505,13 @@ pub async fn handle_complete_multipart_upload(
content_sha256: Option<Hash>,
) -> Result<Response<Body>, Error> {
let body = hyper::body::to_bytes(req.into_body()).await?;
verify_signed_content(content_sha256, &body[..])?;
if let Some(content_sha256) = content_sha256 {
verify_signed_content(content_sha256, &body[..])?;
}
let body_xml = roxmltree::Document::parse(std::str::from_utf8(&body)?)?;
let body_list_of_parts = parse_complete_multpart_upload_body(&body_xml)
let body_list_of_parts = parse_complete_multipart_upload_body(&body_xml)
.ok_or_bad_request("Invalid CompleteMultipartUpload XML")?;
debug!(
"CompleteMultipartUpload list of parts: {:?}",
@@ -456,6 +520,7 @@ pub async fn handle_complete_multipart_upload(
let version_uuid = decode_upload_id(upload_id)?;
// Get object and version
let key = key.to_string();
let (object, version) = futures::try_join!(
garage.object_table.get(&bucket_id, &key),
@@ -480,6 +545,31 @@ pub async fn handle_complete_multipart_upload(
_ => unreachable!(),
};
// Check that part numbers are an increasing sequence.
// (it doesn't need to start at 1 nor to be a continuous sequence,
// see discussion in #192)
if body_list_of_parts.is_empty() {
return Err(Error::EntityTooSmall);
}
if !body_list_of_parts
.iter()
.zip(body_list_of_parts.iter().skip(1))
.all(|(p1, p2)| p1.part_number < p2.part_number)
{
return Err(Error::InvalidPartOrder);
}
// Garage-specific restriction, see #204: part numbers must be
// consecutive starting at 1
if body_list_of_parts[0].part_number != 1
|| !body_list_of_parts
.iter()
.zip(body_list_of_parts.iter().skip(1))
.all(|(p1, p2)| p1.part_number + 1 == p2.part_number)
{
return Err(Error::NotImplemented("Garage does not support completing a Multipart upload with non-consecutive part numbers. This is a restriction of Garage's data model, which might be fixed in a future release. See issue #204 for more information on this topic.".into()));
}
// Check that the list of parts they gave us corresponds to the parts we have here
debug!("Expected parts from request: {:?}", body_list_of_parts);
debug!("Parts stored in version: {:?}", version.parts_etags.items());
@@ -492,18 +582,31 @@ pub async fn handle_complete_multipart_upload(
.iter()
.map(|x| (&x.part_number, &x.etag))
.eq(parts);
if !same_parts {
return Err(Error::InvalidPart);
}
// Check that all blocks belong to one of the parts
let block_parts = version
.blocks
.items()
.iter()
.map(|(bk, _)| bk.part_number)
.collect::<BTreeSet<_>>();
let same_parts = body_list_of_parts
.iter()
.map(|x| x.part_number)
.eq(block_parts.into_iter());
if !same_parts {
return Err(Error::BadRequest(
"We don't have the same parts".to_string(),
"Part numbers in block list and part list do not match. This can happen if a part was partially uploaded. Please abort the multipart upload and try again.".into(),
));
}
// Calculate etag of final object
// To understand how etags are calculated, read more here:
// https://teppen.io/2018/06/23/aws_s3_etags/
let num_parts = version.blocks.items().last().unwrap().0.part_number
- version.blocks.items().first().unwrap().0.part_number
+ 1;
let num_parts = body_list_of_parts.len();
let mut etag_md5_hasher = Md5::new();
for (_, etag) in version.parts_etags.items().iter() {
etag_md5_hasher.update(etag.as_bytes());
@@ -639,7 +742,7 @@ struct CompleteMultipartUploadPart {
part_number: u64,
}
fn parse_complete_multpart_upload_body(
fn parse_complete_multipart_upload_body(
xml: &roxmltree::Document,
) -> Option<Vec<CompleteMultipartUploadPart>> {
let mut parts = vec![];
@@ -651,6 +754,11 @@ fn parse_complete_multpart_upload_body(
}
for item in cmu.children() {
// Only parse <Part> nodes
if !item.is_element() {
continue;
}
if item.has_tag_name("Part") {
let etag = item.children().find(|e| e.has_tag_name("ETag"))?.text()?;
let part_number = item
+121 -215
View File
@@ -8,6 +8,17 @@ use hyper::{HeaderMap, Method, Request};
/// This macro is used to generate very repetitive match {} blocks in this module
/// It is _not_ made to be used anywhere else
macro_rules! s3_match {
(@match $enum:expr , [ $($endpoint:ident,)* ]) => {{
// usage: s3_match {@match my_enum, [ VariantWithField1, VariantWithField2 ..] }
// returns true if the variant was one of the listed variants, false otherwise.
use Endpoint::*;
match $enum {
$(
$endpoint { .. } => true,
)*
_ => false
}
}};
(@extract $enum:expr , $param:ident, [ $($endpoint:ident,)* ]) => {{
// usage: s3_match {@extract my_enum, field_name, [ VariantWithField1, VariantWithField2 ..] }
// returns Some(field_value), or None if the variant was not one of the listed variants.
@@ -19,10 +30,10 @@ macro_rules! s3_match {
_ => None
}
}};
(@gen_parser ($keyword:expr, $key:expr, $bucket:expr, $query:expr, $header:expr),
(@gen_parser ($keyword:expr, $key:expr, $query:expr, $header:expr),
key: [$($kw_k:ident $(if $required_k:ident)? $(header $header_k:expr)? => $api_k:ident $(($($conv_k:ident :: $param_k:ident),*))?,)*],
no_key: [$($kw_nk:ident $(if $required_nk:ident)? $(if_header $header_nk:expr)? => $api_nk:ident $(($($conv_nk:ident :: $param_nk:ident),*))?,)*]) => {{
// usage: s3_match {@gen_parser (keyword, key, bucket, query, header),
// usage: s3_match {@gen_parser (keyword, key, query, header),
// key: [
// SOME_KEYWORD => VariantWithKey,
// ...
@@ -38,7 +49,6 @@ macro_rules! s3_match {
match ($keyword, !$key.is_empty()){
$(
($kw_k, true) if true $(&& $query.$required_k.is_some())? $(&& $header.contains_key($header_k))? => Ok($api_k {
bucket: $bucket,
key: $key,
$($(
$param_k: s3_match!(@@parse_param $query, $conv_k, $param_k),
@@ -47,7 +57,6 @@ macro_rules! s3_match {
)*
$(
($kw_nk, false) $(if $query.$required_nk.is_some())? $(if $header.contains($header_nk))? => Ok($api_nk {
bucket: $bucket,
$($(
$param_nk: s3_match!(@@parse_param $query, $conv_nk, $param_nk),
)*)?
@@ -87,7 +96,6 @@ macro_rules! s3_match {
$(
$(#[$outer:meta])*
$variant:ident $({
bucket: String,
$($name:ident: $ty:ty,)*
})?,
)*
@@ -97,7 +105,6 @@ macro_rules! s3_match {
$(
$(#[$outer])*
$variant $({
bucket: String,
$($name: $ty, )*
})?,
)*
@@ -108,15 +115,6 @@ macro_rules! s3_match {
$(Endpoint::$variant $({ $($name: _,)* .. })? => stringify!($variant),)*
}
}
/// Get the bucket the request target. Returns None for requests not related to a bucket.
pub fn get_bucket(&self) -> Option<&str> {
match self {
$(
Endpoint::$variant $({ bucket, $($name: _,)* .. })? => s3_match!{@if ($(bucket $($name)*)?) then (Some(bucket)) else (None)},
)*
}
}
}
};
(@if ($($cond:tt)+) then ($($then:tt)*) else ($($else:tt)*)) => {
@@ -138,215 +136,158 @@ s3_match! {@func
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum Endpoint {
AbortMultipartUpload {
bucket: String,
key: String,
upload_id: String,
},
CompleteMultipartUpload {
bucket: String,
key: String,
upload_id: String,
},
CopyObject {
bucket: String,
key: String,
},
CreateBucket {
bucket: String,
},
CreateMultipartUpload {
bucket: String,
key: String,
},
DeleteBucket {
bucket: String,
},
DeleteBucketAnalyticsConfiguration {
bucket: String,
id: String,
},
DeleteBucketCors {
bucket: String,
},
DeleteBucketEncryption {
bucket: String,
},
DeleteBucketIntelligentTieringConfiguration {
bucket: String,
id: String,
},
DeleteBucketInventoryConfiguration {
bucket: String,
id: String,
},
DeleteBucketLifecycle {
bucket: String,
},
DeleteBucketMetricsConfiguration {
bucket: String,
id: String,
},
DeleteBucketOwnershipControls {
bucket: String,
},
DeleteBucketPolicy {
bucket: String,
},
DeleteBucketReplication {
bucket: String,
},
DeleteBucketTagging {
bucket: String,
},
DeleteBucketWebsite {
bucket: String,
},
DeleteObject {
bucket: String,
key: String,
version_id: Option<String>,
},
DeleteObjects {
bucket: String,
},
DeleteObjectTagging {
bucket: String,
key: String,
version_id: Option<String>,
},
DeletePublicAccessBlock {
bucket: String,
},
GetBucketAccelerateConfiguration {
bucket: String,
},
GetBucketAcl {
bucket: String,
},
GetBucketAnalyticsConfiguration {
bucket: String,
id: String,
},
GetBucketCors {
bucket: String,
},
GetBucketEncryption {
bucket: String,
},
GetBucketIntelligentTieringConfiguration {
bucket: String,
id: String,
},
GetBucketInventoryConfiguration {
bucket: String,
id: String,
},
GetBucketLifecycleConfiguration {
bucket: String,
},
GetBucketLocation {
bucket: String,
},
GetBucketLogging {
bucket: String,
},
GetBucketMetricsConfiguration {
bucket: String,
id: String,
},
GetBucketNotificationConfiguration {
bucket: String,
},
GetBucketOwnershipControls {
bucket: String,
},
GetBucketPolicy {
bucket: String,
},
GetBucketPolicyStatus {
bucket: String,
},
GetBucketReplication {
bucket: String,
},
GetBucketRequestPayment {
bucket: String,
},
GetBucketTagging {
bucket: String,
},
GetBucketVersioning {
bucket: String,
},
GetBucketWebsite {
bucket: String,
},
/// There are actually many more query parameters, used to add headers to the answer. They were
/// not added here as they are best handled in a dedicated route.
GetObject {
bucket: String,
key: String,
part_number: Option<u64>,
version_id: Option<String>,
},
GetObjectAcl {
bucket: String,
key: String,
version_id: Option<String>,
},
GetObjectLegalHold {
bucket: String,
key: String,
version_id: Option<String>,
},
GetObjectLockConfiguration {
bucket: String,
},
GetObjectRetention {
bucket: String,
key: String,
version_id: Option<String>,
},
GetObjectTagging {
bucket: String,
key: String,
version_id: Option<String>,
},
GetObjectTorrent {
bucket: String,
key: String,
},
GetPublicAccessBlock {
bucket: String,
},
HeadBucket {
bucket: String,
},
HeadObject {
bucket: String,
key: String,
part_number: Option<u64>,
version_id: Option<String>,
},
ListBucketAnalyticsConfigurations {
bucket: String,
continuation_token: Option<String>,
},
ListBucketIntelligentTieringConfigurations {
bucket: String,
continuation_token: Option<String>,
},
ListBucketInventoryConfigurations {
bucket: String,
continuation_token: Option<String>,
},
ListBucketMetricsConfigurations {
bucket: String,
continuation_token: Option<String>,
},
ListBuckets,
ListMultipartUploads {
bucket: String,
delimiter: Option<char>,
encoding_type: Option<String>,
key_marker: Option<String>,
@@ -355,7 +296,6 @@ pub enum Endpoint {
upload_id_marker: Option<String>,
},
ListObjects {
bucket: String,
delimiter: Option<char>,
encoding_type: Option<String>,
marker: Option<String>,
@@ -363,7 +303,6 @@ pub enum Endpoint {
prefix: Option<String>,
},
ListObjectsV2 {
bucket: String,
// This value should always be 2. It is not checked when constructing the struct
list_type: String,
continuation_token: Option<String>,
@@ -375,7 +314,6 @@ pub enum Endpoint {
start_after: Option<String>,
},
ListObjectVersions {
bucket: String,
delimiter: Option<char>,
encoding_type: Option<String>,
key_marker: Option<String>,
@@ -384,119 +322,90 @@ pub enum Endpoint {
version_id_marker: Option<String>,
},
ListParts {
bucket: String,
key: String,
max_parts: Option<u64>,
part_number_marker: Option<u64>,
upload_id: String,
},
Options,
PutBucketAccelerateConfiguration {
bucket: String,
},
PutBucketAcl {
bucket: String,
},
PutBucketAnalyticsConfiguration {
bucket: String,
id: String,
},
PutBucketCors {
bucket: String,
},
PutBucketEncryption {
bucket: String,
},
PutBucketIntelligentTieringConfiguration {
bucket: String,
id: String,
},
PutBucketInventoryConfiguration {
bucket: String,
id: String,
},
PutBucketLifecycleConfiguration {
bucket: String,
},
PutBucketLogging {
bucket: String,
},
PutBucketMetricsConfiguration {
bucket: String,
id: String,
},
PutBucketNotificationConfiguration {
bucket: String,
},
PutBucketOwnershipControls {
bucket: String,
},
PutBucketPolicy {
bucket: String,
},
PutBucketReplication {
bucket: String,
},
PutBucketRequestPayment {
bucket: String,
},
PutBucketTagging {
bucket: String,
},
PutBucketVersioning {
bucket: String,
},
PutBucketWebsite {
bucket: String,
},
PutObject {
bucket: String,
key: String,
},
PutObjectAcl {
bucket: String,
key: String,
version_id: Option<String>,
},
PutObjectLegalHold {
bucket: String,
key: String,
version_id: Option<String>,
},
PutObjectLockConfiguration {
bucket: String,
},
PutObjectRetention {
bucket: String,
key: String,
version_id: Option<String>,
},
PutObjectTagging {
bucket: String,
key: String,
version_id: Option<String>,
},
PutPublicAccessBlock {
bucket: String,
},
RestoreObject {
bucket: String,
key: String,
version_id: Option<String>,
},
SelectObjectContent {
bucket: String,
key: String,
// This value should always be 2. It is not checked when constructing the struct
select_type: String,
},
UploadPart {
bucket: String,
key: String,
part_number: u64,
upload_id: String,
},
UploadPartCopy {
bucket: String,
key: String,
part_number: u64,
upload_id: String,
@@ -506,12 +415,16 @@ pub enum Endpoint {
impl Endpoint {
/// Determine which S3 endpoint a request is for using the request, and a bucket which was
/// possibly extracted from the Host header.
pub fn from_request<T>(req: &Request<T>, bucket: Option<String>) -> Result<Self, Error> {
/// Returns Self plus bucket name, if endpoint is not Endpoint::ListBuckets
pub fn from_request<T>(
req: &Request<T>,
bucket: Option<String>,
) -> Result<(Self, Option<String>), Error> {
let uri = req.uri();
let path = uri.path().trim_start_matches('/');
let query = uri.query();
if bucket.is_none() && path.is_empty() {
return Ok(Self::ListBuckets);
return Ok((Self::ListBuckets, None));
}
let (bucket, key) = if let Some(bucket) = bucket {
@@ -522,6 +435,10 @@ impl Endpoint {
.unwrap_or((path.to_owned(), ""))
};
if *req.method() == Method::OPTIONS {
return Ok((Self::Options, Some(bucket)));
}
let key = percent_encoding::percent_decode_str(key)
.decode_utf8()?
.into_owned();
@@ -529,29 +446,25 @@ impl Endpoint {
let mut query = QueryParameters::from_query(query.unwrap_or_default())?;
let res = match *req.method() {
Method::GET => Self::from_get(bucket, key, &mut query)?,
Method::HEAD => Self::from_head(bucket, key, &mut query)?,
Method::POST => Self::from_post(bucket, key, &mut query)?,
Method::PUT => Self::from_put(bucket, key, &mut query, req.headers())?,
Method::DELETE => Self::from_delete(bucket, key, &mut query)?,
Method::GET => Self::from_get(key, &mut query)?,
Method::HEAD => Self::from_head(key, &mut query)?,
Method::POST => Self::from_post(key, &mut query)?,
Method::PUT => Self::from_put(key, &mut query, req.headers())?,
Method::DELETE => Self::from_delete(key, &mut query)?,
_ => return Err(Error::BadRequest("Unknown method".to_owned())),
};
if let Some(message) = query.nonempty_message() {
debug!("Unused query parameter: {}", message)
}
Ok(res)
Ok((res, Some(bucket)))
}
/// Determine which endpoint a request is for, knowing it is a GET.
fn from_get(
bucket: String,
key: String,
query: &mut QueryParameters<'_>,
) -> Result<Self, Error> {
fn from_get(key: String, query: &mut QueryParameters<'_>) -> Result<Self, Error> {
s3_match! {
@gen_parser
(query.keyword.take().unwrap_or_default().as_ref(), key, bucket, query, None),
(query.keyword.take().unwrap_or_default().as_ref(), key, query, None),
key: [
EMPTY if upload_id => ListParts (query::upload_id, opt_parse::max_parts, opt_parse::part_number_marker),
EMPTY => GetObject (query_opt::version_id, opt_parse::part_number),
@@ -605,14 +518,10 @@ impl Endpoint {
}
/// Determine which endpoint a request is for, knowing it is a HEAD.
fn from_head(
bucket: String,
key: String,
query: &mut QueryParameters<'_>,
) -> Result<Self, Error> {
fn from_head(key: String, query: &mut QueryParameters<'_>) -> Result<Self, Error> {
s3_match! {
@gen_parser
(query.keyword.take().unwrap_or_default().as_ref(), key, bucket, query, None),
(query.keyword.take().unwrap_or_default().as_ref(), key, query, None),
key: [
EMPTY => HeadObject(opt_parse::part_number, query_opt::version_id),
],
@@ -623,14 +532,10 @@ impl Endpoint {
}
/// Determine which endpoint a request is for, knowing it is a POST.
fn from_post(
bucket: String,
key: String,
query: &mut QueryParameters<'_>,
) -> Result<Self, Error> {
fn from_post(key: String, query: &mut QueryParameters<'_>) -> Result<Self, Error> {
s3_match! {
@gen_parser
(query.keyword.take().unwrap_or_default().as_ref(), key, bucket, query, None),
(query.keyword.take().unwrap_or_default().as_ref(), key, query, None),
key: [
EMPTY if upload_id => CompleteMultipartUpload (query::upload_id),
RESTORE => RestoreObject (query_opt::version_id),
@@ -645,14 +550,13 @@ impl Endpoint {
/// Determine which endpoint a request is for, knowing it is a PUT.
fn from_put(
bucket: String,
key: String,
query: &mut QueryParameters<'_>,
headers: &HeaderMap<HeaderValue>,
) -> Result<Self, Error> {
s3_match! {
@gen_parser
(query.keyword.take().unwrap_or_default().as_ref(), key, bucket, query, headers),
(query.keyword.take().unwrap_or_default().as_ref(), key, query, headers),
key: [
EMPTY if part_number header "x-amz-copy-source" => UploadPartCopy (parse::part_number, query::upload_id),
EMPTY header "x-amz-copy-source" => CopyObject,
@@ -691,14 +595,10 @@ impl Endpoint {
}
/// Determine which endpoint a request is for, knowing it is a DELETE.
fn from_delete(
bucket: String,
key: String,
query: &mut QueryParameters<'_>,
) -> Result<Self, Error> {
fn from_delete(key: String, query: &mut QueryParameters<'_>) -> Result<Self, Error> {
s3_match! {
@gen_parser
(query.keyword.take().unwrap_or_default().as_ref(), key, bucket, query, None),
(query.keyword.take().unwrap_or_default().as_ref(), key, query, None),
key: [
EMPTY if upload_id => AbortMultipartUpload (query::upload_id),
EMPTY => DeleteObject (query_opt::version_id),
@@ -759,16 +659,13 @@ impl Endpoint {
}
/// Get the kind of authorization which is required to perform the operation.
pub fn authorization_type(&self) -> Authorization<'_> {
let bucket = if let Some(bucket) = self.get_bucket() {
bucket
} else {
pub fn authorization_type(&self) -> Authorization {
if let Endpoint::ListBuckets = self {
return Authorization::None;
};
let readonly = s3_match! {
@extract
@match
self,
bucket,
[
GetBucketAccelerateConfiguration,
GetBucketAcl,
@@ -809,12 +706,10 @@ impl Endpoint {
ListParts,
SelectObjectContent,
]
}
.is_some();
};
let owner = s3_match! {
@extract
@match
self,
bucket,
[
DeleteBucket,
GetBucketWebsite,
@@ -824,29 +719,28 @@ impl Endpoint {
PutBucketCors,
DeleteBucketCors,
]
}
.is_some();
};
if readonly {
Authorization::Read(bucket)
Authorization::Read
} else if owner {
Authorization::Owner(bucket)
Authorization::Owner
} else {
Authorization::Write(bucket)
Authorization::Write
}
}
}
/// What kind of authorization is required to perform a given action
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum Authorization<'a> {
pub enum Authorization {
/// No authorization is required
None,
/// Having Read permission on bucket .0 is required
Read(&'a str),
/// Having Write permission on bucket .0 is required
Write(&'a str),
/// Having Owner permission on bucket .0 is required
Owner(&'a str),
/// Having Read permission on bucket
Read,
/// Having Write permission on bucket
Write,
/// Having Owner permission on bucket
Owner,
}
/// This macro is used to generate part of the code in this module. It must be called only one, and
@@ -987,7 +881,7 @@ mod tests {
uri: &str,
bucket: Option<String>,
header: Option<(&str, &str)>,
) -> Endpoint {
) -> (Endpoint, Option<String>) {
let mut req = Request::builder().method(method).uri(uri);
if let Some((k, v)) = header {
req = req.header(k, v)
@@ -1002,13 +896,13 @@ mod tests {
$(
assert!(
matches!(
parse(test_cases!{@actual_method $method}, $uri, Some("my_bucket".to_owned()), None),
parse(test_cases!{@actual_method $method}, $uri, Some("my_bucket".to_owned()), None).0,
Endpoint::$variant { .. }
)
);
assert!(
matches!(
parse(test_cases!{@actual_method $method}, concat!("/my_bucket", $uri), None, None),
parse(test_cases!{@actual_method $method}, concat!("/my_bucket", $uri), None, None).0,
Endpoint::$variant { .. }
)
);
@@ -1027,78 +921,82 @@ mod tests {
(@actual_method OWNER_DELETE) => {{ "DELETE" }};
(@auth HEAD $uri:expr) => {{
assert_eq!(parse("HEAD", concat!("/my_bucket", $uri), None, None).authorization_type(),
Authorization::Read("my_bucket"))
assert_eq!(parse("HEAD", concat!("/my_bucket", $uri), None, None).0.authorization_type(),
Authorization::Read)
}};
(@auth GET $uri:expr) => {{
assert_eq!(parse("GET", concat!("/my_bucket", $uri), None, None).authorization_type(),
Authorization::Read("my_bucket"))
assert_eq!(parse("GET", concat!("/my_bucket", $uri), None, None).0.authorization_type(),
Authorization::Read)
}};
(@auth OWNER_GET $uri:expr) => {{
assert_eq!(parse("GET", concat!("/my_bucket", $uri), None, None).authorization_type(),
Authorization::Owner("my_bucket"))
assert_eq!(parse("GET", concat!("/my_bucket", $uri), None, None).0.authorization_type(),
Authorization::Owner)
}};
(@auth PUT $uri:expr) => {{
assert_eq!(parse("PUT", concat!("/my_bucket", $uri), None, None).authorization_type(),
Authorization::Write("my_bucket"))
assert_eq!(parse("PUT", concat!("/my_bucket", $uri), None, None).0.authorization_type(),
Authorization::Write)
}};
(@auth OWNER_PUT $uri:expr) => {{
assert_eq!(parse("PUT", concat!("/my_bucket", $uri), None, None).authorization_type(),
Authorization::Owner("my_bucket"))
assert_eq!(parse("PUT", concat!("/my_bucket", $uri), None, None).0.authorization_type(),
Authorization::Owner)
}};
(@auth POST $uri:expr) => {{
assert_eq!(parse("POST", concat!("/my_bucket", $uri), None, None).authorization_type(),
Authorization::Write("my_bucket"))
assert_eq!(parse("POST", concat!("/my_bucket", $uri), None, None).0.authorization_type(),
Authorization::Write)
}};
(@auth DELETE $uri:expr) => {{
assert_eq!(parse("DELETE", concat!("/my_bucket", $uri), None, None).authorization_type(),
Authorization::Write("my_bucket"))
assert_eq!(parse("DELETE", concat!("/my_bucket", $uri), None, None).0.authorization_type(),
Authorization::Write)
}};
(@auth OWNER_DELETE $uri:expr) => {{
assert_eq!(parse("DELETE", concat!("/my_bucket", $uri), None, None).authorization_type(),
Authorization::Owner("my_bucket"))
assert_eq!(parse("DELETE", concat!("/my_bucket", $uri), None, None).0.authorization_type(),
Authorization::Owner)
}};
}
#[test]
fn test_bucket_extraction() {
assert_eq!(
parse("GET", "/my/key", Some("my_bucket".to_owned()), None).get_bucket(),
parse("GET", "/my_bucket/my/key", None, None).get_bucket()
parse("GET", "/my/key", Some("my_bucket".to_owned()), None).1,
parse("GET", "/my_bucket/my/key", None, None).1
);
assert_eq!(
parse("GET", "/my_bucket/my/key", None, None)
.get_bucket()
.unwrap(),
parse("GET", "/my_bucket/my/key", None, None).1.unwrap(),
"my_bucket"
);
assert!(parse("GET", "/", None, None).get_bucket().is_none());
assert!(parse("GET", "/", None, None).1.is_none());
}
#[test]
fn test_key() {
assert_eq!(
parse("GET", "/my/key", Some("my_bucket".to_owned()), None).get_key(),
parse("GET", "/my_bucket/my/key", None, None).get_key()
parse("GET", "/my/key", Some("my_bucket".to_owned()), None)
.0
.get_key(),
parse("GET", "/my_bucket/my/key", None, None).0.get_key()
);
assert_eq!(
parse("GET", "/my_bucket/my/key", None, None)
.0
.get_key()
.unwrap(),
"my/key"
);
assert_eq!(
parse("GET", "/my_bucket/my/key?acl", None, None)
.0
.get_key()
.unwrap(),
"my/key"
);
assert!(parse("GET", "/my_bucket/?list-type=2", None, None)
.0
.get_key()
.is_none());
assert_eq!(
parse("GET", "/my_bucket/%26%2B%3F%25%C3%A9/something", None, None)
.0
.get_key()
.unwrap(),
"&+?%é/something"
@@ -1270,11 +1168,11 @@ mod tests {
);
// no bucket, won't work with the rest of the test suite
assert!(matches!(
parse("GET", "/", None, None),
parse("GET", "/", None, None).0,
Endpoint::ListBuckets { .. }
));
assert!(matches!(
parse("GET", "/", None, None).authorization_type(),
parse("GET", "/", None, None).0.authorization_type(),
Authorization::None
));
@@ -1285,16 +1183,8 @@ mod tests {
"/Key+",
Some("my_bucket".to_owned()),
Some(("x-amz-copy-source", "some/key"))
),
Endpoint::CopyObject { .. }
));
assert!(matches!(
parse(
"PUT",
"/my_bucket/Key+",
None,
Some(("x-amz-copy-source", "some/key"))
),
)
.0,
Endpoint::CopyObject { .. }
));
assert!(matches!(
@@ -1304,8 +1194,19 @@ mod tests {
None,
Some(("x-amz-copy-source", "some/key"))
)
.0,
Endpoint::CopyObject { .. }
));
assert!(matches!(
parse(
"PUT",
"/my_bucket/Key+",
None,
Some(("x-amz-copy-source", "some/key"))
)
.0
.authorization_type(),
Authorization::Write("my_bucket")
Authorization::Write
));
// require a header
@@ -1315,16 +1216,8 @@ mod tests {
"/Key+?partNumber=2&uploadId=UploadId",
Some("my_bucket".to_owned()),
Some(("x-amz-copy-source", "some/key"))
),
Endpoint::UploadPartCopy { .. }
));
assert!(matches!(
parse(
"PUT",
"/my_bucket/Key+?partNumber=2&uploadId=UploadId",
None,
Some(("x-amz-copy-source", "some/key"))
),
)
.0,
Endpoint::UploadPartCopy { .. }
));
assert!(matches!(
@@ -1334,8 +1227,19 @@ mod tests {
None,
Some(("x-amz-copy-source", "some/key"))
)
.0,
Endpoint::UploadPartCopy { .. }
));
assert!(matches!(
parse(
"PUT",
"/my_bucket/Key+?partNumber=2&uploadId=UploadId",
None,
Some(("x-amz-copy-source", "some/key"))
)
.0
.authorization_type(),
Authorization::Write("my_bucket")
Authorization::Write
));
// POST request, but with GET semantic for permissions purpose
@@ -1345,17 +1249,19 @@ mod tests {
"/{Key+}?select&select-type=2",
Some("my_bucket".to_owned()),
None
),
)
.0,
Endpoint::SelectObjectContent { .. }
));
assert!(matches!(
parse("POST", "/my_bucket/{Key+}?select&select-type=2", None, None),
parse("POST", "/my_bucket/{Key+}?select&select-type=2", None, None).0,
Endpoint::SelectObjectContent { .. }
));
assert!(matches!(
parse("POST", "/my_bucket/{Key+}?select&select-type=2", None, None)
.0
.authorization_type(),
Authorization::Read("my_bucket")
Authorization::Read
));
}
}
+5 -11
View File
@@ -13,16 +13,7 @@ use garage_model::garage::Garage;
use garage_table::*;
use garage_util::data::*;
pub async fn handle_get_website(
garage: Arc<Garage>,
bucket_id: Uuid,
) -> Result<Response<Body>, Error> {
let bucket = garage
.bucket_table
.get(&EmptyKey, &bucket_id)
.await?
.ok_or(Error::NoSuchBucket)?;
pub async fn handle_get_website(bucket: &Bucket) -> Result<Response<Body>, Error> {
let param = bucket
.params()
.ok_or_internal_error("Bucket should not be deleted at this point")?;
@@ -80,7 +71,10 @@ pub async fn handle_put_website(
content_sha256: Option<Hash>,
) -> Result<Response<Body>, Error> {
let body = hyper::body::to_bytes(req.into_body()).await?;
verify_signed_content(content_sha256, &body[..])?;
if let Some(content_sha256) = content_sha256 {
verify_signed_content(content_sha256, &body[..])?;
}
let mut bucket = garage
.bucket_table
+121 -6
View File
@@ -33,12 +33,6 @@ pub struct Bucket {
pub name: Value,
}
#[derive(Debug, Serialize, PartialEq)]
pub struct DisplayName(#[serde(rename = "$value")] pub String);
#[derive(Debug, Serialize, PartialEq)]
pub struct Id(#[serde(rename = "$value")] pub String);
#[derive(Debug, Serialize, PartialEq)]
pub struct Owner {
#[serde(rename = "DisplayName")]
@@ -193,6 +187,46 @@ pub struct ListMultipartUploadsResult {
pub encoding_type: Option<Value>,
}
#[derive(Debug, Serialize, PartialEq)]
pub struct PartItem {
#[serde(rename = "ETag")]
pub etag: Value,
#[serde(rename = "LastModified")]
pub last_modified: Value,
#[serde(rename = "PartNumber")]
pub part_number: IntValue,
#[serde(rename = "Size")]
pub size: IntValue,
}
#[derive(Debug, Serialize, PartialEq)]
pub struct ListPartsResult {
#[serde(serialize_with = "xmlns_tag")]
pub xmlns: (),
#[serde(rename = "Bucket")]
pub bucket: Value,
#[serde(rename = "Key")]
pub key: Value,
#[serde(rename = "UploadId")]
pub upload_id: Value,
#[serde(rename = "PartNumberMarker")]
pub part_number_marker: Option<IntValue>,
#[serde(rename = "NextPartNumberMarker")]
pub next_part_number_marker: Option<IntValue>,
#[serde(rename = "MaxParts")]
pub max_parts: IntValue,
#[serde(rename = "IsTruncated")]
pub is_truncated: Value,
#[serde(rename = "Part", default)]
pub parts: Vec<PartItem>,
#[serde(rename = "Initiator")]
pub initiator: Initiator,
#[serde(rename = "Owner")]
pub owner: Owner,
#[serde(rename = "StorageClass")]
pub storage_class: Value,
}
#[derive(Debug, Serialize, PartialEq)]
pub struct ListBucketItem {
#[serde(rename = "Key")]
@@ -712,4 +746,85 @@ mod tests {
);
Ok(())
}
#[test]
fn list_parts() -> Result<(), ApiError> {
let result = ListPartsResult {
xmlns: (),
bucket: Value("example-bucket".to_string()),
key: Value("example-object".to_string()),
upload_id: Value(
"XXBsb2FkIElEIGZvciBlbHZpbmcncyVcdS1tb3ZpZS5tMnRzEEEwbG9hZA".to_string(),
),
part_number_marker: Some(IntValue(1)),
next_part_number_marker: Some(IntValue(3)),
max_parts: IntValue(2),
is_truncated: Value("true".to_string()),
parts: vec![
PartItem {
etag: Value("\"7778aef83f66abc1fa1e8477f296d394\"".to_string()),
last_modified: Value("2010-11-10T20:48:34.000Z".to_string()),
part_number: IntValue(2),
size: IntValue(10485760),
},
PartItem {
etag: Value("\"aaaa18db4cc2f85cedef654fccc4a4x8\"".to_string()),
last_modified: Value("2010-11-10T20:48:33.000Z".to_string()),
part_number: IntValue(3),
size: IntValue(10485760),
},
],
initiator: Initiator {
display_name: Value("umat-user-11116a31-17b5-4fb7-9df5-b288870f11xx".to_string()),
id: Value(
"arn:aws:iam::111122223333:user/some-user-11116a31-17b5-4fb7-9df5-b288870f11xx"
.to_string(),
),
},
owner: Owner {
display_name: Value("someName".to_string()),
id: Value(
"75aa57f09aa0c8caeab4f8c24e99d10f8e7faeebf76c078efc7c6caea54ba06a".to_string(),
),
},
storage_class: Value("STANDARD".to_string()),
};
assert_eq!(
to_xml_with_header(&result)?,
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>\
<ListPartsResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">\
<Bucket>example-bucket</Bucket>\
<Key>example-object</Key>\
<UploadId>XXBsb2FkIElEIGZvciBlbHZpbmcncyVcdS1tb3ZpZS5tMnRzEEEwbG9hZA</UploadId>\
<PartNumberMarker>1</PartNumberMarker>\
<NextPartNumberMarker>3</NextPartNumberMarker>\
<MaxParts>2</MaxParts>\
<IsTruncated>true</IsTruncated>\
<Part>\
<ETag>&quot;7778aef83f66abc1fa1e8477f296d394&quot;</ETag>\
<LastModified>2010-11-10T20:48:34.000Z</LastModified>\
<PartNumber>2</PartNumber>\
<Size>10485760</Size>\
</Part>\
<Part>\
<ETag>&quot;aaaa18db4cc2f85cedef654fccc4a4x8&quot;</ETag>\
<LastModified>2010-11-10T20:48:33.000Z</LastModified>\
<PartNumber>3</PartNumber>\
<Size>10485760</Size>\
</Part>\
<Initiator>\
<DisplayName>umat-user-11116a31-17b5-4fb7-9df5-b288870f11xx</DisplayName>\
<ID>arn:aws:iam::111122223333:user/some-user-11116a31-17b5-4fb7-9df5-b288870f11xx</ID>\
</Initiator>\
<Owner>\
<DisplayName>someName</DisplayName>\
<ID>75aa57f09aa0c8caeab4f8c24e99d10f8e7faeebf76c078efc7c6caea54ba06a</ID>\
</Owner>\
<StorageClass>STANDARD</StorageClass>\
</ListPartsResult>"
);
Ok(())
}
}
+47
View File
@@ -0,0 +1,47 @@
use chrono::{DateTime, Utc};
use hmac::{Hmac, Mac, NewMac};
use sha2::Sha256;
use garage_util::data::{sha256sum, Hash};
use crate::error::*;
pub mod payload;
pub mod streaming;
pub const SHORT_DATE: &str = "%Y%m%d";
pub const LONG_DATETIME: &str = "%Y%m%dT%H%M%SZ";
type HmacSha256 = Hmac<Sha256>;
pub fn verify_signed_content(expected_sha256: Hash, body: &[u8]) -> Result<(), Error> {
if expected_sha256 != sha256sum(body) {
return Err(Error::BadRequest(
"Request content hash does not match signed hash".to_string(),
));
}
Ok(())
}
pub fn signing_hmac(
datetime: &DateTime<Utc>,
secret_key: &str,
region: &str,
service: &str,
) -> Result<HmacSha256, crypto_mac::InvalidKeyLength> {
let secret = String::from("AWS4") + secret_key;
let mut date_hmac = HmacSha256::new_varkey(secret.as_bytes())?;
date_hmac.update(datetime.format(SHORT_DATE).to_string().as_bytes());
let mut region_hmac = HmacSha256::new_varkey(&date_hmac.finalize().into_bytes())?;
region_hmac.update(region.as_bytes());
let mut service_hmac = HmacSha256::new_varkey(&region_hmac.finalize().into_bytes())?;
service_hmac.update(service.as_bytes());
let mut signing_hmac = HmacSha256::new_varkey(&service_hmac.finalize().into_bytes())?;
signing_hmac.update(b"aws4_request");
let hmac = HmacSha256::new_varkey(&signing_hmac.finalize().into_bytes())?;
Ok(hmac)
}
pub fn compute_scope(datetime: &DateTime<Utc>, region: &str) -> String {
format!("{}/{}/s3/aws4_request", datetime.format(SHORT_DATE), region,)
}
@@ -1,28 +1,26 @@
use std::collections::HashMap;
use chrono::{DateTime, Duration, NaiveDateTime, Utc};
use hmac::{Hmac, Mac, NewMac};
use hmac::Mac;
use hyper::{Body, Method, Request};
use sha2::{Digest, Sha256};
use garage_table::*;
use garage_util::data::{sha256sum, Hash};
use garage_util::data::Hash;
use garage_model::garage::Garage;
use garage_model::key_table::*;
use super::signing_hmac;
use super::{LONG_DATETIME, SHORT_DATE};
use crate::encoding::uri_encode;
use crate::error::*;
const SHORT_DATE: &str = "%Y%m%d";
const LONG_DATETIME: &str = "%Y%m%dT%H%M%SZ";
type HmacSha256 = Hmac<Sha256>;
pub async fn check_signature(
pub async fn check_payload_signature(
garage: &Garage,
request: &Request<Body>,
) -> Result<(Key, Option<Hash>), Error> {
) -> Result<(Option<Key>, Option<Hash>), Error> {
let mut headers = HashMap::new();
for (key, val) in request.headers() {
headers.insert(key.to_string(), val.to_str()?.to_string());
@@ -36,24 +34,24 @@ pub async fn check_signature(
let authorization = if let Some(authorization) = headers.get("authorization") {
parse_authorization(authorization, &headers)?
} else if let Some(algorithm) = headers.get("x-amz-algorithm") {
parse_query_authorization(algorithm, &headers)?
} else {
parse_query_authorization(&headers)?
let content_sha256 = headers.get("x-amz-content-sha256");
if let Some(content_sha256) = content_sha256.filter(|c| "UNSIGNED-PAYLOAD" != c.as_str()) {
let sha256 = hex::decode(content_sha256)
.ok()
.and_then(|bytes| Hash::try_from(&bytes))
.ok_or_bad_request("Invalid content sha256 hash")?;
return Ok((None, Some(sha256)));
} else {
return Ok((None, None));
}
};
let date = headers
.get("x-amz-date")
.ok_or_bad_request("Missing X-Amz-Date field")?;
let date: NaiveDateTime =
NaiveDateTime::parse_from_str(date, LONG_DATETIME).ok_or_bad_request("Invalid date")?;
let date: DateTime<Utc> = DateTime::from_utc(date, Utc);
if Utc::now() - date > Duration::hours(24) {
return Err(Error::BadRequest("Date is too old".to_string()));
}
let scope = format!(
"{}/{}/s3/aws4_request",
date.format(SHORT_DATE),
authorization.date.format(SHORT_DATE),
garage.config.s3_api.s3_region
);
if authorization.scope != scope {
@@ -76,10 +74,10 @@ pub async fn check_signature(
&authorization.signed_headers,
&authorization.content_sha256,
);
let string_to_sign = string_to_sign(&date, &scope, &canonical_request);
let string_to_sign = string_to_sign(&authorization.date, &scope, &canonical_request);
let mut hmac = signing_hmac(
&date,
&authorization.date,
&key_p.secret_key,
&garage.config.s3_api.s3_region,
"s3",
@@ -97,16 +95,16 @@ pub async fn check_signature(
let content_sha256 = if authorization.content_sha256 == "UNSIGNED-PAYLOAD" {
None
} else if authorization.content_sha256 == "STREAMING-AWS4-HMAC-SHA256-PAYLOAD" {
let bytes = hex::decode(authorization.signature).ok_or_bad_request("Invalid signature")?;
Some(Hash::try_from(&bytes).ok_or_bad_request("Invalid signature")?)
} else {
let bytes = hex::decode(authorization.content_sha256)
.ok_or_bad_request("Invalid content sha256 hash")?;
Some(
Hash::try_from(&bytes[..])
.ok_or_else(|| Error::BadRequest("Invalid content sha256 hash".to_string()))?,
)
Some(Hash::try_from(&bytes).ok_or_bad_request("Invalid content sha256 hash")?)
};
Ok((key, content_sha256))
Ok((Some(key), content_sha256))
}
struct Authorization {
@@ -115,6 +113,7 @@ struct Authorization {
signed_headers: String,
signature: String,
content_sha256: String,
date: DateTime<Utc>,
}
fn parse_authorization(
@@ -149,6 +148,17 @@ fn parse_authorization(
.get("x-amz-content-sha256")
.ok_or_bad_request("Missing X-Amz-Content-Sha256 field")?;
let date = headers
.get("x-amz-date")
.ok_or_bad_request("Missing X-Amz-Date field")?;
let date: NaiveDateTime =
NaiveDateTime::parse_from_str(date, LONG_DATETIME).ok_or_bad_request("Invalid date")?;
let date: DateTime<Utc> = DateTime::from_utc(date, Utc);
if Utc::now() - date > Duration::hours(24) {
return Err(Error::BadRequest("Date is too old".to_string()));
}
let auth = Authorization {
key_id,
scope,
@@ -161,15 +171,16 @@ fn parse_authorization(
.ok_or_bad_request("Could not find Signature in Authorization field")?
.to_string(),
content_sha256: content_sha256.to_string(),
date,
};
Ok(auth)
}
fn parse_query_authorization(headers: &HashMap<String, String>) -> Result<Authorization, Error> {
let algo = headers
.get("x-amz-algorithm")
.ok_or_bad_request("X-Amz-Algorithm not found in query parameters")?;
if algo != "AWS4-HMAC-SHA256" {
fn parse_query_authorization(
algorithm: &str,
headers: &HashMap<String, String>,
) -> Result<Authorization, Error> {
if algorithm != "AWS4-HMAC-SHA256" {
return Err(Error::BadRequest(
"Unsupported authorization method".to_string(),
));
@@ -190,12 +201,36 @@ fn parse_query_authorization(headers: &HashMap<String, String>) -> Result<Author
.map(|x| x.as_str())
.unwrap_or("UNSIGNED-PAYLOAD");
let duration = headers
.get("x-amz-expires")
.ok_or_bad_request("X-Amz-Expires not found in query parameters")?
.parse()
.map_err(|_| Error::BadRequest("X-Amz-Expires is not a number".to_string()))?;
if duration > 7 * 24 * 3600 {
return Err(Error::BadRequest(
"X-Amz-Exprires may not exceed a week".to_string(),
));
}
let date = headers
.get("x-amz-date")
.ok_or_bad_request("Missing X-Amz-Date field")?;
let date: NaiveDateTime =
NaiveDateTime::parse_from_str(date, LONG_DATETIME).ok_or_bad_request("Invalid date")?;
let date: DateTime<Utc> = DateTime::from_utc(date, Utc);
if Utc::now() - date > Duration::seconds(duration) {
return Err(Error::BadRequest("Date is too old".to_string()));
}
Ok(Authorization {
key_id,
scope,
signed_headers: signed_headers.to_string(),
signature: signature.to_string(),
content_sha256: content_sha256.to_string(),
date,
})
}
@@ -222,25 +257,6 @@ fn string_to_sign(datetime: &DateTime<Utc>, scope_string: &str, canonical_req: &
.join("\n")
}
fn signing_hmac(
datetime: &DateTime<Utc>,
secret_key: &str,
region: &str,
service: &str,
) -> Result<HmacSha256, crypto_mac::InvalidKeyLength> {
let secret = String::from("AWS4") + secret_key;
let mut date_hmac = HmacSha256::new_varkey(secret.as_bytes())?;
date_hmac.update(datetime.format(SHORT_DATE).to_string().as_bytes());
let mut region_hmac = HmacSha256::new_varkey(&date_hmac.finalize().into_bytes())?;
region_hmac.update(region.as_bytes());
let mut service_hmac = HmacSha256::new_varkey(&region_hmac.finalize().into_bytes())?;
service_hmac.update(service.as_bytes());
let mut signing_hmac = HmacSha256::new_varkey(&service_hmac.finalize().into_bytes())?;
signing_hmac.update(b"aws4_request");
let hmac = HmacSha256::new_varkey(&signing_hmac.finalize().into_bytes())?;
Ok(hmac)
}
fn canonical_request(
method: &Method,
url_path: &str,
@@ -288,14 +304,3 @@ fn canonical_query_string(uri: &hyper::Uri) -> String {
"".to_string()
}
}
pub fn verify_signed_content(content_sha256: Option<Hash>, body: &[u8]) -> Result<(), Error> {
let expected_sha256 =
content_sha256.ok_or_bad_request("Request content hash not signed, aborting.")?;
if expected_sha256 != sha256sum(body) {
return Err(Error::BadRequest(
"Request content hash does not match signed hash".to_string(),
));
}
Ok(())
}
+319
View File
@@ -0,0 +1,319 @@
use std::pin::Pin;
use chrono::{DateTime, Utc};
use futures::prelude::*;
use futures::task;
use hyper::body::Bytes;
use garage_util::data::Hash;
use hmac::Mac;
use super::sha256sum;
use super::HmacSha256;
use super::LONG_DATETIME;
use crate::error::*;
/// Result of `sha256("")`
const EMPTY_STRING_HEX_DIGEST: &str =
"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855";
fn compute_streaming_payload_signature(
signing_hmac: &HmacSha256,
date: DateTime<Utc>,
scope: &str,
previous_signature: Hash,
content_sha256: Hash,
) -> Result<Hash, Error> {
let string_to_sign = [
"AWS4-HMAC-SHA256-PAYLOAD",
&date.format(LONG_DATETIME).to_string(),
scope,
&hex::encode(previous_signature),
EMPTY_STRING_HEX_DIGEST,
&hex::encode(content_sha256),
]
.join("\n");
let mut hmac = signing_hmac.clone();
hmac.update(string_to_sign.as_bytes());
Hash::try_from(&hmac.finalize().into_bytes()).ok_or_internal_error("Invalid signature")
}
mod payload {
use garage_util::data::Hash;
pub enum Error<I> {
Parser(nom::error::Error<I>),
BadSignature,
}
impl<I> Error<I> {
pub fn description(&self) -> &str {
match *self {
Error::Parser(ref e) => e.code.description(),
Error::BadSignature => "Bad signature",
}
}
}
#[derive(Debug, Clone)]
pub struct Header {
pub size: usize,
pub signature: Hash,
}
impl Header {
pub fn parse(input: &[u8]) -> nom::IResult<&[u8], Self, Error<&[u8]>> {
use nom::bytes::streaming::tag;
use nom::character::streaming::hex_digit1;
use nom::combinator::map_res;
use nom::number::streaming::hex_u32;
macro_rules! try_parse {
($expr:expr) => {
$expr.map_err(|e| e.map(Error::Parser))?
};
}
let (input, size) = try_parse!(hex_u32(input));
let (input, _) = try_parse!(tag(";")(input));
let (input, _) = try_parse!(tag("chunk-signature=")(input));
let (input, data) = try_parse!(map_res(hex_digit1, hex::decode)(input));
let signature = Hash::try_from(&data).ok_or(nom::Err::Failure(Error::BadSignature))?;
let (input, _) = try_parse!(tag("\r\n")(input));
let header = Header {
size: size as usize,
signature,
};
Ok((input, header))
}
}
}
#[derive(Debug)]
pub enum SignedPayloadStreamError {
Stream(Error),
InvalidSignature,
Message(String),
}
impl SignedPayloadStreamError {
fn message(msg: &str) -> Self {
SignedPayloadStreamError::Message(msg.into())
}
}
impl From<SignedPayloadStreamError> for Error {
fn from(err: SignedPayloadStreamError) -> Self {
match err {
SignedPayloadStreamError::Stream(e) => e,
SignedPayloadStreamError::InvalidSignature => {
Error::BadRequest("Invalid payload signature".into())
}
SignedPayloadStreamError::Message(e) => {
Error::BadRequest(format!("Chunk format error: {}", e))
}
}
}
}
impl<I> From<payload::Error<I>> for SignedPayloadStreamError {
fn from(err: payload::Error<I>) -> Self {
Self::message(err.description())
}
}
impl<I> From<nom::error::Error<I>> for SignedPayloadStreamError {
fn from(err: nom::error::Error<I>) -> Self {
Self::message(err.code.description())
}
}
struct SignedPayload {
header: payload::Header,
data: Bytes,
}
#[pin_project::pin_project]
pub struct SignedPayloadStream<S>
where
S: Stream<Item = Result<Bytes, Error>>,
{
#[pin]
stream: S,
buf: bytes::BytesMut,
datetime: DateTime<Utc>,
scope: String,
signing_hmac: HmacSha256,
previous_signature: Hash,
}
impl<S> SignedPayloadStream<S>
where
S: Stream<Item = Result<Bytes, Error>>,
{
pub fn new(
stream: S,
signing_hmac: HmacSha256,
datetime: DateTime<Utc>,
scope: &str,
seed_signature: Hash,
) -> Result<Self, Error> {
Ok(Self {
stream,
buf: bytes::BytesMut::new(),
datetime,
scope: scope.into(),
signing_hmac,
previous_signature: seed_signature,
})
}
fn parse_next(input: &[u8]) -> nom::IResult<&[u8], SignedPayload, SignedPayloadStreamError> {
use nom::bytes::streaming::{tag, take};
macro_rules! try_parse {
($expr:expr) => {
$expr.map_err(nom::Err::convert)?
};
}
let (input, header) = try_parse!(payload::Header::parse(input));
// 0-sized chunk is the last
if header.size == 0 {
return Ok((
input,
SignedPayload {
header,
data: Bytes::new(),
},
));
}
let (input, data) = try_parse!(take::<_, _, nom::error::Error<_>>(header.size)(input));
let (input, _) = try_parse!(tag::<_, _, nom::error::Error<_>>("\r\n")(input));
let data = Bytes::from(data.to_vec());
Ok((input, SignedPayload { header, data }))
}
}
impl<S> Stream for SignedPayloadStream<S>
where
S: Stream<Item = Result<Bytes, Error>> + Unpin,
{
type Item = Result<Bytes, SignedPayloadStreamError>;
fn poll_next(
self: Pin<&mut Self>,
cx: &mut task::Context<'_>,
) -> task::Poll<Option<Self::Item>> {
use std::task::Poll;
let mut this = self.project();
loop {
let (input, payload) = match Self::parse_next(this.buf) {
Ok(res) => res,
Err(nom::Err::Incomplete(_)) => {
match futures::ready!(this.stream.as_mut().poll_next(cx)) {
Some(Ok(bytes)) => {
this.buf.extend(bytes);
continue;
}
Some(Err(e)) => {
return Poll::Ready(Some(Err(SignedPayloadStreamError::Stream(e))))
}
None => {
return Poll::Ready(Some(Err(SignedPayloadStreamError::message(
"Unexpected EOF",
))));
}
}
}
Err(nom::Err::Error(e)) | Err(nom::Err::Failure(e)) => {
return Poll::Ready(Some(Err(e)))
}
};
// 0-sized chunk is the last
if payload.data.is_empty() {
return Poll::Ready(None);
}
let data_sha256sum = sha256sum(&payload.data);
let expected_signature = compute_streaming_payload_signature(
this.signing_hmac,
*this.datetime,
this.scope,
*this.previous_signature,
data_sha256sum,
)
.map_err(|e| {
SignedPayloadStreamError::Message(format!("Could not build signature: {}", e))
})?;
if payload.header.signature != expected_signature {
return Poll::Ready(Some(Err(SignedPayloadStreamError::InvalidSignature)));
}
*this.buf = input.into();
*this.previous_signature = payload.header.signature;
return Poll::Ready(Some(Ok(payload.data)));
}
}
fn size_hint(&self) -> (usize, Option<usize>) {
self.stream.size_hint()
}
}
#[cfg(test)]
mod tests {
use futures::prelude::*;
use super::{SignedPayloadStream, SignedPayloadStreamError};
#[tokio::test]
async fn test_interrupted_signed_payload_stream() {
use chrono::{DateTime, Utc};
use garage_util::data::Hash;
let datetime = DateTime::parse_from_rfc3339("2021-12-13T13:12:42+01:00") // TODO UNIX 0
.unwrap()
.with_timezone(&Utc);
let secret_key = "test";
let region = "test";
let scope = crate::signature::compute_scope(&datetime, region);
let signing_hmac =
crate::signature::signing_hmac(&datetime, secret_key, region, "s3").unwrap();
let data: &[&[u8]] = &[b"1"];
let body = futures::stream::iter(data.iter().map(|block| Ok(block.as_ref().into())));
let seed_signature = Hash::default();
let mut stream =
SignedPayloadStream::new(body, signing_hmac, datetime, &scope, seed_signature).unwrap();
assert!(stream.try_next().await.is_err());
match stream.try_next().await {
Err(SignedPayloadStreamError::Message(msg)) if msg == "Unexpected EOF" => {}
item => panic!(
"Unexpected result, expected early EOF error, got {:?}",
item
),
}
}
}
+3
View File
@@ -153,6 +153,9 @@ pub fn print_bucket_info(bucket: &Bucket, relevant_keys: &HashMap<String, Key>)
println!("\nAuthorized keys:");
let mut table = vec![];
for (k, perm) in p.authorized_keys.items().iter() {
if !perm.is_any() {
continue;
}
let rflag = if perm.allow_read { "R" } else { " " };
let wflag = if perm.allow_write { "W" } else { " " };
let oflag = if perm.allow_owner { "O" } else { " " };
-1
View File
@@ -43,7 +43,6 @@ pub struct BucketParams {
/// and if so, the website configuration XML document
pub website_config: crdt::Lww<Option<WebsiteConfig>>,
/// CORS rules
#[serde(default)]
pub cors_config: crdt::Lww<Option<Vec<CorsRule>>>,
}
+1 -5
View File
@@ -31,11 +31,7 @@ where
fn ok_or_bad_request<M: AsRef<str>>(self, reason: M) -> Result<T, Error> {
match self {
Ok(x) => Ok(x),
Err(e) => Err(Error::BadRequest(format!(
"{}: {}",
reason.as_ref(),
e.to_string()
))),
Err(e) => Err(Error::BadRequest(format!("{}: {}", reason.as_ref(), e))),
}
}
}
+16 -71
View File
@@ -2,22 +2,20 @@ use std::{borrow::Cow, convert::Infallible, net::SocketAddr, sync::Arc};
use futures::future::Future;
use http::header::{ACCESS_CONTROL_REQUEST_HEADERS, ACCESS_CONTROL_REQUEST_METHOD};
use hyper::{
header::{HeaderValue, HOST},
server::conn::AddrStream,
service::{make_service_fn, service_fn},
Body, Method, Request, Response, Server, StatusCode,
Body, Method, Request, Response, Server,
};
use crate::error::*;
use garage_api::error::{Error as ApiError, OkOrBadRequest, OkOrInternalError};
use garage_api::helpers::{authority_to_host, host_to_bucket};
use garage_api::s3_cors::{add_cors_headers, cors_rule_matches};
use garage_api::s3_cors::{add_cors_headers, find_matching_cors_rule, handle_options};
use garage_api::s3_get::{handle_get, handle_head};
use garage_model::bucket_table::Bucket;
use garage_model::garage::Garage;
use garage_table::*;
@@ -135,26 +133,23 @@ async fn serve_file(garage: Arc<Garage>, req: &Request<Body>) -> Result<Response
);
let ret_doc = match *req.method() {
Method::OPTIONS => return handle_options(&bucket, req),
Method::HEAD => {
return handle_head(garage.clone(), req, bucket_id, &key)
.await
.map_err(Error::from)
}
Method::GET => handle_get(garage.clone(), req, bucket_id, &key).await,
Method::OPTIONS => handle_options(req, &bucket).await,
Method::HEAD => handle_head(garage.clone(), req, bucket_id, &key, None).await,
Method::GET => handle_get(garage.clone(), req, bucket_id, &key, None).await,
_ => Err(ApiError::BadRequest("HTTP method not supported".into())),
}
.map_err(Error::from);
match ret_doc {
Err(error) => {
// For a HEAD or OPTIONS method, we don't return the error document
// as content, we return above and just return the error message
// For a HEAD or OPTIONS method, and for non-4xx errors,
// we don't return the error document as content,
// we return above and just return the error message
// by relying on err_to_res that is called when we return an Err.
assert!(*req.method() != Method::HEAD && *req.method() != Method::OPTIONS);
if !error.http_status_code().is_client_error() {
// Do not return the error document if it is not a 4xx error code.
if *req.method() == Method::HEAD
|| *req.method() == Method::OPTIONS
|| !error.http_status_code().is_client_error()
{
return Err(error);
}
@@ -171,7 +166,7 @@ async fn serve_file(garage: Arc<Garage>, req: &Request<Body>) -> Result<Response
.body(Body::empty())
.unwrap();
match handle_get(garage, &req2, bucket_id, &error_document).await {
match handle_get(garage, &req2, bucket_id, &error_document, None).await {
Ok(mut error_doc) => {
// The error won't be logged back in handle_request,
// so log it here
@@ -206,65 +201,15 @@ async fn serve_file(garage: Arc<Garage>, req: &Request<Body>) -> Result<Response
}
Ok(mut resp) => {
// Maybe add CORS headers
if let Some(cors_config) = bucket.params().unwrap().cors_config.get() {
if let Some(origin) = req.headers().get("Origin") {
let origin = origin.to_str()?;
let request_headers = match req.headers().get(ACCESS_CONTROL_REQUEST_HEADERS) {
Some(h) => h.to_str()?.split(',').map(|h| h.trim()).collect::<Vec<_>>(),
None => vec![],
};
let matching_rule = cors_config.iter().find(|rule| {
cors_rule_matches(
rule,
origin,
&req.method().to_string(),
request_headers.iter(),
)
});
if let Some(rule) = matching_rule {
add_cors_headers(&mut resp, rule)
.ok_or_internal_error("Invalid CORS configuration")?;
}
}
if let Some(rule) = find_matching_cors_rule(&bucket, req)? {
add_cors_headers(&mut resp, rule)
.ok_or_internal_error("Invalid bucket CORS configuration")?;
}
Ok(resp)
}
}
}
fn handle_options(bucket: &Bucket, req: &Request<Body>) -> Result<Response<Body>, Error> {
let origin = req
.headers()
.get("Origin")
.ok_or_bad_request("Missing Origin header")?
.to_str()?;
let request_method = req
.headers()
.get(ACCESS_CONTROL_REQUEST_METHOD)
.ok_or_bad_request("Missing Access-Control-Request-Method header")?
.to_str()?;
let request_headers = match req.headers().get(ACCESS_CONTROL_REQUEST_HEADERS) {
Some(h) => h.to_str()?.split(',').map(|h| h.trim()).collect::<Vec<_>>(),
None => vec![],
};
if let Some(cors_config) = bucket.params().unwrap().cors_config.get() {
let matching_rule = cors_config
.iter()
.find(|rule| cors_rule_matches(rule, origin, request_method, request_headers.iter()));
if let Some(rule) = matching_rule {
let mut resp = Response::builder()
.status(StatusCode::OK)
.body(Body::empty())
.map_err(ApiError::from)?;
add_cors_headers(&mut resp, rule).ok_or_internal_error("Invalid CORS configuration")?;
return Ok(resp);
}
}
Err(ApiError::Forbidden("No matching CORS rule".into()).into())
}
/// Path to key
///
/// Convert the provided path to the internal key