mirror of
https://github.com/deuxfleurs-org/garage.git
synced 2026-08-13 15:46:53 +00:00
Compare commits
1 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 89c6b38633 |
Generated
+38
-201
@@ -403,49 +403,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a0610544180c38b88101fecf2dd634b174a62eef6946f84dfc6a7127512b381c"
|
||||
dependencies = [
|
||||
"bitflags",
|
||||
"textwrap 0.11.0",
|
||||
"textwrap",
|
||||
"unicode-width",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "clap"
|
||||
version = "3.1.18"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d2dbdf4bdacb33466e854ce889eee8dfd5729abf7ccd7664d0a2d60cd384440b"
|
||||
dependencies = [
|
||||
"atty",
|
||||
"bitflags",
|
||||
"clap_derive",
|
||||
"clap_lex",
|
||||
"indexmap",
|
||||
"lazy_static",
|
||||
"strsim",
|
||||
"termcolor",
|
||||
"textwrap 0.15.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "clap_derive"
|
||||
version = "3.1.18"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "25320346e922cffe59c0bbc5410c8d8784509efb321488971081313cb1e1a33c"
|
||||
dependencies = [
|
||||
"heck 0.4.0",
|
||||
"proc-macro-error",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "clap_lex"
|
||||
version = "0.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a37c35f1112dad5e6e0b1adaff798507497a18fceeb30cceb3bae7d1427b9213"
|
||||
dependencies = [
|
||||
"os_str_bytes",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cloudabi"
|
||||
version = "0.0.3"
|
||||
@@ -543,16 +504,6 @@ dependencies = [
|
||||
"subtle",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crypto-mac"
|
||||
version = "0.11.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b1d1a86f49236c215f271d40892d5fc950490551400b02ef360692c29815c714"
|
||||
dependencies = [
|
||||
"generic-array",
|
||||
"subtle",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ct-logs"
|
||||
version = "0.8.0"
|
||||
@@ -888,23 +839,21 @@ dependencies = [
|
||||
"chrono",
|
||||
"futures",
|
||||
"futures-util",
|
||||
"garage_admin",
|
||||
"garage_api",
|
||||
"garage_model 0.7.0",
|
||||
"garage_rpc 0.7.0",
|
||||
"garage_table 0.7.0",
|
||||
"garage_util 0.7.0",
|
||||
"garage_web",
|
||||
"git-version",
|
||||
"hex",
|
||||
"hmac 0.10.1",
|
||||
"hmac",
|
||||
"http",
|
||||
"hyper",
|
||||
"kuska-sodiumoxide",
|
||||
"netapp 0.4.4",
|
||||
"opentelemetry",
|
||||
"opentelemetry-otlp",
|
||||
"opentelemetry-prometheus",
|
||||
"pretty_env_logger",
|
||||
"prometheus",
|
||||
"rand 0.8.5",
|
||||
"rmp-serde 0.15.5",
|
||||
"serde",
|
||||
@@ -919,6 +868,23 @@ dependencies = [
|
||||
"tracing",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "garage_admin"
|
||||
version = "0.7.0"
|
||||
dependencies = [
|
||||
"futures",
|
||||
"futures-util",
|
||||
"garage_util 0.7.0",
|
||||
"hex",
|
||||
"http",
|
||||
"hyper",
|
||||
"opentelemetry",
|
||||
"opentelemetry-otlp",
|
||||
"opentelemetry-prometheus",
|
||||
"prometheus",
|
||||
"tracing",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "garage_api"
|
||||
version = "0.7.0"
|
||||
@@ -938,7 +904,7 @@ dependencies = [
|
||||
"garage_table 0.7.0",
|
||||
"garage_util 0.7.0",
|
||||
"hex",
|
||||
"hmac 0.10.1",
|
||||
"hmac",
|
||||
"http",
|
||||
"http-range",
|
||||
"httpdate 0.3.2",
|
||||
@@ -948,11 +914,8 @@ dependencies = [
|
||||
"multer",
|
||||
"nom",
|
||||
"opentelemetry",
|
||||
"opentelemetry-otlp",
|
||||
"opentelemetry-prometheus",
|
||||
"percent-encoding",
|
||||
"pin-project 1.0.10",
|
||||
"prometheus",
|
||||
"quick-xml",
|
||||
"roxmltree",
|
||||
"serde",
|
||||
@@ -1077,9 +1040,9 @@ dependencies = [
|
||||
"bytes 1.1.0",
|
||||
"futures",
|
||||
"futures-util",
|
||||
"garage_admin",
|
||||
"garage_util 0.7.0",
|
||||
"gethostname",
|
||||
"git-version",
|
||||
"hex",
|
||||
"hyper",
|
||||
"k8s-openapi",
|
||||
@@ -1298,12 +1261,6 @@ dependencies = [
|
||||
"unicode-segmentation",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "heck"
|
||||
version = "0.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2540771e65fc8cb83cd6e8a237f70c319bd5c29f78ed1084ba5d50eeac86f7f9"
|
||||
|
||||
[[package]]
|
||||
name = "hermit-abi"
|
||||
version = "0.1.19"
|
||||
@@ -1339,16 +1296,6 @@ dependencies = [
|
||||
"digest",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hmac"
|
||||
version = "0.11.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2a2a2320eb7ec0ebe8da8f744d7812d9fc4cb4d09344ac01898dbcb6a20ae69b"
|
||||
dependencies = [
|
||||
"crypto-mac 0.11.1",
|
||||
"digest",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "http"
|
||||
version = "0.2.6"
|
||||
@@ -1576,23 +1523,6 @@ dependencies = [
|
||||
"serde_json",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "k2v-client"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"base64",
|
||||
"clap 3.1.18",
|
||||
"garage_util 0.7.0",
|
||||
"http",
|
||||
"rusoto_core",
|
||||
"rusoto_credential",
|
||||
"rusoto_signature",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"thiserror",
|
||||
"tokio",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "k8s-openapi"
|
||||
version = "0.13.1"
|
||||
@@ -2127,12 +2057,6 @@ dependencies = [
|
||||
"num-traits",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "os_str_bytes"
|
||||
version = "6.0.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "029d8d0b2f198229de29dca79676f2738ff952edf3fde542eb8bf94d8c21b435"
|
||||
|
||||
[[package]]
|
||||
name = "parking_lot"
|
||||
version = "0.11.2"
|
||||
@@ -2382,7 +2306,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "62941722fb675d463659e49c4f3fe1fe792ff24fe5bbaa9c08cd3b98a1c354f5"
|
||||
dependencies = [
|
||||
"bytes 1.1.0",
|
||||
"heck 0.3.3",
|
||||
"heck",
|
||||
"itertools 0.10.3",
|
||||
"lazy_static",
|
||||
"log",
|
||||
@@ -2609,75 +2533,6 @@ dependencies = [
|
||||
"xmlparser",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rusoto_core"
|
||||
version = "0.48.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1db30db44ea73551326269adcf7a2169428a054f14faf9e1768f2163494f2fa2"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"base64",
|
||||
"bytes 1.1.0",
|
||||
"crc32fast",
|
||||
"futures",
|
||||
"http",
|
||||
"hyper",
|
||||
"hyper-tls",
|
||||
"lazy_static",
|
||||
"log",
|
||||
"rusoto_credential",
|
||||
"rusoto_signature",
|
||||
"rustc_version",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"tokio",
|
||||
"xml-rs",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rusoto_credential"
|
||||
version = "0.48.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ee0a6c13db5aad6047b6a44ef023dbbc21a056b6dab5be3b79ce4283d5c02d05"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"chrono",
|
||||
"dirs-next",
|
||||
"futures",
|
||||
"hyper",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"shlex",
|
||||
"tokio",
|
||||
"zeroize",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rusoto_signature"
|
||||
version = "0.48.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a5ae95491c8b4847931e291b151127eccd6ff8ca13f33603eb3d0035ecb05272"
|
||||
dependencies = [
|
||||
"base64",
|
||||
"bytes 1.1.0",
|
||||
"chrono",
|
||||
"digest",
|
||||
"futures",
|
||||
"hex",
|
||||
"hmac 0.11.0",
|
||||
"http",
|
||||
"hyper",
|
||||
"log",
|
||||
"md-5",
|
||||
"percent-encoding",
|
||||
"pin-project-lite",
|
||||
"rusoto_credential",
|
||||
"rustc_version",
|
||||
"serde",
|
||||
"sha2",
|
||||
"tokio",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustc_version"
|
||||
version = "0.4.0"
|
||||
@@ -2814,9 +2669,9 @@ checksum = "a4a3381e03edd24287172047536f20cabde766e2cd3e65e6b00fb3af51c4f38d"
|
||||
|
||||
[[package]]
|
||||
name = "serde"
|
||||
version = "1.0.137"
|
||||
version = "1.0.136"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "61ea8d54c77f8315140a05f4c7237403bf38b72704d031543aa1d16abbf517d1"
|
||||
checksum = "ce31e24b01e1e524df96f1c2fdd054405f8d7376249a5110886fb4b658484789"
|
||||
dependencies = [
|
||||
"serde_derive",
|
||||
]
|
||||
@@ -2842,9 +2697,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "serde_derive"
|
||||
version = "1.0.137"
|
||||
version = "1.0.136"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1f26faba0c3959972377d3b2d306ee9f71faee9714294e41bb777f83f88578be"
|
||||
checksum = "08597e7152fcd306f41838ed3e37be9eaeed2b61c42e2117266a554fab4662f9"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
@@ -2864,9 +2719,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "serde_json"
|
||||
version = "1.0.81"
|
||||
version = "1.0.79"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9b7ce2b32a1aed03c558dc61a5cd328f15aff2dbc17daad8fb8af04d2100e15c"
|
||||
checksum = "8e8d9fa5c3b304765ce1fd9c4c8a3de2c8db365a5b91be52f186efc675681d95"
|
||||
dependencies = [
|
||||
"indexmap",
|
||||
"itoa",
|
||||
@@ -2899,12 +2754,6 @@ dependencies = [
|
||||
"opaque-debug",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "shlex"
|
||||
version = "1.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "43b2853a4d09f215c24cc5489c992ce46052d359b5109343cbafbf26bc62f8a3"
|
||||
|
||||
[[package]]
|
||||
name = "signal-hook-registry"
|
||||
version = "1.4.0"
|
||||
@@ -3027,7 +2876,7 @@ version = "0.3.26"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0c6b5c64445ba8094a6ab0c3cd2ad323e07171012d9c98b0b15651daf1787a10"
|
||||
dependencies = [
|
||||
"clap 2.34.0",
|
||||
"clap",
|
||||
"lazy_static",
|
||||
"structopt-derive",
|
||||
]
|
||||
@@ -3038,7 +2887,7 @@ version = "0.4.18"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "dcb5ae327f9cc13b68763b5749770cb9e048a99bd9dfdfa58d0cf05d5f64afe0"
|
||||
dependencies = [
|
||||
"heck 0.3.3",
|
||||
"heck",
|
||||
"proc-macro-error",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
@@ -3053,9 +2902,9 @@ checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601"
|
||||
|
||||
[[package]]
|
||||
name = "syn"
|
||||
version = "1.0.94"
|
||||
version = "1.0.89"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a07e33e919ebcd69113d5be0e4d70c5707004ff45188910106854f38b960df4a"
|
||||
checksum = "ea297be220d52398dcc07ce15a209fce436d361735ac1db700cab3b6cdfb9f54"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
@@ -3106,26 +2955,20 @@ dependencies = [
|
||||
"unicode-width",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "textwrap"
|
||||
version = "0.15.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b1141d4d61095b28419e22cb0bbf02755f5e54e0526f97f1e3d1d160e60885fb"
|
||||
|
||||
[[package]]
|
||||
name = "thiserror"
|
||||
version = "1.0.31"
|
||||
version = "1.0.30"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bd829fe32373d27f76265620b5309d0340cb8550f523c1dda251d6298069069a"
|
||||
checksum = "854babe52e4df1653706b98fcfc05843010039b406875930a70e4d9644e5c417"
|
||||
dependencies = [
|
||||
"thiserror-impl",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "thiserror-impl"
|
||||
version = "1.0.31"
|
||||
version = "1.0.30"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0396bc89e626244658bef819e22d0cc459e795a5ebe878e6ec336d1674a8d79a"
|
||||
checksum = "aa32fd3f627f367fe16f893e2597ae3c05020f8bba2666a4e6ea73d377e5714b"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
@@ -3692,12 +3535,6 @@ version = "0.32.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "504a2476202769977a040c6364301a3f65d0cc9e3fb08600b2bda150a0488316"
|
||||
|
||||
[[package]]
|
||||
name = "xml-rs"
|
||||
version = "0.8.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d2d7d3948613f75c98fd9328cfdcc45acc4d360655289d0a7d4ec931392200a3"
|
||||
|
||||
[[package]]
|
||||
name = "xmlparser"
|
||||
version = "0.13.3"
|
||||
|
||||
+2
-4
@@ -5,14 +5,12 @@ members = [
|
||||
"src/table",
|
||||
"src/block",
|
||||
"src/model",
|
||||
"src/admin",
|
||||
"src/api",
|
||||
"src/web",
|
||||
"src/garage",
|
||||
"src/k2v-client",
|
||||
"src/garage"
|
||||
]
|
||||
|
||||
default-members = ["src/garage"]
|
||||
|
||||
[profile.dev]
|
||||
lto = "off"
|
||||
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
.PHONY: doc all release shell
|
||||
|
||||
all:
|
||||
clear; cargo build --all-features
|
||||
clear; cargo build --features k2v
|
||||
|
||||
doc:
|
||||
cd doc/book; mdbook build
|
||||
|
||||
@@ -1,587 +0,0 @@
|
||||
# Access control
|
||||
|
||||
The admin API uses two different tokens for acces control, that are specified in the config file's `[admin]` section:
|
||||
|
||||
- `metrics_token`: the token for accessing the Metrics endpoint (if this token is not set in the config file, the Metrics endpoint can be accessed without access control);
|
||||
- `admin_token`: the token for accessing all of the other administration endpoints (if this token is not set in the config file, these endpoints can be accessed without access control).
|
||||
|
||||
# Administration API endpoints
|
||||
|
||||
## Metrics-related endpoints
|
||||
|
||||
### Metrics `GET /metrics`
|
||||
|
||||
Returns internal Garage metrics in Prometheus format.
|
||||
|
||||
## Cluster operations
|
||||
|
||||
### GetClusterStatus `GET /v0/status`
|
||||
|
||||
Returns the cluster's current status in JSON, including:
|
||||
|
||||
- ID of the node being queried and its version of the Garage daemon
|
||||
- Live nodes
|
||||
- Currently configured cluster layout
|
||||
- Staged changes to the cluster layout
|
||||
|
||||
Example response body:
|
||||
|
||||
```json
|
||||
{
|
||||
"node": "ec79480e0ce52ae26fd00c9da684e4fa56658d9c64cdcecb094e936de0bfe71f",
|
||||
"garage_version": "git:v0.8.0",
|
||||
"knownNodes": {
|
||||
"ec79480e0ce52ae26fd00c9da684e4fa56658d9c64cdcecb094e936de0bfe71f": {
|
||||
"addr": "10.0.0.11:3901",
|
||||
"is_up": true,
|
||||
"last_seen_secs_ago": 9,
|
||||
"hostname": "node1"
|
||||
},
|
||||
"4a6ae5a1d0d33bf895f5bb4f0a418b7dc94c47c0dd2eb108d1158f3c8f60b0ff": {
|
||||
"addr": "10.0.0.12:3901",
|
||||
"is_up": true,
|
||||
"last_seen_secs_ago": 1,
|
||||
"hostname": "node2"
|
||||
},
|
||||
"23ffd0cdd375ebff573b20cc5cef38996b51c1a7d6dbcf2c6e619876e507cf27": {
|
||||
"addr": "10.0.0.21:3901",
|
||||
"is_up": true,
|
||||
"last_seen_secs_ago": 7,
|
||||
"hostname": "node3"
|
||||
},
|
||||
"e2ee7984ee65b260682086ec70026165903c86e601a4a5a501c1900afe28d84b": {
|
||||
"addr": "10.0.0.22:3901",
|
||||
"is_up": true,
|
||||
"last_seen_secs_ago": 1,
|
||||
"hostname": "node4"
|
||||
}
|
||||
},
|
||||
"layout": {
|
||||
"version": 12,
|
||||
"roles": {
|
||||
"ec79480e0ce52ae26fd00c9da684e4fa56658d9c64cdcecb094e936de0bfe71f": {
|
||||
"zone": "dc1",
|
||||
"capacity": 4,
|
||||
"tags": [
|
||||
"node1"
|
||||
]
|
||||
},
|
||||
"4a6ae5a1d0d33bf895f5bb4f0a418b7dc94c47c0dd2eb108d1158f3c8f60b0ff": {
|
||||
"zone": "dc1",
|
||||
"capacity": 6,
|
||||
"tags": [
|
||||
"node2"
|
||||
]
|
||||
},
|
||||
"23ffd0cdd375ebff573b20cc5cef38996b51c1a7d6dbcf2c6e619876e507cf27": {
|
||||
"zone": "dc2",
|
||||
"capacity": 10,
|
||||
"tags": [
|
||||
"node3"
|
||||
]
|
||||
}
|
||||
},
|
||||
"stagedRoleChanges": {
|
||||
"e2ee7984ee65b260682086ec70026165903c86e601a4a5a501c1900afe28d84b": {
|
||||
"zone": "dc2",
|
||||
"capacity": 5,
|
||||
"tags": [
|
||||
"node4"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### ConnectClusterNodes `POST /v0/connect`
|
||||
|
||||
Instructs this Garage node to connect to other Garage nodes at specified addresses.
|
||||
|
||||
Example request body:
|
||||
|
||||
```json
|
||||
[
|
||||
"ec79480e0ce52ae26fd00c9da684e4fa56658d9c64cdcecb094e936de0bfe71f@10.0.0.11:3901",
|
||||
"4a6ae5a1d0d33bf895f5bb4f0a418b7dc94c47c0dd2eb108d1158f3c8f60b0ff@10.0.0.12:3901"
|
||||
]
|
||||
```
|
||||
|
||||
The format of the string for a node to connect to is: `<node ID>@<ip address>:<port>`, same as in the `garage node connect` CLI call.
|
||||
|
||||
Example response:
|
||||
|
||||
```json
|
||||
[
|
||||
{
|
||||
"success": true,
|
||||
"error": null,
|
||||
},
|
||||
{
|
||||
"success": false,
|
||||
"error": "Handshake error",
|
||||
}
|
||||
]
|
||||
```
|
||||
|
||||
### GetClusterLayout `GET /v0/layout`
|
||||
|
||||
Returns the cluster's current layout in JSON, including:
|
||||
|
||||
- Currently configured cluster layout
|
||||
- Staged changes to the cluster layout
|
||||
|
||||
(the info returned by this endpoint is a subset of the info returned by GetClusterStatus)
|
||||
|
||||
Example response body:
|
||||
|
||||
```json
|
||||
{
|
||||
"version": 12,
|
||||
"roles": {
|
||||
"ec79480e0ce52ae26fd00c9da684e4fa56658d9c64cdcecb094e936de0bfe71f": {
|
||||
"zone": "dc1",
|
||||
"capacity": 4,
|
||||
"tags": [
|
||||
"node1"
|
||||
]
|
||||
},
|
||||
"4a6ae5a1d0d33bf895f5bb4f0a418b7dc94c47c0dd2eb108d1158f3c8f60b0ff": {
|
||||
"zone": "dc1",
|
||||
"capacity": 6,
|
||||
"tags": [
|
||||
"node2"
|
||||
]
|
||||
},
|
||||
"23ffd0cdd375ebff573b20cc5cef38996b51c1a7d6dbcf2c6e619876e507cf27": {
|
||||
"zone": "dc2",
|
||||
"capacity": 10,
|
||||
"tags": [
|
||||
"node3"
|
||||
]
|
||||
}
|
||||
},
|
||||
"stagedRoleChanges": {
|
||||
"e2ee7984ee65b260682086ec70026165903c86e601a4a5a501c1900afe28d84b": {
|
||||
"zone": "dc2",
|
||||
"capacity": 5,
|
||||
"tags": [
|
||||
"node4"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### UpdateClusterLayout `POST /v0/layout`
|
||||
|
||||
Send modifications to the cluster layout. These modifications will
|
||||
be included in the staged role changes, visible in subsequent calls
|
||||
of `GetClusterLayout`. Once the set of staged changes is satisfactory,
|
||||
the user may call `ApplyClusterLayout` to apply the changed changes,
|
||||
or `Revert ClusterLayout` to clear all of the staged changes in
|
||||
the layout.
|
||||
|
||||
Request body format:
|
||||
|
||||
```json
|
||||
{
|
||||
<node_id>: {
|
||||
"capacity": <new_capacity>,
|
||||
"zone": <new_zone>,
|
||||
"tags": [
|
||||
<new_tag>,
|
||||
...
|
||||
]
|
||||
},
|
||||
<node_id_to_remove>: null,
|
||||
...
|
||||
}
|
||||
```
|
||||
|
||||
Contrary to the CLI that may update only a subset of the fields
|
||||
`capacity`, `zone` and `tags`, when calling this API all of these
|
||||
values must be specified.
|
||||
|
||||
|
||||
### ApplyClusterLayout `POST /v0/layout/apply`
|
||||
|
||||
Applies to the cluster the layout changes currently registered as
|
||||
staged layout changes.
|
||||
|
||||
Request body format:
|
||||
|
||||
```json
|
||||
{
|
||||
"version": 13
|
||||
}
|
||||
```
|
||||
|
||||
Similarly to the CLI, the body must include the version of the new layout
|
||||
that will be created, which MUST be 1 + the value of the currently
|
||||
existing layout in the cluster.
|
||||
|
||||
### RevertClusterLayout `POST /v0/layout/revert`
|
||||
|
||||
Clears all of the staged layout changes.
|
||||
|
||||
Request body format:
|
||||
|
||||
```json
|
||||
{
|
||||
"version": 13
|
||||
}
|
||||
```
|
||||
|
||||
Reverting the staged changes is done by incrementing the version number
|
||||
and clearing the contents of the staged change list.
|
||||
Similarly to the CLI, the body must include the incremented
|
||||
version number, which MUST be 1 + the value of the currently
|
||||
existing layout in the cluster.
|
||||
|
||||
|
||||
## Access key operations
|
||||
|
||||
### ListKeys `GET /v0/key`
|
||||
|
||||
Returns all API access keys in the cluster.
|
||||
|
||||
Example response:
|
||||
|
||||
```json
|
||||
[
|
||||
{
|
||||
"id": "GK31c2f218a2e44f485b94239e",
|
||||
"name": "test"
|
||||
},
|
||||
{
|
||||
"id": "GKe10061ac9c2921f09e4c5540",
|
||||
"name": "test2"
|
||||
}
|
||||
]
|
||||
```
|
||||
|
||||
### CreateKey `POST /v0/key`
|
||||
|
||||
Creates a new API access key.
|
||||
|
||||
Request body format:
|
||||
|
||||
```json
|
||||
{
|
||||
"name": "NameOfMyKey"
|
||||
}
|
||||
```
|
||||
|
||||
### ImportKey `POST /v0/key/import`
|
||||
|
||||
Imports an existing API key.
|
||||
|
||||
Request body format:
|
||||
|
||||
```json
|
||||
{
|
||||
"accessKeyId": "GK31c2f218a2e44f485b94239e",
|
||||
"secretAccessKey": "b892c0665f0ada8a4755dae98baa3b133590e11dae3bcc1f9d769d67f16c3835",
|
||||
"name": "NameOfMyKey"
|
||||
}
|
||||
```
|
||||
|
||||
### GetKeyInfo `GET /v0/key?id=<acces key id>`
|
||||
### GetKeyInfo `GET /v0/key?search=<pattern>`
|
||||
|
||||
Returns information about the requested API access key.
|
||||
|
||||
If `id` is set, the key is looked up using its exact identifier (faster).
|
||||
If `search` is set, the key is looked up using its name or prefix
|
||||
of identifier (slower, all keys are enumerated to do this).
|
||||
|
||||
Example response:
|
||||
|
||||
```json
|
||||
{
|
||||
"name": "test",
|
||||
"accessKeyId": "GK31c2f218a2e44f485b94239e",
|
||||
"secretAccessKey": "b892c0665f0ada8a4755dae98baa3b133590e11dae3bcc1f9d769d67f16c3835",
|
||||
"permissions": {
|
||||
"createBucket": false
|
||||
},
|
||||
"buckets": [
|
||||
{
|
||||
"id": "70dc3bed7fe83a75e46b66e7ddef7d56e65f3c02f9f80b6749fb97eccb5e1033",
|
||||
"globalAliases": [
|
||||
"test2"
|
||||
],
|
||||
"localAliases": [],
|
||||
"permissions": {
|
||||
"read": true,
|
||||
"write": true,
|
||||
"owner": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "d7452a935e663fc1914f3a5515163a6d3724010ce8dfd9e4743ca8be5974f995",
|
||||
"globalAliases": [
|
||||
"test3"
|
||||
],
|
||||
"localAliases": [],
|
||||
"permissions": {
|
||||
"read": true,
|
||||
"write": true,
|
||||
"owner": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "e6a14cd6a27f48684579ec6b381c078ab11697e6bc8513b72b2f5307e25fff9b",
|
||||
"globalAliases": [],
|
||||
"localAliases": [
|
||||
"test"
|
||||
],
|
||||
"permissions": {
|
||||
"read": true,
|
||||
"write": true,
|
||||
"owner": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "96470e0df00ec28807138daf01915cfda2bee8eccc91dea9558c0b4855b5bf95",
|
||||
"globalAliases": [
|
||||
"alex"
|
||||
],
|
||||
"localAliases": [],
|
||||
"permissions": {
|
||||
"read": true,
|
||||
"write": true,
|
||||
"owner": true
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
### DeleteKey `DELETE /v0/key?id=<acces key id>`
|
||||
|
||||
Deletes an API access key.
|
||||
|
||||
### UpdateKey `POST /v0/key?id=<acces key id>`
|
||||
|
||||
Updates information about the specified API access key.
|
||||
|
||||
Request body format:
|
||||
|
||||
```json
|
||||
{
|
||||
"name": "NameOfMyKey",
|
||||
"allow": {
|
||||
"createBucket": true,
|
||||
},
|
||||
"deny": {}
|
||||
}
|
||||
```
|
||||
|
||||
All fields (`name`, `allow` and `deny`) are optionnal.
|
||||
If they are present, the corresponding modifications are applied to the key, otherwise nothing is changed.
|
||||
The possible flags in `allow` and `deny` are: `createBucket`.
|
||||
|
||||
|
||||
## Bucket operations
|
||||
|
||||
### ListBuckets `GET /v0/bucket`
|
||||
|
||||
Returns all storage buckets in the cluster.
|
||||
|
||||
Example response:
|
||||
|
||||
```json
|
||||
[
|
||||
{
|
||||
"id": "70dc3bed7fe83a75e46b66e7ddef7d56e65f3c02f9f80b6749fb97eccb5e1033",
|
||||
"globalAliases": [
|
||||
"test2"
|
||||
],
|
||||
"localAliases": []
|
||||
},
|
||||
{
|
||||
"id": "96470e0df00ec28807138daf01915cfda2bee8eccc91dea9558c0b4855b5bf95",
|
||||
"globalAliases": [
|
||||
"alex"
|
||||
],
|
||||
"localAliases": []
|
||||
},
|
||||
{
|
||||
"id": "d7452a935e663fc1914f3a5515163a6d3724010ce8dfd9e4743ca8be5974f995",
|
||||
"globalAliases": [
|
||||
"test3"
|
||||
],
|
||||
"localAliases": []
|
||||
},
|
||||
{
|
||||
"id": "e6a14cd6a27f48684579ec6b381c078ab11697e6bc8513b72b2f5307e25fff9b",
|
||||
"globalAliases": [],
|
||||
"localAliases": [
|
||||
{
|
||||
"accessKeyId": "GK31c2f218a2e44f485b94239e",
|
||||
"alias": "test"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
```
|
||||
|
||||
### GetBucketInfo `GET /v0/bucket?id=<bucket id>`
|
||||
### GetBucketInfo `GET /v0/bucket?globalAlias=<alias>`
|
||||
|
||||
Returns information about the requested storage bucket.
|
||||
|
||||
If `id` is set, the bucket is looked up using its exact identifier.
|
||||
If `globalAlias` is set, the bucket is looked up using its global alias.
|
||||
(both are fast)
|
||||
|
||||
Example response:
|
||||
|
||||
```json
|
||||
{
|
||||
"id": "e6a14cd6a27f48684579ec6b381c078ab11697e6bc8513b72b2f5307e25fff9b",
|
||||
"globalAliases": [
|
||||
"alex"
|
||||
],
|
||||
"keys": [
|
||||
{
|
||||
"accessKeyId": "GK31c2f218a2e44f485b94239e",
|
||||
"name": "alex",
|
||||
"permissions": {
|
||||
"read": true,
|
||||
"write": true,
|
||||
"owner": true
|
||||
},
|
||||
"bucketLocalAliases": [
|
||||
"test"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
### CreateBucket `POST /v0/bucket`
|
||||
|
||||
Creates a new storage bucket.
|
||||
|
||||
Request body format:
|
||||
|
||||
```json
|
||||
{
|
||||
"globalAlias": "NameOfMyBucket"
|
||||
}
|
||||
```
|
||||
|
||||
OR
|
||||
|
||||
```json
|
||||
{
|
||||
"localAlias": {
|
||||
"accessKeyId": "GK31c2f218a2e44f485b94239e",
|
||||
"alias": "NameOfMyBucket",
|
||||
"allPermissions": true
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
OR
|
||||
|
||||
```json
|
||||
{}
|
||||
```
|
||||
|
||||
Creates a new bucket, either with a global alias, a local one,
|
||||
or no alias at all.
|
||||
|
||||
### DeleteBucket `DELETE /v0/bucket?id=<bucket id>`
|
||||
|
||||
Deletes a storage bucket. A bucket cannot be deleted if it is not empty.
|
||||
|
||||
Warning: this will delete all aliases associated with the bucket!
|
||||
|
||||
### PutBucketWebsite `PUT /v0/bucket/website?id=<bucket id>`
|
||||
|
||||
Sets the website configuration for a bucket (this also enables website access for this bucket).
|
||||
|
||||
Request body format:
|
||||
|
||||
```json
|
||||
{
|
||||
"indexDocument": "index.html",
|
||||
"errorDocument": "404.html",
|
||||
}
|
||||
```
|
||||
|
||||
The field `errorDocument` is optional, if no error document is set a generic error message is displayed when errors happen.
|
||||
|
||||
|
||||
### DeleteBucketWebsite `DELETE /v0/bucket/website?id=<bucket id>`
|
||||
|
||||
Deletes the website configuration for a bucket (disables website access for this bucket).
|
||||
|
||||
|
||||
## Operations on permissions for keys on buckets
|
||||
|
||||
### BucketAllowKey `POST /v0/bucket/allow`
|
||||
|
||||
Allows a key to do read/write/owner operations on a bucket.
|
||||
|
||||
Request body format:
|
||||
|
||||
```json
|
||||
{
|
||||
"bucketId": "e6a14cd6a27f48684579ec6b381c078ab11697e6bc8513b72b2f5307e25fff9b",
|
||||
"accessKeyId": "GK31c2f218a2e44f485b94239e",
|
||||
"permissions": {
|
||||
"read": true,
|
||||
"write": true,
|
||||
"owner": true
|
||||
},
|
||||
}
|
||||
```
|
||||
|
||||
Flags in `permissions` which have the value `true` will be activated.
|
||||
Other flags will remain unchanged.
|
||||
|
||||
### BucketDenyKey `POST /v0/bucket/deny`
|
||||
|
||||
Denies a key from doing read/write/owner operations on a bucket.
|
||||
|
||||
Request body format:
|
||||
|
||||
```json
|
||||
{
|
||||
"bucketId": "e6a14cd6a27f48684579ec6b381c078ab11697e6bc8513b72b2f5307e25fff9b",
|
||||
"accessKeyId": "GK31c2f218a2e44f485b94239e",
|
||||
"permissions": {
|
||||
"read": false,
|
||||
"write": false,
|
||||
"owner": true
|
||||
},
|
||||
}
|
||||
```
|
||||
|
||||
Flags in `permissions` which have the value `true` will be deactivated.
|
||||
Other flags will remain unchanged.
|
||||
|
||||
|
||||
## Operations on bucket aliases
|
||||
|
||||
### GlobalAliasBucket `PUT /v0/bucket/alias/global?id=<bucket id>&alias=<global alias>`
|
||||
|
||||
Empty body. Creates a global alias for a bucket.
|
||||
|
||||
### GlobalUnaliasBucket `DELETE /v0/bucket/alias/global?id=<bucket id>&alias=<global alias>`
|
||||
|
||||
Removes a global alias for a bucket.
|
||||
|
||||
### LocalAliasBucket `PUT /v0/bucket/alias/local?id=<bucket id>&accessKeyId=<access key ID>&alias=<local alias>`
|
||||
|
||||
Empty body. Creates a local alias for a bucket in the namespace of a specific access key.
|
||||
|
||||
### LocalUnaliasBucket `DELETE /v0/bucket/alias/local?id=<bucket id>&accessKeyId<access key ID>&alias=<local alias>`
|
||||
|
||||
Removes a local alias for a bucket in the namespace of a specific access key.
|
||||
|
||||
+7
-44
@@ -195,10 +195,6 @@ TO UNDERSTAND IN ORDER TO USE IT CORRECTLY.**
|
||||
|
||||
## API Endpoints
|
||||
|
||||
**Remark.** Example queries and responses here are given in JSON5 format
|
||||
for clarity. However the actual K2V API uses basic JSON so all examples
|
||||
and responses need to be translated.
|
||||
|
||||
### Operations on single items
|
||||
|
||||
**ReadItem: `GET /<bucket>/<partition key>?sort_key=<sort key>`**
|
||||
@@ -374,11 +370,8 @@ HTTP/1.1 204 NO CONTENT
|
||||
**ReadIndex: `GET /<bucket>?start=<start>&end=<end>&limit=<limit>`**
|
||||
|
||||
Lists all partition keys in the bucket for which some triplets exist, and gives
|
||||
for each the number of triplets, total number of values (which might be bigger
|
||||
than the number of triplets in case of conflicts), total number of bytes of
|
||||
these values, and number of triplets that are in a state of conflict.
|
||||
The values returned are an approximation of the true counts in the bucket,
|
||||
as these values are asynchronously updated, and thus eventually consistent.
|
||||
for each the number of triplets (or an approximation thereof, this value is
|
||||
asynchronously updated, and thus eventually consistent).
|
||||
|
||||
Query parameters:
|
||||
|
||||
@@ -433,41 +426,11 @@ HTTP/1.1 200 OK
|
||||
limit: null,
|
||||
reverse: false,
|
||||
partitionKeys: [
|
||||
{
|
||||
pk: "keys",
|
||||
entries: 3043,
|
||||
conflicts: 0,
|
||||
values: 3043,
|
||||
bytes: 121720,
|
||||
},
|
||||
{
|
||||
pk: "mailbox:INBOX",
|
||||
entries: 42,
|
||||
conflicts: 1,
|
||||
values: 43,
|
||||
bytes: 142029,
|
||||
},
|
||||
{
|
||||
pk: "mailbox:Junk",
|
||||
entries: 2991
|
||||
conflicts: 0,
|
||||
values: 2991,
|
||||
bytes: 12019322,
|
||||
},
|
||||
{
|
||||
pk: "mailbox:Trash",
|
||||
entries: 10,
|
||||
conflicts: 0,
|
||||
values: 10,
|
||||
bytes: 32401,
|
||||
},
|
||||
{
|
||||
pk: "mailboxes",
|
||||
entries: 3,
|
||||
conflicts: 0,
|
||||
values: 3,
|
||||
bytes: 3019,
|
||||
},
|
||||
{ pk: "keys", n: 3043 },
|
||||
{ pk: "mailbox:INBOX", n: 42 },
|
||||
{ pk: "mailbox:Junk", n: 2991 },
|
||||
{ pk: "mailbox:Trash", n: 10 },
|
||||
{ pk: "mailboxes", n: 3 },
|
||||
],
|
||||
more: false,
|
||||
nextStart: null,
|
||||
|
||||
@@ -79,8 +79,6 @@ function refresh_toolchain {
|
||||
pkgs.rustfmt
|
||||
pkgs.perl
|
||||
pkgs.protobuf
|
||||
pkgs.pkg-config
|
||||
pkgs.openssl
|
||||
cargo2nix.packages.x86_64-linux.cargo2nix
|
||||
] else [])
|
||||
++
|
||||
|
||||
@@ -0,0 +1,29 @@
|
||||
[package]
|
||||
name = "garage_admin"
|
||||
version = "0.7.0"
|
||||
authors = ["Maximilien Richer <code@mricher.fr>"]
|
||||
edition = "2018"
|
||||
license = "AGPL-3.0"
|
||||
description = "Administration and metrics REST HTTP server for Garage"
|
||||
repository = "https://git.deuxfleurs.fr/Deuxfleurs/garage"
|
||||
|
||||
[lib]
|
||||
path = "lib.rs"
|
||||
|
||||
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
|
||||
|
||||
[dependencies]
|
||||
garage_util = { version = "0.7.0", path = "../util" }
|
||||
|
||||
hex = "0.4"
|
||||
|
||||
futures = "0.3"
|
||||
futures-util = "0.3"
|
||||
http = "0.2"
|
||||
hyper = "0.14"
|
||||
tracing = "0.1.30"
|
||||
|
||||
opentelemetry = { version = "0.17", features = [ "rt-tokio" ] }
|
||||
opentelemetry-prometheus = "0.10"
|
||||
opentelemetry-otlp = "0.10"
|
||||
prometheus = "0.13"
|
||||
@@ -0,0 +1,6 @@
|
||||
//! Crate for handling the admin and metric HTTP APIs
|
||||
#[macro_use]
|
||||
extern crate tracing;
|
||||
|
||||
pub mod metrics;
|
||||
pub mod tracing_setup;
|
||||
@@ -0,0 +1,146 @@
|
||||
use std::convert::Infallible;
|
||||
use std::net::SocketAddr;
|
||||
use std::sync::Arc;
|
||||
use std::time::SystemTime;
|
||||
|
||||
use futures::future::*;
|
||||
use hyper::{
|
||||
header::CONTENT_TYPE,
|
||||
service::{make_service_fn, service_fn},
|
||||
Body, Method, Request, Response, Server,
|
||||
};
|
||||
|
||||
use opentelemetry::{
|
||||
global,
|
||||
metrics::{BoundCounter, BoundValueRecorder},
|
||||
trace::{FutureExt, TraceContextExt, Tracer},
|
||||
Context,
|
||||
};
|
||||
use opentelemetry_prometheus::PrometheusExporter;
|
||||
|
||||
use prometheus::{Encoder, TextEncoder};
|
||||
|
||||
use garage_util::error::Error as GarageError;
|
||||
use garage_util::metrics::*;
|
||||
|
||||
// serve_req on metric endpoint
|
||||
async fn serve_req(
|
||||
req: Request<Body>,
|
||||
admin_server: Arc<AdminServer>,
|
||||
) -> Result<Response<Body>, hyper::Error> {
|
||||
debug!("Receiving request at path {}", req.uri());
|
||||
let request_start = SystemTime::now();
|
||||
|
||||
admin_server.metrics.http_counter.add(1);
|
||||
|
||||
let response = match (req.method(), req.uri().path()) {
|
||||
(&Method::GET, "/metrics") => {
|
||||
let mut buffer = vec![];
|
||||
let encoder = TextEncoder::new();
|
||||
|
||||
let tracer = opentelemetry::global::tracer("garage");
|
||||
let metric_families = tracer.in_span("admin/gather_metrics", |_| {
|
||||
admin_server.exporter.registry().gather()
|
||||
});
|
||||
|
||||
encoder.encode(&metric_families, &mut buffer).unwrap();
|
||||
admin_server
|
||||
.metrics
|
||||
.http_body_gauge
|
||||
.record(buffer.len() as u64);
|
||||
|
||||
Response::builder()
|
||||
.status(200)
|
||||
.header(CONTENT_TYPE, encoder.format_type())
|
||||
.body(Body::from(buffer))
|
||||
.unwrap()
|
||||
}
|
||||
_ => Response::builder()
|
||||
.status(404)
|
||||
.body(Body::from("Not implemented"))
|
||||
.unwrap(),
|
||||
};
|
||||
|
||||
admin_server
|
||||
.metrics
|
||||
.http_req_histogram
|
||||
.record(request_start.elapsed().map_or(0.0, |d| d.as_secs_f64()));
|
||||
Ok(response)
|
||||
}
|
||||
|
||||
// AdminServer hold the admin server internal admin_server and the metric exporter
|
||||
pub struct AdminServer {
|
||||
exporter: PrometheusExporter,
|
||||
metrics: AdminServerMetrics,
|
||||
}
|
||||
|
||||
// GarageMetricadmin_server holds the metrics counter definition for Garage
|
||||
// FIXME: we would rather have that split up among the different libraries?
|
||||
struct AdminServerMetrics {
|
||||
http_counter: BoundCounter<u64>,
|
||||
http_body_gauge: BoundValueRecorder<u64>,
|
||||
http_req_histogram: BoundValueRecorder<f64>,
|
||||
}
|
||||
|
||||
impl AdminServer {
|
||||
/// init initilialize the AdminServer and background metric server
|
||||
pub fn init() -> AdminServer {
|
||||
let exporter = opentelemetry_prometheus::exporter().init();
|
||||
let meter = global::meter("garage/admin_server");
|
||||
AdminServer {
|
||||
exporter,
|
||||
metrics: AdminServerMetrics {
|
||||
http_counter: meter
|
||||
.u64_counter("admin.http_requests_total")
|
||||
.with_description("Total number of HTTP requests made.")
|
||||
.init()
|
||||
.bind(&[]),
|
||||
http_body_gauge: meter
|
||||
.u64_value_recorder("admin.http_response_size_bytes")
|
||||
.with_description("The metrics HTTP response sizes in bytes.")
|
||||
.init()
|
||||
.bind(&[]),
|
||||
http_req_histogram: meter
|
||||
.f64_value_recorder("admin.http_request_duration_seconds")
|
||||
.with_description("The HTTP request latencies in seconds.")
|
||||
.init()
|
||||
.bind(&[]),
|
||||
},
|
||||
}
|
||||
}
|
||||
/// run execute the admin server on the designated HTTP port and listen for requests
|
||||
pub async fn run(
|
||||
self,
|
||||
bind_addr: SocketAddr,
|
||||
shutdown_signal: impl Future<Output = ()>,
|
||||
) -> Result<(), GarageError> {
|
||||
let admin_server = Arc::new(self);
|
||||
// For every connection, we must make a `Service` to handle all
|
||||
// incoming HTTP requests on said connection.
|
||||
let make_svc = make_service_fn(move |_conn| {
|
||||
let admin_server = admin_server.clone();
|
||||
// This is the `Service` that will handle the connection.
|
||||
// `service_fn` is a helper to convert a function that
|
||||
// returns a Response into a `Service`.
|
||||
async move {
|
||||
Ok::<_, Infallible>(service_fn(move |req| {
|
||||
let tracer = opentelemetry::global::tracer("garage");
|
||||
let span = tracer
|
||||
.span_builder("admin/request")
|
||||
.with_trace_id(gen_trace_id())
|
||||
.start(&tracer);
|
||||
|
||||
serve_req(req, admin_server.clone())
|
||||
.with_context(Context::current_with_span(span))
|
||||
}))
|
||||
}
|
||||
});
|
||||
|
||||
let server = Server::bind(&bind_addr).serve(make_svc);
|
||||
let graceful = server.with_graceful_shutdown(shutdown_signal);
|
||||
info!("Admin server listening on http://{}", bind_addr);
|
||||
|
||||
graceful.await?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -54,9 +54,6 @@ quick-xml = { version = "0.21", features = [ "serialize" ] }
|
||||
url = "2.1"
|
||||
|
||||
opentelemetry = "0.17"
|
||||
opentelemetry-prometheus = "0.10"
|
||||
opentelemetry-otlp = "0.10"
|
||||
prometheus = "0.13"
|
||||
|
||||
[features]
|
||||
k2v = [ "garage_util/k2v", "garage_model/k2v" ]
|
||||
|
||||
@@ -1,189 +0,0 @@
|
||||
use std::sync::Arc;
|
||||
|
||||
use async_trait::async_trait;
|
||||
|
||||
use futures::future::Future;
|
||||
use http::header::{
|
||||
ACCESS_CONTROL_ALLOW_METHODS, ACCESS_CONTROL_ALLOW_ORIGIN, ALLOW, CONTENT_TYPE,
|
||||
};
|
||||
use hyper::{Body, Request, Response};
|
||||
|
||||
use opentelemetry::trace::{SpanRef, Tracer};
|
||||
use opentelemetry_prometheus::PrometheusExporter;
|
||||
use prometheus::{Encoder, TextEncoder};
|
||||
|
||||
use garage_model::garage::Garage;
|
||||
use garage_util::error::Error as GarageError;
|
||||
|
||||
use crate::generic_server::*;
|
||||
|
||||
use crate::admin::bucket::*;
|
||||
use crate::admin::cluster::*;
|
||||
use crate::admin::error::*;
|
||||
use crate::admin::key::*;
|
||||
use crate::admin::router::{Authorization, Endpoint};
|
||||
|
||||
pub struct AdminApiServer {
|
||||
garage: Arc<Garage>,
|
||||
exporter: PrometheusExporter,
|
||||
metrics_token: Option<String>,
|
||||
admin_token: Option<String>,
|
||||
}
|
||||
|
||||
impl AdminApiServer {
|
||||
pub fn new(garage: Arc<Garage>) -> Self {
|
||||
let exporter = opentelemetry_prometheus::exporter().init();
|
||||
let cfg = &garage.config.admin;
|
||||
let metrics_token = cfg
|
||||
.metrics_token
|
||||
.as_ref()
|
||||
.map(|tok| format!("Bearer {}", tok));
|
||||
let admin_token = cfg
|
||||
.admin_token
|
||||
.as_ref()
|
||||
.map(|tok| format!("Bearer {}", tok));
|
||||
Self {
|
||||
garage,
|
||||
exporter,
|
||||
metrics_token,
|
||||
admin_token,
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn run(self, shutdown_signal: impl Future<Output = ()>) -> Result<(), GarageError> {
|
||||
if let Some(bind_addr) = self.garage.config.admin.api_bind_addr {
|
||||
let region = self.garage.config.s3_api.s3_region.clone();
|
||||
ApiServer::new(region, self)
|
||||
.run_server(bind_addr, shutdown_signal)
|
||||
.await
|
||||
} else {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
fn handle_options(&self, _req: &Request<Body>) -> Result<Response<Body>, Error> {
|
||||
Ok(Response::builder()
|
||||
.status(204)
|
||||
.header(ALLOW, "OPTIONS, GET, POST")
|
||||
.header(ACCESS_CONTROL_ALLOW_METHODS, "OPTIONS, GET, POST")
|
||||
.header(ACCESS_CONTROL_ALLOW_ORIGIN, "*")
|
||||
.body(Body::empty())?)
|
||||
}
|
||||
|
||||
fn handle_metrics(&self) -> Result<Response<Body>, Error> {
|
||||
let mut buffer = vec![];
|
||||
let encoder = TextEncoder::new();
|
||||
|
||||
let tracer = opentelemetry::global::tracer("garage");
|
||||
let metric_families = tracer.in_span("admin/gather_metrics", |_| {
|
||||
self.exporter.registry().gather()
|
||||
});
|
||||
|
||||
encoder
|
||||
.encode(&metric_families, &mut buffer)
|
||||
.ok_or_internal_error("Could not serialize metrics")?;
|
||||
|
||||
Ok(Response::builder()
|
||||
.status(200)
|
||||
.header(CONTENT_TYPE, encoder.format_type())
|
||||
.body(Body::from(buffer))?)
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl ApiHandler for AdminApiServer {
|
||||
const API_NAME: &'static str = "admin";
|
||||
const API_NAME_DISPLAY: &'static str = "Admin";
|
||||
|
||||
type Endpoint = Endpoint;
|
||||
type Error = Error;
|
||||
|
||||
fn parse_endpoint(&self, req: &Request<Body>) -> Result<Endpoint, Error> {
|
||||
Endpoint::from_request(req)
|
||||
}
|
||||
|
||||
async fn handle(
|
||||
&self,
|
||||
req: Request<Body>,
|
||||
endpoint: Endpoint,
|
||||
) -> Result<Response<Body>, Error> {
|
||||
let expected_auth_header = match endpoint.authorization_type() {
|
||||
Authorization::MetricsToken => self.metrics_token.as_ref(),
|
||||
Authorization::AdminToken => self.admin_token.as_ref(),
|
||||
};
|
||||
|
||||
if let Some(h) = expected_auth_header {
|
||||
match req.headers().get("Authorization") {
|
||||
None => Err(Error::forbidden("Authorization token must be provided")),
|
||||
Some(v) if v.to_str().map(|hv| hv == h).unwrap_or(false) => Ok(()),
|
||||
_ => Err(Error::forbidden("Invalid authorization token provided")),
|
||||
}?;
|
||||
}
|
||||
|
||||
match endpoint {
|
||||
Endpoint::Options => self.handle_options(&req),
|
||||
Endpoint::Metrics => self.handle_metrics(),
|
||||
Endpoint::GetClusterStatus => handle_get_cluster_status(&self.garage).await,
|
||||
Endpoint::ConnectClusterNodes => handle_connect_cluster_nodes(&self.garage, req).await,
|
||||
// Layout
|
||||
Endpoint::GetClusterLayout => handle_get_cluster_layout(&self.garage).await,
|
||||
Endpoint::UpdateClusterLayout => handle_update_cluster_layout(&self.garage, req).await,
|
||||
Endpoint::ApplyClusterLayout => handle_apply_cluster_layout(&self.garage, req).await,
|
||||
Endpoint::RevertClusterLayout => handle_revert_cluster_layout(&self.garage, req).await,
|
||||
// Keys
|
||||
Endpoint::ListKeys => handle_list_keys(&self.garage).await,
|
||||
Endpoint::GetKeyInfo { id, search } => {
|
||||
handle_get_key_info(&self.garage, id, search).await
|
||||
}
|
||||
Endpoint::CreateKey => handle_create_key(&self.garage, req).await,
|
||||
Endpoint::ImportKey => handle_import_key(&self.garage, req).await,
|
||||
Endpoint::UpdateKey { id } => handle_update_key(&self.garage, id, req).await,
|
||||
Endpoint::DeleteKey { id } => handle_delete_key(&self.garage, id).await,
|
||||
// Buckets
|
||||
Endpoint::ListBuckets => handle_list_buckets(&self.garage).await,
|
||||
Endpoint::GetBucketInfo { id, global_alias } => {
|
||||
handle_get_bucket_info(&self.garage, id, global_alias).await
|
||||
}
|
||||
Endpoint::CreateBucket => handle_create_bucket(&self.garage, req).await,
|
||||
Endpoint::DeleteBucket { id } => handle_delete_bucket(&self.garage, id).await,
|
||||
Endpoint::PutBucketWebsite { id } => {
|
||||
handle_put_bucket_website(&self.garage, id, req).await
|
||||
}
|
||||
Endpoint::DeleteBucketWebsite { id } => {
|
||||
handle_delete_bucket_website(&self.garage, id).await
|
||||
}
|
||||
// Bucket-key permissions
|
||||
Endpoint::BucketAllowKey => {
|
||||
handle_bucket_change_key_perm(&self.garage, req, true).await
|
||||
}
|
||||
Endpoint::BucketDenyKey => {
|
||||
handle_bucket_change_key_perm(&self.garage, req, false).await
|
||||
}
|
||||
// Bucket aliasing
|
||||
Endpoint::GlobalAliasBucket { id, alias } => {
|
||||
handle_global_alias_bucket(&self.garage, id, alias).await
|
||||
}
|
||||
Endpoint::GlobalUnaliasBucket { id, alias } => {
|
||||
handle_global_unalias_bucket(&self.garage, id, alias).await
|
||||
}
|
||||
Endpoint::LocalAliasBucket {
|
||||
id,
|
||||
access_key_id,
|
||||
alias,
|
||||
} => handle_local_alias_bucket(&self.garage, id, access_key_id, alias).await,
|
||||
Endpoint::LocalUnaliasBucket {
|
||||
id,
|
||||
access_key_id,
|
||||
alias,
|
||||
} => handle_local_unalias_bucket(&self.garage, id, access_key_id, alias).await,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl ApiEndpoint for Endpoint {
|
||||
fn name(&self) -> &'static str {
|
||||
Endpoint::name(self)
|
||||
}
|
||||
|
||||
fn add_span_attributes(&self, _span: SpanRef<'_>) {}
|
||||
}
|
||||
@@ -1,542 +0,0 @@
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
|
||||
use hyper::{Body, Request, Response, StatusCode};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use garage_util::crdt::*;
|
||||
use garage_util::data::*;
|
||||
use garage_util::error::Error as GarageError;
|
||||
|
||||
use garage_table::*;
|
||||
|
||||
use garage_model::bucket_alias_table::*;
|
||||
use garage_model::bucket_table::*;
|
||||
use garage_model::garage::Garage;
|
||||
use garage_model::permission::*;
|
||||
|
||||
use crate::admin::error::*;
|
||||
use crate::admin::key::ApiBucketKeyPerm;
|
||||
use crate::common_error::CommonError;
|
||||
use crate::helpers::parse_json_body;
|
||||
|
||||
pub async fn handle_list_buckets(garage: &Arc<Garage>) -> Result<Response<Body>, Error> {
|
||||
let buckets = garage
|
||||
.bucket_table
|
||||
.get_range(
|
||||
&EmptyKey,
|
||||
None,
|
||||
Some(DeletedFilter::NotDeleted),
|
||||
10000,
|
||||
EnumerationOrder::Forward,
|
||||
)
|
||||
.await?;
|
||||
|
||||
let res = buckets
|
||||
.into_iter()
|
||||
.map(|b| {
|
||||
let state = b.state.as_option().unwrap();
|
||||
ListBucketResultItem {
|
||||
id: hex::encode(b.id),
|
||||
global_aliases: state
|
||||
.aliases
|
||||
.items()
|
||||
.iter()
|
||||
.filter(|(_, _, a)| *a)
|
||||
.map(|(n, _, _)| n.to_string())
|
||||
.collect::<Vec<_>>(),
|
||||
local_aliases: state
|
||||
.local_aliases
|
||||
.items()
|
||||
.iter()
|
||||
.filter(|(_, _, a)| *a)
|
||||
.map(|((k, n), _, _)| BucketLocalAlias {
|
||||
access_key_id: k.to_string(),
|
||||
alias: n.to_string(),
|
||||
})
|
||||
.collect::<Vec<_>>(),
|
||||
}
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
let resp_json = serde_json::to_string_pretty(&res).map_err(GarageError::from)?;
|
||||
Ok(Response::builder()
|
||||
.status(StatusCode::OK)
|
||||
.body(Body::from(resp_json))?)
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct ListBucketResultItem {
|
||||
id: String,
|
||||
global_aliases: Vec<String>,
|
||||
local_aliases: Vec<BucketLocalAlias>,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct BucketLocalAlias {
|
||||
access_key_id: String,
|
||||
alias: String,
|
||||
}
|
||||
|
||||
pub async fn handle_get_bucket_info(
|
||||
garage: &Arc<Garage>,
|
||||
id: Option<String>,
|
||||
global_alias: Option<String>,
|
||||
) -> Result<Response<Body>, Error> {
|
||||
let bucket_id = match (id, global_alias) {
|
||||
(Some(id), None) => parse_bucket_id(&id)?,
|
||||
(None, Some(ga)) => garage
|
||||
.bucket_helper()
|
||||
.resolve_global_bucket_name(&ga)
|
||||
.await?
|
||||
.ok_or_else(|| HelperError::NoSuchBucket(ga.to_string()))?,
|
||||
_ => {
|
||||
return Err(Error::bad_request(
|
||||
"Either id or globalAlias must be provided (but not both)",
|
||||
));
|
||||
}
|
||||
};
|
||||
|
||||
bucket_info_results(garage, bucket_id).await
|
||||
}
|
||||
|
||||
async fn bucket_info_results(
|
||||
garage: &Arc<Garage>,
|
||||
bucket_id: Uuid,
|
||||
) -> Result<Response<Body>, Error> {
|
||||
let bucket = garage
|
||||
.bucket_helper()
|
||||
.get_existing_bucket(bucket_id)
|
||||
.await?;
|
||||
|
||||
let mut relevant_keys = HashMap::new();
|
||||
for (k, _) in bucket
|
||||
.state
|
||||
.as_option()
|
||||
.unwrap()
|
||||
.authorized_keys
|
||||
.items()
|
||||
.iter()
|
||||
{
|
||||
if let Some(key) = garage
|
||||
.key_table
|
||||
.get(&EmptyKey, k)
|
||||
.await?
|
||||
.filter(|k| !k.is_deleted())
|
||||
{
|
||||
if !key.state.is_deleted() {
|
||||
relevant_keys.insert(k.clone(), key);
|
||||
}
|
||||
}
|
||||
}
|
||||
for ((k, _), _, _) in bucket
|
||||
.state
|
||||
.as_option()
|
||||
.unwrap()
|
||||
.local_aliases
|
||||
.items()
|
||||
.iter()
|
||||
{
|
||||
if relevant_keys.contains_key(k) {
|
||||
continue;
|
||||
}
|
||||
if let Some(key) = garage.key_table.get(&EmptyKey, k).await? {
|
||||
if !key.state.is_deleted() {
|
||||
relevant_keys.insert(k.clone(), key);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let state = bucket.state.as_option().unwrap();
|
||||
|
||||
let res =
|
||||
GetBucketInfoResult {
|
||||
id: hex::encode(&bucket.id),
|
||||
global_aliases: state
|
||||
.aliases
|
||||
.items()
|
||||
.iter()
|
||||
.filter(|(_, _, a)| *a)
|
||||
.map(|(n, _, _)| n.to_string())
|
||||
.collect::<Vec<_>>(),
|
||||
website_access: state.website_config.get().is_some(),
|
||||
website_config: state.website_config.get().clone().map(|wsc| {
|
||||
GetBucketInfoWebsiteResult {
|
||||
index_document: wsc.index_document,
|
||||
error_document: wsc.error_document,
|
||||
}
|
||||
}),
|
||||
keys: relevant_keys
|
||||
.into_iter()
|
||||
.map(|(_, key)| {
|
||||
let p = key.state.as_option().unwrap();
|
||||
GetBucketInfoKey {
|
||||
access_key_id: key.key_id,
|
||||
name: p.name.get().to_string(),
|
||||
permissions: p
|
||||
.authorized_buckets
|
||||
.get(&bucket.id)
|
||||
.map(|p| ApiBucketKeyPerm {
|
||||
read: p.allow_read,
|
||||
write: p.allow_write,
|
||||
owner: p.allow_owner,
|
||||
})
|
||||
.unwrap_or_default(),
|
||||
bucket_local_aliases: p
|
||||
.local_aliases
|
||||
.items()
|
||||
.iter()
|
||||
.filter(|(_, _, b)| *b == Some(bucket.id))
|
||||
.map(|(n, _, _)| n.to_string())
|
||||
.collect::<Vec<_>>(),
|
||||
}
|
||||
})
|
||||
.collect::<Vec<_>>(),
|
||||
};
|
||||
|
||||
let resp_json = serde_json::to_string_pretty(&res).map_err(GarageError::from)?;
|
||||
Ok(Response::builder()
|
||||
.status(StatusCode::OK)
|
||||
.body(Body::from(resp_json))?)
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct GetBucketInfoResult {
|
||||
id: String,
|
||||
global_aliases: Vec<String>,
|
||||
website_access: bool,
|
||||
#[serde(default)]
|
||||
website_config: Option<GetBucketInfoWebsiteResult>,
|
||||
keys: Vec<GetBucketInfoKey>,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct GetBucketInfoWebsiteResult {
|
||||
index_document: String,
|
||||
error_document: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct GetBucketInfoKey {
|
||||
access_key_id: String,
|
||||
name: String,
|
||||
permissions: ApiBucketKeyPerm,
|
||||
bucket_local_aliases: Vec<String>,
|
||||
}
|
||||
|
||||
pub async fn handle_create_bucket(
|
||||
garage: &Arc<Garage>,
|
||||
req: Request<Body>,
|
||||
) -> Result<Response<Body>, Error> {
|
||||
let req = parse_json_body::<CreateBucketRequest>(req).await?;
|
||||
|
||||
if let Some(ga) = &req.global_alias {
|
||||
if !is_valid_bucket_name(ga) {
|
||||
return Err(Error::bad_request(format!(
|
||||
"{}: {}",
|
||||
ga, INVALID_BUCKET_NAME_MESSAGE
|
||||
)));
|
||||
}
|
||||
|
||||
if let Some(alias) = garage.bucket_alias_table.get(&EmptyKey, ga).await? {
|
||||
if alias.state.get().is_some() {
|
||||
return Err(CommonError::BucketAlreadyExists.into());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(la) = &req.local_alias {
|
||||
if !is_valid_bucket_name(&la.alias) {
|
||||
return Err(Error::bad_request(format!(
|
||||
"{}: {}",
|
||||
la.alias, INVALID_BUCKET_NAME_MESSAGE
|
||||
)));
|
||||
}
|
||||
|
||||
let key = garage
|
||||
.key_helper()
|
||||
.get_existing_key(&la.access_key_id)
|
||||
.await?;
|
||||
let state = key.state.as_option().unwrap();
|
||||
if matches!(state.local_aliases.get(&la.alias), Some(_)) {
|
||||
return Err(Error::bad_request("Local alias already exists"));
|
||||
}
|
||||
}
|
||||
|
||||
let bucket = Bucket::new();
|
||||
garage.bucket_table.insert(&bucket).await?;
|
||||
|
||||
if let Some(ga) = &req.global_alias {
|
||||
garage
|
||||
.bucket_helper()
|
||||
.set_global_bucket_alias(bucket.id, ga)
|
||||
.await?;
|
||||
}
|
||||
|
||||
if let Some(la) = &req.local_alias {
|
||||
garage
|
||||
.bucket_helper()
|
||||
.set_local_bucket_alias(bucket.id, &la.access_key_id, &la.alias)
|
||||
.await?;
|
||||
if la.all_permissions {
|
||||
garage
|
||||
.bucket_helper()
|
||||
.set_bucket_key_permissions(
|
||||
bucket.id,
|
||||
&la.access_key_id,
|
||||
BucketKeyPerm::ALL_PERMISSIONS,
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
}
|
||||
|
||||
bucket_info_results(garage, bucket.id).await
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct CreateBucketRequest {
|
||||
global_alias: Option<String>,
|
||||
local_alias: Option<CreateBucketLocalAlias>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct CreateBucketLocalAlias {
|
||||
access_key_id: String,
|
||||
alias: String,
|
||||
#[serde(default)]
|
||||
all_permissions: bool,
|
||||
}
|
||||
|
||||
pub async fn handle_delete_bucket(
|
||||
garage: &Arc<Garage>,
|
||||
id: String,
|
||||
) -> Result<Response<Body>, Error> {
|
||||
let helper = garage.bucket_helper();
|
||||
|
||||
let bucket_id = parse_bucket_id(&id)?;
|
||||
|
||||
let mut bucket = helper.get_existing_bucket(bucket_id).await?;
|
||||
let state = bucket.state.as_option().unwrap();
|
||||
|
||||
// Check bucket is empty
|
||||
if !helper.is_bucket_empty(bucket_id).await? {
|
||||
return Err(CommonError::BucketNotEmpty.into());
|
||||
}
|
||||
|
||||
// --- done checking, now commit ---
|
||||
// 1. delete authorization from keys that had access
|
||||
for (key_id, perm) in bucket.authorized_keys() {
|
||||
if perm.is_any() {
|
||||
helper
|
||||
.set_bucket_key_permissions(bucket.id, key_id, BucketKeyPerm::NO_PERMISSIONS)
|
||||
.await?;
|
||||
}
|
||||
}
|
||||
// 2. delete all local aliases
|
||||
for ((key_id, alias), _, active) in state.local_aliases.items().iter() {
|
||||
if *active {
|
||||
helper
|
||||
.unset_local_bucket_alias(bucket.id, key_id, alias)
|
||||
.await?;
|
||||
}
|
||||
}
|
||||
// 3. delete all global aliases
|
||||
for (alias, _, active) in state.aliases.items().iter() {
|
||||
if *active {
|
||||
helper.purge_global_bucket_alias(bucket.id, alias).await?;
|
||||
}
|
||||
}
|
||||
|
||||
// 4. delete bucket
|
||||
bucket.state = Deletable::delete();
|
||||
garage.bucket_table.insert(&bucket).await?;
|
||||
|
||||
Ok(Response::builder()
|
||||
.status(StatusCode::NO_CONTENT)
|
||||
.body(Body::empty())?)
|
||||
}
|
||||
|
||||
// ---- BUCKET WEBSITE CONFIGURATION ----
|
||||
|
||||
pub async fn handle_put_bucket_website(
|
||||
garage: &Arc<Garage>,
|
||||
id: String,
|
||||
req: Request<Body>,
|
||||
) -> Result<Response<Body>, Error> {
|
||||
let req = parse_json_body::<PutBucketWebsiteRequest>(req).await?;
|
||||
let bucket_id = parse_bucket_id(&id)?;
|
||||
|
||||
let mut bucket = garage
|
||||
.bucket_helper()
|
||||
.get_existing_bucket(bucket_id)
|
||||
.await?;
|
||||
|
||||
let state = bucket.state.as_option_mut().unwrap();
|
||||
state.website_config.update(Some(WebsiteConfig {
|
||||
index_document: req.index_document,
|
||||
error_document: req.error_document,
|
||||
}));
|
||||
|
||||
garage.bucket_table.insert(&bucket).await?;
|
||||
|
||||
bucket_info_results(garage, bucket_id).await
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct PutBucketWebsiteRequest {
|
||||
index_document: String,
|
||||
#[serde(default)]
|
||||
error_document: Option<String>,
|
||||
}
|
||||
|
||||
pub async fn handle_delete_bucket_website(
|
||||
garage: &Arc<Garage>,
|
||||
id: String,
|
||||
) -> Result<Response<Body>, Error> {
|
||||
let bucket_id = parse_bucket_id(&id)?;
|
||||
|
||||
let mut bucket = garage
|
||||
.bucket_helper()
|
||||
.get_existing_bucket(bucket_id)
|
||||
.await?;
|
||||
|
||||
let state = bucket.state.as_option_mut().unwrap();
|
||||
state.website_config.update(None);
|
||||
|
||||
garage.bucket_table.insert(&bucket).await?;
|
||||
|
||||
bucket_info_results(garage, bucket_id).await
|
||||
}
|
||||
|
||||
// ---- BUCKET/KEY PERMISSIONS ----
|
||||
|
||||
pub async fn handle_bucket_change_key_perm(
|
||||
garage: &Arc<Garage>,
|
||||
req: Request<Body>,
|
||||
new_perm_flag: bool,
|
||||
) -> Result<Response<Body>, Error> {
|
||||
let req = parse_json_body::<BucketKeyPermChangeRequest>(req).await?;
|
||||
|
||||
let bucket_id = parse_bucket_id(&req.bucket_id)?;
|
||||
|
||||
let bucket = garage
|
||||
.bucket_helper()
|
||||
.get_existing_bucket(bucket_id)
|
||||
.await?;
|
||||
let state = bucket.state.as_option().unwrap();
|
||||
|
||||
let key = garage
|
||||
.key_helper()
|
||||
.get_existing_key(&req.access_key_id)
|
||||
.await?;
|
||||
|
||||
let mut perm = state
|
||||
.authorized_keys
|
||||
.get(&key.key_id)
|
||||
.cloned()
|
||||
.unwrap_or(BucketKeyPerm::NO_PERMISSIONS);
|
||||
|
||||
if req.permissions.read {
|
||||
perm.allow_read = new_perm_flag;
|
||||
}
|
||||
if req.permissions.write {
|
||||
perm.allow_write = new_perm_flag;
|
||||
}
|
||||
if req.permissions.owner {
|
||||
perm.allow_owner = new_perm_flag;
|
||||
}
|
||||
|
||||
garage
|
||||
.bucket_helper()
|
||||
.set_bucket_key_permissions(bucket.id, &key.key_id, perm)
|
||||
.await?;
|
||||
|
||||
bucket_info_results(garage, bucket.id).await
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct BucketKeyPermChangeRequest {
|
||||
bucket_id: String,
|
||||
access_key_id: String,
|
||||
permissions: ApiBucketKeyPerm,
|
||||
}
|
||||
|
||||
// ---- BUCKET ALIASES ----
|
||||
|
||||
pub async fn handle_global_alias_bucket(
|
||||
garage: &Arc<Garage>,
|
||||
bucket_id: String,
|
||||
alias: String,
|
||||
) -> Result<Response<Body>, Error> {
|
||||
let bucket_id = parse_bucket_id(&bucket_id)?;
|
||||
|
||||
garage
|
||||
.bucket_helper()
|
||||
.set_global_bucket_alias(bucket_id, &alias)
|
||||
.await?;
|
||||
|
||||
bucket_info_results(garage, bucket_id).await
|
||||
}
|
||||
|
||||
pub async fn handle_global_unalias_bucket(
|
||||
garage: &Arc<Garage>,
|
||||
bucket_id: String,
|
||||
alias: String,
|
||||
) -> Result<Response<Body>, Error> {
|
||||
let bucket_id = parse_bucket_id(&bucket_id)?;
|
||||
|
||||
garage
|
||||
.bucket_helper()
|
||||
.unset_global_bucket_alias(bucket_id, &alias)
|
||||
.await?;
|
||||
|
||||
bucket_info_results(garage, bucket_id).await
|
||||
}
|
||||
|
||||
pub async fn handle_local_alias_bucket(
|
||||
garage: &Arc<Garage>,
|
||||
bucket_id: String,
|
||||
access_key_id: String,
|
||||
alias: String,
|
||||
) -> Result<Response<Body>, Error> {
|
||||
let bucket_id = parse_bucket_id(&bucket_id)?;
|
||||
|
||||
garage
|
||||
.bucket_helper()
|
||||
.set_local_bucket_alias(bucket_id, &access_key_id, &alias)
|
||||
.await?;
|
||||
|
||||
bucket_info_results(garage, bucket_id).await
|
||||
}
|
||||
|
||||
pub async fn handle_local_unalias_bucket(
|
||||
garage: &Arc<Garage>,
|
||||
bucket_id: String,
|
||||
access_key_id: String,
|
||||
alias: String,
|
||||
) -> Result<Response<Body>, Error> {
|
||||
let bucket_id = parse_bucket_id(&bucket_id)?;
|
||||
|
||||
garage
|
||||
.bucket_helper()
|
||||
.unset_local_bucket_alias(bucket_id, &access_key_id, &alias)
|
||||
.await?;
|
||||
|
||||
bucket_info_results(garage, bucket_id).await
|
||||
}
|
||||
|
||||
// ---- HELPER ----
|
||||
|
||||
fn parse_bucket_id(id: &str) -> Result<Uuid, Error> {
|
||||
let id_hex = hex::decode(&id).ok_or_bad_request("Invalid bucket id")?;
|
||||
Ok(Uuid::try_from(&id_hex).ok_or_bad_request("Invalid bucket id")?)
|
||||
}
|
||||
@@ -1,198 +0,0 @@
|
||||
use std::collections::HashMap;
|
||||
use std::net::SocketAddr;
|
||||
use std::sync::Arc;
|
||||
|
||||
use hyper::{Body, Request, Response, StatusCode};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use garage_util::crdt::*;
|
||||
use garage_util::data::*;
|
||||
use garage_util::error::Error as GarageError;
|
||||
|
||||
use garage_rpc::layout::*;
|
||||
|
||||
use garage_model::garage::Garage;
|
||||
|
||||
use crate::admin::error::*;
|
||||
use crate::helpers::parse_json_body;
|
||||
|
||||
pub async fn handle_get_cluster_status(garage: &Arc<Garage>) -> Result<Response<Body>, Error> {
|
||||
let res = GetClusterStatusResponse {
|
||||
node: hex::encode(garage.system.id),
|
||||
garage_version: garage.system.garage_version(),
|
||||
known_nodes: garage
|
||||
.system
|
||||
.get_known_nodes()
|
||||
.into_iter()
|
||||
.map(|i| {
|
||||
(
|
||||
hex::encode(i.id),
|
||||
KnownNodeResp {
|
||||
addr: i.addr,
|
||||
is_up: i.is_up,
|
||||
last_seen_secs_ago: i.last_seen_secs_ago,
|
||||
hostname: i.status.hostname,
|
||||
},
|
||||
)
|
||||
})
|
||||
.collect(),
|
||||
layout: get_cluster_layout(garage),
|
||||
};
|
||||
|
||||
let resp_json = serde_json::to_string_pretty(&res).map_err(GarageError::from)?;
|
||||
Ok(Response::builder()
|
||||
.status(StatusCode::OK)
|
||||
.body(Body::from(resp_json))?)
|
||||
}
|
||||
|
||||
pub async fn handle_connect_cluster_nodes(
|
||||
garage: &Arc<Garage>,
|
||||
req: Request<Body>,
|
||||
) -> Result<Response<Body>, Error> {
|
||||
let req = parse_json_body::<Vec<String>>(req).await?;
|
||||
|
||||
let res = futures::future::join_all(req.iter().map(|node| garage.system.connect(node)))
|
||||
.await
|
||||
.into_iter()
|
||||
.map(|r| match r {
|
||||
Ok(()) => ConnectClusterNodesResponse {
|
||||
success: true,
|
||||
error: None,
|
||||
},
|
||||
Err(e) => ConnectClusterNodesResponse {
|
||||
success: false,
|
||||
error: Some(format!("{}", e)),
|
||||
},
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
let resp_json = serde_json::to_string_pretty(&res).map_err(GarageError::from)?;
|
||||
Ok(Response::builder()
|
||||
.status(StatusCode::OK)
|
||||
.body(Body::from(resp_json))?)
|
||||
}
|
||||
|
||||
pub async fn handle_get_cluster_layout(garage: &Arc<Garage>) -> Result<Response<Body>, Error> {
|
||||
let res = get_cluster_layout(garage);
|
||||
let resp_json = serde_json::to_string_pretty(&res).map_err(GarageError::from)?;
|
||||
Ok(Response::builder()
|
||||
.status(StatusCode::OK)
|
||||
.body(Body::from(resp_json))?)
|
||||
}
|
||||
|
||||
fn get_cluster_layout(garage: &Arc<Garage>) -> GetClusterLayoutResponse {
|
||||
let layout = garage.system.get_cluster_layout();
|
||||
|
||||
GetClusterLayoutResponse {
|
||||
version: layout.version,
|
||||
roles: layout
|
||||
.roles
|
||||
.items()
|
||||
.iter()
|
||||
.filter(|(_, _, v)| v.0.is_some())
|
||||
.map(|(k, _, v)| (hex::encode(k), v.0.clone()))
|
||||
.collect(),
|
||||
staged_role_changes: layout
|
||||
.staging
|
||||
.items()
|
||||
.iter()
|
||||
.filter(|(k, _, v)| layout.roles.get(k) != Some(v))
|
||||
.map(|(k, _, v)| (hex::encode(k), v.0.clone()))
|
||||
.collect(),
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct GetClusterStatusResponse {
|
||||
node: String,
|
||||
garage_version: &'static str,
|
||||
known_nodes: HashMap<String, KnownNodeResp>,
|
||||
layout: GetClusterLayoutResponse,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct ConnectClusterNodesResponse {
|
||||
success: bool,
|
||||
error: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct GetClusterLayoutResponse {
|
||||
version: u64,
|
||||
roles: HashMap<String, Option<NodeRole>>,
|
||||
staged_role_changes: HashMap<String, Option<NodeRole>>,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct KnownNodeResp {
|
||||
addr: SocketAddr,
|
||||
is_up: bool,
|
||||
last_seen_secs_ago: Option<u64>,
|
||||
hostname: String,
|
||||
}
|
||||
|
||||
pub async fn handle_update_cluster_layout(
|
||||
garage: &Arc<Garage>,
|
||||
req: Request<Body>,
|
||||
) -> Result<Response<Body>, Error> {
|
||||
let updates = parse_json_body::<UpdateClusterLayoutRequest>(req).await?;
|
||||
|
||||
let mut layout = garage.system.get_cluster_layout();
|
||||
|
||||
let mut roles = layout.roles.clone();
|
||||
roles.merge(&layout.staging);
|
||||
|
||||
for (node, role) in updates {
|
||||
let node = hex::decode(node).ok_or_bad_request("Invalid node identifier")?;
|
||||
let node = Uuid::try_from(&node).ok_or_bad_request("Invalid node identifier")?;
|
||||
|
||||
layout
|
||||
.staging
|
||||
.merge(&roles.update_mutator(node, NodeRoleV(role)));
|
||||
}
|
||||
|
||||
garage.system.update_cluster_layout(&layout).await?;
|
||||
|
||||
Ok(Response::builder()
|
||||
.status(StatusCode::OK)
|
||||
.body(Body::empty())?)
|
||||
}
|
||||
|
||||
pub async fn handle_apply_cluster_layout(
|
||||
garage: &Arc<Garage>,
|
||||
req: Request<Body>,
|
||||
) -> Result<Response<Body>, Error> {
|
||||
let param = parse_json_body::<ApplyRevertLayoutRequest>(req).await?;
|
||||
|
||||
let layout = garage.system.get_cluster_layout();
|
||||
let layout = layout.apply_staged_changes(Some(param.version))?;
|
||||
garage.system.update_cluster_layout(&layout).await?;
|
||||
|
||||
Ok(Response::builder()
|
||||
.status(StatusCode::OK)
|
||||
.body(Body::empty())?)
|
||||
}
|
||||
|
||||
pub async fn handle_revert_cluster_layout(
|
||||
garage: &Arc<Garage>,
|
||||
req: Request<Body>,
|
||||
) -> Result<Response<Body>, Error> {
|
||||
let param = parse_json_body::<ApplyRevertLayoutRequest>(req).await?;
|
||||
|
||||
let layout = garage.system.get_cluster_layout();
|
||||
let layout = layout.revert_staged_changes(Some(param.version))?;
|
||||
garage.system.update_cluster_layout(&layout).await?;
|
||||
|
||||
Ok(Response::builder()
|
||||
.status(StatusCode::OK)
|
||||
.body(Body::empty())?)
|
||||
}
|
||||
|
||||
type UpdateClusterLayoutRequest = HashMap<String, Option<NodeRole>>;
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct ApplyRevertLayoutRequest {
|
||||
version: u64,
|
||||
}
|
||||
@@ -1,96 +0,0 @@
|
||||
use err_derive::Error;
|
||||
use hyper::header::HeaderValue;
|
||||
use hyper::{Body, HeaderMap, StatusCode};
|
||||
|
||||
pub use garage_model::helper::error::Error as HelperError;
|
||||
|
||||
use crate::common_error::CommonError;
|
||||
pub use crate::common_error::{CommonErrorDerivative, OkOrBadRequest, OkOrInternalError};
|
||||
use crate::generic_server::ApiError;
|
||||
use crate::helpers::CustomApiErrorBody;
|
||||
|
||||
/// Errors of this crate
|
||||
#[derive(Debug, Error)]
|
||||
pub enum Error {
|
||||
#[error(display = "{}", _0)]
|
||||
/// Error from common error
|
||||
Common(CommonError),
|
||||
|
||||
// Category: cannot process
|
||||
/// The API access key does not exist
|
||||
#[error(display = "Access key not found: {}", _0)]
|
||||
NoSuchAccessKey(String),
|
||||
|
||||
/// In Import key, the key already exists
|
||||
#[error(
|
||||
display = "Key {} already exists in data store. Even if it is deleted, we can't let you create a new key with the same ID. Sorry.",
|
||||
_0
|
||||
)]
|
||||
KeyAlreadyExists(String),
|
||||
}
|
||||
|
||||
impl<T> From<T> for Error
|
||||
where
|
||||
CommonError: From<T>,
|
||||
{
|
||||
fn from(err: T) -> Self {
|
||||
Error::Common(CommonError::from(err))
|
||||
}
|
||||
}
|
||||
|
||||
impl CommonErrorDerivative for Error {}
|
||||
|
||||
impl From<HelperError> for Error {
|
||||
fn from(err: HelperError) -> Self {
|
||||
match err {
|
||||
HelperError::Internal(i) => Self::Common(CommonError::InternalError(i)),
|
||||
HelperError::BadRequest(b) => Self::Common(CommonError::BadRequest(b)),
|
||||
HelperError::InvalidBucketName(n) => Self::Common(CommonError::InvalidBucketName(n)),
|
||||
HelperError::NoSuchBucket(n) => Self::Common(CommonError::NoSuchBucket(n)),
|
||||
HelperError::NoSuchAccessKey(n) => Self::NoSuchAccessKey(n),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Error {
|
||||
fn code(&self) -> &'static str {
|
||||
match self {
|
||||
Error::Common(c) => c.aws_code(),
|
||||
Error::NoSuchAccessKey(_) => "NoSuchAccessKey",
|
||||
Error::KeyAlreadyExists(_) => "KeyAlreadyExists",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl ApiError for Error {
|
||||
/// Get the HTTP status code that best represents the meaning of the error for the client
|
||||
fn http_status_code(&self) -> StatusCode {
|
||||
match self {
|
||||
Error::Common(c) => c.http_status_code(),
|
||||
Error::NoSuchAccessKey(_) => StatusCode::NOT_FOUND,
|
||||
Error::KeyAlreadyExists(_) => StatusCode::CONFLICT,
|
||||
}
|
||||
}
|
||||
|
||||
fn add_http_headers(&self, _header_map: &mut HeaderMap<HeaderValue>) {
|
||||
// nothing
|
||||
}
|
||||
|
||||
fn http_body(&self, garage_region: &str, path: &str) -> Body {
|
||||
let error = CustomApiErrorBody {
|
||||
code: self.code().to_string(),
|
||||
message: format!("{}", self),
|
||||
path: path.to_string(),
|
||||
region: garage_region.to_string(),
|
||||
};
|
||||
Body::from(serde_json::to_string_pretty(&error).unwrap_or_else(|_| {
|
||||
r#"
|
||||
{
|
||||
"code": "InternalError",
|
||||
"message": "JSON encoding of error failed"
|
||||
}
|
||||
"#
|
||||
.into()
|
||||
}))
|
||||
}
|
||||
}
|
||||
@@ -1,264 +0,0 @@
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
|
||||
use hyper::{Body, Request, Response, StatusCode};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use garage_util::error::Error as GarageError;
|
||||
|
||||
use garage_table::*;
|
||||
|
||||
use garage_model::garage::Garage;
|
||||
use garage_model::key_table::*;
|
||||
|
||||
use crate::admin::error::*;
|
||||
use crate::helpers::parse_json_body;
|
||||
|
||||
pub async fn handle_list_keys(garage: &Arc<Garage>) -> Result<Response<Body>, Error> {
|
||||
let res = garage
|
||||
.key_table
|
||||
.get_range(
|
||||
&EmptyKey,
|
||||
None,
|
||||
Some(KeyFilter::Deleted(DeletedFilter::NotDeleted)),
|
||||
10000,
|
||||
EnumerationOrder::Forward,
|
||||
)
|
||||
.await?
|
||||
.iter()
|
||||
.map(|k| ListKeyResultItem {
|
||||
id: k.key_id.to_string(),
|
||||
name: k.params().unwrap().name.get().clone(),
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
let resp_json = serde_json::to_string_pretty(&res).map_err(GarageError::from)?;
|
||||
Ok(Response::builder()
|
||||
.status(StatusCode::OK)
|
||||
.body(Body::from(resp_json))?)
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct ListKeyResultItem {
|
||||
id: String,
|
||||
name: String,
|
||||
}
|
||||
|
||||
pub async fn handle_get_key_info(
|
||||
garage: &Arc<Garage>,
|
||||
id: Option<String>,
|
||||
search: Option<String>,
|
||||
) -> Result<Response<Body>, Error> {
|
||||
let key = if let Some(id) = id {
|
||||
garage.key_helper().get_existing_key(&id).await?
|
||||
} else if let Some(search) = search {
|
||||
garage
|
||||
.key_helper()
|
||||
.get_existing_matching_key(&search)
|
||||
.await?
|
||||
} else {
|
||||
unreachable!();
|
||||
};
|
||||
|
||||
key_info_results(garage, key).await
|
||||
}
|
||||
|
||||
pub async fn handle_create_key(
|
||||
garage: &Arc<Garage>,
|
||||
req: Request<Body>,
|
||||
) -> Result<Response<Body>, Error> {
|
||||
let req = parse_json_body::<CreateKeyRequest>(req).await?;
|
||||
|
||||
let key = Key::new(&req.name);
|
||||
garage.key_table.insert(&key).await?;
|
||||
|
||||
key_info_results(garage, key).await
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct CreateKeyRequest {
|
||||
name: String,
|
||||
}
|
||||
|
||||
pub async fn handle_import_key(
|
||||
garage: &Arc<Garage>,
|
||||
req: Request<Body>,
|
||||
) -> Result<Response<Body>, Error> {
|
||||
let req = parse_json_body::<ImportKeyRequest>(req).await?;
|
||||
|
||||
let prev_key = garage.key_table.get(&EmptyKey, &req.access_key_id).await?;
|
||||
if prev_key.is_some() {
|
||||
return Err(Error::KeyAlreadyExists(req.access_key_id.to_string()));
|
||||
}
|
||||
|
||||
let imported_key = Key::import(&req.access_key_id, &req.secret_access_key, &req.name);
|
||||
garage.key_table.insert(&imported_key).await?;
|
||||
|
||||
key_info_results(garage, imported_key).await
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct ImportKeyRequest {
|
||||
access_key_id: String,
|
||||
secret_access_key: String,
|
||||
name: String,
|
||||
}
|
||||
|
||||
pub async fn handle_update_key(
|
||||
garage: &Arc<Garage>,
|
||||
id: String,
|
||||
req: Request<Body>,
|
||||
) -> Result<Response<Body>, Error> {
|
||||
let req = parse_json_body::<UpdateKeyRequest>(req).await?;
|
||||
|
||||
let mut key = garage.key_helper().get_existing_key(&id).await?;
|
||||
|
||||
let key_state = key.state.as_option_mut().unwrap();
|
||||
|
||||
if let Some(new_name) = req.name {
|
||||
key_state.name.update(new_name);
|
||||
}
|
||||
if let Some(allow) = req.allow {
|
||||
if allow.create_bucket {
|
||||
key_state.allow_create_bucket.update(true);
|
||||
}
|
||||
}
|
||||
if let Some(deny) = req.deny {
|
||||
if deny.create_bucket {
|
||||
key_state.allow_create_bucket.update(false);
|
||||
}
|
||||
}
|
||||
|
||||
garage.key_table.insert(&key).await?;
|
||||
|
||||
key_info_results(garage, key).await
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct UpdateKeyRequest {
|
||||
name: Option<String>,
|
||||
allow: Option<KeyPerm>,
|
||||
deny: Option<KeyPerm>,
|
||||
}
|
||||
|
||||
pub async fn handle_delete_key(garage: &Arc<Garage>, id: String) -> Result<Response<Body>, Error> {
|
||||
let mut key = garage.key_helper().get_existing_key(&id).await?;
|
||||
|
||||
key.state.as_option().unwrap();
|
||||
|
||||
garage.key_helper().delete_key(&mut key).await?;
|
||||
|
||||
Ok(Response::builder()
|
||||
.status(StatusCode::NO_CONTENT)
|
||||
.body(Body::empty())?)
|
||||
}
|
||||
|
||||
async fn key_info_results(garage: &Arc<Garage>, key: Key) -> Result<Response<Body>, Error> {
|
||||
let mut relevant_buckets = HashMap::new();
|
||||
|
||||
let key_state = key.state.as_option().unwrap();
|
||||
|
||||
for id in key_state
|
||||
.authorized_buckets
|
||||
.items()
|
||||
.iter()
|
||||
.map(|(id, _)| id)
|
||||
.chain(
|
||||
key_state
|
||||
.local_aliases
|
||||
.items()
|
||||
.iter()
|
||||
.filter_map(|(_, _, v)| v.as_ref()),
|
||||
) {
|
||||
if !relevant_buckets.contains_key(id) {
|
||||
if let Some(b) = garage.bucket_table.get(&EmptyKey, id).await? {
|
||||
if b.state.as_option().is_some() {
|
||||
relevant_buckets.insert(*id, b);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let res = GetKeyInfoResult {
|
||||
name: key_state.name.get().clone(),
|
||||
access_key_id: key.key_id.clone(),
|
||||
secret_access_key: key_state.secret_key.clone(),
|
||||
permissions: KeyPerm {
|
||||
create_bucket: *key_state.allow_create_bucket.get(),
|
||||
},
|
||||
buckets: relevant_buckets
|
||||
.into_iter()
|
||||
.map(|(_, bucket)| {
|
||||
let state = bucket.state.as_option().unwrap();
|
||||
KeyInfoBucketResult {
|
||||
id: hex::encode(bucket.id),
|
||||
global_aliases: state
|
||||
.aliases
|
||||
.items()
|
||||
.iter()
|
||||
.filter(|(_, _, a)| *a)
|
||||
.map(|(n, _, _)| n.to_string())
|
||||
.collect::<Vec<_>>(),
|
||||
local_aliases: state
|
||||
.local_aliases
|
||||
.items()
|
||||
.iter()
|
||||
.filter(|((k, _), _, a)| *a && *k == key.key_id)
|
||||
.map(|((_, n), _, _)| n.to_string())
|
||||
.collect::<Vec<_>>(),
|
||||
permissions: key_state
|
||||
.authorized_buckets
|
||||
.get(&bucket.id)
|
||||
.map(|p| ApiBucketKeyPerm {
|
||||
read: p.allow_read,
|
||||
write: p.allow_write,
|
||||
owner: p.allow_owner,
|
||||
})
|
||||
.unwrap_or_default(),
|
||||
}
|
||||
})
|
||||
.collect::<Vec<_>>(),
|
||||
};
|
||||
|
||||
let resp_json = serde_json::to_string_pretty(&res).map_err(GarageError::from)?;
|
||||
Ok(Response::builder()
|
||||
.status(StatusCode::OK)
|
||||
.body(Body::from(resp_json))?)
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct GetKeyInfoResult {
|
||||
name: String,
|
||||
access_key_id: String,
|
||||
secret_access_key: String,
|
||||
permissions: KeyPerm,
|
||||
buckets: Vec<KeyInfoBucketResult>,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct KeyPerm {
|
||||
#[serde(default)]
|
||||
create_bucket: bool,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct KeyInfoBucketResult {
|
||||
id: String,
|
||||
global_aliases: Vec<String>,
|
||||
local_aliases: Vec<String>,
|
||||
permissions: ApiBucketKeyPerm,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Default)]
|
||||
pub(crate) struct ApiBucketKeyPerm {
|
||||
#[serde(default)]
|
||||
pub(crate) read: bool,
|
||||
#[serde(default)]
|
||||
pub(crate) write: bool,
|
||||
#[serde(default)]
|
||||
pub(crate) owner: bool,
|
||||
}
|
||||
@@ -1,7 +0,0 @@
|
||||
pub mod api_server;
|
||||
mod error;
|
||||
mod router;
|
||||
|
||||
mod bucket;
|
||||
mod cluster;
|
||||
mod key;
|
||||
@@ -1,149 +0,0 @@
|
||||
use std::borrow::Cow;
|
||||
|
||||
use hyper::{Method, Request};
|
||||
|
||||
use crate::admin::error::*;
|
||||
use crate::router_macros::*;
|
||||
|
||||
pub enum Authorization {
|
||||
MetricsToken,
|
||||
AdminToken,
|
||||
}
|
||||
|
||||
router_match! {@func
|
||||
|
||||
/// List of all Admin API endpoints.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum Endpoint {
|
||||
Options,
|
||||
Metrics,
|
||||
GetClusterStatus,
|
||||
ConnectClusterNodes,
|
||||
// Layout
|
||||
GetClusterLayout,
|
||||
UpdateClusterLayout,
|
||||
ApplyClusterLayout,
|
||||
RevertClusterLayout,
|
||||
// Keys
|
||||
ListKeys,
|
||||
CreateKey,
|
||||
ImportKey,
|
||||
GetKeyInfo {
|
||||
id: Option<String>,
|
||||
search: Option<String>,
|
||||
},
|
||||
DeleteKey {
|
||||
id: String,
|
||||
},
|
||||
UpdateKey {
|
||||
id: String,
|
||||
},
|
||||
// Buckets
|
||||
ListBuckets,
|
||||
CreateBucket,
|
||||
GetBucketInfo {
|
||||
id: Option<String>,
|
||||
global_alias: Option<String>,
|
||||
},
|
||||
DeleteBucket {
|
||||
id: String,
|
||||
},
|
||||
PutBucketWebsite {
|
||||
id: String,
|
||||
},
|
||||
DeleteBucketWebsite {
|
||||
id: String,
|
||||
},
|
||||
// Bucket-Key Permissions
|
||||
BucketAllowKey,
|
||||
BucketDenyKey,
|
||||
// Bucket aliases
|
||||
GlobalAliasBucket {
|
||||
id: String,
|
||||
alias: String,
|
||||
},
|
||||
GlobalUnaliasBucket {
|
||||
id: String,
|
||||
alias: String,
|
||||
},
|
||||
LocalAliasBucket {
|
||||
id: String,
|
||||
access_key_id: String,
|
||||
alias: String,
|
||||
},
|
||||
LocalUnaliasBucket {
|
||||
id: String,
|
||||
access_key_id: String,
|
||||
alias: String,
|
||||
},
|
||||
}}
|
||||
|
||||
impl Endpoint {
|
||||
/// Determine which S3 endpoint a request is for using the request, and a bucket which was
|
||||
/// possibly extracted from the Host header.
|
||||
/// Returns Self plus bucket name, if endpoint is not Endpoint::ListBuckets
|
||||
pub fn from_request<T>(req: &Request<T>) -> Result<Self, Error> {
|
||||
let uri = req.uri();
|
||||
let path = uri.path();
|
||||
let query = uri.query();
|
||||
|
||||
let mut query = QueryParameters::from_query(query.unwrap_or_default())?;
|
||||
|
||||
let res = router_match!(@gen_path_parser (req.method(), path, query) [
|
||||
OPTIONS _ => Options,
|
||||
GET "/metrics" => Metrics,
|
||||
GET "/v0/status" => GetClusterStatus,
|
||||
POST "/v0/connect" => ConnectClusterNodes,
|
||||
// Layout endpoints
|
||||
GET "/v0/layout" => GetClusterLayout,
|
||||
POST "/v0/layout" => UpdateClusterLayout,
|
||||
POST "/v0/layout/apply" => ApplyClusterLayout,
|
||||
POST "/v0/layout/revert" => RevertClusterLayout,
|
||||
// API key endpoints
|
||||
GET "/v0/key" if id => GetKeyInfo (query_opt::id, query_opt::search),
|
||||
GET "/v0/key" if search => GetKeyInfo (query_opt::id, query_opt::search),
|
||||
POST "/v0/key" if id => UpdateKey (query::id),
|
||||
POST "/v0/key" => CreateKey,
|
||||
POST "/v0/key/import" => ImportKey,
|
||||
DELETE "/v0/key" if id => DeleteKey (query::id),
|
||||
GET "/v0/key" => ListKeys,
|
||||
// Bucket endpoints
|
||||
GET "/v0/bucket" if id => GetBucketInfo (query_opt::id, query_opt::global_alias),
|
||||
GET "/v0/bucket" if global_alias => GetBucketInfo (query_opt::id, query_opt::global_alias),
|
||||
GET "/v0/bucket" => ListBuckets,
|
||||
POST "/v0/bucket" => CreateBucket,
|
||||
DELETE "/v0/bucket" if id => DeleteBucket (query::id),
|
||||
PUT "/v0/bucket/website" if id => PutBucketWebsite (query::id),
|
||||
DELETE "/v0/bucket/website" if id => DeleteBucketWebsite (query::id),
|
||||
// Bucket-key permissions
|
||||
POST "/v0/bucket/allow" => BucketAllowKey,
|
||||
POST "/v0/bucket/deny" => BucketDenyKey,
|
||||
// Bucket aliases
|
||||
PUT "/v0/bucket/alias/global" => GlobalAliasBucket (query::id, query::alias),
|
||||
DELETE "/v0/bucket/alias/global" => GlobalUnaliasBucket (query::id, query::alias),
|
||||
PUT "/v0/bucket/alias/local" => LocalAliasBucket (query::id, query::access_key_id, query::alias),
|
||||
DELETE "/v0/bucket/alias/local" => LocalUnaliasBucket (query::id, query::access_key_id, query::alias),
|
||||
]);
|
||||
|
||||
if let Some(message) = query.nonempty_message() {
|
||||
debug!("Unused query parameter: {}", message)
|
||||
}
|
||||
|
||||
Ok(res)
|
||||
}
|
||||
/// Get the kind of authorization which is required to perform the operation.
|
||||
pub fn authorization_type(&self) -> Authorization {
|
||||
match self {
|
||||
Self::Metrics => Authorization::MetricsToken,
|
||||
_ => Authorization::AdminToken,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
generateQueryParameters! {
|
||||
"id" => id,
|
||||
"search" => search,
|
||||
"globalAlias" => global_alias,
|
||||
"alias" => alias,
|
||||
"accessKeyId" => access_key_id
|
||||
}
|
||||
@@ -1,177 +0,0 @@
|
||||
use err_derive::Error;
|
||||
use hyper::StatusCode;
|
||||
|
||||
use garage_util::error::Error as GarageError;
|
||||
|
||||
/// Errors of this crate
|
||||
#[derive(Debug, Error)]
|
||||
pub enum CommonError {
|
||||
// ---- INTERNAL ERRORS ----
|
||||
/// Error related to deeper parts of Garage
|
||||
#[error(display = "Internal error: {}", _0)]
|
||||
InternalError(#[error(source)] GarageError),
|
||||
|
||||
/// Error related to Hyper
|
||||
#[error(display = "Internal error (Hyper error): {}", _0)]
|
||||
Hyper(#[error(source)] hyper::Error),
|
||||
|
||||
/// Error related to HTTP
|
||||
#[error(display = "Internal error (HTTP error): {}", _0)]
|
||||
Http(#[error(source)] http::Error),
|
||||
|
||||
// ---- GENERIC CLIENT ERRORS ----
|
||||
/// Proper authentication was not provided
|
||||
#[error(display = "Forbidden: {}", _0)]
|
||||
Forbidden(String),
|
||||
|
||||
/// Generic bad request response with custom message
|
||||
#[error(display = "Bad request: {}", _0)]
|
||||
BadRequest(String),
|
||||
|
||||
// ---- SPECIFIC ERROR CONDITIONS ----
|
||||
// These have to be error codes referenced in the S3 spec here:
|
||||
// https://docs.aws.amazon.com/AmazonS3/latest/API/ErrorResponses.html#ErrorCodeList
|
||||
/// The bucket requested don't exists
|
||||
#[error(display = "Bucket not found: {}", _0)]
|
||||
NoSuchBucket(String),
|
||||
|
||||
/// Tried to create a bucket that already exist
|
||||
#[error(display = "Bucket already exists")]
|
||||
BucketAlreadyExists,
|
||||
|
||||
/// Tried to delete a non-empty bucket
|
||||
#[error(display = "Tried to delete a non-empty bucket")]
|
||||
BucketNotEmpty,
|
||||
|
||||
// Category: bad request
|
||||
/// Bucket name is not valid according to AWS S3 specs
|
||||
#[error(display = "Invalid bucket name: {}", _0)]
|
||||
InvalidBucketName(String),
|
||||
}
|
||||
|
||||
impl CommonError {
|
||||
pub fn http_status_code(&self) -> StatusCode {
|
||||
match self {
|
||||
CommonError::InternalError(
|
||||
GarageError::Timeout
|
||||
| GarageError::RemoteError(_)
|
||||
| GarageError::Quorum(_, _, _, _),
|
||||
) => StatusCode::SERVICE_UNAVAILABLE,
|
||||
CommonError::InternalError(_) | CommonError::Hyper(_) | CommonError::Http(_) => {
|
||||
StatusCode::INTERNAL_SERVER_ERROR
|
||||
}
|
||||
CommonError::BadRequest(_) => StatusCode::BAD_REQUEST,
|
||||
CommonError::Forbidden(_) => StatusCode::FORBIDDEN,
|
||||
CommonError::NoSuchBucket(_) => StatusCode::NOT_FOUND,
|
||||
CommonError::BucketNotEmpty | CommonError::BucketAlreadyExists => StatusCode::CONFLICT,
|
||||
CommonError::InvalidBucketName(_) => StatusCode::BAD_REQUEST,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn aws_code(&self) -> &'static str {
|
||||
match self {
|
||||
CommonError::Forbidden(_) => "AccessDenied",
|
||||
CommonError::InternalError(
|
||||
GarageError::Timeout
|
||||
| GarageError::RemoteError(_)
|
||||
| GarageError::Quorum(_, _, _, _),
|
||||
) => "ServiceUnavailable",
|
||||
CommonError::InternalError(_) | CommonError::Hyper(_) | CommonError::Http(_) => {
|
||||
"InternalError"
|
||||
}
|
||||
CommonError::BadRequest(_) => "InvalidRequest",
|
||||
CommonError::NoSuchBucket(_) => "NoSuchBucket",
|
||||
CommonError::BucketAlreadyExists => "BucketAlreadyExists",
|
||||
CommonError::BucketNotEmpty => "BucketNotEmpty",
|
||||
CommonError::InvalidBucketName(_) => "InvalidBucketName",
|
||||
}
|
||||
}
|
||||
|
||||
pub fn bad_request<M: ToString>(msg: M) -> Self {
|
||||
CommonError::BadRequest(msg.to_string())
|
||||
}
|
||||
}
|
||||
|
||||
pub trait CommonErrorDerivative: From<CommonError> {
|
||||
fn internal_error<M: ToString>(msg: M) -> Self {
|
||||
Self::from(CommonError::InternalError(GarageError::Message(
|
||||
msg.to_string(),
|
||||
)))
|
||||
}
|
||||
|
||||
fn bad_request<M: ToString>(msg: M) -> Self {
|
||||
Self::from(CommonError::BadRequest(msg.to_string()))
|
||||
}
|
||||
|
||||
fn forbidden<M: ToString>(msg: M) -> Self {
|
||||
Self::from(CommonError::Forbidden(msg.to_string()))
|
||||
}
|
||||
}
|
||||
|
||||
/// Trait to map error to the Bad Request error code
|
||||
pub trait OkOrBadRequest {
|
||||
type S;
|
||||
fn ok_or_bad_request<M: AsRef<str>>(self, reason: M) -> Result<Self::S, CommonError>;
|
||||
}
|
||||
|
||||
impl<T, E> OkOrBadRequest for Result<T, E>
|
||||
where
|
||||
E: std::fmt::Display,
|
||||
{
|
||||
type S = T;
|
||||
fn ok_or_bad_request<M: AsRef<str>>(self, reason: M) -> Result<T, CommonError> {
|
||||
match self {
|
||||
Ok(x) => Ok(x),
|
||||
Err(e) => Err(CommonError::BadRequest(format!(
|
||||
"{}: {}",
|
||||
reason.as_ref(),
|
||||
e
|
||||
))),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<T> OkOrBadRequest for Option<T> {
|
||||
type S = T;
|
||||
fn ok_or_bad_request<M: AsRef<str>>(self, reason: M) -> Result<T, CommonError> {
|
||||
match self {
|
||||
Some(x) => Ok(x),
|
||||
None => Err(CommonError::BadRequest(reason.as_ref().to_string())),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Trait to map an error to an Internal Error code
|
||||
pub trait OkOrInternalError {
|
||||
type S;
|
||||
fn ok_or_internal_error<M: AsRef<str>>(self, reason: M) -> Result<Self::S, CommonError>;
|
||||
}
|
||||
|
||||
impl<T, E> OkOrInternalError for Result<T, E>
|
||||
where
|
||||
E: std::fmt::Display,
|
||||
{
|
||||
type S = T;
|
||||
fn ok_or_internal_error<M: AsRef<str>>(self, reason: M) -> Result<T, CommonError> {
|
||||
match self {
|
||||
Ok(x) => Ok(x),
|
||||
Err(e) => Err(CommonError::InternalError(GarageError::Message(format!(
|
||||
"{}: {}",
|
||||
reason.as_ref(),
|
||||
e
|
||||
)))),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<T> OkOrInternalError for Option<T> {
|
||||
type S = T;
|
||||
fn ok_or_internal_error<M: AsRef<str>>(self, reason: M) -> Result<T, CommonError> {
|
||||
match self {
|
||||
Some(x) => Ok(x),
|
||||
None => Err(CommonError::InternalError(GarageError::Message(
|
||||
reason.as_ref().to_string(),
|
||||
))),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -2,24 +2,34 @@ use std::convert::TryInto;
|
||||
|
||||
use err_derive::Error;
|
||||
use hyper::header::HeaderValue;
|
||||
use hyper::{Body, HeaderMap, StatusCode};
|
||||
use hyper::{HeaderMap, StatusCode};
|
||||
|
||||
use garage_model::helper::error::Error as HelperError;
|
||||
use garage_util::error::Error as GarageError;
|
||||
|
||||
use crate::common_error::CommonError;
|
||||
pub use crate::common_error::{CommonErrorDerivative, OkOrBadRequest, OkOrInternalError};
|
||||
use crate::generic_server::ApiError;
|
||||
use crate::s3::xml as s3_xml;
|
||||
use crate::signature::error::Error as SignatureError;
|
||||
|
||||
/// Errors of this crate
|
||||
#[derive(Debug, Error)]
|
||||
pub enum Error {
|
||||
#[error(display = "{}", _0)]
|
||||
/// Error from common error
|
||||
Common(CommonError),
|
||||
// Category: internal error
|
||||
/// Error related to deeper parts of Garage
|
||||
#[error(display = "Internal error: {}", _0)]
|
||||
InternalError(#[error(source)] GarageError),
|
||||
|
||||
/// Error related to Hyper
|
||||
#[error(display = "Internal error (Hyper error): {}", _0)]
|
||||
Hyper(#[error(source)] hyper::Error),
|
||||
|
||||
/// Error related to HTTP
|
||||
#[error(display = "Internal error (HTTP error): {}", _0)]
|
||||
Http(#[error(source)] http::Error),
|
||||
|
||||
// Category: cannot process
|
||||
/// No proper api key was used, or the signature was invalid
|
||||
#[error(display = "Forbidden: {}", _0)]
|
||||
Forbidden(String),
|
||||
|
||||
/// Authorization Header Malformed
|
||||
#[error(display = "Authorization header malformed, expected scope: {}", _0)]
|
||||
AuthorizationHeaderMalformed(String),
|
||||
@@ -28,10 +38,22 @@ pub enum Error {
|
||||
#[error(display = "Key not found")]
|
||||
NoSuchKey,
|
||||
|
||||
/// The bucket requested don't exists
|
||||
#[error(display = "Bucket not found")]
|
||||
NoSuchBucket,
|
||||
|
||||
/// The multipart upload requested don't exists
|
||||
#[error(display = "Upload not found")]
|
||||
NoSuchUpload,
|
||||
|
||||
/// Tried to create a bucket that already exist
|
||||
#[error(display = "Bucket already exists")]
|
||||
BucketAlreadyExists,
|
||||
|
||||
/// Tried to delete a non-empty bucket
|
||||
#[error(display = "Tried to delete a non-empty bucket")]
|
||||
BucketNotEmpty,
|
||||
|
||||
/// Precondition failed (e.g. x-amz-copy-source-if-match)
|
||||
#[error(display = "At least one of the preconditions you specified did not hold")]
|
||||
PreconditionFailed,
|
||||
@@ -58,6 +80,10 @@ pub enum Error {
|
||||
#[error(display = "Invalid UTF-8: {}", _0)]
|
||||
InvalidUtf8String(#[error(source)] std::string::FromUtf8Error),
|
||||
|
||||
/// Some base64 encoded data was badly encoded
|
||||
#[error(display = "Invalid base64: {}", _0)]
|
||||
InvalidBase64(#[error(source)] base64::DecodeError),
|
||||
|
||||
/// The client sent invalid XML data
|
||||
#[error(display = "Invalid XML: {}", _0)]
|
||||
InvalidXml(String),
|
||||
@@ -70,34 +96,19 @@ pub enum Error {
|
||||
#[error(display = "Invalid HTTP range: {:?}", _0)]
|
||||
InvalidRange(#[error(from)] (http_range::HttpRangeParseError, u64)),
|
||||
|
||||
/// The client sent an invalid request
|
||||
#[error(display = "Bad request: {}", _0)]
|
||||
BadRequest(String),
|
||||
|
||||
/// The client asked for an invalid return format (invalid Accept header)
|
||||
#[error(display = "Not acceptable: {}", _0)]
|
||||
NotAcceptable(String),
|
||||
|
||||
/// The client sent a request for an action not supported by garage
|
||||
#[error(display = "Unimplemented action: {}", _0)]
|
||||
NotImplemented(String),
|
||||
}
|
||||
|
||||
impl<T> From<T> for Error
|
||||
where
|
||||
CommonError: From<T>,
|
||||
{
|
||||
fn from(err: T) -> Self {
|
||||
Error::Common(CommonError::from(err))
|
||||
}
|
||||
}
|
||||
|
||||
impl CommonErrorDerivative for Error {}
|
||||
|
||||
impl From<HelperError> for Error {
|
||||
fn from(err: HelperError) -> Self {
|
||||
match err {
|
||||
HelperError::Internal(i) => Self::Common(CommonError::InternalError(i)),
|
||||
HelperError::BadRequest(b) => Self::Common(CommonError::BadRequest(b)),
|
||||
HelperError::InvalidBucketName(n) => Self::Common(CommonError::InvalidBucketName(n)),
|
||||
HelperError::NoSuchBucket(n) => Self::Common(CommonError::NoSuchBucket(n)),
|
||||
e => Self::bad_request(format!("{}", e)),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<roxmltree::Error> for Error {
|
||||
fn from(err: roxmltree::Error) -> Self {
|
||||
Self::InvalidXml(format!("{}", err))
|
||||
@@ -110,67 +121,88 @@ impl From<quick_xml::de::DeError> for Error {
|
||||
}
|
||||
}
|
||||
|
||||
impl From<SignatureError> for Error {
|
||||
fn from(err: SignatureError) -> Self {
|
||||
impl From<HelperError> for Error {
|
||||
fn from(err: HelperError) -> Self {
|
||||
match err {
|
||||
SignatureError::Common(c) => Self::Common(c),
|
||||
SignatureError::AuthorizationHeaderMalformed(c) => {
|
||||
Self::AuthorizationHeaderMalformed(c)
|
||||
}
|
||||
SignatureError::InvalidUtf8Str(i) => Self::InvalidUtf8Str(i),
|
||||
SignatureError::InvalidHeader(h) => Self::InvalidHeader(h),
|
||||
HelperError::Internal(i) => Self::InternalError(i),
|
||||
HelperError::BadRequest(b) => Self::BadRequest(b),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<multer::Error> for Error {
|
||||
fn from(err: multer::Error) -> Self {
|
||||
Self::bad_request(err)
|
||||
Self::BadRequest(err.to_string())
|
||||
}
|
||||
}
|
||||
|
||||
impl Error {
|
||||
/// Get the HTTP status code that best represents the meaning of the error for the client
|
||||
pub fn http_status_code(&self) -> StatusCode {
|
||||
match self {
|
||||
Error::NoSuchKey | Error::NoSuchBucket | Error::NoSuchUpload => StatusCode::NOT_FOUND,
|
||||
Error::BucketNotEmpty | Error::BucketAlreadyExists => StatusCode::CONFLICT,
|
||||
Error::PreconditionFailed => StatusCode::PRECONDITION_FAILED,
|
||||
Error::Forbidden(_) => StatusCode::FORBIDDEN,
|
||||
Error::NotAcceptable(_) => StatusCode::NOT_ACCEPTABLE,
|
||||
Error::InternalError(
|
||||
GarageError::Timeout
|
||||
| GarageError::RemoteError(_)
|
||||
| GarageError::Quorum(_, _, _, _),
|
||||
) => StatusCode::SERVICE_UNAVAILABLE,
|
||||
Error::InternalError(_) | Error::Hyper(_) | Error::Http(_) => {
|
||||
StatusCode::INTERNAL_SERVER_ERROR
|
||||
}
|
||||
Error::InvalidRange(_) => StatusCode::RANGE_NOT_SATISFIABLE,
|
||||
Error::NotImplemented(_) => StatusCode::NOT_IMPLEMENTED,
|
||||
_ => StatusCode::BAD_REQUEST,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn aws_code(&self) -> &'static str {
|
||||
match self {
|
||||
Error::Common(c) => c.aws_code(),
|
||||
Error::NoSuchKey => "NoSuchKey",
|
||||
Error::NoSuchBucket => "NoSuchBucket",
|
||||
Error::NoSuchUpload => "NoSuchUpload",
|
||||
Error::BucketAlreadyExists => "BucketAlreadyExists",
|
||||
Error::BucketNotEmpty => "BucketNotEmpty",
|
||||
Error::PreconditionFailed => "PreconditionFailed",
|
||||
Error::InvalidPart => "InvalidPart",
|
||||
Error::InvalidPartOrder => "InvalidPartOrder",
|
||||
Error::EntityTooSmall => "EntityTooSmall",
|
||||
Error::Forbidden(_) => "AccessDenied",
|
||||
Error::AuthorizationHeaderMalformed(_) => "AuthorizationHeaderMalformed",
|
||||
Error::NotImplemented(_) => "NotImplemented",
|
||||
Error::InvalidXml(_) => "MalformedXML",
|
||||
Error::InvalidRange(_) => "InvalidRange",
|
||||
Error::InvalidUtf8Str(_) | Error::InvalidUtf8String(_) | Error::InvalidHeader(_) => {
|
||||
"InvalidRequest"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl ApiError for Error {
|
||||
/// Get the HTTP status code that best represents the meaning of the error for the client
|
||||
fn http_status_code(&self) -> StatusCode {
|
||||
match self {
|
||||
Error::Common(c) => c.http_status_code(),
|
||||
Error::NoSuchKey | Error::NoSuchUpload => StatusCode::NOT_FOUND,
|
||||
Error::PreconditionFailed => StatusCode::PRECONDITION_FAILED,
|
||||
Error::InvalidRange(_) => StatusCode::RANGE_NOT_SATISFIABLE,
|
||||
Error::NotImplemented(_) => StatusCode::NOT_IMPLEMENTED,
|
||||
Error::AuthorizationHeaderMalformed(_)
|
||||
| Error::InvalidPart
|
||||
| Error::InvalidPartOrder
|
||||
| Error::EntityTooSmall
|
||||
| Error::InvalidXml(_)
|
||||
| Error::InvalidUtf8Str(_)
|
||||
| Error::InvalidUtf8String(_)
|
||||
| Error::InvalidHeader(_) => StatusCode::BAD_REQUEST,
|
||||
Error::InternalError(
|
||||
GarageError::Timeout
|
||||
| GarageError::RemoteError(_)
|
||||
| GarageError::Quorum(_, _, _, _),
|
||||
) => "ServiceUnavailable",
|
||||
Error::InternalError(_) | Error::Hyper(_) | Error::Http(_) => "InternalError",
|
||||
_ => "InvalidRequest",
|
||||
}
|
||||
}
|
||||
|
||||
fn add_http_headers(&self, header_map: &mut HeaderMap<HeaderValue>) {
|
||||
pub fn aws_xml(&self, garage_region: &str, path: &str) -> String {
|
||||
let error = s3_xml::Error {
|
||||
code: s3_xml::Value(self.aws_code().to_string()),
|
||||
message: s3_xml::Value(format!("{}", self)),
|
||||
resource: Some(s3_xml::Value(path.to_string())),
|
||||
region: Some(s3_xml::Value(garage_region.to_string())),
|
||||
};
|
||||
s3_xml::to_xml_with_header(&error).unwrap_or_else(|_| {
|
||||
r#"
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<Error>
|
||||
<Code>InternalError</Code>
|
||||
<Message>XML encoding of error failed</Message>
|
||||
</Error>
|
||||
"#
|
||||
.into()
|
||||
})
|
||||
}
|
||||
|
||||
pub fn add_headers(&self, header_map: &mut HeaderMap<HeaderValue>) {
|
||||
use hyper::header;
|
||||
#[allow(clippy::single_match)]
|
||||
match self {
|
||||
@@ -185,23 +217,68 @@ impl ApiError for Error {
|
||||
_ => (),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn http_body(&self, garage_region: &str, path: &str) -> Body {
|
||||
let error = s3_xml::Error {
|
||||
code: s3_xml::Value(self.aws_code().to_string()),
|
||||
message: s3_xml::Value(format!("{}", self)),
|
||||
resource: Some(s3_xml::Value(path.to_string())),
|
||||
region: Some(s3_xml::Value(garage_region.to_string())),
|
||||
};
|
||||
Body::from(s3_xml::to_xml_with_header(&error).unwrap_or_else(|_| {
|
||||
r#"
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<Error>
|
||||
<Code>InternalError</Code>
|
||||
<Message>XML encoding of error failed</Message>
|
||||
</Error>
|
||||
"#
|
||||
.into()
|
||||
}))
|
||||
/// Trait to map error to the Bad Request error code
|
||||
pub trait OkOrBadRequest {
|
||||
type S;
|
||||
fn ok_or_bad_request<M: AsRef<str>>(self, reason: M) -> Result<Self::S, Error>;
|
||||
}
|
||||
|
||||
impl<T, E> OkOrBadRequest for Result<T, E>
|
||||
where
|
||||
E: std::fmt::Display,
|
||||
{
|
||||
type S = T;
|
||||
fn ok_or_bad_request<M: AsRef<str>>(self, reason: M) -> Result<T, Error> {
|
||||
match self {
|
||||
Ok(x) => Ok(x),
|
||||
Err(e) => Err(Error::BadRequest(format!("{}: {}", reason.as_ref(), e))),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<T> OkOrBadRequest for Option<T> {
|
||||
type S = T;
|
||||
fn ok_or_bad_request<M: AsRef<str>>(self, reason: M) -> Result<T, Error> {
|
||||
match self {
|
||||
Some(x) => Ok(x),
|
||||
None => Err(Error::BadRequest(reason.as_ref().to_string())),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Trait to map an error to an Internal Error code
|
||||
pub trait OkOrInternalError {
|
||||
type S;
|
||||
fn ok_or_internal_error<M: AsRef<str>>(self, reason: M) -> Result<Self::S, Error>;
|
||||
}
|
||||
|
||||
impl<T, E> OkOrInternalError for Result<T, E>
|
||||
where
|
||||
E: std::fmt::Display,
|
||||
{
|
||||
type S = T;
|
||||
fn ok_or_internal_error<M: AsRef<str>>(self, reason: M) -> Result<T, Error> {
|
||||
match self {
|
||||
Ok(x) => Ok(x),
|
||||
Err(e) => Err(Error::InternalError(GarageError::Message(format!(
|
||||
"{}: {}",
|
||||
reason.as_ref(),
|
||||
e
|
||||
)))),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<T> OkOrInternalError for Option<T> {
|
||||
type S = T;
|
||||
fn ok_or_internal_error<M: AsRef<str>>(self, reason: M) -> Result<T, Error> {
|
||||
match self {
|
||||
Some(x) => Ok(x),
|
||||
None => Err(Error::InternalError(GarageError::Message(
|
||||
reason.as_ref().to_string(),
|
||||
))),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -5,11 +5,9 @@ use async_trait::async_trait;
|
||||
|
||||
use futures::future::Future;
|
||||
|
||||
use hyper::header::HeaderValue;
|
||||
use hyper::server::conn::AddrStream;
|
||||
use hyper::service::{make_service_fn, service_fn};
|
||||
use hyper::{Body, Request, Response, Server};
|
||||
use hyper::{HeaderMap, StatusCode};
|
||||
|
||||
use opentelemetry::{
|
||||
global,
|
||||
@@ -21,31 +19,26 @@ use opentelemetry::{
|
||||
use garage_util::error::Error as GarageError;
|
||||
use garage_util::metrics::{gen_trace_id, RecordDuration};
|
||||
|
||||
use crate::error::*;
|
||||
|
||||
pub(crate) trait ApiEndpoint: Send + Sync + 'static {
|
||||
fn name(&self) -> &'static str;
|
||||
fn add_span_attributes(&self, span: SpanRef<'_>);
|
||||
}
|
||||
|
||||
pub trait ApiError: std::error::Error + Send + Sync + 'static {
|
||||
fn http_status_code(&self) -> StatusCode;
|
||||
fn add_http_headers(&self, header_map: &mut HeaderMap<HeaderValue>);
|
||||
fn http_body(&self, garage_region: &str, path: &str) -> Body;
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
pub(crate) trait ApiHandler: Send + Sync + 'static {
|
||||
const API_NAME: &'static str;
|
||||
const API_NAME_DISPLAY: &'static str;
|
||||
|
||||
type Endpoint: ApiEndpoint;
|
||||
type Error: ApiError;
|
||||
|
||||
fn parse_endpoint(&self, r: &Request<Body>) -> Result<Self::Endpoint, Self::Error>;
|
||||
fn parse_endpoint(&self, r: &Request<Body>) -> Result<Self::Endpoint, Error>;
|
||||
async fn handle(
|
||||
&self,
|
||||
req: Request<Body>,
|
||||
endpoint: Self::Endpoint,
|
||||
) -> Result<Response<Body>, Self::Error>;
|
||||
) -> Result<Response<Body>, Error>;
|
||||
}
|
||||
|
||||
pub(crate) struct ApiServer<A: ApiHandler> {
|
||||
@@ -149,13 +142,13 @@ impl<A: ApiHandler> ApiServer<A> {
|
||||
Ok(x)
|
||||
}
|
||||
Err(e) => {
|
||||
let body: Body = e.http_body(&self.region, uri.path());
|
||||
let body: Body = Body::from(e.aws_xml(&self.region, uri.path()));
|
||||
let mut http_error_builder = Response::builder()
|
||||
.status(e.http_status_code())
|
||||
.header("Content-Type", "application/xml");
|
||||
|
||||
if let Some(header_map) = http_error_builder.headers_mut() {
|
||||
e.add_http_headers(header_map)
|
||||
e.add_headers(header_map)
|
||||
}
|
||||
|
||||
let http_error = http_error_builder.body(body)?;
|
||||
@@ -170,7 +163,7 @@ impl<A: ApiHandler> ApiServer<A> {
|
||||
}
|
||||
}
|
||||
|
||||
async fn handler_stage2(&self, req: Request<Body>) -> Result<Response<Body>, A::Error> {
|
||||
async fn handler_stage2(&self, req: Request<Body>) -> Result<Response<Body>, Error> {
|
||||
let endpoint = self.api_handler.parse_endpoint(&req)?;
|
||||
debug!("Endpoint: {}", endpoint.name());
|
||||
|
||||
|
||||
+32
-21
@@ -1,8 +1,11 @@
|
||||
use hyper::{Body, Request};
|
||||
use idna::domain_to_unicode;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::common_error::{CommonError as Error, *};
|
||||
use garage_util::data::*;
|
||||
|
||||
use garage_model::garage::Garage;
|
||||
use garage_model::key_table::Key;
|
||||
|
||||
use crate::error::*;
|
||||
|
||||
/// What kind of authorization is required to perform a given action
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
@@ -47,7 +50,7 @@ pub fn authority_to_host(authority: &str) -> Result<String, Error> {
|
||||
let mut iter = authority.chars().enumerate();
|
||||
let (_, first_char) = iter
|
||||
.next()
|
||||
.ok_or_else(|| Error::bad_request("Authority is empty".to_string()))?;
|
||||
.ok_or_else(|| Error::BadRequest("Authority is empty".to_string()))?;
|
||||
|
||||
let split = match first_char {
|
||||
'[' => {
|
||||
@@ -55,7 +58,7 @@ pub fn authority_to_host(authority: &str) -> Result<String, Error> {
|
||||
match iter.next() {
|
||||
Some((_, ']')) => iter.next(),
|
||||
_ => {
|
||||
return Err(Error::bad_request(format!(
|
||||
return Err(Error::BadRequest(format!(
|
||||
"Authority {} has an illegal format",
|
||||
authority
|
||||
)))
|
||||
@@ -68,7 +71,7 @@ pub fn authority_to_host(authority: &str) -> Result<String, Error> {
|
||||
let authority = match split {
|
||||
Some((i, ':')) => Ok(&authority[..i]),
|
||||
None => Ok(authority),
|
||||
Some((_, _)) => Err(Error::bad_request(format!(
|
||||
Some((_, _)) => Err(Error::BadRequest(format!(
|
||||
"Authority {} has an illegal format",
|
||||
authority
|
||||
))),
|
||||
@@ -76,6 +79,28 @@ pub fn authority_to_host(authority: &str) -> Result<String, Error> {
|
||||
authority.map(|h| domain_to_unicode(h).0)
|
||||
}
|
||||
|
||||
#[allow(clippy::ptr_arg)]
|
||||
pub async fn resolve_bucket(
|
||||
garage: &Garage,
|
||||
bucket_name: &String,
|
||||
api_key: &Key,
|
||||
) -> Result<Uuid, Error> {
|
||||
let api_key_params = api_key
|
||||
.state
|
||||
.as_option()
|
||||
.ok_or_internal_error("Key should not be deleted at this point")?;
|
||||
|
||||
if let Some(Some(bucket_id)) = api_key_params.local_aliases.get(bucket_name) {
|
||||
Ok(*bucket_id)
|
||||
} else {
|
||||
Ok(garage
|
||||
.bucket_helper()
|
||||
.resolve_global_bucket_name(bucket_name)
|
||||
.await?
|
||||
.ok_or(Error::NoSuchBucket)?)
|
||||
}
|
||||
}
|
||||
|
||||
/// Extract the bucket name and the key name from an HTTP path and possibly a bucket provided in
|
||||
/// the host header of the request
|
||||
///
|
||||
@@ -107,7 +132,7 @@ pub fn parse_bucket_key<'a>(
|
||||
None => (path, None),
|
||||
};
|
||||
if bucket.is_empty() {
|
||||
return Err(Error::bad_request("No bucket specified"));
|
||||
return Err(Error::BadRequest("No bucket specified".to_string()));
|
||||
}
|
||||
Ok((bucket, key))
|
||||
}
|
||||
@@ -138,12 +163,6 @@ pub fn key_after_prefix(pfx: &str) -> Option<String> {
|
||||
None
|
||||
}
|
||||
|
||||
pub async fn parse_json_body<T: for<'de> Deserialize<'de>>(req: Request<Body>) -> Result<T, Error> {
|
||||
let body = hyper::body::to_bytes(req.into_body()).await?;
|
||||
let resp: T = serde_json::from_slice(&body).ok_or_bad_request("Invalid JSON")?;
|
||||
Ok(resp)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
@@ -279,11 +298,3 @@ mod tests {
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
pub(crate) struct CustomApiErrorBody {
|
||||
pub(crate) code: String,
|
||||
pub(crate) message: String,
|
||||
pub(crate) region: String,
|
||||
pub(crate) path: String,
|
||||
}
|
||||
|
||||
+16
-17
@@ -7,12 +7,13 @@ use hyper::{Body, Method, Request, Response};
|
||||
|
||||
use opentelemetry::{trace::SpanRef, KeyValue};
|
||||
|
||||
use garage_table::util::*;
|
||||
use garage_util::error::Error as GarageError;
|
||||
|
||||
use garage_model::garage::Garage;
|
||||
|
||||
use crate::error::*;
|
||||
use crate::generic_server::*;
|
||||
use crate::k2v::error::*;
|
||||
|
||||
use crate::signature::payload::check_payload_signature;
|
||||
use crate::signature::streaming::*;
|
||||
@@ -59,7 +60,6 @@ impl ApiHandler for K2VApiServer {
|
||||
const API_NAME_DISPLAY: &'static str = "K2V";
|
||||
|
||||
type Endpoint = K2VApiEndpoint;
|
||||
type Error = Error;
|
||||
|
||||
fn parse_endpoint(&self, req: &Request<Body>) -> Result<K2VApiEndpoint, Error> {
|
||||
let (endpoint, bucket_name) = Endpoint::from_request(req)?;
|
||||
@@ -83,14 +83,13 @@ impl ApiHandler for K2VApiServer {
|
||||
|
||||
// The OPTIONS method is procesed early, before we even check for an API key
|
||||
if let Endpoint::Options = endpoint {
|
||||
return Ok(handle_options_s3api(garage, &req, Some(bucket_name))
|
||||
.await
|
||||
.ok_or_bad_request("Error handling OPTIONS")?);
|
||||
return handle_options_s3api(garage, &req, Some(bucket_name)).await;
|
||||
}
|
||||
|
||||
let (api_key, mut content_sha256) = check_payload_signature(&garage, "k2v", &req).await?;
|
||||
let api_key = api_key
|
||||
.ok_or_else(|| Error::forbidden("Garage does not support anonymous access yet"))?;
|
||||
let api_key = api_key.ok_or_else(|| {
|
||||
Error::Forbidden("Garage does not support anonymous access yet".to_string())
|
||||
})?;
|
||||
|
||||
let req = parse_streaming_body(
|
||||
&api_key,
|
||||
@@ -100,14 +99,13 @@ impl ApiHandler for K2VApiServer {
|
||||
"k2v",
|
||||
)?;
|
||||
|
||||
let bucket_id = garage
|
||||
.bucket_helper()
|
||||
.resolve_bucket(&bucket_name, &api_key)
|
||||
.await?;
|
||||
let bucket_id = resolve_bucket(&garage, &bucket_name, &api_key).await?;
|
||||
let bucket = garage
|
||||
.bucket_helper()
|
||||
.get_existing_bucket(bucket_id)
|
||||
.await?;
|
||||
.bucket_table
|
||||
.get(&EmptyKey, &bucket_id)
|
||||
.await?
|
||||
.filter(|b| !b.state.is_deleted())
|
||||
.ok_or(Error::NoSuchBucket)?;
|
||||
|
||||
let allowed = match endpoint.authorization_type() {
|
||||
Authorization::Read => api_key.allow_read(&bucket_id),
|
||||
@@ -117,7 +115,9 @@ impl ApiHandler for K2VApiServer {
|
||||
};
|
||||
|
||||
if !allowed {
|
||||
return Err(Error::forbidden("Operation is not allowed for this key."));
|
||||
return Err(Error::Forbidden(
|
||||
"Operation is not allowed for this key.".to_string(),
|
||||
));
|
||||
}
|
||||
|
||||
// Look up what CORS rule might apply to response.
|
||||
@@ -125,8 +125,7 @@ impl ApiHandler for K2VApiServer {
|
||||
// are always preflighted, i.e. the browser should make
|
||||
// an OPTIONS call before to check it is allowed
|
||||
let matching_cors_rule = match *req.method() {
|
||||
Method::GET | Method::HEAD | Method::POST => find_matching_cors_rule(&bucket, &req)
|
||||
.ok_or_internal_error("Error looking up CORS rule")?,
|
||||
Method::GET | Method::HEAD | Method::POST => find_matching_cors_rule(&bucket, &req)?,
|
||||
_ => None,
|
||||
};
|
||||
|
||||
|
||||
+12
-7
@@ -12,8 +12,7 @@ use garage_model::garage::Garage;
|
||||
use garage_model::k2v::causality::*;
|
||||
use garage_model::k2v::item_table::*;
|
||||
|
||||
use crate::helpers::*;
|
||||
use crate::k2v::error::*;
|
||||
use crate::error::*;
|
||||
use crate::k2v::range::read_range;
|
||||
|
||||
pub async fn handle_insert_batch(
|
||||
@@ -21,7 +20,9 @@ pub async fn handle_insert_batch(
|
||||
bucket_id: Uuid,
|
||||
req: Request<Body>,
|
||||
) -> Result<Response<Body>, Error> {
|
||||
let items = parse_json_body::<Vec<InsertBatchItem>>(req).await?;
|
||||
let body = hyper::body::to_bytes(req.into_body()).await?;
|
||||
let items: Vec<InsertBatchItem> =
|
||||
serde_json::from_slice(&body).ok_or_bad_request("Invalid JSON")?;
|
||||
|
||||
let mut items2 = vec![];
|
||||
for it in items {
|
||||
@@ -51,7 +52,9 @@ pub async fn handle_read_batch(
|
||||
bucket_id: Uuid,
|
||||
req: Request<Body>,
|
||||
) -> Result<Response<Body>, Error> {
|
||||
let queries = parse_json_body::<Vec<ReadBatchQuery>>(req).await?;
|
||||
let body = hyper::body::to_bytes(req.into_body()).await?;
|
||||
let queries: Vec<ReadBatchQuery> =
|
||||
serde_json::from_slice(&body).ok_or_bad_request("Invalid JSON")?;
|
||||
|
||||
let resp_results = futures::future::join_all(
|
||||
queries
|
||||
@@ -88,7 +91,7 @@ async fn handle_read_batch_query(
|
||||
|
||||
let (items, more, next_start) = if query.single_item {
|
||||
if query.prefix.is_some() || query.end.is_some() || query.limit.is_some() || query.reverse {
|
||||
return Err(Error::bad_request("Batch query parameters 'prefix', 'end', 'limit' and 'reverse' must not be set when singleItem is true."));
|
||||
return Err(Error::BadRequest("Batch query parameters 'prefix', 'end', 'limit' and 'reverse' must not be set when singleItem is true.".into()));
|
||||
}
|
||||
let sk = query
|
||||
.start
|
||||
@@ -146,7 +149,9 @@ pub async fn handle_delete_batch(
|
||||
bucket_id: Uuid,
|
||||
req: Request<Body>,
|
||||
) -> Result<Response<Body>, Error> {
|
||||
let queries = parse_json_body::<Vec<DeleteBatchQuery>>(req).await?;
|
||||
let body = hyper::body::to_bytes(req.into_body()).await?;
|
||||
let queries: Vec<DeleteBatchQuery> =
|
||||
serde_json::from_slice(&body).ok_or_bad_request("Invalid JSON")?;
|
||||
|
||||
let resp_results = futures::future::join_all(
|
||||
queries
|
||||
@@ -183,7 +188,7 @@ async fn handle_delete_batch_query(
|
||||
|
||||
let deleted_items = if query.single_item {
|
||||
if query.prefix.is_some() || query.end.is_some() {
|
||||
return Err(Error::bad_request("Batch query parameters 'prefix' and 'end' must not be set when singleItem is true."));
|
||||
return Err(Error::BadRequest("Batch query parameters 'prefix' and 'end' must not be set when singleItem is true.".into()));
|
||||
}
|
||||
let sk = query
|
||||
.start
|
||||
|
||||
@@ -1,134 +0,0 @@
|
||||
use err_derive::Error;
|
||||
use hyper::header::HeaderValue;
|
||||
use hyper::{Body, HeaderMap, StatusCode};
|
||||
|
||||
use garage_model::helper::error::Error as HelperError;
|
||||
|
||||
use crate::common_error::CommonError;
|
||||
pub use crate::common_error::{CommonErrorDerivative, OkOrBadRequest, OkOrInternalError};
|
||||
use crate::generic_server::ApiError;
|
||||
use crate::helpers::CustomApiErrorBody;
|
||||
use crate::signature::error::Error as SignatureError;
|
||||
|
||||
/// Errors of this crate
|
||||
#[derive(Debug, Error)]
|
||||
pub enum Error {
|
||||
#[error(display = "{}", _0)]
|
||||
/// Error from common error
|
||||
Common(CommonError),
|
||||
|
||||
// Category: cannot process
|
||||
/// Authorization Header Malformed
|
||||
#[error(display = "Authorization header malformed, expected scope: {}", _0)]
|
||||
AuthorizationHeaderMalformed(String),
|
||||
|
||||
/// The object requested don't exists
|
||||
#[error(display = "Key not found")]
|
||||
NoSuchKey,
|
||||
|
||||
/// Some base64 encoded data was badly encoded
|
||||
#[error(display = "Invalid base64: {}", _0)]
|
||||
InvalidBase64(#[error(source)] base64::DecodeError),
|
||||
|
||||
/// The client sent a header with invalid value
|
||||
#[error(display = "Invalid header value: {}", _0)]
|
||||
InvalidHeader(#[error(source)] hyper::header::ToStrError),
|
||||
|
||||
/// The client asked for an invalid return format (invalid Accept header)
|
||||
#[error(display = "Not acceptable: {}", _0)]
|
||||
NotAcceptable(String),
|
||||
|
||||
/// The request contained an invalid UTF-8 sequence in its path or in other parameters
|
||||
#[error(display = "Invalid UTF-8: {}", _0)]
|
||||
InvalidUtf8Str(#[error(source)] std::str::Utf8Error),
|
||||
}
|
||||
|
||||
impl<T> From<T> for Error
|
||||
where
|
||||
CommonError: From<T>,
|
||||
{
|
||||
fn from(err: T) -> Self {
|
||||
Error::Common(CommonError::from(err))
|
||||
}
|
||||
}
|
||||
|
||||
impl CommonErrorDerivative for Error {}
|
||||
|
||||
impl From<HelperError> for Error {
|
||||
fn from(err: HelperError) -> Self {
|
||||
match err {
|
||||
HelperError::Internal(i) => Self::Common(CommonError::InternalError(i)),
|
||||
HelperError::BadRequest(b) => Self::Common(CommonError::BadRequest(b)),
|
||||
HelperError::InvalidBucketName(n) => Self::Common(CommonError::InvalidBucketName(n)),
|
||||
HelperError::NoSuchBucket(n) => Self::Common(CommonError::NoSuchBucket(n)),
|
||||
e => Self::Common(CommonError::BadRequest(format!("{}", e))),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<SignatureError> for Error {
|
||||
fn from(err: SignatureError) -> Self {
|
||||
match err {
|
||||
SignatureError::Common(c) => Self::Common(c),
|
||||
SignatureError::AuthorizationHeaderMalformed(c) => {
|
||||
Self::AuthorizationHeaderMalformed(c)
|
||||
}
|
||||
SignatureError::InvalidUtf8Str(i) => Self::InvalidUtf8Str(i),
|
||||
SignatureError::InvalidHeader(h) => Self::InvalidHeader(h),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Error {
|
||||
/// This returns a keyword for the corresponding error.
|
||||
/// Here, these keywords are not necessarily those from AWS S3,
|
||||
/// as we are building a custom API
|
||||
fn code(&self) -> &'static str {
|
||||
match self {
|
||||
Error::Common(c) => c.aws_code(),
|
||||
Error::NoSuchKey => "NoSuchKey",
|
||||
Error::NotAcceptable(_) => "NotAcceptable",
|
||||
Error::AuthorizationHeaderMalformed(_) => "AuthorizationHeaderMalformed",
|
||||
Error::InvalidBase64(_) => "InvalidBase64",
|
||||
Error::InvalidHeader(_) => "InvalidHeaderValue",
|
||||
Error::InvalidUtf8Str(_) => "InvalidUtf8String",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl ApiError for Error {
|
||||
/// Get the HTTP status code that best represents the meaning of the error for the client
|
||||
fn http_status_code(&self) -> StatusCode {
|
||||
match self {
|
||||
Error::Common(c) => c.http_status_code(),
|
||||
Error::NoSuchKey => StatusCode::NOT_FOUND,
|
||||
Error::NotAcceptable(_) => StatusCode::NOT_ACCEPTABLE,
|
||||
Error::AuthorizationHeaderMalformed(_)
|
||||
| Error::InvalidBase64(_)
|
||||
| Error::InvalidHeader(_)
|
||||
| Error::InvalidUtf8Str(_) => StatusCode::BAD_REQUEST,
|
||||
}
|
||||
}
|
||||
|
||||
fn add_http_headers(&self, _header_map: &mut HeaderMap<HeaderValue>) {
|
||||
// nothing
|
||||
}
|
||||
|
||||
fn http_body(&self, garage_region: &str, path: &str) -> Body {
|
||||
let error = CustomApiErrorBody {
|
||||
code: self.code().to_string(),
|
||||
message: format!("{}", self),
|
||||
path: path.to_string(),
|
||||
region: garage_region.to_string(),
|
||||
};
|
||||
Body::from(serde_json::to_string_pretty(&error).unwrap_or_else(|_| {
|
||||
r#"
|
||||
{
|
||||
"code": "InternalError",
|
||||
"message": "JSON encoding of error failed"
|
||||
}
|
||||
"#
|
||||
.into()
|
||||
}))
|
||||
}
|
||||
}
|
||||
@@ -12,7 +12,7 @@ use garage_table::util::*;
|
||||
use garage_model::garage::Garage;
|
||||
use garage_model::k2v::counter_table::{BYTES, CONFLICTS, ENTRIES, VALUES};
|
||||
|
||||
use crate::k2v::error::*;
|
||||
use crate::error::*;
|
||||
use crate::k2v::range::read_range;
|
||||
|
||||
pub async fn handle_read_index(
|
||||
|
||||
+1
-1
@@ -10,7 +10,7 @@ use garage_model::garage::Garage;
|
||||
use garage_model::k2v::causality::*;
|
||||
use garage_model::k2v::item_table::*;
|
||||
|
||||
use crate::k2v::error::*;
|
||||
use crate::error::*;
|
||||
|
||||
pub const X_GARAGE_CAUSALITY_TOKEN: &str = "X-Garage-Causality-Token";
|
||||
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
pub mod api_server;
|
||||
mod error;
|
||||
mod router;
|
||||
|
||||
mod batch;
|
||||
|
||||
@@ -7,8 +7,8 @@ use std::sync::Arc;
|
||||
use garage_table::replication::TableShardedReplication;
|
||||
use garage_table::*;
|
||||
|
||||
use crate::error::*;
|
||||
use crate::helpers::key_after_prefix;
|
||||
use crate::k2v::error::*;
|
||||
|
||||
/// Read range in a Garage table.
|
||||
/// Returns (entries, more?, nextStart)
|
||||
@@ -31,7 +31,7 @@ where
|
||||
(None, Some(s)) => (Some(s.clone()), false),
|
||||
(Some(p), Some(s)) => {
|
||||
if !s.starts_with(p) {
|
||||
return Err(Error::bad_request(format!(
|
||||
return Err(Error::BadRequest(format!(
|
||||
"Start key '{}' does not start with prefix '{}'",
|
||||
s, p
|
||||
)));
|
||||
@@ -74,11 +74,7 @@ where
|
||||
}
|
||||
}
|
||||
if let Some(e) = end {
|
||||
let is_finished = match enumeration_order {
|
||||
EnumerationOrder::Forward => entry.sort_key() >= e,
|
||||
EnumerationOrder::Reverse => entry.sort_key() <= e,
|
||||
};
|
||||
if is_finished {
|
||||
if entry.sort_key() == e {
|
||||
return Ok((entries, false, None));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
use crate::k2v::error::*;
|
||||
use crate::error::*;
|
||||
|
||||
use std::borrow::Cow;
|
||||
|
||||
@@ -62,7 +62,7 @@ impl Endpoint {
|
||||
.unwrap_or((path.to_owned(), ""));
|
||||
|
||||
if bucket.is_empty() {
|
||||
return Err(Error::bad_request("Missing bucket name"));
|
||||
return Err(Error::BadRequest("Missing bucket name".to_owned()));
|
||||
}
|
||||
|
||||
if *req.method() == Method::OPTIONS {
|
||||
@@ -83,7 +83,7 @@ impl Endpoint {
|
||||
Method::PUT => Self::from_put(partition_key, &mut query)?,
|
||||
Method::DELETE => Self::from_delete(partition_key, &mut query)?,
|
||||
_ if req.method() == method_search => Self::from_search(partition_key, &mut query)?,
|
||||
_ => return Err(Error::bad_request("Unknown method")),
|
||||
_ => return Err(Error::BadRequest("Unknown method".to_owned())),
|
||||
};
|
||||
|
||||
if let Some(message) = query.nonempty_message() {
|
||||
|
||||
+3
-3
@@ -2,16 +2,16 @@
|
||||
#[macro_use]
|
||||
extern crate tracing;
|
||||
|
||||
pub mod common_error;
|
||||
pub mod error;
|
||||
pub use error::Error;
|
||||
|
||||
mod encoding;
|
||||
pub mod generic_server;
|
||||
mod generic_server;
|
||||
pub mod helpers;
|
||||
mod router_macros;
|
||||
/// This mode is public only to help testing. Don't expect stability here
|
||||
pub mod signature;
|
||||
|
||||
pub mod admin;
|
||||
#[cfg(feature = "k2v")]
|
||||
pub mod k2v;
|
||||
pub mod s3;
|
||||
|
||||
@@ -23,29 +23,6 @@ macro_rules! router_match {
|
||||
_ => None
|
||||
}
|
||||
}};
|
||||
(@gen_path_parser ($method:expr, $reqpath:expr, $query:expr)
|
||||
[
|
||||
$($meth:ident $path:pat $(if $required:ident)? => $api:ident $(($($conv:ident :: $param:ident),*))?,)*
|
||||
]) => {{
|
||||
{
|
||||
use Endpoint::*;
|
||||
match ($method, $reqpath) {
|
||||
$(
|
||||
(&Method::$meth, $path) if true $(&& $query.$required.is_some())? => $api {
|
||||
$($(
|
||||
$param: router_match!(@@parse_param $query, $conv, $param),
|
||||
)*)?
|
||||
},
|
||||
)*
|
||||
(m, p) => {
|
||||
return Err(Error::bad_request(format!(
|
||||
"Unknown API endpoint: {} {}",
|
||||
m, p
|
||||
)))
|
||||
}
|
||||
}
|
||||
}
|
||||
}};
|
||||
(@gen_parser ($keyword:expr, $key:ident, $query:expr, $header:expr),
|
||||
key: [$($kw_k:ident $(if $required_k:ident)? $(header $header_k:expr)? => $api_k:ident $(($($conv_k:ident :: $param_k:ident),*))?,)*],
|
||||
no_key: [$($kw_nk:ident $(if $required_nk:ident)? $(if_header $header_nk:expr)? => $api_nk:ident $(($($conv_nk:ident :: $param_nk:ident),*))?,)*]) => {{
|
||||
@@ -78,7 +55,7 @@ macro_rules! router_match {
|
||||
)*)?
|
||||
}),
|
||||
)*
|
||||
(kw, _) => Err(Error::bad_request(format!("Invalid endpoint: {}", kw)))
|
||||
(kw, _) => Err(Error::BadRequest(format!("Invalid endpoint: {}", kw)))
|
||||
}
|
||||
}};
|
||||
|
||||
@@ -97,14 +74,14 @@ macro_rules! router_match {
|
||||
.take()
|
||||
.map(|param| param.parse())
|
||||
.transpose()
|
||||
.map_err(|_| Error::bad_request("Failed to parse query parameter"))?
|
||||
.map_err(|_| Error::BadRequest("Failed to parse query parameter".to_owned()))?
|
||||
}};
|
||||
(@@parse_param $query:expr, parse, $param:ident) => {{
|
||||
// extract and parse mandatory query parameter
|
||||
// both missing and un-parseable parameters are reported as errors
|
||||
$query.$param.take().ok_or_bad_request("Missing argument for endpoint")?
|
||||
.parse()
|
||||
.map_err(|_| Error::bad_request("Failed to parse query parameter"))?
|
||||
.map_err(|_| Error::BadRequest("Failed to parse query parameter".to_owned()))?
|
||||
}};
|
||||
(@func
|
||||
$(#[$doc:meta])*
|
||||
@@ -173,7 +150,7 @@ macro_rules! generateQueryParameters {
|
||||
false
|
||||
} else if v.as_ref().is_empty() {
|
||||
if res.keyword.replace(k).is_some() {
|
||||
return Err(Error::bad_request("Multiple keywords"));
|
||||
return Err(Error::BadRequest("Multiple keywords".to_owned()));
|
||||
}
|
||||
continue;
|
||||
} else {
|
||||
@@ -183,7 +160,7 @@ macro_rules! generateQueryParameters {
|
||||
}
|
||||
};
|
||||
if repeated {
|
||||
return Err(Error::bad_request(format!(
|
||||
return Err(Error::BadRequest(format!(
|
||||
"Query parameter repeated: '{}'",
|
||||
k
|
||||
)));
|
||||
|
||||
+15
-13
@@ -8,13 +8,14 @@ use hyper::{Body, Method, Request, Response};
|
||||
|
||||
use opentelemetry::{trace::SpanRef, KeyValue};
|
||||
|
||||
use garage_table::util::*;
|
||||
use garage_util::error::Error as GarageError;
|
||||
|
||||
use garage_model::garage::Garage;
|
||||
use garage_model::key_table::Key;
|
||||
|
||||
use crate::error::*;
|
||||
use crate::generic_server::*;
|
||||
use crate::s3::error::*;
|
||||
|
||||
use crate::signature::payload::check_payload_signature;
|
||||
use crate::signature::streaming::*;
|
||||
@@ -74,7 +75,6 @@ impl ApiHandler for S3ApiServer {
|
||||
const API_NAME_DISPLAY: &'static str = "S3";
|
||||
|
||||
type Endpoint = S3ApiEndpoint;
|
||||
type Error = Error;
|
||||
|
||||
fn parse_endpoint(&self, req: &Request<Body>) -> Result<S3ApiEndpoint, Error> {
|
||||
let authority = req
|
||||
@@ -122,8 +122,9 @@ impl ApiHandler for S3ApiServer {
|
||||
}
|
||||
|
||||
let (api_key, mut content_sha256) = check_payload_signature(&garage, "s3", &req).await?;
|
||||
let api_key = api_key
|
||||
.ok_or_else(|| Error::forbidden("Garage does not support anonymous access yet"))?;
|
||||
let api_key = api_key.ok_or_else(|| {
|
||||
Error::Forbidden("Garage does not support anonymous access yet".to_string())
|
||||
})?;
|
||||
|
||||
let req = parse_streaming_body(
|
||||
&api_key,
|
||||
@@ -147,14 +148,13 @@ impl ApiHandler for S3ApiServer {
|
||||
return handle_create_bucket(&garage, req, content_sha256, api_key, bucket_name).await;
|
||||
}
|
||||
|
||||
let bucket_id = garage
|
||||
.bucket_helper()
|
||||
.resolve_bucket(&bucket_name, &api_key)
|
||||
.await?;
|
||||
let bucket_id = resolve_bucket(&garage, &bucket_name, &api_key).await?;
|
||||
let bucket = garage
|
||||
.bucket_helper()
|
||||
.get_existing_bucket(bucket_id)
|
||||
.await?;
|
||||
.bucket_table
|
||||
.get(&EmptyKey, &bucket_id)
|
||||
.await?
|
||||
.filter(|b| !b.state.is_deleted())
|
||||
.ok_or(Error::NoSuchBucket)?;
|
||||
|
||||
let allowed = match endpoint.authorization_type() {
|
||||
Authorization::Read => api_key.allow_read(&bucket_id),
|
||||
@@ -164,7 +164,9 @@ impl ApiHandler for S3ApiServer {
|
||||
};
|
||||
|
||||
if !allowed {
|
||||
return Err(Error::forbidden("Operation is not allowed for this key."));
|
||||
return Err(Error::Forbidden(
|
||||
"Operation is not allowed for this key.".to_string(),
|
||||
));
|
||||
}
|
||||
|
||||
// Look up what CORS rule might apply to response.
|
||||
@@ -307,7 +309,7 @@ impl ApiHandler for S3ApiServer {
|
||||
)
|
||||
.await
|
||||
} else {
|
||||
Err(Error::bad_request(format!(
|
||||
Err(Error::BadRequest(format!(
|
||||
"Invalid endpoint: list-type={}",
|
||||
list_type
|
||||
)))
|
||||
|
||||
+17
-7
@@ -8,13 +8,13 @@ use garage_model::bucket_table::Bucket;
|
||||
use garage_model::garage::Garage;
|
||||
use garage_model::key_table::Key;
|
||||
use garage_model::permission::BucketKeyPerm;
|
||||
use garage_model::s3::object_table::ObjectFilter;
|
||||
use garage_table::util::*;
|
||||
use garage_util::crdt::*;
|
||||
use garage_util::data::*;
|
||||
use garage_util::time::*;
|
||||
|
||||
use crate::common_error::CommonError;
|
||||
use crate::s3::error::*;
|
||||
use crate::error::*;
|
||||
use crate::s3::xml as s3_xml;
|
||||
use crate::signature::verify_signed_content;
|
||||
|
||||
@@ -130,7 +130,7 @@ pub async fn handle_create_bucket(
|
||||
|
||||
if let Some(location_constraint) = cmd {
|
||||
if location_constraint != garage.config.s3_api.s3_region {
|
||||
return Err(Error::bad_request(format!(
|
||||
return Err(Error::BadRequest(format!(
|
||||
"Cannot satisfy location constraint `{}`: buckets can only be created in region `{}`",
|
||||
location_constraint,
|
||||
garage.config.s3_api.s3_region
|
||||
@@ -158,12 +158,12 @@ pub async fn handle_create_bucket(
|
||||
// otherwise return a forbidden error.
|
||||
let kp = api_key.bucket_permissions(&bucket_id);
|
||||
if !(kp.allow_write || kp.allow_owner) {
|
||||
return Err(CommonError::BucketAlreadyExists.into());
|
||||
return Err(Error::BucketAlreadyExists);
|
||||
}
|
||||
} else {
|
||||
// Create the bucket!
|
||||
if !is_valid_bucket_name(&bucket_name) {
|
||||
return Err(Error::bad_request(format!(
|
||||
return Err(Error::BadRequest(format!(
|
||||
"{}: {}",
|
||||
bucket_name, INVALID_BUCKET_NAME_MESSAGE
|
||||
)));
|
||||
@@ -228,8 +228,18 @@ pub async fn handle_delete_bucket(
|
||||
// Delete bucket
|
||||
|
||||
// Check bucket is empty
|
||||
if !garage.bucket_helper().is_bucket_empty(bucket_id).await? {
|
||||
return Err(CommonError::BucketNotEmpty.into());
|
||||
let objects = garage
|
||||
.object_table
|
||||
.get_range(
|
||||
&bucket_id,
|
||||
None,
|
||||
Some(ObjectFilter::IsData),
|
||||
10,
|
||||
EnumerationOrder::Forward,
|
||||
)
|
||||
.await?;
|
||||
if !objects.is_empty() {
|
||||
return Err(Error::BucketNotEmpty);
|
||||
}
|
||||
|
||||
// --- done checking, now commit ---
|
||||
|
||||
+11
-14
@@ -18,8 +18,8 @@ use garage_model::s3::block_ref_table::*;
|
||||
use garage_model::s3::object_table::*;
|
||||
use garage_model::s3::version_table::*;
|
||||
|
||||
use crate::helpers::parse_bucket_key;
|
||||
use crate::s3::error::*;
|
||||
use crate::error::*;
|
||||
use crate::helpers::{parse_bucket_key, resolve_bucket};
|
||||
use crate::s3::put::{decode_upload_id, get_headers};
|
||||
use crate::s3::xml::{self as s3_xml, xmlns_tag};
|
||||
|
||||
@@ -201,8 +201,8 @@ pub async fn handle_upload_part_copy(
|
||||
let mut ranges = http_range::HttpRange::parse(range_str, source_version_meta.size)
|
||||
.map_err(|e| (e, source_version_meta.size))?;
|
||||
if ranges.len() != 1 {
|
||||
return Err(Error::bad_request(
|
||||
"Invalid x-amz-copy-source-range header: exactly 1 range must be given",
|
||||
return Err(Error::BadRequest(
|
||||
"Invalid x-amz-copy-source-range header: exactly 1 range must be given".into(),
|
||||
));
|
||||
} else {
|
||||
ranges.pop().unwrap()
|
||||
@@ -230,8 +230,8 @@ pub async fn handle_upload_part_copy(
|
||||
// This is only for small files, we don't bother handling this.
|
||||
// (in AWS UploadPartCopy works for parts at least 5MB which
|
||||
// is never the case of an inline object)
|
||||
return Err(Error::bad_request(
|
||||
"Source object is too small (minimum part size is 5Mb)",
|
||||
return Err(Error::BadRequest(
|
||||
"Source object is too small (minimum part size is 5Mb)".into(),
|
||||
));
|
||||
}
|
||||
ObjectVersionData::FirstBlock(_meta, _first_block_hash) => (),
|
||||
@@ -250,7 +250,7 @@ pub async fn handle_upload_part_copy(
|
||||
// Check this part number hasn't yet been uploaded
|
||||
if let Some(dv) = dest_version {
|
||||
if dv.has_part_number(part_number) {
|
||||
return Err(Error::bad_request(format!(
|
||||
return Err(Error::BadRequest(format!(
|
||||
"Part number {} has already been uploaded",
|
||||
part_number
|
||||
)));
|
||||
@@ -413,13 +413,10 @@ async fn get_copy_source(
|
||||
let copy_source = percent_encoding::percent_decode_str(copy_source).decode_utf8()?;
|
||||
|
||||
let (source_bucket, source_key) = parse_bucket_key(©_source, None)?;
|
||||
let source_bucket_id = garage
|
||||
.bucket_helper()
|
||||
.resolve_bucket(&source_bucket.to_string(), api_key)
|
||||
.await?;
|
||||
let source_bucket_id = resolve_bucket(garage, &source_bucket.to_string(), api_key).await?;
|
||||
|
||||
if !api_key.allow_read(&source_bucket_id) {
|
||||
return Err(Error::forbidden(format!(
|
||||
return Err(Error::Forbidden(format!(
|
||||
"Reading from bucket {} not allowed for this key",
|
||||
source_bucket
|
||||
)));
|
||||
@@ -539,8 +536,8 @@ impl CopyPreconditionHeaders {
|
||||
(None, None, None, Some(ims)) => v_date > *ims,
|
||||
(None, None, None, None) => true,
|
||||
_ => {
|
||||
return Err(Error::bad_request(
|
||||
"Invalid combination of x-amz-copy-source-if-xxxxx headers",
|
||||
return Err(Error::BadRequest(
|
||||
"Invalid combination of x-amz-copy-source-if-xxxxx headers".into(),
|
||||
))
|
||||
}
|
||||
};
|
||||
|
||||
+23
-11
@@ -9,12 +9,13 @@ use hyper::{header::HeaderName, Body, Method, Request, Response, StatusCode};
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::s3::error::*;
|
||||
use crate::error::*;
|
||||
use crate::s3::xml::{to_xml_with_header, xmlns_tag, IntValue, Value};
|
||||
use crate::signature::verify_signed_content;
|
||||
|
||||
use garage_model::bucket_table::{Bucket, CorsRule as GarageCorsRule};
|
||||
use garage_model::garage::Garage;
|
||||
use garage_table::*;
|
||||
use garage_util::data::*;
|
||||
|
||||
pub async fn handle_get_cors(bucket: &Bucket) -> Result<Response<Body>, Error> {
|
||||
@@ -47,11 +48,14 @@ pub async fn handle_delete_cors(
|
||||
bucket_id: Uuid,
|
||||
) -> Result<Response<Body>, Error> {
|
||||
let mut bucket = garage
|
||||
.bucket_helper()
|
||||
.get_existing_bucket(bucket_id)
|
||||
.await?;
|
||||
.bucket_table
|
||||
.get(&EmptyKey, &bucket_id)
|
||||
.await?
|
||||
.ok_or(Error::NoSuchBucket)?;
|
||||
|
||||
let param = bucket.params_mut().unwrap();
|
||||
let param = bucket
|
||||
.params_mut()
|
||||
.ok_or_internal_error("Bucket should not be deleted at this point")?;
|
||||
|
||||
param.cors_config.update(None);
|
||||
garage.bucket_table.insert(&bucket).await?;
|
||||
@@ -74,11 +78,14 @@ pub async fn handle_put_cors(
|
||||
}
|
||||
|
||||
let mut bucket = garage
|
||||
.bucket_helper()
|
||||
.get_existing_bucket(bucket_id)
|
||||
.await?;
|
||||
.bucket_table
|
||||
.get(&EmptyKey, &bucket_id)
|
||||
.await?
|
||||
.ok_or(Error::NoSuchBucket)?;
|
||||
|
||||
let param = bucket.params_mut().unwrap();
|
||||
let param = bucket
|
||||
.params_mut()
|
||||
.ok_or_internal_error("Bucket should not be deleted at this point")?;
|
||||
|
||||
let conf: CorsConfiguration = from_reader(&body as &[u8])?;
|
||||
conf.validate()?;
|
||||
@@ -112,7 +119,12 @@ pub async fn handle_options_s3api(
|
||||
let helper = garage.bucket_helper();
|
||||
let bucket_id = helper.resolve_global_bucket_name(&bn).await?;
|
||||
if let Some(id) = bucket_id {
|
||||
let bucket = garage.bucket_helper().get_existing_bucket(id).await?;
|
||||
let bucket = garage
|
||||
.bucket_table
|
||||
.get(&EmptyKey, &id)
|
||||
.await?
|
||||
.filter(|b| !b.state.is_deleted())
|
||||
.ok_or(Error::NoSuchBucket)?;
|
||||
handle_options_for_bucket(req, &bucket)
|
||||
} else {
|
||||
// If there is a bucket name in the request, but that name
|
||||
@@ -173,7 +185,7 @@ pub fn handle_options_for_bucket(
|
||||
}
|
||||
}
|
||||
|
||||
Err(Error::forbidden("This CORS request is not allowed."))
|
||||
Err(Error::Forbidden("This CORS request is not allowed.".into()))
|
||||
}
|
||||
|
||||
pub fn find_matching_cors_rule<'a>(
|
||||
|
||||
@@ -8,7 +8,7 @@ use garage_util::time::*;
|
||||
use garage_model::garage::Garage;
|
||||
use garage_model::s3::object_table::*;
|
||||
|
||||
use crate::s3::error::*;
|
||||
use crate::error::*;
|
||||
use crate::s3::xml as s3_xml;
|
||||
use crate::signature::verify_signed_content;
|
||||
|
||||
|
||||
+5
-5
@@ -17,7 +17,7 @@ use garage_model::garage::Garage;
|
||||
use garage_model::s3::object_table::*;
|
||||
use garage_model::s3::version_table::*;
|
||||
|
||||
use crate::s3::error::*;
|
||||
use crate::error::*;
|
||||
|
||||
const X_AMZ_MP_PARTS_COUNT: &str = "x-amz-mp-parts-count";
|
||||
|
||||
@@ -210,8 +210,8 @@ pub async fn handle_get(
|
||||
|
||||
match (part_number, parse_range_header(req, last_v_meta.size)?) {
|
||||
(Some(_), Some(_)) => {
|
||||
return Err(Error::bad_request(
|
||||
"Cannot specify both partNumber and Range header",
|
||||
return Err(Error::BadRequest(
|
||||
"Cannot specify both partNumber and Range header".into(),
|
||||
));
|
||||
}
|
||||
(Some(pn), None) => {
|
||||
@@ -302,9 +302,9 @@ async fn handle_get_range(
|
||||
let body: Body = Body::from(bytes[begin as usize..end as usize].to_vec());
|
||||
Ok(resp_builder.body(body)?)
|
||||
} else {
|
||||
Err(Error::internal_error(
|
||||
None.ok_or_internal_error(
|
||||
"Requested range not present in inline bytes when it should have been",
|
||||
))
|
||||
)
|
||||
}
|
||||
}
|
||||
ObjectVersionData::FirstBlock(_meta, _first_block_hash) => {
|
||||
|
||||
+4
-10
@@ -16,8 +16,8 @@ use garage_model::s3::version_table::Version;
|
||||
use garage_table::{EmptyKey, EnumerationOrder};
|
||||
|
||||
use crate::encoding::*;
|
||||
use crate::error::*;
|
||||
use crate::helpers::key_after_prefix;
|
||||
use crate::s3::error::*;
|
||||
use crate::s3::put as s3_put;
|
||||
use crate::s3::xml as s3_xml;
|
||||
|
||||
@@ -582,19 +582,13 @@ impl ListObjectsQuery {
|
||||
// representing the key to start with.
|
||||
(Some(token), _) => match &token[..1] {
|
||||
"[" => Ok(RangeBegin::IncludingKey {
|
||||
key: String::from_utf8(
|
||||
base64::decode(token[1..].as_bytes())
|
||||
.ok_or_bad_request("Invalid continuation token")?,
|
||||
)?,
|
||||
key: String::from_utf8(base64::decode(token[1..].as_bytes())?)?,
|
||||
fallback_key: None,
|
||||
}),
|
||||
"]" => Ok(RangeBegin::AfterKey {
|
||||
key: String::from_utf8(
|
||||
base64::decode(token[1..].as_bytes())
|
||||
.ok_or_bad_request("Invalid continuation token")?,
|
||||
)?,
|
||||
key: String::from_utf8(base64::decode(token[1..].as_bytes())?)?,
|
||||
}),
|
||||
_ => Err(Error::bad_request("Invalid continuation token")),
|
||||
_ => Err(Error::BadRequest("Invalid continuation token".to_string())),
|
||||
},
|
||||
|
||||
// StartAfter has defined semantics in the spec:
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
pub mod api_server;
|
||||
pub mod error;
|
||||
|
||||
mod bucket;
|
||||
mod copy;
|
||||
|
||||
+34
-28
@@ -14,7 +14,8 @@ use serde::Deserialize;
|
||||
|
||||
use garage_model::garage::Garage;
|
||||
|
||||
use crate::s3::error::*;
|
||||
use crate::error::*;
|
||||
use crate::helpers::resolve_bucket;
|
||||
use crate::s3::put::{get_headers, save_stream};
|
||||
use crate::s3::xml as s3_xml;
|
||||
use crate::signature::payload::{parse_date, verify_v4};
|
||||
@@ -47,7 +48,9 @@ pub async fn handle_post_object(
|
||||
let field = if let Some(field) = multipart.next_field().await? {
|
||||
field
|
||||
} else {
|
||||
return Err(Error::bad_request("Request did not contain a file"));
|
||||
return Err(Error::BadRequest(
|
||||
"Request did not contain a file".to_owned(),
|
||||
));
|
||||
};
|
||||
let name: HeaderName = if let Some(Ok(name)) = field.name().map(TryInto::try_into) {
|
||||
name
|
||||
@@ -63,14 +66,14 @@ pub async fn handle_post_object(
|
||||
"tag" => (/* tag need to be reencoded, but we don't support them yet anyway */),
|
||||
"acl" => {
|
||||
if params.insert("x-amz-acl", content).is_some() {
|
||||
return Err(Error::bad_request(
|
||||
"Field 'acl' provided more than one time",
|
||||
return Err(Error::BadRequest(
|
||||
"Field 'acl' provided more than one time".to_string(),
|
||||
));
|
||||
}
|
||||
}
|
||||
_ => {
|
||||
if params.insert(&name, content).is_some() {
|
||||
return Err(Error::bad_request(format!(
|
||||
return Err(Error::BadRequest(format!(
|
||||
"Field '{}' provided more than one time",
|
||||
name
|
||||
)));
|
||||
@@ -87,7 +90,9 @@ pub async fn handle_post_object(
|
||||
.to_str()?;
|
||||
let credential = params
|
||||
.get("x-amz-credential")
|
||||
.ok_or_else(|| Error::forbidden("Garage does not support anonymous access yet"))?
|
||||
.ok_or_else(|| {
|
||||
Error::Forbidden("Garage does not support anonymous access yet".to_string())
|
||||
})?
|
||||
.to_str()?;
|
||||
let policy = params
|
||||
.get("policy")
|
||||
@@ -124,16 +129,15 @@ pub async fn handle_post_object(
|
||||
)
|
||||
.await?;
|
||||
|
||||
let bucket_id = garage
|
||||
.bucket_helper()
|
||||
.resolve_bucket(&bucket, &api_key)
|
||||
.await?;
|
||||
let bucket_id = resolve_bucket(&garage, &bucket, &api_key).await?;
|
||||
|
||||
if !api_key.allow_write(&bucket_id) {
|
||||
return Err(Error::forbidden("Operation is not allowed for this key."));
|
||||
return Err(Error::Forbidden(
|
||||
"Operation is not allowed for this key.".to_string(),
|
||||
));
|
||||
}
|
||||
|
||||
let decoded_policy = base64::decode(&policy).ok_or_bad_request("Invalid policy")?;
|
||||
let decoded_policy = base64::decode(&policy)?;
|
||||
let decoded_policy: Policy =
|
||||
serde_json::from_slice(&decoded_policy).ok_or_bad_request("Invalid policy")?;
|
||||
|
||||
@@ -141,7 +145,9 @@ pub async fn handle_post_object(
|
||||
.ok_or_bad_request("Invalid expiration date")?
|
||||
.into();
|
||||
if Utc::now() - expiration > Duration::zero() {
|
||||
return Err(Error::bad_request("Expiration date is in the paste"));
|
||||
return Err(Error::BadRequest(
|
||||
"Expiration date is in the paste".to_string(),
|
||||
));
|
||||
}
|
||||
|
||||
let mut conditions = decoded_policy.into_conditions()?;
|
||||
@@ -153,7 +159,7 @@ pub async fn handle_post_object(
|
||||
"policy" | "x-amz-signature" => (), // this is always accepted, as it's required to validate other fields
|
||||
"content-type" => {
|
||||
let conds = conditions.params.remove("content-type").ok_or_else(|| {
|
||||
Error::bad_request(format!("Key '{}' is not allowed in policy", param_key))
|
||||
Error::BadRequest(format!("Key '{}' is not allowed in policy", param_key))
|
||||
})?;
|
||||
for cond in conds {
|
||||
let ok = match cond {
|
||||
@@ -163,7 +169,7 @@ pub async fn handle_post_object(
|
||||
}
|
||||
};
|
||||
if !ok {
|
||||
return Err(Error::bad_request(format!(
|
||||
return Err(Error::BadRequest(format!(
|
||||
"Key '{}' has value not allowed in policy",
|
||||
param_key
|
||||
)));
|
||||
@@ -172,7 +178,7 @@ pub async fn handle_post_object(
|
||||
}
|
||||
"key" => {
|
||||
let conds = conditions.params.remove("key").ok_or_else(|| {
|
||||
Error::bad_request(format!("Key '{}' is not allowed in policy", param_key))
|
||||
Error::BadRequest(format!("Key '{}' is not allowed in policy", param_key))
|
||||
})?;
|
||||
for cond in conds {
|
||||
let ok = match cond {
|
||||
@@ -180,7 +186,7 @@ pub async fn handle_post_object(
|
||||
Operation::StartsWith(s) => key.starts_with(&s),
|
||||
};
|
||||
if !ok {
|
||||
return Err(Error::bad_request(format!(
|
||||
return Err(Error::BadRequest(format!(
|
||||
"Key '{}' has value not allowed in policy",
|
||||
param_key
|
||||
)));
|
||||
@@ -195,7 +201,7 @@ pub async fn handle_post_object(
|
||||
continue;
|
||||
}
|
||||
let conds = conditions.params.remove(¶m_key).ok_or_else(|| {
|
||||
Error::bad_request(format!("Key '{}' is not allowed in policy", param_key))
|
||||
Error::BadRequest(format!("Key '{}' is not allowed in policy", param_key))
|
||||
})?;
|
||||
for cond in conds {
|
||||
let ok = match cond {
|
||||
@@ -203,7 +209,7 @@ pub async fn handle_post_object(
|
||||
Operation::StartsWith(s) => value.to_str()?.starts_with(s.as_str()),
|
||||
};
|
||||
if !ok {
|
||||
return Err(Error::bad_request(format!(
|
||||
return Err(Error::BadRequest(format!(
|
||||
"Key '{}' has value not allowed in policy",
|
||||
param_key
|
||||
)));
|
||||
@@ -214,7 +220,7 @@ pub async fn handle_post_object(
|
||||
}
|
||||
|
||||
if let Some((param_key, _)) = conditions.params.iter().next() {
|
||||
return Err(Error::bad_request(format!(
|
||||
return Err(Error::BadRequest(format!(
|
||||
"Key '{}' is required in policy, but no value was provided",
|
||||
param_key
|
||||
)));
|
||||
@@ -320,7 +326,7 @@ impl Policy {
|
||||
match condition {
|
||||
PolicyCondition::Equal(map) => {
|
||||
if map.len() != 1 {
|
||||
return Err(Error::bad_request("Invalid policy item"));
|
||||
return Err(Error::BadRequest("Invalid policy item".to_owned()));
|
||||
}
|
||||
let (mut k, v) = map.into_iter().next().expect("size was verified");
|
||||
k.make_ascii_lowercase();
|
||||
@@ -328,7 +334,7 @@ impl Policy {
|
||||
}
|
||||
PolicyCondition::OtherOp([cond, mut key, value]) => {
|
||||
if key.remove(0) != '$' {
|
||||
return Err(Error::bad_request("Invalid policy item"));
|
||||
return Err(Error::BadRequest("Invalid policy item".to_owned()));
|
||||
}
|
||||
key.make_ascii_lowercase();
|
||||
match cond.as_str() {
|
||||
@@ -341,7 +347,7 @@ impl Policy {
|
||||
.or_default()
|
||||
.push(Operation::StartsWith(value));
|
||||
}
|
||||
_ => return Err(Error::bad_request("Invalid policy item")),
|
||||
_ => return Err(Error::BadRequest("Invalid policy item".to_owned())),
|
||||
}
|
||||
}
|
||||
PolicyCondition::SizeRange(key, min, max) => {
|
||||
@@ -349,7 +355,7 @@ impl Policy {
|
||||
length.0 = length.0.max(min);
|
||||
length.1 = length.1.min(max);
|
||||
} else {
|
||||
return Err(Error::bad_request("Invalid policy item"));
|
||||
return Err(Error::BadRequest("Invalid policy item".to_owned()));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -414,15 +420,15 @@ where
|
||||
self.read += bytes.len() as u64;
|
||||
// optimization to fail early when we know before the end it's too long
|
||||
if self.length.end() < &self.read {
|
||||
return Poll::Ready(Some(Err(Error::bad_request(
|
||||
"File size does not match policy",
|
||||
return Poll::Ready(Some(Err(Error::BadRequest(
|
||||
"File size does not match policy".to_owned(),
|
||||
))));
|
||||
}
|
||||
}
|
||||
Poll::Ready(None) => {
|
||||
if !self.length.contains(&self.read) {
|
||||
return Poll::Ready(Some(Err(Error::bad_request(
|
||||
"File size does not match policy",
|
||||
return Poll::Ready(Some(Err(Error::BadRequest(
|
||||
"File size does not match policy".to_owned(),
|
||||
))));
|
||||
}
|
||||
}
|
||||
|
||||
+10
-8
@@ -19,7 +19,7 @@ use garage_model::s3::block_ref_table::*;
|
||||
use garage_model::s3::object_table::*;
|
||||
use garage_model::s3::version_table::*;
|
||||
|
||||
use crate::s3::error::*;
|
||||
use crate::error::*;
|
||||
use crate::s3::xml as s3_xml;
|
||||
use crate::signature::verify_signed_content;
|
||||
|
||||
@@ -183,8 +183,8 @@ fn ensure_checksum_matches(
|
||||
) -> Result<(), Error> {
|
||||
if let Some(expected_sha256) = content_sha256 {
|
||||
if expected_sha256 != data_sha256sum {
|
||||
return Err(Error::bad_request(
|
||||
"Unable to validate x-amz-content-sha256",
|
||||
return Err(Error::BadRequest(
|
||||
"Unable to validate x-amz-content-sha256".to_string(),
|
||||
));
|
||||
} else {
|
||||
trace!("Successfully validated x-amz-content-sha256");
|
||||
@@ -192,7 +192,9 @@ fn ensure_checksum_matches(
|
||||
}
|
||||
if let Some(expected_md5) = content_md5 {
|
||||
if expected_md5.trim_matches('"') != base64::encode(data_md5sum) {
|
||||
return Err(Error::bad_request("Unable to validate content-md5"));
|
||||
return Err(Error::BadRequest(
|
||||
"Unable to validate content-md5".to_string(),
|
||||
));
|
||||
} else {
|
||||
trace!("Successfully validated content-md5");
|
||||
}
|
||||
@@ -426,7 +428,7 @@ pub async fn handle_put_part(
|
||||
// Check part hasn't already been uploaded
|
||||
if let Some(v) = version {
|
||||
if v.has_part_number(part_number) {
|
||||
return Err(Error::bad_request(format!(
|
||||
return Err(Error::BadRequest(format!(
|
||||
"Part number {} has already been uploaded",
|
||||
part_number
|
||||
)));
|
||||
@@ -511,7 +513,7 @@ pub async fn handle_complete_multipart_upload(
|
||||
|
||||
let version = version.ok_or(Error::NoSuchKey)?;
|
||||
if version.blocks.is_empty() {
|
||||
return Err(Error::bad_request("No data was uploaded"));
|
||||
return Err(Error::BadRequest("No data was uploaded".to_string()));
|
||||
}
|
||||
|
||||
let headers = match object_version.state {
|
||||
@@ -572,8 +574,8 @@ pub async fn handle_complete_multipart_upload(
|
||||
.map(|x| x.part_number)
|
||||
.eq(block_parts.into_iter());
|
||||
if !same_parts {
|
||||
return Err(Error::bad_request(
|
||||
"Part numbers in block list and part list do not match. This can happen if a part was partially uploaded. Please abort the multipart upload and try again."
|
||||
return Err(Error::BadRequest(
|
||||
"Part numbers in block list and part list do not match. This can happen if a part was partially uploaded. Please abort the multipart upload and try again.".into(),
|
||||
));
|
||||
}
|
||||
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
use crate::error::{Error, OkOrBadRequest};
|
||||
|
||||
use std::borrow::Cow;
|
||||
|
||||
use hyper::header::HeaderValue;
|
||||
@@ -5,7 +7,6 @@ use hyper::{HeaderMap, Method, Request};
|
||||
|
||||
use crate::helpers::Authorization;
|
||||
use crate::router_macros::{generateQueryParameters, router_match};
|
||||
use crate::s3::error::*;
|
||||
|
||||
router_match! {@func
|
||||
|
||||
@@ -342,7 +343,7 @@ impl Endpoint {
|
||||
Method::POST => Self::from_post(key, &mut query)?,
|
||||
Method::PUT => Self::from_put(key, &mut query, req.headers())?,
|
||||
Method::DELETE => Self::from_delete(key, &mut query)?,
|
||||
_ => return Err(Error::bad_request("Unknown method")),
|
||||
_ => return Err(Error::BadRequest("Unknown method".to_owned())),
|
||||
};
|
||||
|
||||
if let Some(message) = query.nonempty_message() {
|
||||
|
||||
+28
-21
@@ -4,12 +4,13 @@ use std::sync::Arc;
|
||||
use hyper::{Body, Request, Response, StatusCode};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::s3::error::*;
|
||||
use crate::error::*;
|
||||
use crate::s3::xml::{to_xml_with_header, xmlns_tag, IntValue, Value};
|
||||
use crate::signature::verify_signed_content;
|
||||
|
||||
use garage_model::bucket_table::*;
|
||||
use garage_model::garage::Garage;
|
||||
use garage_table::*;
|
||||
use garage_util::data::*;
|
||||
|
||||
pub async fn handle_get_website(bucket: &Bucket) -> Result<Response<Body>, Error> {
|
||||
@@ -46,11 +47,14 @@ pub async fn handle_delete_website(
|
||||
bucket_id: Uuid,
|
||||
) -> Result<Response<Body>, Error> {
|
||||
let mut bucket = garage
|
||||
.bucket_helper()
|
||||
.get_existing_bucket(bucket_id)
|
||||
.await?;
|
||||
.bucket_table
|
||||
.get(&EmptyKey, &bucket_id)
|
||||
.await?
|
||||
.ok_or(Error::NoSuchBucket)?;
|
||||
|
||||
let param = bucket.params_mut().unwrap();
|
||||
let param = bucket
|
||||
.params_mut()
|
||||
.ok_or_internal_error("Bucket should not be deleted at this point")?;
|
||||
|
||||
param.website_config.update(None);
|
||||
garage.bucket_table.insert(&bucket).await?;
|
||||
@@ -73,11 +77,14 @@ pub async fn handle_put_website(
|
||||
}
|
||||
|
||||
let mut bucket = garage
|
||||
.bucket_helper()
|
||||
.get_existing_bucket(bucket_id)
|
||||
.await?;
|
||||
.bucket_table
|
||||
.get(&EmptyKey, &bucket_id)
|
||||
.await?
|
||||
.ok_or(Error::NoSuchBucket)?;
|
||||
|
||||
let param = bucket.params_mut().unwrap();
|
||||
let param = bucket
|
||||
.params_mut()
|
||||
.ok_or_internal_error("Bucket should not be deleted at this point")?;
|
||||
|
||||
let conf: WebsiteConfiguration = from_reader(&body as &[u8])?;
|
||||
conf.validate()?;
|
||||
@@ -169,8 +176,8 @@ impl WebsiteConfiguration {
|
||||
|| self.index_document.is_some()
|
||||
|| self.routing_rules.is_some())
|
||||
{
|
||||
return Err(Error::bad_request(
|
||||
"Bad XML: can't have RedirectAllRequestsTo and other fields",
|
||||
return Err(Error::BadRequest(
|
||||
"Bad XML: can't have RedirectAllRequestsTo and other fields".to_owned(),
|
||||
));
|
||||
}
|
||||
if let Some(ref ed) = self.error_document {
|
||||
@@ -215,8 +222,8 @@ impl WebsiteConfiguration {
|
||||
impl Key {
|
||||
pub fn validate(&self) -> Result<(), Error> {
|
||||
if self.key.0.is_empty() {
|
||||
Err(Error::bad_request(
|
||||
"Bad XML: error document specified but empty",
|
||||
Err(Error::BadRequest(
|
||||
"Bad XML: error document specified but empty".to_owned(),
|
||||
))
|
||||
} else {
|
||||
Ok(())
|
||||
@@ -227,8 +234,8 @@ impl Key {
|
||||
impl Suffix {
|
||||
pub fn validate(&self) -> Result<(), Error> {
|
||||
if self.suffix.0.is_empty() | self.suffix.0.contains('/') {
|
||||
Err(Error::bad_request(
|
||||
"Bad XML: index document is empty or contains /",
|
||||
Err(Error::BadRequest(
|
||||
"Bad XML: index document is empty or contains /".to_owned(),
|
||||
))
|
||||
} else {
|
||||
Ok(())
|
||||
@@ -240,7 +247,7 @@ impl Target {
|
||||
pub fn validate(&self) -> Result<(), Error> {
|
||||
if let Some(ref protocol) = self.protocol {
|
||||
if protocol.0 != "http" && protocol.0 != "https" {
|
||||
return Err(Error::bad_request("Bad XML: invalid protocol"));
|
||||
return Err(Error::BadRequest("Bad XML: invalid protocol".to_owned()));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
@@ -262,19 +269,19 @@ impl Redirect {
|
||||
pub fn validate(&self, has_prefix: bool) -> Result<(), Error> {
|
||||
if self.replace_prefix.is_some() {
|
||||
if self.replace_full.is_some() {
|
||||
return Err(Error::bad_request(
|
||||
"Bad XML: both ReplaceKeyPrefixWith and ReplaceKeyWith are set",
|
||||
return Err(Error::BadRequest(
|
||||
"Bad XML: both ReplaceKeyPrefixWith and ReplaceKeyWith are set".to_owned(),
|
||||
));
|
||||
}
|
||||
if !has_prefix {
|
||||
return Err(Error::bad_request(
|
||||
"Bad XML: ReplaceKeyPrefixWith is set, but KeyPrefixEquals isn't",
|
||||
return Err(Error::BadRequest(
|
||||
"Bad XML: ReplaceKeyPrefixWith is set, but KeyPrefixEquals isn't".to_owned(),
|
||||
));
|
||||
}
|
||||
}
|
||||
if let Some(ref protocol) = self.protocol {
|
||||
if protocol.0 != "http" && protocol.0 != "https" {
|
||||
return Err(Error::bad_request("Bad XML: invalid protocol"));
|
||||
return Err(Error::BadRequest("Bad XML: invalid protocol".to_owned()));
|
||||
}
|
||||
}
|
||||
// TODO there are probably more invalide cases, but which ones?
|
||||
|
||||
+1
-1
@@ -1,7 +1,7 @@
|
||||
use quick_xml::se::to_string;
|
||||
use serde::{Deserialize, Serialize, Serializer};
|
||||
|
||||
use crate::s3::error::Error as ApiError;
|
||||
use crate::Error as ApiError;
|
||||
|
||||
pub fn to_xml_with_header<T: Serialize>(x: &T) -> Result<String, ApiError> {
|
||||
let mut xml = r#"<?xml version="1.0" encoding="UTF-8"?>"#.to_string();
|
||||
|
||||
@@ -1,36 +0,0 @@
|
||||
use err_derive::Error;
|
||||
|
||||
use crate::common_error::CommonError;
|
||||
pub use crate::common_error::{CommonErrorDerivative, OkOrBadRequest, OkOrInternalError};
|
||||
|
||||
/// Errors of this crate
|
||||
#[derive(Debug, Error)]
|
||||
pub enum Error {
|
||||
#[error(display = "{}", _0)]
|
||||
/// Error from common error
|
||||
Common(CommonError),
|
||||
|
||||
/// Authorization Header Malformed
|
||||
#[error(display = "Authorization header malformed, expected scope: {}", _0)]
|
||||
AuthorizationHeaderMalformed(String),
|
||||
|
||||
// Category: bad request
|
||||
/// The request contained an invalid UTF-8 sequence in its path or in other parameters
|
||||
#[error(display = "Invalid UTF-8: {}", _0)]
|
||||
InvalidUtf8Str(#[error(source)] std::str::Utf8Error),
|
||||
|
||||
/// The client sent a header with invalid value
|
||||
#[error(display = "Invalid header value: {}", _0)]
|
||||
InvalidHeader(#[error(source)] hyper::header::ToStrError),
|
||||
}
|
||||
|
||||
impl<T> From<T> for Error
|
||||
where
|
||||
CommonError: From<T>,
|
||||
{
|
||||
fn from(err: T) -> Self {
|
||||
Error::Common(CommonError::from(err))
|
||||
}
|
||||
}
|
||||
|
||||
impl CommonErrorDerivative for Error {}
|
||||
@@ -4,12 +4,11 @@ use sha2::Sha256;
|
||||
|
||||
use garage_util::data::{sha256sum, Hash};
|
||||
|
||||
pub mod error;
|
||||
use crate::error::*;
|
||||
|
||||
pub mod payload;
|
||||
pub mod streaming;
|
||||
|
||||
use error::*;
|
||||
|
||||
pub const SHORT_DATE: &str = "%Y%m%d";
|
||||
pub const LONG_DATETIME: &str = "%Y%m%dT%H%M%SZ";
|
||||
|
||||
@@ -17,7 +16,7 @@ type HmacSha256 = Hmac<Sha256>;
|
||||
|
||||
pub fn verify_signed_content(expected_sha256: Hash, body: &[u8]) -> Result<(), Error> {
|
||||
if expected_sha256 != sha256sum(body) {
|
||||
return Err(Error::bad_request(
|
||||
return Err(Error::BadRequest(
|
||||
"Request content hash does not match signed hash".to_string(),
|
||||
));
|
||||
}
|
||||
|
||||
@@ -15,7 +15,7 @@ use super::LONG_DATETIME;
|
||||
use super::{compute_scope, signing_hmac};
|
||||
|
||||
use crate::encoding::uri_encode;
|
||||
use crate::signature::error::*;
|
||||
use crate::error::*;
|
||||
|
||||
pub async fn check_payload_signature(
|
||||
garage: &Garage,
|
||||
@@ -105,7 +105,7 @@ fn parse_authorization(
|
||||
let (auth_kind, rest) = authorization.split_at(first_space);
|
||||
|
||||
if auth_kind != "AWS4-HMAC-SHA256" {
|
||||
return Err(Error::bad_request("Unsupported authorization method"));
|
||||
return Err(Error::BadRequest("Unsupported authorization method".into()));
|
||||
}
|
||||
|
||||
let mut auth_params = HashMap::new();
|
||||
@@ -129,11 +129,10 @@ fn parse_authorization(
|
||||
let date = headers
|
||||
.get("x-amz-date")
|
||||
.ok_or_bad_request("Missing X-Amz-Date field")
|
||||
.map_err(Error::from)
|
||||
.and_then(|d| parse_date(d))?;
|
||||
|
||||
if Utc::now() - date > Duration::hours(24) {
|
||||
return Err(Error::bad_request("Date is too old".to_string()));
|
||||
return Err(Error::BadRequest("Date is too old".to_string()));
|
||||
}
|
||||
|
||||
let auth = Authorization {
|
||||
@@ -157,7 +156,7 @@ fn parse_query_authorization(
|
||||
headers: &HashMap<String, String>,
|
||||
) -> Result<Authorization, Error> {
|
||||
if algorithm != "AWS4-HMAC-SHA256" {
|
||||
return Err(Error::bad_request(
|
||||
return Err(Error::BadRequest(
|
||||
"Unsupported authorization method".to_string(),
|
||||
));
|
||||
}
|
||||
@@ -180,10 +179,10 @@ fn parse_query_authorization(
|
||||
.get("x-amz-expires")
|
||||
.ok_or_bad_request("X-Amz-Expires not found in query parameters")?
|
||||
.parse()
|
||||
.map_err(|_| Error::bad_request("X-Amz-Expires is not a number".to_string()))?;
|
||||
.map_err(|_| Error::BadRequest("X-Amz-Expires is not a number".to_string()))?;
|
||||
|
||||
if duration > 7 * 24 * 3600 {
|
||||
return Err(Error::bad_request(
|
||||
return Err(Error::BadRequest(
|
||||
"X-Amz-Exprires may not exceed a week".to_string(),
|
||||
));
|
||||
}
|
||||
@@ -191,11 +190,10 @@ fn parse_query_authorization(
|
||||
let date = headers
|
||||
.get("x-amz-date")
|
||||
.ok_or_bad_request("Missing X-Amz-Date field")
|
||||
.map_err(Error::from)
|
||||
.and_then(|d| parse_date(d))?;
|
||||
|
||||
if Utc::now() - date > Duration::seconds(duration) {
|
||||
return Err(Error::bad_request("Date is too old".to_string()));
|
||||
return Err(Error::BadRequest("Date is too old".to_string()));
|
||||
}
|
||||
|
||||
Ok(Authorization {
|
||||
@@ -303,7 +301,7 @@ pub async fn verify_v4(
|
||||
.get(&EmptyKey, &key_id)
|
||||
.await?
|
||||
.filter(|k| !k.state.is_deleted())
|
||||
.ok_or_else(|| Error::forbidden(format!("No such key: {}", &key_id)))?;
|
||||
.ok_or_else(|| Error::Forbidden(format!("No such key: {}", &key_id)))?;
|
||||
let key_p = key.params().unwrap();
|
||||
|
||||
let mut hmac = signing_hmac(
|
||||
@@ -316,7 +314,7 @@ pub async fn verify_v4(
|
||||
hmac.update(payload);
|
||||
let our_signature = hex::encode(hmac.finalize().into_bytes());
|
||||
if signature != our_signature {
|
||||
return Err(Error::forbidden("Invalid signature".to_string()));
|
||||
return Err(Error::Forbidden("Invalid signature".to_string()));
|
||||
}
|
||||
|
||||
Ok(key)
|
||||
|
||||
@@ -12,7 +12,7 @@ use garage_util::data::Hash;
|
||||
|
||||
use super::{compute_scope, sha256sum, HmacSha256, LONG_DATETIME};
|
||||
|
||||
use crate::signature::error::*;
|
||||
use crate::error::*;
|
||||
|
||||
pub fn parse_streaming_body(
|
||||
api_key: &Key,
|
||||
@@ -87,7 +87,7 @@ fn compute_streaming_payload_signature(
|
||||
let mut hmac = signing_hmac.clone();
|
||||
hmac.update(string_to_sign.as_bytes());
|
||||
|
||||
Ok(Hash::try_from(&hmac.finalize().into_bytes()).ok_or_internal_error("Invalid signature")?)
|
||||
Hash::try_from(&hmac.finalize().into_bytes()).ok_or_internal_error("Invalid signature")
|
||||
}
|
||||
|
||||
mod payload {
|
||||
@@ -163,10 +163,10 @@ impl From<SignedPayloadStreamError> for Error {
|
||||
match err {
|
||||
SignedPayloadStreamError::Stream(e) => e,
|
||||
SignedPayloadStreamError::InvalidSignature => {
|
||||
Error::bad_request("Invalid payload signature")
|
||||
Error::BadRequest("Invalid payload signature".into())
|
||||
}
|
||||
SignedPayloadStreamError::Message(e) => {
|
||||
Error::bad_request(format!("Chunk format error: {}", e))
|
||||
Error::BadRequest(format!("Chunk format error: {}", e))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -27,8 +27,10 @@ garage_rpc = { version = "0.7.0", path = "../rpc" }
|
||||
garage_table = { version = "0.7.0", path = "../table" }
|
||||
garage_util = { version = "0.7.0", path = "../util" }
|
||||
garage_web = { version = "0.7.0", path = "../web" }
|
||||
garage_admin = { version = "0.7.0", path = "../admin" }
|
||||
|
||||
bytes = "1.0"
|
||||
git-version = "0.3.4"
|
||||
hex = "0.4"
|
||||
tracing = { version = "0.1.30", features = ["log-always"] }
|
||||
pretty_env_logger = "0.4"
|
||||
@@ -52,11 +54,6 @@ tokio = { version = "1.0", default-features = false, features = ["rt", "rt-multi
|
||||
#netapp = { version = "0.4", path = "../../../netapp" }
|
||||
netapp = "0.4"
|
||||
|
||||
opentelemetry = { version = "0.17", features = [ "rt-tokio" ] }
|
||||
opentelemetry-prometheus = "0.10"
|
||||
opentelemetry-otlp = "0.10"
|
||||
prometheus = "0.13"
|
||||
|
||||
[dev-dependencies]
|
||||
aws-sdk-s3 = "0.8"
|
||||
chrono = "0.4"
|
||||
|
||||
+54
-23
@@ -22,6 +22,7 @@ use garage_model::helper::error::{Error, OkOrBadRequest};
|
||||
use garage_model::key_table::*;
|
||||
use garage_model::migrate::Migrate;
|
||||
use garage_model::permission::*;
|
||||
use garage_model::s3::object_table::ObjectFilter;
|
||||
|
||||
use crate::cli::*;
|
||||
use crate::repair::Repair;
|
||||
@@ -212,7 +213,18 @@ impl AdminRpcHandler {
|
||||
}
|
||||
|
||||
// Check bucket is empty
|
||||
if !helper.is_bucket_empty(bucket_id).await? {
|
||||
let objects = self
|
||||
.garage
|
||||
.object_table
|
||||
.get_range(
|
||||
&bucket_id,
|
||||
None,
|
||||
Some(ObjectFilter::IsData),
|
||||
10,
|
||||
EnumerationOrder::Forward,
|
||||
)
|
||||
.await?;
|
||||
if !objects.is_empty() {
|
||||
return Err(Error::BadRequest(format!(
|
||||
"Bucket {} is not empty",
|
||||
query.name
|
||||
@@ -249,7 +261,6 @@ impl AdminRpcHandler {
|
||||
|
||||
async fn handle_alias_bucket(&self, query: &AliasBucketOpt) -> Result<AdminRpc, Error> {
|
||||
let helper = self.garage.bucket_helper();
|
||||
let key_helper = self.garage.key_helper();
|
||||
|
||||
let bucket_id = helper
|
||||
.resolve_global_bucket_name(&query.existing_bucket)
|
||||
@@ -257,7 +268,7 @@ impl AdminRpcHandler {
|
||||
.ok_or_bad_request("Bucket not found")?;
|
||||
|
||||
if let Some(key_pattern) = &query.local {
|
||||
let key = key_helper.get_existing_matching_key(key_pattern).await?;
|
||||
let key = helper.get_existing_matching_key(key_pattern).await?;
|
||||
|
||||
helper
|
||||
.set_local_bucket_alias(bucket_id, &key.key_id, &query.new_name)
|
||||
@@ -279,10 +290,9 @@ impl AdminRpcHandler {
|
||||
|
||||
async fn handle_unalias_bucket(&self, query: &UnaliasBucketOpt) -> Result<AdminRpc, Error> {
|
||||
let helper = self.garage.bucket_helper();
|
||||
let key_helper = self.garage.key_helper();
|
||||
|
||||
if let Some(key_pattern) = &query.local {
|
||||
let key = key_helper.get_existing_matching_key(key_pattern).await?;
|
||||
let key = helper.get_existing_matching_key(key_pattern).await?;
|
||||
|
||||
let bucket_id = key
|
||||
.state
|
||||
@@ -321,15 +331,12 @@ impl AdminRpcHandler {
|
||||
|
||||
async fn handle_bucket_allow(&self, query: &PermBucketOpt) -> Result<AdminRpc, Error> {
|
||||
let helper = self.garage.bucket_helper();
|
||||
let key_helper = self.garage.key_helper();
|
||||
|
||||
let bucket_id = helper
|
||||
.resolve_global_bucket_name(&query.bucket)
|
||||
.await?
|
||||
.ok_or_bad_request("Bucket not found")?;
|
||||
let key = key_helper
|
||||
.get_existing_matching_key(&query.key_pattern)
|
||||
.await?;
|
||||
let key = helper.get_existing_matching_key(&query.key_pattern).await?;
|
||||
|
||||
let allow_read = query.read || key.allow_read(&bucket_id);
|
||||
let allow_write = query.write || key.allow_write(&bucket_id);
|
||||
@@ -356,15 +363,12 @@ impl AdminRpcHandler {
|
||||
|
||||
async fn handle_bucket_deny(&self, query: &PermBucketOpt) -> Result<AdminRpc, Error> {
|
||||
let helper = self.garage.bucket_helper();
|
||||
let key_helper = self.garage.key_helper();
|
||||
|
||||
let bucket_id = helper
|
||||
.resolve_global_bucket_name(&query.bucket)
|
||||
.await?
|
||||
.ok_or_bad_request("Bucket not found")?;
|
||||
let key = key_helper
|
||||
.get_existing_matching_key(&query.key_pattern)
|
||||
.await?;
|
||||
let key = helper.get_existing_matching_key(&query.key_pattern).await?;
|
||||
|
||||
let allow_read = !query.read && key.allow_read(&bucket_id);
|
||||
let allow_write = !query.write && key.allow_write(&bucket_id);
|
||||
@@ -465,7 +469,7 @@ impl AdminRpcHandler {
|
||||
async fn handle_key_info(&self, query: &KeyOpt) -> Result<AdminRpc, Error> {
|
||||
let key = self
|
||||
.garage
|
||||
.key_helper()
|
||||
.bucket_helper()
|
||||
.get_existing_matching_key(&query.key_pattern)
|
||||
.await?;
|
||||
self.key_info_result(key).await
|
||||
@@ -480,7 +484,7 @@ impl AdminRpcHandler {
|
||||
async fn handle_rename_key(&self, query: &KeyRenameOpt) -> Result<AdminRpc, Error> {
|
||||
let mut key = self
|
||||
.garage
|
||||
.key_helper()
|
||||
.bucket_helper()
|
||||
.get_existing_matching_key(&query.key_pattern)
|
||||
.await?;
|
||||
key.params_mut()
|
||||
@@ -492,11 +496,9 @@ impl AdminRpcHandler {
|
||||
}
|
||||
|
||||
async fn handle_delete_key(&self, query: &KeyDeleteOpt) -> Result<AdminRpc, Error> {
|
||||
let key_helper = self.garage.key_helper();
|
||||
let helper = self.garage.bucket_helper();
|
||||
|
||||
let mut key = key_helper
|
||||
.get_existing_matching_key(&query.key_pattern)
|
||||
.await?;
|
||||
let mut key = helper.get_existing_matching_key(&query.key_pattern).await?;
|
||||
|
||||
if !query.yes {
|
||||
return Err(Error::BadRequest(
|
||||
@@ -504,7 +506,32 @@ impl AdminRpcHandler {
|
||||
));
|
||||
}
|
||||
|
||||
key_helper.delete_key(&mut key).await?;
|
||||
let state = key.state.as_option_mut().unwrap();
|
||||
|
||||
// --- done checking, now commit ---
|
||||
// (the step at unset_local_bucket_alias will fail if a bucket
|
||||
// does not have another alias, the deletion will be
|
||||
// interrupted in the middle if that happens)
|
||||
|
||||
// 1. Delete local aliases
|
||||
for (alias, _, to) in state.local_aliases.items().iter() {
|
||||
if let Some(bucket_id) = to {
|
||||
helper
|
||||
.unset_local_bucket_alias(*bucket_id, &key.key_id, alias)
|
||||
.await?;
|
||||
}
|
||||
}
|
||||
|
||||
// 2. Remove permissions on all authorized buckets
|
||||
for (ab_id, _auth) in state.authorized_buckets.items().iter() {
|
||||
helper
|
||||
.set_bucket_key_permissions(*ab_id, &key.key_id, BucketKeyPerm::NO_PERMISSIONS)
|
||||
.await?;
|
||||
}
|
||||
|
||||
// 3. Actually delete key
|
||||
key.state = Deletable::delete();
|
||||
self.garage.key_table.insert(&key).await?;
|
||||
|
||||
Ok(AdminRpc::Ok(format!(
|
||||
"Key {} was deleted successfully.",
|
||||
@@ -515,7 +542,7 @@ impl AdminRpcHandler {
|
||||
async fn handle_allow_key(&self, query: &KeyPermOpt) -> Result<AdminRpc, Error> {
|
||||
let mut key = self
|
||||
.garage
|
||||
.key_helper()
|
||||
.bucket_helper()
|
||||
.get_existing_matching_key(&query.key_pattern)
|
||||
.await?;
|
||||
if query.create_bucket {
|
||||
@@ -528,7 +555,7 @@ impl AdminRpcHandler {
|
||||
async fn handle_deny_key(&self, query: &KeyPermOpt) -> Result<AdminRpc, Error> {
|
||||
let mut key = self
|
||||
.garage
|
||||
.key_helper()
|
||||
.bucket_helper()
|
||||
.get_existing_matching_key(&query.key_pattern)
|
||||
.await?;
|
||||
if query.create_bucket {
|
||||
@@ -669,7 +696,11 @@ impl AdminRpcHandler {
|
||||
writeln!(
|
||||
&mut ret,
|
||||
"\nGarage version: {}",
|
||||
self.garage.system.garage_version(),
|
||||
option_env!("GIT_VERSION").unwrap_or(git_version::git_version!(
|
||||
prefix = "git:",
|
||||
cargo_prefix = "cargo:",
|
||||
fallback = "unknown"
|
||||
))
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
use std::collections::HashSet;
|
||||
|
||||
use garage_util::error::*;
|
||||
use garage_util::formater::format_table;
|
||||
|
||||
use garage_rpc::layout::*;
|
||||
use garage_rpc::system::*;
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
use garage_util::crdt::Crdt;
|
||||
use garage_util::data::*;
|
||||
use garage_util::error::*;
|
||||
use garage_util::formater::format_table;
|
||||
|
||||
use garage_rpc::layout::*;
|
||||
use garage_rpc::system::*;
|
||||
@@ -211,9 +211,31 @@ pub async fn cmd_apply_layout(
|
||||
rpc_host: NodeID,
|
||||
apply_opt: ApplyLayoutOpt,
|
||||
) -> Result<(), Error> {
|
||||
let layout = fetch_layout(rpc_cli, rpc_host).await?;
|
||||
let mut layout = fetch_layout(rpc_cli, rpc_host).await?;
|
||||
|
||||
let layout = layout.apply_staged_changes(apply_opt.version)?;
|
||||
match apply_opt.version {
|
||||
None => {
|
||||
println!("Please pass the --version flag to ensure that you are writing the correct version of the cluster layout.");
|
||||
println!("To know the correct value of the --version flag, invoke `garage layout show` and review the proposed changes.");
|
||||
return Err(Error::Message("--version flag is missing".into()));
|
||||
}
|
||||
Some(v) => {
|
||||
if v != layout.version + 1 {
|
||||
return Err(Error::Message("Invalid value of --version flag".into()));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
layout.roles.merge(&layout.staging);
|
||||
|
||||
if !layout.calculate_partition_assignation() {
|
||||
return Err(Error::Message("Could not calculate new assignation of partitions to nodes. This can happen if there are less nodes than the desired number of copies of your data (see the replication_mode configuration parameter).".into()));
|
||||
}
|
||||
|
||||
layout.staging.clear();
|
||||
layout.staging_hash = blake2sum(&rmp_to_vec_all_named(&layout.staging).unwrap()[..]);
|
||||
|
||||
layout.version += 1;
|
||||
|
||||
send_layout(rpc_cli, rpc_host, layout).await?;
|
||||
|
||||
@@ -228,9 +250,25 @@ pub async fn cmd_revert_layout(
|
||||
rpc_host: NodeID,
|
||||
revert_opt: RevertLayoutOpt,
|
||||
) -> Result<(), Error> {
|
||||
let layout = fetch_layout(rpc_cli, rpc_host).await?;
|
||||
let mut layout = fetch_layout(rpc_cli, rpc_host).await?;
|
||||
|
||||
let layout = layout.revert_staged_changes(revert_opt.version)?;
|
||||
match revert_opt.version {
|
||||
None => {
|
||||
println!("Please pass the --version flag to ensure that you are writing the correct version of the cluster layout.");
|
||||
println!("To know the correct value of the --version flag, invoke `garage layout show` and review the proposed changes.");
|
||||
return Err(Error::Message("--version flag is missing".into()));
|
||||
}
|
||||
Some(v) => {
|
||||
if v != layout.version + 1 {
|
||||
return Err(Error::Message("Invalid value of --version flag".into()));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
layout.staging.clear();
|
||||
layout.staging_hash = blake2sum(&rmp_to_vec_all_named(&layout.staging).unwrap()[..]);
|
||||
|
||||
layout.version += 1;
|
||||
|
||||
send_layout(rpc_cli, rpc_host, layout).await?;
|
||||
|
||||
|
||||
+29
-1
@@ -3,7 +3,6 @@ use std::collections::HashMap;
|
||||
use garage_util::crdt::*;
|
||||
use garage_util::data::Uuid;
|
||||
use garage_util::error::*;
|
||||
use garage_util::formater::format_table;
|
||||
|
||||
use garage_model::bucket_table::*;
|
||||
use garage_model::key_table::*;
|
||||
@@ -174,6 +173,35 @@ pub fn print_bucket_info(bucket: &Bucket, relevant_keys: &HashMap<String, Key>)
|
||||
};
|
||||
}
|
||||
|
||||
pub fn format_table(data: Vec<String>) {
|
||||
let data = data
|
||||
.iter()
|
||||
.map(|s| s.split('\t').collect::<Vec<_>>())
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
let columns = data.iter().map(|row| row.len()).fold(0, std::cmp::max);
|
||||
let mut column_size = vec![0; columns];
|
||||
|
||||
let mut out = String::new();
|
||||
|
||||
for row in data.iter() {
|
||||
for (i, col) in row.iter().enumerate() {
|
||||
column_size[i] = std::cmp::max(column_size[i], col.chars().count());
|
||||
}
|
||||
}
|
||||
|
||||
for row in data.iter() {
|
||||
for (col, col_len) in row[..row.len() - 1].iter().zip(column_size.iter()) {
|
||||
out.push_str(col);
|
||||
(0..col_len - col.chars().count() + 2).for_each(|_| out.push(' '));
|
||||
}
|
||||
out.push_str(row[row.len() - 1]);
|
||||
out.push('\n');
|
||||
}
|
||||
|
||||
print!("{}", out);
|
||||
}
|
||||
|
||||
pub fn find_matching_node(
|
||||
cand: impl std::iter::Iterator<Item = Uuid>,
|
||||
pattern: &str,
|
||||
|
||||
@@ -8,7 +8,6 @@ mod admin;
|
||||
mod cli;
|
||||
mod repair;
|
||||
mod server;
|
||||
mod tracing_setup;
|
||||
|
||||
use std::net::SocketAddr;
|
||||
use std::path::PathBuf;
|
||||
@@ -142,7 +141,6 @@ async fn cli_command(opt: Opt) -> Result<(), Error> {
|
||||
match cli_command_dispatch(opt.cmd, &system_rpc_endpoint, &admin_rpc_endpoint, id).await {
|
||||
Err(HelperError::Internal(i)) => Err(Error::Message(format!("Internal error: {}", i))),
|
||||
Err(HelperError::BadRequest(b)) => Err(Error::Message(b)),
|
||||
Err(e) => Err(Error::Message(format!("{}", e))),
|
||||
Ok(x) => Ok(x),
|
||||
}
|
||||
}
|
||||
|
||||
+17
-19
@@ -6,7 +6,8 @@ use garage_util::background::*;
|
||||
use garage_util::config::*;
|
||||
use garage_util::error::Error;
|
||||
|
||||
use garage_api::admin::api_server::AdminApiServer;
|
||||
use garage_admin::metrics::*;
|
||||
use garage_admin::tracing_setup::*;
|
||||
use garage_api::s3::api_server::S3ApiServer;
|
||||
use garage_model::garage::Garage;
|
||||
use garage_web::run_web_server;
|
||||
@@ -15,7 +16,6 @@ use garage_web::run_web_server;
|
||||
use garage_api::k2v::api_server::K2VApiServer;
|
||||
|
||||
use crate::admin::*;
|
||||
use crate::tracing_setup::*;
|
||||
|
||||
async fn wait_from(mut chan: watch::Receiver<bool>) {
|
||||
while !*chan.borrow() {
|
||||
@@ -39,6 +39,9 @@ pub async fn run_server(config_file: PathBuf) -> Result<(), Error> {
|
||||
.open()
|
||||
.expect("Unable to open sled DB");
|
||||
|
||||
info!("Initialize admin web server and metric backend...");
|
||||
let admin_server_init = AdminServer::init();
|
||||
|
||||
info!("Initializing background runner...");
|
||||
let watch_cancel = netapp::util::watch_ctrl_c();
|
||||
let (background, await_background_done) = BackgroundRunner::new(16, watch_cancel.clone());
|
||||
@@ -51,9 +54,6 @@ pub async fn run_server(config_file: PathBuf) -> Result<(), Error> {
|
||||
init_tracing(&export_to, garage.system.id)?;
|
||||
}
|
||||
|
||||
info!("Initialize Admin API server and metrics collector...");
|
||||
let admin_server = AdminApiServer::new(garage.clone());
|
||||
|
||||
let run_system = tokio::spawn(garage.system.clone().run(watch_cancel.clone()));
|
||||
|
||||
info!("Create admin RPC handler...");
|
||||
@@ -80,41 +80,39 @@ pub async fn run_server(config_file: PathBuf) -> Result<(), Error> {
|
||||
wait_from(watch_cancel.clone()),
|
||||
));
|
||||
|
||||
info!("Launching Admin API server...");
|
||||
let admin_server = tokio::spawn(admin_server.run(wait_from(watch_cancel.clone())));
|
||||
let admin_server = if let Some(admin_bind_addr) = config.admin.api_bind_addr {
|
||||
info!("Configure and run admin web server...");
|
||||
Some(tokio::spawn(
|
||||
admin_server_init.run(admin_bind_addr, wait_from(watch_cancel.clone())),
|
||||
))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
// Stuff runs
|
||||
|
||||
// When a cancel signal is sent, stuff stops
|
||||
if let Err(e) = s3_api_server.await? {
|
||||
warn!("S3 API server exited with error: {}", e);
|
||||
} else {
|
||||
info!("S3 API server exited without error.");
|
||||
}
|
||||
#[cfg(feature = "k2v")]
|
||||
if let Err(e) = k2v_api_server.await? {
|
||||
warn!("K2V API server exited with error: {}", e);
|
||||
} else {
|
||||
info!("K2V API server exited without error.");
|
||||
}
|
||||
if let Err(e) = web_server.await? {
|
||||
warn!("Web server exited with error: {}", e);
|
||||
} else {
|
||||
info!("Web server exited without error.");
|
||||
}
|
||||
if let Err(e) = admin_server.await? {
|
||||
warn!("Admin web server exited with error: {}", e);
|
||||
} else {
|
||||
info!("Admin API server exited without error.");
|
||||
if let Some(a) = admin_server {
|
||||
if let Err(e) = a.await? {
|
||||
warn!("Admin web server exited with error: {}", e);
|
||||
}
|
||||
}
|
||||
|
||||
// Remove RPC handlers for system to break reference cycles
|
||||
garage.system.netapp.drop_all_handlers();
|
||||
opentelemetry::global::shutdown_tracer_provider();
|
||||
|
||||
// Await for netapp RPC system to end
|
||||
run_system.await?;
|
||||
info!("Netapp exited");
|
||||
|
||||
// Drop all references so that stuff can terminate properly
|
||||
drop(garage);
|
||||
|
||||
@@ -92,9 +92,7 @@ async fn test_batch() {
|
||||
br#"[
|
||||
{"partitionKey": "root"},
|
||||
{"partitionKey": "root", "start": "c"},
|
||||
{"partitionKey": "root", "start": "c", "end": "dynamite"},
|
||||
{"partitionKey": "root", "start": "c", "reverse": true, "end": "a"},
|
||||
{"partitionKey": "root", "start": "c", "reverse": true, "end": "azerty"},
|
||||
{"partitionKey": "root", "limit": 1},
|
||||
{"partitionKey": "root", "prefix": "d"}
|
||||
]"#
|
||||
@@ -149,24 +147,6 @@ async fn test_batch() {
|
||||
"more": false,
|
||||
"nextStart": null,
|
||||
},
|
||||
{
|
||||
"partitionKey": "root",
|
||||
"prefix": null,
|
||||
"start": "c",
|
||||
"end": "dynamite",
|
||||
"limit": null,
|
||||
"reverse": false,
|
||||
"conflictsOnly": false,
|
||||
"tombstones": false,
|
||||
"singleItem": false,
|
||||
"items": [
|
||||
{"sk": "c", "ct": ct.get("c").unwrap(), "v": [base64::encode(values.get("c").unwrap())]},
|
||||
{"sk": "d.1", "ct": ct.get("d.1").unwrap(), "v": [base64::encode(values.get("d.1").unwrap())]},
|
||||
{"sk": "d.2", "ct": ct.get("d.2").unwrap(), "v": [base64::encode(values.get("d.2").unwrap())]},
|
||||
],
|
||||
"more": false,
|
||||
"nextStart": null,
|
||||
},
|
||||
{
|
||||
"partitionKey": "root",
|
||||
"prefix": null,
|
||||
@@ -184,23 +164,6 @@ async fn test_batch() {
|
||||
"more": false,
|
||||
"nextStart": null,
|
||||
},
|
||||
{
|
||||
"partitionKey": "root",
|
||||
"prefix": null,
|
||||
"start": "c",
|
||||
"end": "azerty",
|
||||
"limit": null,
|
||||
"reverse": true,
|
||||
"conflictsOnly": false,
|
||||
"tombstones": false,
|
||||
"singleItem": false,
|
||||
"items": [
|
||||
{"sk": "c", "ct": ct.get("c").unwrap(), "v": [base64::encode(values.get("c").unwrap())]},
|
||||
{"sk": "b", "ct": ct.get("b").unwrap(), "v": [base64::encode(values.get("b").unwrap())]},
|
||||
],
|
||||
"more": false,
|
||||
"nextStart": null,
|
||||
},
|
||||
{
|
||||
"partitionKey": "root",
|
||||
"prefix": null,
|
||||
@@ -502,34 +465,6 @@ async fn test_batch() {
|
||||
])
|
||||
);
|
||||
|
||||
// update our known tombstones
|
||||
for sk in ["a", "b", "d.1", "d.2"] {
|
||||
let res = ctx
|
||||
.k2v
|
||||
.request
|
||||
.builder(bucket.clone())
|
||||
.path("root")
|
||||
.query_param("sort_key", Some(sk))
|
||||
.signed_header("accept", "application/octet-stream")
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(res.status(), 204);
|
||||
assert_eq!(
|
||||
res.headers().get("content-type").unwrap().to_str().unwrap(),
|
||||
"application/octet-stream"
|
||||
);
|
||||
ct.insert(
|
||||
sk,
|
||||
res.headers()
|
||||
.get("x-garage-causality-token")
|
||||
.unwrap()
|
||||
.to_str()
|
||||
.unwrap()
|
||||
.to_string(),
|
||||
);
|
||||
}
|
||||
|
||||
let res = ctx
|
||||
.k2v
|
||||
.request
|
||||
@@ -538,8 +473,7 @@ async fn test_batch() {
|
||||
.body(
|
||||
br#"[
|
||||
{"partitionKey": "root"},
|
||||
{"partitionKey": "root", "reverse": true},
|
||||
{"partitionKey": "root", "tombstones": true}
|
||||
{"partitionKey": "root", "reverse": true}
|
||||
]"#
|
||||
.to_vec(),
|
||||
)
|
||||
@@ -586,27 +520,6 @@ async fn test_batch() {
|
||||
"more": false,
|
||||
"nextStart": null,
|
||||
},
|
||||
{
|
||||
"partitionKey": "root",
|
||||
"prefix": null,
|
||||
"start": null,
|
||||
"end": null,
|
||||
"limit": null,
|
||||
"reverse": false,
|
||||
"conflictsOnly": false,
|
||||
"tombstones": true,
|
||||
"singleItem": false,
|
||||
"items": [
|
||||
{"sk": "a", "ct": ct.get("a").unwrap(), "v": [null]},
|
||||
{"sk": "b", "ct": ct.get("b").unwrap(), "v": [null]},
|
||||
{"sk": "c", "ct": ct.get("c").unwrap(), "v": [base64::encode(values.get("c").unwrap()), base64::encode(values.get("c'").unwrap())]},
|
||||
{"sk": "d.1", "ct": ct.get("d.1").unwrap(), "v": [null]},
|
||||
{"sk": "d.2", "ct": ct.get("d.2").unwrap(), "v": [null]},
|
||||
{"sk": "e", "ct": ct.get("e").unwrap(), "v": [base64::encode(values.get("e").unwrap())]},
|
||||
],
|
||||
"more": false,
|
||||
"nextStart": null,
|
||||
},
|
||||
])
|
||||
);
|
||||
}
|
||||
|
||||
@@ -1,27 +0,0 @@
|
||||
[package]
|
||||
name = "k2v-client"
|
||||
version = "0.1.0"
|
||||
edition = "2018"
|
||||
|
||||
[dependencies]
|
||||
base64 = "0.13.0"
|
||||
http = "0.2.6"
|
||||
rusoto_core = "0.48.0"
|
||||
rusoto_credential = "0.48.0"
|
||||
rusoto_signature = "0.48.0"
|
||||
serde = "1.0.137"
|
||||
serde_json = "1.0.81"
|
||||
thiserror = "1.0.31"
|
||||
tokio = "1.17.0"
|
||||
|
||||
# cli deps
|
||||
clap = { version = "3.1.18", optional = true, features = ["derive", "env"] }
|
||||
garage_util = { path = "../util", optional = true }
|
||||
|
||||
|
||||
[features]
|
||||
cli = ["clap", "tokio/fs", "tokio/io-std", "garage_util"]
|
||||
|
||||
[[bin]]
|
||||
name = "k2v-cli"
|
||||
required-features = ["cli"]
|
||||
@@ -1,25 +0,0 @@
|
||||
Example usage:
|
||||
```sh
|
||||
# all these values can be provided on the cli instead
|
||||
export AWS_ACCESS_KEY_ID=GK123456
|
||||
export AWS_SECRET_ACCESS_KEY=0123..789
|
||||
export AWS_REGION=garage
|
||||
export K2V_ENDPOINT=http://172.30.2.1:3903
|
||||
export K2V_BUCKET=my-bucket
|
||||
|
||||
cargo run --features=cli -- read-range my-partition-key --all
|
||||
|
||||
cargo run --features=cli -- insert my-partition-key my-sort-key --text "my string1"
|
||||
cargo run --features=cli -- insert my-partition-key my-sort-key --text "my string2"
|
||||
cargo run --features=cli -- insert my-partition-key my-sort-key2 --text "my string"
|
||||
|
||||
cargo run --features=cli -- read-range my-partition-key --all
|
||||
|
||||
causality=$(cargo run --features=cli -- read my-partition-key my-sort-key2 -b | head -n1)
|
||||
cargo run --features=cli -- delete my-partition-key my-sort-key2 -c $causality
|
||||
|
||||
causality=$(cargo run --features=cli -- read my-partition-key my-sort-key -b | head -n1)
|
||||
cargo run --features=cli -- insert my-partition-key my-sort-key --text "my string3" -c $causality
|
||||
|
||||
cargo run --features=cli -- read-range my-partition-key --all
|
||||
```
|
||||
@@ -1,466 +0,0 @@
|
||||
use k2v_client::*;
|
||||
|
||||
use garage_util::formater::format_table;
|
||||
|
||||
use rusoto_core::credential::AwsCredentials;
|
||||
use rusoto_core::Region;
|
||||
|
||||
use clap::{Parser, Subcommand};
|
||||
|
||||
/// K2V command line interface
|
||||
#[derive(Parser, Debug)]
|
||||
#[clap(author, version, about, long_about = None)]
|
||||
struct Args {
|
||||
/// Name of the region to use
|
||||
#[clap(short, long, env = "AWS_REGION", default_value = "garage")]
|
||||
region: String,
|
||||
/// Url of the endpoint to connect to
|
||||
#[clap(short, long, env = "K2V_ENDPOINT")]
|
||||
endpoint: String,
|
||||
/// Access key ID
|
||||
#[clap(short, long, env = "AWS_ACCESS_KEY_ID")]
|
||||
key_id: String,
|
||||
/// Access key ID
|
||||
#[clap(short, long, env = "AWS_SECRET_ACCESS_KEY")]
|
||||
secret: String,
|
||||
/// Bucket name
|
||||
#[clap(short, long, env = "K2V_BUCKET")]
|
||||
bucket: String,
|
||||
#[clap(subcommand)]
|
||||
command: Command,
|
||||
}
|
||||
|
||||
#[derive(Subcommand, Debug)]
|
||||
enum Command {
|
||||
/// Insert a single value
|
||||
Insert {
|
||||
/// Partition key to insert to
|
||||
partition_key: String,
|
||||
/// Sort key to insert to
|
||||
sort_key: String,
|
||||
/// Causality of the insertion
|
||||
#[clap(short, long)]
|
||||
causality: Option<String>,
|
||||
/// Value to insert
|
||||
#[clap(flatten)]
|
||||
value: Value,
|
||||
},
|
||||
/// Read a single value
|
||||
Read {
|
||||
/// Partition key to read from
|
||||
partition_key: String,
|
||||
/// Sort key to read from
|
||||
sort_key: String,
|
||||
/// Output formating
|
||||
#[clap(flatten)]
|
||||
output_kind: ReadOutputKind,
|
||||
},
|
||||
/// Delete a single value
|
||||
Delete {
|
||||
/// Partition key to delete from
|
||||
partition_key: String,
|
||||
/// Sort key to delete from
|
||||
sort_key: String,
|
||||
/// Causality information
|
||||
#[clap(short, long)]
|
||||
causality: String,
|
||||
},
|
||||
/// List partition keys
|
||||
ReadIndex {
|
||||
/// Output formating
|
||||
#[clap(flatten)]
|
||||
output_kind: BatchOutputKind,
|
||||
/// Output only partition keys matching this filter
|
||||
#[clap(flatten)]
|
||||
filter: Filter,
|
||||
},
|
||||
/// Read a range of sort keys
|
||||
ReadRange {
|
||||
/// Partition key to read from
|
||||
partition_key: String,
|
||||
/// Output formating
|
||||
#[clap(flatten)]
|
||||
output_kind: BatchOutputKind,
|
||||
/// Output only sort keys matching this filter
|
||||
#[clap(flatten)]
|
||||
filter: Filter,
|
||||
},
|
||||
/// Delete a range of sort keys
|
||||
DeleteRange {
|
||||
/// Partition key to delete from
|
||||
partition_key: String,
|
||||
/// Output formating
|
||||
#[clap(flatten)]
|
||||
output_kind: BatchOutputKind,
|
||||
/// Delete only sort keys matching this filter
|
||||
#[clap(flatten)]
|
||||
filter: Filter,
|
||||
},
|
||||
}
|
||||
|
||||
/// Where to read a value from
|
||||
#[derive(Parser, Debug)]
|
||||
#[clap(group = clap::ArgGroup::new("value").multiple(false).required(true))]
|
||||
struct Value {
|
||||
/// Read value from a file. use - to read from stdin
|
||||
#[clap(short, long, group = "value")]
|
||||
file: Option<String>,
|
||||
/// Read a base64 value from commandline
|
||||
#[clap(short, long, group = "value")]
|
||||
b64: Option<String>,
|
||||
/// Read a raw (UTF-8) value from the commandline
|
||||
#[clap(short, long, group = "value")]
|
||||
text: Option<String>,
|
||||
}
|
||||
|
||||
impl Value {
|
||||
async fn to_data(&self) -> Result<Vec<u8>, Error> {
|
||||
if let Some(ref text) = self.text {
|
||||
Ok(text.as_bytes().to_vec())
|
||||
} else if let Some(ref b64) = self.b64 {
|
||||
base64::decode(b64).map_err(|_| Error::Message("invalid base64 input".into()))
|
||||
} else if let Some(ref path) = self.file {
|
||||
use tokio::io::AsyncReadExt;
|
||||
if path == "-" {
|
||||
let mut file = tokio::io::stdin();
|
||||
let mut vec = Vec::new();
|
||||
file.read_to_end(&mut vec).await?;
|
||||
Ok(vec)
|
||||
} else {
|
||||
let mut file = tokio::fs::File::open(path).await?;
|
||||
let mut vec = Vec::new();
|
||||
file.read_to_end(&mut vec).await?;
|
||||
Ok(vec)
|
||||
}
|
||||
} else {
|
||||
unreachable!("Value must have one option set")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Parser, Debug)]
|
||||
#[clap(group = clap::ArgGroup::new("output-kind").multiple(false).required(false))]
|
||||
struct ReadOutputKind {
|
||||
/// Base64 output. Conflicts are line separated, first line is causality token
|
||||
#[clap(short, long, group = "output-kind")]
|
||||
b64: bool,
|
||||
/// Raw output. Conflicts generate error, causality token is not returned
|
||||
#[clap(short, long, group = "output-kind")]
|
||||
raw: bool,
|
||||
/// Human formated output
|
||||
#[clap(short = 'H', long, group = "output-kind")]
|
||||
human: bool,
|
||||
/// JSON formated output
|
||||
#[clap(short, long, group = "output-kind")]
|
||||
json: bool,
|
||||
}
|
||||
|
||||
impl ReadOutputKind {
|
||||
fn display_output(&self, val: CausalValue) -> ! {
|
||||
use std::io::Write;
|
||||
use std::process::exit;
|
||||
|
||||
if self.json {
|
||||
let stdout = std::io::stdout();
|
||||
serde_json::to_writer_pretty(stdout, &val).unwrap();
|
||||
exit(0);
|
||||
}
|
||||
|
||||
if self.raw {
|
||||
let mut val = val.value;
|
||||
if val.len() != 1 {
|
||||
eprintln!(
|
||||
"Raw mode can only read non-concurent values, found {} values, expected 1",
|
||||
val.len()
|
||||
);
|
||||
exit(1);
|
||||
}
|
||||
let val = val.pop().unwrap();
|
||||
match val {
|
||||
K2vValue::Value(v) => {
|
||||
std::io::stdout().write_all(&v).unwrap();
|
||||
exit(0);
|
||||
}
|
||||
K2vValue::Tombstone => {
|
||||
eprintln!("Expected value, found tombstone");
|
||||
exit(2);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let causality: String = val.causality.into();
|
||||
if self.b64 {
|
||||
println!("{}", causality);
|
||||
for val in val.value {
|
||||
match val {
|
||||
K2vValue::Value(v) => {
|
||||
println!("{}", base64::encode(&v))
|
||||
}
|
||||
K2vValue::Tombstone => {
|
||||
println!();
|
||||
}
|
||||
}
|
||||
}
|
||||
exit(0);
|
||||
}
|
||||
|
||||
// human
|
||||
println!("causality: {}", causality);
|
||||
println!("values:");
|
||||
for val in val.value {
|
||||
match val {
|
||||
K2vValue::Value(v) => {
|
||||
if let Ok(string) = std::str::from_utf8(&v) {
|
||||
println!(" utf-8: {}", string);
|
||||
} else {
|
||||
println!(" base64: {}", base64::encode(&v));
|
||||
}
|
||||
}
|
||||
K2vValue::Tombstone => {
|
||||
println!(" tombstone");
|
||||
}
|
||||
}
|
||||
}
|
||||
exit(0);
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Parser, Debug)]
|
||||
#[clap(group = clap::ArgGroup::new("output-kind").multiple(false).required(false))]
|
||||
struct BatchOutputKind {
|
||||
/// Human formated output
|
||||
#[clap(short = 'H', long, group = "output-kind")]
|
||||
human: bool,
|
||||
/// JSON formated output
|
||||
#[clap(short, long, group = "output-kind")]
|
||||
json: bool,
|
||||
}
|
||||
|
||||
/// Filter for batch operations
|
||||
#[derive(Parser, Debug)]
|
||||
#[clap(group = clap::ArgGroup::new("filter").multiple(true).required(true))]
|
||||
struct Filter {
|
||||
/// Match only keys starting with this prefix
|
||||
#[clap(short, long, group = "filter")]
|
||||
prefix: Option<String>,
|
||||
/// Match only keys lexicographically after this key (including this key itself)
|
||||
#[clap(short, long, group = "filter")]
|
||||
start: Option<String>,
|
||||
/// Match only keys lexicographically before this key (excluding this key)
|
||||
#[clap(short, long, group = "filter")]
|
||||
end: Option<String>,
|
||||
/// Only match the first X keys
|
||||
#[clap(short, long)]
|
||||
limit: Option<u64>,
|
||||
/// Return keys in reverse order
|
||||
#[clap(short, long)]
|
||||
reverse: bool,
|
||||
/// Return only keys where conflict happened
|
||||
#[clap(short, long)]
|
||||
conflicts_only: bool,
|
||||
/// Also include keys storing only tombstones
|
||||
#[clap(short, long)]
|
||||
tombstones: bool,
|
||||
/// Return any key
|
||||
#[clap(short, long, group = "filter")]
|
||||
all: bool,
|
||||
}
|
||||
|
||||
impl Filter {
|
||||
fn k2v_filter(&self) -> k2v_client::Filter<'_> {
|
||||
k2v_client::Filter {
|
||||
start: self.start.as_deref(),
|
||||
end: self.end.as_deref(),
|
||||
prefix: self.prefix.as_deref(),
|
||||
limit: self.limit,
|
||||
reverse: self.reverse,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() -> Result<(), Error> {
|
||||
let args = Args::parse();
|
||||
|
||||
let region = Region::Custom {
|
||||
name: args.region,
|
||||
endpoint: args.endpoint,
|
||||
};
|
||||
|
||||
let creds = AwsCredentials::new(args.key_id, args.secret, None, None);
|
||||
|
||||
let client = K2vClient::new(region, args.bucket, creds, None)?;
|
||||
|
||||
match args.command {
|
||||
Command::Insert {
|
||||
partition_key,
|
||||
sort_key,
|
||||
causality,
|
||||
value,
|
||||
} => {
|
||||
client
|
||||
.insert_item(
|
||||
&partition_key,
|
||||
&sort_key,
|
||||
value.to_data().await?,
|
||||
causality.map(Into::into),
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
Command::Delete {
|
||||
partition_key,
|
||||
sort_key,
|
||||
causality,
|
||||
} => {
|
||||
client
|
||||
.delete_item(&partition_key, &sort_key, causality.into())
|
||||
.await?;
|
||||
}
|
||||
Command::Read {
|
||||
partition_key,
|
||||
sort_key,
|
||||
output_kind,
|
||||
} => {
|
||||
let res = client.read_item(&partition_key, &sort_key).await?;
|
||||
output_kind.display_output(res);
|
||||
}
|
||||
Command::ReadIndex {
|
||||
output_kind,
|
||||
filter,
|
||||
} => {
|
||||
if filter.conflicts_only || filter.tombstones {
|
||||
return Err(Error::Message(
|
||||
"conlicts-only and tombstones are invalid for read-index".into(),
|
||||
));
|
||||
}
|
||||
let res = client.read_index(filter.k2v_filter()).await?;
|
||||
if output_kind.json {
|
||||
let values = res
|
||||
.items
|
||||
.into_iter()
|
||||
.map(|(k, v)| {
|
||||
let mut value = serde_json::to_value(v).unwrap();
|
||||
value
|
||||
.as_object_mut()
|
||||
.unwrap()
|
||||
.insert("sort_key".to_owned(), k.into());
|
||||
value
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
let json = serde_json::json!({
|
||||
"next_key": res.next_start,
|
||||
"values": values,
|
||||
});
|
||||
|
||||
let stdout = std::io::stdout();
|
||||
serde_json::to_writer_pretty(stdout, &json).unwrap();
|
||||
} else {
|
||||
if let Some(next) = res.next_start {
|
||||
println!("next key: {}", next);
|
||||
}
|
||||
|
||||
let mut to_print = Vec::new();
|
||||
to_print.push(format!("key:\tentries\tconflicts\tvalues\tbytes"));
|
||||
for (k, v) in res.items {
|
||||
to_print.push(format!(
|
||||
"{}\t{}\t{}\t{}\t{}",
|
||||
k, v.entries, v.conflicts, v.values, v.bytes
|
||||
));
|
||||
}
|
||||
format_table(to_print);
|
||||
}
|
||||
}
|
||||
Command::ReadRange {
|
||||
partition_key,
|
||||
output_kind,
|
||||
filter,
|
||||
} => {
|
||||
let op = BatchReadOp {
|
||||
partition_key: &partition_key,
|
||||
filter: filter.k2v_filter(),
|
||||
conflicts_only: filter.conflicts_only,
|
||||
tombstones: filter.tombstones,
|
||||
single_item: false,
|
||||
};
|
||||
let mut res = client.read_batch(&[op]).await?;
|
||||
let res = res.pop().unwrap();
|
||||
if output_kind.json {
|
||||
let values = res
|
||||
.items
|
||||
.into_iter()
|
||||
.map(|(k, v)| {
|
||||
let mut value = serde_json::to_value(v).unwrap();
|
||||
value
|
||||
.as_object_mut()
|
||||
.unwrap()
|
||||
.insert("sort_key".to_owned(), k.into());
|
||||
value
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
let json = serde_json::json!({
|
||||
"next_key": res.next_start,
|
||||
"values": values,
|
||||
});
|
||||
|
||||
let stdout = std::io::stdout();
|
||||
serde_json::to_writer_pretty(stdout, &json).unwrap();
|
||||
} else {
|
||||
if let Some(next) = res.next_start {
|
||||
println!("next key: {}", next);
|
||||
}
|
||||
for (key, values) in res.items {
|
||||
println!("key: {}", key);
|
||||
let causality: String = values.causality.into();
|
||||
println!("causality: {}", causality);
|
||||
for value in values.value {
|
||||
match value {
|
||||
K2vValue::Value(v) => {
|
||||
if let Ok(string) = std::str::from_utf8(&v) {
|
||||
println!(" value(utf-8): {}", string);
|
||||
} else {
|
||||
println!(" value(base64): {}", base64::encode(&v));
|
||||
}
|
||||
}
|
||||
K2vValue::Tombstone => {
|
||||
println!(" tombstone");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Command::DeleteRange {
|
||||
partition_key,
|
||||
output_kind,
|
||||
filter,
|
||||
} => {
|
||||
let op = BatchDeleteOp {
|
||||
partition_key: &partition_key,
|
||||
prefix: filter.prefix.as_deref(),
|
||||
start: filter.start.as_deref(),
|
||||
end: filter.end.as_deref(),
|
||||
single_item: false,
|
||||
};
|
||||
if filter.reverse
|
||||
|| filter.conflicts_only
|
||||
|| filter.tombstones
|
||||
|| filter.limit.is_some()
|
||||
{
|
||||
return Err(Error::Message(
|
||||
"limit, conlicts-only, reverse and tombstones are invalid for delete-range"
|
||||
.into(),
|
||||
));
|
||||
}
|
||||
|
||||
let res = client.delete_batch(&[op]).await?;
|
||||
|
||||
if output_kind.json {
|
||||
println!("{}", res[0]);
|
||||
} else {
|
||||
println!("deleted {} keys", res[0]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -1,22 +0,0 @@
|
||||
use std::borrow::Cow;
|
||||
|
||||
use thiserror::Error;
|
||||
|
||||
/// Errors returned by this crate
|
||||
#[derive(Error, Debug)]
|
||||
pub enum Error {
|
||||
#[error("received invalid response: {0}")]
|
||||
InvalidResponse(Cow<'static, str>),
|
||||
#[error("not found")]
|
||||
NotFound,
|
||||
#[error("io error: {0}")]
|
||||
IoError(#[from] std::io::Error),
|
||||
#[error("rusoto tls error: {0}")]
|
||||
RusotoTls(#[from] rusoto_core::request::TlsError),
|
||||
#[error("rusoto http error: {0}")]
|
||||
RusotoHttp(#[from] rusoto_core::HttpDispatchError),
|
||||
#[error("deserialization error: {0}")]
|
||||
Deserialization(#[from] serde_json::Error),
|
||||
#[error("{0}")]
|
||||
Message(Cow<'static, str>),
|
||||
}
|
||||
@@ -1,566 +0,0 @@
|
||||
use std::collections::BTreeMap;
|
||||
use std::time::Duration;
|
||||
|
||||
use http::header::{ACCEPT, CONTENT_LENGTH, CONTENT_TYPE};
|
||||
use http::status::StatusCode;
|
||||
use http::HeaderMap;
|
||||
|
||||
use rusoto_core::{ByteStream, DispatchSignedRequest, HttpClient};
|
||||
use rusoto_credential::AwsCredentials;
|
||||
use rusoto_signature::region::Region;
|
||||
use rusoto_signature::signature::SignedRequest;
|
||||
use serde::de::Error as DeError;
|
||||
use serde::{Deserialize, Deserializer, Serialize, Serializer};
|
||||
|
||||
use tokio::io::AsyncReadExt;
|
||||
|
||||
mod error;
|
||||
|
||||
pub use error::Error;
|
||||
|
||||
const DEFAULT_TIMEOUT: Duration = Duration::from_secs(5);
|
||||
const DEFAULT_POLL_TIMEOUT: Duration = Duration::from_secs(300);
|
||||
const SERVICE: &str = "k2v";
|
||||
const GARAGE_CAUSALITY_TOKEN: &str = "X-Garage-Causality-Token";
|
||||
|
||||
/// Client used to query a K2V server.
|
||||
pub struct K2vClient {
|
||||
region: Region,
|
||||
bucket: String,
|
||||
creds: AwsCredentials,
|
||||
client: HttpClient,
|
||||
}
|
||||
|
||||
impl K2vClient {
|
||||
/// Create a new K2V client.
|
||||
pub fn new(
|
||||
region: Region,
|
||||
bucket: String,
|
||||
creds: AwsCredentials,
|
||||
user_agent: Option<String>,
|
||||
) -> Result<Self, Error> {
|
||||
let mut client = HttpClient::new()?;
|
||||
if let Some(ua) = user_agent {
|
||||
client.local_agent_prepend(ua);
|
||||
} else {
|
||||
client.local_agent_prepend(format!("k2v/{}", env!("CARGO_PKG_VERSION")));
|
||||
}
|
||||
Ok(K2vClient {
|
||||
region,
|
||||
bucket,
|
||||
creds,
|
||||
client,
|
||||
})
|
||||
}
|
||||
|
||||
/// Perform a ReadItem request, reading the value(s) stored for a single pk+sk.
|
||||
pub async fn read_item(
|
||||
&self,
|
||||
partition_key: &str,
|
||||
sort_key: &str,
|
||||
) -> Result<CausalValue, Error> {
|
||||
let mut req = SignedRequest::new(
|
||||
"GET",
|
||||
SERVICE,
|
||||
&self.region,
|
||||
&format!("/{}/{}", self.bucket, partition_key),
|
||||
);
|
||||
req.add_param("sort_key", sort_key);
|
||||
req.add_header(ACCEPT, "application/octet-stream, application/json");
|
||||
|
||||
let res = self.dispatch(req, None).await?;
|
||||
|
||||
let causality = res
|
||||
.causality_token
|
||||
.ok_or_else(|| Error::InvalidResponse("missing causality token".into()))?;
|
||||
|
||||
if res.status == StatusCode::NO_CONTENT {
|
||||
return Ok(CausalValue {
|
||||
causality,
|
||||
value: vec![K2vValue::Tombstone],
|
||||
});
|
||||
}
|
||||
|
||||
match res.content_type.as_deref() {
|
||||
Some("application/octet-stream") => Ok(CausalValue {
|
||||
causality,
|
||||
value: vec![K2vValue::Value(res.body)],
|
||||
}),
|
||||
Some("application/json") => {
|
||||
let value = serde_json::from_slice(&res.body)?;
|
||||
Ok(CausalValue { causality, value })
|
||||
}
|
||||
Some(ct) => Err(Error::InvalidResponse(
|
||||
format!("invalid content type: {}", ct).into(),
|
||||
)),
|
||||
None => Err(Error::InvalidResponse("missing content type".into())),
|
||||
}
|
||||
}
|
||||
|
||||
/// Perform a PollItem request, waiting for the value(s) stored for a single pk+sk to be
|
||||
/// updated.
|
||||
pub async fn poll_item(
|
||||
&self,
|
||||
partition_key: &str,
|
||||
sort_key: &str,
|
||||
causality: CausalityToken,
|
||||
timeout: Option<Duration>,
|
||||
) -> Result<Option<CausalValue>, Error> {
|
||||
let timeout = timeout.unwrap_or(DEFAULT_POLL_TIMEOUT);
|
||||
|
||||
let mut req = SignedRequest::new(
|
||||
"GET",
|
||||
SERVICE,
|
||||
&self.region,
|
||||
&format!("/{}/{}", self.bucket, partition_key),
|
||||
);
|
||||
req.add_param("sort_key", sort_key);
|
||||
req.add_param("causality_token", &causality.0);
|
||||
req.add_param("timeout", &timeout.as_secs().to_string());
|
||||
req.add_header(ACCEPT, "application/octet-stream, application/json");
|
||||
|
||||
let res = self.dispatch(req, Some(timeout + DEFAULT_TIMEOUT)).await?;
|
||||
|
||||
let causality = res
|
||||
.causality_token
|
||||
.ok_or_else(|| Error::InvalidResponse("missing causality token".into()))?;
|
||||
|
||||
if res.status == StatusCode::NOT_MODIFIED {
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
if res.status == StatusCode::NO_CONTENT {
|
||||
return Ok(Some(CausalValue {
|
||||
causality,
|
||||
value: vec![K2vValue::Tombstone],
|
||||
}));
|
||||
}
|
||||
|
||||
match res.content_type.as_deref() {
|
||||
Some("application/octet-stream") => Ok(Some(CausalValue {
|
||||
causality,
|
||||
value: vec![K2vValue::Value(res.body)],
|
||||
})),
|
||||
Some("application/json") => {
|
||||
let value = serde_json::from_slice(&res.body)?;
|
||||
Ok(Some(CausalValue { causality, value }))
|
||||
}
|
||||
Some(ct) => Err(Error::InvalidResponse(
|
||||
format!("invalid content type: {}", ct).into(),
|
||||
)),
|
||||
None => Err(Error::InvalidResponse("missing content type".into())),
|
||||
}
|
||||
}
|
||||
|
||||
/// Perform an InsertItem request, inserting a value for a single pk+sk.
|
||||
pub async fn insert_item(
|
||||
&self,
|
||||
partition_key: &str,
|
||||
sort_key: &str,
|
||||
value: Vec<u8>,
|
||||
causality: Option<CausalityToken>,
|
||||
) -> Result<(), Error> {
|
||||
let mut req = SignedRequest::new(
|
||||
"PUT",
|
||||
SERVICE,
|
||||
&self.region,
|
||||
&format!("/{}/{}", self.bucket, partition_key),
|
||||
);
|
||||
req.add_param("sort_key", sort_key);
|
||||
req.set_payload(Some(value));
|
||||
|
||||
if let Some(causality) = causality {
|
||||
req.add_header(GARAGE_CAUSALITY_TOKEN, &causality.0);
|
||||
}
|
||||
|
||||
self.dispatch(req, None).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Perform a DeleteItem request, deleting the value(s) stored for a single pk+sk.
|
||||
pub async fn delete_item(
|
||||
&self,
|
||||
partition_key: &str,
|
||||
sort_key: &str,
|
||||
causality: CausalityToken,
|
||||
) -> Result<(), Error> {
|
||||
let mut req = SignedRequest::new(
|
||||
"DELETE",
|
||||
SERVICE,
|
||||
&self.region,
|
||||
&format!("/{}/{}", self.bucket, partition_key),
|
||||
);
|
||||
req.add_param("sort_key", sort_key);
|
||||
req.add_header(GARAGE_CAUSALITY_TOKEN, &causality.0);
|
||||
|
||||
self.dispatch(req, None).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Perform a ReadIndex request, listing partition key which have at least one associated
|
||||
/// sort key, and which matches the filter.
|
||||
pub async fn read_index(
|
||||
&self,
|
||||
filter: Filter<'_>,
|
||||
) -> Result<PaginatedRange<PartitionInfo>, Error> {
|
||||
let mut req =
|
||||
SignedRequest::new("GET", SERVICE, &self.region, &format!("/{}", self.bucket));
|
||||
filter.insert_params(&mut req);
|
||||
|
||||
let res = self.dispatch(req, None).await?;
|
||||
|
||||
let resp: ReadIndexResponse = serde_json::from_slice(&res.body)?;
|
||||
|
||||
let items = resp
|
||||
.partition_keys
|
||||
.into_iter()
|
||||
.map(|ReadIndexItem { pk, info }| (pk, info))
|
||||
.collect();
|
||||
|
||||
Ok(PaginatedRange {
|
||||
items,
|
||||
next_start: resp.next_start,
|
||||
})
|
||||
}
|
||||
|
||||
/// Perform an InsertBatch request, inserting multiple values at once. Note: this operation is
|
||||
/// *not* atomic: it is possible for some sub-operations to fails and others to success. In
|
||||
/// that case, failure is reported.
|
||||
pub async fn insert_batch(&self, operations: &[BatchInsertOp<'_>]) -> Result<(), Error> {
|
||||
let mut req =
|
||||
SignedRequest::new("POST", SERVICE, &self.region, &format!("/{}", self.bucket));
|
||||
|
||||
let payload = serde_json::to_vec(operations)?;
|
||||
req.set_payload(Some(payload));
|
||||
self.dispatch(req, None).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Perform a ReadBatch request, reading multiple values or range of values at once.
|
||||
pub async fn read_batch(
|
||||
&self,
|
||||
operations: &[BatchReadOp<'_>],
|
||||
) -> Result<Vec<PaginatedRange<CausalValue>>, Error> {
|
||||
let mut req =
|
||||
SignedRequest::new("POST", SERVICE, &self.region, &format!("/{}", self.bucket));
|
||||
req.add_param("search", "");
|
||||
|
||||
let payload = serde_json::to_vec(operations)?;
|
||||
req.set_payload(Some(payload));
|
||||
let res = self.dispatch(req, None).await?;
|
||||
|
||||
let resp: Vec<BatchReadResponse> = serde_json::from_slice(&res.body)?;
|
||||
|
||||
Ok(resp
|
||||
.into_iter()
|
||||
.map(|e| PaginatedRange {
|
||||
items: e
|
||||
.items
|
||||
.into_iter()
|
||||
.map(|BatchReadItem { sk, ct, v }| {
|
||||
(
|
||||
sk,
|
||||
CausalValue {
|
||||
causality: ct,
|
||||
value: v,
|
||||
},
|
||||
)
|
||||
})
|
||||
.collect(),
|
||||
next_start: e.next_start,
|
||||
})
|
||||
.collect())
|
||||
}
|
||||
|
||||
/// Perform a DeleteBatch request, deleting mutiple values or range of values at once, without
|
||||
/// providing causality information.
|
||||
pub async fn delete_batch(&self, operations: &[BatchDeleteOp<'_>]) -> Result<Vec<u64>, Error> {
|
||||
let mut req =
|
||||
SignedRequest::new("POST", SERVICE, &self.region, &format!("/{}", self.bucket));
|
||||
req.add_param("delete", "");
|
||||
|
||||
let payload = serde_json::to_vec(operations)?;
|
||||
req.set_payload(Some(payload));
|
||||
let res = self.dispatch(req, None).await?;
|
||||
|
||||
let resp: Vec<BatchDeleteResponse> = serde_json::from_slice(&res.body)?;
|
||||
|
||||
Ok(resp.into_iter().map(|r| r.deleted_items).collect())
|
||||
}
|
||||
|
||||
async fn dispatch(
|
||||
&self,
|
||||
mut req: SignedRequest,
|
||||
timeout: Option<Duration>,
|
||||
) -> Result<Response, Error> {
|
||||
req.sign(&self.creds);
|
||||
let mut res = self
|
||||
.client
|
||||
.dispatch(req, Some(timeout.unwrap_or(DEFAULT_TIMEOUT)))
|
||||
.await?;
|
||||
|
||||
let causality_token = res
|
||||
.headers
|
||||
.remove(GARAGE_CAUSALITY_TOKEN)
|
||||
.map(CausalityToken);
|
||||
let content_type = res.headers.remove(CONTENT_TYPE);
|
||||
|
||||
let body = match res.status {
|
||||
StatusCode::OK => read_body(&mut res.headers, res.body).await?,
|
||||
StatusCode::NO_CONTENT => Vec::new(),
|
||||
StatusCode::NOT_FOUND => return Err(Error::NotFound),
|
||||
StatusCode::NOT_MODIFIED => Vec::new(),
|
||||
_ => {
|
||||
return Err(Error::InvalidResponse(
|
||||
format!("invalid error code: {}", res.status).into(),
|
||||
))
|
||||
}
|
||||
};
|
||||
|
||||
Ok(Response {
|
||||
body,
|
||||
status: res.status,
|
||||
causality_token,
|
||||
content_type,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
async fn read_body(headers: &mut HeaderMap<String>, body: ByteStream) -> Result<Vec<u8>, Error> {
|
||||
let body_len = headers
|
||||
.get(CONTENT_LENGTH)
|
||||
.and_then(|h| h.parse().ok())
|
||||
.unwrap_or(0);
|
||||
let mut res = Vec::with_capacity(body_len);
|
||||
body.into_async_read().read_to_end(&mut res).await?;
|
||||
Ok(res)
|
||||
}
|
||||
|
||||
/// An opaque token used to convey causality between operations.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Deserialize, Serialize)]
|
||||
#[serde(transparent)]
|
||||
pub struct CausalityToken(String);
|
||||
|
||||
impl From<String> for CausalityToken {
|
||||
fn from(v: String) -> Self {
|
||||
CausalityToken(v)
|
||||
}
|
||||
}
|
||||
|
||||
impl From<CausalityToken> for String {
|
||||
fn from(v: CausalityToken) -> Self {
|
||||
v.0
|
||||
}
|
||||
}
|
||||
|
||||
/// A value in K2V. can be either a binary value, or a tombstone.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum K2vValue {
|
||||
Tombstone,
|
||||
Value(Vec<u8>),
|
||||
}
|
||||
|
||||
impl From<Vec<u8>> for K2vValue {
|
||||
fn from(v: Vec<u8>) -> Self {
|
||||
K2vValue::Value(v)
|
||||
}
|
||||
}
|
||||
|
||||
impl From<Option<Vec<u8>>> for K2vValue {
|
||||
fn from(v: Option<Vec<u8>>) -> Self {
|
||||
match v {
|
||||
Some(v) => K2vValue::Value(v),
|
||||
None => K2vValue::Tombstone,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<'de> Deserialize<'de> for K2vValue {
|
||||
fn deserialize<D>(d: D) -> Result<Self, D::Error>
|
||||
where
|
||||
D: Deserializer<'de>,
|
||||
{
|
||||
let val: Option<&str> = Option::deserialize(d)?;
|
||||
Ok(match val {
|
||||
Some(s) => {
|
||||
K2vValue::Value(base64::decode(s).map_err(|_| DeError::custom("invalid base64"))?)
|
||||
}
|
||||
None => K2vValue::Tombstone,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl Serialize for K2vValue {
|
||||
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
|
||||
where
|
||||
S: Serializer,
|
||||
{
|
||||
match self {
|
||||
K2vValue::Tombstone => serializer.serialize_none(),
|
||||
K2vValue::Value(v) => {
|
||||
let b64 = base64::encode(v);
|
||||
serializer.serialize_str(&b64)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A set of K2vValue and associated causality information.
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub struct CausalValue {
|
||||
pub causality: CausalityToken,
|
||||
pub value: Vec<K2vValue>,
|
||||
}
|
||||
|
||||
/// Result of paginated requests.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct PaginatedRange<V> {
|
||||
pub items: BTreeMap<String, V>,
|
||||
pub next_start: Option<String>,
|
||||
}
|
||||
|
||||
/// Filter for batch operations.
|
||||
#[derive(Debug, Default, Clone, Deserialize, Serialize)]
|
||||
pub struct Filter<'a> {
|
||||
pub start: Option<&'a str>,
|
||||
pub end: Option<&'a str>,
|
||||
pub prefix: Option<&'a str>,
|
||||
pub limit: Option<u64>,
|
||||
#[serde(default)]
|
||||
pub reverse: bool,
|
||||
}
|
||||
|
||||
impl<'a> Filter<'a> {
|
||||
fn insert_params(&self, req: &mut SignedRequest) {
|
||||
if let Some(start) = &self.start {
|
||||
req.add_param("start", start);
|
||||
}
|
||||
if let Some(end) = &self.end {
|
||||
req.add_param("end", end);
|
||||
}
|
||||
if let Some(prefix) = &self.prefix {
|
||||
req.add_param("prefix", prefix);
|
||||
}
|
||||
if let Some(limit) = &self.limit {
|
||||
req.add_param("limit", &limit.to_string());
|
||||
}
|
||||
if self.reverse {
|
||||
req.add_param("reverse", "true");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct ReadIndexResponse<'a> {
|
||||
#[serde(flatten, borrow)]
|
||||
#[allow(dead_code)]
|
||||
filter: Filter<'a>,
|
||||
partition_keys: Vec<ReadIndexItem>,
|
||||
#[allow(dead_code)]
|
||||
more: bool,
|
||||
next_start: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
struct ReadIndexItem {
|
||||
pk: String,
|
||||
#[serde(flatten)]
|
||||
info: PartitionInfo,
|
||||
}
|
||||
|
||||
/// Information about data stored with a given partition key.
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct PartitionInfo {
|
||||
pub entries: u64,
|
||||
pub conflicts: u64,
|
||||
pub values: u64,
|
||||
pub bytes: u64,
|
||||
}
|
||||
|
||||
/// Single sub-operation of an InsertBatch.
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub struct BatchInsertOp<'a> {
|
||||
#[serde(rename = "pk")]
|
||||
pub partition_key: &'a str,
|
||||
#[serde(rename = "sk")]
|
||||
pub sort_key: &'a str,
|
||||
#[serde(rename = "ct")]
|
||||
pub causality: Option<CausalityToken>,
|
||||
#[serde(rename = "v")]
|
||||
pub value: K2vValue,
|
||||
}
|
||||
|
||||
/// Single sub-operation of a ReadBatch.
|
||||
#[derive(Debug, Default, Clone, Deserialize, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct BatchReadOp<'a> {
|
||||
pub partition_key: &'a str,
|
||||
#[serde(flatten, borrow)]
|
||||
pub filter: Filter<'a>,
|
||||
#[serde(default)]
|
||||
pub single_item: bool,
|
||||
#[serde(default)]
|
||||
pub conflicts_only: bool,
|
||||
#[serde(default)]
|
||||
pub tombstones: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct BatchReadResponse<'a> {
|
||||
#[serde(flatten, borrow)]
|
||||
#[allow(dead_code)]
|
||||
op: BatchReadOp<'a>,
|
||||
items: Vec<BatchReadItem>,
|
||||
#[allow(dead_code)]
|
||||
more: bool,
|
||||
next_start: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
struct BatchReadItem {
|
||||
sk: String,
|
||||
ct: CausalityToken,
|
||||
v: Vec<K2vValue>,
|
||||
}
|
||||
|
||||
/// Single sub-operation of a DeleteBatch
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct BatchDeleteOp<'a> {
|
||||
pub partition_key: &'a str,
|
||||
pub prefix: Option<&'a str>,
|
||||
pub start: Option<&'a str>,
|
||||
pub end: Option<&'a str>,
|
||||
#[serde(default)]
|
||||
pub single_item: bool,
|
||||
}
|
||||
|
||||
impl<'a> BatchDeleteOp<'a> {
|
||||
pub fn new(partition_key: &'a str) -> Self {
|
||||
BatchDeleteOp {
|
||||
partition_key,
|
||||
prefix: None,
|
||||
start: None,
|
||||
end: None,
|
||||
single_item: false,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct BatchDeleteResponse<'a> {
|
||||
#[serde(flatten, borrow)]
|
||||
#[allow(dead_code)]
|
||||
filter: BatchDeleteOp<'a>,
|
||||
deleted_items: u64,
|
||||
}
|
||||
|
||||
struct Response {
|
||||
body: Vec<u8>,
|
||||
status: StatusCode,
|
||||
causality_token: Option<CausalityToken>,
|
||||
content_type: Option<String>,
|
||||
}
|
||||
@@ -191,10 +191,6 @@ impl Garage {
|
||||
pub fn bucket_helper(&self) -> helper::bucket::BucketHelper {
|
||||
helper::bucket::BucketHelper(self)
|
||||
}
|
||||
|
||||
pub fn key_helper(&self) -> helper::key::KeyHelper {
|
||||
helper::key::KeyHelper(self)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "k2v")]
|
||||
|
||||
+73
-79
@@ -1,18 +1,15 @@
|
||||
use garage_table::util::*;
|
||||
use garage_util::crdt::*;
|
||||
use garage_util::data::*;
|
||||
use garage_util::error::{Error as GarageError, OkOrMessage};
|
||||
use garage_util::time::*;
|
||||
|
||||
use garage_table::util::*;
|
||||
|
||||
use crate::bucket_alias_table::*;
|
||||
use crate::bucket_table::*;
|
||||
use crate::garage::Garage;
|
||||
use crate::helper::error::*;
|
||||
use crate::helper::key::KeyHelper;
|
||||
use crate::key_table::*;
|
||||
use crate::key_table::{Key, KeyFilter};
|
||||
use crate::permission::BucketKeyPerm;
|
||||
use crate::s3::object_table::ObjectFilter;
|
||||
|
||||
pub struct BucketHelper<'a>(pub(crate) &'a Garage);
|
||||
|
||||
@@ -52,23 +49,6 @@ impl<'a> BucketHelper<'a> {
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(clippy::ptr_arg)]
|
||||
pub async fn resolve_bucket(&self, bucket_name: &String, api_key: &Key) -> Result<Uuid, Error> {
|
||||
let api_key_params = api_key
|
||||
.state
|
||||
.as_option()
|
||||
.ok_or_message("Key should not be deleted at this point")?;
|
||||
|
||||
if let Some(Some(bucket_id)) = api_key_params.local_aliases.get(bucket_name) {
|
||||
Ok(*bucket_id)
|
||||
} else {
|
||||
Ok(self
|
||||
.resolve_global_bucket_name(bucket_name)
|
||||
.await?
|
||||
.ok_or_else(|| Error::NoSuchBucket(bucket_name.to_string()))?)
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns a Bucket if it is present in bucket table,
|
||||
/// even if it is in deleted state. Querying a non-existing
|
||||
/// bucket ID returns an internal error.
|
||||
@@ -91,7 +71,64 @@ impl<'a> BucketHelper<'a> {
|
||||
.get(&EmptyKey, &bucket_id)
|
||||
.await?
|
||||
.filter(|b| !b.is_deleted())
|
||||
.ok_or_else(|| Error::NoSuchBucket(hex::encode(bucket_id)))
|
||||
.ok_or_bad_request(format!(
|
||||
"Bucket {:?} does not exist or has been deleted",
|
||||
bucket_id
|
||||
))
|
||||
}
|
||||
|
||||
/// Returns a Key if it is present in key table,
|
||||
/// even if it is in deleted state. Querying a non-existing
|
||||
/// key ID returns an internal error.
|
||||
pub async fn get_internal_key(&self, key_id: &String) -> Result<Key, Error> {
|
||||
Ok(self
|
||||
.0
|
||||
.key_table
|
||||
.get(&EmptyKey, key_id)
|
||||
.await?
|
||||
.ok_or_message(format!("Key {} does not exist", key_id))?)
|
||||
}
|
||||
|
||||
/// Returns a Key if it is present in key table,
|
||||
/// only if it is in non-deleted state.
|
||||
/// Querying a non-existing key ID or a deleted key
|
||||
/// returns a bad request error.
|
||||
pub async fn get_existing_key(&self, key_id: &String) -> Result<Key, Error> {
|
||||
self.0
|
||||
.key_table
|
||||
.get(&EmptyKey, key_id)
|
||||
.await?
|
||||
.filter(|b| !b.state.is_deleted())
|
||||
.ok_or_bad_request(format!("Key {} does not exist or has been deleted", key_id))
|
||||
}
|
||||
|
||||
/// Returns a Key if it is present in key table,
|
||||
/// looking it up by key ID or by a match on its name,
|
||||
/// only if it is in non-deleted state.
|
||||
/// Querying a non-existing key ID or a deleted key
|
||||
/// returns a bad request error.
|
||||
pub async fn get_existing_matching_key(&self, pattern: &str) -> Result<Key, Error> {
|
||||
let candidates = self
|
||||
.0
|
||||
.key_table
|
||||
.get_range(
|
||||
&EmptyKey,
|
||||
None,
|
||||
Some(KeyFilter::MatchesAndNotDeleted(pattern.to_string())),
|
||||
10,
|
||||
EnumerationOrder::Forward,
|
||||
)
|
||||
.await?
|
||||
.into_iter()
|
||||
.collect::<Vec<_>>();
|
||||
if candidates.len() != 1 {
|
||||
Err(Error::BadRequest(format!(
|
||||
"{} matching keys",
|
||||
candidates.len()
|
||||
)))
|
||||
} else {
|
||||
Ok(candidates.into_iter().next().unwrap())
|
||||
}
|
||||
}
|
||||
|
||||
/// Sets a new alias for a bucket in global namespace.
|
||||
@@ -105,7 +142,10 @@ impl<'a> BucketHelper<'a> {
|
||||
alias_name: &String,
|
||||
) -> Result<(), Error> {
|
||||
if !is_valid_bucket_name(alias_name) {
|
||||
return Err(Error::InvalidBucketName(alias_name.to_string()));
|
||||
return Err(Error::BadRequest(format!(
|
||||
"{}: {}",
|
||||
alias_name, INVALID_BUCKET_NAME_MESSAGE
|
||||
)));
|
||||
}
|
||||
|
||||
let mut bucket = self.get_existing_bucket(bucket_id).await?;
|
||||
@@ -136,7 +176,7 @@ impl<'a> BucketHelper<'a> {
|
||||
|
||||
let alias = match alias {
|
||||
None => BucketAlias::new(alias_name.clone(), alias_ts, Some(bucket_id))
|
||||
.ok_or_else(|| Error::InvalidBucketName(alias_name.clone()))?,
|
||||
.ok_or_bad_request(format!("{}: {}", alias_name, INVALID_BUCKET_NAME_MESSAGE))?,
|
||||
Some(mut a) => {
|
||||
a.state = Lww::raw(alias_ts, Some(bucket_id));
|
||||
a
|
||||
@@ -224,7 +264,7 @@ impl<'a> BucketHelper<'a> {
|
||||
.bucket_alias_table
|
||||
.get(&EmptyKey, alias_name)
|
||||
.await?
|
||||
.ok_or_else(|| Error::NoSuchBucket(alias_name.to_string()))?;
|
||||
.ok_or_message(format!("Alias {} not found", alias_name))?;
|
||||
|
||||
// Checks ok, remove alias
|
||||
let alias_ts = match bucket.state.as_option() {
|
||||
@@ -263,14 +303,15 @@ impl<'a> BucketHelper<'a> {
|
||||
key_id: &String,
|
||||
alias_name: &String,
|
||||
) -> Result<(), Error> {
|
||||
let key_helper = KeyHelper(self.0);
|
||||
|
||||
if !is_valid_bucket_name(alias_name) {
|
||||
return Err(Error::InvalidBucketName(alias_name.to_string()));
|
||||
return Err(Error::BadRequest(format!(
|
||||
"{}: {}",
|
||||
alias_name, INVALID_BUCKET_NAME_MESSAGE
|
||||
)));
|
||||
}
|
||||
|
||||
let mut bucket = self.get_existing_bucket(bucket_id).await?;
|
||||
let mut key = key_helper.get_existing_key(key_id).await?;
|
||||
let mut key = self.get_existing_key(key_id).await?;
|
||||
|
||||
let mut key_param = key.state.as_option_mut().unwrap();
|
||||
|
||||
@@ -319,10 +360,8 @@ impl<'a> BucketHelper<'a> {
|
||||
key_id: &String,
|
||||
alias_name: &String,
|
||||
) -> Result<(), Error> {
|
||||
let key_helper = KeyHelper(self.0);
|
||||
|
||||
let mut bucket = self.get_existing_bucket(bucket_id).await?;
|
||||
let mut key = key_helper.get_existing_key(key_id).await?;
|
||||
let mut key = self.get_existing_key(key_id).await?;
|
||||
|
||||
let mut bucket_p = bucket.state.as_option_mut().unwrap();
|
||||
|
||||
@@ -390,10 +429,8 @@ impl<'a> BucketHelper<'a> {
|
||||
key_id: &String,
|
||||
mut perm: BucketKeyPerm,
|
||||
) -> Result<(), Error> {
|
||||
let key_helper = KeyHelper(self.0);
|
||||
|
||||
let mut bucket = self.get_internal_bucket(bucket_id).await?;
|
||||
let mut key = key_helper.get_internal_key(key_id).await?;
|
||||
let mut key = self.get_internal_key(key_id).await?;
|
||||
|
||||
if let Some(bstate) = bucket.state.as_option() {
|
||||
if let Some(kp) = bstate.authorized_keys.get(key_id) {
|
||||
@@ -429,47 +466,4 @@ impl<'a> BucketHelper<'a> {
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn is_bucket_empty(&self, bucket_id: Uuid) -> Result<bool, Error> {
|
||||
let objects = self
|
||||
.0
|
||||
.object_table
|
||||
.get_range(
|
||||
&bucket_id,
|
||||
None,
|
||||
Some(ObjectFilter::IsData),
|
||||
10,
|
||||
EnumerationOrder::Forward,
|
||||
)
|
||||
.await?;
|
||||
if !objects.is_empty() {
|
||||
return Ok(false);
|
||||
}
|
||||
|
||||
#[cfg(feature = "k2v")]
|
||||
{
|
||||
use garage_rpc::ring::Ring;
|
||||
use std::sync::Arc;
|
||||
|
||||
let ring: Arc<Ring> = self.0.system.ring.borrow().clone();
|
||||
let k2vindexes = self
|
||||
.0
|
||||
.k2v
|
||||
.counter_table
|
||||
.table
|
||||
.get_range(
|
||||
&bucket_id,
|
||||
None,
|
||||
Some((DeletedFilter::NotDeleted, ring.layout.node_id_vec.clone())),
|
||||
10,
|
||||
EnumerationOrder::Forward,
|
||||
)
|
||||
.await?;
|
||||
if !k2vindexes.is_empty() {
|
||||
return Ok(false);
|
||||
}
|
||||
}
|
||||
|
||||
Ok(true)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -10,16 +10,6 @@ pub enum Error {
|
||||
|
||||
#[error(display = "Bad request: {}", _0)]
|
||||
BadRequest(String),
|
||||
|
||||
/// Bucket name is not valid according to AWS S3 specs
|
||||
#[error(display = "Invalid bucket name: {}", _0)]
|
||||
InvalidBucketName(String),
|
||||
|
||||
#[error(display = "Access key not found: {}", _0)]
|
||||
NoSuchAccessKey(String),
|
||||
|
||||
#[error(display = "Bucket not found: {}", _0)]
|
||||
NoSuchBucket(String),
|
||||
}
|
||||
|
||||
impl From<netapp::error::Error> for Error {
|
||||
|
||||
@@ -1,102 +0,0 @@
|
||||
use garage_table::util::*;
|
||||
use garage_util::crdt::*;
|
||||
use garage_util::error::OkOrMessage;
|
||||
|
||||
use crate::garage::Garage;
|
||||
use crate::helper::bucket::BucketHelper;
|
||||
use crate::helper::error::*;
|
||||
use crate::key_table::{Key, KeyFilter};
|
||||
use crate::permission::BucketKeyPerm;
|
||||
|
||||
pub struct KeyHelper<'a>(pub(crate) &'a Garage);
|
||||
|
||||
#[allow(clippy::ptr_arg)]
|
||||
impl<'a> KeyHelper<'a> {
|
||||
/// Returns a Key if it is present in key table,
|
||||
/// even if it is in deleted state. Querying a non-existing
|
||||
/// key ID returns an internal error.
|
||||
pub async fn get_internal_key(&self, key_id: &String) -> Result<Key, Error> {
|
||||
Ok(self
|
||||
.0
|
||||
.key_table
|
||||
.get(&EmptyKey, key_id)
|
||||
.await?
|
||||
.ok_or_message(format!("Key {} does not exist", key_id))?)
|
||||
}
|
||||
|
||||
/// Returns a Key if it is present in key table,
|
||||
/// only if it is in non-deleted state.
|
||||
/// Querying a non-existing key ID or a deleted key
|
||||
/// returns a bad request error.
|
||||
pub async fn get_existing_key(&self, key_id: &String) -> Result<Key, Error> {
|
||||
self.0
|
||||
.key_table
|
||||
.get(&EmptyKey, key_id)
|
||||
.await?
|
||||
.filter(|b| !b.state.is_deleted())
|
||||
.ok_or_else(|| Error::NoSuchAccessKey(key_id.to_string()))
|
||||
}
|
||||
|
||||
/// Returns a Key if it is present in key table,
|
||||
/// looking it up by key ID or by a match on its name,
|
||||
/// only if it is in non-deleted state.
|
||||
/// Querying a non-existing key ID or a deleted key
|
||||
/// returns a bad request error.
|
||||
pub async fn get_existing_matching_key(&self, pattern: &str) -> Result<Key, Error> {
|
||||
let candidates = self
|
||||
.0
|
||||
.key_table
|
||||
.get_range(
|
||||
&EmptyKey,
|
||||
None,
|
||||
Some(KeyFilter::MatchesAndNotDeleted(pattern.to_string())),
|
||||
10,
|
||||
EnumerationOrder::Forward,
|
||||
)
|
||||
.await?
|
||||
.into_iter()
|
||||
.collect::<Vec<_>>();
|
||||
if candidates.len() != 1 {
|
||||
Err(Error::BadRequest(format!(
|
||||
"{} matching keys",
|
||||
candidates.len()
|
||||
)))
|
||||
} else {
|
||||
Ok(candidates.into_iter().next().unwrap())
|
||||
}
|
||||
}
|
||||
|
||||
/// Deletes an API access key
|
||||
pub async fn delete_key(&self, key: &mut Key) -> Result<(), Error> {
|
||||
let bucket_helper = BucketHelper(self.0);
|
||||
|
||||
let state = key.state.as_option_mut().unwrap();
|
||||
|
||||
// --- done checking, now commit ---
|
||||
// (the step at unset_local_bucket_alias will fail if a bucket
|
||||
// does not have another alias, the deletion will be
|
||||
// interrupted in the middle if that happens)
|
||||
|
||||
// 1. Delete local aliases
|
||||
for (alias, _, to) in state.local_aliases.items().iter() {
|
||||
if let Some(bucket_id) = to {
|
||||
bucket_helper
|
||||
.unset_local_bucket_alias(*bucket_id, &key.key_id, alias)
|
||||
.await?;
|
||||
}
|
||||
}
|
||||
|
||||
// 2. Remove permissions on all authorized buckets
|
||||
for (ab_id, _auth) in state.authorized_buckets.items().iter() {
|
||||
bucket_helper
|
||||
.set_bucket_key_permissions(*ab_id, &key.key_id, BucketKeyPerm::NO_PERMISSIONS)
|
||||
.await?;
|
||||
}
|
||||
|
||||
// 3. Actually delete key
|
||||
key.state = Deletable::delete();
|
||||
self.0.key_table.insert(key).await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -1,3 +1,2 @@
|
||||
pub mod bucket;
|
||||
pub mod error;
|
||||
pub mod key;
|
||||
|
||||
+1
-1
@@ -15,11 +15,11 @@ path = "lib.rs"
|
||||
|
||||
[dependencies]
|
||||
garage_util = { version = "0.7.0", path = "../util" }
|
||||
garage_admin = { version = "0.7.0", path = "../admin" }
|
||||
|
||||
arc-swap = "1.0"
|
||||
bytes = "1.0"
|
||||
gethostname = "0.2"
|
||||
git-version = "0.3.4"
|
||||
hex = "0.4"
|
||||
tracing = "0.1.30"
|
||||
rand = "0.8"
|
||||
|
||||
@@ -5,7 +5,6 @@ use serde::{Deserialize, Serialize};
|
||||
|
||||
use garage_util::crdt::{AutoCrdt, Crdt, LwwMap};
|
||||
use garage_util::data::*;
|
||||
use garage_util::error::*;
|
||||
|
||||
use crate::ring::*;
|
||||
|
||||
@@ -101,61 +100,6 @@ impl ClusterLayout {
|
||||
}
|
||||
}
|
||||
|
||||
pub fn apply_staged_changes(mut self, version: Option<u64>) -> Result<Self, Error> {
|
||||
match version {
|
||||
None => {
|
||||
let error = r#"
|
||||
Please pass the new layout version number to ensure that you are writing the correct version of the cluster layout.
|
||||
To know the correct value of the new layout version, invoke `garage layout show` and review the proposed changes.
|
||||
"#;
|
||||
return Err(Error::Message(error.into()));
|
||||
}
|
||||
Some(v) => {
|
||||
if v != self.version + 1 {
|
||||
return Err(Error::Message("Invalid new layout version".into()));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
self.roles.merge(&self.staging);
|
||||
self.roles.retain(|(_, _, v)| v.0.is_some());
|
||||
|
||||
if !self.calculate_partition_assignation() {
|
||||
return Err(Error::Message("Could not calculate new assignation of partitions to nodes. This can happen if there are less nodes than the desired number of copies of your data (see the replication_mode configuration parameter).".into()));
|
||||
}
|
||||
|
||||
self.staging.clear();
|
||||
self.staging_hash = blake2sum(&rmp_to_vec_all_named(&self.staging).unwrap()[..]);
|
||||
|
||||
self.version += 1;
|
||||
|
||||
Ok(self)
|
||||
}
|
||||
|
||||
pub fn revert_staged_changes(mut self, version: Option<u64>) -> Result<Self, Error> {
|
||||
match version {
|
||||
None => {
|
||||
let error = r#"
|
||||
Please pass the new layout version number to ensure that you are writing the correct version of the cluster layout.
|
||||
To know the correct value of the new layout version, invoke `garage layout show` and review the proposed changes.
|
||||
"#;
|
||||
return Err(Error::Message(error.into()));
|
||||
}
|
||||
Some(v) => {
|
||||
if v != self.version + 1 {
|
||||
return Err(Error::Message("Invalid new layout version".into()));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
self.staging.clear();
|
||||
self.staging_hash = blake2sum(&rmp_to_vec_all_named(&self.staging).unwrap()[..]);
|
||||
|
||||
self.version += 1;
|
||||
|
||||
Ok(self)
|
||||
}
|
||||
|
||||
/// Returns a list of IDs of nodes that currently have
|
||||
/// a role in the cluster
|
||||
pub fn node_ids(&self) -> &[Uuid] {
|
||||
|
||||
+48
-84
@@ -312,84 +312,6 @@ impl System {
|
||||
);
|
||||
}
|
||||
|
||||
// ---- Administrative operations (directly available and
|
||||
// also available through RPC) ----
|
||||
|
||||
pub fn garage_version(&self) -> &'static str {
|
||||
option_env!("GIT_VERSION").unwrap_or(git_version::git_version!(
|
||||
prefix = "git:",
|
||||
cargo_prefix = "cargo:",
|
||||
fallback = "unknown"
|
||||
))
|
||||
}
|
||||
|
||||
pub fn get_known_nodes(&self) -> Vec<KnownNodeInfo> {
|
||||
let node_status = self.node_status.read().unwrap();
|
||||
let known_nodes = self
|
||||
.fullmesh
|
||||
.get_peer_list()
|
||||
.iter()
|
||||
.map(|n| KnownNodeInfo {
|
||||
id: n.id.into(),
|
||||
addr: n.addr,
|
||||
is_up: n.is_up(),
|
||||
last_seen_secs_ago: n.last_seen.map(|t| (Instant::now() - t).as_secs()),
|
||||
status: node_status
|
||||
.get(&n.id.into())
|
||||
.cloned()
|
||||
.map(|(_, st)| st)
|
||||
.unwrap_or(NodeStatus {
|
||||
hostname: "?".to_string(),
|
||||
replication_factor: 0,
|
||||
cluster_layout_version: 0,
|
||||
cluster_layout_staging_hash: Hash::from([0u8; 32]),
|
||||
}),
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
known_nodes
|
||||
}
|
||||
|
||||
pub fn get_cluster_layout(&self) -> ClusterLayout {
|
||||
self.ring.borrow().layout.clone()
|
||||
}
|
||||
|
||||
pub async fn update_cluster_layout(
|
||||
self: &Arc<Self>,
|
||||
layout: &ClusterLayout,
|
||||
) -> Result<(), Error> {
|
||||
self.handle_advertise_cluster_layout(layout).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn connect(&self, node: &str) -> Result<(), Error> {
|
||||
let (pubkey, addrs) = parse_and_resolve_peer_addr(node).ok_or_else(|| {
|
||||
Error::Message(format!(
|
||||
"Unable to parse or resolve node specification: {}",
|
||||
node
|
||||
))
|
||||
})?;
|
||||
let mut errors = vec![];
|
||||
for ip in addrs.iter() {
|
||||
match self
|
||||
.netapp
|
||||
.clone()
|
||||
.try_connect(*ip, pubkey)
|
||||
.await
|
||||
.err_context(CONNECT_ERROR_MESSAGE)
|
||||
{
|
||||
Ok(()) => return Ok(()),
|
||||
Err(e) => {
|
||||
errors.push((*ip, e));
|
||||
}
|
||||
}
|
||||
}
|
||||
if errors.len() == 1 {
|
||||
Err(Error::Message(errors[0].1.to_string()))
|
||||
} else {
|
||||
Err(Error::Message(format!("{:?}", errors)))
|
||||
}
|
||||
}
|
||||
|
||||
// ---- INTERNALS ----
|
||||
|
||||
async fn advertise_to_consul(self: Arc<Self>) -> Result<(), Error> {
|
||||
@@ -462,11 +384,32 @@ impl System {
|
||||
self.local_status.swap(Arc::new(new_si));
|
||||
}
|
||||
|
||||
// --- RPC HANDLERS ---
|
||||
|
||||
async fn handle_connect(&self, node: &str) -> Result<SystemRpc, Error> {
|
||||
self.connect(node).await?;
|
||||
Ok(SystemRpc::Ok)
|
||||
let (pubkey, addrs) = parse_and_resolve_peer_addr(node).ok_or_else(|| {
|
||||
Error::Message(format!(
|
||||
"Unable to parse or resolve node specification: {}",
|
||||
node
|
||||
))
|
||||
})?;
|
||||
let mut errors = vec![];
|
||||
for ip in addrs.iter() {
|
||||
match self
|
||||
.netapp
|
||||
.clone()
|
||||
.try_connect(*ip, pubkey)
|
||||
.await
|
||||
.err_context(CONNECT_ERROR_MESSAGE)
|
||||
{
|
||||
Ok(()) => return Ok(SystemRpc::Ok),
|
||||
Err(e) => {
|
||||
errors.push((*ip, e));
|
||||
}
|
||||
}
|
||||
}
|
||||
return Err(Error::Message(format!(
|
||||
"Could not connect to specified peers. Errors: {:?}",
|
||||
errors
|
||||
)));
|
||||
}
|
||||
|
||||
fn handle_pull_cluster_layout(&self) -> SystemRpc {
|
||||
@@ -475,7 +418,28 @@ impl System {
|
||||
}
|
||||
|
||||
fn handle_get_known_nodes(&self) -> SystemRpc {
|
||||
let known_nodes = self.get_known_nodes();
|
||||
let node_status = self.node_status.read().unwrap();
|
||||
let known_nodes = self
|
||||
.fullmesh
|
||||
.get_peer_list()
|
||||
.iter()
|
||||
.map(|n| KnownNodeInfo {
|
||||
id: n.id.into(),
|
||||
addr: n.addr,
|
||||
is_up: n.is_up(),
|
||||
last_seen_secs_ago: n.last_seen.map(|t| (Instant::now() - t).as_secs()),
|
||||
status: node_status
|
||||
.get(&n.id.into())
|
||||
.cloned()
|
||||
.map(|(_, st)| st)
|
||||
.unwrap_or(NodeStatus {
|
||||
hostname: "?".to_string(),
|
||||
replication_factor: 0,
|
||||
cluster_layout_version: 0,
|
||||
cluster_layout_staging_hash: Hash::from([0u8; 32]),
|
||||
}),
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
SystemRpc::ReturnKnownNodes(known_nodes)
|
||||
}
|
||||
|
||||
@@ -512,7 +476,7 @@ impl System {
|
||||
}
|
||||
|
||||
async fn handle_advertise_cluster_layout(
|
||||
self: &Arc<Self>,
|
||||
self: Arc<Self>,
|
||||
adv: &ClusterLayout,
|
||||
) -> Result<SystemRpc, Error> {
|
||||
let update_ring = self.update_ring.lock().await;
|
||||
|
||||
+15
-22
@@ -6,9 +6,7 @@ use std::time::Duration;
|
||||
|
||||
use futures::future::*;
|
||||
use futures::select;
|
||||
use futures::stream::FuturesUnordered;
|
||||
use futures::StreamExt;
|
||||
use tokio::sync::{mpsc, mpsc::error::TryRecvError, watch, Mutex};
|
||||
use tokio::sync::{mpsc, watch, Mutex};
|
||||
|
||||
use crate::error::Error;
|
||||
|
||||
@@ -32,31 +30,26 @@ impl BackgroundRunner {
|
||||
|
||||
let stop_signal_2 = stop_signal.clone();
|
||||
let await_all_done = tokio::spawn(async move {
|
||||
let mut workers = FuturesUnordered::new();
|
||||
let mut shutdown_timer = 0;
|
||||
loop {
|
||||
let closed = match worker_out.try_recv() {
|
||||
Ok(wkr) => {
|
||||
workers.push(wkr);
|
||||
false
|
||||
}
|
||||
Err(TryRecvError::Empty) => false,
|
||||
Err(TryRecvError::Disconnected) => true,
|
||||
};
|
||||
select! {
|
||||
res = workers.next() => {
|
||||
if let Some(Err(e)) = res {
|
||||
error!("Worker exited with error: {}", e);
|
||||
let wkr = {
|
||||
select! {
|
||||
item = worker_out.recv().fuse() => {
|
||||
match item {
|
||||
Some(x) => x,
|
||||
None => break,
|
||||
}
|
||||
}
|
||||
}
|
||||
_ = tokio::time::sleep(Duration::from_secs(1)).fuse() => {
|
||||
if closed || *stop_signal_2.borrow() {
|
||||
shutdown_timer += 1;
|
||||
if shutdown_timer >= 10 {
|
||||
_ = tokio::time::sleep(Duration::from_secs(5)).fuse() => {
|
||||
if *stop_signal_2.borrow() {
|
||||
break;
|
||||
} else {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
if let Err(e) = wkr.await {
|
||||
error!("Error while awaiting for worker: {}", e);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
@@ -121,10 +121,6 @@ pub struct WebConfig {
|
||||
pub struct AdminConfig {
|
||||
/// Address and port to bind for admin API serving
|
||||
pub api_bind_addr: Option<SocketAddr>,
|
||||
/// Bearer token to use to scrape metrics
|
||||
pub metrics_token: Option<String>,
|
||||
/// Bearer token to use to access Admin API endpoints
|
||||
pub admin_token: Option<String>,
|
||||
/// OTLP server to where to export traces
|
||||
pub trace_sink: Option<String>,
|
||||
}
|
||||
|
||||
@@ -140,11 +140,6 @@ where
|
||||
self.vals.clear();
|
||||
}
|
||||
|
||||
/// Retain only values that match a certain predicate
|
||||
pub fn retain(&mut self, pred: impl FnMut(&(K, u64, V)) -> bool) {
|
||||
self.vals.retain(pred);
|
||||
}
|
||||
|
||||
/// Get a reference to the value assigned to a key
|
||||
pub fn get(&self, k: &K) -> Option<&V> {
|
||||
match self.vals.binary_search_by(|(k2, _, _)| k2.cmp(k)) {
|
||||
|
||||
@@ -1,28 +0,0 @@
|
||||
pub fn format_table(data: Vec<String>) {
|
||||
let data = data
|
||||
.iter()
|
||||
.map(|s| s.split('\t').collect::<Vec<_>>())
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
let columns = data.iter().map(|row| row.len()).fold(0, std::cmp::max);
|
||||
let mut column_size = vec![0; columns];
|
||||
|
||||
let mut out = String::new();
|
||||
|
||||
for row in data.iter() {
|
||||
for (i, col) in row.iter().enumerate() {
|
||||
column_size[i] = std::cmp::max(column_size[i], col.chars().count());
|
||||
}
|
||||
}
|
||||
|
||||
for row in data.iter() {
|
||||
for (col, col_len) in row[..row.len() - 1].iter().zip(column_size.iter()) {
|
||||
out.push_str(col);
|
||||
(0..col_len - col.chars().count() + 2).for_each(|_| out.push(' '));
|
||||
}
|
||||
out.push_str(row[row.len() - 1]);
|
||||
out.push('\n');
|
||||
}
|
||||
|
||||
print!("{}", out);
|
||||
}
|
||||
@@ -8,7 +8,6 @@ pub mod config;
|
||||
pub mod crdt;
|
||||
pub mod data;
|
||||
pub mod error;
|
||||
pub mod formater;
|
||||
pub mod metrics;
|
||||
pub mod persister;
|
||||
pub mod sled_counter;
|
||||
|
||||
+23
-13
@@ -2,47 +2,57 @@ use err_derive::Error;
|
||||
use hyper::header::HeaderValue;
|
||||
use hyper::{HeaderMap, StatusCode};
|
||||
|
||||
use garage_api::generic_server::ApiError;
|
||||
use garage_util::error::Error as GarageError;
|
||||
|
||||
/// Errors of this crate
|
||||
#[derive(Debug, Error)]
|
||||
pub enum Error {
|
||||
/// An error received from the API crate
|
||||
#[error(display = "API error: {}", _0)]
|
||||
ApiError(garage_api::s3::error::Error),
|
||||
ApiError(#[error(source)] garage_api::Error),
|
||||
|
||||
// Category: internal error
|
||||
/// Error internal to garage
|
||||
#[error(display = "Internal error: {}", _0)]
|
||||
InternalError(#[error(source)] GarageError),
|
||||
|
||||
/// The file does not exist
|
||||
#[error(display = "Not found")]
|
||||
NotFound,
|
||||
|
||||
/// The request contained an invalid UTF-8 sequence in its path or in other parameters
|
||||
#[error(display = "Invalid UTF-8: {}", _0)]
|
||||
InvalidUtf8(#[error(source)] std::str::Utf8Error),
|
||||
|
||||
/// The client send a header with invalid value
|
||||
#[error(display = "Invalid header value: {}", _0)]
|
||||
InvalidHeader(#[error(source)] hyper::header::ToStrError),
|
||||
|
||||
/// The client sent a request without host, or with unsupported method
|
||||
#[error(display = "Bad request: {}", _0)]
|
||||
BadRequest(String),
|
||||
}
|
||||
|
||||
impl<T> From<T> for Error
|
||||
where
|
||||
garage_api::s3::error::Error: From<T>,
|
||||
{
|
||||
fn from(err: T) -> Self {
|
||||
Error::ApiError(garage_api::s3::error::Error::from(err))
|
||||
}
|
||||
}
|
||||
|
||||
impl Error {
|
||||
/// Transform errors into http status code
|
||||
pub fn http_status_code(&self) -> StatusCode {
|
||||
match self {
|
||||
Error::NotFound => StatusCode::NOT_FOUND,
|
||||
Error::ApiError(e) => e.http_status_code(),
|
||||
Error::BadRequest(_) => StatusCode::BAD_REQUEST,
|
||||
Error::InternalError(
|
||||
GarageError::Timeout
|
||||
| GarageError::RemoteError(_)
|
||||
| GarageError::Quorum(_, _, _, _),
|
||||
) => StatusCode::SERVICE_UNAVAILABLE,
|
||||
Error::InternalError(_) => StatusCode::INTERNAL_SERVER_ERROR,
|
||||
_ => StatusCode::BAD_REQUEST,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn add_headers(&self, header_map: &mut HeaderMap<HeaderValue>) {
|
||||
#[allow(clippy::single_match)]
|
||||
match self {
|
||||
Error::ApiError(e) => e.add_http_headers(header_map),
|
||||
Error::ApiError(e) => e.add_headers(header_map),
|
||||
_ => (),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -18,11 +18,9 @@ use opentelemetry::{
|
||||
|
||||
use crate::error::*;
|
||||
|
||||
use garage_api::error::{Error as ApiError, OkOrBadRequest, OkOrInternalError};
|
||||
use garage_api::helpers::{authority_to_host, host_to_bucket};
|
||||
use garage_api::s3::cors::{add_cors_headers, find_matching_cors_rule, handle_options_for_bucket};
|
||||
use garage_api::s3::error::{
|
||||
CommonErrorDerivative, Error as ApiError, OkOrBadRequest, OkOrInternalError,
|
||||
};
|
||||
use garage_api::s3::get::{handle_get, handle_head};
|
||||
|
||||
use garage_model::garage::Garage;
|
||||
@@ -209,7 +207,7 @@ async fn serve_file(garage: Arc<Garage>, req: &Request<Body>) -> Result<Response
|
||||
Method::OPTIONS => handle_options_for_bucket(req, &bucket),
|
||||
Method::HEAD => handle_head(garage.clone(), req, bucket_id, &key, None).await,
|
||||
Method::GET => handle_get(garage.clone(), req, bucket_id, &key, None).await,
|
||||
_ => Err(ApiError::bad_request("HTTP method not supported")),
|
||||
_ => Err(ApiError::BadRequest("HTTP method not supported".into())),
|
||||
}
|
||||
.map_err(Error::from);
|
||||
|
||||
@@ -292,7 +290,9 @@ fn path_to_key<'a>(path: &'a str, index: &str) -> Result<Cow<'a, str>, Error> {
|
||||
let path_utf8 = percent_encoding::percent_decode_str(path).decode_utf8()?;
|
||||
|
||||
if !path_utf8.starts_with('/') {
|
||||
return Err(Error::BadRequest("Path must start with a / (slash)".into()));
|
||||
return Err(Error::BadRequest(
|
||||
"Path must start with a / (slash)".to_string(),
|
||||
));
|
||||
}
|
||||
|
||||
match path_utf8.chars().last() {
|
||||
|
||||
Reference in New Issue
Block a user