Compare commits

..

11 Commits

Author SHA1 Message Date
Alex Auvolat 073dd715ce bump version to v2.4.0 and update api json schema 2026-09-06 19:43:04 +02:00
Alex 845db1afa1 Merge pull request 'api/s3: don't panic when a version has no blocks' (#1522) from rajsinghtech/garage:fix/multipart-empty-blocks into main-v2
Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/1522
2026-09-06 07:49:47 +00:00
Raj Singh c54077af73 api/s3: don't panic when reading a version with no blocks
body_from_blocks_range indexed all_blocks[0] to size a capacity hint. A
version with zero blocks made that index panic and kill the process on a
plain GET, the read-side twin of the CompleteMultipartUpload panic at
multipart.rs:479. Fall back to the existing 1024 floor when the slice is
empty; with a non-empty slice the hint is unchanged.
2026-09-04 03:19:07 +00:00
Raj Singh f38f001bad api/s3: don't panic when final multipart version has no blocks
CompleteMultipartUpload builds the final version and then reads
`final_version.blocks.items()[0]` unconditionally. When the assembled
version ends up with an empty block list the index panics and the
process dies before it can answer the request:

    panicked at src/api/s3/multipart.rs:479:3:
    index out of bounds: the len is 0 but the index is 0

Use `.first()` and return an internal error instead, matching how the
rest of this file handles missing data. The empty-block case is an
internal consistency condition rather than a malformed request, so it
maps to an internal error rather than a 4xx.

The same panic was reported earlier in #1403, at multipart.rs:483 on
v2.2.0. That report was closed without a fix after the reporter moved
off consistency_mode = "dangerous".

Closes #1521
2026-09-04 00:01:32 +00:00
i5-650 c85597fd18 Better handling of wildcards in CORS rules (fix #1105) (#1516)
Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/1516
2026-09-03 07:50:59 +00:00
maximilien 05e294307e Merge pull request 'doc: update on k2v inclusion in builds' (#1513) from vincent/garage:main-v2 into main-v2
Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/1513
2026-08-23 19:57:51 +00:00
Vincent 109fbd49b1 Update information about k2v inclusion in builds 2026-08-23 20:57:21 +02:00
i5-650 d0176f8e30 deregister from consul on shutdown (fix #1062) (#1507)
Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/1507
2026-08-22 17:19:51 +00:00
i5-650 d9b1dba137 1250 - add NO_COLOR env var to disable logs colors (#1503)
Fix #1250

Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/1503
Reviewed-by: Alex <lx@deuxfleurs.fr>
2026-08-19 08:03:44 +00:00
Agathe Porte c3c8af9a66 doc: quick-start: rework flag description (#1496)
Add a link to the manual install procedure, use a
quote to make the manual install path more visible
and mention the --default-access-key option.

Closes: #1446

Rendered output:

![image](/attachments/84bbe650-daff-40d1-b4bd-629abece27b0)

Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/1496
2026-07-25 14:48:53 +00:00
Alex Auvolat ad78b9ee5c CI: revert spurrious changed introduced in #1487 that broke CI for main-v2 (#1499)
Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/1499
2026-07-25 13:25:17 +00:00
39 changed files with 459 additions and 344 deletions
Generated
+13 -13
View File
@@ -1513,7 +1513,7 @@ dependencies = [
[[package]]
name = "garage"
version = "2.3.0"
version = "2.4.0"
dependencies = [
"assert-json-diff",
"async-trait",
@@ -1581,7 +1581,7 @@ dependencies = [
[[package]]
name = "garage_api_admin"
version = "2.3.0"
version = "2.4.0"
dependencies = [
"argon2",
"async-trait",
@@ -1613,7 +1613,7 @@ dependencies = [
[[package]]
name = "garage_api_common"
version = "2.3.0"
version = "2.4.0"
dependencies = [
"base64 0.22.1",
"bytes",
@@ -1649,7 +1649,7 @@ dependencies = [
[[package]]
name = "garage_api_k2v"
version = "2.3.0"
version = "2.4.0"
dependencies = [
"base64 0.22.1",
"futures",
@@ -1672,7 +1672,7 @@ dependencies = [
[[package]]
name = "garage_api_s3"
version = "2.3.0"
version = "2.4.0"
dependencies = [
"aes-gcm",
"async-compression",
@@ -1717,7 +1717,7 @@ dependencies = [
[[package]]
name = "garage_block"
version = "2.3.0"
version = "2.4.0"
dependencies = [
"arc-swap",
"async-compression",
@@ -1741,7 +1741,7 @@ dependencies = [
[[package]]
name = "garage_db"
version = "2.3.0"
version = "2.4.0"
dependencies = [
"fjall",
"heed",
@@ -1756,7 +1756,7 @@ dependencies = [
[[package]]
name = "garage_model"
version = "2.3.0"
version = "2.4.0"
dependencies = [
"arbitrary",
"argon2",
@@ -1784,7 +1784,7 @@ dependencies = [
[[package]]
name = "garage_net"
version = "2.3.0"
version = "2.4.0"
dependencies = [
"arc-swap",
"bytes",
@@ -1810,7 +1810,7 @@ dependencies = [
[[package]]
name = "garage_rpc"
version = "2.3.0"
version = "2.4.0"
dependencies = [
"arc-swap",
"async-trait",
@@ -1843,7 +1843,7 @@ dependencies = [
[[package]]
name = "garage_table"
version = "2.3.0"
version = "2.4.0"
dependencies = [
"arc-swap",
"async-trait",
@@ -1864,7 +1864,7 @@ dependencies = [
[[package]]
name = "garage_util"
version = "2.3.0"
version = "2.4.0"
dependencies = [
"arbitrary",
"arc-swap",
@@ -1899,7 +1899,7 @@ dependencies = [
[[package]]
name = "garage_web"
version = "2.3.0"
version = "2.4.0"
dependencies = [
"garage_api_common",
"garage_api_s3",
+12 -12
View File
@@ -25,18 +25,18 @@ default-members = ["src/garage"]
# Internal Garage crates
format_table = { version = "0.1.1", path = "src/format-table" }
garage_api_common = { version = "2.3.0", path = "src/api/common" }
garage_api_admin = { version = "2.3.0", path = "src/api/admin" }
garage_api_s3 = { version = "2.3.0", path = "src/api/s3" }
garage_api_k2v = { version = "2.3.0", path = "src/api/k2v" }
garage_block = { version = "2.3.0", path = "src/block" }
garage_db = { version = "2.3.0", path = "src/db", default-features = false }
garage_model = { version = "2.3.0", path = "src/model", default-features = false }
garage_net = { version = "2.3.0", path = "src/net" }
garage_rpc = { version = "2.3.0", path = "src/rpc" }
garage_table = { version = "2.3.0", path = "src/table" }
garage_util = { version = "2.3.0", path = "src/util" }
garage_web = { version = "2.3.0", path = "src/web" }
garage_api_common = { version = "2.4.0", path = "src/api/common" }
garage_api_admin = { version = "2.4.0", path = "src/api/admin" }
garage_api_s3 = { version = "2.4.0", path = "src/api/s3" }
garage_api_k2v = { version = "2.4.0", path = "src/api/k2v" }
garage_block = { version = "2.4.0", path = "src/block" }
garage_db = { version = "2.4.0", path = "src/db", default-features = false }
garage_model = { version = "2.4.0", path = "src/model", default-features = false }
garage_net = { version = "2.4.0", path = "src/net" }
garage_rpc = { version = "2.4.0", path = "src/rpc" }
garage_table = { version = "2.4.0", path = "src/table" }
garage_util = { version = "2.4.0", path = "src/util" }
garage_web = { version = "2.4.0", path = "src/web" }
k2v-client = { version = "0.0.4", path = "src/k2v-client" }
# External crates from crates.io
+8 -88
View File
@@ -1243,36 +1243,6 @@
],
"description": "List all the buckets on the cluster with their UUID and their global and local aliases.",
"operationId": "ListBuckets",
"parameters": [
{
"name": "details",
"in": "query",
"description": "Returned detailed informations in the same format as GetBucketInfo for each bucket",
"required": false,
"schema": {
"type": "boolean"
}
},
{
"name": "offset",
"in": "query",
"description": "Bucket ID of the first bucket to return",
"required": false,
"schema": {
"type": "string"
}
},
{
"name": "limit",
"in": "query",
"description": "Maximum number of buckets to return in a single call",
"required": false,
"schema": {
"type": "integer",
"minimum": 0
}
}
],
"responses": {
"200": {
"description": "Returns the UUID of all the buckets and all their aliases",
@@ -1297,36 +1267,6 @@
],
"description": "Returns all API access keys in the cluster.",
"operationId": "ListKeys",
"parameters": [
{
"name": "details",
"in": "query",
"description": "Returned detailed informations in the same format as GetKeyInfo for each bucket",
"required": false,
"schema": {
"type": "boolean"
}
},
{
"name": "offset",
"in": "query",
"description": "Key ID of the first key to return",
"required": false,
"schema": {
"type": "string"
}
},
{
"name": "limit",
"in": "query",
"description": "Maximum number of keys to return in a single call",
"required": false,
"schema": {
"type": "integer",
"minimum": 0
}
}
],
"responses": {
"200": {
"description": "Returns the key identifier (aka `AWS_ACCESS_KEY_ID`) and its associated, human friendly, name if any (otherwise return an empty string)",
@@ -3260,20 +3200,10 @@
}
},
"ListBucketsResponse": {
"oneOf": [
{
"type": "array",
"items": {
"$ref": "#/components/schemas/ListBucketsResponseItem"
}
},
{
"type": "array",
"items": {
"$ref": "#/components/schemas/GetBucketInfoResponse"
}
}
]
"type": "array",
"items": {
"$ref": "#/components/schemas/ListBucketsResponseItem"
}
},
"ListBucketsResponseItem": {
"type": "object",
@@ -3306,20 +3236,10 @@
}
},
"ListKeysResponse": {
"oneOf": [
{
"type": "array",
"items": {
"$ref": "#/components/schemas/ListKeysResponseItem"
}
},
{
"type": "array",
"items": {
"$ref": "#/components/schemas/GetKeyInfoResponse"
}
}
]
"type": "array",
"items": {
"$ref": "#/components/schemas/ListKeysResponseItem"
}
},
"ListKeysResponseItem": {
"type": "object",
+5 -5
View File
@@ -96,14 +96,14 @@ to store 2 TB of data in total.
## Get a Docker image
Our docker image is currently named `dxflrs/garage` and is stored on the [Docker Hub](https://hub.docker.com/r/dxflrs/garage/tags?page=1&ordering=last_updated).
We encourage you to use a fixed tag (eg. `v2.3.0`) and not the `latest` tag.
For this example, we will use the latest published version at the time of the writing which is `v2.3.0` but it's up to you
We encourage you to use a fixed tag (eg. `v2.4.0`) and not the `latest` tag.
For this example, we will use the latest published version at the time of the writing which is `v2.4.0` but it's up to you
to check [the most recent versions on the Docker Hub](https://hub.docker.com/r/dxflrs/garage/tags?page=1&ordering=last_updated).
For example:
```
docker pull dxflrs/garage:v2.3.0
docker pull dxflrs/garage:v2.4.0
```
## Deploying and configuring Garage
@@ -171,7 +171,7 @@ docker run \
-v /etc/garage.toml:/etc/garage.toml \
-v /var/lib/garage/meta:/var/lib/garage/meta \
-v /var/lib/garage/data:/var/lib/garage/data \
dxflrs/garage:v2.3.0
dxflrs/garage:v2.4.0
```
With this command line, Garage should be started automatically at each boot.
@@ -185,7 +185,7 @@ If you want to use `docker-compose`, you may use the following `docker-compose.y
version: "3"
services:
garage:
image: dxflrs/garage:v2.3.0
image: dxflrs/garage:v2.4.0
network_mode: "host"
restart: unless-stopped
volumes:
+14 -9
View File
@@ -111,7 +111,7 @@ your data to be persisted properly.
### Configuring initial access credentials
Since `v2.3.0`, Garage can automatically create a default access key and a default storage bucket,
Since `v2.n.0`, Garage can automatically create a default access key and a default storage bucket,
based on values provided in environment variables.
To use this feature, export the following environment variables:
@@ -133,12 +133,17 @@ Use the following command to launch the Garage server:
garage server --single-node --default-bucket
```
The `--single-node` flag instructs Garage to automatically configure a single-node cluster without data replication.
The `--default-bucket` flag instructs Garage to create a default access key and a default bucket using the environment variables we defined above.
Both flags are optional and can be omitted, in which case you will have to follow manual configuration steps described below.
- the `--single-node` flag instructs Garage to automatically configure a
single-node cluster without data replication;
- the `--default-bucket` flag instructs Garage to create a default access key
and a default bucket using the environment variables we defined above (it
implies `--default-access-key`).
**For older versions of Garage (before v2.3.0):** automatic configuration using `--single-node` and `--default-bucket` is not available,
you must follow the manual configuration steps.
> You can refer to the [manual configuration
> steps](#manual-configuration) if:
>
> - you decide to no use these optional flags;
> - you are running an **older version of Garage (before v2.3.0)**.
Alternatively, if you cannot or do not wish to run the Garage binary directly,
you may use Docker to run Garage in a container using the following command:
@@ -152,7 +157,7 @@ docker run \
-e GARAGE_DEFAULT_ACCESS_KEY \
-e GARAGE_DEFAULT_SECRET_KEY \
-e GARAGE_DEFAULT_BUCKET \
dxflrs/garage:v2.3.0
dxflrs/garage:v2.4.0
/garage server --single-node --default-bucket
```
@@ -292,7 +297,7 @@ An exhaustive list is maintained in the ["Integrations" > "Browsing tools" secti
## Manual configuration
## Manual configuration {#manual-configuration}
This section provides instructions that are equivalent to using the
`--single-node` and `--default-bucket` flags for automatic configuration. If
@@ -332,7 +337,7 @@ When you first start a cluster without automatic configuration, the output of `g
```
==== HEALTHY NODES ====
ID Hostname Address Tags Zone Capacity DataAvail Version
563e1ac825ee3323 linuxbox 127.0.0.1:3901 NO ROLE ASSIGNED v2.3.0
563e1ac825ee3323 linuxbox 127.0.0.1:3901 NO ROLE ASSIGNED v2.4.0
```
Creating a cluster layout for a Garage deployment means informing Garage of the
@@ -175,6 +175,9 @@ they do not exist in the configuration file:
Garage daemon send its logs to `journald` (using the native protocol of `systemd-journald`)
instead of printing to stderr.
- `NO_COLOR` (since `v2.4.0`): set this to `0` or `false` to disable
ANSI color codes in Garage's logs.
The following environment variables can be used to override the corresponding
values in the configuration file:
+5 -6
View File
@@ -8,12 +8,11 @@ which is an alternative storage API designed to help efficiently store
many small values in buckets (in opposition to S3 which is more designed
to store large blobs).
K2V is currently disabled at compile time in all builds, as the
specification is still subject to changes. To build a Garage version with
K2V, the Cargo feature flag `k2v` must be activated. Special builds with
the `k2v` feature flag enabled can be obtained from our download page under
"Extra builds": such builds can be identified easily as their tag name ends
with `-k2v` (example: `v0.7.2-k2v`).
K2V is included in release builds since version 0.8.0. Precompiled builds
of earlier versions including `k2v` can be found in our download page under
"Extra builds": they can be easily identified as their tag name ends with
`-k2v` (example: `v0.7.2-k2v`). Otherwise, when compiling Garage, the Cargo
feature flag `k2v` must be activated.
The specification of the K2V API can be found
[here](https://git.deuxfleurs.fr/Deuxfleurs/garage/src/commit/f8be15c37db857e177d543de7be863692628d567/doc/drafts/k2v-spec.md).
+1 -1
View File
@@ -18,7 +18,7 @@ fi
$GARAGE_BIN -c /tmp/config.1.toml bucket create eprouvette
if [ "$GARAGE_OLDVER" = "v08" ]; then
KEY_INFO=$($GARAGE_BIN -c /tmp/config.1.toml key create opérateur)
KEY_INFO=$($GARAGE_BIN -c /tmp/config.1.toml key new --name opérateur)
ACCESS_KEY=`echo $KEY_INFO|grep -Po 'GK[a-f0-9]+'`
SECRET_KEY=`echo $KEY_INFO|grep -Po 'Secret key: [a-f0-9]+'|grep -Po '[a-f0-9]+$'`
elif [ "$GARAGE_OLDVER" = "v1" ]; then
+2 -2
View File
@@ -2,8 +2,8 @@ apiVersion: v2
name: garage
description: S3-compatible object store for small self-hosted geo-distributed deployments
type: application
version: 0.9.3
appVersion: "v2.3.0"
version: 0.9.4
appVersion: "v2.4.0"
home: https://garagehq.deuxfleurs.fr/
icon: https://garagehq.deuxfleurs.fr/images/garage-logo.svg
+1 -1
View File
@@ -1,6 +1,6 @@
# garage
![Version: 0.9.3](https://img.shields.io/badge/Version-0.9.3-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: v2.3.0](https://img.shields.io/badge/AppVersion-v2.3.0-informational?style=flat-square)
![Version: 0.9.4](https://img.shields.io/badge/Version-0.9.4-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: v2.4.0](https://img.shields.io/badge/AppVersion-v2.4.0-informational?style=flat-square)
S3-compatible object store for small self-hosted geo-distributed deployments
+1 -1
View File
@@ -1,6 +1,6 @@
[package]
name = "garage_api_admin"
version = "2.3.0"
version = "2.4.0"
authors = ["Alex Auvolat <alex@adnab.me>"]
edition = "2018"
license = "AGPL-3.0"
+6 -36
View File
@@ -688,26 +688,11 @@ pub struct ClusterLayoutSkipDeadNodesResponse {
// ---- ListKeys ----
#[derive(Debug, Clone, Serialize, Deserialize, Default, IntoParams)]
#[into_params(parameter_in = Query)]
pub struct ListKeysRequest {
/// Returned detailed informations in the same format as GetKeyInfo for each bucket
#[serde(default)]
pub details: bool,
/// Key ID of the first key to return
#[serde(default)]
pub offset: Option<String>,
/// Maximum number of keys to return in a single call
#[serde(default)]
pub limit: Option<usize>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ListKeysRequest;
#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)]
#[serde(untagged)]
pub enum ListKeysResponse {
WithoutDetails(Vec<ListKeysResponseItem>),
WithDetails(Vec<GetKeyInfoResponse>),
}
pub struct ListKeysResponse(pub Vec<ListKeysResponseItem>);
#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)]
#[serde(rename_all = "camelCase")]
@@ -845,26 +830,11 @@ pub struct DeleteKeyResponse;
// ---- ListBuckets ----
#[derive(Debug, Clone, Serialize, Deserialize, Default, IntoParams)]
#[into_params(parameter_in = Query)]
pub struct ListBucketsRequest {
/// Returned detailed informations in the same format as GetBucketInfo for each bucket
#[serde(default)]
pub details: bool,
/// Bucket ID of the first bucket to return
#[serde(default)]
pub offset: Option<String>,
/// Maximum number of buckets to return in a single call
#[serde(default)]
pub limit: Option<usize>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ListBucketsRequest;
#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)]
#[serde(untagged)]
pub enum ListBucketsResponse {
WithoutDetails(Vec<ListBucketsResponseItem>),
WithDetails(Vec<GetBucketInfoResponse>),
}
pub struct ListBucketsResponse(pub Vec<ListBucketsResponseItem>);
#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)]
#[serde(rename_all = "camelCase")]
+31 -55
View File
@@ -3,7 +3,6 @@ use std::sync::Arc;
use std::time::Duration;
use chrono::DateTime;
use futures::StreamExt;
use garage_util::crdt::*;
use garage_util::data::*;
@@ -33,70 +32,47 @@ impl RequestHandler for ListBucketsRequest {
garage: &Arc<Garage>,
_admin: &Admin,
) -> Result<ListBucketsResponse, Error> {
let limit = self
.limit
.unwrap_or_else(|| if self.details { 1000 } else { 10_000 });
let offset = match self.offset {
Some(id) => Some(parse_bucket_id(&id)?),
None => None,
};
let buckets = garage
.bucket_table
.get_range(
&EmptyKey,
offset,
None,
Some(DeletedFilter::NotDeleted),
limit,
1_000_000,
EnumerationOrder::Forward,
)
.await?;
if self.details {
let mut stream = buckets
.into_iter()
.map(|b| bucket_info_results(garage, b.id))
.collect::<futures::stream::FuturesOrdered<_>>();
let res = buckets
.into_iter()
.map(|b| {
let state = b.state.as_option().unwrap();
ListBucketsResponseItem {
id: hex::encode(b.id),
created: DateTime::from_timestamp_millis(state.creation_date as i64)
.expect("invalid timestamp stored in db"),
global_aliases: state
.aliases
.items()
.iter()
.filter(|(_, _, a)| *a)
.map(|(n, _, _)| n.to_string())
.collect::<Vec<_>>(),
local_aliases: state
.local_aliases
.items()
.iter()
.filter(|(_, _, a)| *a)
.map(|((k, n), _, _)| BucketLocalAlias {
access_key_id: k.to_string(),
alias: n.to_string(),
})
.collect::<Vec<_>>(),
}
})
.collect::<Vec<_>>();
let mut res = vec![];
while let Some(next) = stream.next().await {
res.push(next?);
}
Ok(ListBucketsResponse::WithDetails(res))
} else {
let res = buckets
.into_iter()
.map(|b| {
let state = b.state.as_option().unwrap();
ListBucketsResponseItem {
id: hex::encode(b.id),
created: DateTime::from_timestamp_millis(state.creation_date as i64)
.expect("invalid timestamp stored in db"),
global_aliases: state
.aliases
.items()
.iter()
.filter(|(_, _, a)| *a)
.map(|(n, _, _)| n.to_string())
.collect::<Vec<_>>(),
local_aliases: state
.local_aliases
.items()
.iter()
.filter(|(_, _, a)| *a)
.map(|((k, n), _, _)| BucketLocalAlias {
access_key_id: k.to_string(),
alias: n.to_string(),
})
.collect::<Vec<_>>(),
}
})
.collect::<Vec<_>>();
Ok(ListBucketsResponse::WithoutDetails(res))
}
Ok(ListBucketsResponse(res))
}
}
+23 -44
View File
@@ -2,7 +2,6 @@ use std::collections::HashMap;
use std::sync::Arc;
use chrono::DateTime;
use futures::StreamExt;
use garage_table::*;
use garage_util::time::now_msec;
@@ -21,57 +20,37 @@ impl RequestHandler for ListKeysRequest {
async fn handle(self, garage: &Arc<Garage>, _admin: &Admin) -> Result<ListKeysResponse, Error> {
let now = now_msec();
let limit = self
.limit
.unwrap_or_else(|| if self.details { 1000 } else { 10_000 });
let keys = garage
let res = garage
.key_table
.get_range(
&EmptyKey,
self.offset,
None,
Some(KeyFilter::Deleted(DeletedFilter::NotDeleted)),
limit,
10000,
EnumerationOrder::Forward,
)
.await?;
.await?
.iter()
.map(|k| {
let p = k.params().unwrap();
if self.details {
let mut stream = keys
.into_iter()
.map(|k| key_info_results(garage, k, false))
.collect::<futures::stream::FuturesOrdered<_>>();
ListKeysResponseItem {
id: k.key_id.to_string(),
name: p.name.get().clone(),
created: p.created.map(|x| {
DateTime::from_timestamp_millis(x as i64)
.expect("invalid timestamp stored in db")
}),
expiration: p.expiration.get().inner().map(|x| {
DateTime::from_timestamp_millis(x.0 as i64)
.expect("invalid timestamp stored in db")
}),
expired: p.is_expired(now),
}
})
.collect::<Vec<_>>();
let mut res = vec![];
while let Some(next) = stream.next().await {
res.push(next?);
}
Ok(ListKeysResponse::WithDetails(res))
} else {
let res = keys
.iter()
.map(|k| {
let p = k.params().unwrap();
ListKeysResponseItem {
id: k.key_id.to_string(),
name: p.name.get().clone(),
created: p.created.map(|x| {
DateTime::from_timestamp_millis(x as i64)
.expect("invalid timestamp stored in db")
}),
expiration: p.expiration.get().inner().map(|x| {
DateTime::from_timestamp_millis(x.0 as i64)
.expect("invalid timestamp stored in db")
}),
expired: p.is_expired(now),
}
})
.collect::<Vec<_>>();
Ok(ListKeysResponse::WithoutDetails(res))
}
Ok(ListKeysResponse(res))
}
}
-2
View File
@@ -364,7 +364,6 @@ fn ClusterLayoutSkipDeadNodes() {}
path = "/v2/ListKeys",
tag = "Access key",
description = "Returns all API access keys in the cluster.",
params(ListKeysRequest),
responses(
(status = 200, description = "Returns the key identifier (aka `AWS_ACCESS_KEY_ID`) and its associated, human friendly, name if any (otherwise return an empty string)", body = ListKeysResponse),
(status = 500, description = "Internal server error")
@@ -454,7 +453,6 @@ fn DeleteKey() {}
path = "/v2/ListBuckets",
tag = "Bucket",
description = "List all the buckets on the cluster with their UUID and their global and local aliases.",
params(ListBucketsRequest),
responses(
(status = 200, description = "Returns the UUID of all the buckets and all their aliases", body = ListBucketsResponse),
(status = 500, description = "Internal server error")
+5 -10
View File
@@ -55,10 +55,10 @@ impl AdminApiRequest {
POST CreateKey (body),
POST ImportKey (body),
POST DeleteKey (query::id),
GET ListKeys (parse_default(false)::details, query_opt::offset, opt_parse::limit),
GET ListKeys (),
// Bucket endpoints
GET GetBucketInfo (query_opt::id, query_opt::global_alias, query_opt::search),
GET ListBuckets (parse_default(false)::details, query_opt::offset, opt_parse::limit),
GET ListBuckets (),
POST CreateBucket (body),
POST DeleteBucket (query::id),
POST UpdateBucket (body_field, query::id),
@@ -129,7 +129,7 @@ impl AdminApiRequest {
)),
// Keys
Endpoint::ListKeys => Ok(AdminApiRequest::ListKeys(ListKeysRequest::default())),
Endpoint::ListKeys => Ok(AdminApiRequest::ListKeys(ListKeysRequest)),
Endpoint::GetKeyInfo {
id,
search,
@@ -161,9 +161,7 @@ impl AdminApiRequest {
// Endpoint::DeleteKey { id } => Ok(AdminApiRequest::DeleteKey(DeleteKeyRequest { id })),
// Buckets
Endpoint::ListBuckets => {
Ok(AdminApiRequest::ListBuckets(ListBucketsRequest::default()))
}
Endpoint::ListBuckets => Ok(AdminApiRequest::ListBuckets(ListBucketsRequest)),
Endpoint::GetBucketInfo { id, global_alias } => {
Ok(AdminApiRequest::GetBucketInfo(GetBucketInfoRequest {
id,
@@ -273,9 +271,6 @@ generateQueryParameters! {
"accessKeyId" => access_key_id,
"showSecretKey" => show_secret_key,
"bucketId" => bucket_id,
"key" => key,
"details" => details,
"offset" => offset,
"limit" => limit
"key" => key
]
}
+1 -1
View File
@@ -1,6 +1,6 @@
[package]
name = "garage_api_common"
version = "2.3.0"
version = "2.4.0"
authors = ["Alex Auvolat <alex@adnab.me>"]
edition = "2018"
license = "AGPL-3.0"
+228 -2
View File
@@ -47,15 +47,26 @@ where
HI: Iterator<Item = S>,
S: AsRef<str>,
{
rule.allow_origins.iter().any(|x| x == "*" || x == origin)
rule.allow_origins.iter().any(|x| wildcard_match(x, origin))
&& rule.allow_methods.iter().any(|x| x == "*" || x == method)
&& request_headers.all(|h| {
rule.allow_headers
.iter()
.any(|x| x == "*" || x == h.as_ref())
.any(|x| wildcard_match(x, h.as_ref()))
})
}
/// Checks whether `candidate` matches the pattern `allowed_wildcard`.
#[inline]
fn wildcard_match(allowed_wildcard: &String, candidate: &str) -> bool {
if allowed_wildcard.contains("*") {
let parts = allowed_wildcard.split("*").collect::<Vec<&str>>();
parts.len() == 2 && candidate.starts_with(parts[0]) && candidate.ends_with(parts[1])
} else {
candidate == allowed_wildcard
}
}
pub fn add_cors_headers(
resp: &mut Response<impl Body>,
rule: &GarageCorsRule,
@@ -190,6 +201,221 @@ pub fn handle_options_for_bucket<B>(
mod tests {
use super::*;
fn cors_rule(
allow_origins: &[&str],
allow_methods: &[&str],
allow_headers: &[&str],
) -> GarageCorsRule {
GarageCorsRule {
id: None,
max_age_seconds: None,
allow_origins: allow_origins.iter().map(|s| s.to_string()).collect(),
allow_methods: allow_methods.iter().map(|s| s.to_string()).collect(),
allow_headers: allow_headers.iter().map(|s| s.to_string()).collect(),
expose_headers: vec![],
}
}
#[test]
fn matches_when_origin_method_and_headers_are_explicitly_allowed() {
let rule = cors_rule(
&["https://app.example.test"],
&["GET", "PUT"],
&["content-type", "x-custom"],
);
let headers = vec!["content-type", "x-custom"];
assert!(cors_rule_matches(
&rule,
"https://app.example.test",
"PUT",
headers.iter(),
));
}
#[test]
fn does_not_match_when_origin_is_not_allowed() {
let rule = cors_rule(&["https://app.example.test"], &["GET"], &["*"]);
assert!(!cors_rule_matches(
&rule,
"https://evil.example.test",
"GET",
std::iter::empty::<&str>(),
));
}
#[test]
fn does_not_match_when_method_is_not_allowed() {
let rule = cors_rule(&["*"], &["GET"], &["*"]);
assert!(!cors_rule_matches(
&rule,
"https://app.example.test",
"DELETE",
std::iter::empty::<&str>(),
));
}
#[test]
fn does_not_match_when_a_requested_header_is_not_allowed() {
let rule = cors_rule(&["*"], &["GET"], &["content-type"]);
let headers = vec!["content-type", "x-not-allowed"];
assert!(!cors_rule_matches(
&rule,
"https://app.example.test",
"GET",
headers.iter(),
));
}
#[test]
fn wildcard_origin_method_and_headers_match_anything() {
let rule = cors_rule(&["*"], &["*"], &["*"]);
let headers = vec!["x-anything"];
assert!(cors_rule_matches(
&rule,
"https://app.example.test",
"DELETE",
headers.iter(),
));
}
#[test]
fn wildcard_origin_regex() {
let rule = cors_rule(&["https://*.localhost.com"], &["*"], &["*"]);
let headers = vec!["x-anything"];
assert!(cors_rule_matches(
&rule,
"https://s3.localhost.com",
"DELETE",
headers.iter(),
));
}
#[test]
fn origin_matching_cases() {
// (allow_origins, origin, expect_match)
let cases: &[(&[&str], &str, bool)] = &[
// exact match
(
&["https://app.example.test"],
"https://app.example.test",
true,
),
(
&["https://app.example.test"],
"https://other.example.test",
false,
),
// full wildcard
(&["*"], "https://anything.example.test", true),
// subdomain glob
(
&["https://*.example.test"],
"https://foo.example.test",
true,
),
(&["https://*.example.test"], "https://example.test", false),
(
&["https://*.example.test"],
"http://foo.example.test",
false,
),
// multiple allowed origins, at least one should match
(
&["https://a.example.test", "https://b.example.test"],
"https://b.example.test",
true,
),
// match multiple origins
(
&["https://a*.example.test", "https://ab*.example.test"],
"https://abc.example.test",
true,
),
(
&["https://a.example.test", "https://b.example.test"],
"https://c.example.test",
false,
),
// at most one '*' in a pattern is allowed
(&["https://*.example.*"], "https://a.example.test", false),
// domain changed with wildcard
(
&["https://*example.test"],
"https://garageexample.test",
true,
),
// trailing '*' matches any suffix, including the empty string,
// so this also matches origins with anything (or nothing) after
// "example."
(&["https://example.*"], "https://example.test", true),
(&["https://*example.test"], "https://example.test", true),
(&["https://example.*"], "https://example.", true),
];
for (allow_origins, origin, expect_match) in cases {
let rule = cors_rule(allow_origins, &["GET"], &["*"]);
let got = cors_rule_matches(&rule, origin, "GET", std::iter::empty::<&str>());
assert_eq!(
got, *expect_match,
"allow_origins={allow_origins:?}, origin={origin:?}: expected match={expect_match}, got {got}"
);
}
}
#[test]
fn header_matching_cases() {
// (allow_headers, requested_headers, expect_match)
let cases: &[(&[&str], &[&str], bool)] = &[
// exact match
(&["content-type"], &["content-type"], true),
(&["content-type"], &["x-custom"], false),
// full wildcard
(&["*"], &["x-anything"], true),
// no headers requested always matches, regardless of allow_headers
(&["content-type"], &[], true),
(&[], &[], true),
// prefix glob
(&["x-amz-*"], &["x-amz-meta-foo"], true),
(&["x-amz-*"], &["x-amz-"], true),
(&["x-amz-*"], &["x-other"], false),
// suffix glob
(&["*-meta"], &["foo-meta"], true),
(&["*-meta"], &["-meta"], true),
(&["*-meta"], &["foo-meta-bar"], false),
// multiple allowed headers, at least one should match per requested header
(
&["content-type", "x-amz-*"],
&["content-type", "x-amz-meta-foo"],
true,
),
(&["content-type", "x-amz-*"], &["x-other"], false),
// all requested headers must be covered
(&["content-type"], &["content-type", "x-custom"], false),
// at most one '*' in a pattern is allowed
(&["x-*-*"], &["x-a-b"], false),
];
for (allow_headers, requested_headers, expect_match) in cases {
let rule = cors_rule(&["*"], &["GET"], allow_headers);
let got = cors_rule_matches(
&rule,
"https://app.example.test",
"GET",
requested_headers.iter(),
);
assert_eq!(
got, *expect_match,
"allow_headers={allow_headers:?}, requested_headers={requested_headers:?}: expected match={expect_match}, got {got}"
);
}
}
fn bucket_params_with_rule(allow_origins: Vec<&str>) -> BucketParams {
let mut bucket_params = BucketParams::default();
bucket_params.cors_config.update(
+1 -1
View File
@@ -1,6 +1,6 @@
[package]
name = "garage_api_k2v"
version = "2.3.0"
version = "2.4.0"
authors = ["Alex Auvolat <alex@adnab.me>"]
edition = "2018"
license = "AGPL-3.0"
+1 -1
View File
@@ -1,6 +1,6 @@
[package]
name = "garage_api_s3"
version = "2.3.0"
version = "2.4.0"
authors = ["Alex Auvolat <alex@adnab.me>"]
edition = "2018"
license = "AGPL-3.0"
+6 -1
View File
@@ -698,9 +698,14 @@ fn body_from_blocks_range(
// range, as well as their "true offset", which is their actual offset in the complete
// file (whereas block.offset designates the offset of the block WITHIN THE PART
// block.part_number, which is not the same in the case of a multipart upload)
// A version with no blocks yields no data, so the capacity hint must not index
// into an empty slice.
let capacity_block_size = all_blocks
.first()
.map_or(1024, |(_, b)| std::cmp::max(b.size, 1024));
let mut blocks: Vec<(VersionBlock, u64)> = Vec::with_capacity(std::cmp::min(
all_blocks.len(),
4 + ((end - begin) / std::cmp::max(all_blocks[0].1.size, 1024)) as usize,
4 + ((end - begin) / capacity_block_size) as usize,
));
let mut block_offset: u64 = 0;
for (_, b) in all_blocks.iter() {
+7 -1
View File
@@ -476,7 +476,13 @@ pub async fn handle_complete_multipart_upload(
size: total_size,
etag: etag.clone(),
},
final_version.blocks.items()[0].1.hash,
final_version
.blocks
.items()
.first()
.ok_or_internal_error("Multipart completion produced a final version with no blocks")?
.1
.hash,
));
let final_object = Object::new(*bucket_id, key.clone(), vec![object_version]);
+1 -1
View File
@@ -1,6 +1,6 @@
[package]
name = "garage_block"
version = "2.3.0"
version = "2.4.0"
authors = ["Alex Auvolat <alex@adnab.me>"]
edition = "2018"
license = "AGPL-3.0"
+1 -1
View File
@@ -1,6 +1,6 @@
[package]
name = "garage_db"
version = "2.3.0"
version = "2.4.0"
authors = ["Alex Auvolat <alex@adnab.me>"]
edition = "2018"
license = "AGPL-3.0"
+1 -1
View File
@@ -1,6 +1,6 @@
[package]
name = "garage"
version = "2.3.0"
version = "2.4.0"
authors = ["Alex Auvolat <alex@adnab.me>"]
edition = "2018"
license = "AGPL-3.0"
+3 -10
View File
@@ -31,19 +31,12 @@ impl Cli {
}
pub async fn cmd_list_buckets(&self) -> Result<(), Error> {
let mut buckets = match self.api_request(ListBucketsRequest::default()).await? {
ListBucketsResponse::WithoutDetails(list) => list,
_ => {
return Err(Error::Message(
"Unexpected ListBuckets response format".into(),
))
}
};
let mut buckets = self.api_request(ListBucketsRequest).await?;
buckets.sort_by_key(|x| x.created);
buckets.0.sort_by_key(|x| x.created);
let mut table = vec!["ID\tCreated\tGlobal aliases\tLocal aliases".to_string()];
for bucket in buckets.iter() {
for bucket in buckets.0.iter() {
table.push(format!(
"{:.16}\t{}\t{}\t{}",
bucket.id,
+4 -10
View File
@@ -28,15 +28,12 @@ impl Cli {
}
pub async fn cmd_list_keys(&self) -> Result<(), Error> {
let mut keys = match self.api_request(ListKeysRequest::default()).await? {
ListKeysResponse::WithoutDetails(list) => list,
_ => return Err(Error::Message("Unexpected ListKeys response format".into())),
};
let mut keys = self.api_request(ListKeysRequest).await?;
keys.sort_by_key(|x| x.created);
keys.0.sort_by_key(|x| x.created);
let mut table = vec!["ID\tCreated\tName\tExpiration".to_string()];
for key in keys.iter() {
for key in keys.0.iter() {
let exp = if key.expired {
Cow::from("expired")
} else {
@@ -246,10 +243,7 @@ impl Cli {
}
pub async fn cmd_delete_expired_keys(&self, yes: bool) -> Result<(), Error> {
let mut list = match self.api_request(ListKeysRequest::default()).await? {
ListKeysResponse::WithoutDetails(list) => list,
_ => return Err(Error::Message("Unexpected ListKeys response format".into())),
};
let mut list = self.api_request(ListKeysRequest).await?.0;
list.retain(|key| key.expired);
+8 -21
View File
@@ -110,30 +110,17 @@ impl Cli {
Ok(resp.success.into_iter().next().unwrap().1)
}
pub async fn cmd_json_api(
&self,
endpoint: String,
payload: Option<String>,
) -> Result<(), Error> {
let request: AdminApiRequest = if let Some(payload) = payload {
let payload: serde_json::Value = if payload == "-" {
serde_json::from_reader(&std::io::stdin())?
} else {
serde_json::from_str(&payload)?
};
serde_json::from_value(serde_json::json!({
endpoint.clone(): payload,
}))?
pub async fn cmd_json_api(&self, endpoint: String, payload: String) -> Result<(), Error> {
let payload: serde_json::Value = if payload == "-" {
serde_json::from_reader(&std::io::stdin())?
} else {
serde_json::from_value(serde_json::json!({
endpoint.clone(): null,
}))
.or(serde_json::from_value(serde_json::json!({
endpoint.clone(): {},
})))?
serde_json::from_str(&payload)?
};
let request: AdminApiRequest = serde_json::from_value(serde_json::json!({
endpoint.clone(): payload,
}))?;
let resp = match self
.proxy_rpc_endpoint
.call(&self.rpc_host, ProxyRpc::Proxy(request), PRIO_NORMAL)
+2 -1
View File
@@ -78,7 +78,8 @@ pub enum Command {
/// The admin API endpoint to invoke, e.g. `GetClusterStatus`
endpoint: String,
/// The JSON payload, or `-` to read from `stdin`
payload: Option<String>,
#[structopt(default_value = "null")]
payload: String,
},
/// Generate completions for a shell
+5
View File
@@ -276,6 +276,11 @@ fn init_logging(opt: &Opt) {
tracing_subscriber::fmt()
.with_writer(std::io::stderr)
.with_env_filter(env_filter)
.with_ansi(
std::env::var("NO_COLOR")
.map(|x| x != "0" && !x.eq_ignore_ascii_case("false"))
.unwrap_or(true),
)
.init();
}
+14
View File
@@ -152,6 +152,14 @@ pub async fn run_server(
}
}
// Deregister from Consul (if enabled) in the background, in parallel with the
// rest of the shutdown sequence, so that it doesn't add to shutdown latency.
#[cfg(feature = "consul-discovery")]
let deregister_consul_task = tokio::spawn({
let system = garage.system.clone();
async move { system.deregister_from_discovery().await }
});
// Remove RPC handlers for system to break reference cycles
info!("Deregistering RPC handlers for shutdown...");
garage.system.netapp.drop_all_handlers();
@@ -168,6 +176,12 @@ pub async fn run_server(
// Await for all background tasks to end
await_background_done.await?;
// Await for Consul deregistration to end, if it hasn't already
#[cfg(feature = "consul-discovery")]
if let Err(e) = deregister_consul_task.await {
error!("Error while joining Consul deregistration task: {}", e);
}
info!("Cleaning up...");
Ok(())
+1 -1
View File
@@ -1,6 +1,6 @@
[package]
name = "garage_model"
version = "2.3.0"
version = "2.4.0"
authors = ["Alex Auvolat <alex@adnab.me>"]
edition = "2018"
license = "AGPL-3.0"
+1 -1
View File
@@ -1,6 +1,6 @@
[package]
name = "garage_net"
version = "2.3.0"
version = "2.4.0"
authors = ["Alex Auvolat <alex@adnab.me>"]
edition = "2018"
license = "AGPL-3.0"
+1 -1
View File
@@ -1,6 +1,6 @@
[package]
name = "garage_rpc"
version = "2.3.0"
version = "2.4.0"
authors = ["Alex Auvolat <alex@adnab.me>"]
edition = "2018"
license = "AGPL-3.0"
+30
View File
@@ -183,6 +183,36 @@ impl ConsulDiscovery {
}
// ---- PUBLISHING TO CONSUL CATALOG ----
#[cfg(feature = "consul-discovery")]
pub async fn deregister_consul_service(&self, node_id: NodeID) -> Result<(), ConsulError> {
let node = format!("garage:{}", hex::encode(&node_id[..8]));
let url = format!(
"{}/v1/{}",
self.config.consul_http_addr,
(match &self.config.api {
ConsulDiscoveryAPI::Catalog => format!("catalog/deregister"),
ConsulDiscoveryAPI::Agent => format!("agent/service/deregister/{}", node),
})
);
let req = self.client.put(&url);
let http = if matches!(&self.config.api, ConsulDiscoveryAPI::Catalog) {
let deregister_request = serde_json::json!({
"Node": node,
"ServiceID": node,
});
let req = req.json(&deregister_request);
req.send().await?
} else {
req.send().await?
};
http.error_for_status()?;
debug!("Deregistered service {} from Consul", node);
Ok(())
}
pub async fn publish_consul_service(
&self,
node_id: NodeID,
+10 -1
View File
@@ -358,7 +358,7 @@ impl System {
);
}
pub fn cleanup(&self) {
pub fn cleanup(self: &Arc<Self>) {
// Break reference cycle
self.metrics.store(None);
}
@@ -650,6 +650,15 @@ impl System {
}
}
#[cfg(feature = "consul-discovery")]
pub async fn deregister_from_discovery(self: &Arc<Self>) {
if let Some(c) = &self.consul_discovery {
if let Err(e) = c.deregister_consul_service(self.netapp.id).await {
error!("Error while deregistering from Consul: {}", e);
}
}
}
async fn discovery_loop(self: &Arc<Self>, mut stop_signal: watch::Receiver<bool>) {
while !*stop_signal.borrow() {
let peers_up = self
+1 -1
View File
@@ -1,6 +1,6 @@
[package]
name = "garage_table"
version = "2.3.0"
version = "2.4.0"
authors = ["Alex Auvolat <alex@adnab.me>"]
edition = "2018"
license = "AGPL-3.0"
+1 -1
View File
@@ -1,6 +1,6 @@
[package]
name = "garage_util"
version = "2.3.0"
version = "2.4.0"
authors = ["Alex Auvolat <alex@adnab.me>"]
edition = "2018"
license = "AGPL-3.0"
+1 -1
View File
@@ -1,6 +1,6 @@
[package]
name = "garage_web"
version = "2.3.0"
version = "2.4.0"
authors = ["Alex Auvolat <alex@adnab.me>", "Quentin Dufour <quentin@dufour.io>"]
edition = "2018"
license = "AGPL-3.0"