Compare commits

..

1 Commits

Author SHA1 Message Date
Alex Auvolat 5e4e870403 add boto3 test for STREAMING-UNSIGNED-PAYLOAD-TRAILER 2025-05-22 17:44:51 +02:00
169 changed files with 2408 additions and 6202 deletions
+6 -14
View File
@@ -1,6 +1,3 @@
labels:
nix: "enabled"
when:
event:
- push
@@ -12,32 +9,27 @@ when:
steps:
- name: check formatting
image: nixpkgs/nix:nixos-24.05
image: nixpkgs/nix:nixos-22.05
commands:
- nix-build -j4 --attr flakePackages.fmt
- nix-shell --attr devShell --run "cargo fmt -- --check"
- name: build
image: nixpkgs/nix:nixos-24.05
image: nixpkgs/nix:nixos-22.05
commands:
- nix-build -j4 --attr flakePackages.dev
- name: unit + func tests (lmdb)
image: nixpkgs/nix:nixos-24.05
image: nixpkgs/nix:nixos-22.05
commands:
- nix-build -j4 --attr flakePackages.tests-lmdb
- name: unit + func tests (sqlite)
image: nixpkgs/nix:nixos-24.05
image: nixpkgs/nix:nixos-22.05
commands:
- nix-build -j4 --attr flakePackages.tests-sqlite
- name: unit + func tests (fjall)
image: nixpkgs/nix:nixos-24.05
commands:
- nix-build -j4 --attr flakePackages.tests-fjall
- name: integration tests
image: nixpkgs/nix:nixos-24.05
image: nixpkgs/nix:nixos-22.05
commands:
- nix-build -j4 --attr flakePackages.dev
- nix-shell --attr ci --run ./script/test-smoke.sh || (cat /tmp/garage.log; false)
+2 -5
View File
@@ -1,6 +1,3 @@
labels:
nix: "enabled"
when:
event:
- deployment
@@ -11,7 +8,7 @@ depends_on:
steps:
- name: refresh-index
image: nixpkgs/nix:nixos-24.05
image: nixpkgs/nix:nixos-22.05
environment:
AWS_ACCESS_KEY_ID:
from_secret: garagehq_aws_access_key_id
@@ -22,7 +19,7 @@ steps:
- nix-shell --attr ci --run "refresh_index"
- name: multiarch-docker
image: nixpkgs/nix:nixos-24.05
image: nixpkgs/nix:nixos-22.05
environment:
DOCKER_AUTH:
from_secret: docker_auth
+7 -10
View File
@@ -1,6 +1,3 @@
labels:
nix: "enabled"
when:
event:
- deployment
@@ -19,17 +16,17 @@ matrix:
steps:
- name: build
image: nixpkgs/nix:nixos-24.05
image: nixpkgs/nix:nixos-22.05
commands:
- nix-build --attr releasePackages.${ARCH} --argstr git_version ${CI_COMMIT_TAG:-$CI_COMMIT_SHA}
- name: check is static binary
image: nixpkgs/nix:nixos-24.05
image: nixpkgs/nix:nixos-22.05
commands:
- nix-shell --attr ci --run "./script/not-dynamic.sh result/bin/garage"
- name: integration tests
image: nixpkgs/nix:nixos-24.05
image: nixpkgs/nix:nixos-22.05
commands:
- nix-shell --attr ci --run ./script/test-smoke.sh || (cat /tmp/garage.log; false)
when:
@@ -39,7 +36,7 @@ steps:
ARCH: i386
- name: upgrade tests from v1.0.0
image: nixpkgs/nix:nixos-24.05
image: nixpkgs/nix:nixos-22.05
commands:
- nix-shell --attr ci --run "./script/test-upgrade.sh v1.0.0 x86_64-unknown-linux-musl" || (cat /tmp/garage.log; false)
when:
@@ -47,7 +44,7 @@ steps:
ARCH: amd64
- name: upgrade tests from v0.8.4
image: nixpkgs/nix:nixos-24.05
image: nixpkgs/nix:nixos-22.05
commands:
- nix-shell --attr ci --run "./script/test-upgrade.sh v0.8.4 x86_64-unknown-linux-musl" || (cat /tmp/garage.log; false)
when:
@@ -55,7 +52,7 @@ steps:
ARCH: amd64
- name: push static binary
image: nixpkgs/nix:nixos-24.05
image: nixpkgs/nix:nixos-22.05
environment:
TARGET: "${TARGET}"
AWS_ACCESS_KEY_ID:
@@ -66,7 +63,7 @@ steps:
- nix-shell --attr ci --run "to_s3"
- name: docker build and publish
image: nixpkgs/nix:nixos-24.05
image: nixpkgs/nix:nixos-22.05
environment:
DOCKER_PLATFORM: "linux/${ARCH}"
CONTAINER_NAME: "dxflrs/${ARCH}_garage"
Generated
+861 -1119
View File
File diff suppressed because it is too large Load Diff
+21 -20
View File
@@ -24,18 +24,18 @@ default-members = ["src/garage"]
# Internal Garage crates
format_table = { version = "0.1.1", path = "src/format-table" }
garage_api_common = { version = "2.2.0", path = "src/api/common" }
garage_api_admin = { version = "2.2.0", path = "src/api/admin" }
garage_api_s3 = { version = "2.2.0", path = "src/api/s3" }
garage_api_k2v = { version = "2.2.0", path = "src/api/k2v" }
garage_block = { version = "2.2.0", path = "src/block" }
garage_db = { version = "2.2.0", path = "src/db", default-features = false }
garage_model = { version = "2.2.0", path = "src/model", default-features = false }
garage_net = { version = "2.2.0", path = "src/net" }
garage_rpc = { version = "2.2.0", path = "src/rpc" }
garage_table = { version = "2.2.0", path = "src/table" }
garage_util = { version = "2.2.0", path = "src/util" }
garage_web = { version = "2.2.0", path = "src/web" }
garage_api_common = { version = "2.0.0", path = "src/api/common" }
garage_api_admin = { version = "2.0.0", path = "src/api/admin" }
garage_api_s3 = { version = "2.0.0", path = "src/api/s3" }
garage_api_k2v = { version = "2.0.0", path = "src/api/k2v" }
garage_block = { version = "2.0.0", path = "src/block" }
garage_db = { version = "2.0.0", path = "src/db", default-features = false }
garage_model = { version = "2.0.0", path = "src/model", default-features = false }
garage_net = { version = "2.0.0", path = "src/net" }
garage_rpc = { version = "2.0.0", path = "src/rpc" }
garage_table = { version = "2.0.0", path = "src/table" }
garage_util = { version = "2.0.0", path = "src/util" }
garage_web = { version = "2.0.0", path = "src/web" }
k2v-client = { version = "0.0.4", path = "src/k2v-client" }
# External crates from crates.io
@@ -49,13 +49,17 @@ bytes = "1.0"
bytesize = "1.1"
cfg-if = "1.0"
chrono = { version = "0.4", features = ["serde"] }
crc-fast = "1.6"
crc32fast = "1.4"
crc32c = "0.6"
crc64fast-nvme = "1.2"
crypto-common = "0.1"
err-derive = "0.3"
gethostname = "0.4"
git-version = "0.3.4"
hex = "0.4"
hexdump = "0.1"
hmac = "0.12"
idna = "0.5"
itertools = "0.12"
ipnet = "2.9.0"
lazy_static = "1.4"
@@ -63,7 +67,6 @@ md-5 = "0.10"
mktemp = "0.5"
nix = { version = "0.29", default-features = false, features = ["fs"] }
nom = "7.1"
parking_lot = "0.12"
parse_duration = "2.1"
paste = "1.0"
pin-project = "1.0.12"
@@ -83,14 +86,12 @@ pretty_env_logger = "0.5"
structopt = { version = "0.3", default-features = false }
syslog-tracing = "0.3"
tracing = "0.1"
tracing-journald = "0.3.1"
tracing-subscriber = { version = "0.3", features = ["env-filter"] }
heed = { version = "0.11", default-features = false, features = ["lmdb"] }
rusqlite = "0.37"
rusqlite = "0.31.0"
r2d2 = "0.8"
r2d2_sqlite = "0.31"
fjall = "2.4"
r2d2_sqlite = "0.24"
async-compression = { version = "0.4", features = ["tokio", "zstd"] }
zstd = { version = "0.13", default-features = false }
@@ -137,7 +138,7 @@ prometheus = "0.13"
aws-sigv4 = { version = "1.1", default-features = false }
hyper-rustls = { version = "0.26", default-features = false, features = ["http1", "http2", "ring", "rustls-native-certs"] }
log = "0.4"
thiserror = "2.0"
thiserror = "1.0"
# ---- used only as build / dev dependencies ----
assert-json-diff = "2.0"
@@ -154,5 +155,5 @@ lto = "off"
[profile.release]
lto = true
codegen-units = 1
opt-level = 3
opt-level = "s"
strip = true
+1 -1
View File
@@ -1,7 +1,7 @@
<!DOCTYPE html>
<html>
<head>
<title>Garage administration API v0</title>
<title>Garage adminstration API v0</title>
<!-- needed for adaptive design -->
<meta charset="utf-8"/>
<meta name="viewport" content="width=device-width, initial-scale=1">
+1 -1
View File
@@ -1,7 +1,7 @@
<!DOCTYPE html>
<html>
<head>
<title>Garage administration API v1</title>
<title>Garage adminstration API v1</title>
<!-- needed for adaptive design -->
<meta charset="utf-8"/>
<meta name="viewport" content="width=device-width, initial-scale=1">
+1 -1
View File
@@ -1,7 +1,7 @@
<!DOCTYPE html>
<html>
<head>
<title>Garage administration API v2</title>
<title>Garage adminstration API v2</title>
<!-- needed for adaptive design -->
<meta charset="utf-8"/>
<meta name="viewport" content="width=device-width, initial-scale=1">
+157 -255
View File
@@ -12,7 +12,7 @@
"name": "AGPL-3.0",
"identifier": "AGPL-3.0"
},
"version": "v2.2.0"
"version": "v2.0.0"
},
"servers": [
{
@@ -31,12 +31,9 @@
"parameters": [
{
"name": "domain",
"in": "query",
"in": "path",
"description": "The domain name to check for",
"required": true,
"schema": {
"type": "string"
}
"required": true
}
],
"responses": {
@@ -103,7 +100,7 @@
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/BucketAliasEnum"
"$ref": "#/components/schemas/AddBucketAliasRequest"
}
}
},
@@ -408,12 +405,9 @@
"parameters": [
{
"name": "node",
"in": "query",
"in": "path",
"description": "Node ID to query, or `*` for all nodes, or `self` for the node responding to the request",
"required": true,
"schema": {
"type": "string"
}
"required": true
}
],
"responses": {
@@ -443,12 +437,9 @@
"parameters": [
{
"name": "id",
"in": "query",
"in": "path",
"description": "Admin API token ID",
"required": true,
"schema": {
"type": "string"
}
"required": true
}
],
"responses": {
@@ -471,12 +462,9 @@
"parameters": [
{
"name": "id",
"in": "query",
"in": "path",
"description": "ID of the bucket to delete",
"required": true,
"schema": {
"type": "string"
}
"required": true
}
],
"responses": {
@@ -505,12 +493,9 @@
"parameters": [
{
"name": "id",
"in": "query",
"in": "path",
"description": "Access key ID",
"required": true,
"schema": {
"type": "string"
}
"required": true
}
],
"responses": {
@@ -567,20 +552,26 @@
"parameters": [
{
"name": "id",
"in": "query",
"in": "path",
"description": "Admin API token ID",
"required": false,
"required": true,
"schema": {
"type": "string"
"type": [
"string",
"null"
]
}
},
{
"name": "search",
"in": "query",
"in": "path",
"description": "Partial token ID or name to search for",
"required": false,
"required": true,
"schema": {
"type": "string"
"type": [
"string",
"null"
]
}
}
],
@@ -611,12 +602,9 @@
"parameters": [
{
"name": "node",
"in": "query",
"in": "path",
"description": "Node ID to query, or `*` for all nodes, or `self` for the node responding to the request",
"required": true,
"schema": {
"type": "string"
}
"required": true
}
],
"requestBody": {
@@ -656,29 +644,38 @@
"parameters": [
{
"name": "id",
"in": "query",
"in": "path",
"description": "Exact bucket ID to look up",
"required": false,
"required": true,
"schema": {
"type": "string"
"type": [
"string",
"null"
]
}
},
{
"name": "globalAlias",
"in": "query",
"in": "path",
"description": "Global alias of bucket to look up",
"required": false,
"required": true,
"schema": {
"type": "string"
"type": [
"string",
"null"
]
}
},
{
"name": "search",
"in": "query",
"in": "path",
"description": "Partial ID or alias to search for",
"required": false,
"required": true,
"schema": {
"type": "string"
"type": [
"string",
"null"
]
}
}
],
@@ -816,30 +813,6 @@
}
}
},
"/v2/GetCurrentAdminTokenInfo": {
"get": {
"tags": [
"Admin API token"
],
"description": "\nReturn information about the calling admin API token.\n ",
"operationId": "GetCurrentAdminTokenInfo",
"responses": {
"200": {
"description": "Information about the admin token",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/GetCurrentAdminTokenInfoResponse"
}
}
}
},
"500": {
"description": "Internal server error"
}
}
}
},
"/v2/GetKeyInfo": {
"get": {
"tags": [
@@ -850,27 +823,33 @@
"parameters": [
{
"name": "id",
"in": "query",
"in": "path",
"description": "Access key ID",
"required": false,
"required": true,
"schema": {
"type": "string"
"type": [
"string",
"null"
]
}
},
{
"name": "search",
"in": "query",
"in": "path",
"description": "Partial key ID or name to search for",
"required": false,
"required": true,
"schema": {
"type": "string"
"type": [
"string",
"null"
]
}
},
{
"name": "showSecretKey",
"in": "query",
"in": "path",
"description": "Whether to return the secret access key",
"required": false,
"required": true,
"schema": {
"type": "boolean"
}
@@ -903,12 +882,9 @@
"parameters": [
{
"name": "node",
"in": "query",
"in": "path",
"description": "Node ID to query, or `*` for all nodes, or `self` for the node responding to the request",
"required": true,
"schema": {
"type": "string"
}
"required": true
}
],
"responses": {
@@ -938,12 +914,9 @@
"parameters": [
{
"name": "node",
"in": "query",
"in": "path",
"description": "Node ID to query, or `*` for all nodes, or `self` for the node responding to the request",
"required": true,
"schema": {
"type": "string"
}
"required": true
}
],
"responses": {
@@ -973,12 +946,9 @@
"parameters": [
{
"name": "node",
"in": "query",
"in": "path",
"description": "Node ID to query, or `*` for all nodes, or `self` for the node responding to the request",
"required": true,
"schema": {
"type": "string"
}
"required": true
}
],
"requestBody": {
@@ -1018,12 +988,9 @@
"parameters": [
{
"name": "node",
"in": "query",
"in": "path",
"description": "Node ID to query, or `*` for all nodes, or `self` for the node responding to the request",
"required": true,
"schema": {
"type": "string"
}
"required": true
}
],
"requestBody": {
@@ -1097,7 +1064,7 @@
"parameters": [
{
"name": "bucketId",
"in": "query",
"in": "path",
"required": true,
"schema": {
"type": "string"
@@ -1105,7 +1072,7 @@
},
{
"name": "key",
"in": "query",
"in": "path",
"required": true,
"schema": {
"type": "string"
@@ -1142,12 +1109,9 @@
"parameters": [
{
"name": "node",
"in": "query",
"in": "path",
"description": "Node ID to query, or `*` for all nodes, or `self` for the node responding to the request",
"required": true,
"schema": {
"type": "string"
}
"required": true
}
],
"requestBody": {
@@ -1211,12 +1175,9 @@
"parameters": [
{
"name": "node",
"in": "query",
"in": "path",
"description": "Node ID to query, or `*` for all nodes, or `self` for the node responding to the request",
"required": true,
"schema": {
"type": "string"
}
"required": true
}
],
"responses": {
@@ -1294,12 +1255,9 @@
"parameters": [
{
"name": "node",
"in": "query",
"in": "path",
"description": "Node ID to query, or `*` for all nodes, or `self` for the node responding to the request",
"required": true,
"schema": {
"type": "string"
}
"required": true
}
],
"requestBody": {
@@ -1363,12 +1321,9 @@
"parameters": [
{
"name": "node",
"in": "query",
"in": "path",
"description": "Node ID to query, or `*` for all nodes, or `self` for the node responding to the request",
"required": true,
"schema": {
"type": "string"
}
"required": true
}
],
"requestBody": {
@@ -1409,7 +1364,7 @@
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/BucketAliasEnum"
"$ref": "#/components/schemas/RemoveBucketAliasRequest"
}
}
},
@@ -1442,12 +1397,9 @@
"parameters": [
{
"name": "node",
"in": "query",
"in": "path",
"description": "Node ID to query, or `*` for all nodes, or `self` for the node responding to the request",
"required": true,
"schema": {
"type": "string"
}
"required": true
}
],
"requestBody": {
@@ -1511,12 +1463,9 @@
"parameters": [
{
"name": "node",
"in": "query",
"in": "path",
"description": "Node ID to query, or `*` for all nodes, or `self` for the node responding to the request",
"required": true,
"schema": {
"type": "string"
}
"required": true
}
],
"requestBody": {
@@ -1556,12 +1505,9 @@
"parameters": [
{
"name": "id",
"in": "query",
"in": "path",
"description": "Admin API token ID",
"required": true,
"schema": {
"type": "string"
}
"required": true
}
],
"requestBody": {
@@ -1601,12 +1547,9 @@
"parameters": [
{
"name": "id",
"in": "query",
"in": "path",
"description": "ID of the bucket to update",
"required": true,
"schema": {
"type": "string"
}
"required": true
}
],
"requestBody": {
@@ -1684,12 +1627,9 @@
"parameters": [
{
"name": "id",
"in": "query",
"in": "path",
"description": "Access key ID",
"required": true,
"schema": {
"type": "string"
}
"required": true
}
],
"requestBody": {
@@ -1722,6 +1662,24 @@
},
"components": {
"schemas": {
"AddBucketAliasRequest": {
"allOf": [
{
"$ref": "#/components/schemas/BucketAliasEnum"
},
{
"type": "object",
"required": [
"bucketId"
],
"properties": {
"bucketId": {
"type": "string"
}
}
}
]
},
"AddBucketAliasResponse": {
"$ref": "#/components/schemas/GetBucketInfoResponse"
},
@@ -1939,13 +1897,9 @@
{
"type": "object",
"required": [
"bucketId",
"globalAlias"
],
"properties": {
"bucketId": {
"type": "string"
},
"globalAlias": {
"type": "string"
}
@@ -1954,7 +1908,6 @@
{
"type": "object",
"required": [
"bucketId",
"localAlias",
"accessKeyId"
],
@@ -1962,9 +1915,6 @@
"accessKeyId": {
"type": "string"
},
"bucketId": {
"type": "string"
},
"localAlias": {
"type": "string"
}
@@ -2433,7 +2383,7 @@
"knownNodes",
"connectedNodes",
"storageNodes",
"storageNodesUp",
"storageNodesOk",
"partitions",
"partitionsQuorum",
"partitionsAllOk"
@@ -2473,7 +2423,7 @@
"description": "the number of storage nodes currently registered in the cluster layout",
"minimum": 0
},
"storageNodesUp": {
"storageNodesOk": {
"type": "integer",
"description": "the number of storage nodes to which a connection is currently open",
"minimum": 0
@@ -2608,9 +2558,6 @@
}
}
},
"GetCurrentAdminTokenInfoResponse": {
"$ref": "#/components/schemas/GetAdminTokenInfoResponse"
},
"GetKeyInfoResponse": {
"type": "object",
"required": [
@@ -3173,15 +3120,9 @@
"blocksPurged",
"objectsDeleted",
"uploadsDeleted",
"versionsDeleted",
"blockRefsPurged"
"versionsDeleted"
],
"properties": {
"blockRefsPurged": {
"type": "integer",
"format": "int64",
"minimum": 0
},
"blocksPurged": {
"type": "integer",
"format": "int64",
@@ -3633,15 +3574,9 @@
"blocksPurged",
"objectsDeleted",
"uploadsDeleted",
"versionsDeleted",
"blockRefsPurged"
"versionsDeleted"
],
"properties": {
"blockRefsPurged": {
"type": "integer",
"format": "int64",
"minimum": 0
},
"blocksPurged": {
"type": "integer",
"format": "int64",
@@ -3902,49 +3837,6 @@
}
]
},
"NodeRoleChangeRequest": {
"oneOf": [
{
"type": "object",
"required": [
"id",
"remove"
],
"properties": {
"id": {
"type": "string",
"description": "ID of the node for which this change applies"
},
"remove": {
"type": "boolean",
"description": "Set `remove` to `true` to remove the node from the layout"
}
}
},
{
"$ref": "#/components/schemas/NodeRoleUpdate"
}
]
},
"NodeRoleUpdate": {
"allOf": [
{
"$ref": "#/components/schemas/NodeAssignedRole"
},
{
"type": "object",
"required": [
"id"
],
"properties": {
"id": {
"type": "string",
"description": "ID of the node for which this change applies"
}
}
}
]
},
"NodeUpdateTrackers": {
"type": "object",
"required": [
@@ -4006,6 +3898,24 @@
}
]
},
"RemoveBucketAliasRequest": {
"allOf": [
{
"$ref": "#/components/schemas/BucketAliasEnum"
},
{
"type": "object",
"required": [
"bucketId"
],
"properties": {
"bucketId": {
"type": "string"
}
}
}
]
},
"RemoveBucketAliasResponse": {
"$ref": "#/components/schemas/GetBucketInfoResponse"
},
@@ -4063,18 +3973,6 @@
"$ref": "#/components/schemas/ScrubCommand"
}
}
},
{
"type": "string",
"enum": [
"aliases"
]
},
{
"type": "string",
"enum": [
"clearResyncQueue"
]
}
]
},
@@ -4195,7 +4093,7 @@
"roles": {
"type": "array",
"items": {
"$ref": "#/components/schemas/NodeRoleChangeRequest"
"$ref": "#/components/schemas/NodeRoleChange"
},
"description": "New node roles to assign or remove in the cluster layout"
}
@@ -4353,39 +4251,43 @@
"WorkerStateResp": {
"oneOf": [
{
"$ref": "#/components/schemas/Yolo"
},
"type": "string",
"enum": [
"busy"
]
},
{
"$ref": "#/components/schemas/WorkerStateRespStrs"
}
]
},
"Yolo": {
"type": "object",
"required": [
"throttled"
],
"properties": {
"throttled": {
"type": "object",
"required": [
"durationSecs"
"throttled"
],
"properties": {
"durationSecs": {
"type": "number",
"format": "float"
"throttled": {
"type": "object",
"required": [
"durationSecs"
],
"properties": {
"durationSecs": {
"type": "number",
"format": "float"
}
}
}
}
},
{
"type": "string",
"enum": [
"idle"
]
},
{
"type": "string",
"enum": [
"done"
]
}
}
},
"WorkerStateRespStrs": {
"type": "string",
"enum": [
"busy",
"idle",
"done"
]
},
"ZoneRedundancy": {
+10 -184
View File
@@ -12,9 +12,8 @@ In this section, we cover the following web applications:
| [Mastodon](#mastodon) | ✅ | Natively supported |
| [Matrix](#matrix) | ✅ | Tested with `synapse-s3-storage-provider` |
| [ejabberd](#ejabberd) | ✅ | `mod_s3_upload` |
| [Ente](#ente) | | Natively supported |
| [Pixelfed](#pixelfed) | ❓ | Natively supported |
| [Pleroma](#pleroma) | ✅ | Natively supported |
| [Pixelfed](#pixelfed) | | Not yet tested |
| [Pleroma](#pleroma) | ❓ | Not yet tested |
| [Lemmy](#lemmy) | ✅ | Supported with pict-rs |
| [Funkwhale](#funkwhale) | ❓ | Not yet tested |
| [Misskey](#misskey) | ❓ | Not yet tested |
@@ -70,7 +69,7 @@ $CONFIG = array(
'hostname' => '127.0.0.1', // Can also be a domain name, eg. garage.example.com
'port' => 3900, // Put your reverse proxy port or your S3 API port
'use_ssl' => false, // Set it to true if you have a TLS enabled reverse proxy
'region' => 'garage', // Garage default region is named "garage", edit according to your cluster config
'region' => 'garage', // Garage has only one region named "garage"
'use_path_style' => true // Garage supports only path style, must be set to true
],
],
@@ -136,7 +135,7 @@ bucket but doesn't also know the secret encryption key.
*Click on the picture to zoom*
Add a new external storage. Put what you want in "folder name" (eg. "shared"). Select "Amazon S3". Keep "Access Key" for the Authentication field.
In Configuration, put your bucket name (eg. nextcloud), the host (eg. 127.0.0.1), the port (eg. 3900 or 443), the region ("garage" if you use the default, or the one your configured in your `garage.toml`). Tick the SSL box if you have put an HTTPS proxy in front of garage. You must tick the "Path access" box and you must leave the "Legacy authentication (v2)" box empty. Put your Key ID (eg. GK...) and your Secret Key in the last two input boxes. Finally click on the tick symbol on the right of your screen.
In Configuration, put your bucket name (eg. nextcloud), the host (eg. 127.0.0.1), the port (eg. 3900 or 443), the region (garage). Tick the SSL box if you have put an HTTPS proxy in front of garage. You must tick the "Path access" box and you must leave the "Legacy authentication (v2)" box empty. Put your Key ID (eg. GK...) and your Secret Key in the last two input boxes. Finally click on the tick symbol on the right of your screen.
Now go to your "Files" app and a new "linked folder" has appeared with the name you chose earlier (eg. "shared").
@@ -192,10 +191,10 @@ garage key create peertube-key
Keep the Key ID and the Secret key in a pad, they will be needed later.
We need two buckets, one for normal videos (named peertube-videos) and one for webtorrent videos (named peertube-playlists).
We need two buckets, one for normal videos (named peertube-video) and one for webtorrent videos (named peertube-playlist).
```bash
garage bucket create peertube-videos
garage bucket create peertube-playlists
garage bucket create peertube-playlist
```
Now we allow our key to read and write on these buckets:
@@ -239,7 +238,7 @@ object_storage:
# Put localhost only if you have a garage instance running on that node
endpoint: 'http://localhost:3900' # or "garage.example.com" if you have TLS on port 443
# Garage default region is named "garage", edit according to your config
# Garage supports only one region for now, named garage
region: 'garage'
credentials:
@@ -254,7 +253,7 @@ object_storage:
proxify_private_files: false
streaming_playlists:
bucket_name: 'peertube-playlists'
bucket_name: 'peertube-playlist'
# Keep it empty for our example
prefix: ''
@@ -442,7 +441,7 @@ media_storage_providers:
store_synchronous: True # do we want to wait that the file has been written before returning?
config:
bucket: matrix # the name of our bucket, we chose matrix earlier
region_name: garage # "garage" by default, edit according to your cluster config
region_name: garage # only "garage" is supported for the region field
endpoint_url: http://localhost:3900 # the path to the S3 endpoint
access_key_id: "GKxxx" # your Key ID
secret_access_key: "xxxx" # your Secret Key
@@ -568,186 +567,13 @@ The module can then be configured with:
Other configuration options can be found in the
[configuration YAML file](https://github.com/processone/ejabberd-contrib/blob/master/mod_s3_upload/conf/mod_s3_upload.yml).
## Ente
Ente is an alternative for Google Photos and Apple Photos. It [can be selfhosted](https://help.ente.io/self-hosting/) and is working fine with Garage as of May 2024.
As a first step we need to create a bucket and a key for Ente:
```bash
garage bucket create ente
garage key create ente-key
# For the CORS setup to work, the key needs to be --owner as well, at least temporarily.
garage bucket allow ente --read --write --owner --key ente-key
```
We also need to setup some CORS rules to allow the Ente frontend to access the bucket:
```bash
export CORS='{"CORSRules":[{"AllowedHeaders":["*"],"AllowedMethods":["GET", "PUT", "POST", "DELETE"],"AllowedOrigins":["*"], "ExposeHeaders":["ETag"]}]}'
aws s3api put-bucket-cors --bucket ente --cors-configuration $CORS
```
Now we need to configure ente-server to use our bucket. This is explained [in the Ente S3 documentation](https://help.ente.io/self-hosting/guides/external-s3).
Prepare a configuration file for ente's backend as `museum.yaml`:
```yaml
credentials-file: /credentials.yaml
apps:
public-albums: https://albums.example.tld # If you want to use the share album feature
internal:
hardcoded-ott:
local-domain-suffix: "@example.com" # Your domain
local-domain-value: 123456 # Custom One-Time Password since we are not sending mail by default
key:
# WARNING -- You MUST CHANGE the values below
# Someone has made an image that can do it for you : https://github.com/EdyTheCow/ente-selfhost/blob/main/images/ente-server-tools/Dockerfile
# Simply build it yourself or run docker run --rm ghcr.io/edythecow/ente-server-tools go run tools/gen-random-keys/main.go
encryption: yvmG/RnzKrbCb9L3mgsmoxXr9H7i2Z4qlbT0mL3ln4w= # CHANGE THIS VALUE
hash: KXYiG07wC7GIgvCSdg+WmyWdXDAn6XKYJtp/wkEU7x573+byBRAYtpTP0wwvi8i/4l37uicX1dVTUzwH3sLZyw== # CHANGE THIS VALUE
jwt:
secret: i2DecQmfGreG6q1vBj5tCokhlN41gcfS2cjOs9Po-u8= # CHANGE THIS VALUE
```
The full configuration file can be found [here](https://github.com/ente-io/ente/blob/main/server/configurations/local.yaml)
Then prepare a credentials file as `credentials.yaml`
```yaml
db:
host: postgres
port: 5432
name: <ente_db_name>
user: <pguser>
password: <pgpass>
s3:
# Override the primary and secondary hot storage. The commented out values
# are the defaults.
#
hot_storage:
primary: b2-eu-cen
# secondary: wasabi-eu-central-2-v3
# If true, enable some workarounds to allow us to use a local minio instance
# for object storage.
#
# 1. Disable SSL.
# 2. Use "path" style S3 URLs (see `use_path_style_urls` below).
# 3. Directly download the file during replication instead of going via the
# Cloudflare worker.
# 4. Do not specify storage classes when uploading objects (since minio does
# not support them, specifically it doesn't support GLACIER).
are_local_buckets: true
# To use "path" style S3 URLs instead of DNS-based bucket access
# default to true if you set "are_local_buckets: true"
# use_path_style_urls: true
b2-eu-cen: # Don't change this key, it is hardcoded
key: <keyID>
secret: <keySecret>
endpoint: garage:3900 # publically accessible endpoint of your garage instance
region: garage
bucket: <yourbucketName>
use_path_style: true
# you can specify secondary locations, names are hardcoded as well
# wasabi-eu-central-2-v3:
# scw-eu-fr-v3:
# and you can also specify a bucket to be used for embeddings, preview etc..
# default to the first bucket
# derived-storage: wasabi-eu-central-2-derived
```
Finally you can run it with Docker :
```bash
docker run -d --name ente-server --restart unless-stopped -v /path/to/museum.yaml:/museum.yaml -v /path/to/credentials.yaml:/credentials.yaml -p 8080:8080 ghcr.io/ente-io/ente-server
```
For more information on deployment you can check the [ente documentation](https://help.ente.io/self-hosting/)
## Pixelfed
[Pixelfed Technical Documentation > Configuration](https://docs.pixelfed.org/technical-documentation/env.html#filesystem)
## Pleroma
### Creating your bucket
This is the usual Garage setup:
```bash
garage key new --name pleroma-key
garage bucket create pleroma
garage bucket allow pleroma --read --write --owner --key pleroma-key
```
We also need to expose these buckets publicly to serve their content to users:
```bash
garage bucket website --allow pleroma
```
Note the Key ID and Secret Key.
### Configure Pleroma
Update your Pleroma configuration like that in `/etc/pleroma/config.exs`.
```
config :pleroma, Pleroma.Upload,
uploader: Pleroma.Uploaders.S3,
base_url: "https://pleroma.garage.example.tld"
config :ex_aws, :s3,
access_key_id: "GW...",
secret_access_key: "XXX",
region: "garage",
host: "api.garage.example.tld"
```
And restart Pleroma.
You can found more information in [Pleroma Documentation > Pleroma.Uploaders.S3](https://docs-develop.pleroma.social/backend/configuration/cheatsheet/#pleromauploaderss3)
### Migrating your data
Pleroma have an internal migration tool that can encounter some fatal error
```
** (EXIT from #PID<0.98.0>) an exception was raised:
** (File.Error) could not stream "/var/lib/pleroma/uploads/09/f8": illegal operation on a directory
(elixir 1.17.3) lib/file/stream.ex:100: anonymous fn/3 in Enumerable.File.Stream.reduce/3
(elixir 1.17.3) lib/stream.ex:1675: anonymous fn/5 in Stream.resource/3
(elixir 1.17.3) lib/stream.ex:1891: Enumerable.Stream.do_each/4
(elixir 1.17.3) lib/task/supervised.ex:370: Task.Supervised.stream_reduce/7
(elixir 1.17.3) lib/enum.ex:4423: Enum.map/2
(ex_aws_s3 2.5.8) lib/ex_aws/s3/upload.ex:141: ExAws.Operation.ExAws.S3.Upload.perform/2
(pleroma 2.10.0) lib/pleroma/uploaders/s3.ex:60: Pleroma.Uploaders.S3.put_file/1
(pleroma 2.10.0) lib/pleroma/uploaders/uploader.ex:49: Pleroma.Uploaders.Uploader.put_file/2
```
So, use [your best tool](https://garagehq.deuxfleurs.fr/documentation/connect/cli/) to sync `/var/lib/pleroma/uploads/` in your S3.
Then, to avoid some non existant problem (just in case of), run this command
```bash
while true
do
rm -vr $(./bin/pleroma_ctl uploads migrate_local S3 2>&1 | grep "could not stream" | awk -F '"' '{print $2}')
sleep 5
done
```
If you have many files, stop this command sometime and the command bellow (interactive) to delete local
file after upload. Then restart the loop.
```bash
./bin/pleroma_ctl uploads migrate_local S3 --delete
```
And *voilà*
[Pleroma Documentation > Pleroma.Uploaders.S3](https://docs-develop.pleroma.social/backend/configuration/cheatsheet/#pleromauploaderss3)
## Lemmy
-56
View File
@@ -161,59 +161,3 @@ kopia repository validate-provider
You can then run all the standard kopia commands: `kopia snapshot create`, `kopia mount`...
Everything should work out-of-the-box.
## Plakar
Create your key and bucket on Garage server:
```bash
garage key create my-plakar-key
garage bucket create plakar-backups
garage bucket allow plakar-backups --read --write --key my-plakar-key
```
On Plakar server, add your Garage as a storage location:
```bash
plakar store add garageS3 s3://my-garage.tld/plakar-backups \
region=garage # Or as you've specified in garage.toml \
access_key=<Key ID from "garage key info my-plakar-key"> \
secret_access_key=<Secret key from "garage key info my-plakar-key">
```
Then create the repository.
```bash
plakar at @garageS3 create -plaintext # Unencrypted
# or
plakar at @garageS3 create #encrypted
```
If you encrypt your backups (Plakar default), you will need to define a strong passphrase. Do not forget to save your password safely. It will be needed to decrypt your backups.
After the repository has been created, check that everything works as expected (that might give an empty result as no file has been added yet, but no error message):
```bash
plakar at @garageS3 check
```
Now that everything is configure, you can use Garage as your backups storage. For instance sync it with a local backup storage:
```bash
$ plakar at ~/backups sync to @garageS3
```
Or list the S3 storage content:
```bash
$ plakar at @garageS3 ls
```
More information in Plakar documentation: https://www.plakar.io/docs/main/quickstart/
## Synology HyperBackup
HyperBackup can be configured to upload backups to garage using a custom S3 destination. However, the HyperBackup client hardcodes the `us-east-1` region that is a critical input to the v4 signature process. If garage is not set to `us-east-1`, HyperBackup will recognize available buckets, but fail during the final setup stage.
In garage.toml:
```toml
[s3_api]
s3_region = "us-east-1"
```
+1 -9
View File
@@ -149,15 +149,6 @@ rclone help
This will tremendously accelerate operations such as `rclone sync` or `rclone ncdu` by reducing the number
of ListObjects calls that are made.
**Garage behind Cloudflare proxy:** when running Garage behind Cloudflare proxy, you might see `Response: error 403 Forbidden, Forbidden: Invalid signature` error in your garage logs or `AccessDenied: Forbidden: Invalid signature` error in rclone logs. Try adding `--s3-sign-accept-encoding=false` flag to your rclone command and see if the issue is resolved.
```bash
# this throws an error
rclone lsd garage:
# this should work
rclone lsd --s3-sign-accept-encoding=false garage:
```
## `s3cmd`
@@ -323,3 +314,4 @@ ls
```
And through the web interface at http://[::1]:8080/web/client
+12 -24
View File
@@ -8,18 +8,18 @@ have published Ansible roles. We list them and compare them below.
## Comparison of Ansible roles
| Feature | [ansible-role-garage](#zorun-ansible-role-garage) | [garage-docker-ansible-deploy](#moan0s-garage-docker-ansible-deploy) | [eddster2309 ansible-role-garage](#eddster2309-ansible-role-garage) |
|------------------------------------|---------------------------------------------|---------------------------------------------------------------|---------------------------------|
| **Runtime** | Systemd | Docker | Systemd |
| **Target OS** | Any Linux | Any Linux | Any Linux |
| **Architecture** | amd64, arm64, i686 | amd64, arm64 | arm64, arm, 386, amd64 |
| **Additional software** | None | Traefik | Ngnix and Keepalived (optional) |
| **Automatic node connection** | ❌ | ✅ | ✅ |
| **Layout management** | ❌ | ✅ | ✅ |
| **Manage buckets & keys** | ❌ | ✅ (basic) | ✅ |
| **Allow custom Garage config** | ✅ | ❌ | ❌ |
| **Facilitate Garage upgrades** | ✅ | ❌ | ✅ |
| **Multiple instances on one host** | ✅ | ✅ | ❌ |
| Feature | [ansible-role-garage](#zorun-ansible-role-garage) | [garage-docker-ansible-deploy](#moan0s-garage-docker-ansible-deploy) |
|------------------------------------|---------------------------------------------|---------------------------------------------------------------|
| **Runtime** | Systemd | Docker |
| **Target OS** | Any Linux | Any Linux |
| **Architecture** | amd64, arm64, i686 | amd64, arm64 |
| **Additional software** | None | Traefik |
| **Automatic node connection** | ❌ | ✅ |
| **Layout management** | ❌ | ✅ |
| **Manage buckets & keys** | ❌ | ✅ (basic) |
| **Allow custom Garage config** | ✅ | ❌ |
| **Facilitate Garage upgrades** | ✅ | ❌ |
| **Multiple instances on one host** | ✅ | ✅ |
## zorun/ansible-role-garage
@@ -49,15 +49,3 @@ structured DNS names, etc).
As a result, this role makes it easier to start with Garage on Ansible,
but is less flexible.
## eddster2309/ansible-role-garage
[Source code](https://github.com/eddster2309/ansible-role-garage), [Ansible galaxy](https://galaxy.ansible.com/ui/standalone/roles/eddster2309/garage/)
This role is a opinionated but customisable role using the official Garage
static binaries and only requires Systemd. As such it should work on any
Linux based host. It includes all the nesscary configuration to
automatically setup a clustered Garage deployment. Most Garage
configuration options are exposed through Ansible variables so while you
can't provide a custom config you can get very close. It can optionally
installed a HA nginx deployment with Keepalived.
+4 -15
View File
@@ -15,10 +15,9 @@ Alpine Linux repositories (available since v3.17):
apk add garage
```
The default configuration file is installed to `/etc/garage/garage.toml`. You can run
Garage using: `rc-service garage start`.
If you don't specify `rpc_secret`, it will be automatically replaced with a random string on the first start.
The default configuration file is installed to `/etc/garage.toml`. You can run
Garage using: `rc-service garage start`. If you don't specify `rpc_secret`, it
will be automatically replaced with a random string on the first start.
Please note that this package is built without Consul discovery, Kubernetes
discovery, OpenTelemetry exporter, and K2V features (K2V will be enabled once
@@ -27,11 +26,7 @@ it's stable).
## Arch Linux
Garage is available in the official repositories under [extra](https://archlinux.org/packages/extra/x86_64/garage).
```bash
pacman -S garage
```
Garage is available in the [AUR](https://aur.archlinux.org/packages/garage).
## FreeBSD
@@ -44,9 +39,3 @@ pkg install garage
```bash
nix-shell -p garage
```
## conda-forge
```bash
pixi global install garage
```
+6 -9
View File
@@ -20,10 +20,10 @@ sudo apt-get update
sudo apt-get install build-essential
```
## Building from source from the Forgejo repository
## Building from source from the Gitea repository
The primary location for Garage's source code is the
[Forgejo repository](https://git.deuxfleurs.fr/Deuxfleurs/garage),
[Gitea repository](https://git.deuxfleurs.fr/Deuxfleurs/garage),
which contains all of the released versions as well as the code
for the developpement of the next version.
@@ -85,14 +85,11 @@ The following feature flags are available in v0.8.0:
| Feature flag | Enabled | Description |
| ------------ | ------- | ----------- |
| `bundled-libs` | *by default* | Use bundled version of sqlite3, zstd, lmdb and libsodium |
| `consul-discovery` | optional | Enable automatic registration and discovery<br>of cluster nodes through the Consul API |
| `fjall` | experimental | Enable using Fjall to store Garage's metadata |
| `journald` | optional | Enable logging to systemd-journald with<br>`GARAGE_LOG_TO_JOURNALD=true` environment variable set |
| `system-libs` | optional | Use system version of sqlite3, zstd, lmdb and libsodium<br>if available (exclusive with `bundled-libs`, build using<br>`cargo build --no-default-features --features system-libs`) |
| `k2v` | optional | Enable the experimental K2V API (if used, all nodes on your<br>Garage cluster must have it enabled as well) |
| `kubernetes-discovery` | optional | Enable automatic registration and discovery<br>of cluster nodes through the Kubernetes API |
| `lmdb` | *by default* | Enable using LMDB to store Garage's metadata |
| `metrics` | *by default* | Enable collection of metrics in Prometheus format on the admin API |
| `sqlite` | *by default* | Enable using Sqlite3 to store Garage's metadata |
| `syslog` | optional | Enable logging to Syslog with<br>`GARAGE_LOG_TO_SYSLOG=true` environment variable set |
| `system-libs` | optional | Use system version of sqlite3, zstd, lmdb and libsodium<br>if available (exclusive with `bundled-libs`, build using<br>`cargo build --no-default-features --features system-libs`) |
| `telemetry-otlp` | optional | Enable collection of execution traces using OpenTelemetry |
| `syslog` | optional | Enable logging to Syslog |
| `lmdb` | *by default* | Enable using LMDB to store Garage's metadata |
| `sqlite` | *by default* | Enable using Sqlite3 to store Garage's metadata |
+2 -9
View File
@@ -11,7 +11,7 @@ Firstly clone the repository:
```bash
git clone https://git.deuxfleurs.fr/Deuxfleurs/garage
cd garage/script/helm
cd garage/scripts/helm
```
Deploy with default options:
@@ -26,13 +26,6 @@ Or deploy with custom values:
helm install --create-namespace --namespace garage garage ./garage -f values.override.yaml
```
If you want to manage the CustomRessourceDefinition used by garage for its `kubernetes_discovery` outside of the helm chart, add `garage.kubernetesSkipCrd: true` to your custom values and use the kustomization before deploying the helm chart:
```bash
kubectl apply -k ../k8s/crd
helm install --create-namespace --namespace garage garage ./garage -f values.override.yaml
```
After deploying, cluster layout must be configured manually as described in [Creating a cluster layout](@/documentation/quick-start/_index.md#creating-a-cluster-layout). Use the following command to access garage CLI:
```bash
@@ -52,7 +45,7 @@ This is an example `values.overrride.yaml` for deploying in a microk8s cluster w
```yaml
garage:
# Use only 2 replicas per object
replicationFactor: 2
replicationMode: "2"
# Start 4 instances (StatefulSets) of garage
deployment:
+5 -5
View File
@@ -96,14 +96,14 @@ to store 2 TB of data in total.
## Get a Docker image
Our docker image is currently named `dxflrs/garage` and is stored on the [Docker Hub](https://hub.docker.com/r/dxflrs/garage/tags?page=1&ordering=last_updated).
We encourage you to use a fixed tag (eg. `v2.2.0`) and not the `latest` tag.
For this example, we will use the latest published version at the time of the writing which is `v2.2.0` but it's up to you
We encourage you to use a fixed tag (eg. `v2.0.0`) and not the `latest` tag.
For this example, we will use the latest published version at the time of the writing which is `v2.0.0` but it's up to you
to check [the most recent versions on the Docker Hub](https://hub.docker.com/r/dxflrs/garage/tags?page=1&ordering=last_updated).
For example:
```
sudo docker pull dxflrs/garage:v2.2.0
sudo docker pull dxflrs/garage:v2.0.0
```
## Deploying and configuring Garage
@@ -171,7 +171,7 @@ docker run \
-v /etc/garage.toml:/etc/garage.toml \
-v /var/lib/garage/meta:/var/lib/garage/meta \
-v /var/lib/garage/data:/var/lib/garage/data \
dxflrs/garage:v2.2.0
dxflrs/garage:v2.0.0
```
With this command line, Garage should be started automatically at each boot.
@@ -185,7 +185,7 @@ If you want to use `docker-compose`, you may use the following `docker-compose.y
version: "3"
services:
garage:
image: dxflrs/garage:v2.2.0
image: dxflrs/garage:v2.0.0
network_mode: "host"
restart: unless-stopped
volumes:
+2 -2
View File
@@ -7,7 +7,7 @@ The main reason to add a reverse proxy in front of Garage is to provide TLS to y
In production you will likely need your certificates signed by a certificate authority.
The most automated way is to use a provider supporting the [ACME protocol](https://datatracker.ietf.org/doc/html/rfc8555)
such as [Let's Encrypt](https://letsencrypt.org/) or [ZeroSSL](https://zerossl.com/).
such as [Let's Encrypt](https://letsencrypt.org/), [ZeroSSL](https://zerossl.com/) or [Buypass Go SSL](https://www.buypass.com/ssl/products/acme).
If you are only testing Garage, you can generate a self-signed certificate to follow the documentation:
@@ -97,7 +97,7 @@ server {
location / {
proxy_pass http://s3_backend;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header Host $http_host;
proxy_set_header Host $host;
# Disable buffering to a temporary file.
proxy_max_temp_file_size 0;
}
-1
View File
@@ -28,7 +28,6 @@ StateDirectory=garage
DynamicUser=true
ProtectHome=true
NoNewPrivileges=true
LimitNOFILE=42000
[Install]
WantedBy=multi-user.target
+2 -4
View File
@@ -59,13 +59,11 @@ Garage themselves for the following tasks:
- Hosting of their homepage, [privacyguides.org](https://www.privacyguides.org/), and various other static sites
- As a PowerDNS authoritative zone backend through [Lightning Stream](https://doc.powerdns.com/lightningstream/latest/index.html) and [LMDB](https://doc.powerdns.com/authoritative/backends/lmdb.html)
- As a Mastodon media storage backend for [mstdn.party](https://mstdn.party/) and [mstdn.plus](https://mstdn.plus/)
- As a Mastodon object storage backend for [mstdn.party](https://mstdn.party/) and [mstdn.plus](https://mstdn.plus/)
- As a PeerTube storage backend for [neat.tube](https://neat.tube/)
- As a [Matrix media backend](https://github.com/matrix-org/synapse-s3-storage-provider)
Triplebit's Garage cluster is a multi-site cluster currently composed of
15 storage nodes in 3 physical locations.
10 nodes in 3 physical locations.
+1 -1
View File
@@ -42,7 +42,7 @@ You may pause an ongoing scrub using `garage repair scrub pause`, but note that
the scrub will resume automatically 24 hours later as Garage will not let your
cluster run without a regular scrub. If the scrub procedure is too intensive
for your servers and is slowing down your workload, the recommended solution
is to increase the "scrub tranquility" using `garage worker set scrub-tranquility`.
is to increase the "scrub tranquility" using `garage repair scrub set-tranquility`.
A higher tranquility value will make Garage take longer pauses between two block
verifications. Of course, scrubbing the entire data store will also take longer.
+1 -4
View File
@@ -161,7 +161,4 @@ your recovery options are as follows:
- **Option 3: restoring a filesystem-level snapshot.** If you are using ZFS or
BTRFS to snapshot your metadata partition, refer to their specific
documentation on rolling back or copying files from an old snapshot.
Note that, depending on the properties of the filesystem and of the DB engine,
if these snapshots were taken during a write operation to the database, they may
also be corrupted and thus unfit for recovery.
documentation on rolling back or copying files from an old snapshot.
+8 -9
View File
@@ -129,10 +129,10 @@ docker run \
-d \
--name garaged \
-p 3900:3900 -p 3901:3901 -p 3902:3902 -p 3903:3903 \
-v /path/to/garage.toml:/etc/garage.toml \
-v /path/to/garage/meta:/var/lib/garage/meta \
-v /path/to/garage/data:/var/lib/garage/data \
dxflrs/garage:v2.2.0
-v /etc/garage.toml:/path/to/garage.toml \
-v /var/lib/garage/meta:/path/to/garage/meta \
-v /var/lib/garage/data:/path/to/garage/data \
dxflrs/garage:v2.0.0
```
Under Linux, you can substitute `--network host` for `-p 3900:3900 -p 3901:3901 -p 3902:3902 -p 3903:3903`
@@ -182,12 +182,11 @@ ID Hostname Address Tag Zone Capacit
## Creating a cluster layout
Creating a cluster layout for a Garage deployment means informing Garage
of the disk space available on each node of the cluster, `-c`,
as well as the name of the zone (e.g. datacenter), `-z`, each machine is located in.
of the disk space available on each node of the cluster
as well as the zone (e.g. datacenter) each machine is located in.
For our test deployment, we are have only one node with zone named `dc1` and a
capacity of `1G`, though the capacity is ignored for a single node deployment
and can be changed later when adding new nodes.
For our test deployment, we are using only one node. The way in which we configure
it does not matter, you can simply write:
```bash
garage layout assign -z dc1 -c 1G <node_id>
+40 -150
View File
@@ -6,167 +6,41 @@ weight = 40
The Garage administration API is accessible through a dedicated server whose
listen address is specified in the `[admin]` section of the configuration
file (see [configuration file
reference](@/documentation/reference-manual/configuration.md)).
reference](@/documentation/reference-manual/configuration.md))
The current version of the admin API is v2. No breaking changes to the Garage
administration API will be published outside of a major release.
**WARNING.** At this point, there is no commitment to the stability of the APIs described in this document.
We will bump the version numbers prefixed to each API endpoint each time the syntax
or semantics change, meaning that code that relies on these endpoint will break
when changes are introduced.
Versions:
- Before Garage 0.7.2 - no admin API
- Garage 0.7.2 - admin APIv0
- Garage 0.9.0 - admin APIv1, deprecate admin APIv0
History of previous versions:
- Before Garage v0.7.2 - no admin API
- Garage v0.7.2 - admin API v0
- Garage v0.9.0 - admin API v1, deprecate admin API v0
- Garage v2.0.0 - admin API v2, deprecate admin API v1
## Access control
### Using an API token
The admin API uses two different tokens for access control, that are specified in the config file's `[admin]` section:
Administration API tokens tokens are used as simple HTTP bearer tokens. In
other words, to authenticate access to an admin API endpoint, add the following
HTTP header to your request:
- `metrics_token`: the token for accessing the Metrics endpoint (if this token
is not set in the config file, the Metrics endpoint can be accessed without
access control);
- `admin_token`: the token for accessing all of the other administration
endpoints (if this token is not set in the config file, access to these
endpoints is disabled entirely).
These tokens are used as simple HTTP bearer tokens. In other words, to
authenticate access to an admin API endpoint, add the following HTTP header
to your request:
```
Authorization: Bearer <token>
```
### User-defined API tokens
Cluster administrators may dynamically define administration tokens using the CLI commands under `garage admin-token`.
Such tokens may be limited in scope, meaning that they may enable access to only a subset of API calls.
They may also have an expiration date to limit their use in time.
Here is an example to create an administration token that is valid for 30 days
and gives access to only a subset of API calls, allowing it to create buckets
and access keys and give keys permissions on buckets:
```bash
$ garage admin-token create --expires-in 30d \
--scope ListBuckets,GetBucketInfo,ListKeys,GetKeyInfo,CreateBucket,CreateKey,AllowBucketKey,DenyBucketKey \
my-token
This is your secret bearer token, it will not be shown again by Garage:
8ed1830b10a276ff57061950.kOSIpxWK9zSGbTO9Xadpv3YndSFWma0_snXcYHaORXk
==== ADMINISTRATION TOKEN INFORMATION ====
Token ID: 8ed1830b10a276ff57061950
Token name: my-token
Created: 2025-06-15 15:12:44.160 +02:00
Validity: valid
Expiration: 2025-07-15 15:12:44.117 +02:00
Scope: ListBuckets
GetBucketInfo
ListKeys
GetKeyInfo
CreateBucket
CreateKey
AllowBucketKey
DenyBucketKey
```
When running this command, your token will be shown only once and **will never
be shown again by Garage**, so make sure to save it directly. The token is
hashed internally, and is identified by its prefix (32 hex digits followed by a
dot) which is saved in clear.
When running `garage admin-token list`, you might see something like this:
```
ID Created Name Expiration Scope
- - metrics_token (from daemon configuration) never Metrics
8ed1830b10a276ff57061950 2025-06-15 my-token 2025-07-15 15:12:44.117 +02:00 ListBuckets, ... (8)
```
### Master API tokens
The admin API can also use two different master tokens for access control,
specified in the config file's `[admin]` section:
- `metrics_token`: the token for accessing the Metrics endpoint. If this token
is not set in the config file, the Metrics endpoint can be accessed without
access control.
- `admin_token`: the token for accessing all of the other administration
endpoints. If this token is not set in the config file, access to these
endpoints is only possible with a user-defined admin token.
With the introduction of multiple user-defined admin tokens, the use of master
API tokens is now discouraged.
## Using the admin API
All of the admin API endpoints are described in the OpenAPI specification:
- APIv2 - [HTML spec](https://garagehq.deuxfleurs.fr/api/garage-admin-v2.html) - [OpenAPI JSON](https://garagehq.deuxfleurs.fr/api/garage-admin-v2.json)
- APIv1 (deprecated) - [HTML spec](https://garagehq.deuxfleurs.fr/api/garage-admin-v1.html) - [OpenAPI YAML](https://garagehq.deuxfleurs.fr/api/garage-admin-v1.yml)
- APIv0 (deprecated) - [HTML spec](https://garagehq.deuxfleurs.fr/api/garage-admin-v0.html) - [OpenAPI YAML](https://garagehq.deuxfleurs.fr/api/garage-admin-v0.yml)
Making a request to the API from the command line can be as simple as running:
```bash
curl -H 'Authorization: Bearer s3cr3t' http://localhost:3903/v2/GetClusterStatus | jq
```
For more advanced use cases, we recommend using an SDK.
[Go to the "Build your own app" section to know how to use our SDKs](@/documentation/build/_index.md)
### Making API calls from the `garage` CLI
Since v2.0.0, the `garage` binary provides a subcommand `garage json-api` that
allows you to invoke the API without making an HTTP request. This can be
useful for scripting Garage deployments.
`garage json-api` proxies API calls through Garage's internal RPC protocol,
therefore it does not require any form of authentication: RPC connection
parameters are discovered automatically to contact the locally-running Garage
instance (as when running any other `garage` CLI command).
For simple calls that take no parameters, usage is as follows:
```
$ garage json-api GetClusterHealth
{
"connectedNodes": 3,
"knownNodes": 3,
"partitions": 256,
"partitionsAllOk": 256,
"partitionsQuorum": 256,
"status": "healthy",
"storageNodes": 3,
"storageNodesOk": 3
}
```
If you need to specify a JSON body for your call, you can add it directly after
the name of the function you are calling:
```
$ garage json-api CreateAdminToken '{"name": "test"}'
```
Or you can feed it through stdin by adding a `-` as the last command parameter:
```
$ garage json-api CreateAdminToken -
{"name": "test"}
<EOF>
```
For admin API calls that would have taken query parameters in their HTTP version, these parameters can be passed in the JSON body object:
```
$ garage json-api GetAdminTokenInfo '{"id":"b0e6e0ace2c0b2aca4cdb2de"}'
```
For admin API calls that take both query parameters and a JSON body, combine them in the following fashion:
```
$ garage json-api UpdateAdminToken '{"id":"b0e6e0ace2c0b2aca4cdb2de", "body":{"name":"not a test"}}'
```
## Special administration API endpoints
## Administration API endpoints
### Metrics `GET /metrics`
@@ -209,7 +83,7 @@ content-length: 102
date: Tue, 08 Aug 2023 07:22:38 GMT
Garage is fully operational
Consult the full health check API endpoint at /v2/GetClusterHealth for more details
Consult the full health check API endpoint at /v0/health for more details
```
### On-demand TLS `GET /check`
@@ -252,7 +126,23 @@ $ curl -so /dev/null -w "%{http_code}" http://localhost:3903/check?domain=exampl
200
```
**References:**
- [Using On-Demand TLS](https://caddyserver.com/docs/automatic-https#using-on-demand-tls)
- [Add option for a backend check to approve use of on-demand TLS](https://github.com/caddyserver/caddy/pull/1939)
- [Serving tens of thousands of domains over HTTPS with Caddy](https://caddy.community/t/serving-tens-of-thousands-of-domains-over-https-with-caddy/11179)
### Cluster operations
These endpoints have a dedicated OpenAPI spec.
- APIv1 - [HTML spec](https://garagehq.deuxfleurs.fr/api/garage-admin-v1.html) - [OpenAPI YAML](https://garagehq.deuxfleurs.fr/api/garage-admin-v1.yml)
- APIv0 (deprecated) - [HTML spec](https://garagehq.deuxfleurs.fr/api/garage-admin-v0.html) - [OpenAPI YAML](https://garagehq.deuxfleurs.fr/api/garage-admin-v0.yml)
Requesting the API from the command line can be as simple as running:
```bash
curl -H 'Authorization: Bearer s3cr3t' http://localhost:3903/v0/status | jq
```
For more advanced use cases, we recommend using a SDK.
[Go to the "Build your own app" section to know how to use our SDKs](@/documentation/build/_index.md)
+31 -110
View File
@@ -24,8 +24,7 @@ db_engine = "lmdb"
block_size = "1M"
block_ram_buffer_max = "256MiB"
block_max_concurrent_reads = 16
block_max_concurrent_writes_per_request =10
lmdb_map_size = "1T"
compression_level = 1
@@ -47,24 +46,20 @@ bootstrap_peers = [
"212fd62eeaca72c122b45a7f4fa0f55e012aa5e24ac384a72a3016413fa724ff@[fc00:F::1]:3901",
]
allow_punycode = false
[consul_discovery]
api = "catalog"
consul_http_addr = "https://127.0.0.1:8500"
tls_skip_verify = false
consul_http_addr = "http://127.0.0.1:8500"
service_name = "garage-daemon"
ca_cert = "/etc/consul/consul-ca.crt"
client_cert = "/etc/consul/consul-client.crt"
client_key = "/etc/consul/consul-key.crt"
# for `agent` API mode, unset client_cert and client_key, and optionally enable `token`
# token = "abcdef-01234-56789"
tls_skip_verify = false
tags = [ "dns-enabled" ]
meta = { dns-acl = "allow trusted" }
datacenters = ["dc1", "dc2", "dc3"]
[kubernetes_discovery]
namespace = "garage"
@@ -98,32 +93,29 @@ The following gives details about each available configuration option.
[Environment variables](#env_variables).
Top-level configuration options, in alphabetical order:
[`allow_punycode`](#allow_punycode),
Top-level configuration options:
[`allow_world_readable_secrets`](#allow_world_readable_secrets),
[`block_max_concurrent_reads`](#block_max_concurrent_reads),
[`block_max_concurrent_writes_per_request`](#block_max_concurrent_writes_per_request),
[`block_ram_buffer_max`](#block_ram_buffer_max),
[`block_size`](#block_size),
[`bootstrap_peers`](#bootstrap_peers),
[`compression_level`](#compression_level),
[`consistency_mode`](#consistency_mode),
[`data_dir`](#data_dir),
[`data_fsync`](#data_fsync),
[`db_engine`](#db_engine),
[`disable_scrub`](#disable_scrub),
[`use_local_tz`](#use_local_tz),
[`lmdb_map_size`](#lmdb_map_size),
[`metadata_auto_snapshot_interval`](#metadata_auto_snapshot_interval),
[`metadata_dir`](#metadata_dir),
[`metadata_fsync`](#metadata_fsync),
[`metadata_snapshots_dir`](#metadata_snapshots_dir),
[`replication_factor`](#replication_factor),
[`consistency_mode`](#consistency_mode),
[`rpc_bind_addr`](#rpc_bind_addr),
[`rpc_bind_outgoing`](#rpc_bind_outgoing),
[`rpc_public_addr`](#rpc_public_addr),
[`rpc_public_addr_subnet`](#rpc_public_addr_subnet)
[`rpc_secret`/`rpc_secret_file`](#rpc_secret),
[`use_local_tz`](#use_local_tz).
[`rpc_secret`/`rpc_secret_file`](#rpc_secret).
The `[consul_discovery]` section:
[`api`](#consul_api),
@@ -131,14 +123,12 @@ The `[consul_discovery]` section:
[`client_cert`](#consul_client_cert_and_key),
[`client_key`](#consul_client_cert_and_key),
[`consul_http_addr`](#consul_http_addr),
[`datacenters`](#consul_datacenters)
[`meta`](#consul_tags_and_meta),
[`service_name`](#consul_service_name),
[`tags`](#consul_tags_and_meta),
[`tls_skip_verify`](#consul_tls_skip_verify),
[`token`](#consul_token).
The `[kubernetes_discovery]` section:
[`namespace`](#kube_namespace),
[`service_name`](#kube_service_name),
@@ -163,17 +153,13 @@ The `[admin]` section:
### Environment variables {#env_variables}
The following configuration parameters must be specified as environment variables,
they do not exist in the configuration file:
The following configuration parameter must be specified as an environment
variable, it does not exist in the configuration file:
- `GARAGE_LOG_TO_SYSLOG` (since `v0.9.4`): set this to `1` or `true` to make the
Garage daemon send its logs to `syslog` (using the libc `syslog` function)
instead of printing to stderr.
- `GARAGE_LOG_TO_JOURNALD` (since `v1.2.0`): set this to `1` or `true` to make the
Garage daemon send its logs to `journald` (using the native protocol of `systemd-journald`)
instead of printing to stderr.
The following environment variables can be used to override the corresponding
values in the configuration file:
@@ -185,7 +171,7 @@ values in the configuration file:
### Top-level configuration options
#### `replication_factor` (since `v1.0.0`) {#replication_factor}
#### `replication_factor` {#replication_factor}
The replication factor can be any positive integer smaller or equal the node count in your cluster.
The chosen replication factor has a big impact on the cluster's failure tolerancy and performance characteristics.
@@ -233,7 +219,7 @@ is in progress. In theory, no data should be lost as rebalancing is a
routine operation for Garage, although we cannot guarantee you that everything
will go right in such an extreme scenario.
#### `consistency_mode` (since `v1.0.0`) {#consistency_mode}
#### `consistency_mode` {#consistency_mode}
The consistency mode setting determines the read and write behaviour of your cluster.
@@ -343,7 +329,6 @@ Since `v0.8.0`, Garage can use alternative storage backends as follows:
| --------- | ----------------- | ------------- |
| [LMDB](https://www.symas.com/lmdb) (since `v0.8.0`, default since `v0.9.0`) | `"lmdb"` | `<metadata_dir>/db.lmdb/` |
| [Sqlite](https://sqlite.org) (since `v0.8.0`) | `"sqlite"` | `<metadata_dir>/db.sqlite` |
| [Fjall](https://github.com/fjall-rs/fjall) (**experimental support** since `v1.3.0`/`v2.1.0`) | `"fjall"` | `<metadata_dir>/db.fjall/` |
| [Sled](https://sled.rs) (old default, removed since `v1.0`) | `"sled"` | `<metadata_dir>/db/` |
Sled was supported until Garage v0.9.x, and was removed in Garage v1.0.
@@ -352,16 +337,8 @@ old Sled metadata databases to another engine.
Performance characteristics of the different DB engines are as follows:
- **LMDB:** the recommended database engine for high-performance distributed clusters
with `replication_factor` ≥ 2.
LMDB works well, but is known to have the following limitations:
- LMDB is prone to database corruption after an unclean shutdown (e.g. a process kill
or a power outage). It is recommended to configure
[`metadata_auto_snapshot_interval`](#metadata_auto_snapshot_interval) to be
able to easily recover from this situation. With `replication_factor` ≥ 2,
metadata can also be reconstructed from remote nodes upon corruption
(see [Recovering from failures](@/documentation/operations/recovering.md#corrupted_meta)).
- LMDB: the recommended database engine for high-performance distributed clusters.
LMDB works very well, but is known to have the following limitations:
- The data format of LMDB is not portable between architectures, so for
instance the Garage database of an x86-64 node cannot be moved to an ARM64
@@ -371,21 +348,22 @@ Performance characteristics of the different DB engines are as follows:
node to very small database sizes due to how LMDB works; it is therefore
not recommended.
- Several users have reported corrupted LMDB database files after an unclean
shutdown (e.g. a power outage). This situation can generally be recovered
from if your cluster is geo-replicated (by rebuilding your metadata db from
other nodes), or if you have saved regular snapshots at the filesystem
level.
- Keys in LMDB are limited to 511 bytes. This limit translates to limits on
object keys in S3 and sort keys in K2V that are limted to 479 bytes.
- **Sqlite:** Garage supports Sqlite as an alternative storage backend for
metadata, which does not have the issues listed above for LMDB. Sqlite is
slower than LMDB, so it is not the best choice for high-performance storage
clusters.
- **Fjall:** a storage engine based on LSM trees, which theoretically allow for
higher write throughput than other storage engines that are based on B-trees.
Using Fjall could potentially improve Garage's performance significantly in
write-heavy workloads. **Support for Fjall is experimental at this point**,
we have added it to Garage for evaluation purposes only. **Use it only with
test data, and report any issues to our bug tracker. Do not use it for
production workloads.**
- Sqlite: Garage supports Sqlite as an alternative storage backend for
metadata, which does not have the issues listed above for LMDB.
On versions 0.8.x and earlier, Sqlite should be avoided due to abysmal
performance, which was fixed with the addition of `metadata_fsync`.
Sqlite is still probably slower than LMDB due to the way we use it,
so it is not the best choice for high-performance storage clusters,
but it should work fine in many cases.
It is possible to convert Garage's metadata directory from one format to another
using the `garage convert-db` command, which should be used as follows:
@@ -424,7 +402,6 @@ Here is how this option impacts the different database engines:
|----------|------------------------------------|-------------------------------|
| Sqlite | `PRAGMA synchronous = OFF` | `PRAGMA synchronous = NORMAL` |
| LMDB | `MDB_NOMETASYNC` + `MDB_NOSYNC` | `MDB_NOMETASYNC` |
| Fjall | default options | not supported |
Note that the Sqlite database is always ran in `WAL` mode (`PRAGMA journal_mode = WAL`).
@@ -444,8 +421,7 @@ if geographical replication is used.
#### `metadata_auto_snapshot_interval` (since `v0.9.4`) {#metadata_auto_snapshot_interval}
If this value is set, Garage will automatically take a snapshot of the metadata
DB file at a regular interval and save it in the metadata directory,
or in [`metadata_snapshots_dir`](#metadata_snapshots_dir) if it is set.
DB file at a regular interval and save it in the metadata directory.
This parameter can take any duration string that can be parsed by
the [`parse_duration`](https://docs.rs/parse_duration/latest/parse_duration/#syntax) crate.
@@ -454,19 +430,14 @@ corrupted, for instance after an unclean shutdown. See [this
page](@/documentation/operations/recovering.md#corrupted_meta) for details.
Garage keeps only the two most recent snapshots of the metadata DB and deletes
older ones automatically.
You can also create metadata snapshots manually at any point using the
`garage meta snapshot` command.
Using snapshots created by Garage is the best option to make snapshots of your
node's metadata for potential recovery, as they are guaranteed to be clean and
consistent, contrarily to filesystem-level snapshots that may be taken while
some writes are in-flight and thus might be corrupted.
Note that taking a metadata snapshot is a relatively intensive operation as the
entire data file is copied. A snapshot being taken might have performance
impacts on the Garage node while it is running. If the cluster is under heavy
write load when a snapshot operation is running, this might also cause the
database file to grow in size significantly as pages cannot be recycled easily.
For this reason, it might be better to use filesystem-level snapshots instead
if possible.
#### `disable_scrub` {#disable_scrub}
@@ -537,37 +508,6 @@ node.
The default value is 256MiB.
#### `block_max_concurrent_reads` (since `v1.3.0` / `v2.1.0`) {#block_max_concurrent_reads}
The maximum number of blocks (individual files in the data directory) open
simultaneously for reading.
Reducing this number does not limit the number of data blocks that can be
transferred through the network simultaneously. This mechanism was just added
as a backpressure mechanism for HDD read speed: it helps avoid a situation
where too many requests are coming in and Garage is reading too many block
files simultaneously, thus not making timely progress on any of the reads.
When a request to read a data block comes in through the network, the requests
awaits for one of the `block_max_concurrent_reads` slots to be available
(internally implemented using a Semaphore object). Once it acquired a read
slot, it reads the entire block file to RAM and frees the slot as soon as the
block file is finished reading. Only after the slot is released will the
block's data start being transferred over the network. If the request fails to
acquire a reading slot wihtin 15 seconds, it fails with a timeout error.
Timeout events can be monitored through the `block_read_semaphore_timeouts`
metric in Prometheus: a non-zero number of such events indicates an I/O
bottleneck on HDD read speed.
#### `block_max_concurrent_writes_per_request` (since `v1.3.1` / `v2.2.0`) {#block_max_concurrent_writes_per_request}
This parameter is designed to adapt to the concurrent write performance of
different storage media. Maximum number of parallel block writes per put request.
Higher values may improve throughput but increase memory usage.
Default value: 3. Recommended values: 10-30 for NVMe, 3-10 for spinning HDD.
#### `lmdb_map_size` {#lmdb_map_size}
This parameters can be used to set the map size used by LMDB,
@@ -666,7 +606,7 @@ be obtained by running `garage node id` and then included directly in the
key will be returned by `garage node id` and you will have to add the IP
yourself.
#### `allow_world_readable_secrets` or `GARAGE_ALLOW_WORLD_READABLE_SECRETS` (env) {#allow_world_readable_secrets}
### `allow_world_readable_secrets` or `GARAGE_ALLOW_WORLD_READABLE_SECRETS` (env) {#allow_world_readable_secrets}
Garage checks the permissions of your secret files to make sure they're not
world-readable. In some cases, the check might fail and consider your files as
@@ -678,13 +618,6 @@ permission verification.
Alternatively, you can set the `GARAGE_ALLOW_WORLD_READABLE_SECRETS`
environment variable to `true` to bypass the permissions check.
#### `allow_punycode` {#allow_punycode}
Allow creating buckets with names containing punycode. When used for buckets served
as websites, this allows using almost any unicode character in the domain name.
Default to `false`.
### The `[consul_discovery]` section
Garage supports discovering other nodes of the cluster using Consul. For this
@@ -740,18 +673,6 @@ node_prefix "" {
}
```
#### `datacenters` {#consul_datacenters}
Optional list of datacenters that allow garage to do service discovery when Consul is configured in WAN federation.
Example: `datacenters = ["dc1", "dc2", "dc3"]`
In a WAN configuration, by default the Consul services API only responds with
local LAN services. When a list of datacenters is specified using this option,
Garage will query the consul server API by datacenter directly, allowing for
Garage to discover nodes across the Consul WAN.
#### `tags` and `meta` {#consul_tags_and_meta}
Additional list of tags and map of service meta to add during service registration.
+1 -1
View File
@@ -129,5 +129,5 @@ related to objects stored in an S3 bucket.
In the context of our research project, [Aérogramme](https://aerogramme.deuxfleurs.fr),
K2V is used to provide metadata and log storage for operations on encrypted e-mail storage.
Learn more on the specification of K2V [here](https://git.deuxfleurs.fr/Deuxfleurs/garage/src/commit/f8be15c37db857e177d543de7be863692628d567/doc/drafts/k2v-spec.md)
Learn more on the specification of K2V [here](https://git.deuxfleurs.fr/Deuxfleurs/garage/src/branch/k2v/doc/drafts/k2v-spec.md)
and on how to enable it in Garage [here](@/documentation/reference-manual/k2v.md).
+1 -1
View File
@@ -16,7 +16,7 @@ the `k2v` feature flag enabled can be obtained from our download page under
with `-k2v` (example: `v0.7.2-k2v`).
The specification of the K2V API can be found
[here](https://git.deuxfleurs.fr/Deuxfleurs/garage/src/commit/f8be15c37db857e177d543de7be863692628d567/doc/drafts/k2v-spec.md).
[here](https://git.deuxfleurs.fr/Deuxfleurs/garage/src/branch/main/doc/drafts/k2v-spec.md).
This document also includes a high-level overview of K2V's design.
The K2V API uses AWSv4 signatures for authentification, same as the S3 API.
@@ -23,17 +23,17 @@ Feel free to open a PR to suggest fixes this table. Minio is missing because the
- 2022-05-25 - Many Ceph S3 endpoints are not documented but implemented. Following a notification from the Ceph community, we added them.
## High-level features
| Feature | Garage | [Openstack Swift](https://docs.openstack.org/swift/latest/s3_compat.html) | [Ceph Object Gateway](https://docs.ceph.com/en/latest/radosgw/s3/) | [Riak CS](https://docs.riak.com/riak/cs/2.1.1/references/apis/storage/s3/index.html) | [OpenIO](https://docs.openio.io/latest/source/arch-design/s3_compliancy.html) |
|------------------------------|----------------------------------|-----------------|---------------|---------|-----|
| [signature v2](https://docs.aws.amazon.com/AmazonS3/latest/API/Appendix-Sigv2.html) (deprecated) | ❌ Missing | ✅ | ✅ | ✅ | ✅ |
| [signature v2](https://docs.aws.amazon.com/general/latest/gr/signature-version-2.html) (deprecated) | ❌ Missing | ✅ | ✅ | ✅ | ✅ |
| [signature v4](https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-authenticating-requests.html) | ✅ Implemented | ✅ | ✅ | ❌ | ✅ |
| [URL path-style](https://docs.aws.amazon.com/AmazonS3/latest/userguide/VirtualHosting.html#path-style-access) (eg. `host.tld/bucket/key`) | ✅ Implemented | ✅ | ✅ | ❓| ✅ |
| [URL vhost-style](https://docs.aws.amazon.com/AmazonS3/latest/userguide/VirtualHosting.html#virtual-hosted-style-access) URL (eg. `bucket.host.tld/key`) | ✅ Implemented | ❌| ✅| ✅ | ✅ |
| [Presigned URLs](https://docs.aws.amazon.com/AmazonS3/latest/userguide/ShareObjectPreSignedURL.html) | ✅ Implemented | ❌| ✅ | ✅ | ✅(❓) |
| [SSE-C encryption](https://docs.aws.amazon.com/AmazonS3/latest/userguide/ServerSideEncryptionCustomerKeys.html) | ✅ Implemented | ❓ | ✅ | ❌ | ✅ |
| [Bucket versioning](https://docs.aws.amazon.com/AmazonS3/latest/userguide/Versioning.html) | ❌ Missing | ✅ | ✅ | ❌ | ✅ |
*Note:* OpenIO does not says if it supports presigned URLs. Because it is part
of signature v4 and they claim they support it without additional precisions,
+1 -1
View File
@@ -1,6 +1,6 @@
+++
title = "Migrating from 0.3 to 0.4"
weight = 80
weight = 20
+++
**Migrating from 0.3 to 0.4 is unsupported. This document is only intended to
+1 -1
View File
@@ -1,6 +1,6 @@
+++
title = "Migrating from 0.5 to 0.6"
weight = 75
weight = 15
+++
**This guide explains how to migrate to 0.6 if you have an existing 0.5 cluster.
+1 -1
View File
@@ -1,6 +1,6 @@
+++
title = "Migrating from 0.6 to 0.7"
weight = 74
weight = 14
+++
**This guide explains how to migrate to 0.7 if you have an existing 0.6 cluster.
We don't recommend trying to migrate to 0.7 directly from 0.5 or older.**
+1 -1
View File
@@ -1,6 +1,6 @@
+++
title = "Migrating from 0.7 to 0.8"
weight = 73
weight = 13
+++
**This guide explains how to migrate to 0.8 if you have an existing 0.7 cluster.
+1 -1
View File
@@ -1,6 +1,6 @@
+++
title = "Migrating from 0.8 to 0.9"
weight = 72
weight = 12
+++
**This guide explains how to migrate to 0.9 if you have an existing 0.8 cluster.
+1 -1
View File
@@ -1,6 +1,6 @@
+++
title = "Migrating from 0.9 to 1.0"
weight = 71
weight = 11
+++
**This guide explains how to migrate to 1.0 if you have an existing 0.9 cluster.
-70
View File
@@ -1,70 +0,0 @@
+++
title = "Migrating from 1.0 to 2.0"
weight = 70
+++
**This guide explains how to migrate to v2.x if you have an existing v1.x.x cluster.
We don't recommend trying to migrate to v2.x directly from v0.9.x or older.**
This migration procedure has been tested on several clusters without issues.
However, it is still a *critical procedure* that might cause issues.
**Make sure to back up all your data before attempting it!**
You might also want to read our [general documentation on upgrading Garage](@/documentation/operations/upgrading.md).
## Changes introduced in v2.0
The following are **breaking changes** in Garage v2.0 that require your attention when migrating:
- The administration API has been completely reworked.
Some calls to the `/v1/` endpoints will still work but most will not.
New endpoints are prefixed by `/v2/`. **You will need to update all your code that makes use of the admin API.**
- `replication_mode` is no longer a supported configuration parameter,
please use `replication_factor` and `consistency_mode` instead.
## Migration procedure
The migration to Garage v2.0 can be done with almost no downtime,
by restarting all nodes at once in the new version.
The migration steps are as follows:
1. Do a `garage repair --all-nodes --yes tables`, check the logs and check that
all data seems to be synced correctly between nodes. If you have time, do
additional `garage repair` procedures (`blocks`, `versions`, `block_refs`,
etc.)
2. Ensure you have a snapshot of your Garage installation that you can restore
to in case the upgrade goes wrong, with one of the following options:
- You may use the `garage meta snapshot --all` command
to make a backup snapshot of the metadata directories of your nodes
for backup purposes. Once this command has completed, copy the following
files and directories from the `metadata_dir` of all your nodes
to somewhere safe: `snapshots`, `cluster_layout`, `data_layout`,
`node_key`, `node_key.pub`. (If you have set the `metadata_snapshots_dir`
to a different value in your config file, back up that directory instead.)
- If you are running a filesystem such as ZFS or BTRFS that support
snapshotting, you can create a filesystem-level snapshot of the `metadata_dir`
of all your nodes to be used as a restoration point if needed.
- You may also make a back-up manually: turn off each node
individually; back up its metadata folder (for instance, use the following
command if your metadata directory is `/var/lib/garage/meta`: `cd
/var/lib/garage ; tar -acf meta-v1.0.tar.zst meta/`); turn it back on
again. This will allow you to take a backup of all nodes without
impacting global cluster availability. You can do all nodes of a single
zone at once as this does not impact the availability of Garage.
3. Prepare your updated binaries and configuration files for Garage v2.0.
**Remember to update your configuration file to remove `replication_mode` and replace it by `replication_factor`.**
4. Shut down all v1.0 nodes simultaneously, and restart them all simultaneously
in v2.0. Use your favorite deployment tool (Ansible, Kubernetes, Nomad) to
achieve this as fast as possible. Garage v2.0 should be in a working state
as soon as enough nodes have started.
5. Monitor your cluster in the following hours to see if it works well under
your production load.
@@ -1,6 +1,6 @@
+++
title = "Testing strategy"
weight = 100
weight = 30
+++
+1 -4
View File
@@ -15,10 +15,7 @@ when changes are introduced.
The Garage administration API was introduced in version 0.7.2, and was
changed several times.
**THIS DOCUMENT IS DEPRECATED.** We now have an OpenAPI spec which is automatically generated
from Garage's source code and is always up-to-date. See `doc/api/garage-admin-v2.html`.
Text in this document is no longer kept in sync with the admin API's actual behavior.
This document applies only to the Garage v2 API (starting with Garage v2.0.0).
## Access control
-17
View File
@@ -1,17 +0,0 @@
*
!*.txt
!*.md
!assets
!.gitignore
!*.svg
!*.png
!*.jpg
!*.tex
!Makefile
!.gitignore
!assets/*.drawio.pdf
!talk.pdf
-19
View File
@@ -1,19 +0,0 @@
ASSETS=../assets/lattice/lattice1.pdf_tex \
../assets/lattice/lattice2.pdf_tex \
../assets/lattice/lattice3.pdf_tex \
../assets/lattice/lattice4.pdf_tex \
../assets/lattice/lattice5.pdf_tex \
../assets/lattice/lattice6.pdf_tex \
../assets/lattice/lattice7.pdf_tex \
../assets/lattice/lattice8.pdf_tex \
../assets/logos/deuxfleurs.pdf \
../assets/timeline-22-24.pdf
talk.pdf: talk.tex $(ASSETS)
pdflatex talk.tex
%.pdf: %.svg
inkscape -D -z --file=$^ --export-pdf=$@
%.pdf_tex: %.svg
inkscape -D -z --file=$^ --export-pdf=$@ --export-latex
Binary file not shown.
-702
View File
@@ -1,702 +0,0 @@
\nonstopmode
\documentclass[aspectratio=169,xcolor={svgnames}]{beamer}
\usepackage[utf8]{inputenc}
% \usepackage[frenchb]{babel}
\usepackage{amsmath}
\usepackage{mathtools}
\usepackage{breqn}
\usepackage{multirow}
\usetheme{boxes}
\usepackage{graphicx}
\usepackage{import}
\usepackage{adjustbox}
\usepackage[absolute,overlay]{textpos}
%\useoutertheme[footline=authortitle,subsection=false]{miniframes}
%\useoutertheme[footline=authorinstitute,subsection=false]{miniframes}
\useoutertheme{infolines}
\setbeamertemplate{headline}{}
\beamertemplatenavigationsymbolsempty
\definecolor{TitleOrange}{RGB}{255,137,0}
\setbeamercolor{title}{fg=TitleOrange}
\setbeamercolor{frametitle}{fg=TitleOrange}
\definecolor{ListOrange}{RGB}{255,145,5}
\setbeamertemplate{itemize item}{\color{ListOrange}$\blacktriangleright$}
\definecolor{verygrey}{RGB}{70,70,70}
\setbeamercolor{normal text}{fg=verygrey}
\usepackage{tabu}
\usepackage{multicol}
\usepackage{vwcol}
\usepackage{stmaryrd}
\usepackage{graphicx}
\usepackage[normalem]{ulem}
\AtBeginSection[]{
\begin{frame}
\vfill
\centering
\begin{beamercolorbox}[sep=8pt,center,shadow=true,rounded=true]{title}
\usebeamerfont{title}\insertsectionhead\par%
\end{beamercolorbox}
\vfill
\end{frame}
}
\title{Garage, an S3 backend as reliable as possible}
\author{Garage Authors}
\date{JoSy S3, 2025-10-08}
\begin{document}
\begin{frame}
\centering
\includegraphics[width=.3\linewidth]{../../sticker/Garage.png}
\vspace{1em}
{\large\bf Garage, an S3 backend as reliable as possible}
\vspace{1em}
\url{https://garagehq.deuxfleurs.fr/}\\
\url{mailto:garagehq@deuxfleurs.fr}\\
\texttt{\#garage:deuxfleurs.fr} on Matrix
\end{frame}
\section{Meet Garage}
\begin{frame}
\frametitle{A non-profit initiative}
\begin{columns}[t]
\begin{column}{.2\textwidth}
\centering
\adjincludegraphics[width=.5\linewidth, valign=t]{../assets/logos/deuxfleurs.pdf}
\end{column}
\begin{column}{.8\textwidth}
\textbf{Part of a degrowth initiative}\\
Garage has been created at Deuxfleurs where we experiment running Internet services without datacenter on commodity and refurbished hardware.
\end{column}
\end{columns}
\vspace{2em}
\begin{columns}[t]
\begin{column}{.2\textwidth}
\centering
\adjincludegraphics[width=.5\linewidth, valign=t]{../assets/community.png}
\end{column}
\begin{column}{.8\textwidth}
\textbf{Developed by a community}\\
{\small Some recent contributors: Arthur C, Charles H, dongdigua, Etienne L, Jonah A, Julien K, Lapineige, MagicRR, Milas B, Niklas M, RockWolf, Schwitzd, trinity-1686a, Xavier S, babykart, Baptiste J, eddster2309, James O'C, Joker9944, Maximilien R, Renjaya RZ, Yureka...}
\end{column}
\end{columns}
\vspace{2em}
\begin{columns}[t]
\begin{column}{.2\textwidth}
\centering
\adjincludegraphics[width=.5\linewidth, valign=t]{../assets/logos/AGPLv3_Logo.png}
\end{column}
\begin{column}{.8\textwidth}
\textbf{Owned by nobody, open-core is impossible, zero VC money}\\
AGPL + no Contributor License Agreement = Garage ownership spreads among hundredth of contributors.
\end{column}
\end{columns}
\end{frame}
\begin{frame}
\frametitle{Getting support for Garage}
\begin{columns}[t]
\begin{column}{.2\textwidth}
\centering
\adjincludegraphics[width=.4\linewidth, valign=t]{../assets/alex.jpg}
\end{column}
\begin{column}{.4\textwidth}
\textbf{Alex Auvolat}\\
PhD; co-founder of Deuxfleurs\\
Garage maintainer, Freelance
\end{column}
\begin{column}{.3\textwidth}
\centering
\adjincludegraphics[width=.4\linewidth, valign=t]{../assets/support.png}
\end{column}
\begin{column}{.1\textwidth}
~
\end{column}
\end{columns}
\vspace{2em}
\begin{columns}[t]
\begin{column}{.2\textwidth}
\centering
\adjincludegraphics[width=.4\linewidth, valign=t]{../assets/quentin.jpg}
\end{column}
\begin{column}{.4\textwidth}
\textbf{Quentin Dufour}\\
PhD; co-founder of Deuxfleurs\\
Garage contributor, Freelance
\end{column}
\begin{column}{.4\textwidth}
For support requests, write at: \\
\url{garagehq@deuxfleurs.fr}
\end{column}
\end{columns}
\vspace{2em}
\begin{columns}[t]
\begin{column}{.2\textwidth}
\centering
\adjincludegraphics[width=.4\linewidth, valign=t]{../assets/armael.jpg}
\end{column}
\begin{column}{.4\textwidth}
\textbf{Armaël Guéneau}\\
PhD; member of Deuxfleurs\\
Garage contributor, Freelance
\end{column}
\begin{column}{.4\textwidth}
Eligible: email support, architecture design, specific feature development, etc.
\end{column}
\end{columns}
\end{frame}
\begin{frame}
\frametitle{Our initial goal}
\centering
\Large
Being a self-sovereign community to be free of our degrowth choice
$\big\downarrow$
As web citizens, datacenters are big black boxes. \\
We want to leave them to autonoumously manage our servers.
$\big\downarrow$
We want reliable services without relying on dedicated hardware or places.
\end{frame}
\begin{frame}
\frametitle{Building a resilient system with cheap stuff}
\only<1,4-7>{
\begin{itemize}
\item \textcolor<5->{gray}{Commodity hardware (e.g. old desktop PCs)\\
\vspace{.5em}
\visible<4->{{\footnotesize (can die at any time)}}}
\vspace{1.5em}
\item<5-> \textcolor<7->{gray}{Regular Internet (e.g. FTTB, FTTH) and power grid connections\\
\vspace{.5em}
\visible<6->{{\footnotesize (can be unavailable randomly)}}}
\vspace{1.5em}
\item<7-> \textbf{Geographical redundancy} (multi-site replication)
\end{itemize}
}
\only<2>{
\begin{center}
\includegraphics[width=.8\linewidth]{../assets/neptune.jpg}
\end{center}
}
\only<3>{
\begin{center}
\includegraphics[width=.8\linewidth]{../assets/atuin.jpg}
\end{center}
}
\only<8>{
\begin{center}
\includegraphics[width=.8\linewidth]{../assets/inframap_jdll2023.pdf}
\end{center}
}
\end{frame}
\begin{frame}
\frametitle{Object storage: a crucial component}
\begin{center}
\includegraphics[height=6em]{../assets/logos/Amazon-S3.jpg}
\hspace{3em}
\visible<2->{\includegraphics[height=5em]{../assets/logos/minio.png}}
\hspace{3em}
\visible<3>{\includegraphics[height=6em]{../../logo/garage_hires_crop.png}}
\end{center}
\vspace{1em}
S3: a de-facto standard, many compatible applications
\vspace{1em}
\visible<2->{MinIO is self-hostable but not suited for geo-distributed deployments}
\vspace{1em}
\visible<3->{\textbf{Garage is a self-hosted drop-in replacement for the Amazon S3 object store}}
\end{frame}
\begin{frame}
\frametitle{CRDTs / weak consistency instead of consensus}
\underline{Internally, Garage uses only CRDTs} (conflict-free replicated data types)
\vspace{2em}
Why not Raft, Paxos, ...? Issues of consensus algorithms:
\vspace{1em}
\begin{itemize}
\item<2-> \textbf{Software complexity}
\vspace{1em}
\item<3-> \textbf{Performance issues:}
\vspace{.5em}
\begin{itemize}
\item<4-> The leader is a \textbf{bottleneck} for all requests\\
\vspace{.5em}
\item<5-> \textbf{Sensitive to higher latency} between nodes
\vspace{.5em}
\item<6-> \textbf{Takes time to reconverge} when disrupted (e.g. node going down)
\end{itemize}
\end{itemize}
\end{frame}
\begin{frame}
\frametitle{The data model of object storage}
Object storage is basically a \textbf{key-value store}:
\vspace{.5em}
{\scriptsize
\begin{center}
\begin{tabular}{|l|p{7cm}|}
\hline
\textbf{Key: file path + name} & \textbf{Value: file data + metadata} \\
\hline
\hline
\texttt{index.html} &
\texttt{Content-Type: text/html; charset=utf-8} \newline
\texttt{Content-Length: 24929} \newline
\texttt{<binary blob>} \\
\hline
\texttt{img/logo.svg} &
\texttt{Content-Type: text/svg+xml} \newline
\texttt{Content-Length: 13429} \newline
\texttt{<binary blob>} \\
\hline
\texttt{download/index.html} &
\texttt{Content-Type: text/html; charset=utf-8} \newline
\texttt{Content-Length: 26563} \newline
\texttt{<binary blob>} \\
\hline
\end{tabular}
\end{center}
}
\vspace{1em}
\begin{itemize}
\item<2> Maps well to CRDT data types
\end{itemize}
\end{frame}
\begin{frame}
\frametitle{Performance gains in practice}
\begin{center}
\includegraphics[width=.8\linewidth]{../assets/perf/endpoint_latency_0.7_0.8_minio.png}
\end{center}
\end{frame}
% ======================================== OPERATING
% ======================================== OPERATING
% ======================================== OPERATING
\section{Production clusters}
\begin{frame}
\frametitle{Deployment kinds}
\includegraphics[width=.9\linewidth]{../assets/cluster_kind.png}
\vspace{1em}
\end{frame}
\begin{frame}
\frametitle{How big they are?}
\includegraphics[width=.9\linewidth]{../assets/cluster_size.png}
\vspace{1em}
\textit{"Petabyte storage setup for a video site. Nginx as CDN in-front using garage-s3-website feature. Each storage node has ~64TB storage with raid10, no replication within garage. 25gbit nic. haproxy to loadbalance across 5 nodes. mostly reads with very few writes."}
\vspace{1em}
\textit{"We currently manage 7 Garage nodes, 28TB total storage, 6M blocks for 3M objects and 4TB of object data. We have been running Garage in production for 2.5 years."}
\end{frame}
\begin{frame}
\frametitle{Operating Garage}
\begin{center}
\only<1-2>{
\includegraphics[width=.9\linewidth]{../assets/screenshots/garage_status_0.10.png}
\\\vspace{1em}
\visible<2>{\includegraphics[width=.9\linewidth]{../assets/screenshots/garage_status_unhealthy_0.10.png}}
}
\end{center}
\end{frame}
\begin{frame}
\frametitle{Garage's architecture}
\begin{center}
\only<1>{\includegraphics[width=.45\linewidth]{../assets/garage.drawio.pdf}}%
\only<2>{\includegraphics[width=.6\linewidth]{../assets/garage_sync.drawio.pdf}}%
\end{center}
\end{frame}
\begin{frame}
\frametitle{Digging deeper}
\begin{center}
\only<1>{\includegraphics[width=.9\linewidth]{../assets/screenshots/garage_stats_0.10.png}}
\only<2>{\includegraphics[width=.5\linewidth]{../assets/screenshots/garage_worker_list_0.10.png}}
\only<3>{\includegraphics[width=.6\linewidth]{../assets/screenshots/garage_worker_param_0.10.png}}
\end{center}
\end{frame}
\begin{frame}
\frametitle{Potential limitations and bottlenecks}
\begin{itemize}
\item Global:
\begin{itemize}
\item Max. $\sim$100 nodes per cluster (excluding gateways)
\end{itemize}
\vspace{1em}
\item Metadata:
\begin{itemize}
\item One big bucket = bottleneck, object list on 3 nodes only
\end{itemize}
\vspace{1em}
\item Block manager:
\begin{itemize}
\item Lots of small files on disk
\item Processing the resync queue can be slow
\end{itemize}
\end{itemize}
\end{frame}
\begin{frame}
\frametitle{Deployment advice for very large clusters}
\begin{itemize}
\item Metadata storage:
\begin{itemize}
\item ZFS mirror (x2) on fast NVMe
\item Use LMDB storage engine
\end{itemize}
\vspace{.5em}
\item Data block storage:
\begin{itemize}
\item Use Garage's native multi-HDD support
\item XFS on individual drives
\item Increase block size (1MB $\to$ 10MB, requires more RAM and good networking)
\item Tune \texttt{resync-tranquility} and \texttt{resync-worker-count} dynamically
\end{itemize}
\vspace{.5em}
\item Other :
\begin{itemize}
\item Split data over several buckets
\item Use less than 100 storage nodes
\item Use gateway nodes
\end{itemize}
\vspace{.5em}
\end{itemize}
\end{frame}
\begin{frame}
\frametitle{Focus on Deuxfleurs}
Host institutional websites, partnership with a web agency.
Matrix media backend.
Plan to use it as an email backend for an internally developed email server.
\end{frame}
% ======================================== TIMELINE
% ======================================== TIMELINE
% ======================================== TIMELINE
\section{Recent developments}
% ====================== v0.7.0 ===============================
\begin{frame}
\begin{center}
\includegraphics[width=.8\linewidth]{../assets/tl.drawio.png}
\end{center}
\end{frame}
\begin{frame}
\frametitle{April 2022 - Garage v0.7.0}
Focus on \underline{observability and ecosystem integration}
\vspace{2em}
\begin{itemize}
\item \textbf{Monitoring:} metrics and traces, using OpenTelemetry
\vspace{1em}
\item Replication modes with 1 or 2 copies / weaker consistency
\vspace{1em}
\item Kubernetes integration for node discovery
\vspace{1em}
\item Admin API (v0.7.2)
\end{itemize}
\end{frame}
\begin{frame}
\frametitle{Metrics (Prometheus + Grafana)}
\begin{center}
\includegraphics[width=.9\linewidth]{../assets/screenshots/grafana_dashboard.png}
\end{center}
\end{frame}
\begin{frame}
\frametitle{Traces (Jaeger)}
\begin{center}
\includegraphics[width=.8\linewidth]{../assets/screenshots/jaeger_listobjects.png}
\end{center}
\end{frame}
% ====================== v0.8.0 ===============================
\begin{frame}
\begin{center}
\includegraphics[width=.8\linewidth]{../assets/tl.drawio.png}
\end{center}
\end{frame}
\begin{frame}
\frametitle{November 2022 - Garage v0.8.0}
Focus on \underline{performance}
\vspace{2em}
\begin{itemize}
\item \textbf{Alternative metadata DB engines} (LMDB, Sqlite)
\vspace{1em}
\item \textbf{Performance improvements:} block streaming, various optimizations...
\vspace{1em}
\item Bucket quotas (max size, max \#objects)
\vspace{1em}
\item Quality of life improvements, observability, etc.
\end{itemize}
\end{frame}
\begin{frame}
\frametitle{About metadata DB engines}
\textbf{Issues with Sled:}
\vspace{1em}
\begin{itemize}
\item Huge files on disk
\vspace{.5em}
\item Unpredictable performance, especially on HDD
\vspace{.5em}
\item API limitations
\vspace{.5em}
\item Not actively maintained
\end{itemize}
\vspace{2em}
\textbf{LMDB:} very stable, good performance, file size is reasonable\\
\textbf{Sqlite} also available as a second choice
\vspace{1em}
Sled will be removed in Garage v1.0
\end{frame}
\begin{frame}
\frametitle{DB engine performance comparison}
\begin{center}
\includegraphics[width=.6\linewidth]{../assets/perf/db_engine.png}
\end{center}
NB: Sqlite was slow due to synchronous mode, now configurable
\end{frame}
\begin{frame}
\frametitle{Block streaming}
\begin{center}
\only<1>{\includegraphics[width=.8\linewidth]{../assets/schema-streaming-1.png}}
\only<2>{\includegraphics[width=.8\linewidth]{../assets/schema-streaming-2.png}}
\end{center}
\end{frame}
\begin{frame}
\frametitle{TTFB benchmark}
\begin{center}
\includegraphics[width=.8\linewidth]{../assets/perf/ttfb.png}
\end{center}
\end{frame}
\begin{frame}
\frametitle{Throughput benchmark}
\begin{center}
\includegraphics[width=.7\linewidth]{../assets/perf/io-0.7-0.8-minio.png}
\end{center}
\end{frame}
% ====================== v0.9.0 ===============================
\begin{frame}
\begin{center}
\includegraphics[width=.8\linewidth]{../assets/tl.drawio.png}
\end{center}
\end{frame}
\begin{frame}
\frametitle{October 2023 - Garage v0.9.0}
Focus on \underline{streamlining \& usability}
\vspace{2em}
\begin{itemize}
\item Support multiple HDDs per node
\vspace{1em}
\item S3 compatibility:
\vspace{1em}
\begin{itemize}
\item support basic lifecycle configurations
\vspace{.5em}
\item allow for multipart upload part retries
\end{itemize}
\vspace{1em}
\item LMDB by default, deprecation of Sled
\vspace{1em}
\item New layout computation algorithm
\end{itemize}
\end{frame}
\begin{frame}
\frametitle{Layout computation}
\begin{overprint}
\onslide<1>
\begin{center}
\includegraphics[width=\linewidth, trim=0 0 0 -4cm]{../assets/screenshots/garage_status_0.9_prod_zonehl.png}
\end{center}
\onslide<2>
\begin{center}
\includegraphics[width=.7\linewidth]{../assets/map.png}
\end{center}
\end{overprint}
\vspace{1em}
Garage stores replicas on different zones when possible
\end{frame}
\begin{frame}
\frametitle{What a "layout" is}
\textbf{A layout is a precomputed index table:}
\vspace{1em}
{\footnotesize
\begin{center}
\begin{tabular}{|l|l|l|l|}
\hline
\textbf{Partition} & \textbf{Node 1} & \textbf{Node 2} & \textbf{Node 3} \\
\hline
\hline
Partition 0 & df-ymk (bespin) & Abricot (scorpio) & Courgette (neptune) \\
\hline
Partition 1 & Ananas (scorpio) & Courgette (neptune) & df-ykl (bespin) \\
\hline
Partition 2 & df-ymf (bespin) & Celeri (neptune) & Abricot (scorpio) \\
\hline
\hspace{1em}$\vdots$ & \hspace{1em}$\vdots$ & \hspace{1em}$\vdots$ & \hspace{1em}$\vdots$ \\
\hline
Partition 255 & Concombre (neptune) & df-ykl (bespin) & Abricot (scorpio) \\
\hline
\end{tabular}
\end{center}
}
\vspace{2em}
\visible<2->{
The index table is built centrally using an optimal algorithm,\\
then propagated to all nodes
}
\vspace{1em}
\visible<3->{
\footnotesize
Oulamara, M., \& Auvolat, A. (2023). \emph{An algorithm for geo-distributed and redundant storage in Garage}.\\ arXiv preprint arXiv:2302.13798.
}
\end{frame}
% ====================== v1.0.0 ===============================
\begin{frame}
\begin{center}
\includegraphics[width=.8\linewidth]{../assets/tl.drawio.png}
\end{center}
\end{frame}
\begin{frame}
\frametitle{April 2024 - Garage v1.0.0}
Focus on \underline{consistency, security \& stability}
\vspace{2em}
\begin{itemize}
\item Fix consistency issues when reshuffling data (Jepsen testing)
\vspace{1em}
\item \textbf{Security audit} by Radically Open Security
\vspace{1em}
\item Misc. S3 features (SSE-C, checksums, ...) and compatibility fixes
\end{itemize}
\end{frame}
% ====================== v2.0.0 ===============================
\begin{frame}
\begin{center}
\includegraphics[width=.8\linewidth]{../assets/tl.drawio.png}
\end{center}
\end{frame}
\begin{frame}
\frametitle{Garage v2.0.0}
Focus on \underline{}
\vspace{2em}
\begin{itemize}
\item TODO
\end{itemize}
\end{frame}
\begin{frame}
\frametitle{Currently funding...}
\textit{...}
\end{frame}
\begin{frame}
\frametitle{We run community surveys}
\begin{center}
\includegraphics[width=.6\linewidth]{../assets/survey_requested_features.png}
\end{center}
\end{frame}
% ======================================== END
% ======================================== END
% ======================================== END
\begin{frame}
\frametitle{Where to find us}
\begin{center}
\includegraphics[width=.25\linewidth]{../../logo/garage_hires.png}\\
\vspace{-1em}
\url{https://garagehq.deuxfleurs.fr/}\\
\url{mailto:garagehq@deuxfleurs.fr}\\
\texttt{\#garage:deuxfleurs.fr} on Matrix
\vspace{1.5em}
\includegraphics[width=.06\linewidth]{../assets/logos/rust_logo.png}
\includegraphics[width=.13\linewidth]{../assets/logos/AGPLv3_Logo.png}
\end{center}
\end{frame}
\end{document}
%% vim: set ts=4 sw=4 tw=0 noet spelllang=en :
Binary file not shown.

Before

Width:  |  Height:  |  Size: 30 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 50 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 24 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 6.1 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 123 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 7.9 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 183 KiB

Generated
+11 -12
View File
@@ -12,17 +12,16 @@
"original": {
"owner": "ipetkov",
"repo": "crane",
"rev": "6fe74265bbb6d016d663b1091f015e2976c4a527",
"type": "github"
}
},
"flake-compat": {
"locked": {
"lastModified": 1761640442,
"narHash": "sha256-AtrEP6Jmdvrqiv4x2xa5mrtaIp3OEe8uBYCDZDS+hu8=",
"lastModified": 1717312683,
"narHash": "sha256-FrlieJH50AuvagamEvWMIE6D2OAnERuDboFDYAED/dE=",
"owner": "nix-community",
"repo": "flake-compat",
"rev": "4a56054d8ffc173222d09dad23adf4ba946c8884",
"rev": "38fd3954cf65ce6faf3d0d45cd26059e059f07ea",
"type": "github"
},
"original": {
@@ -51,17 +50,17 @@
},
"nixpkgs": {
"locked": {
"lastModified": 1763977559,
"narHash": "sha256-g4MKqsIRy5yJwEsI+fYODqLUnAqIY4kZai0nldAP6EM=",
"lastModified": 1747825515,
"narHash": "sha256-BWpMQymVI73QoKZdcVCxUCCK3GNvr/xa2Dc4DM1o2BE=",
"owner": "NixOS",
"repo": "nixpkgs",
"rev": "cfe2c7d5b5d3032862254e68c37a6576b633d632",
"rev": "cd2812de55cf87df88a9e09bf3be1ce63d50c1a6",
"type": "github"
},
"original": {
"owner": "NixOS",
"repo": "nixpkgs",
"rev": "cfe2c7d5b5d3032862254e68c37a6576b633d632",
"rev": "cd2812de55cf87df88a9e09bf3be1ce63d50c1a6",
"type": "github"
}
},
@@ -81,17 +80,17 @@
]
},
"locked": {
"lastModified": 1763952169,
"narHash": "sha256-+PeDBD8P+NKauH+w7eO/QWCIp8Cx4mCfWnh9sJmy9CM=",
"lastModified": 1738549608,
"narHash": "sha256-GdyT9QEUSx5k/n8kILuNy83vxxdyUfJ8jL5mMpQZWfw=",
"owner": "oxalica",
"repo": "rust-overlay",
"rev": "ab726555a9a72e6dc80649809147823a813fa95b",
"rev": "35c6f8c4352f995ecd53896200769f80a3e8f22d",
"type": "github"
},
"original": {
"owner": "oxalica",
"repo": "rust-overlay",
"rev": "ab726555a9a72e6dc80649809147823a813fa95b",
"rev": "35c6f8c4352f995ecd53896200769f80a3e8f22d",
"type": "github"
}
},
+5 -20
View File
@@ -2,17 +2,16 @@
description =
"Garage, an S3-compatible distributed object store for self-hosted deployments";
# Nixpkgs 25.05 as of 2025-11-24
# Nixpkgs 25.05 as of 2025-05-22
inputs.nixpkgs.url =
"github:NixOS/nixpkgs/cfe2c7d5b5d3032862254e68c37a6576b633d632";
"github:NixOS/nixpkgs/cd2812de55cf87df88a9e09bf3be1ce63d50c1a6";
# Rust overlay as of 2025-11-24
# Rust overlay as of 2025-02-03
inputs.rust-overlay.url =
"github:oxalica/rust-overlay/ab726555a9a72e6dc80649809147823a813fa95b";
"github:oxalica/rust-overlay/35c6f8c4352f995ecd53896200769f80a3e8f22d";
inputs.rust-overlay.inputs.nixpkgs.follows = "nixpkgs";
# Crane as of 2025-01-24
inputs.crane.url = "github:ipetkov/crane/6fe74265bbb6d016d663b1091f015e2976c4a527";
inputs.crane.url = "github:ipetkov/crane";
inputs.flake-compat.url = "github:nix-community/flake-compat";
inputs.flake-utils.url = "github:numtide/flake-utils";
@@ -31,10 +30,6 @@
inherit system nixpkgs crane rust-overlay extraTestEnv;
release = false;
}).garage-test;
lints = (compile {
inherit system nixpkgs crane rust-overlay;
release = false;
});
in
{
packages = {
@@ -58,13 +53,6 @@
tests-sqlite = testWith {
GARAGE_TEST_INTEGRATION_DB_ENGINE = "sqlite";
};
tests-fjall = testWith {
GARAGE_TEST_INTEGRATION_DB_ENGINE = "fjall";
};
# lints (fmt, clippy)
fmt = lints.garage-cargo-fmt;
clippy = lints.garage-cargo-clippy;
};
# ---- developpment shell, for making native builds only ----
@@ -90,9 +78,6 @@
cargo-outdated
cargo-machete
nixpkgs-fmt
openssl
socat
killall
];
};
};
+1 -1
View File
@@ -167,7 +167,7 @@ let
</ul></p>
<p> Sources:
<ul>
<li><a href="https://git.deuxfleurs.fr/Deuxfleurs/garage/src/${r.type}/${x.version}">Forgejo</a></li>
<li><a href="https://git.deuxfleurs.fr/Deuxfleurs/garage/src/${r.type}/${x.version}">gitea</a></li>
<li><a href="https://git.deuxfleurs.fr/Deuxfleurs/garage/archive/${x.version}.zip">.zip</a></li>
<li><a href="https://git.deuxfleurs.fr/Deuxfleurs/garage/archive/${x.version}.tar.gz">.tar.gz</a></li>
</ul></p>
+2 -14
View File
@@ -48,7 +48,7 @@ let
inherit (pkgs) lib stdenv;
toolchainFn = (p: p.rust-bin.stable."1.91.0".default.override {
toolchainFn = (p: p.rust-bin.stable."1.82.0".default.override {
targets = lib.optionals (target != null) [ rustTarget ];
extensions = [
"rust-src"
@@ -68,13 +68,12 @@ let
rootFeatures = if features != null then
features
else
([ "bundled-libs" "lmdb" "sqlite" "fjall" "k2v" ] ++ (lib.optionals release [
([ "bundled-libs" "lmdb" "sqlite" "k2v" ] ++ (lib.optionals release [
"consul-discovery"
"kubernetes-discovery"
"metrics"
"telemetry-otlp"
"syslog"
"journald"
]));
featuresStr = lib.concatStringsSep "," rootFeatures;
@@ -190,15 +189,4 @@ in rec {
pkgs.cacert
];
} // extraTestEnv);
# ---- source code linting ----
garage-cargo-fmt = craneLib.cargoFmt (commonArgs // {
cargoExtraArgs = "";
});
garage-cargo-clippy = craneLib.cargoClippy (commonArgs // {
cargoArtifacts = garage-deps;
cargoClippyExtraArgs = "--all-targets -- -D warnings";
});
}
+1 -7
View File
@@ -30,12 +30,6 @@ for count in $(seq 1 3); do
CONF_PATH="/tmp/config.$count.toml"
LABEL="\e[${FANCYCOLORS[$count]}[$count]\e[49m"
if [ "$GARAGE_OLDVER" == "v08" ]; then
REPLICATION_MODE="replication_mode = \"3\""
else
REPLICATION_MODE="replication_factor = 3"
fi
cat > $CONF_PATH <<EOF
block_size = 1048576 # objects are split in blocks of maximum this number of bytes
metadata_dir = "/tmp/garage-meta-$count"
@@ -44,7 +38,7 @@ data_dir = "/tmp/garage-data-$count"
rpc_bind_addr = "0.0.0.0:$((3900+$count))" # the port other Garage nodes will use to talk to this node
rpc_public_addr = "127.0.0.1:$((3900+$count))"
bootstrap_peers = []
$REPLICATION_MODE
replication_mode = "3"
rpc_secret = "$NETWORK_SECRET"
[s3_api]
+18 -12
View File
@@ -1,18 +1,24 @@
apiVersion: v2
name: garage
description: S3-compatible object store for small self-hosted geo-distributed deployments
# A chart can be either an 'application' or a 'library' chart.
#
# Application charts are a collection of templates that can be packaged into versioned archives
# to be deployed.
#
# Library charts provide useful utilities or functions for the chart developer. They're included as
# a dependency of application charts to inject those utilities and functions into the rendering
# pipeline. Library charts do not define any templates and therefore cannot be deployed.
type: application
version: 0.9.2
appVersion: "v2.2.0"
home: https://garagehq.deuxfleurs.fr/
icon: https://garagehq.deuxfleurs.fr/images/garage-logo.svg
keywords:
- geo-distributed
- read-after-write-consistency
- s3-compatible
# This is the chart version. This version number should be incremented each time you make changes
# to the chart and its templates, including the app version.
# Versions are expected to follow Semantic Versioning (https://semver.org/)
version: 0.8.0
sources:
- https://git.deuxfleurs.fr/Deuxfleurs/garage.git
maintainers: []
# This is the version number of the application being deployed. This version number should be
# incremented each time you make changes to the application. Versions are not expected to
# follow Semantic Versioning. They should reflect the version the application is using.
# It is recommended to use it with quotes.
appVersion: "v2.0.0"
+5 -16
View File
@@ -1,21 +1,14 @@
# garage
![Version: 0.9.2](https://img.shields.io/badge/Version-0.9.2-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: v2.2.0](https://img.shields.io/badge/AppVersion-v2.2.0-informational?style=flat-square)
![Version: 0.6.0](https://img.shields.io/badge/Version-0.6.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: v1.0.1](https://img.shields.io/badge/AppVersion-v1.0.1-informational?style=flat-square)
S3-compatible object store for small self-hosted geo-distributed deployments
**Homepage:** <https://garagehq.deuxfleurs.fr/>
## Source Code
* <https://git.deuxfleurs.fr/Deuxfleurs/garage.git>
## Values
| Key | Type | Default | Description |
|-----|------|---------|-------------|
| affinity | object | `{}` | |
| commonLabels | object | `{}` | Extra labels for all resources |
| deployment.kind | string | `"StatefulSet"` | Switchable to DaemonSet |
| deployment.podManagementPolicy | string | `"OrderedReady"` | If using statefulset, allow Parallel or OrderedReady (default) |
| deployment.replicaCount | int | `3` | Number of StatefulSet replicas/garage nodes to start |
@@ -23,16 +16,14 @@ S3-compatible object store for small self-hosted geo-distributed deployments
| extraVolumeMounts | object | `{}` | |
| extraVolumes | object | `{}` | |
| fullnameOverride | string | `""` | |
| garage.blockSize | string | `"1048576"` | Defaults is 1MB An increase can result in better performance in certain scenarios https://garagehq.deuxfleurs.fr/documentation/reference-manual/configuration/#block_size |
| garage.blockSize | string | `"1048576"` | Defaults is 1MB An increase can result in better performance in certain scenarios https://garagehq.deuxfleurs.fr/documentation/reference-manual/configuration/#block-size |
| garage.bootstrapPeers | list | `[]` | This is not required if you use the integrated kubernetes discovery |
| garage.compressionLevel | string | `"1"` | zstd compression level of stored blocks https://garagehq.deuxfleurs.fr/documentation/reference-manual/configuration/#compression_level |
| garage.dbEngine | string | `"lmdb"` | Can be changed for better performance on certain systems https://garagehq.deuxfleurs.fr/documentation/reference-manual/configuration/#db_engine |
| garage.compressionLevel | string | `"1"` | zstd compression level of stored blocks https://garagehq.deuxfleurs.fr/documentation/reference-manual/configuration/#compression-level |
| garage.dbEngine | string | `"lmdb"` | Can be changed for better performance on certain systems https://garagehq.deuxfleurs.fr/documentation/reference-manual/configuration/#db-engine-since-v0-8-0 |
| garage.existingConfigMap | string | `""` | if not empty string, allow using an existing ConfigMap for the garage.toml, if set, ignores garage.toml |
| garage.garageTomlString | string | `""` | String Template for the garage configuration if set, ignores above values. Values can be templated, see https://garagehq.deuxfleurs.fr/documentation/reference-manual/configuration/ |
| garage.kubernetesSkipCrd | bool | `false` | Set to true if you want to use k8s discovery but install the CRDs manually outside of the helm chart, for example if you operate at namespace level without cluster ressources |
| garage.replicationFactor | string | `"3"` | Default to 3 replicas, see the replication_factor section at https://garagehq.deuxfleurs.fr/documentation/reference-manual/configuration/#replication_factor |
| garage.consistencyMode | string | `"consistent"` | Default to read-after-write consistency, see the consistency_mode section at https://garagehq.deuxfleurs.fr/documentation/reference-manual/configuration/#consistency_mode |
| garage.metadataAutoSnapshotInterval | string | `""` | If this value is set, Garage will automatically take a snapshot of the metadata DB file at a regular interval and save it in the metadata directory. https://garagehq.deuxfleurs.fr/documentation/reference-manual/configuration/#metadata_auto_snapshot_interval |
| garage.replicationMode | string | `"3"` | Default to 3 replicas, see the replication_mode section at https://garagehq.deuxfleurs.fr/documentation/reference-manual/configuration/#replication-mode |
| garage.rpcBindAddr | string | `"[::]:3901"` | |
| garage.rpcSecret | string | `""` | If not given, a random secret will be generated and stored in a Secret object |
| garage.s3.api.region | string | `"garage"` | |
@@ -58,7 +49,6 @@ S3-compatible object store for small self-hosted geo-distributed deployments
| initImage.pullPolicy | string | `"IfNotPresent"` | |
| initImage.repository | string | `"busybox"` | |
| initImage.tag | string | `"stable"` | |
| livenessProbe | object | `{}` | Specifies a livenessProbe |
| monitoring.metrics.enabled | bool | `false` | If true, a service for monitoring is created with a prometheus.io/scrape annotation |
| monitoring.metrics.serviceMonitor.enabled | bool | `false` | If true, a ServiceMonitor CRD is created for a prometheus operator https://github.com/coreos/prometheus-operator |
| monitoring.metrics.serviceMonitor.interval | string | `"15s"` | |
@@ -81,7 +71,6 @@ S3-compatible object store for small self-hosted geo-distributed deployments
| podSecurityContext.runAsGroup | int | `1000` | |
| podSecurityContext.runAsNonRoot | bool | `true` | |
| podSecurityContext.runAsUser | int | `1000` | |
| readinessProbe | object | `{}` | Specifies a readinessProbe |
| resources | object | `{}` | |
| securityContext.capabilities | object | `{"drop":["ALL"]}` | The default security context is heavily restricted, feel free to tune it to your requirements |
| securityContext.readOnlyRootFilesystem | bool | `true` | |
+1 -4
View File
@@ -27,7 +27,7 @@ If release name contains chart name it will be used as a full name.
Create the name of the rpc secret
*/}}
{{- define "garage.rpcSecretName" -}}
{{- .Values.garage.existingRpcSecret | default (printf "%s-rpc-secret" (include "garage.fullname" .)) -}}
{{- printf "%s-rpc-secret" (include "garage.fullname" .) -}}
{{- end }}
{{/*
@@ -47,9 +47,6 @@ helm.sh/chart: {{ include "garage.chart" . }}
app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
{{- end }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
{{- with .Values.commonLabels }}
{{- toYaml . | nindent 0 }}
{{- end }}
{{- end }}
{{/*
+2 -15
View File
@@ -15,29 +15,16 @@ data:
block_size = {{ .Values.garage.blockSize }}
replication_factor = {{ .Values.garage.replicationFactor }}
consistency_mode = "{{ .Values.garage.consistencyMode }}"
replication_mode = "{{ .Values.garage.replicationMode }}"
compression_level = {{ .Values.garage.compressionLevel }}
{{- if .Values.garage.metadataAutoSnapshotInterval }}
metadata_auto_snapshot_interval = {{ .Values.garage.metadataAutoSnapshotInterval | quote }}
{{- end }}
rpc_bind_addr = "{{ .Values.garage.rpcBindAddr }}"
# rpc_secret will be populated by the init container from a k8s secret object
rpc_secret = "__RPC_SECRET_REPLACE__"
bootstrap_peers = [
{{- range $index, $peer := .Values.garage.bootstrapPeers }}
{{- if $index}}, {{ end }}{{ $peer | quote }}
{{ end }}
]
bootstrap_peers = {{ .Values.garage.bootstrapPeers }}
{{- if .Values.garage.additionalTopLevelConfig }}
{{ .Values.garage.additionalTopLevelConfig | nindent 4 }}
{{- end }}
[kubernetes_discovery]
namespace = "{{ .Release.Namespace }}"
service_name = "{{ include "garage.fullname" . }}"
-2
View File
@@ -1,4 +1,3 @@
{{- if not .Values.garage.existingRpcSecret }}
apiVersion: v1
kind: Secret
metadata:
@@ -13,4 +12,3 @@ data:
{{- $prevRpcSecret := $prevSecretData.rpcSecret | default "" | b64dec }}
{{/* Priority is: 1. from values, 2. previous value, 3. generate random */}}
rpcSecret: {{ .Values.garage.rpcSecret | default $prevRpcSecret | default (include "jupyterhub.randHex" 64) | b64enc | quote }}
{{- end }}
+1 -5
View File
@@ -4,10 +4,6 @@ metadata:
name: {{ include "garage.fullname" . }}
labels:
{{- include "garage.labels" . | nindent 4 }}
{{- with .Values.service.annotations }}
annotations:
{{- toYaml . | nindent 4 }}
{{- end }}
spec:
type: {{ .Values.service.type }}
ports:
@@ -41,4 +37,4 @@ spec:
name: metrics
selector:
{{- include "garage.selectorLabels" . | nindent 4 }}
{{- end }}
{{- end }}
+10 -9
View File
@@ -21,7 +21,7 @@ spec:
{{- toYaml . | nindent 8 }}
{{- end }}
labels:
{{- include "garage.labels" . | nindent 8 }}
{{- include "garage.selectorLabels" . | nindent 8 }}
spec:
{{- with .Values.imagePullSecrets }}
imagePullSecrets:
@@ -78,14 +78,15 @@ spec:
{{- with .Values.extraVolumeMounts }}
{{- toYaml . | nindent 12 }}
{{- end }}
{{- with .Values.livenessProbe }}
livenessProbe:
{{- toYaml . | nindent 12 }}
{{- end }}
{{- with .Values.readinessProbe }}
readinessProbe:
{{- toYaml . | nindent 12 }}
{{- end }}
# TODO
# livenessProbe:
# httpGet:
# path: /
# port: 3900
# readinessProbe:
# httpGet:
# path: /
# port: 3900
resources:
{{- toYaml .Values.resources | nindent 12 }}
volumes:
+6 -47
View File
@@ -2,45 +2,28 @@
# This is a YAML-formatted file.
# Declare variables to be passed into your templates.
# -- Additional labels to add to all resources created by this chart
commonLabels: {}
# app.kubernetes.io/part-of: storage
# team: platform
# Garage configuration. These values go to garage.toml
garage:
# -- Can be changed for better performance on certain systems
# https://garagehq.deuxfleurs.fr/documentation/reference-manual/configuration/#db_engine
# https://garagehq.deuxfleurs.fr/documentation/reference-manual/configuration/#db-engine-since-v0-8-0
dbEngine: "lmdb"
# -- Defaults is 1MB
# An increase can result in better performance in certain scenarios
# https://garagehq.deuxfleurs.fr/documentation/reference-manual/configuration/#block_size
# https://garagehq.deuxfleurs.fr/documentation/reference-manual/configuration/#block-size
blockSize: "1048576"
# -- Default to 3 replicas, see the replication_factor section at
# https://garagehq.deuxfleurs.fr/documentation/reference-manual/configuration/#replication_factor
replicationFactor: "3"
# -- By default, enable read-after-write consistency guarantees, see the consistency_mode section at
# https://garagehq.deuxfleurs.fr/documentation/reference-manual/configuration/#consistency_mode
consistencyMode: "consistent"
# -- Default to 3 replicas, see the replication_mode section at
# https://garagehq.deuxfleurs.fr/documentation/reference-manual/configuration/#replication-mode
replicationMode: "3"
# -- zstd compression level of stored blocks
# https://garagehq.deuxfleurs.fr/documentation/reference-manual/configuration/#compression_level
# https://garagehq.deuxfleurs.fr/documentation/reference-manual/configuration/#compression-level
compressionLevel: "1"
# -- If this value is set, Garage will automatically take a snapshot of the metadata DB file at a regular interval and save it in the metadata directory.
# https://garagehq.deuxfleurs.fr/documentation/reference-manual/configuration/#metadata_auto_snapshot_interval
metadataAutoSnapshotInterval: ""
rpcBindAddr: "[::]:3901"
# -- If not given, a random secret will be generated and stored in a Secret object
rpcSecret: ""
# -- If you want to provide an rpcSecret within an existing k8s secret,
# specify the secret name here, and store the value under the secret key `rpcSecret`
# the default secret will not be created
existingRpcSecret: ""
# -- This is not required if you use the integrated kubernetes discovery
bootstrapPeers: []
# -- Set to true if you want to use k8s discovery but install the CRDs manually outside
@@ -54,12 +37,6 @@ garage:
rootDomain: ".web.garage.tld"
index: "index.html"
# -- Additional configuration to append to garage.toml. Use a multi-line string for custom config.
# Example:
# additionalTopLevelConfig: |-
# data_fsync = true
additionalTopLevelConfig: ""
# -- if not empty string, allow using an existing ConfigMap for the garage.toml,
# if set, ignores garage.toml
existingConfigMap: ""
@@ -127,7 +104,6 @@ podSecurityContext:
runAsUser: 1000
runAsGroup: 1000
fsGroup: 1000
fsGroupChangePolicy: "OnRootMismatch"
runAsNonRoot: true
securityContext:
@@ -144,8 +120,6 @@ service:
# - NodePort (+ Ingress)
# - LoadBalancer
type: ClusterIP
# -- Annotations to add to the service
annotations: {}
s3:
api:
port: 3900
@@ -217,21 +191,6 @@ resources: {}
# cpu: 100m
# memory: 512Mi
# -- Specifies a livenessProbe
livenessProbe: {}
#httpGet:
# path: /health
# port: 3903
#initialDelaySeconds: 5
#periodSeconds: 30
# -- Specifies a readinessProbe
readinessProbe: {}
#httpGet:
# path: /health
# port: 3903
#initialDelaySeconds: 5
#periodSeconds: 30
nodeSelector: {}
tolerations: []
@@ -43,7 +43,7 @@
"rpc_bind_addr = \"0.0.0.0:3901\"\n"
"rpc_public_addr = \"" node ":3901\"\n"
"db_engine = \"lmdb\"\n"
"replication_factor = 3\n"
"replication_mode = \"3\"\n"
"data_dir = \"" data-dir "\"\n"
"metadata_dir = \"" meta-dir "\"\n"
"[s3_api]\n"
+1 -1
View File
@@ -8,7 +8,7 @@ data:
metadata_dir = "/tmp/meta"
data_dir = "/tmp/data"
replication_factor = 3
replication_mode = "3"
rpc_bind_addr = "[::]:3901"
rpc_secret = "1799bccfd7411eddcf9ebd316bc1f5287ad12a68094e1c6ac6abde7e6feae1ec"
@@ -1,43 +0,0 @@
apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
name: garagenodes.deuxfleurs.fr
spec:
conversion:
strategy: None
group: deuxfleurs.fr
names:
kind: GarageNode
listKind: GarageNodeList
plural: garagenodes
singular: garagenode
scope: Namespaced
versions:
- name: v1
schema:
openAPIV3Schema:
description: Auto-generated derived type for Node via `CustomResource`
properties:
spec:
properties:
address:
format: ip
type: string
hostname:
type: string
port:
format: uint16
minimum: 0
type: integer
required:
- address
- hostname
- port
type: object
required:
- spec
title: GarageNode
type: object
served: true
storage: true
subresources: {}
-5
View File
@@ -1,5 +0,0 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- garagenodes.deuxfleurs.fr.yaml
@@ -694,7 +694,32 @@
]
}
},
"overrides": []
"overrides": [
{
"__systemRef": "hideSeriesFrom",
"matcher": {
"id": "byNames",
"options": {
"mode": "exclude",
"names": [
"10.83.2.3:3903"
],
"prefix": "All except:",
"readOnly": true
}
},
"properties": [
{
"id": "custom.hideFrom",
"value": {
"legend": false,
"tooltip": false,
"viz": true
}
}
]
}
]
},
"gridPos": {
"h": 8,
-4
View File
@@ -119,12 +119,8 @@ if [ -z "$SKIP_BOTO3" ]; then
AWS_ENDPOINT_URL=https://localhost:4443 python <<EOF
import boto3
client = boto3.client('s3', verify=False)
print("Put&delete hello world object")
client.put_object(Body=b'hello world', Bucket='eprouvette', Key='test.s3.txt')
client.delete_object(Bucket='eprouvette', Key='test.s3.txt')
print("Put&delete big object")
client.upload_file("/tmp/garage.3.rnd", 'eprouvette', 'garage.3.rnd')
client.delete_object(Bucket='eprouvette', Key='garage.3.rnd')
print("OK!")
EOF
fi
+1 -6
View File
@@ -26,7 +26,7 @@ in
s3cmd
minio-client
rclone
(python313.withPackages (ps: [ ps.boto3 ]))
(python312.withPackages (ps: [ ps.boto3 ]))
socat
psmisc
@@ -36,10 +36,7 @@ in
jq
];
shellHook = ''
export AWS_REQUEST_CHECKSUM_CALCULATION='when_required'
function to_s3 {
AWS_REQUEST_CHECKSUM_CALCULATION=WHEN_REQUIRED AWS_RESPONSE_CHECKSUM_VALIDATION=WHEN_REQUIRED \
aws \
--endpoint-url https://garage.deuxfleurs.fr \
--region garage \
@@ -96,7 +93,6 @@ in
nix-build nix/build_index.nix
AWS_REQUEST_CHECKSUM_CALCULATION=WHEN_REQUIRED AWS_RESPONSE_CHECKSUM_VALIDATION=WHEN_REQUIRED \
aws \
--endpoint-url https://garage.deuxfleurs.fr \
--region garage \
@@ -104,7 +100,6 @@ in
result/share/_releases.json \
s3://garagehq.deuxfleurs.fr/
AWS_REQUEST_CHECKSUM_CALCULATION=WHEN_REQUIRED AWS_RESPONSE_CHECKSUM_VALIDATION=WHEN_REQUIRED \
aws \
--endpoint-url https://garage.deuxfleurs.fr \
--region garage \
+2 -2
View File
@@ -1,6 +1,6 @@
[package]
name = "garage_api_admin"
version = "2.2.0"
version = "2.0.0"
authors = ["Alex Auvolat <alex@adnab.me>"]
edition = "2018"
license = "AGPL-3.0"
@@ -26,7 +26,7 @@ argon2.workspace = true
async-trait.workspace = true
bytesize.workspace = true
chrono.workspace = true
thiserror.workspace = true
err-derive.workspace = true
hex.workspace = true
paste.workspace = true
tracing.workspace = true
-57
View File
@@ -175,63 +175,6 @@ impl RequestHandler for DeleteAdminTokenRequest {
}
}
impl RequestHandler for GetCurrentAdminTokenInfoRequest {
type Response = GetCurrentAdminTokenInfoResponse;
async fn handle(
self,
garage: &Arc<Garage>,
_admin: &Admin,
) -> Result<GetCurrentAdminTokenInfoResponse, Error> {
let now = now_msec();
if garage
.config
.admin
.metrics_token
.as_ref()
.is_some_and(|s| s == &self.admin_token)
{
return Ok(GetCurrentAdminTokenInfoResponse(
GetAdminTokenInfoResponse {
id: None,
created: None,
name: "metrics_token (from daemon configuration)".into(),
expiration: None,
expired: false,
scope: vec!["Metrics".into()],
},
));
}
if garage
.config
.admin
.admin_token
.as_ref()
.is_some_and(|s| s == &self.admin_token)
{
return Ok(GetCurrentAdminTokenInfoResponse(
GetAdminTokenInfoResponse {
id: None,
created: None,
name: "admin_token (from daemon configuration)".into(),
expiration: None,
expired: false,
scope: vec!["*".into()],
},
));
}
let (prefix, _) = self.admin_token.split_once('.').unwrap();
let token = get_existing_admin_token(&garage, &prefix.to_string()).await?;
Ok(GetCurrentAdminTokenInfoResponse(admin_token_info_results(
&token, now,
)))
}
}
// ---- helpers ----
fn admin_token_info_results(token: &AdminApiToken, now: u64) -> GetAdminTokenInfoResponse {
+9 -50
View File
@@ -12,7 +12,7 @@ use garage_rpc::*;
use garage_model::garage::Garage;
use garage_api_common::{common_error::CommonError, helpers::is_default};
use garage_api_common::helpers::is_default;
use crate::api_server::{find_matching_nodes, AdminRpc, AdminRpcResponse};
use crate::error::Error;
@@ -56,7 +56,6 @@ admin_endpoints![
CreateAdminToken,
UpdateAdminToken,
DeleteAdminToken,
GetCurrentAdminTokenInfo,
// Layout operations
GetClusterLayout,
@@ -145,13 +144,6 @@ pub struct MultiResponse<RB> {
pub error: HashMap<String, String>,
}
#[derive(Debug, Clone, Serialize, Deserialize, IntoParams)]
#[into_params(parameter_in = Query)]
pub struct MultiRequestQueryParams {
/// Node ID to query, or `*` for all nodes, or `self` for the node responding to the request
pub node: String,
}
// **********************************************
// Special endpoints
//
@@ -162,10 +154,8 @@ pub struct MultiRequestQueryParams {
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct OptionsRequest;
#[derive(Debug, Clone, Serialize, Deserialize, IntoParams)]
#[into_params(parameter_in = Query)]
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct CheckDomainRequest {
/// The domain name to check for
pub domain: String,
}
@@ -267,7 +257,7 @@ pub struct GetClusterHealthResponse {
/// the number of storage nodes currently registered in the cluster layout
pub storage_nodes: usize,
/// the number of storage nodes to which a connection is currently open
pub storage_nodes_up: usize,
pub storage_nodes_ok: usize,
/// the total number of partitions of the data (currently always 256)
pub partitions: usize,
/// the number of partitions for which a quorum of write nodes is available
@@ -318,7 +308,6 @@ pub struct ListAdminTokensResponse(pub Vec<GetAdminTokenInfoResponse>);
// ---- GetAdminTokenInfo ----
#[derive(Debug, Clone, Serialize, Deserialize, IntoParams)]
#[into_params(parameter_in = Query)]
#[serde(rename_all = "camelCase")]
pub struct GetAdminTokenInfoRequest {
/// Admin API token ID
@@ -364,12 +353,9 @@ pub struct CreateAdminTokenResponse {
// ---- UpdateAdminToken ----
#[derive(Debug, Clone, Serialize, Deserialize, IntoParams)]
#[into_params(parameter_in = Query)]
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct UpdateAdminTokenRequest {
/// Admin API token ID
pub id: String,
#[param(ignore = true)]
pub body: UpdateAdminTokenRequestBody,
}
@@ -396,25 +382,14 @@ pub struct UpdateAdminTokenResponse(pub GetAdminTokenInfoResponse);
// ---- DeleteAdminToken ----
#[derive(Debug, Clone, Serialize, Deserialize, IntoParams)]
#[into_params(parameter_in = Query)]
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct DeleteAdminTokenRequest {
/// Admin API token ID
pub id: String,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct DeleteAdminTokenResponse;
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct GetCurrentAdminTokenInfoRequest {
pub admin_token: String,
}
#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)]
#[serde(rename_all = "camelCase")]
pub struct GetCurrentAdminTokenInfoResponse(pub GetAdminTokenInfoResponse);
// **********************************************
// Layout operations
// **********************************************
@@ -673,7 +648,6 @@ pub struct ListKeysResponseItem {
// ---- GetKeyInfo ----
#[derive(Debug, Clone, Serialize, Deserialize, IntoParams)]
#[into_params(parameter_in = Query)]
#[serde(rename_all = "camelCase")]
pub struct GetKeyInfoRequest {
/// Access key ID
@@ -750,12 +724,9 @@ pub struct ImportKeyResponse(pub GetKeyInfoResponse);
// ---- UpdateKey ----
#[derive(Debug, Clone, Serialize, Deserialize, IntoParams)]
#[into_params(parameter_in = Query)]
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct UpdateKeyRequest {
/// Access key ID
pub id: String,
#[param(ignore = true)]
pub body: UpdateKeyRequestBody,
}
@@ -780,10 +751,8 @@ pub struct UpdateKeyRequestBody {
// ---- DeleteKey ----
#[derive(Debug, Clone, Serialize, Deserialize, IntoParams)]
#[into_params(parameter_in = Query)]
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct DeleteKeyRequest {
/// Access key ID
pub id: String,
}
@@ -821,7 +790,6 @@ pub struct BucketLocalAlias {
// ---- GetBucketInfo ----
#[derive(Debug, Clone, Serialize, Deserialize, IntoParams)]
#[into_params(parameter_in = Query)]
#[serde(rename_all = "camelCase")]
pub struct GetBucketInfoRequest {
/// Exact bucket ID to look up
@@ -910,12 +878,9 @@ pub struct CreateBucketLocalAlias {
// ---- UpdateBucket ----
#[derive(Debug, Clone, Serialize, Deserialize, IntoParams)]
#[into_params(parameter_in = Query)]
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct UpdateBucketRequest {
/// ID of the bucket to update
pub id: String,
#[param(ignore = true)]
pub body: UpdateBucketRequestBody,
}
@@ -939,10 +904,8 @@ pub struct UpdateBucketWebsiteAccess {
// ---- DeleteBucket ----
#[derive(Debug, Clone, Serialize, Deserialize, IntoParams)]
#[into_params(parameter_in = Query)]
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct DeleteBucketRequest {
/// ID of the bucket to delete
pub id: String,
}
@@ -965,7 +928,6 @@ pub struct CleanupIncompleteUploadsResponse {
}
#[derive(Debug, Clone, Serialize, Deserialize, IntoParams)]
#[into_params(parameter_in = Query)]
#[serde(rename_all = "camelCase")]
pub struct InspectObjectRequest {
pub bucket_id: String,
@@ -1152,8 +1114,6 @@ pub enum RepairType {
BlockRc,
Rebalance,
Scrub(ScrubCommand),
Aliases,
ClearResyncQueue,
}
#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)]
@@ -1348,5 +1308,4 @@ pub struct LocalPurgeBlocksResponse {
pub objects_deleted: u64,
pub uploads_deleted: u64,
pub versions_deleted: u64,
pub block_refs_purged: u64,
}
+32 -43
View File
@@ -217,13 +217,6 @@ impl ApiHandler for ArcAdminApiServer {
) -> Result<Response<ResBody>, Error> {
self.0.handle_http_api(req, endpoint).await
}
fn key_id_from_request(&self, req: &Request<IncomingBody>) -> Option<String> {
let auth_header = req.headers().get(AUTHORIZATION)?;
let token = parse_authorization(auth_header).ok()?;
let key_id = token.split_once('.')?.0;
Some(key_id.to_string())
}
}
impl ApiEndpoint for HttpEndpoint {
@@ -251,15 +244,6 @@ fn hash_bearer_token(token: &str) -> String {
.to_string()
}
fn parse_authorization(auth_header: &hyper::http::HeaderValue) -> Result<&str, Error> {
let token = auth_header
.to_str()?
.strip_prefix("Bearer ")
.ok_or_else(|| Error::forbidden("Invalid Authorization header"))?
.trim();
Ok(token)
}
fn verify_authorization(
garage: &Garage,
global_token_hash: Option<&str>,
@@ -276,7 +260,11 @@ fn verify_authorization(
"Bearer token must be provided in Authorization header",
))
}
Some(authorization) => parse_authorization(authorization)?,
Some(authorization) => authorization
.to_str()?
.strip_prefix("Bearer ")
.ok_or_else(|| Error::forbidden("Invalid Authorization header"))?
.trim(),
};
let token_hash_string = if let Some((prefix, _)) = token.split_once('.') {
@@ -285,8 +273,7 @@ fn verify_authorization(
.get_local(&EmptyKey, &prefix.to_string())?
.and_then(|k| k.state.into_option())
.filter(|p| !p.is_expired(now_msec()))
// GetCurrentAdminTokenInfo endpoint must be accessible even if it is not in the token scopes
.filter(|p| p.has_scope(endpoint_name) || endpoint_name == "GetCurrentAdminTokenInfo")
.filter(|p| p.has_scope(endpoint_name))
.ok_or_else(|| Error::forbidden(invalid_msg))?
.token_hash
} else {
@@ -306,36 +293,38 @@ fn verify_authorization(
}
pub(crate) fn find_matching_nodes(garage: &Garage, spec: &str) -> Result<Vec<Uuid>, Error> {
if spec == "self" {
Ok(vec![garage.system.id])
} else {
// Collect all nodes currently up and/or in cluster layout
let mut res = vec![];
if let Ok(all_nodes) = garage.system.cluster_layout().all_nodes() {
res = all_nodes.to_vec();
}
let mut res = vec![];
if spec == "*" {
res = garage.system.cluster_layout().all_nodes().to_vec();
for node in garage.system.get_known_nodes() {
if node.is_up && !res.contains(&node.id) {
res.push(node.id);
}
}
if spec == "*" {
// match all nodes
Ok(res)
} else {
// filter nodes that match spec
res.retain(|node| hex::encode(node).starts_with(spec));
if res.is_empty() {
Err(Error::bad_request(format!("No nodes matching {}", spec)))
} else if res.len() > 1 {
Err(Error::bad_request(format!(
"Multiple nodes matching {}: {:?}",
spec, res
)))
} else {
Ok(res)
} else if spec == "self" {
res.push(garage.system.id);
} else {
let layout = garage.system.cluster_layout();
let known_nodes = garage.system.get_known_nodes();
let all_nodes = layout
.all_nodes()
.iter()
.copied()
.chain(known_nodes.iter().filter(|x| x.is_up).map(|x| x.id));
for node in all_nodes {
if !res.contains(&node) && hex::encode(node).starts_with(spec) {
res.push(node);
}
}
if res.is_empty() {
return Err(Error::bad_request(format!("No nodes matching {}", spec)));
}
if res.len() > 1 {
return Err(Error::bad_request(format!(
"Multiple nodes matching {}: {:?}",
spec, res
)));
}
}
Ok(res)
}
-8
View File
@@ -151,7 +151,6 @@ impl RequestHandler for LocalPurgeBlocksRequest {
let mut obj_dels = 0;
let mut mpu_dels = 0;
let mut ver_dels = 0;
let mut br_dels = 0;
for hash in self.0.iter() {
let hash = hex::decode(hash).ok_or_bad_request("invalid hash")?;
@@ -177,18 +176,11 @@ impl RequestHandler for LocalPurgeBlocksRequest {
ver_dels += 1;
}
}
if !br.deleted.get() {
let mut br = br;
br.deleted.set();
garage.block_ref_table.insert(&br).await?;
br_dels += 1;
}
}
}
Ok(LocalPurgeBlocksResponse {
blocks_purged: self.0.len() as u64,
block_refs_purged: br_dels,
versions_deleted: ver_dels,
objects_deleted: obj_dels,
uploads_deleted: mpu_dels,
+15 -16
View File
@@ -159,7 +159,7 @@ impl RequestHandler for CreateBucketRequest {
let helper = garage.locked_helper().await;
if let Some(ga) = &self.global_alias {
if !is_valid_bucket_name(ga, garage.config.allow_punycode) {
if !is_valid_bucket_name(ga) {
return Err(Error::bad_request(format!(
"{}: {}",
ga, INVALID_BUCKET_NAME_MESSAGE
@@ -174,7 +174,7 @@ impl RequestHandler for CreateBucketRequest {
}
if let Some(la) = &self.local_alias {
if !is_valid_bucket_name(&la.alias, garage.config.allow_punycode) {
if !is_valid_bucket_name(&la.alias) {
return Err(Error::bad_request(format!(
"{}: {}",
la.alias, INVALID_BUCKET_NAME_MESSAGE
@@ -255,7 +255,7 @@ impl RequestHandler for DeleteBucketRequest {
for ((key_id, alias), _, active) in state.local_aliases.items().iter() {
if *active {
helper
.purge_local_bucket_alias(bucket.id, key_id, alias)
.unset_local_bucket_alias(bucket.id, key_id, alias)
.await?;
}
}
@@ -697,21 +697,20 @@ async fn bucket_info_results(
}),
keys: relevant_keys
.into_values()
.filter_map(|key| {
.map(|key| {
let p = key.state.as_option().unwrap();
let permissions = p
.authorized_buckets
.get(&bucket.id)
.filter(|p| p.is_any())
.map(|p| ApiBucketKeyPerm {
read: p.allow_read,
write: p.allow_write,
owner: p.allow_owner,
})?;
Some(GetBucketInfoKey {
GetBucketInfoKey {
access_key_id: key.key_id,
name: p.name.get().to_string(),
permissions,
permissions: p
.authorized_buckets
.get(&bucket.id)
.map(|p| ApiBucketKeyPerm {
read: p.allow_read,
write: p.allow_write,
owner: p.allow_owner,
})
.unwrap_or_default(),
bucket_local_aliases: p
.local_aliases
.items()
@@ -719,7 +718,7 @@ async fn bucket_info_results(
.filter(|(_, _, b)| *b == Some(bucket.id))
.map(|(n, _, _)| n.to_string())
.collect::<Vec<_>>(),
})
}
})
.collect::<Vec<_>>(),
objects: *counters.get(OBJECTS).unwrap_or(&0),
+42 -54
View File
@@ -56,52 +56,48 @@ impl RequestHandler for GetClusterStatusRequest {
})
.collect::<HashMap<_, _>>();
if let Ok(current_layout) = layout.current() {
for (id, _, role) in current_layout.roles.items().iter() {
if let layout::NodeRoleV(Some(r)) = role {
let role = NodeAssignedRole {
zone: r.zone.to_string(),
capacity: r.capacity,
tags: r.tags.clone(),
};
match nodes.get_mut(id) {
None => {
nodes.insert(
*id,
NodeResp {
id: hex::encode(id),
role: Some(role),
..Default::default()
},
);
}
Some(n) => {
n.role = Some(role);
}
for (id, _, role) in layout.current().roles.items().iter() {
if let layout::NodeRoleV(Some(r)) = role {
let role = NodeAssignedRole {
zone: r.zone.to_string(),
capacity: r.capacity,
tags: r.tags.clone(),
};
match nodes.get_mut(id) {
None => {
nodes.insert(
*id,
NodeResp {
id: hex::encode(id),
role: Some(role),
..Default::default()
},
);
}
Some(n) => {
n.role = Some(role);
}
}
}
}
if let Ok(layout_versions) = layout.versions() {
for ver in layout_versions.iter().rev().skip(1) {
for (id, _, role) in ver.roles.items().iter() {
if let layout::NodeRoleV(Some(r)) = role {
if r.capacity.is_some() {
if let Some(n) = nodes.get_mut(id) {
if n.role.is_none() {
n.draining = true;
}
} else {
nodes.insert(
*id,
NodeResp {
id: hex::encode(id),
draining: true,
..Default::default()
},
);
for ver in layout.versions().iter().rev().skip(1) {
for (id, _, role) in ver.roles.items().iter() {
if let layout::NodeRoleV(Some(r)) = role {
if r.capacity.is_some() {
if let Some(n) = nodes.get_mut(id) {
if n.role.is_none() {
n.draining = true;
}
} else {
nodes.insert(
*id,
NodeResp {
id: hex::encode(id),
draining: true,
..Default::default()
},
);
}
}
}
@@ -112,7 +108,7 @@ impl RequestHandler for GetClusterStatusRequest {
nodes.sort_by(|x, y| x.id.cmp(&y.id));
Ok(GetClusterStatusResponse {
layout_version: layout.inner().current().version,
layout_version: layout.current().version,
nodes,
})
}
@@ -138,9 +134,7 @@ impl RequestHandler for GetClusterHealthRequest {
known_nodes: health.known_nodes,
connected_nodes: health.connected_nodes,
storage_nodes: health.storage_nodes,
// Translating storage_nodes_up (admin API context) to storage_nodes_ok (metrics context)
// TODO: when releasing major release, consider renaming all the fields in the metrics to storage_nodes_up
storage_nodes_up: health.storage_nodes_ok,
storage_nodes_ok: health.storage_nodes_ok,
partitions: health.partitions,
partitions_quorum: health.partitions_quorum,
partitions_all_ok: health.partitions_all_ok,
@@ -163,11 +157,9 @@ impl RequestHandler for GetClusterStatisticsRequest {
// Gather storage node and free space statistics for current nodes
let layout = &garage.system.cluster_layout();
let mut node_partition_count = HashMap::<Uuid, u64>::new();
if let Ok(current_layout) = layout.current() {
for short_id in current_layout.ring_assignment_data.iter() {
let id = current_layout.node_id_vec[*short_id as usize];
*node_partition_count.entry(id).or_default() += 1;
}
for short_id in layout.current().ring_assignment_data.iter() {
let id = layout.current().node_id_vec[*short_id as usize];
*node_partition_count.entry(id).or_default() += 1;
}
let node_info = garage
.system
@@ -180,11 +172,7 @@ impl RequestHandler for GetClusterStatisticsRequest {
for (id, parts) in node_partition_count.iter() {
let info = node_info.get(id);
let status = info.map(|x| &x.status);
let role = layout
.current()
.ok()
.and_then(|l| l.roles.get(id))
.and_then(|x| x.0.as_ref());
let role = layout.current().roles.get(id).and_then(|x| x.0.as_ref());
let hostname = status.and_then(|x| x.hostname.as_deref()).unwrap_or("?");
let zone = role.map(|x| x.zone.as_str()).unwrap_or("?");
let capacity = role
+12 -10
View File
@@ -1,8 +1,8 @@
use std::convert::TryFrom;
use err_derive::Error;
use hyper::header::HeaderValue;
use hyper::{HeaderMap, StatusCode};
use thiserror::Error;
pub use garage_model::helper::error::Error as HelperError;
@@ -16,33 +16,36 @@ use garage_api_common::helpers::*;
/// Errors of this crate
#[derive(Debug, Error)]
pub enum Error {
#[error("{0}")]
#[error(display = "{}", _0)]
/// Error from common error
Common(#[from] CommonError),
Common(#[error(source)] CommonError),
// Category: cannot process
/// The admin API token does not exist
#[error("Admin token not found: {0}")]
#[error(display = "Admin token not found: {}", _0)]
NoSuchAdminToken(String),
/// The API access key does not exist
#[error("Access key not found: {0}")]
#[error(display = "Access key not found: {}", _0)]
NoSuchAccessKey(String),
/// The requested block does not exist
#[error("Block not found: {0}")]
#[error(display = "Block not found: {}", _0)]
NoSuchBlock(String),
/// The requested worker does not exist
#[error("Worker not found: {0}")]
#[error(display = "Worker not found: {}", _0)]
NoSuchWorker(u64),
/// The object requested don't exists
#[error("Key not found")]
#[error(display = "Key not found")]
NoSuchKey,
/// In Import key, the key already exists
#[error("Key {0} already exists in data store. Even if it is deleted, we can't let you create a new key with the same ID. Sorry.")]
#[error(
display = "Key {} already exists in data store. Even if it is deleted, we can't let you create a new key with the same ID. Sorry.",
_0
)]
KeyAlreadyExists(String),
}
@@ -92,7 +95,6 @@ impl ApiError for Error {
fn add_http_headers(&self, header_map: &mut HeaderMap<HeaderValue>) {
use hyper::header;
header_map.append(header::CONTENT_TYPE, "application/json".parse().unwrap());
header_map.append(header::ACCESS_CONTROL_ALLOW_ORIGIN, "*".parse().unwrap());
}
fn http_body(&self, garage_region: &str, path: &str) -> ErrorBody {
+12 -13
View File
@@ -230,18 +230,9 @@ async fn key_info_results(
},
buckets: relevant_buckets
.into_values()
.filter_map(|bucket| {
.map(|bucket| {
let state = bucket.state.as_option().unwrap();
let permissions = key_state
.authorized_buckets
.get(&bucket.id)
.filter(|p| p.is_any())
.map(|p| ApiBucketKeyPerm {
read: p.allow_read,
write: p.allow_write,
owner: p.allow_owner,
})?;
Some(KeyInfoBucketResponse {
KeyInfoBucketResponse {
id: hex::encode(bucket.id),
global_aliases: state
.aliases
@@ -257,8 +248,16 @@ async fn key_info_results(
.filter(|((k, _), _, a)| *a && *k == key.key_id)
.map(|((_, n), _, _)| n.to_string())
.collect::<Vec<_>>(),
permissions,
})
permissions: key_state
.authorized_buckets
.get(&bucket.id)
.map(|p| ApiBucketKeyPerm {
read: p.allow_read,
write: p.allow_write,
owner: p.allow_owner,
})
.unwrap_or_default(),
}
})
.collect::<Vec<_>>(),
};
+5 -7
View File
@@ -74,18 +74,16 @@ macro_rules! admin_endpoints {
type Response = AdminApiResponse;
async fn handle(self, garage: &Arc<Garage>, admin: &Admin) -> Result<AdminApiResponse, Error> {
match self {
Ok(match self {
$(
AdminApiRequest::$special_endpoint(_) => Err(
Error::Common(CommonError::BadRequest(
concat!(stringify!($special_endpoint), " cannot be used outside of the HTTP Admin API").into()
))
AdminApiRequest::$special_endpoint(_) => panic!(
concat!(stringify!($special_endpoint), " needs to go through a special handler")
),
)*
$(
AdminApiRequest::$endpoint(req) => Ok(AdminApiResponse::$endpoint(req.handle(garage, admin).await?)),
AdminApiRequest::$endpoint(req) => AdminApiResponse::$endpoint(req.handle(garage, admin).await?),
)*
}
})
}
}
}
+8 -13
View File
@@ -106,17 +106,17 @@ impl RequestHandler for LocalGetNodeStatisticsRequest {
// Gather block manager statistics
writeln!(&mut ret, "\nBlock manager stats:").unwrap();
let rc_len = garage.block_manager.rc_approximate_len()?.to_string();
let rc_len = garage.block_manager.rc_len()?.to_string();
ret += &format_table_to_string(vec![
format!(" number of RC entries:\t{} (~= number of blocks)", rc_len),
format!(
" resync queue length:\t{}",
garage.block_manager.resync.queue_approximate_len()?
garage.block_manager.resync.queue_len()?
),
format!(
" blocks with resync errors:\t{}",
garage.block_manager.resync.errors_approximate_len()?
garage.block_manager.resync.errors_len()?
),
]);
@@ -129,21 +129,16 @@ where
F: TableSchema + 'static,
R: TableReplication + 'static,
{
let data_len = t
.data
.store
.approximate_len()
.map_err(GarageError::from)?
.to_string();
let mkl_len = t.merkle_updater.merkle_tree_approximate_len()?.to_string();
let data_len = t.data.store.len().map_err(GarageError::from)?.to_string();
let mkl_len = t.merkle_updater.merkle_tree_len()?.to_string();
Ok(format!(
" {}\t{}\t{}\t{}\t{}\t{}",
F::TABLE_NAME,
data_len,
mkl_len,
t.merkle_updater.todo_approximate_len()?,
t.data.insert_queue_approximate_len()?,
t.data.gc_todo_approximate_len()?
t.merkle_updater.todo_len()?,
t.data.insert_queue_len()?,
t.data.gc_todo_len()?
))
}
+62 -99
View File
@@ -1,8 +1,7 @@
#![allow(dead_code)]
#![allow(non_snake_case)]
use serde::{Deserialize, Serialize};
use utoipa::{Modify, OpenApi, ToSchema};
use utoipa::{Modify, OpenApi};
use crate::api::*;
@@ -47,7 +46,9 @@ a static website for the requested domain. This is used by reverse proxies such
as Caddy or Tricot, to avoid requesting TLS certificates for domain names that
do not correspond to an actual website.
",
params(CheckDomainRequest),
params(
("domain", description = "The domain name to check for"),
),
security(()),
responses(
(status = 200, description = "The domain name redirects to a static website bucket"),
@@ -166,7 +167,9 @@ fn CreateAdminToken() -> () {}
Updates information about the specified admin API token.
",
request_body = UpdateAdminTokenRequestBody,
params(UpdateAdminTokenRequest),
params(
("id", description = "Admin API token ID"),
),
responses(
(status = 200, description = "Admin token has been updated", body = UpdateAdminTokenResponse),
(status = 500, description = "Internal server error")
@@ -178,7 +181,9 @@ fn UpdateAdminToken() -> () {}
path = "/v2/DeleteAdminToken",
tag = "Admin API token",
description = "Delete an admin API token from the cluster, revoking all its permissions.",
params(DeleteAdminTokenRequest),
params(
("id", description = "Admin API token ID"),
),
responses(
(status = 200, description = "Admin token has been deleted"),
(status = 500, description = "Internal server error")
@@ -186,19 +191,6 @@ fn UpdateAdminToken() -> () {}
)]
fn DeleteAdminToken() -> () {}
#[utoipa::path(get,
path = "/v2/GetCurrentAdminTokenInfo",
tag = "Admin API token",
description = "
Return information about the calling admin API token.
",
responses(
(status = 200, description = "Information about the admin token", body = GetCurrentAdminTokenInfoResponse),
(status = 500, description = "Internal server error")
),
)]
fn GetCurrentAdminTokenInfo() -> () {}
// **********************************************
// Layout operations
// **********************************************
@@ -247,7 +239,7 @@ For example to declare 100GB, you must set `capacity: 100000000000`.
Garage uses internally the International System of Units (SI), it assumes that 1kB = 1000 bytes, and displays storage as kB, MB, GB (and not KiB, MiB, GiB that assume 1KiB = 1024 bytes).
",
request_body(
content=UpdateClusterLayoutRequestOpenapi,
content=UpdateClusterLayoutRequest,
description="
To add a new node to the layout or to change the configuration of an existing node, simply set the values you want (`zone`, `capacity`, and `tags`).
To remove a node, simply pass the `remove: true` field.
@@ -263,48 +255,6 @@ Contrary to the CLI that may update only a subset of the fields capacity, zone a
)]
fn UpdateClusterLayout() -> () {}
// Hack: we cannot use the UpdateClusterLayoutRequest from api.rs,
// as it contains (via NodeRoleChange) an untagged enum flattenned into
// a struct, which breaks the openapi generator.
// See issue #1249.
// Instead, we use a rewritten version of the NodeRoleChange struct where
// the struct fields are distributed into the enum variants (this is an equivalent
// representation, but this way we avoid having to rewrite all uses of the original
// struct in the Garage codebase).
#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)]
#[schema(as = UpdateClusterLayoutRequest)]
pub struct UpdateClusterLayoutRequestOpenapi {
/// New node roles to assign or remove in the cluster layout
#[serde(default)]
pub roles: Vec<NodeRoleChangeOpenapi>,
/// New layout computation parameters to use
#[serde(default)]
pub parameters: Option<LayoutParameters>,
}
#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)]
#[schema(as = NodeRoleChangeRequest)]
#[serde(untagged)]
pub enum NodeRoleChangeOpenapi {
#[serde(rename_all = "camelCase")]
Remove {
/// ID of the node for which this change applies
id: String,
/// Set `remove` to `true` to remove the node from the layout
remove: bool,
},
Update(NodeRoleUpdate),
}
#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)]
#[serde(rename_all = "camelCase")]
pub struct NodeRoleUpdate {
/// ID of the node for which this change applies
id: String,
#[serde(flatten)]
role: NodeAssignedRole,
}
#[utoipa::path(post,
path = "/v2/PreviewClusterLayoutChanges",
tag = "Cluster layout",
@@ -428,7 +378,9 @@ Updates information about the specified API access key.
*Note: the secret key is not returned in the response, `null` is sent instead.*
",
request_body = UpdateKeyRequestBody,
params(UpdateKeyRequest),
params(
("id", description = "Access key ID"),
),
responses(
(status = 200, description = "Access key has been updated", body = UpdateKeyResponse),
(status = 500, description = "Internal server error")
@@ -440,7 +392,9 @@ fn UpdateKey() -> () {}
path = "/v2/DeleteKey",
tag = "Access key",
description = "Delete a key from the cluster. Its access will be removed from all the buckets. Buckets are not automatically deleted and can be dangling. You should manually delete them before. ",
params(DeleteKeyRequest),
params(
("id", description = "Access key ID"),
),
responses(
(status = 200, description = "Access key has been deleted"),
(status = 500, description = "Internal server error")
@@ -511,7 +465,9 @@ In `quotas`: new values of `maxSize` and `maxObjects` must both be specified, or
to remove the quotas. An absent value will be considered the same as a `null`. It is not possible
to change only one of the two quotas.
",
params(UpdateBucketRequest),
params(
("id", description = "ID of the bucket to update"),
),
request_body = UpdateBucketRequestBody,
responses(
(status = 200, description = "Bucket has been updated", body = UpdateBucketResponse),
@@ -529,7 +485,9 @@ Deletes a storage bucket. A bucket cannot be deleted if it is not empty.
**Warning:** this will delete all aliases associated with the bucket!
",
params(DeleteBucketRequest),
params(
("id", description = "ID of the bucket to delete"),
),
responses(
(status = 200, description = "Bucket has been deleted"),
(status = 400, description = "Bucket is not empty"),
@@ -629,7 +587,7 @@ fn DenyBucketKey() -> () {}
path = "/v2/AddBucketAlias",
tag = "Bucket alias",
description = "Add an alias for the target bucket. This can be either a global or a local alias, depending on which fields are specified.",
request_body = BucketAliasEnumOpenapi,
request_body = AddBucketAliasRequest,
responses(
(status = 200, description = "Returns exhaustive information about the bucket", body = AddBucketAliasResponse),
(status = 500, description = "Internal server error")
@@ -641,7 +599,7 @@ fn AddBucketAlias() -> () {}
path = "/v2/RemoveBucketAlias",
tag = "Bucket alias",
description = "Remove an alias for the target bucket. This can be either a global or a local alias, depending on which fields are specified.",
request_body = BucketAliasEnumOpenapi,
request_body = RemoveBucketAliasRequest,
responses(
(status = 200, description = "Returns exhaustive information about the bucket", body = RemoveBucketAliasResponse),
(status = 500, description = "Internal server error")
@@ -649,24 +607,6 @@ fn AddBucketAlias() -> () {}
)]
fn RemoveBucketAlias() -> () {}
// Hack for issue #1249 (see UpdateClusterLayout)
#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)]
#[serde(untagged)]
#[schema(as = BucketAliasEnum)]
pub enum BucketAliasEnumOpenapi {
#[serde(rename_all = "camelCase")]
Global {
bucket_id: String,
global_alias: String,
},
#[serde(rename_all = "camelCase")]
Local {
bucket_id: String,
local_alias: String,
access_key_id: String,
},
}
// **********************************************
// Node operations
// **********************************************
@@ -677,7 +617,9 @@ pub enum BucketAliasEnumOpenapi {
description = "
Return information about the Garage daemon running on one or several nodes.
",
params(MultiRequestQueryParams),
params(
("node", description = "Node ID to query, or `*` for all nodes, or `self` for the node responding to the request"),
),
responses(
(status = 200, description = "Responses from individual cluster nodes", body = MultiResponse<LocalGetNodeInfoResponse>),
(status = 500, description = "Internal server error")
@@ -693,7 +635,9 @@ Fetch statistics for one or several Garage nodes.
*Note: do not try to parse the `freeform` field of the response, it is given as a string specifically because its format is not stable.*
",
params(MultiRequestQueryParams),
params(
("node", description = "Node ID to query, or `*` for all nodes, or `self` for the node responding to the request"),
),
responses(
(status = 200, description = "Responses from individual cluster nodes", body = MultiResponse<LocalGetNodeStatisticsResponse>),
(status = 500, description = "Internal server error")
@@ -707,7 +651,9 @@ fn GetNodeStatistics() -> () {}
description = "
Instruct one or several nodes to take a snapshot of their metadata databases.
",
params(MultiRequestQueryParams),
params(
("node", description = "Node ID to query, or `*` for all nodes, or `self` for the node responding to the request"),
),
responses(
(status = 200, description = "Responses from individual cluster nodes", body = MultiResponse<LocalCreateMetadataSnapshotResponse>),
(status = 500, description = "Internal server error")
@@ -721,7 +667,9 @@ fn CreateMetadataSnapshot() -> () {}
description = "
Launch a repair operation on one or several cluster nodes.
",
params(MultiRequestQueryParams),
params(
("node", description = "Node ID to query, or `*` for all nodes, or `self` for the node responding to the request"),
),
request_body = LocalLaunchRepairOperationRequest,
responses(
(status = 200, description = "Responses from individual cluster nodes", body = MultiResponse<LocalLaunchRepairOperationResponse>),
@@ -740,7 +688,9 @@ fn LaunchRepairOperation() -> () {}
description = "
List background workers currently running on one or several cluster nodes.
",
params(MultiRequestQueryParams),
params(
("node", description = "Node ID to query, or `*` for all nodes, or `self` for the node responding to the request"),
),
request_body = LocalListWorkersRequest,
responses(
(status = 200, description = "Responses from individual cluster nodes", body = MultiResponse<LocalListWorkersResponse>),
@@ -755,7 +705,9 @@ fn ListWorkers() -> () {}
description = "
Get information about the specified background worker on one or several cluster nodes.
",
params(MultiRequestQueryParams),
params(
("node", description = "Node ID to query, or `*` for all nodes, or `self` for the node responding to the request"),
),
request_body = LocalGetWorkerInfoRequest,
responses(
(status = 200, description = "Responses from individual cluster nodes", body = MultiResponse<LocalGetWorkerInfoResponse>),
@@ -770,7 +722,9 @@ fn GetWorkerInfo() -> () {}
description = "
Fetch values of one or several worker variables, from one or several cluster nodes.
",
params(MultiRequestQueryParams),
params(
("node", description = "Node ID to query, or `*` for all nodes, or `self` for the node responding to the request"),
),
request_body = LocalGetWorkerVariableRequest,
responses(
(status = 200, description = "Responses from individual cluster nodes", body = MultiResponse<LocalGetWorkerVariableResponse>),
@@ -785,7 +739,9 @@ fn GetWorkerVariable() -> () {}
description = "
Set the value for a worker variable, on one or several cluster nodes.
",
params(MultiRequestQueryParams),
params(
("node", description = "Node ID to query, or `*` for all nodes, or `self` for the node responding to the request"),
),
request_body = LocalSetWorkerVariableRequest,
responses(
(status = 200, description = "Responses from individual cluster nodes", body = MultiResponse<LocalSetWorkerVariableResponse>),
@@ -804,7 +760,9 @@ fn SetWorkerVariable() -> () {}
description = "
List data blocks that are currently in an errored state on one or several Garage nodes.
",
params(MultiRequestQueryParams),
params(
("node", description = "Node ID to query, or `*` for all nodes, or `self` for the node responding to the request"),
),
responses(
(status = 200, description = "Responses from individual cluster nodes", body = MultiResponse<LocalListBlockErrorsResponse>),
(status = 500, description = "Internal server error")
@@ -818,7 +776,9 @@ fn ListBlockErrors() -> () {}
description = "
Get detailed information about a data block stored on a Garage node, including all object versions and in-progress multipart uploads that contain a reference to this block.
",
params(MultiRequestQueryParams),
params(
("node", description = "Node ID to query, or `*` for all nodes, or `self` for the node responding to the request"),
),
request_body = LocalGetBlockInfoRequest,
responses(
(status = 200, description = "Detailed block information", body = MultiResponse<LocalGetBlockInfoResponse>),
@@ -833,7 +793,9 @@ fn GetBlockInfo() -> () {}
description = "
Instruct Garage node(s) to retry the resynchronization of one or several missing data block(s).
",
params(MultiRequestQueryParams),
params(
("node", description = "Node ID to query, or `*` for all nodes, or `self` for the node responding to the request"),
),
request_body = LocalRetryBlockResyncRequest,
responses(
(status = 200, description = "Responses from individual cluster nodes", body = MultiResponse<LocalRetryBlockResyncResponse>),
@@ -850,7 +812,9 @@ Purge references to one or several missing data blocks.
This will remove all objects and in-progress multipart uploads that contain the specified data block(s). The objects will be permanently deleted from the buckets in which they appear. Use with caution.
",
params(MultiRequestQueryParams),
params(
("node", description = "Node ID to query, or `*` for all nodes, or `self` for the node responding to the request"),
),
request_body = LocalPurgeBlocksRequest,
responses(
(status = 200, description = "Responses from individual cluster nodes", body = MultiResponse<LocalPurgeBlocksResponse>),
@@ -879,7 +843,7 @@ impl Modify for SecurityAddon {
#[derive(OpenApi)]
#[openapi(
info(
version = "v2.2.0",
version = "v2.0.0",
title = "Garage administration API",
description = "Administrate your Garage cluster programatically, including status, layout, keys, buckets, and maintainance tasks.
@@ -908,7 +872,6 @@ impl Modify for SecurityAddon {
CreateAdminToken,
UpdateAdminToken,
DeleteAdminToken,
GetCurrentAdminTokenInfo,
// Layout operations
GetClusterLayout,
GetClusterLayoutHistory,
-13
View File
@@ -89,19 +89,6 @@ impl RequestHandler for LocalLaunchRepairOperationRequest {
garage.block_manager.clone(),
));
}
RepairType::Aliases => {
info!("Repairing bucket aliases (foreground)");
garage.locked_helper().await.repair_aliases().await?;
}
RepairType::ClearResyncQueue => {
info!("Clearing resync queue (foreground)");
let garage = garage.clone();
tokio::task::spawn_blocking(move || {
garage.block_manager.resync.clear_resync_queue()
})
.await
.map_err(garage_util::error::Error::from)??;
}
}
Ok(LocalLaunchRepairOperationResponse)
}
-1
View File
@@ -40,7 +40,6 @@ impl AdminApiRequest {
POST CreateAdminToken (body),
POST UpdateAdminToken (body_field, query::id),
POST DeleteAdminToken (query::id),
GET GetCurrentAdminTokenInfo (admin_token),
// Layout endpoints
GET GetClusterLayout (),
GET GetClusterLayoutHistory (),
+6 -3
View File
@@ -1,6 +1,6 @@
[package]
name = "garage_api_common"
version = "2.2.0"
version = "2.0.0"
authors = ["Alex Auvolat <alex@adnab.me>"]
edition = "2018"
license = "AGPL-3.0"
@@ -21,12 +21,15 @@ garage_util.workspace = true
base64.workspace = true
bytes.workspace = true
chrono.workspace = true
crc-fast.workspace = true
crc32fast.workspace = true
crc32c.workspace = true
crc64fast-nvme.workspace = true
crypto-common.workspace = true
thiserror.workspace = true
err-derive.workspace = true
hex.workspace = true
hmac.workspace = true
md-5.workspace = true
idna.workspace = true
tracing.workspace = true
nom.workspace = true
pin-project.workspace = true
+15 -15
View File
@@ -1,7 +1,7 @@
use std::convert::TryFrom;
use err_derive::Error;
use hyper::StatusCode;
use thiserror::Error;
use garage_util::error::Error as GarageError;
@@ -12,48 +12,48 @@ use garage_model::helper::error::Error as HelperError;
pub enum CommonError {
// ---- INTERNAL ERRORS ----
/// Error related to deeper parts of Garage
#[error("Internal error: {0}")]
InternalError(#[from] GarageError),
#[error(display = "Internal error: {}", _0)]
InternalError(#[error(source)] GarageError),
/// Error related to Hyper
#[error("Internal error (Hyper error): {0}")]
Hyper(#[from] hyper::Error),
#[error(display = "Internal error (Hyper error): {}", _0)]
Hyper(#[error(source)] hyper::Error),
/// Error related to HTTP
#[error("Internal error (HTTP error): {0}")]
Http(#[from] http::Error),
#[error(display = "Internal error (HTTP error): {}", _0)]
Http(#[error(source)] http::Error),
// ---- GENERIC CLIENT ERRORS ----
/// Proper authentication was not provided
#[error("Forbidden: {0}")]
#[error(display = "Forbidden: {}", _0)]
Forbidden(String),
/// Generic bad request response with custom message
#[error("Bad request: {0}")]
#[error(display = "Bad request: {}", _0)]
BadRequest(String),
/// The client sent a header with invalid value
#[error("Invalid header value: {0}")]
InvalidHeader(#[from] hyper::header::ToStrError),
#[error(display = "Invalid header value: {}", _0)]
InvalidHeader(#[error(source)] hyper::header::ToStrError),
// ---- SPECIFIC ERROR CONDITIONS ----
// These have to be error codes referenced in the S3 spec here:
// https://docs.aws.amazon.com/AmazonS3/latest/API/ErrorResponses.html#ErrorCodeList
/// The bucket requested don't exists
#[error("Bucket not found: {0}")]
#[error(display = "Bucket not found: {}", _0)]
NoSuchBucket(String),
/// Tried to create a bucket that already exist
#[error("Bucket already exists")]
#[error(display = "Bucket already exists")]
BucketAlreadyExists,
/// Tried to delete a non-empty bucket
#[error("Tried to delete a non-empty bucket")]
#[error(display = "Tried to delete a non-empty bucket")]
BucketNotEmpty,
// Category: bad request
/// Bucket name is not valid according to AWS S3 specs
#[error("Invalid bucket name: {0}")]
#[error(display = "Invalid bucket name: {}", _0)]
InvalidBucketName(String),
}
+11 -22
View File
@@ -59,12 +59,6 @@ pub trait ApiHandler: Send + Sync + 'static {
req: Request<IncomingBody>,
endpoint: Self::Endpoint,
) -> impl Future<Output = Result<Response<BoxBody<Self::Error>>, Self::Error>> + Send;
/// Returns the key id used to authenticate this request. The ID returned must be safe to
/// log.
fn key_id_from_request(&self, _req: &Request<IncomingBody>) -> Option<String> {
None
}
}
pub struct ApiServer<A: ApiHandler> {
@@ -149,20 +143,19 @@ impl<A: ApiHandler> ApiServer<A> {
) -> Result<Response<BoxBody<A::Error>>, http::Error> {
let uri = req.uri().clone();
let source = if let Ok(forwarded_for_ip_addr) =
if let Ok(forwarded_for_ip_addr) =
forwarded_headers::handle_forwarded_for_headers(req.headers())
{
format!("{forwarded_for_ip_addr} (via {addr})")
info!(
"{} (via {}) {} {}",
forwarded_for_ip_addr,
addr,
req.method(),
uri
);
} else {
format!("{addr}")
};
// we only do this to log the access key, so we can discard any error
let key = self
.api_handler
.key_id_from_request(&req)
.map(|k| format!("(key {k}) "))
.unwrap_or_default();
info!("{source} {key}{} {uri}", req.method());
info!("{} {} {}", addr, req.method(), uri);
}
debug!("{:?}", req);
let tracer = opentelemetry::global::tracer("garage");
@@ -351,11 +344,7 @@ where
while !*must_exit.borrow() {
let (stream, client_addr) = tokio::select! {
acc = listener.accept() => match acc {
Ok(r) => r,
Err(e) if e.kind() == std::io::ErrorKind::ConnectionAborted => continue,
Err(e) => return Err(e.into()),
},
acc = listener.accept() => acc?,
_ = must_exit.changed() => continue,
};
+2 -1
View File
@@ -8,6 +8,7 @@ use hyper::{
body::{Body, Bytes},
Request, Response,
};
use idna::domain_to_unicode;
use serde::{Deserialize, Serialize};
use garage_model::bucket_table::BucketParams;
@@ -96,7 +97,7 @@ pub fn authority_to_host(authority: &str) -> Result<String, Error> {
authority
))),
};
authority.map(|h| h.to_ascii_lowercase())
authority.map(|h| domain_to_unicode(h).0)
}
/// Extract the bucket name and the key name from an HTTP path and possibly a bucket provided in
-15
View File
@@ -83,21 +83,6 @@ macro_rules! router_match {
parse_json_body::< [<$api Request>], _, Error>($req).await?
})
}};
(@@gen_parse_request $api:ident, (admin_token), $query: expr, $req:expr) => {{
paste!({
let auth_header = $req.headers()
.get(hyper::header::AUTHORIZATION)
.ok_or_else(|| Error::bad_request("Missing Authorization header"))?
.to_str()
.map_err(|_| Error::bad_request("Invalid Authorization header"))?;
let admin_token = auth_header.strip_prefix("Bearer ")
.ok_or_else(|| Error::bad_request("Authorization header must be Bearer token"))?
.to_string();
[< $api Request >] { admin_token }
})
}};
(@@gen_parse_request $api:ident, (body_field, $($conv:ident $(($conv_arg:expr))? :: $param:ident),*), $query: expr, $req:expr)
=>
{{
+24 -40
View File
@@ -1,7 +1,10 @@
use std::convert::TryInto;
use std::convert::{TryFrom, TryInto};
use std::hash::Hasher;
use base64::prelude::*;
use crc_fast::{CrcAlgorithm, Digest as CrcDigest};
use crc32c::Crc32cHasher as Crc32c;
use crc32fast::Hasher as Crc32;
use crc64fast_nvme::Digest as Crc64Nvme;
use md5::{Digest, Md5};
use sha1::Sha1;
use sha2::Sha256;
@@ -19,7 +22,6 @@ pub const CONTENT_MD5: HeaderName = HeaderName::from_static("content-md5");
pub const X_AMZ_CHECKSUM_ALGORITHM: HeaderName =
HeaderName::from_static("x-amz-checksum-algorithm");
pub const X_AMZ_CHECKSUM_MODE: HeaderName = HeaderName::from_static("x-amz-checksum-mode");
pub const X_AMZ_CHECKSUM_TYPE: HeaderName = HeaderName::from_static("x-amz-checksum-type");
pub const X_AMZ_CHECKSUM_CRC32: HeaderName = HeaderName::from_static("x-amz-checksum-crc32");
pub const X_AMZ_CHECKSUM_CRC32C: HeaderName = HeaderName::from_static("x-amz-checksum-crc32c");
pub const X_AMZ_CHECKSUM_CRC64NVME: HeaderName =
@@ -27,10 +29,6 @@ pub const X_AMZ_CHECKSUM_CRC64NVME: HeaderName =
pub const X_AMZ_CHECKSUM_SHA1: HeaderName = HeaderName::from_static("x-amz-checksum-sha1");
pub const X_AMZ_CHECKSUM_SHA256: HeaderName = HeaderName::from_static("x-amz-checksum-sha256");
// Values for x-amz-checksum-type
pub const COMPOSITE: &str = "COMPOSITE";
pub const FULL_OBJECT: &str = "FULL_OBJECT";
pub type Crc32Checksum = [u8; 4];
pub type Crc32cChecksum = [u8; 4];
pub type Crc64NvmeChecksum = [u8; 8];
@@ -38,21 +36,6 @@ pub type Md5Checksum = [u8; 16];
pub type Sha1Checksum = [u8; 20];
pub type Sha256Checksum = [u8; 32];
// -- MAP OF CRC ALGORITHMS :
// CRC32 -> CrcAlgorithm::Crc32IsoHdlc
// CRC32C -> CrcAlgorithm::Crc32Iscsi
// CRC64NVME -> CrcAlgorithm::Crc64Nvme
pub fn new_crc32() -> CrcDigest {
CrcDigest::new(CrcAlgorithm::Crc32IsoHdlc)
}
pub fn new_crc32c() -> CrcDigest {
CrcDigest::new(CrcAlgorithm::Crc32Iscsi)
}
pub fn new_crc64nvme() -> CrcDigest {
CrcDigest::new(CrcAlgorithm::Crc64Nvme)
}
#[derive(Debug, Default, Clone)]
pub struct ExpectedChecksums {
// base64-encoded md5 (content-md5 header)
@@ -64,9 +47,9 @@ pub struct ExpectedChecksums {
}
pub struct Checksummer {
pub crc32: Option<CrcDigest>,
pub crc32c: Option<CrcDigest>,
pub crc64nvme: Option<CrcDigest>,
pub crc32: Option<Crc32>,
pub crc32c: Option<Crc32c>,
pub crc64nvme: Option<Crc64Nvme>,
pub md5: Option<Md5>,
pub sha1: Option<Sha1>,
pub sha256: Option<Sha256>,
@@ -115,13 +98,13 @@ impl Checksummer {
self.sha256 = Some(Sha256::new());
}
if matches!(&expected.extra, Some(ChecksumValue::Crc32(_))) {
self.crc32 = Some(new_crc32());
self.crc32 = Some(Crc32::new());
}
if matches!(&expected.extra, Some(ChecksumValue::Crc32c(_))) {
self.crc32c = Some(new_crc32c());
self.crc32c = Some(Crc32c::default());
}
if matches!(&expected.extra, Some(ChecksumValue::Crc64Nvme(_))) {
self.crc64nvme = Some(new_crc64nvme());
self.crc64nvme = Some(Crc64Nvme::default());
}
if matches!(&expected.extra, Some(ChecksumValue::Sha1(_))) {
self.sha1 = Some(Sha1::new());
@@ -131,13 +114,13 @@ impl Checksummer {
pub fn add(mut self, algo: Option<ChecksumAlgorithm>) -> Self {
match algo {
Some(ChecksumAlgorithm::Crc32) => {
self.crc32 = Some(new_crc32());
self.crc32 = Some(Crc32::new());
}
Some(ChecksumAlgorithm::Crc32c) => {
self.crc32c = Some(new_crc32c());
self.crc32c = Some(Crc32c::default());
}
Some(ChecksumAlgorithm::Crc64Nvme) => {
self.crc64nvme = Some(new_crc64nvme());
self.crc64nvme = Some(Crc64Nvme::default());
}
Some(ChecksumAlgorithm::Sha1) => {
self.sha1 = Some(Sha1::new());
@@ -155,10 +138,10 @@ impl Checksummer {
crc32.update(bytes);
}
if let Some(crc32c) = &mut self.crc32c {
crc32c.update(bytes);
crc32c.write(bytes);
}
if let Some(crc64nvme) = &mut self.crc64nvme {
crc64nvme.update(bytes);
crc64nvme.write(bytes);
}
if let Some(md5) = &mut self.md5 {
md5.update(bytes);
@@ -173,9 +156,11 @@ impl Checksummer {
pub fn finalize(self) -> Checksums {
Checksums {
crc32: self.crc32.map(|x| u32::to_be_bytes(x.finalize() as u32)),
crc32c: self.crc32c.map(|x| u32::to_be_bytes(x.finalize() as u32)),
crc64nvme: self.crc64nvme.map(|x| u64::to_be_bytes(x.finalize())),
crc32: self.crc32.map(|x| u32::to_be_bytes(x.finalize())),
crc32c: self
.crc32c
.map(|x| u32::to_be_bytes(u32::try_from(x.finish()).unwrap())),
crc64nvme: self.crc64nvme.map(|x| u64::to_be_bytes(x.sum64())),
md5: self.md5.map(|x| x.finalize()[..].try_into().unwrap()),
sha1: self.sha1.map(|x| x.finalize()[..].try_into().unwrap()),
sha256: self.sha256.map(|x| x.finalize()[..].try_into().unwrap()),
@@ -207,11 +192,10 @@ impl Checksums {
}
if let Some(extra) = expected.extra {
let algo = extra.algorithm();
let calculated = self.extract(Some(algo));
if calculated != Some(extra) {
if self.extract(Some(algo)) != Some(extra) {
return Err(Error::InvalidDigest(format!(
"Failed to validate checksum for algorithm {:?}: calculated {:?}, expected {:?}",
algo, calculated, extra
"Failed to validate checksum for algorithm {:?}",
algo
)));
}
}
+6 -6
View File
@@ -1,4 +1,4 @@
use thiserror::Error;
use err_derive::Error;
use crate::common_error::CommonError;
pub use crate::common_error::{CommonErrorDerivative, OkOrBadRequest, OkOrInternalError};
@@ -6,21 +6,21 @@ pub use crate::common_error::{CommonErrorDerivative, OkOrBadRequest, OkOrInterna
/// Errors of this crate
#[derive(Debug, Error)]
pub enum Error {
#[error("{0}")]
#[error(display = "{}", _0)]
/// Error from common error
Common(CommonError),
/// Authorization Header Malformed
#[error("Authorization header malformed, unexpected scope: {0}")]
#[error(display = "Authorization header malformed, unexpected scope: {}", _0)]
AuthorizationHeaderMalformed(String),
// Category: bad request
/// The request contained an invalid UTF-8 sequence in its path or in other parameters
#[error("Invalid UTF-8: {0}")]
InvalidUtf8Str(#[from] std::str::Utf8Error),
#[error(display = "Invalid UTF-8: {}", _0)]
InvalidUtf8Str(#[error(source)] std::str::Utf8Error),
/// The provided digest (checksum) value was invalid
#[error("Invalid digest: {0}")]
#[error(display = "Invalid digest: {}", _0)]
InvalidDigest(String),
}
+7 -8
View File
@@ -105,7 +105,7 @@ fn check_standard_signature(
// Verify that all necessary request headers are included in signed_headers
// The following must be included for all signatures:
// - the Host header (mandatory)
// - all x-amz-* headers used in the request (except x-amz-content-sha256)
// - all x-amz-* headers used in the request
// AWS also indicates that the Content-Type header should be signed if
// it is used, but Minio client doesn't sign it so we don't check it for compatibility.
let signed_headers = split_signed_headers(&authorization)?;
@@ -152,7 +152,7 @@ fn check_presigned_signature(
// Verify that all necessary request headers are included in signed_headers
// For AWSv4 pre-signed URLs, the following must be included:
// - the Host header (mandatory)
// - all x-amz-* headers used in the request (except x-amz-content-sha256)
// - all x-amz-* headers used in the request
let signed_headers = split_signed_headers(&authorization)?;
verify_signed_headers(request.headers(), &signed_headers)?;
@@ -269,9 +269,7 @@ fn verify_signed_headers(headers: &HeaderMap, signed_headers: &[HeaderName]) ->
return Err(Error::bad_request("Header `Host` should be signed"));
}
for (name, _) in headers.iter() {
// Enforce signature of all x-amz-* headers, except x-amz-content-sh256
// because it is included in the canonical request in all cases
if name.as_str().starts_with("x-amz-") && name != X_AMZ_CONTENT_SHA256 {
if name.as_str().starts_with("x-amz-") {
if !signed_headers.contains(name) {
return Err(Error::bad_request(format!(
"Header `{}` should be signed",
@@ -426,7 +424,7 @@ pub fn verify_v4(
// ============ Authorization header, or X-Amz-* query params =========
pub struct Authorization {
pub key_id: String,
key_id: String,
scope: String,
signed_headers: String,
signature: String,
@@ -435,7 +433,7 @@ pub struct Authorization {
}
impl Authorization {
pub fn parse_header(headers: &HeaderMap) -> Result<Self, Error> {
fn parse_header(headers: &HeaderMap) -> Result<Self, Error> {
let authorization = headers
.get(AUTHORIZATION)
.ok_or_bad_request("Missing authorization header")?
@@ -477,7 +475,8 @@ impl Authorization {
let date = headers
.get(X_AMZ_DATE)
.ok_or_bad_request("Missing X-Amz-Date field")?
.ok_or_bad_request("Missing X-Amz-Date field")
.map_err(Error::from)?
.to_str()?;
let date = parse_date(date)?;
+2 -2
View File
@@ -1,6 +1,6 @@
[package]
name = "garage_api_k2v"
version = "2.2.0"
version = "2.0.0"
authors = ["Alex Auvolat <alex@adnab.me>"]
edition = "2018"
license = "AGPL-3.0"
@@ -20,7 +20,7 @@ garage_util = { workspace = true, features = [ "k2v" ] }
garage_api_common.workspace = true
base64.workspace = true
thiserror.workspace = true
err-derive.workspace = true
tracing.workspace = true
futures.workspace = true
-6
View File
@@ -171,12 +171,6 @@ impl ApiHandler for K2VApiServer {
Ok(resp_ok)
}
fn key_id_from_request(&self, req: &Request<IncomingBody>) -> Option<String> {
garage_api_common::signature::payload::Authorization::parse_header(req.headers())
.map(|auth| auth.key_id)
.ok()
}
}
impl ApiEndpoint for K2VApiEndpoint {
+12 -13
View File
@@ -1,6 +1,6 @@
use err_derive::Error;
use hyper::header::HeaderValue;
use hyper::{HeaderMap, StatusCode};
use thiserror::Error;
pub(crate) use garage_api_common::common_error::pass_helper_error;
use garage_api_common::common_error::{commonErrorDerivative, CommonError};
@@ -14,38 +14,38 @@ use garage_api_common::signature::error::Error as SignatureError;
/// Errors of this crate
#[derive(Debug, Error)]
pub enum Error {
#[error("{0}")]
#[error(display = "{}", _0)]
/// Error from common error
Common(#[from] CommonError),
Common(#[error(source)] CommonError),
// Category: cannot process
/// Authorization Header Malformed
#[error("Authorization header malformed, unexpected scope: {0}")]
#[error(display = "Authorization header malformed, unexpected scope: {}", _0)]
AuthorizationHeaderMalformed(String),
/// The provided digest (checksum) value was invalid
#[error("Invalid digest: {0}")]
#[error(display = "Invalid digest: {}", _0)]
InvalidDigest(String),
/// The object requested don't exists
#[error("Key not found")]
#[error(display = "Key not found")]
NoSuchKey,
/// Some base64 encoded data was badly encoded
#[error("Invalid base64: {0}")]
InvalidBase64(#[from] base64::DecodeError),
#[error(display = "Invalid base64: {}", _0)]
InvalidBase64(#[error(source)] base64::DecodeError),
/// Invalid causality token
#[error("Invalid causality token")]
#[error(display = "Invalid causality token")]
InvalidCausalityToken,
/// The client asked for an invalid return format (invalid Accept header)
#[error("Not acceptable: {0}")]
#[error(display = "Not acceptable: {}", _0)]
NotAcceptable(String),
/// The request contained an invalid UTF-8 sequence in its path or in other parameters
#[error("Invalid UTF-8: {0}")]
InvalidUtf8Str(#[from] std::str::Utf8Error),
#[error(display = "Invalid UTF-8: {}", _0)]
InvalidUtf8Str(#[error(source)] std::str::Utf8Error),
}
commonErrorDerivative!(Error);
@@ -99,7 +99,6 @@ impl ApiError for Error {
fn add_http_headers(&self, header_map: &mut HeaderMap<HeaderValue>) {
use hyper::header;
header_map.append(header::CONTENT_TYPE, "application/json".parse().unwrap());
header_map.append(header::ACCESS_CONTROL_ALLOW_ORIGIN, "*".parse().unwrap());
}
fn http_body(&self, garage_region: &str, path: &str) -> ErrorBody {
+2 -2
View File
@@ -28,7 +28,7 @@ pub async fn handle_read_index(
let node_id_vec = garage
.system
.cluster_layout()
.all_nongateway_nodes()?
.all_nongateway_nodes()
.to_vec();
let (partition_keys, more, next_start) = read_range(
@@ -66,7 +66,7 @@ pub async fn handle_read_index(
bytes: *vals.get(&s_bytes).unwrap_or(&0),
}
})
.collect(),
.collect::<Vec<_>>(),
more,
next_start,
};
+5 -3
View File
@@ -1,6 +1,6 @@
[package]
name = "garage_api_s3"
version = "2.2.0"
version = "2.0.0"
authors = ["Alex Auvolat <alex@adnab.me>"]
edition = "2018"
license = "AGPL-3.0"
@@ -27,8 +27,10 @@ async-compression.workspace = true
base64.workspace = true
bytes.workspace = true
chrono.workspace = true
crc-fast.workspace = true
thiserror.workspace = true
crc32fast.workspace = true
crc32c.workspace = true
crc64fast-nvme.workspace = true
err-derive.workspace = true
hex.workspace = true
hmac.workspace = true
tracing.workspace = true
-7
View File
@@ -223,7 +223,6 @@ impl ApiHandler for S3ApiServer {
Endpoint::DeleteBucket {} => handle_delete_bucket(ctx).await,
Endpoint::GetBucketLocation {} => handle_get_bucket_location(ctx),
Endpoint::GetBucketVersioning {} => handle_get_bucket_versioning(),
Endpoint::GetBucketAcl {} => handle_get_bucket_acl(ctx),
Endpoint::ListObjects {
delimiter,
encoding_type,
@@ -340,12 +339,6 @@ impl ApiHandler for S3ApiServer {
Ok(resp_ok)
}
fn key_id_from_request(&self, req: &Request<IncomingBody>) -> Option<String> {
garage_api_common::signature::payload::Authorization::parse_header(req.headers())
.map(|auth| auth.key_id)
.ok()
}
}
impl ApiEndpoint for S3ApiEndpoint {
+4 -62
View File
@@ -5,7 +5,7 @@ use hyper::{Request, Response, StatusCode};
use garage_model::bucket_alias_table::*;
use garage_model::bucket_table::Bucket;
use garage_model::garage::Garage;
use garage_model::key_table::{Key, KeyParams};
use garage_model::key_table::Key;
use garage_model::permission::BucketKeyPerm;
use garage_table::util::*;
use garage_util::crdt::*;
@@ -44,55 +44,6 @@ pub fn handle_get_bucket_versioning() -> Result<Response<ResBody>, Error> {
.body(string_body(xml))?)
}
pub fn handle_get_bucket_acl(ctx: ReqCtx) -> Result<Response<ResBody>, Error> {
let ReqCtx {
bucket_id, api_key, ..
} = ctx;
let key_p = api_key.params().ok_or_internal_error(
"Key should not be in deleted state at this point (in handle_get_bucket_acl)",
)?;
let mut grants: Vec<s3_xml::Grant> = vec![];
let kp = api_key.bucket_permissions(&bucket_id);
if kp.allow_owner {
grants.push(s3_xml::Grant {
grantee: create_grantee(&key_p, &api_key),
permission: s3_xml::Value("FULL_CONTROL".to_string()),
});
} else {
if kp.allow_read {
grants.push(s3_xml::Grant {
grantee: create_grantee(&key_p, &api_key),
permission: s3_xml::Value("READ".to_string()),
});
grants.push(s3_xml::Grant {
grantee: create_grantee(&key_p, &api_key),
permission: s3_xml::Value("READ_ACP".to_string()),
});
}
if kp.allow_write {
grants.push(s3_xml::Grant {
grantee: create_grantee(&key_p, &api_key),
permission: s3_xml::Value("WRITE".to_string()),
});
}
}
let access_control_policy = s3_xml::AccessControlPolicy {
xmlns: (),
owner: None,
acl: s3_xml::AccessControlList { entries: grants },
};
let xml = s3_xml::to_xml_with_header(&access_control_policy)?;
trace!("xml: {}", xml);
Ok(Response::builder()
.header("Content-Type", "application/xml")
.body(string_body(xml))?)
}
pub async fn handle_list_buckets(
garage: &Garage,
api_key: &Key,
@@ -216,7 +167,7 @@ pub async fn handle_create_bucket(
}
// Create the bucket!
if !is_valid_bucket_name(&bucket_name, garage.config.allow_punycode) {
if !is_valid_bucket_name(&bucket_name) {
return Err(Error::bad_request(format!(
"{}: {}",
bucket_name, INVALID_BUCKET_NAME_MESSAGE
@@ -285,11 +236,11 @@ pub async fn handle_delete_bucket(ctx: ReqCtx) -> Result<Response<ResBody>, Erro
// 1. delete bucket alias
if is_local_alias {
helper
.purge_local_bucket_alias(*bucket_id, &api_key.key_id, bucket_name)
.unset_local_bucket_alias(*bucket_id, &api_key.key_id, bucket_name)
.await?;
} else {
helper
.purge_global_bucket_alias(*bucket_id, bucket_name)
.unset_global_bucket_alias(*bucket_id, bucket_name)
.await?;
}
@@ -355,15 +306,6 @@ fn parse_create_bucket_xml(xml_bytes: &[u8]) -> Option<Option<String>> {
Some(ret)
}
fn create_grantee(key_params: &KeyParams, api_key: &Key) -> s3_xml::Grantee {
s3_xml::Grantee {
xmlns_xsi: (),
typ: "CanonicalUser".to_string(),
display_name: Some(s3_xml::Value(key_params.name.get().to_string())),
id: Some(s3_xml::Value(api_key.key_id.to_string())),
}
}
#[cfg(test)]
mod tests {
use super::*;
+43 -95
View File
@@ -26,10 +26,9 @@ use garage_api_common::signature::checksum::*;
use crate::api_server::{ReqBody, ResBody};
use crate::encryption::{EncryptionParams, OekDerivationInfo};
use crate::error::*;
use crate::get::{check_version_not_deleted, full_object_byte_stream, PreconditionHeaders};
use crate::get::{full_object_byte_stream, PreconditionHeaders};
use crate::multipart;
use crate::put::{extract_metadata_headers, save_stream, ChecksumMode, SaveStreamResult};
use crate::website::X_AMZ_WEBSITE_REDIRECT_LOCATION;
use crate::xml::{self as s3_xml, xmlns_tag};
pub const X_AMZ_COPY_SOURCE_IF_MATCH: HeaderName =
@@ -79,24 +78,8 @@ pub async fn handle_copy(
},
)?;
let was_multipart = source_version_meta.etag.contains('-') // HACK
|| source_object_meta_inner.checksum_type == Some(ChecksumType::Composite);
// Extract source checksum info before source_object_meta_inner is consumed
let source_checksum = source_object_meta_inner.checksum;
let source_checksum_type = match (source_object_meta_inner.checksum_type, source_checksum) {
(Some(ct), _) => Some(ct),
(None, Some(_)) => {
// Migrated object from garage v1.x or older
// determine checksum type depending if this is a multipart upload or not
if was_multipart {
Some(ChecksumType::Composite)
} else {
Some(ChecksumType::FullObject)
}
}
(None, None) => None,
};
let source_checksum_algorithm = source_checksum.map(|x| x.algorithm());
// If source object has a checksum, the destination object must as well.
@@ -105,26 +88,15 @@ pub async fn handle_copy(
let checksum_algorithm = checksum_algorithm.or(source_checksum_algorithm);
// Determine metadata of destination object
let was_multipart = source_version_meta.etag.contains('-');
let dest_object_meta = ObjectVersionMetaInner {
headers: match req.headers().get("x-amz-metadata-directive") {
Some(v) if v == hyper::header::HeaderValue::from_static("REPLACE") => {
extract_metadata_headers(req.headers())?
}
_ => {
// The x-amz-website-redirect-location header is not copied, instead
// it is replaced by the value from the request (or removed if no
// value was specified)
let is_redirect =
|(key, _): &(String, String)| key == X_AMZ_WEBSITE_REDIRECT_LOCATION.as_str();
let mut headers: Vec<_> = source_object_meta_inner.headers.clone();
headers.retain(|h| !is_redirect(h));
let new_headers = extract_metadata_headers(req.headers())?;
headers.extend(new_headers.into_iter().filter(is_redirect));
headers
}
_ => source_object_meta_inner.into_owned().headers,
},
checksum: source_checksum,
checksum_type: source_checksum_type,
};
// Do actual object copying
@@ -144,8 +116,8 @@ pub async fn handle_copy(
// See: https://docs.aws.amazon.com/AmazonS3/latest/userguide/checking-object-integrity.html
let must_recopy = !EncryptionParams::is_same(&source_encryption, &dest_encryption)
|| (checksum_algorithm.is_some()
&& (was_multipart || checksum_algorithm != source_checksum_algorithm));
|| source_checksum_algorithm != checksum_algorithm
|| (was_multipart && checksum_algorithm.is_some());
let res = if !must_recopy {
// In most cases, we can just copy the metadata and link blocks of the
@@ -162,21 +134,18 @@ pub async fn handle_copy(
)
.await?
} else {
let expected_checksum = ExpectedChecksums {
md5: None,
sha256: None,
extra: source_checksum,
};
let checksum_mode = if was_multipart || source_checksum_algorithm != checksum_algorithm {
ChecksumMode::Calculate(checksum_algorithm)
} else {
ChecksumMode::Verify(ExpectedChecksums {
md5: None,
sha256: None,
extra: source_checksum,
})
};
// For multipart uploads that had a composite checksum, set checksum type
// to full object as it will be recalculated.
let dest_object_meta = ObjectVersionMetaInner {
checksum_type: checksum_algorithm.map(|_| ChecksumType::FullObject),
..dest_object_meta
ChecksumMode::Verify(&expected_checksum)
};
// If source and dest encryption use different keys,
// we must decrypt content and re-encrypt, so rewrite all data blocks.
handle_copy_reencrypt(
ctx,
dest_key,
@@ -268,7 +237,6 @@ async fn handle_copy_metaonly(
.get(&source_version.uuid, &EmptyKey)
.await?;
let source_version = source_version.ok_or(Error::NoSuchKey)?;
check_version_not_deleted(&source_version)?;
// Write an "uploading" marker in Object table
// This holds a reference to the object in the Version table
@@ -357,7 +325,7 @@ async fn handle_copy_reencrypt(
source_version: &ObjectVersion,
source_version_data: &ObjectVersionData,
source_encryption: EncryptionParams,
checksum_mode: ChecksumMode,
checksum_mode: ChecksumMode<'_>,
) -> Result<SaveStreamResult, Error> {
// basically we will read the source data (decrypt if necessary)
// and save that in a new object (encrypt if necessary),
@@ -469,7 +437,6 @@ pub async fn handle_upload_part_copy(
.get(&source_object_version.uuid, &EmptyKey)
.await?
.ok_or(Error::NoSuchKey)?;
check_version_not_deleted(&source_version)?;
// We want to reuse blocks from the source version as much as possible.
// However, we still need to get the data from these blocks
@@ -545,7 +512,7 @@ pub async fn handle_upload_part_copy(
// Now, actually copy the blocks
let mut checksummer = Checksummer::init(&Default::default(), !dest_encryption.is_encrypted())
.add(dest_object_checksum_algorithm.map(|(algo, _)| algo));
.add(dest_object_checksum_algorithm);
// First, create a stream that is able to read the source blocks
// and extract the subrange if necessary.
@@ -601,7 +568,6 @@ pub async fn handle_upload_part_copy(
let mut current_offset = 0;
let mut next_block = defragmenter.next().await?;
let mut blocks_to_dup = dest_version.clone();
// TODO this could be optimized similarly to read_and_put_blocks
// low priority because uploadpartcopy is rarely used
@@ -631,7 +597,8 @@ pub async fn handle_upload_part_copy(
.unwrap()?;
checksummer = checksummer_updated;
let (version_block_key, version_block) = (
dest_version.blocks.clear();
dest_version.blocks.put(
VersionBlockKey {
part_number,
offset: current_offset,
@@ -643,59 +610,40 @@ pub async fn handle_upload_part_copy(
);
current_offset += data_len;
let next = if let Some(final_data) = data_to_upload {
dest_version.blocks.clear();
dest_version.blocks.put(version_block_key, version_block);
let block_ref = BlockRef {
block: final_hash,
version: dest_version_id,
deleted: false.into(),
};
let (_, _, _, next) = futures::try_join!(
// Thing 1: if the block is not exactly a block that existed before,
// we need to insert that data as a new block.
garage.block_manager.rpc_put_block(
final_hash,
final_data,
dest_encryption.is_encrypted(),
None
),
// Thing 2: we need to insert the block in the version
garage.version_table.insert(&dest_version),
// Thing 3: we need to add a block reference
garage.block_ref_table.insert(&block_ref),
// Thing 4: we need to read the next block
defragmenter.next(),
)?;
next
} else {
blocks_to_dup.blocks.put(version_block_key, version_block);
defragmenter.next().await?
let block_ref = BlockRef {
block: final_hash,
version: dest_version_id,
deleted: false.into(),
};
let (_, _, _, next) = futures::try_join!(
// Thing 1: if the block is not exactly a block that existed before,
// we need to insert that data as a new block.
async {
if let Some(final_data) = data_to_upload {
garage
.block_manager
.rpc_put_block(final_hash, final_data, dest_encryption.is_encrypted(), None)
.await
} else {
Ok(())
}
},
// Thing 2: we need to insert the block in the version
garage.version_table.insert(&dest_version),
// Thing 3: we need to add a block reference
garage.block_ref_table.insert(&block_ref),
// Thing 4: we need to read the next block
defragmenter.next(),
)?;
next_block = next;
}
assert_eq!(current_offset, source_range.length);
// Put the duplicated blocks into the version & block_refs tables
let block_refs_to_put = blocks_to_dup
.blocks
.items()
.iter()
.map(|b| BlockRef {
block: b.1.hash,
version: dest_version_id,
deleted: false.into(),
})
.collect::<Vec<_>>();
futures::try_join!(
garage.version_table.insert(&blocks_to_dup),
garage.block_ref_table.insert_many(&block_refs_to_put[..]),
)?;
let checksums = checksummer.finalize();
let etag = dest_encryption.etag_from_md5(&checksums.md5);
let checksum = checksums.extract(dest_object_checksum_algorithm.map(|(algo, _)| algo));
let checksum = checksums.extract(dest_object_checksum_algorithm);
// Put the part's ETag in the Versiontable
dest_mpu.parts.put(
+1 -1
View File
@@ -29,7 +29,7 @@ pub async fn handle_get_cors(ctx: ReqCtx) -> Result<Response<ResBody>, Error> {
.body(string_body(xml))?)
} else {
Ok(Response::builder()
.status(StatusCode::NOT_FOUND)
.status(StatusCode::NO_CONTENT)
.body(empty_body())?)
}
}
+20 -27
View File
@@ -1,8 +1,8 @@
use std::convert::TryInto;
use err_derive::Error;
use hyper::header::HeaderValue;
use hyper::{HeaderMap, StatusCode};
use thiserror::Error;
use garage_model::helper::error::Error as HelperError;
@@ -25,67 +25,67 @@ use crate::xml as s3_xml;
/// Errors of this crate
#[derive(Debug, Error)]
pub enum Error {
#[error("{0}")]
#[error(display = "{}", _0)]
/// Error from common error
Common(#[from] CommonError),
Common(#[error(source)] CommonError),
// Category: cannot process
/// Authorization Header Malformed
#[error("Authorization header malformed, unexpected scope: {0}")]
#[error(display = "Authorization header malformed, unexpected scope: {}", _0)]
AuthorizationHeaderMalformed(String),
/// The object requested don't exists
#[error("Key not found")]
#[error(display = "Key not found")]
NoSuchKey,
/// The multipart upload requested don't exists
#[error("Upload not found")]
#[error(display = "Upload not found")]
NoSuchUpload,
/// Precondition failed (e.g. x-amz-copy-source-if-match)
#[error("At least one of the preconditions you specified did not hold")]
#[error(display = "At least one of the preconditions you specified did not hold")]
PreconditionFailed,
/// Parts specified in CMU request do not match parts actually uploaded
#[error("Parts given to CompleteMultipartUpload do not match uploaded parts")]
#[error(display = "Parts given to CompleteMultipartUpload do not match uploaded parts")]
InvalidPart,
/// Parts given to CompleteMultipartUpload were not in ascending order
#[error("Parts given to CompleteMultipartUpload were not in ascending order")]
#[error(display = "Parts given to CompleteMultipartUpload were not in ascending order")]
InvalidPartOrder,
/// In CompleteMultipartUpload: not enough data
/// (here we are more lenient than AWS S3)
#[error("Proposed upload is smaller than the minimum allowed object size")]
#[error(display = "Proposed upload is smaller than the minimum allowed object size")]
EntityTooSmall,
// Category: bad request
/// The request contained an invalid UTF-8 sequence in its path or in other parameters
#[error("Invalid UTF-8: {0}")]
InvalidUtf8Str(#[from] std::str::Utf8Error),
#[error(display = "Invalid UTF-8: {}", _0)]
InvalidUtf8Str(#[error(source)] std::str::Utf8Error),
/// The request used an invalid path
#[error("Invalid UTF-8: {0}")]
InvalidUtf8String(#[from] std::string::FromUtf8Error),
#[error(display = "Invalid UTF-8: {}", _0)]
InvalidUtf8String(#[error(source)] std::string::FromUtf8Error),
/// The client sent invalid XML data
#[error("Invalid XML: {0}")]
#[error(display = "Invalid XML: {}", _0)]
InvalidXml(String),
/// The client sent a range header with invalid value
#[error("Invalid HTTP range: {0:?}")]
InvalidRange((http_range::HttpRangeParseError, u64)),
#[error(display = "Invalid HTTP range: {:?}", _0)]
InvalidRange(#[error(from)] (http_range::HttpRangeParseError, u64)),
/// The client sent a range header with invalid value
#[error("Invalid encryption algorithm: {0:?}, should be AES256")]
#[error(display = "Invalid encryption algorithm: {:?}, should be AES256", _0)]
InvalidEncryptionAlgorithm(String),
/// The provided digest (checksum) value was invalid
#[error("Invalid digest: {0}")]
#[error(display = "Invalid digest: {}", _0)]
InvalidDigest(String),
/// The client sent a request for an action not supported by garage
#[error("Unimplemented action: {0}")]
#[error(display = "Unimplemented action: {}", _0)]
NotImplemented(String),
}
@@ -99,12 +99,6 @@ impl From<HelperError> for Error {
}
}
impl From<(http_range::HttpRangeParseError, u64)> for Error {
fn from(err: (http_range::HttpRangeParseError, u64)) -> Error {
Error::InvalidRange(err)
}
}
impl From<roxmltree::Error> for Error {
fn from(err: roxmltree::Error) -> Self {
Self::InvalidXml(format!("{}", err))
@@ -182,7 +176,6 @@ impl ApiError for Error {
use hyper::header;
header_map.append(header::CONTENT_TYPE, "application/xml".parse().unwrap());
header_map.append(header::ACCESS_CONTROL_ALLOW_ORIGIN, "*".parse().unwrap());
#[allow(clippy::single_match)]
match self {

Some files were not shown because too many files have changed in this diff Show More