mirror of
https://github.com/deuxfleurs-org/garage.git
synced 2026-08-14 16:13:11 +00:00
Compare commits
1 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Alex Auvolat
|
6a6f01ee09 |
Generated
+208
-343
File diff suppressed because it is too large
Load Diff
-12
@@ -11,22 +11,10 @@ members = [
|
||||
"src/web",
|
||||
"src/garage",
|
||||
"src/k2v-client",
|
||||
"src/format-table",
|
||||
]
|
||||
|
||||
default-members = ["src/garage"]
|
||||
|
||||
[workspace.dependencies]
|
||||
format_table = { version = "0.1.1", path = "src/format-table" }
|
||||
garage_api = { version = "0.8.2", path = "src/api" }
|
||||
garage_block = { version = "0.8.2", path = "src/block" }
|
||||
garage_db = { version = "0.8.2", path = "src/db", default-features = false }
|
||||
garage_model = { version = "0.8.2", path = "src/model", default-features = false }
|
||||
garage_rpc = { version = "0.8.2", path = "src/rpc" }
|
||||
garage_table = { version = "0.8.2", path = "src/table" }
|
||||
garage_util = { version = "0.8.2", path = "src/util" }
|
||||
garage_web = { version = "0.8.2", path = "src/web" }
|
||||
|
||||
[profile.dev]
|
||||
lto = "off"
|
||||
|
||||
|
||||
+49
-25
@@ -12,7 +12,6 @@ These tools are particularly suitable for debug, backups, website deployments or
|
||||
| [AWS CLI](#aws-cli) | ✅ | Recommended |
|
||||
| [rclone](#rclone) | ✅ | |
|
||||
| [s3cmd](#s3cmd) | ✅ | |
|
||||
| [s5cmd](#s5cmd) | ✅ | |
|
||||
| [(Cyber)duck](#cyberduck) | ✅ | |
|
||||
| [WinSCP (libs3)](#winscp) | ✅ | CLI instructions only |
|
||||
| [sftpgo](#sftpgo) | ✅ | |
|
||||
@@ -179,34 +178,59 @@ s3cmd put /tmp/hello.txt s3://my-bucket/
|
||||
s3cmd get s3://my-bucket/hello.txt hello.txt
|
||||
```
|
||||
|
||||
## `s5cmd`
|
||||
|
||||
Configure a credentials file as follows:
|
||||
|
||||
```bash
|
||||
export AWS_ACCESS_KEY_ID=GK...
|
||||
export AWS_SECRET_ACCESS_KEY=
|
||||
export AWS_DEFAULT_REGION='garage'
|
||||
export AWS_ENDPOINT='http://localhost:3900'
|
||||
```
|
||||
|
||||
After adding these environment variables in your shell, `s5cmd` can be used
|
||||
with:
|
||||
|
||||
```bash
|
||||
s5cmd --endpoint-url=$AWS_ENDPOINT ls
|
||||
```
|
||||
|
||||
See its usage output for other commands available.
|
||||
|
||||
## Cyberduck & duck {#cyberduck}
|
||||
|
||||
Both Cyberduck (the GUI) and duck (the CLI) have a concept of "Connection Profiles" that contain some presets for a specific provider.
|
||||
We wrote the following connection profile for Garage:
|
||||
|
||||
Within Cyberduck, a
|
||||
[Garage connection profile](https://docs.cyberduck.io/protocols/s3/garage/) is
|
||||
available within the `Preferences -> Profiles` section. This can enabled and
|
||||
then connections to Garage may be configured.
|
||||
```xml
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
|
||||
<plist version="1.0">
|
||||
<dict>
|
||||
<key>Protocol</key>
|
||||
<string>s3</string>
|
||||
<key>Vendor</key>
|
||||
<string>garage</string>
|
||||
<key>Scheme</key>
|
||||
<string>https</string>
|
||||
<key>Description</key>
|
||||
<string>GarageS3</string>
|
||||
<key>Default Hostname</key>
|
||||
<string>127.0.0.1</string>
|
||||
<key>Default Port</key>
|
||||
<string>4443</string>
|
||||
<key>Hostname Configurable</key>
|
||||
<false/>
|
||||
<key>Port Configurable</key>
|
||||
<false/>
|
||||
<key>Username Configurable</key>
|
||||
<true/>
|
||||
<key>Username Placeholder</key>
|
||||
<string>Access Key ID (GK...)</string>
|
||||
<key>Password Placeholder</key>
|
||||
<string>Secret Key</string>
|
||||
<key>Properties</key>
|
||||
<array>
|
||||
<string>s3service.disable-dns-buckets=true</string>
|
||||
</array>
|
||||
<key>Region</key>
|
||||
<string>garage</string>
|
||||
<key>Regions</key>
|
||||
<array>
|
||||
<string>garage</string>
|
||||
</array>
|
||||
</dict>
|
||||
</plist>
|
||||
```
|
||||
|
||||
*Note: If your garage instance is configured with vhost access style, you can remove `s3service.disable-dns-buckets=true`.*
|
||||
|
||||
### Instructions for the GUI
|
||||
|
||||
Copy the connection profile, and save it anywhere as `garage.cyberduckprofile`.
|
||||
Then find this file with your file explorer and double click on it: Cyberduck will open a connection wizard for this profile.
|
||||
Simply follow the wizard and you should be done!
|
||||
|
||||
### Instuctions for the CLI
|
||||
|
||||
|
||||
@@ -6,7 +6,7 @@ sort_by = "weight"
|
||||
+++
|
||||
|
||||
A cookbook, when you cook, is a collection of recipes.
|
||||
Similarly, Garage's cookbook contains a collection of recipes that are known to work well!
|
||||
Similarly, Garage's cookbook contains a collection of recipes that are known to works well!
|
||||
This chapter could also be referred as "Tutorials" or "Best practices".
|
||||
|
||||
- **[Multi-node deployment](@/documentation/cookbook/real-world.md):** This page will walk you through all of the necessary
|
||||
@@ -16,10 +16,6 @@ This chapter could also be referred as "Tutorials" or "Best practices".
|
||||
source in case a binary is not provided for your architecture, or if you want to
|
||||
hack with us!
|
||||
|
||||
- **[Binary packages](@/documentation/cookbook/binary-packages.md):** This page
|
||||
lists the different platforms that provide ready-built software packages for
|
||||
Garage.
|
||||
|
||||
- **[Integration with Systemd](@/documentation/cookbook/systemd.md):** This page explains how to run Garage
|
||||
as a Systemd service (instead of as a Docker container).
|
||||
|
||||
@@ -30,10 +26,6 @@ This chapter could also be referred as "Tutorials" or "Best practices".
|
||||
|
||||
- **[Configuring a reverse-proxy](@/documentation/cookbook/reverse-proxy.md):** This page explains how to configure a reverse-proxy to add TLS support to your S3 api endpoint.
|
||||
|
||||
- **[Deploying on Kubernetes](@/documentation/cookbook/kubernetes.md):** This page explains how to deploy Garage on Kubernetes using our Helm chart.
|
||||
|
||||
- **[Deploying with Ansible](@/documentation/cookbook/ansible.md):** This page lists available Ansible roles developed by the community to deploy Garage.
|
||||
|
||||
- **[Monitoring Garage](@/documentation/cookbook/monitoring.md)** This page
|
||||
explains the Prometheus metrics available for monitoring the Garage
|
||||
cluster/nodes.
|
||||
|
||||
@@ -1,51 +0,0 @@
|
||||
+++
|
||||
title = "Deploying with Ansible"
|
||||
weight = 35
|
||||
+++
|
||||
|
||||
While Ansible is not officially supported to deploy Garage, several community members
|
||||
have published Ansible roles. We list them and compare them below.
|
||||
|
||||
## Comparison of Ansible roles
|
||||
|
||||
| Feature | [ansible-role-garage](#zorun-ansible-role-garage) | [garage-docker-ansible-deploy](#moan0s-garage-docker-ansible-deploy) |
|
||||
|------------------------------------|---------------------------------------------|---------------------------------------------------------------|
|
||||
| **Runtime** | Systemd | Docker |
|
||||
| **Target OS** | Any Linux | Any Linux |
|
||||
| **Architecture** | amd64, arm64, i686 | amd64, arm64 |
|
||||
| **Additional software** | None | Traefik |
|
||||
| **Automatic node connection** | ❌ | ✅ |
|
||||
| **Layout management** | ❌ | ✅ |
|
||||
| **Manage buckets & keys** | ❌ | ✅ (basic) |
|
||||
| **Allow custom Garage config** | ✅ | ❌ |
|
||||
| **Facilitate Garage upgrades** | ✅ | ❌ |
|
||||
| **Multiple instances on one host** | ✅ | ✅ |
|
||||
|
||||
|
||||
## zorun/ansible-role-garage
|
||||
|
||||
[Source code](https://github.com/zorun/ansible-role-garage), [Ansible galaxy](https://galaxy.ansible.com/zorun/garage)
|
||||
|
||||
This role is voluntarily simple: it relies on the official Garage static
|
||||
binaries and only requires Systemd. As such, it should work on any
|
||||
Linux-based OS.
|
||||
|
||||
To make things more flexible, the user has to provide a Garage
|
||||
configuration template. This allows to customize Garage configuration in
|
||||
any way.
|
||||
|
||||
Some more features might be added, such as a way to automatically connect
|
||||
nodes to each other or to define a layout.
|
||||
|
||||
## moan0s/garage-docker-ansible-deploy
|
||||
|
||||
[Source code](https://github.com/moan0s/garage-docker-ansible-deploy), [Blog post](https://hyteck.de/post/garage/)
|
||||
|
||||
This role is based on the Docker image for Garage, and comes with
|
||||
"batteries included": it will additionally install Docker and Traefik. In
|
||||
addition, it is "opinionated" in the sense that it expects a particular
|
||||
deployment structure (one instance per disk, one gateway per host,
|
||||
structured DNS names, etc).
|
||||
|
||||
As a result, this role makes it easier to start with Garage on Ansible,
|
||||
but is less flexible.
|
||||
@@ -1,28 +0,0 @@
|
||||
+++
|
||||
title = "Binary packages"
|
||||
weight = 11
|
||||
+++
|
||||
|
||||
Garage is also available in binary packages on:
|
||||
|
||||
## Alpine Linux
|
||||
|
||||
```bash
|
||||
apk install garage
|
||||
```
|
||||
|
||||
## Arch Linux
|
||||
|
||||
Garage is available in the [AUR](https://aur.archlinux.org/packages/garage).
|
||||
|
||||
## FreeBSD
|
||||
|
||||
```bash
|
||||
pkg install garage
|
||||
```
|
||||
|
||||
## NixOS
|
||||
|
||||
```bash
|
||||
nix-shell -p garage
|
||||
```
|
||||
@@ -349,5 +349,5 @@ and is covered in the [quick start guide](@/documentation/quick-start/_index.md)
|
||||
Remember also that the CLI is self-documented thanks to the `--help` flag and
|
||||
the `help` subcommand (e.g. `garage help`, `garage key --help`).
|
||||
|
||||
Configuring S3-compatible applications to interact with Garage
|
||||
Configuring S3-compatible applicatiosn to interact with Garage
|
||||
is covered in the [Integrations](@/documentation/connect/_index.md) section.
|
||||
|
||||
@@ -290,13 +290,13 @@ sourcing the right file.*
|
||||
aws s3 ls
|
||||
|
||||
# list objects of a bucket
|
||||
aws s3 ls s3://nextcloud-bucket
|
||||
aws s3 ls s3://my_files
|
||||
|
||||
# copy from your filesystem to garage
|
||||
aws s3 cp /proc/cpuinfo s3://nextcloud-bucket/cpuinfo.txt
|
||||
aws s3 cp /proc/cpuinfo s3://my_files/cpuinfo.txt
|
||||
|
||||
# copy from garage to your filesystem
|
||||
aws s3 cp s3://nextcloud-bucket/cpuinfo.txt /tmp/cpuinfo.txt
|
||||
aws s3 cp s3/my_files/cpuinfo.txt /tmp/cpuinfo.txt
|
||||
```
|
||||
|
||||
Note that you can use `awscli` for more advanced operations like
|
||||
|
||||
|
Before Width: | Height: | Size: 16 KiB After Width: | Height: | Size: 16 KiB |
+1
-181
File diff suppressed because one or more lines are too long
|
Before Width: | Height: | Size: 15 KiB After Width: | Height: | Size: 74 KiB |
File diff suppressed because one or more lines are too long
|
Before Width: | Height: | Size: 74 KiB |
Generated
+23
-76
@@ -41,15 +41,12 @@
|
||||
}
|
||||
},
|
||||
"flake-utils": {
|
||||
"inputs": {
|
||||
"systems": "systems"
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1681202837,
|
||||
"narHash": "sha256-H+Rh19JDwRtpVPAWp64F+rlEtxUWBAQW28eAi3SRSzg=",
|
||||
"lastModified": 1659877975,
|
||||
"narHash": "sha256-zllb8aq3YO3h8B/U0/J1WBgAL8EX5yWf5pMj3G0NAmc=",
|
||||
"owner": "numtide",
|
||||
"repo": "flake-utils",
|
||||
"rev": "cfacdce06f30d2b68473a46042957675eebb3401",
|
||||
"rev": "c0e246b9b83f637f4681389ecabcb2681b4f3af0",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
@@ -59,15 +56,12 @@
|
||||
}
|
||||
},
|
||||
"flake-utils_2": {
|
||||
"inputs": {
|
||||
"systems": "systems_2"
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1681202837,
|
||||
"narHash": "sha256-H+Rh19JDwRtpVPAWp64F+rlEtxUWBAQW28eAi3SRSzg=",
|
||||
"lastModified": 1667395993,
|
||||
"narHash": "sha256-nuEHfE/LcWyuSWnS8t12N1wc105Qtau+/OdUAjtQ0rA=",
|
||||
"owner": "numtide",
|
||||
"repo": "flake-utils",
|
||||
"rev": "cfacdce06f30d2b68473a46042957675eebb3401",
|
||||
"rev": "5aed5285a952e0b949eb3ba02c12fa4fcfef535f",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
@@ -78,61 +72,44 @@
|
||||
},
|
||||
"nixpkgs": {
|
||||
"locked": {
|
||||
"lastModified": 1682109806,
|
||||
"narHash": "sha256-d9g7RKNShMLboTWwukM+RObDWWpHKaqTYXB48clBWXI=",
|
||||
"lastModified": 1665657542,
|
||||
"narHash": "sha256-mojxNyzbvmp8NtVtxqiHGhRfjCALLfk9i/Uup68Y5q8=",
|
||||
"owner": "NixOS",
|
||||
"repo": "nixpkgs",
|
||||
"rev": "2362848adf8def2866fabbffc50462e929d7fffb",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "NixOS",
|
||||
"ref": "nixpkgs-unstable",
|
||||
"repo": "nixpkgs",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"nixpkgs_2": {
|
||||
"locked": {
|
||||
"lastModified": 1682423271,
|
||||
"narHash": "sha256-WHhl1GiOij1ob4cTLL+yhqr+vFOUH8E5wAX8Ir8fvjE=",
|
||||
"owner": "NixOS",
|
||||
"repo": "nixpkgs",
|
||||
"rev": "94517a501434a627c5d9e72ac6e7f26174b978d3",
|
||||
"rev": "a3073c49bc0163fea6a121c276f526837672b555",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "NixOS",
|
||||
"repo": "nixpkgs",
|
||||
"rev": "94517a501434a627c5d9e72ac6e7f26174b978d3",
|
||||
"rev": "a3073c49bc0163fea6a121c276f526837672b555",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"root": {
|
||||
"inputs": {
|
||||
"cargo2nix": "cargo2nix",
|
||||
"flake-compat": [
|
||||
"cargo2nix",
|
||||
"flake-compat"
|
||||
],
|
||||
"flake-utils": [
|
||||
"cargo2nix",
|
||||
"flake-utils"
|
||||
],
|
||||
"nixpkgs": "nixpkgs_2"
|
||||
"flake-utils": "flake-utils_2",
|
||||
"nixpkgs": "nixpkgs"
|
||||
}
|
||||
},
|
||||
"rust-overlay": {
|
||||
"inputs": {
|
||||
"flake-utils": "flake-utils_2",
|
||||
"nixpkgs": "nixpkgs"
|
||||
"flake-utils": [
|
||||
"cargo2nix",
|
||||
"flake-utils"
|
||||
],
|
||||
"nixpkgs": [
|
||||
"cargo2nix",
|
||||
"nixpkgs"
|
||||
]
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1682389182,
|
||||
"narHash": "sha256-8t2nmFnH+8V48+IJsf8AK51ebXNlVbOSVYOpiqJKvJE=",
|
||||
"lastModified": 1664247556,
|
||||
"narHash": "sha256-J4vazHU3609ekn7dr+3wfqPo5WGlZVAgV7jfux352L0=",
|
||||
"owner": "oxalica",
|
||||
"repo": "rust-overlay",
|
||||
"rev": "74f1a64dd28faeeb85ef081f32cad2989850322c",
|
||||
"rev": "524db9c9ea7bc7743bb74cdd45b6d46ea3fcc2ab",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
@@ -140,36 +117,6 @@
|
||||
"repo": "rust-overlay",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"systems": {
|
||||
"locked": {
|
||||
"lastModified": 1681028828,
|
||||
"narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
|
||||
"owner": "nix-systems",
|
||||
"repo": "default",
|
||||
"rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "nix-systems",
|
||||
"repo": "default",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"systems_2": {
|
||||
"locked": {
|
||||
"lastModified": 1681028828,
|
||||
"narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
|
||||
"owner": "nix-systems",
|
||||
"repo": "default",
|
||||
"rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "nix-systems",
|
||||
"repo": "default",
|
||||
"type": "github"
|
||||
}
|
||||
}
|
||||
},
|
||||
"root": "root",
|
||||
|
||||
@@ -2,30 +2,16 @@
|
||||
description =
|
||||
"Garage, an S3-compatible distributed object store for self-hosted deployments";
|
||||
|
||||
# Nixpkgs unstable as of 2023-04-25, has rustc v1.68
|
||||
inputs.nixpkgs.url =
|
||||
"github:NixOS/nixpkgs/94517a501434a627c5d9e72ac6e7f26174b978d3";
|
||||
|
||||
"github:NixOS/nixpkgs/a3073c49bc0163fea6a121c276f526837672b555";
|
||||
inputs.cargo2nix = {
|
||||
# As of 2022-10-18: two small patches over unstable branch, one for clippy and one to fix feature detection
|
||||
url = "github:Alexis211/cargo2nix/a7a61179b66054904ef6a195d8da736eaaa06c36";
|
||||
|
||||
# As of 2023-04-25:
|
||||
# - my two patches were merged into unstable (one for clippy and one to "fix" feature detection)
|
||||
# - rustc v1.66
|
||||
# url = "github:cargo2nix/cargo2nix/8fb57a670f7993bfc24099c33eb9c5abb51f29a2";
|
||||
|
||||
# Rust overlay as of 2023-04-25
|
||||
inputs.rust-overlay.url =
|
||||
"github:oxalica/rust-overlay/74f1a64dd28faeeb85ef081f32cad2989850322c";
|
||||
|
||||
inputs.nixpkgs.follows = "nixpkgs";
|
||||
};
|
||||
inputs.flake-utils.url = "github:numtide/flake-utils";
|
||||
|
||||
inputs.flake-utils.follows = "cargo2nix/flake-utils";
|
||||
inputs.flake-compat.follows = "cargo2nix/flake-compat";
|
||||
|
||||
outputs = { self, nixpkgs, cargo2nix, flake-utils, ... }:
|
||||
outputs = { self, nixpkgs, cargo2nix, flake-utils }:
|
||||
let
|
||||
git_version = self.lastModifiedDate;
|
||||
compile = import ./nix/compile.nix;
|
||||
|
||||
+15
-10
@@ -1,14 +1,19 @@
|
||||
let
|
||||
lock = builtins.fromJSON (builtins.readFile ../flake.lock);
|
||||
flakeCompatRev = lock.nodes.flake-compat.locked.rev;
|
||||
flakeCompat = fetchTarball {
|
||||
rec {
|
||||
# * Fixed dependencies
|
||||
pkgsSrc = fetchTarball {
|
||||
# As of 2022-10-13
|
||||
url =
|
||||
"https://github.com/edolstra/flake-compat/archive/${flakeCompatRev}.tar.gz";
|
||||
sha256 = lock.nodes.flake-compat.locked.narHash;
|
||||
"https://github.com/NixOS/nixpkgs/archive/a3073c49bc0163fea6a121c276f526837672b555.zip";
|
||||
sha256 = "1bz632psfbpmicyzjb8b4265y50shylccvfm6ry6mgnv5hvz324s";
|
||||
};
|
||||
flake = ((import flakeCompat) { src = ../.; }).defaultNix;
|
||||
in rec {
|
||||
pkgsSrc = flake.inputs.nixpkgs;
|
||||
cargo2nix = flake.inputs.cargo2nix;
|
||||
cargo2nixSrc = fetchGit {
|
||||
# As of 2022-10-18: two small patches over unstable branch, one for clippy and one to fix feature detection
|
||||
url = "https://github.com/Alexis211/cargo2nix";
|
||||
ref = "custom_unstable";
|
||||
rev = "a7a61179b66054904ef6a195d8da736eaaa06c36";
|
||||
};
|
||||
|
||||
# * Shared objects
|
||||
cargo2nix = import cargo2nixSrc;
|
||||
cargo2nixOverlay = cargo2nix.overlays.default;
|
||||
}
|
||||
|
||||
+1
-1
@@ -32,7 +32,7 @@ let
|
||||
*/
|
||||
toolchainOptions = if target == null || target == "x86_64-unknown-linux-musl"
|
||||
|| target == "aarch64-unknown-linux-musl" then {
|
||||
rustVersion = "1.68.0";
|
||||
rustVersion = "1.63.0";
|
||||
extraRustComponents = [ "clippy" ];
|
||||
} else {
|
||||
rustToolchain = pkgs.symlinkJoin {
|
||||
|
||||
+2
-2
@@ -1,13 +1,13 @@
|
||||
pkgs:
|
||||
pkgs.buildGoModule rec {
|
||||
pname = "kaniko";
|
||||
version = "1.9.2";
|
||||
version = "1.6.0";
|
||||
|
||||
src = pkgs.fetchFromGitHub {
|
||||
owner = "GoogleContainerTools";
|
||||
repo = "kaniko";
|
||||
rev = "v${version}";
|
||||
sha256 = "dXQ0/o1qISv+sjNVIpfF85bkbM9sGOGwqVbWZpMWfMY=";
|
||||
sha256 = "1fnclr556avxay6pvgw5ya3xbxfnf2gv4njq2hr4fd6fcjyslq5h";
|
||||
};
|
||||
|
||||
vendorSha256 = null;
|
||||
|
||||
@@ -15,10 +15,10 @@ type: application
|
||||
# This is the chart version. This version number should be incremented each time you make changes
|
||||
# to the chart and its templates, including the app version.
|
||||
# Versions are expected to follow Semantic Versioning (https://semver.org/)
|
||||
version: 0.4.1
|
||||
version: 0.3.0
|
||||
|
||||
# This is the version number of the application being deployed. This version number should be
|
||||
# incremented each time you make changes to the application. Versions are not expected to
|
||||
# follow Semantic Versioning. They should reflect the version the application is using.
|
||||
# It is recommended to use it with quotes.
|
||||
appVersion: "v0.8.2"
|
||||
appVersion: "v0.7.2.1"
|
||||
|
||||
@@ -4,28 +4,9 @@
|
||||
|
||||
# Garage configuration. These values go to garage.toml
|
||||
garage:
|
||||
# Can be changed for better performance on certain systems
|
||||
# https://garagehq.deuxfleurs.fr/documentation/reference-manual/configuration/#db-engine-since-v0-8-0
|
||||
dbEngine: "sled"
|
||||
|
||||
# Defaults is 1MB
|
||||
# An increase can result in better performance in certain scenarios
|
||||
# https://garagehq.deuxfleurs.fr/documentation/reference-manual/configuration/#block-size
|
||||
blockSize: "1048576"
|
||||
|
||||
# Tuning parameters for the sled DB engine
|
||||
# https://garagehq.deuxfleurs.fr/documentation/reference-manual/configuration/#sled-cache-capacity
|
||||
sledCacheCapacity: "134217728"
|
||||
sledFlushEveryMs: "2000"
|
||||
|
||||
# Default to 3 replicas, see the replication_mode section at
|
||||
# https://garagehq.deuxfleurs.fr/documentation/reference-manual/configuration/#replication-mode
|
||||
# https://garagehq.deuxfleurs.fr/documentation/reference-manual/configuration/
|
||||
replicationMode: "3"
|
||||
|
||||
# zstd compression level of stored blocks
|
||||
# https://garagehq.deuxfleurs.fr/documentation/reference-manual/configuration/#compression-level
|
||||
compressionLevel: "1"
|
||||
|
||||
rpcBindAddr: "[::]:3901"
|
||||
# If not given, a random secret will be generated and stored in a Secret object
|
||||
rpcSecret: ""
|
||||
@@ -46,19 +27,8 @@ garage:
|
||||
metadata_dir = "/mnt/meta"
|
||||
data_dir = "/mnt/data"
|
||||
|
||||
db_engine = "{{ .Values.garage.dbEngine }}"
|
||||
|
||||
block_size = {{ .Values.garage.blockSize }}
|
||||
|
||||
{{- if eq .Values.garage.dbEngine "sled"}}
|
||||
sled_cache_capacity = {{ .Values.garage.sledCacheCapacity }}
|
||||
sled_flush_every_ms = {{ .Values.garage.sledFlushEveryMs }}
|
||||
{{- end }}
|
||||
|
||||
replication_mode = "{{ .Values.garage.replicationMode }}"
|
||||
|
||||
compression_level = {{ .Values.garage.compressionLevel }}
|
||||
|
||||
rpc_bind_addr = "{{ .Values.garage.rpcBindAddr }}"
|
||||
# rpc_secret will be populated by the init container from a k8s secret object
|
||||
rpc_secret = "__RPC_SECRET_REPLACE__"
|
||||
@@ -79,7 +49,7 @@ garage:
|
||||
bind_addr = "[::]:3902"
|
||||
root_domain = "{{ .Values.garage.s3.web.rootDomain }}"
|
||||
index = "{{ .Values.garage.s3.web.index }}"
|
||||
|
||||
|
||||
[admin]
|
||||
api_bind_addr = "[::]:3903"
|
||||
{{- if .Values.monitoring.tracing.sink }}
|
||||
@@ -241,4 +211,4 @@ monitoring:
|
||||
scrapeTimeout: 10s
|
||||
relabelings: []
|
||||
tracing:
|
||||
sink: ""
|
||||
sink: ""
|
||||
+7
-7
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "garage_api"
|
||||
version = "0.8.2"
|
||||
version = "0.8.1"
|
||||
authors = ["Alex Auvolat <alex@adnab.me>"]
|
||||
edition = "2018"
|
||||
license = "AGPL-3.0"
|
||||
@@ -14,11 +14,11 @@ path = "lib.rs"
|
||||
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
|
||||
|
||||
[dependencies]
|
||||
garage_model.workspace = true
|
||||
garage_table.workspace = true
|
||||
garage_block.workspace = true
|
||||
garage_util.workspace = true
|
||||
garage_rpc.workspace = true
|
||||
garage_model = { version = "0.8.1", path = "../model" }
|
||||
garage_table = { version = "0.8.1", path = "../table" }
|
||||
garage_block = { version = "0.8.1", path = "../block" }
|
||||
garage_util = { version = "0.8.1", path = "../util" }
|
||||
garage_rpc = { version = "0.8.1", path = "../rpc" }
|
||||
|
||||
async-trait = "0.1.7"
|
||||
base64 = "0.21"
|
||||
@@ -51,7 +51,7 @@ roxmltree = "0.14"
|
||||
serde = { version = "1.0", features = ["derive"] }
|
||||
serde_bytes = "0.11"
|
||||
serde_json = "1.0"
|
||||
quick-xml = { version = "0.26", features = [ "serialize" ] }
|
||||
quick-xml = { version = "0.21", features = [ "serialize" ] }
|
||||
url = "2.3"
|
||||
|
||||
opentelemetry = "0.17"
|
||||
|
||||
@@ -105,7 +105,7 @@ impl AdminApiServer {
|
||||
let bucket_id = self
|
||||
.garage
|
||||
.bucket_helper()
|
||||
.resolve_global_bucket_name(domain)
|
||||
.resolve_global_bucket_name(&domain)
|
||||
.await?
|
||||
.ok_or(HelperError::NoSuchBucket(domain.to_string()))?;
|
||||
|
||||
|
||||
@@ -183,8 +183,8 @@ async fn bucket_info_results(
|
||||
}
|
||||
}),
|
||||
keys: relevant_keys
|
||||
.into_values()
|
||||
.map(|key| {
|
||||
.into_iter()
|
||||
.map(|(_, key)| {
|
||||
let p = key.state.as_option().unwrap();
|
||||
GetBucketInfoKey {
|
||||
access_key_id: key.key_id,
|
||||
|
||||
@@ -20,7 +20,6 @@ pub async fn handle_get_cluster_status(garage: &Arc<Garage>) -> Result<Response<
|
||||
node: hex::encode(garage.system.id),
|
||||
garage_version: garage_util::version::garage_version(),
|
||||
garage_features: garage_util::version::garage_features(),
|
||||
rust_version: garage_util::version::rust_version(),
|
||||
db_engine: garage.db.engine(),
|
||||
known_nodes: garage
|
||||
.system
|
||||
@@ -107,7 +106,6 @@ struct GetClusterStatusResponse {
|
||||
node: String,
|
||||
garage_version: &'static str,
|
||||
garage_features: Option<&'static [&'static str]>,
|
||||
rust_version: &'static str,
|
||||
db_engine: String,
|
||||
known_nodes: HashMap<String, KnownNodeResp>,
|
||||
layout: GetClusterLayoutResponse,
|
||||
|
||||
@@ -183,8 +183,8 @@ async fn key_info_results(garage: &Arc<Garage>, key: Key) -> Result<Response<Bod
|
||||
create_bucket: *key_state.allow_create_bucket.get(),
|
||||
},
|
||||
buckets: relevant_buckets
|
||||
.into_values()
|
||||
.map(|bucket| {
|
||||
.into_iter()
|
||||
.map(|(_, bucket)| {
|
||||
let state = bucket.state.as_option().unwrap();
|
||||
KeyInfoBucketResult {
|
||||
id: hex::encode(bucket.id),
|
||||
|
||||
@@ -19,7 +19,6 @@ use opentelemetry::{
|
||||
};
|
||||
|
||||
use garage_util::error::Error as GarageError;
|
||||
use garage_util::forwarded_headers;
|
||||
use garage_util::metrics::{gen_trace_id, RecordDuration};
|
||||
|
||||
pub(crate) trait ApiEndpoint: Send + Sync + 'static {
|
||||
@@ -127,9 +126,15 @@ impl<A: ApiHandler> ApiServer<A> {
|
||||
) -> Result<Response<Body>, GarageError> {
|
||||
let uri = req.uri().clone();
|
||||
|
||||
if let Ok(forwarded_for_ip_addr) =
|
||||
forwarded_headers::handle_forwarded_for_headers(req.headers())
|
||||
{
|
||||
let has_forwarded_for_header = req.headers().contains_key("x-forwarded-for");
|
||||
if has_forwarded_for_header {
|
||||
let forwarded_for_ip_addr = &req
|
||||
.headers()
|
||||
.get("x-forwarded-for")
|
||||
.expect("Could not parse X-Forwarded-For header")
|
||||
.to_str()
|
||||
.unwrap_or_default();
|
||||
|
||||
info!(
|
||||
"{} (via {}) {} {}",
|
||||
forwarded_for_ip_addr,
|
||||
|
||||
@@ -282,8 +282,8 @@ pub(crate) async fn handle_poll_range(
|
||||
if let Some((items, seen_marker)) = resp {
|
||||
let resp = PollRangeResponse {
|
||||
items: items
|
||||
.into_values()
|
||||
.map(ReadBatchResponseItem::from)
|
||||
.into_iter()
|
||||
.map(|(_k, i)| ReadBatchResponseItem::from(i))
|
||||
.collect::<Vec<_>>(),
|
||||
seen_marker,
|
||||
};
|
||||
|
||||
+2
-2
@@ -443,7 +443,7 @@ fn body_from_blocks_range(
|
||||
// block.part_number, which is not the same in the case of a multipart upload)
|
||||
let mut blocks: Vec<(VersionBlock, u64)> = Vec::with_capacity(std::cmp::min(
|
||||
all_blocks.len(),
|
||||
4 + ((end - begin) / std::cmp::max(all_blocks[0].1.size, 1024)) as usize,
|
||||
4 + ((end - begin) / std::cmp::max(all_blocks[0].1.size as u64, 1024)) as usize,
|
||||
));
|
||||
let mut block_offset: u64 = 0;
|
||||
for (_, b) in all_blocks.iter() {
|
||||
@@ -454,7 +454,7 @@ fn body_from_blocks_range(
|
||||
if block_offset < end && block_offset + b.size > begin {
|
||||
blocks.push((*b, block_offset));
|
||||
}
|
||||
block_offset += b.size;
|
||||
block_offset += b.size as u64;
|
||||
}
|
||||
|
||||
let order_stream = OrderTag::stream();
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "garage_block"
|
||||
version = "0.8.2"
|
||||
version = "0.8.1"
|
||||
authors = ["Alex Auvolat <alex@adnab.me>"]
|
||||
edition = "2018"
|
||||
license = "AGPL-3.0"
|
||||
@@ -14,10 +14,10 @@ path = "lib.rs"
|
||||
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
|
||||
|
||||
[dependencies]
|
||||
garage_db.workspace = true
|
||||
garage_rpc.workspace = true
|
||||
garage_util.workspace = true
|
||||
garage_table.workspace = true
|
||||
garage_db = { version = "0.8.1", path = "../db" }
|
||||
garage_rpc = { version = "0.8.1", path = "../rpc" }
|
||||
garage_util = { version = "0.8.1", path = "../util" }
|
||||
garage_table = { version = "0.8.1", path = "../table" }
|
||||
|
||||
opentelemetry = "0.17"
|
||||
|
||||
@@ -37,7 +37,7 @@ serde_bytes = "0.11"
|
||||
futures = "0.3"
|
||||
futures-util = "0.3"
|
||||
tokio = { version = "1.0", default-features = false, features = ["rt", "rt-multi-thread", "io-util", "net", "time", "macros", "sync", "signal", "fs"] }
|
||||
tokio-util = { version = "0.7", features = ["io"] }
|
||||
tokio-util = { version = "0.6", features = ["io"] }
|
||||
|
||||
[features]
|
||||
system-libs = [ "zstd/pkg-config" ]
|
||||
|
||||
+5
-29
@@ -152,7 +152,6 @@ impl BlockManager {
|
||||
tx_scrub_command: ArcSwapOption::new(None),
|
||||
});
|
||||
block_manager.endpoint.set_handler(block_manager.clone());
|
||||
block_manager.scrub_persister.set_with(|_| ()).unwrap();
|
||||
|
||||
block_manager
|
||||
}
|
||||
@@ -186,9 +185,6 @@ impl BlockManager {
|
||||
vars.register_ro(&self.scrub_persister, "scrub-last-completed", |p| {
|
||||
p.get_with(|x| msec_to_rfc3339(x.time_last_complete_scrub))
|
||||
});
|
||||
vars.register_ro(&self.scrub_persister, "scrub-next-run", |p| {
|
||||
p.get_with(|x| msec_to_rfc3339(x.time_next_run_scrub))
|
||||
});
|
||||
vars.register_ro(&self.scrub_persister, "scrub-corruptions_detected", |p| {
|
||||
p.get_with(|x| x.corruptions_detected)
|
||||
});
|
||||
@@ -600,32 +596,12 @@ impl BlockManager {
|
||||
/// Utility: check if block is stored compressed. Error if block is not stored
|
||||
async fn is_block_compressed(&self, hash: &Hash) -> Result<bool, Error> {
|
||||
let mut path = self.block_path(hash);
|
||||
|
||||
// If compression is disabled on node - check for the raw block
|
||||
// first and then a compressed one (as compression may have been
|
||||
// previously enabled).
|
||||
match self.compression_level {
|
||||
None => {
|
||||
if fs::metadata(&path).await.is_ok() {
|
||||
return Ok(false);
|
||||
}
|
||||
|
||||
path.set_extension("zst");
|
||||
|
||||
fs::metadata(&path).await.map(|_| true).map_err(Into::into)
|
||||
}
|
||||
_ => {
|
||||
path.set_extension("zst");
|
||||
|
||||
if fs::metadata(&path).await.is_ok() {
|
||||
return Ok(true);
|
||||
}
|
||||
|
||||
path.set_extension("");
|
||||
|
||||
fs::metadata(&path).await.map(|_| false).map_err(Into::into)
|
||||
}
|
||||
path.set_extension("zst");
|
||||
if fs::metadata(&path).await.is_ok() {
|
||||
return Ok(true);
|
||||
}
|
||||
path.set_extension("");
|
||||
fs::metadata(&path).await.map(|_| false).map_err(Into::into)
|
||||
}
|
||||
|
||||
async fn lock_mutate(&self, hash: &Hash) -> MutexGuard<'_, BlockManagerLocked> {
|
||||
|
||||
+24
-89
@@ -4,7 +4,7 @@ use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use async_trait::async_trait;
|
||||
use rand::Rng;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use tokio::fs;
|
||||
use tokio::select;
|
||||
use tokio::sync::mpsc;
|
||||
@@ -19,8 +19,8 @@ use garage_util::tranquilizer::Tranquilizer;
|
||||
|
||||
use crate::manager::*;
|
||||
|
||||
// Full scrub every 25 days with a random element of 10 days mixed in below
|
||||
const SCRUB_INTERVAL: Duration = Duration::from_secs(3600 * 24 * 25);
|
||||
// Full scrub every 30 days
|
||||
const SCRUB_INTERVAL: Duration = Duration::from_secs(3600 * 24 * 30);
|
||||
// Scrub tranquility is initially set to 4, but can be changed in the CLI
|
||||
// and the updated version is persisted over Garage restarts
|
||||
const INITIAL_SCRUB_TRANQUILITY: u32 = 4;
|
||||
@@ -161,51 +161,6 @@ impl Worker for RepairWorker {
|
||||
// and whose parameter (esp. speed) can be controlled at runtime.
|
||||
// ---- ---- ----
|
||||
|
||||
mod v081 {
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
#[derive(Serialize, Deserialize)]
|
||||
pub struct ScrubWorkerPersisted {
|
||||
pub tranquility: u32,
|
||||
pub(crate) time_last_complete_scrub: u64,
|
||||
pub(crate) corruptions_detected: u64,
|
||||
}
|
||||
|
||||
impl garage_util::migrate::InitialFormat for ScrubWorkerPersisted {}
|
||||
}
|
||||
|
||||
mod v082 {
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use super::v081;
|
||||
|
||||
#[derive(Serialize, Deserialize)]
|
||||
pub struct ScrubWorkerPersisted {
|
||||
pub tranquility: u32,
|
||||
pub(crate) time_last_complete_scrub: u64,
|
||||
pub(crate) time_next_run_scrub: u64,
|
||||
pub(crate) corruptions_detected: u64,
|
||||
}
|
||||
|
||||
impl garage_util::migrate::Migrate for ScrubWorkerPersisted {
|
||||
type Previous = v081::ScrubWorkerPersisted;
|
||||
const VERSION_MARKER: &'static [u8] = b"G082bswp";
|
||||
|
||||
fn migrate(old: v081::ScrubWorkerPersisted) -> ScrubWorkerPersisted {
|
||||
use crate::repair::randomize_next_scrub_run_time;
|
||||
|
||||
ScrubWorkerPersisted {
|
||||
tranquility: old.tranquility,
|
||||
time_last_complete_scrub: old.time_last_complete_scrub,
|
||||
time_next_run_scrub: randomize_next_scrub_run_time(old.time_last_complete_scrub),
|
||||
corruptions_detected: old.corruptions_detected,
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub use v082::*;
|
||||
|
||||
pub struct ScrubWorker {
|
||||
manager: Arc<BlockManager>,
|
||||
rx_cmd: mpsc::Receiver<ScrubWorkerCommand>,
|
||||
@@ -216,37 +171,35 @@ pub struct ScrubWorker {
|
||||
persister: PersisterShared<ScrubWorkerPersisted>,
|
||||
}
|
||||
|
||||
fn randomize_next_scrub_run_time(timestamp: u64) -> u64 {
|
||||
// Take SCRUB_INTERVAL and mix in a random interval of 10 days to attempt to
|
||||
// balance scrub load across different cluster nodes.
|
||||
|
||||
timestamp
|
||||
+ SCRUB_INTERVAL
|
||||
.saturating_add(Duration::from_secs(
|
||||
rand::thread_rng().gen_range(0..3600 * 24 * 10),
|
||||
))
|
||||
.as_millis() as u64
|
||||
#[derive(Serialize, Deserialize)]
|
||||
pub struct ScrubWorkerPersisted {
|
||||
pub tranquility: u32,
|
||||
pub(crate) time_last_complete_scrub: u64,
|
||||
pub(crate) corruptions_detected: u64,
|
||||
}
|
||||
|
||||
impl garage_util::migrate::InitialFormat for ScrubWorkerPersisted {}
|
||||
impl Default for ScrubWorkerPersisted {
|
||||
fn default() -> Self {
|
||||
ScrubWorkerPersisted {
|
||||
time_last_complete_scrub: 0,
|
||||
time_next_run_scrub: randomize_next_scrub_run_time(now_msec()),
|
||||
tranquility: INITIAL_SCRUB_TRANQUILITY,
|
||||
corruptions_detected: 0,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
enum ScrubWorkerState {
|
||||
Running(BlockStoreIterator),
|
||||
Paused(BlockStoreIterator, u64), // u64 = time when to resume scrub
|
||||
#[default]
|
||||
Finished,
|
||||
}
|
||||
|
||||
impl Default for ScrubWorkerState {
|
||||
fn default() -> Self {
|
||||
ScrubWorkerState::Finished
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub enum ScrubWorkerCommand {
|
||||
Start,
|
||||
@@ -275,7 +228,6 @@ impl ScrubWorker {
|
||||
ScrubWorkerCommand::Start => {
|
||||
self.work = match std::mem::take(&mut self.work) {
|
||||
ScrubWorkerState::Finished => {
|
||||
info!("Scrub worker initializing, now performing datastore scrub");
|
||||
let iterator = BlockStoreIterator::new(&self.manager);
|
||||
ScrubWorkerState::Running(iterator)
|
||||
}
|
||||
@@ -327,13 +279,12 @@ impl Worker for ScrubWorker {
|
||||
}
|
||||
|
||||
fn status(&self) -> WorkerStatus {
|
||||
let (corruptions_detected, tranquility, time_last_complete_scrub, time_next_run_scrub) =
|
||||
let (corruptions_detected, tranquility, time_last_complete_scrub) =
|
||||
self.persister.get_with(|p| {
|
||||
(
|
||||
p.corruptions_detected,
|
||||
p.tranquility,
|
||||
p.time_last_complete_scrub,
|
||||
p.time_next_run_scrub,
|
||||
)
|
||||
});
|
||||
|
||||
@@ -351,16 +302,10 @@ impl Worker for ScrubWorker {
|
||||
s.freeform = vec![format!("Scrub paused, resumes at {}", msec_to_rfc3339(*rt))];
|
||||
}
|
||||
ScrubWorkerState::Finished => {
|
||||
s.freeform = vec![
|
||||
format!(
|
||||
"Last scrub completed at {}",
|
||||
msec_to_rfc3339(time_last_complete_scrub),
|
||||
),
|
||||
format!(
|
||||
"Next scrub scheduled for {}",
|
||||
msec_to_rfc3339(time_next_run_scrub)
|
||||
),
|
||||
];
|
||||
s.freeform = vec![format!(
|
||||
"Last scrub completed at {}",
|
||||
msec_to_rfc3339(time_last_complete_scrub)
|
||||
)];
|
||||
}
|
||||
}
|
||||
s
|
||||
@@ -389,21 +334,10 @@ impl Worker for ScrubWorker {
|
||||
.tranquilizer
|
||||
.tranquilize_worker(self.persister.get_with(|p| p.tranquility)))
|
||||
} else {
|
||||
let now = now_msec();
|
||||
let next_scrub_timestamp = randomize_next_scrub_run_time(now);
|
||||
|
||||
self.persister.set_with(|p| {
|
||||
p.time_last_complete_scrub = now;
|
||||
p.time_next_run_scrub = next_scrub_timestamp;
|
||||
})?;
|
||||
self.persister
|
||||
.set_with(|p| p.time_last_complete_scrub = now_msec())?;
|
||||
self.work = ScrubWorkerState::Finished;
|
||||
self.tranquilizer.clear();
|
||||
|
||||
info!(
|
||||
"Datastore scrub completed, next scrub scheduled for {}",
|
||||
msec_to_rfc3339(next_scrub_timestamp)
|
||||
);
|
||||
|
||||
Ok(WorkerState::Idle)
|
||||
}
|
||||
}
|
||||
@@ -416,7 +350,8 @@ impl Worker for ScrubWorker {
|
||||
ScrubWorkerState::Running(_) => return WorkerState::Busy,
|
||||
ScrubWorkerState::Paused(_, resume_time) => (*resume_time, ScrubWorkerCommand::Resume),
|
||||
ScrubWorkerState::Finished => (
|
||||
self.persister.get_with(|p| p.time_next_run_scrub),
|
||||
self.persister.get_with(|p| p.time_last_complete_scrub)
|
||||
+ SCRUB_INTERVAL.as_millis() as u64,
|
||||
ScrubWorkerCommand::Start,
|
||||
),
|
||||
};
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "garage_db"
|
||||
version = "0.8.2"
|
||||
version = "0.8.1"
|
||||
authors = ["Alex Auvolat <alex@adnab.me>"]
|
||||
edition = "2018"
|
||||
license = "AGPL-3.0"
|
||||
|
||||
@@ -349,6 +349,6 @@ pub fn recommended_map_size() -> usize {
|
||||
|
||||
#[cfg(target_pointer_width = "32")]
|
||||
pub fn recommended_map_size() -> usize {
|
||||
tracing::warn!("LMDB is not recommended on 32-bit systems, database size will be limited");
|
||||
warn!("LMDB is not recommended on 32-bit systems, database size will be limited");
|
||||
1usize << 30
|
||||
}
|
||||
|
||||
@@ -1,12 +0,0 @@
|
||||
[package]
|
||||
name = "format_table"
|
||||
version = "0.1.1"
|
||||
authors = ["Alex Auvolat <alex@adnab.me>"]
|
||||
edition = "2018"
|
||||
license = "AGPL-3.0"
|
||||
description = "Format tables with a stupid API"
|
||||
repository = "https://git.deuxfleurs.fr/Deuxfleurs/garage"
|
||||
readme = "README.md"
|
||||
|
||||
[lib]
|
||||
path = "lib.rs"
|
||||
@@ -1,13 +0,0 @@
|
||||
# `format_table`
|
||||
|
||||
Format tables with a stupid API. [Documentation](https://docs.rs/format_table).
|
||||
|
||||
Example:
|
||||
|
||||
```rust
|
||||
let mut table = vec!["product\tquantity\tprice".to_string()];
|
||||
for (p, q, r) in [("tomato", 12, 15), ("potato", 10, 20), ("rice", 5, 12)] {
|
||||
table.push(format!("{}\t{}\t{}", p, q, r));
|
||||
}
|
||||
format_table::format_table(table);
|
||||
```
|
||||
+10
-12
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "garage"
|
||||
version = "0.8.2"
|
||||
version = "0.8.1"
|
||||
authors = ["Alex Auvolat <alex@adnab.me>"]
|
||||
edition = "2018"
|
||||
license = "AGPL-3.0"
|
||||
@@ -21,20 +21,19 @@ path = "tests/lib.rs"
|
||||
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
|
||||
|
||||
[dependencies]
|
||||
format_table.workspace = true
|
||||
garage_db.workspace = true
|
||||
garage_api.workspace = true
|
||||
garage_block.workspace = true
|
||||
garage_model.workspace = true
|
||||
garage_rpc.workspace = true
|
||||
garage_table.workspace = true
|
||||
garage_util.workspace = true
|
||||
garage_web.workspace = true
|
||||
garage_db = { version = "0.8.1", path = "../db" }
|
||||
garage_api = { version = "0.8.1", path = "../api" }
|
||||
garage_block = { version = "0.8.1", path = "../block" }
|
||||
garage_model = { version = "0.8.1", path = "../model" }
|
||||
garage_rpc = { version = "0.8.1", path = "../rpc" }
|
||||
garage_table = { version = "0.8.1", path = "../table" }
|
||||
garage_util = { version = "0.8.1", path = "../util" }
|
||||
garage_web = { version = "0.8.1", path = "../web" }
|
||||
|
||||
backtrace = "0.3"
|
||||
bytes = "1.0"
|
||||
bytesize = "1.1"
|
||||
timeago = { version = "0.4", default-features = false }
|
||||
timeago = "0.4"
|
||||
parse_duration = "2.1"
|
||||
hex = "0.4"
|
||||
tracing = { version = "0.1" }
|
||||
@@ -42,7 +41,6 @@ tracing-subscriber = { version = "0.3", features = ["env-filter"] }
|
||||
rand = "0.8"
|
||||
async-trait = "0.1.7"
|
||||
sodiumoxide = { version = "0.2.5-0", package = "kuska-sodiumoxide" }
|
||||
git-version = "0.3.4"
|
||||
|
||||
serde = { version = "1.0", default-features = false, features = ["derive", "rc"] }
|
||||
serde_bytes = "0.11"
|
||||
|
||||
+2
-3
@@ -5,11 +5,11 @@ use std::sync::Arc;
|
||||
use async_trait::async_trait;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use format_table::format_table_to_string;
|
||||
use garage_util::background::BackgroundRunner;
|
||||
use garage_util::crdt::*;
|
||||
use garage_util::data::*;
|
||||
use garage_util::error::Error as GarageError;
|
||||
use garage_util::formater::format_table_to_string;
|
||||
use garage_util::time::*;
|
||||
|
||||
use garage_table::replication::*;
|
||||
@@ -820,12 +820,11 @@ impl AdminRpcHandler {
|
||||
let mut ret = String::new();
|
||||
writeln!(
|
||||
&mut ret,
|
||||
"\nGarage version: {} [features: {}]\nRust compiler version: {}",
|
||||
"\nGarage version: {} [features: {}]",
|
||||
garage_util::version::garage_version(),
|
||||
garage_util::version::garage_features()
|
||||
.map(|list| list.join(", "))
|
||||
.unwrap_or_else(|| "(unknown)".into()),
|
||||
garage_util::version::rust_version(),
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
use std::collections::HashSet;
|
||||
use std::time::Duration;
|
||||
|
||||
use format_table::format_table;
|
||||
use garage_util::error::*;
|
||||
use garage_util::formater::format_table;
|
||||
|
||||
use garage_rpc::layout::*;
|
||||
use garage_rpc::system::*;
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
use format_table::format_table;
|
||||
use garage_util::crdt::Crdt;
|
||||
use garage_util::error::*;
|
||||
use garage_util::formater::format_table;
|
||||
|
||||
use garage_rpc::layout::*;
|
||||
use garage_rpc::system::*;
|
||||
|
||||
@@ -1,11 +1,11 @@
|
||||
use std::collections::HashMap;
|
||||
use std::time::Duration;
|
||||
|
||||
use format_table::format_table;
|
||||
use garage_util::background::*;
|
||||
use garage_util::crdt::*;
|
||||
use garage_util::data::*;
|
||||
use garage_util::error::*;
|
||||
use garage_util::formater::format_table;
|
||||
use garage_util::time::*;
|
||||
|
||||
use garage_block::manager::BlockResyncErrorInfo;
|
||||
|
||||
@@ -108,12 +108,6 @@ async fn main() {
|
||||
][..];
|
||||
if let Some(git_version) = option_env!("GIT_VERSION") {
|
||||
garage_util::version::init_version(git_version);
|
||||
} else {
|
||||
garage_util::version::init_version(git_version::git_version!(
|
||||
prefix = "git:",
|
||||
cargo_prefix = "cargo:",
|
||||
fallback = "unknown"
|
||||
));
|
||||
}
|
||||
garage_util::version::init_features(features);
|
||||
|
||||
|
||||
@@ -21,7 +21,14 @@ async fn test_admin_bucket_perms() {
|
||||
|
||||
ctx.garage
|
||||
.command()
|
||||
.args(["bucket", "allow", "--read", "--key", &ctx.key.id, BCKT_NAME])
|
||||
.args([
|
||||
"bucket",
|
||||
"allow",
|
||||
"--read",
|
||||
"--key",
|
||||
&ctx.garage.key.id,
|
||||
BCKT_NAME,
|
||||
])
|
||||
.quiet()
|
||||
.expect_success_status("Could not create bucket");
|
||||
|
||||
@@ -29,7 +36,14 @@ async fn test_admin_bucket_perms() {
|
||||
|
||||
ctx.garage
|
||||
.command()
|
||||
.args(["bucket", "deny", "--read", "--key", &ctx.key.id, BCKT_NAME])
|
||||
.args([
|
||||
"bucket",
|
||||
"deny",
|
||||
"--read",
|
||||
"--key",
|
||||
&ctx.garage.key.name,
|
||||
BCKT_NAME,
|
||||
])
|
||||
.quiet()
|
||||
.expect_success_status("Could not create bucket");
|
||||
|
||||
@@ -37,7 +51,14 @@ async fn test_admin_bucket_perms() {
|
||||
|
||||
ctx.garage
|
||||
.command()
|
||||
.args(["bucket", "allow", "--read", "--key", &ctx.key.id, BCKT_NAME])
|
||||
.args([
|
||||
"bucket",
|
||||
"allow",
|
||||
"--read",
|
||||
"--key",
|
||||
&ctx.garage.key.name,
|
||||
BCKT_NAME,
|
||||
])
|
||||
.quiet()
|
||||
.expect_success_status("Could not create bucket");
|
||||
|
||||
|
||||
@@ -13,7 +13,7 @@ async fn test_bucket_all() {
|
||||
ctx.garage
|
||||
.command()
|
||||
.args(["key", "deny"])
|
||||
.args(["--create-bucket", &ctx.key.id])
|
||||
.args(["--create-bucket", &ctx.garage.key.id])
|
||||
.quiet()
|
||||
.expect_success_output("Could not deny key to create buckets");
|
||||
|
||||
@@ -26,7 +26,7 @@ async fn test_bucket_all() {
|
||||
ctx.garage
|
||||
.command()
|
||||
.args(["key", "allow"])
|
||||
.args(["--create-bucket", &ctx.key.id])
|
||||
.args(["--create-bucket", &ctx.garage.key.id])
|
||||
.quiet()
|
||||
.expect_success_output("Could not deny key to create buckets");
|
||||
|
||||
|
||||
@@ -1,9 +1,15 @@
|
||||
use aws_sdk_s3::{Client, Config, Credentials, Endpoint};
|
||||
|
||||
use super::garage::{Instance, Key};
|
||||
use super::garage::Instance;
|
||||
|
||||
pub fn build_client(instance: &Instance, key: &Key) -> Client {
|
||||
let credentials = Credentials::new(&key.id, &key.secret, None, None, "garage-integ-test");
|
||||
pub fn build_client(instance: &Instance) -> Client {
|
||||
let credentials = Credentials::new(
|
||||
&instance.key.id,
|
||||
&instance.key.secret,
|
||||
None,
|
||||
None,
|
||||
"garage-integ-test",
|
||||
);
|
||||
let endpoint = Endpoint::immutable(instance.s3_uri());
|
||||
|
||||
let config = Config::builder()
|
||||
|
||||
@@ -14,7 +14,6 @@ use garage_api::signature;
|
||||
/// You should ever only use this to send requests AWS sdk won't send,
|
||||
/// like to reproduce behavior of unusual implementations found to be
|
||||
/// problematic.
|
||||
#[derive(Clone)]
|
||||
pub struct CustomRequester {
|
||||
key: Key,
|
||||
uri: Uri,
|
||||
@@ -23,18 +22,18 @@ pub struct CustomRequester {
|
||||
}
|
||||
|
||||
impl CustomRequester {
|
||||
pub fn new_s3(instance: &Instance, key: &Key) -> Self {
|
||||
pub fn new_s3(instance: &Instance) -> Self {
|
||||
CustomRequester {
|
||||
key: key.clone(),
|
||||
key: instance.key.clone(),
|
||||
uri: instance.s3_uri(),
|
||||
service: "s3",
|
||||
client: Client::new(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn new_k2v(instance: &Instance, key: &Key) -> Self {
|
||||
pub fn new_k2v(instance: &Instance) -> Self {
|
||||
CustomRequester {
|
||||
key: key.clone(),
|
||||
key: instance.key.clone(),
|
||||
uri: instance.k2v_uri(),
|
||||
service: "k2v",
|
||||
client: Client::new(),
|
||||
|
||||
@@ -13,7 +13,7 @@ static GARAGE_TEST_SECRET: &str =
|
||||
|
||||
#[derive(Debug, Default, Clone)]
|
||||
pub struct Key {
|
||||
pub name: Option<String>,
|
||||
pub name: String,
|
||||
pub id: String,
|
||||
pub secret: String,
|
||||
}
|
||||
@@ -21,7 +21,7 @@ pub struct Key {
|
||||
pub struct Instance {
|
||||
process: process::Child,
|
||||
pub path: PathBuf,
|
||||
pub default_key: Key,
|
||||
pub key: Key,
|
||||
pub s3_port: u16,
|
||||
pub k2v_port: u16,
|
||||
pub web_port: u16,
|
||||
@@ -102,7 +102,7 @@ api_bind_addr = "127.0.0.1:{admin_port}"
|
||||
Instance {
|
||||
process: child,
|
||||
path,
|
||||
default_key: Key::default(),
|
||||
key: Key::default(),
|
||||
s3_port: port,
|
||||
k2v_port: port + 1,
|
||||
web_port: port + 3,
|
||||
@@ -111,27 +111,14 @@ api_bind_addr = "127.0.0.1:{admin_port}"
|
||||
}
|
||||
|
||||
fn setup(&mut self) {
|
||||
self.wait_for_boot();
|
||||
self.setup_layout();
|
||||
self.default_key = self.key(Some("garage_test"));
|
||||
}
|
||||
|
||||
fn wait_for_boot(&mut self) {
|
||||
use std::{thread, time::Duration};
|
||||
|
||||
// 60 * 2 seconds = 120 seconds = 2min
|
||||
for _ in 0..60 {
|
||||
let termination = self
|
||||
.command()
|
||||
.args(["status"])
|
||||
.quiet()
|
||||
.status()
|
||||
.expect("Unable to run command");
|
||||
if termination.success() {
|
||||
break;
|
||||
}
|
||||
thread::sleep(Duration::from_secs(2));
|
||||
}
|
||||
// Wait for node to be ready
|
||||
thread::sleep(Duration::from_secs(2));
|
||||
|
||||
self.setup_layout();
|
||||
|
||||
self.key = self.new_key("garage_test");
|
||||
}
|
||||
|
||||
fn setup_layout(&self) {
|
||||
@@ -182,17 +169,14 @@ api_bind_addr = "127.0.0.1:{admin_port}"
|
||||
.expect("Could not build garage endpoint URI")
|
||||
}
|
||||
|
||||
pub fn key(&self, maybe_name: Option<&str>) -> Key {
|
||||
pub fn new_key(&self, name: &str) -> Key {
|
||||
let mut key = Key::default();
|
||||
|
||||
let mut cmd = self.command();
|
||||
let base = cmd.args(["key", "new"]);
|
||||
let with_name = match maybe_name {
|
||||
Some(name) => base.args(["--name", name]),
|
||||
None => base,
|
||||
};
|
||||
|
||||
let output = with_name.expect_success_output("Could not create key");
|
||||
let output = self
|
||||
.command()
|
||||
.args(["key", "new"])
|
||||
.args(["--name", name])
|
||||
.expect_success_output("Could not create key");
|
||||
let stdout = String::from_utf8(output.stdout).unwrap();
|
||||
|
||||
for line in stdout.lines() {
|
||||
@@ -209,7 +193,7 @@ api_bind_addr = "127.0.0.1:{admin_port}"
|
||||
assert!(!key.secret.is_empty(), "Invalid key: Key secret is empty");
|
||||
|
||||
Key {
|
||||
name: maybe_name.map(String::from),
|
||||
name: name.to_owned(),
|
||||
..key
|
||||
}
|
||||
}
|
||||
|
||||
@@ -13,16 +13,13 @@ use custom_requester::CustomRequester;
|
||||
|
||||
const REGION: Region = Region::from_static("garage-integ-test");
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct Context {
|
||||
pub garage: &'static garage::Instance,
|
||||
pub key: garage::Key,
|
||||
pub client: Client,
|
||||
pub custom_request: CustomRequester,
|
||||
pub k2v: K2VContext,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct K2VContext {
|
||||
pub request: CustomRequester,
|
||||
}
|
||||
@@ -30,15 +27,13 @@ pub struct K2VContext {
|
||||
impl Context {
|
||||
fn new() -> Self {
|
||||
let garage = garage::instance();
|
||||
let key = garage.key(None);
|
||||
let client = client::build_client(garage, &key);
|
||||
let custom_request = CustomRequester::new_s3(garage, &key);
|
||||
let k2v_request = CustomRequester::new_k2v(garage, &key);
|
||||
let client = client::build_client(garage);
|
||||
let custom_request = CustomRequester::new_s3(garage);
|
||||
let k2v_request = CustomRequester::new_k2v(garage);
|
||||
|
||||
Context {
|
||||
garage,
|
||||
client,
|
||||
key,
|
||||
custom_request,
|
||||
k2v: K2VContext {
|
||||
request: k2v_request,
|
||||
@@ -62,7 +57,7 @@ impl Context {
|
||||
.args(["bucket", "allow"])
|
||||
.args(["--owner", "--read", "--write"])
|
||||
.arg(&bucket_name)
|
||||
.args(["--key", &self.key.id])
|
||||
.args(["--key", &self.garage.key.name])
|
||||
.quiet()
|
||||
.expect_success_status("Could not allow key for bucket");
|
||||
|
||||
|
||||
@@ -57,8 +57,8 @@ async fn test_poll_item() {
|
||||
let poll = {
|
||||
let bucket = bucket.clone();
|
||||
let ct = ct.clone();
|
||||
let ctx = ctx.clone();
|
||||
tokio::spawn(async move {
|
||||
let ctx = common::context();
|
||||
ctx.k2v
|
||||
.request
|
||||
.builder(bucket.clone())
|
||||
@@ -171,8 +171,8 @@ async fn test_poll_range() {
|
||||
// Second poll range, which will complete later
|
||||
let poll = {
|
||||
let bucket = bucket.clone();
|
||||
let ctx = ctx.clone();
|
||||
tokio::spawn(async move {
|
||||
let ctx = common::context();
|
||||
ctx.k2v
|
||||
.request
|
||||
.builder(bucket.clone())
|
||||
@@ -220,8 +220,8 @@ async fn test_poll_range() {
|
||||
// Start a new poll operation
|
||||
let poll = {
|
||||
let bucket = bucket.clone();
|
||||
let ctx = ctx.clone();
|
||||
tokio::spawn(async move {
|
||||
let ctx = common::context();
|
||||
ctx.k2v
|
||||
.request
|
||||
.builder(bucket.clone())
|
||||
|
||||
@@ -109,7 +109,7 @@ async fn test_create_bucket_streaming() {
|
||||
ctx.garage
|
||||
.command()
|
||||
.args(["key", "allow"])
|
||||
.args(["--create-bucket", &ctx.key.id])
|
||||
.args(["--create-bucket", &ctx.garage.key.id])
|
||||
.quiet()
|
||||
.expect_success_output("Could not allow key to create buckets");
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "k2v-client"
|
||||
version = "0.0.2"
|
||||
version = "0.1.1"
|
||||
authors = ["Trinity Pointard <trinity.pointard@gmail.com>", "Alex Auvolat <alex@adnab.me>"]
|
||||
edition = "2018"
|
||||
license = "AGPL-3.0"
|
||||
@@ -19,15 +19,15 @@ hyper-rustls = { version = "0.23", default-features = false, features = [ "http1
|
||||
serde = "1.0"
|
||||
serde_json = "1.0"
|
||||
thiserror = "1.0"
|
||||
tokio = { version = "1.0", default-features = false, features = ["rt", "rt-multi-thread", "io-util", "net", "time", "macros", "sync", "signal", "fs"] }
|
||||
tokio = "1.24"
|
||||
|
||||
# cli deps
|
||||
clap = { version = "4.1", optional = true, features = ["derive", "env"] }
|
||||
format_table = { workspace = true, optional = true }
|
||||
garage_util = { version = "0.8.1", path = "../util", optional = true }
|
||||
|
||||
|
||||
[features]
|
||||
cli = ["clap", "tokio/fs", "tokio/io-std", "format_table"]
|
||||
cli = ["clap", "tokio/fs", "tokio/io-std", "garage_util"]
|
||||
|
||||
[lib]
|
||||
path = "lib.rs"
|
||||
|
||||
@@ -2,11 +2,9 @@ use std::collections::BTreeMap;
|
||||
use std::process::exit;
|
||||
use std::time::Duration;
|
||||
|
||||
use base64::prelude::*;
|
||||
|
||||
use k2v_client::*;
|
||||
|
||||
use format_table::format_table;
|
||||
use garage_util::formater::format_table;
|
||||
|
||||
use rusoto_core::credential::AwsCredentials;
|
||||
use rusoto_core::Region;
|
||||
@@ -157,9 +155,7 @@ impl Value {
|
||||
if let Some(ref text) = self.text {
|
||||
Ok(text.as_bytes().to_vec())
|
||||
} else if let Some(ref b64) = self.b64 {
|
||||
BASE64_STANDARD
|
||||
.decode(b64)
|
||||
.map_err(|_| Error::Message("invalid base64 input".into()))
|
||||
base64::decode(b64).map_err(|_| Error::Message("invalid base64 input".into()))
|
||||
} else if let Some(ref path) = self.file {
|
||||
use tokio::io::AsyncReadExt;
|
||||
if path == "-" {
|
||||
@@ -234,7 +230,7 @@ impl ReadOutputKind {
|
||||
for val in val.value {
|
||||
match val {
|
||||
K2vValue::Value(v) => {
|
||||
println!("{}", BASE64_STANDARD.encode(&v))
|
||||
println!("{}", base64::encode(&v))
|
||||
}
|
||||
K2vValue::Tombstone => {
|
||||
println!();
|
||||
@@ -253,7 +249,7 @@ impl ReadOutputKind {
|
||||
if let Ok(string) = std::str::from_utf8(&v) {
|
||||
println!(" utf-8: {}", string);
|
||||
} else {
|
||||
println!(" base64: {}", BASE64_STANDARD.encode(&v));
|
||||
println!(" base64: {}", base64::encode(&v));
|
||||
}
|
||||
}
|
||||
K2vValue::Tombstone => {
|
||||
@@ -279,7 +275,7 @@ struct BatchOutputKind {
|
||||
impl BatchOutputKind {
|
||||
fn display_human_output(&self, values: BTreeMap<String, CausalValue>) -> ! {
|
||||
for (key, values) in values {
|
||||
println!("sort_key: {}", key);
|
||||
println!("key: {}", key);
|
||||
let causality: String = values.causality.into();
|
||||
println!("causality: {}", causality);
|
||||
for value in values.value {
|
||||
@@ -288,7 +284,7 @@ impl BatchOutputKind {
|
||||
if let Ok(string) = std::str::from_utf8(&v) {
|
||||
println!(" value(utf-8): {}", string);
|
||||
} else {
|
||||
println!(" value(base64): {}", BASE64_STANDARD.encode(&v));
|
||||
println!(" value(base64): {}", base64::encode(&v));
|
||||
}
|
||||
}
|
||||
K2vValue::Tombstone => {
|
||||
@@ -524,7 +520,7 @@ async fn main() -> Result<(), Error> {
|
||||
value
|
||||
.as_object_mut()
|
||||
.unwrap()
|
||||
.insert("partition_key".to_owned(), k.into());
|
||||
.insert("sort_key".to_owned(), k.into());
|
||||
value
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
@@ -541,7 +537,7 @@ async fn main() -> Result<(), Error> {
|
||||
}
|
||||
|
||||
let mut to_print = Vec::new();
|
||||
to_print.push(format!("partition_key\tentries\tconflicts\tvalues\tbytes"));
|
||||
to_print.push(format!("key:\tentries\tconflicts\tvalues\tbytes"));
|
||||
for (k, v) in res.items {
|
||||
to_print.push(format!(
|
||||
"{}\t{}\t{}\t{}\t{}",
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
use std::collections::BTreeMap;
|
||||
use std::time::Duration;
|
||||
|
||||
use base64::prelude::*;
|
||||
use http::header::{ACCEPT, CONTENT_LENGTH, CONTENT_TYPE};
|
||||
use http::status::StatusCode;
|
||||
use http::HeaderMap;
|
||||
@@ -376,7 +375,7 @@ impl K2vClient {
|
||||
.unwrap_or_default();
|
||||
let err_body_str = std::str::from_utf8(&err_body)
|
||||
.map(String::from)
|
||||
.unwrap_or_else(|_| BASE64_STANDARD.encode(&err_body));
|
||||
.unwrap_or_else(|_| base64::encode(&err_body));
|
||||
|
||||
if s.is_client_error() || s.is_server_error() {
|
||||
error!("Error response {}: {}", res.status, err_body_str);
|
||||
@@ -409,7 +408,7 @@ impl K2vClient {
|
||||
"Response body: {}",
|
||||
std::str::from_utf8(&body)
|
||||
.map(String::from)
|
||||
.unwrap_or_else(|_| BASE64_STANDARD.encode(&body))
|
||||
.unwrap_or_else(|_| base64::encode(&body))
|
||||
);
|
||||
|
||||
Ok(Response {
|
||||
@@ -483,11 +482,9 @@ impl<'de> Deserialize<'de> for K2vValue {
|
||||
{
|
||||
let val: Option<&str> = Option::deserialize(d)?;
|
||||
Ok(match val {
|
||||
Some(s) => K2vValue::Value(
|
||||
BASE64_STANDARD
|
||||
.decode(s)
|
||||
.map_err(|_| DeError::custom("invalid base64"))?,
|
||||
),
|
||||
Some(s) => {
|
||||
K2vValue::Value(base64::decode(s).map_err(|_| DeError::custom("invalid base64"))?)
|
||||
}
|
||||
None => K2vValue::Tombstone,
|
||||
})
|
||||
}
|
||||
@@ -501,7 +498,7 @@ impl Serialize for K2vValue {
|
||||
match self {
|
||||
K2vValue::Tombstone => serializer.serialize_none(),
|
||||
K2vValue::Value(v) => {
|
||||
let b64 = BASE64_STANDARD.encode(v);
|
||||
let b64 = base64::encode(v);
|
||||
serializer.serialize_str(&b64)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "garage_model"
|
||||
version = "0.8.2"
|
||||
version = "0.8.1"
|
||||
authors = ["Alex Auvolat <alex@adnab.me>"]
|
||||
edition = "2018"
|
||||
license = "AGPL-3.0"
|
||||
@@ -14,11 +14,11 @@ path = "lib.rs"
|
||||
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
|
||||
|
||||
[dependencies]
|
||||
garage_db.workspace = true
|
||||
garage_rpc.workspace = true
|
||||
garage_table.workspace = true
|
||||
garage_block.workspace = true
|
||||
garage_util.workspace = true
|
||||
garage_db = { version = "0.8.1", default-features = false, path = "../db" }
|
||||
garage_rpc = { version = "0.8.1", path = "../rpc" }
|
||||
garage_table = { version = "0.8.1", path = "../table" }
|
||||
garage_block = { version = "0.8.1", path = "../block" }
|
||||
garage_util = { version = "0.8.1", path = "../util" }
|
||||
|
||||
async-trait = "0.1.7"
|
||||
arc-swap = "1.0"
|
||||
|
||||
+3
-10
@@ -136,16 +136,9 @@ impl Garage {
|
||||
env_builder.flag(heed::flags::Flags::MdbNoSync);
|
||||
env_builder.flag(heed::flags::Flags::MdbNoMetaSync);
|
||||
}
|
||||
let db = match env_builder.open(&db_path) {
|
||||
Err(heed::Error::Io(e)) if e.kind() == std::io::ErrorKind::OutOfMemory => {
|
||||
return Err(Error::Message(
|
||||
"OutOfMemory error while trying to open LMDB database. This can happen \
|
||||
if your operating system is not allowing you to use sufficient virtual \
|
||||
memory address space. Please check that no limit is set (ulimit -v). \
|
||||
On 32-bit machines, you should probably switch to another database engine.".into()))
|
||||
}
|
||||
x => x.ok_or_message("Unable to open LMDB DB")?,
|
||||
};
|
||||
let db = env_builder
|
||||
.open(&db_path)
|
||||
.ok_or_message("Unable to open LMDB DB")?;
|
||||
db::lmdb_adapter::LmdbDb::init(db)
|
||||
}
|
||||
#[cfg(not(feature = "lmdb"))]
|
||||
|
||||
@@ -269,6 +269,7 @@ impl CountedItem for K2VItem {
|
||||
&self.partition.partition_key
|
||||
}
|
||||
|
||||
#[allow(clippy::bool_to_int_with_if)]
|
||||
fn counts(&self) -> Vec<(&'static str, i64)> {
|
||||
let values = self.values();
|
||||
|
||||
|
||||
@@ -79,7 +79,7 @@ impl RangeSeenMarker {
|
||||
|
||||
let bytes = nonversioned_encode(&self)?;
|
||||
let bytes = zstd::stream::encode_all(&mut &bytes[..], zstd::DEFAULT_COMPRESSION_LEVEL)?;
|
||||
Ok(BASE64_STANDARD.encode(bytes))
|
||||
Ok(BASE64_STANDARD.encode(&bytes))
|
||||
}
|
||||
|
||||
/// Decode from msgpack+zstd+b64 representation, returns None on error.
|
||||
|
||||
@@ -355,6 +355,7 @@ impl CountedItem for Object {
|
||||
|
||||
fn counts(&self) -> Vec<(&'static str, i64)> {
|
||||
let versions = self.versions();
|
||||
#[allow(clippy::bool_to_int_with_if)]
|
||||
let n_objects = if versions.iter().any(|v| v.is_data()) {
|
||||
1
|
||||
} else {
|
||||
|
||||
+2
-2
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "garage_rpc"
|
||||
version = "0.8.2"
|
||||
version = "0.8.1"
|
||||
authors = ["Alex Auvolat <alex@adnab.me>"]
|
||||
edition = "2018"
|
||||
license = "AGPL-3.0"
|
||||
@@ -14,7 +14,7 @@ path = "lib.rs"
|
||||
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
|
||||
|
||||
[dependencies]
|
||||
garage_util.workspace = true
|
||||
garage_util = { version = "0.8.1", path = "../util" }
|
||||
|
||||
arc-swap = "1.0"
|
||||
bytes = "1.0"
|
||||
|
||||
@@ -31,10 +31,10 @@ impl SystemMetrics {
|
||||
.u64_value_observer("garage_build_info", move |observer| {
|
||||
observer.observe(
|
||||
1,
|
||||
&[
|
||||
KeyValue::new("rustversion", garage_util::version::rust_version()),
|
||||
KeyValue::new("version", garage_util::version::garage_version()),
|
||||
],
|
||||
&[KeyValue::new(
|
||||
"version",
|
||||
garage_util::version::garage_version(),
|
||||
)],
|
||||
)
|
||||
})
|
||||
.with_description("Garage build info")
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "garage_table"
|
||||
version = "0.8.2"
|
||||
version = "0.8.1"
|
||||
authors = ["Alex Auvolat <alex@adnab.me>"]
|
||||
edition = "2018"
|
||||
license = "AGPL-3.0"
|
||||
@@ -14,9 +14,9 @@ path = "lib.rs"
|
||||
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
|
||||
|
||||
[dependencies]
|
||||
garage_db.workspace = true
|
||||
garage_rpc.workspace = true
|
||||
garage_util.workspace = true
|
||||
garage_db = { version = "0.8.1", path = "../db" }
|
||||
garage_rpc = { version = "0.8.1", path = "../rpc" }
|
||||
garage_util = { version = "0.8.1", path = "../util" }
|
||||
|
||||
opentelemetry = "0.17"
|
||||
|
||||
|
||||
+8
-8
@@ -44,22 +44,22 @@ pub struct TableData<F: TableSchema, R: TableReplication> {
|
||||
impl<F: TableSchema, R: TableReplication> TableData<F, R> {
|
||||
pub fn new(system: Arc<System>, instance: F, replication: R, db: &db::Db) -> Arc<Self> {
|
||||
let store = db
|
||||
.open_tree(format!("{}:table", F::TABLE_NAME))
|
||||
.open_tree(&format!("{}:table", F::TABLE_NAME))
|
||||
.expect("Unable to open DB tree");
|
||||
|
||||
let merkle_tree = db
|
||||
.open_tree(format!("{}:merkle_tree", F::TABLE_NAME))
|
||||
.open_tree(&format!("{}:merkle_tree", F::TABLE_NAME))
|
||||
.expect("Unable to open DB Merkle tree tree");
|
||||
let merkle_todo = db
|
||||
.open_tree(format!("{}:merkle_todo", F::TABLE_NAME))
|
||||
.open_tree(&format!("{}:merkle_todo", F::TABLE_NAME))
|
||||
.expect("Unable to open DB Merkle TODO tree");
|
||||
|
||||
let insert_queue = db
|
||||
.open_tree(format!("{}:insert_queue", F::TABLE_NAME))
|
||||
.open_tree(&format!("{}:insert_queue", F::TABLE_NAME))
|
||||
.expect("Unable to open insert queue DB tree");
|
||||
|
||||
let gc_todo = db
|
||||
.open_tree(format!("{}:gc_todo_v2", F::TABLE_NAME))
|
||||
.open_tree(&format!("{}:gc_todo_v2", F::TABLE_NAME))
|
||||
.expect("Unable to open GC DB tree");
|
||||
let gc_todo = CountedTree::new(gc_todo).expect("Cannot count gc_todo_v2");
|
||||
|
||||
@@ -90,7 +90,7 @@ impl<F: TableSchema, R: TableReplication> TableData<F, R> {
|
||||
|
||||
pub fn read_entry(&self, p: &F::P, s: &F::S) -> Result<Option<ByteBuf>, Error> {
|
||||
let tree_key = self.tree_key(p, s);
|
||||
if let Some(bytes) = self.store.get(tree_key)? {
|
||||
if let Some(bytes) = self.store.get(&tree_key)? {
|
||||
Ok(Some(ByteBuf::from(bytes.to_vec())))
|
||||
} else {
|
||||
Ok(None)
|
||||
@@ -132,10 +132,10 @@ impl<F: TableSchema, R: TableReplication> TableData<F, R> {
|
||||
}
|
||||
}
|
||||
|
||||
fn read_range_aux(
|
||||
fn read_range_aux<'a>(
|
||||
&self,
|
||||
partition_hash: Hash,
|
||||
range: db::ValueIter,
|
||||
range: db::ValueIter<'a>,
|
||||
filter: &Option<F::Filter>,
|
||||
limit: usize,
|
||||
) -> Result<Vec<Arc<ByteBuf>>, Error> {
|
||||
|
||||
+7
-2
@@ -34,9 +34,8 @@ impl DeletedFilter {
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
|
||||
#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq)]
|
||||
pub enum EnumerationOrder {
|
||||
#[default]
|
||||
Forward,
|
||||
Reverse,
|
||||
}
|
||||
@@ -50,3 +49,9 @@ impl EnumerationOrder {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for EnumerationOrder {
|
||||
fn default() -> Self {
|
||||
EnumerationOrder::Forward
|
||||
}
|
||||
}
|
||||
|
||||
+3
-5
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "garage_util"
|
||||
version = "0.8.2"
|
||||
version = "0.8.1"
|
||||
authors = ["Alex Auvolat <alex@adnab.me>"]
|
||||
edition = "2018"
|
||||
license = "AGPL-3.0"
|
||||
@@ -14,7 +14,7 @@ path = "lib.rs"
|
||||
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
|
||||
|
||||
[dependencies]
|
||||
garage_db.workspace = true
|
||||
garage_db = { version = "0.8.1", path = "../db" }
|
||||
|
||||
arc-swap = "1.0"
|
||||
async-trait = "0.1"
|
||||
@@ -22,6 +22,7 @@ blake2 = "0.10"
|
||||
bytes = "1.0"
|
||||
digest = "0.10"
|
||||
err-derive = "0.3"
|
||||
git-version = "0.3.4"
|
||||
hexdump = "0.1"
|
||||
xxhash-rust = { version = "0.8", default-features = false, features = ["xxh3"] }
|
||||
hex = "0.4"
|
||||
@@ -46,9 +47,6 @@ hyper = "0.14"
|
||||
|
||||
opentelemetry = { version = "0.17", features = [ "rt-tokio", "metrics", "trace" ] }
|
||||
|
||||
[build-dependencies]
|
||||
rustc_version = "0.4.0"
|
||||
|
||||
[dev-dependencies]
|
||||
mktemp = "0.5"
|
||||
|
||||
|
||||
@@ -1,8 +0,0 @@
|
||||
use rustc_version::version;
|
||||
|
||||
fn main() {
|
||||
// Acquire the version of Rust used to compile, this is added as a label to
|
||||
// the garage_build_info metric.
|
||||
let v = version().unwrap();
|
||||
println!("cargo:rustc-env=RUSTC_VERSION={v}");
|
||||
}
|
||||
+1
-1
@@ -223,7 +223,7 @@ fn secret_from_file(
|
||||
#[cfg(unix)]
|
||||
if std::env::var("GARAGE_ALLOW_WORLD_READABLE_SECRETS").as_deref() != Ok("true") {
|
||||
use std::os::unix::fs::MetadataExt;
|
||||
let metadata = std::fs::metadata(file_path)?;
|
||||
let metadata = std::fs::metadata(&file_path)?;
|
||||
if metadata.mode() & 0o077 != 0 {
|
||||
return Err(format!("File {} is world-readable! (mode: 0{:o}, expected 0600)\nRefusing to start until this is fixed, or environment variable GARAGE_ALLOW_WORLD_READABLE_SECRETS is set to true.", file_path, metadata.mode()).into());
|
||||
}
|
||||
|
||||
@@ -1,19 +1,3 @@
|
||||
//! Format tables with a stupid API.
|
||||
//!
|
||||
//! Example:
|
||||
//!
|
||||
//! ```rust
|
||||
//! let mut table = vec!["product\tquantity\tprice".to_string()];
|
||||
//! for (p, q, r) in [("tomato", 12, 15), ("potato", 10, 20), ("rice", 5, 12)] {
|
||||
//! table.push(format!("{}\t{}\t{}", p, q, r));
|
||||
//! }
|
||||
//! format_table::format_table(table);
|
||||
//! ```
|
||||
//!
|
||||
//! A table to be formatted is a `Vec<String>`, containing one string per line.
|
||||
//! Table columns in each line are separated by a `\t` character.
|
||||
|
||||
/// Format a table and return the result as a string.
|
||||
pub fn format_table_to_string(data: Vec<String>) -> String {
|
||||
let data = data
|
||||
.iter()
|
||||
@@ -43,7 +27,6 @@ pub fn format_table_to_string(data: Vec<String>) -> String {
|
||||
out
|
||||
}
|
||||
|
||||
/// Format a table and print the result to stdout.
|
||||
pub fn format_table(data: Vec<String>) {
|
||||
print!("{}", format_table_to_string(data));
|
||||
}
|
||||
@@ -1,63 +0,0 @@
|
||||
use http::{HeaderMap, HeaderValue};
|
||||
use std::net::IpAddr;
|
||||
use std::str::FromStr;
|
||||
|
||||
use crate::error::{Error, OkOrMessage};
|
||||
|
||||
pub fn handle_forwarded_for_headers(headers: &HeaderMap<HeaderValue>) -> Result<String, Error> {
|
||||
let forwarded_for_header = headers
|
||||
.get("x-forwarded-for")
|
||||
.ok_or_message("X-Forwarded-For header not provided")?;
|
||||
|
||||
let forwarded_for_ip_str = forwarded_for_header
|
||||
.to_str()
|
||||
.ok_or_message("Error parsing X-Forwarded-For header")?;
|
||||
|
||||
let client_ip = IpAddr::from_str(forwarded_for_ip_str)
|
||||
.ok_or_message("Valid IP address not found in X-Forwarded-For header")?;
|
||||
|
||||
Ok(client_ip.to_string())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod test {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_handle_forwarded_for_headers_ipv4_client() {
|
||||
let mut test_headers = HeaderMap::new();
|
||||
test_headers.insert("X-Forwarded-For", "192.0.2.100".parse().unwrap());
|
||||
|
||||
if let Ok(forwarded_ip) = handle_forwarded_for_headers(&test_headers) {
|
||||
assert_eq!(forwarded_ip, "192.0.2.100");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_handle_forwarded_for_headers_ipv6_client() {
|
||||
let mut test_headers = HeaderMap::new();
|
||||
test_headers.insert("X-Forwarded-For", "2001:db8::f00d:cafe".parse().unwrap());
|
||||
|
||||
if let Ok(forwarded_ip) = handle_forwarded_for_headers(&test_headers) {
|
||||
assert_eq!(forwarded_ip, "2001:db8::f00d:cafe");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_handle_forwarded_for_headers_invalid_ip() {
|
||||
let mut test_headers = HeaderMap::new();
|
||||
test_headers.insert("X-Forwarded-For", "www.example.com".parse().unwrap());
|
||||
|
||||
let result = handle_forwarded_for_headers(&test_headers);
|
||||
assert!(result.is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_handle_forwarded_for_headers_missing() {
|
||||
let mut test_headers = HeaderMap::new();
|
||||
test_headers.insert("Host", "www.deuxfleurs.fr".parse().unwrap());
|
||||
|
||||
let result = handle_forwarded_for_headers(&test_headers);
|
||||
assert!(result.is_err());
|
||||
}
|
||||
}
|
||||
+1
-1
@@ -10,7 +10,7 @@ pub mod crdt;
|
||||
pub mod data;
|
||||
pub mod encode;
|
||||
pub mod error;
|
||||
pub mod forwarded_headers;
|
||||
pub mod formater;
|
||||
pub mod metrics;
|
||||
pub mod migrate;
|
||||
pub mod persister;
|
||||
|
||||
+8
-8
@@ -1,14 +1,18 @@
|
||||
use std::sync::Arc;
|
||||
|
||||
use arc_swap::ArcSwapOption;
|
||||
use arc_swap::{ArcSwap, ArcSwapOption};
|
||||
|
||||
lazy_static::lazy_static! {
|
||||
static ref VERSION: ArcSwapOption<&'static str> = ArcSwapOption::new(None);
|
||||
static ref VERSION: ArcSwap<&'static str> = ArcSwap::new(Arc::new(git_version::git_version!(
|
||||
prefix = "git:",
|
||||
cargo_prefix = "cargo:",
|
||||
fallback = "unknown"
|
||||
)));
|
||||
static ref FEATURES: ArcSwapOption<&'static [&'static str]> = ArcSwapOption::new(None);
|
||||
}
|
||||
|
||||
pub fn garage_version() -> &'static str {
|
||||
VERSION.load().as_ref().unwrap()
|
||||
&VERSION.load()
|
||||
}
|
||||
|
||||
pub fn garage_features() -> Option<&'static [&'static str]> {
|
||||
@@ -16,13 +20,9 @@ pub fn garage_features() -> Option<&'static [&'static str]> {
|
||||
}
|
||||
|
||||
pub fn init_version(version: &'static str) {
|
||||
VERSION.store(Some(Arc::new(version)));
|
||||
VERSION.store(Arc::new(version));
|
||||
}
|
||||
|
||||
pub fn init_features(features: &'static [&'static str]) {
|
||||
FEATURES.store(Some(Arc::new(features)));
|
||||
}
|
||||
|
||||
pub fn rust_version() -> &'static str {
|
||||
env!("RUSTC_VERSION")
|
||||
}
|
||||
|
||||
+5
-5
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "garage_web"
|
||||
version = "0.8.2"
|
||||
version = "0.8.1"
|
||||
authors = ["Alex Auvolat <alex@adnab.me>", "Quentin Dufour <quentin@dufour.io>"]
|
||||
edition = "2018"
|
||||
license = "AGPL-3.0"
|
||||
@@ -14,10 +14,10 @@ path = "lib.rs"
|
||||
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
|
||||
|
||||
[dependencies]
|
||||
garage_api.workspace = true
|
||||
garage_model.workspace = true
|
||||
garage_util.workspace = true
|
||||
garage_table.workspace = true
|
||||
garage_api = { version = "0.8.1", path = "../api" }
|
||||
garage_model = { version = "0.8.1", path = "../model" }
|
||||
garage_util = { version = "0.8.1", path = "../util" }
|
||||
garage_table = { version = "0.8.1", path = "../table" }
|
||||
|
||||
err-derive = "0.3"
|
||||
tracing = "0.1"
|
||||
|
||||
+1
-14
@@ -29,7 +29,6 @@ use garage_model::garage::Garage;
|
||||
|
||||
use garage_table::*;
|
||||
use garage_util::error::Error as GarageError;
|
||||
use garage_util::forwarded_headers;
|
||||
use garage_util::metrics::{gen_trace_id, RecordDuration};
|
||||
|
||||
struct WebMetrics {
|
||||
@@ -105,19 +104,7 @@ impl WebServer {
|
||||
req: Request<Body>,
|
||||
addr: SocketAddr,
|
||||
) -> Result<Response<Body>, Infallible> {
|
||||
if let Ok(forwarded_for_ip_addr) =
|
||||
forwarded_headers::handle_forwarded_for_headers(req.headers())
|
||||
{
|
||||
info!(
|
||||
"{} (via {}) {} {}",
|
||||
forwarded_for_ip_addr,
|
||||
addr,
|
||||
req.method(),
|
||||
req.uri()
|
||||
);
|
||||
} else {
|
||||
info!("{} {} {}", addr, req.method(), req.uri());
|
||||
}
|
||||
info!("{} {} {}", addr, req.method(), req.uri());
|
||||
|
||||
// Lots of instrumentation
|
||||
let tracer = opentelemetry::global::tracer("garage");
|
||||
|
||||
Reference in New Issue
Block a user