mirror of
https://github.com/deuxfleurs-org/garage.git
synced 2026-08-15 00:23:11 +00:00
Compare commits
1 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 4756db4d29 |
Generated
+39
-10
@@ -1048,7 +1048,7 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "garage"
|
name = "garage"
|
||||||
version = "0.8.1"
|
version = "0.8.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"assert-json-diff",
|
"assert-json-diff",
|
||||||
"async-trait",
|
"async-trait",
|
||||||
@@ -1080,6 +1080,7 @@ dependencies = [
|
|||||||
"parse_duration",
|
"parse_duration",
|
||||||
"prometheus",
|
"prometheus",
|
||||||
"rand 0.8.5",
|
"rand 0.8.5",
|
||||||
|
"rmp-serde",
|
||||||
"serde",
|
"serde",
|
||||||
"serde_bytes",
|
"serde_bytes",
|
||||||
"serde_json",
|
"serde_json",
|
||||||
@@ -1095,7 +1096,7 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "garage_api"
|
name = "garage_api"
|
||||||
version = "0.8.1"
|
version = "0.8.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"async-trait",
|
"async-trait",
|
||||||
"base64",
|
"base64",
|
||||||
@@ -1140,7 +1141,7 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "garage_block"
|
name = "garage_block"
|
||||||
version = "0.8.1"
|
version = "0.8.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"arc-swap",
|
"arc-swap",
|
||||||
"async-compression",
|
"async-compression",
|
||||||
@@ -1155,6 +1156,7 @@ dependencies = [
|
|||||||
"hex",
|
"hex",
|
||||||
"opentelemetry",
|
"opentelemetry",
|
||||||
"rand 0.8.5",
|
"rand 0.8.5",
|
||||||
|
"rmp-serde",
|
||||||
"serde",
|
"serde",
|
||||||
"serde_bytes",
|
"serde_bytes",
|
||||||
"tokio",
|
"tokio",
|
||||||
@@ -1165,22 +1167,27 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "garage_db"
|
name = "garage_db"
|
||||||
version = "0.8.1"
|
version = "0.8.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"clap 3.1.18",
|
"clap 3.1.18",
|
||||||
"err-derive",
|
"err-derive",
|
||||||
"heed",
|
"heed",
|
||||||
|
"hex",
|
||||||
"hexdump",
|
"hexdump",
|
||||||
"mktemp",
|
"mktemp",
|
||||||
|
"nettext",
|
||||||
"pretty_env_logger",
|
"pretty_env_logger",
|
||||||
|
"rmp-serde",
|
||||||
"rusqlite",
|
"rusqlite",
|
||||||
|
"serde",
|
||||||
|
"serde-transcode",
|
||||||
"sled",
|
"sled",
|
||||||
"tracing",
|
"tracing",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "garage_model"
|
name = "garage_model"
|
||||||
version = "0.8.1"
|
version = "0.8.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"arc-swap",
|
"arc-swap",
|
||||||
"async-trait",
|
"async-trait",
|
||||||
@@ -1198,6 +1205,7 @@ dependencies = [
|
|||||||
"netapp",
|
"netapp",
|
||||||
"opentelemetry",
|
"opentelemetry",
|
||||||
"rand 0.8.5",
|
"rand 0.8.5",
|
||||||
|
"rmp-serde",
|
||||||
"serde",
|
"serde",
|
||||||
"serde_bytes",
|
"serde_bytes",
|
||||||
"tokio",
|
"tokio",
|
||||||
@@ -1207,7 +1215,7 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "garage_rpc"
|
name = "garage_rpc"
|
||||||
version = "0.8.1"
|
version = "0.8.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"arc-swap",
|
"arc-swap",
|
||||||
"async-trait",
|
"async-trait",
|
||||||
@@ -1226,6 +1234,7 @@ dependencies = [
|
|||||||
"pnet_datalink",
|
"pnet_datalink",
|
||||||
"rand 0.8.5",
|
"rand 0.8.5",
|
||||||
"reqwest",
|
"reqwest",
|
||||||
|
"rmp-serde",
|
||||||
"schemars",
|
"schemars",
|
||||||
"serde",
|
"serde",
|
||||||
"serde_bytes",
|
"serde_bytes",
|
||||||
@@ -1237,7 +1246,7 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "garage_table"
|
name = "garage_table"
|
||||||
version = "0.8.1"
|
version = "0.8.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"arc-swap",
|
"arc-swap",
|
||||||
"async-trait",
|
"async-trait",
|
||||||
@@ -1251,6 +1260,7 @@ dependencies = [
|
|||||||
"hexdump",
|
"hexdump",
|
||||||
"opentelemetry",
|
"opentelemetry",
|
||||||
"rand 0.8.5",
|
"rand 0.8.5",
|
||||||
|
"rmp-serde",
|
||||||
"serde",
|
"serde",
|
||||||
"serde_bytes",
|
"serde_bytes",
|
||||||
"tokio",
|
"tokio",
|
||||||
@@ -1259,7 +1269,7 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "garage_util"
|
name = "garage_util"
|
||||||
version = "0.8.1"
|
version = "0.8.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"arc-swap",
|
"arc-swap",
|
||||||
"async-trait",
|
"async-trait",
|
||||||
@@ -1272,7 +1282,6 @@ dependencies = [
|
|||||||
"garage_db",
|
"garage_db",
|
||||||
"git-version",
|
"git-version",
|
||||||
"hex",
|
"hex",
|
||||||
"hexdump",
|
|
||||||
"http",
|
"http",
|
||||||
"hyper",
|
"hyper",
|
||||||
"lazy_static",
|
"lazy_static",
|
||||||
@@ -1291,7 +1300,7 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "garage_web"
|
name = "garage_web"
|
||||||
version = "0.8.1"
|
version = "0.8.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"err-derive",
|
"err-derive",
|
||||||
"futures",
|
"futures",
|
||||||
@@ -2137,6 +2146,17 @@ dependencies = [
|
|||||||
"tokio-util 0.7.0",
|
"tokio-util 0.7.0",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "nettext"
|
||||||
|
version = "0.3.2"
|
||||||
|
dependencies = [
|
||||||
|
"base64",
|
||||||
|
"err-derive",
|
||||||
|
"hex",
|
||||||
|
"nom",
|
||||||
|
"serde",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "nom"
|
name = "nom"
|
||||||
version = "7.1.1"
|
version = "7.1.1"
|
||||||
@@ -3278,6 +3298,15 @@ dependencies = [
|
|||||||
"serde_derive",
|
"serde_derive",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "serde-transcode"
|
||||||
|
version = "1.1.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "590c0e25c2a5bb6e85bf5c1bce768ceb86b316e7a01bdf07d2cb4ec2271990e2"
|
||||||
|
dependencies = [
|
||||||
|
"serde",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "serde-value"
|
name = "serde-value"
|
||||||
version = "0.7.0"
|
version = "0.7.0"
|
||||||
|
|||||||
@@ -32,7 +32,7 @@ args@{
|
|||||||
ignoreLockHash,
|
ignoreLockHash,
|
||||||
}:
|
}:
|
||||||
let
|
let
|
||||||
nixifiedLockHash = "b6aeefc112eb232904b24398f4e5da776c8ee2c13d427a26dbdf1732205d4fc9";
|
nixifiedLockHash = "a1d84930f23d3d8abc8abbed59b8ce3c9adf9f25d06bc1f39cbdf5bd90aceead";
|
||||||
workspaceSrc = if args.workspaceSrc == null then ./. else args.workspaceSrc;
|
workspaceSrc = if args.workspaceSrc == null then ./. else args.workspaceSrc;
|
||||||
currentLockHash = builtins.hashFile "sha256" (workspaceSrc + /Cargo.lock);
|
currentLockHash = builtins.hashFile "sha256" (workspaceSrc + /Cargo.lock);
|
||||||
lockHashIgnored = if ignoreLockHash
|
lockHashIgnored = if ignoreLockHash
|
||||||
@@ -56,15 +56,15 @@ in
|
|||||||
{
|
{
|
||||||
cargo2nixVersion = "0.11.0";
|
cargo2nixVersion = "0.11.0";
|
||||||
workspace = {
|
workspace = {
|
||||||
garage_db = rustPackages.unknown.garage_db."0.8.1";
|
garage_db = rustPackages.unknown.garage_db."0.8.0";
|
||||||
garage_util = rustPackages.unknown.garage_util."0.8.1";
|
garage_util = rustPackages.unknown.garage_util."0.8.0";
|
||||||
garage_rpc = rustPackages.unknown.garage_rpc."0.8.1";
|
garage_rpc = rustPackages.unknown.garage_rpc."0.8.0";
|
||||||
garage_table = rustPackages.unknown.garage_table."0.8.1";
|
garage_table = rustPackages.unknown.garage_table."0.8.0";
|
||||||
garage_block = rustPackages.unknown.garage_block."0.8.1";
|
garage_block = rustPackages.unknown.garage_block."0.8.0";
|
||||||
garage_model = rustPackages.unknown.garage_model."0.8.1";
|
garage_model = rustPackages.unknown.garage_model."0.8.0";
|
||||||
garage_api = rustPackages.unknown.garage_api."0.8.1";
|
garage_api = rustPackages.unknown.garage_api."0.8.0";
|
||||||
garage_web = rustPackages.unknown.garage_web."0.8.1";
|
garage_web = rustPackages.unknown.garage_web."0.8.0";
|
||||||
garage = rustPackages.unknown.garage."0.8.1";
|
garage = rustPackages.unknown.garage."0.8.0";
|
||||||
k2v-client = rustPackages.unknown.k2v-client."0.0.1";
|
k2v-client = rustPackages.unknown.k2v-client."0.0.1";
|
||||||
};
|
};
|
||||||
"registry+https://github.com/rust-lang/crates.io-index".addr2line."0.17.0" = overridableMkRustCrate (profileName: rec {
|
"registry+https://github.com/rust-lang/crates.io-index".addr2line."0.17.0" = overridableMkRustCrate (profileName: rec {
|
||||||
@@ -1494,16 +1494,16 @@ in
|
|||||||
};
|
};
|
||||||
});
|
});
|
||||||
|
|
||||||
"unknown".garage."0.8.1" = overridableMkRustCrate (profileName: rec {
|
"unknown".garage."0.8.0" = overridableMkRustCrate (profileName: rec {
|
||||||
name = "garage";
|
name = "garage";
|
||||||
version = "0.8.1";
|
version = "0.8.0";
|
||||||
registry = "unknown";
|
registry = "unknown";
|
||||||
src = fetchCrateLocal (workspaceSrc + "/src/garage");
|
src = fetchCrateLocal (workspaceSrc + "/src/garage");
|
||||||
features = builtins.concatLists [
|
features = builtins.concatLists [
|
||||||
(lib.optional (rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default") "bundled-libs")
|
(lib.optional (rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default") "bundled-libs")
|
||||||
(lib.optional (rootFeatures' ? "garage/consul-discovery") "consul-discovery")
|
(lib.optional (rootFeatures' ? "garage/consul-discovery") "consul-discovery")
|
||||||
(lib.optional (rootFeatures' ? "garage/default") "default")
|
(lib.optional (rootFeatures' ? "garage/default") "default")
|
||||||
(lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/k2v") "k2v")
|
(lib.optional (rootFeatures' ? "garage/k2v") "k2v")
|
||||||
(lib.optional (rootFeatures' ? "garage/kubernetes-discovery") "kubernetes-discovery")
|
(lib.optional (rootFeatures' ? "garage/kubernetes-discovery") "kubernetes-discovery")
|
||||||
(lib.optional (rootFeatures' ? "garage/lmdb") "lmdb")
|
(lib.optional (rootFeatures' ? "garage/lmdb") "lmdb")
|
||||||
(lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/metrics") "metrics")
|
(lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/metrics") "metrics")
|
||||||
@@ -1522,14 +1522,14 @@ in
|
|||||||
bytesize = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytesize."1.1.0" { inherit profileName; }).out;
|
bytesize = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytesize."1.1.0" { inherit profileName; }).out;
|
||||||
futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.21" { inherit profileName; }).out;
|
futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.21" { inherit profileName; }).out;
|
||||||
futures_util = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.21" { inherit profileName; }).out;
|
futures_util = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.21" { inherit profileName; }).out;
|
||||||
garage_api = (rustPackages."unknown".garage_api."0.8.1" { inherit profileName; }).out;
|
garage_api = (rustPackages."unknown".garage_api."0.8.0" { inherit profileName; }).out;
|
||||||
garage_block = (rustPackages."unknown".garage_block."0.8.1" { inherit profileName; }).out;
|
garage_block = (rustPackages."unknown".garage_block."0.8.0" { inherit profileName; }).out;
|
||||||
garage_db = (rustPackages."unknown".garage_db."0.8.1" { inherit profileName; }).out;
|
garage_db = (rustPackages."unknown".garage_db."0.8.0" { inherit profileName; }).out;
|
||||||
garage_model = (rustPackages."unknown".garage_model."0.8.1" { inherit profileName; }).out;
|
garage_model = (rustPackages."unknown".garage_model."0.8.0" { inherit profileName; }).out;
|
||||||
garage_rpc = (rustPackages."unknown".garage_rpc."0.8.1" { inherit profileName; }).out;
|
garage_rpc = (rustPackages."unknown".garage_rpc."0.8.0" { inherit profileName; }).out;
|
||||||
garage_table = (rustPackages."unknown".garage_table."0.8.1" { inherit profileName; }).out;
|
garage_table = (rustPackages."unknown".garage_table."0.8.0" { inherit profileName; }).out;
|
||||||
garage_util = (rustPackages."unknown".garage_util."0.8.1" { inherit profileName; }).out;
|
garage_util = (rustPackages."unknown".garage_util."0.8.0" { inherit profileName; }).out;
|
||||||
garage_web = (rustPackages."unknown".garage_web."0.8.1" { inherit profileName; }).out;
|
garage_web = (rustPackages."unknown".garage_web."0.8.0" { inherit profileName; }).out;
|
||||||
hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out;
|
hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out;
|
||||||
sodiumoxide = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".kuska-sodiumoxide."0.2.5-0" { inherit profileName; }).out;
|
sodiumoxide = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".kuska-sodiumoxide."0.2.5-0" { inherit profileName; }).out;
|
||||||
netapp = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".netapp."0.5.2" { inherit profileName; }).out;
|
netapp = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".netapp."0.5.2" { inherit profileName; }).out;
|
||||||
@@ -1539,6 +1539,7 @@ in
|
|||||||
parse_duration = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".parse_duration."2.1.1" { inherit profileName; }).out;
|
parse_duration = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".parse_duration."2.1.1" { inherit profileName; }).out;
|
||||||
${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/metrics" || rootFeatures' ? "garage/prometheus" then "prometheus" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".prometheus."0.13.0" { inherit profileName; }).out;
|
${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/metrics" || rootFeatures' ? "garage/prometheus" then "prometheus" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".prometheus."0.13.0" { inherit profileName; }).out;
|
||||||
rand = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand."0.8.5" { inherit profileName; }).out;
|
rand = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand."0.8.5" { inherit profileName; }).out;
|
||||||
|
rmp_serde = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rmp-serde."0.15.5" { inherit profileName; }).out;
|
||||||
serde = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde."1.0.137" { inherit profileName; }).out;
|
serde = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde."1.0.137" { inherit profileName; }).out;
|
||||||
serde_bytes = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde_bytes."0.11.5" { inherit profileName; }).out;
|
serde_bytes = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde_bytes."0.11.5" { inherit profileName; }).out;
|
||||||
structopt = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".structopt."0.3.26" { inherit profileName; }).out;
|
structopt = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".structopt."0.3.26" { inherit profileName; }).out;
|
||||||
@@ -1562,13 +1563,13 @@ in
|
|||||||
};
|
};
|
||||||
});
|
});
|
||||||
|
|
||||||
"unknown".garage_api."0.8.1" = overridableMkRustCrate (profileName: rec {
|
"unknown".garage_api."0.8.0" = overridableMkRustCrate (profileName: rec {
|
||||||
name = "garage_api";
|
name = "garage_api";
|
||||||
version = "0.8.1";
|
version = "0.8.0";
|
||||||
registry = "unknown";
|
registry = "unknown";
|
||||||
src = fetchCrateLocal (workspaceSrc + "/src/api");
|
src = fetchCrateLocal (workspaceSrc + "/src/api");
|
||||||
features = builtins.concatLists [
|
features = builtins.concatLists [
|
||||||
(lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/k2v" || rootFeatures' ? "garage_api/k2v") "k2v")
|
(lib.optional (rootFeatures' ? "garage/k2v" || rootFeatures' ? "garage_api/k2v") "k2v")
|
||||||
(lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/metrics" || rootFeatures' ? "garage_api/metrics") "metrics")
|
(lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/metrics" || rootFeatures' ? "garage_api/metrics") "metrics")
|
||||||
(lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/metrics" || rootFeatures' ? "garage_api/metrics" || rootFeatures' ? "garage_api/opentelemetry-prometheus") "opentelemetry-prometheus")
|
(lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/metrics" || rootFeatures' ? "garage_api/metrics" || rootFeatures' ? "garage_api/opentelemetry-prometheus") "opentelemetry-prometheus")
|
||||||
(lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/metrics" || rootFeatures' ? "garage_api/metrics" || rootFeatures' ? "garage_api/prometheus") "prometheus")
|
(lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/metrics" || rootFeatures' ? "garage_api/metrics" || rootFeatures' ? "garage_api/prometheus") "prometheus")
|
||||||
@@ -1583,11 +1584,11 @@ in
|
|||||||
form_urlencoded = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".form_urlencoded."1.0.1" { inherit profileName; }).out;
|
form_urlencoded = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".form_urlencoded."1.0.1" { inherit profileName; }).out;
|
||||||
futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.21" { inherit profileName; }).out;
|
futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.21" { inherit profileName; }).out;
|
||||||
futures_util = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.21" { inherit profileName; }).out;
|
futures_util = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.21" { inherit profileName; }).out;
|
||||||
garage_block = (rustPackages."unknown".garage_block."0.8.1" { inherit profileName; }).out;
|
garage_block = (rustPackages."unknown".garage_block."0.8.0" { inherit profileName; }).out;
|
||||||
garage_model = (rustPackages."unknown".garage_model."0.8.1" { inherit profileName; }).out;
|
garage_model = (rustPackages."unknown".garage_model."0.8.0" { inherit profileName; }).out;
|
||||||
garage_rpc = (rustPackages."unknown".garage_rpc."0.8.1" { inherit profileName; }).out;
|
garage_rpc = (rustPackages."unknown".garage_rpc."0.8.0" { inherit profileName; }).out;
|
||||||
garage_table = (rustPackages."unknown".garage_table."0.8.1" { inherit profileName; }).out;
|
garage_table = (rustPackages."unknown".garage_table."0.8.0" { inherit profileName; }).out;
|
||||||
garage_util = (rustPackages."unknown".garage_util."0.8.1" { inherit profileName; }).out;
|
garage_util = (rustPackages."unknown".garage_util."0.8.0" { inherit profileName; }).out;
|
||||||
hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out;
|
hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out;
|
||||||
hmac = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hmac."0.12.1" { inherit profileName; }).out;
|
hmac = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hmac."0.12.1" { inherit profileName; }).out;
|
||||||
http = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.8" { inherit profileName; }).out;
|
http = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.8" { inherit profileName; }).out;
|
||||||
@@ -1616,9 +1617,9 @@ in
|
|||||||
};
|
};
|
||||||
});
|
});
|
||||||
|
|
||||||
"unknown".garage_block."0.8.1" = overridableMkRustCrate (profileName: rec {
|
"unknown".garage_block."0.8.0" = overridableMkRustCrate (profileName: rec {
|
||||||
name = "garage_block";
|
name = "garage_block";
|
||||||
version = "0.8.1";
|
version = "0.8.0";
|
||||||
registry = "unknown";
|
registry = "unknown";
|
||||||
src = fetchCrateLocal (workspaceSrc + "/src/block");
|
src = fetchCrateLocal (workspaceSrc + "/src/block");
|
||||||
features = builtins.concatLists [
|
features = builtins.concatLists [
|
||||||
@@ -1631,13 +1632,14 @@ in
|
|||||||
bytes = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.2.0" { inherit profileName; }).out;
|
bytes = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.2.0" { inherit profileName; }).out;
|
||||||
futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.21" { inherit profileName; }).out;
|
futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.21" { inherit profileName; }).out;
|
||||||
futures_util = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.21" { inherit profileName; }).out;
|
futures_util = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.21" { inherit profileName; }).out;
|
||||||
garage_db = (rustPackages."unknown".garage_db."0.8.1" { inherit profileName; }).out;
|
garage_db = (rustPackages."unknown".garage_db."0.8.0" { inherit profileName; }).out;
|
||||||
garage_rpc = (rustPackages."unknown".garage_rpc."0.8.1" { inherit profileName; }).out;
|
garage_rpc = (rustPackages."unknown".garage_rpc."0.8.0" { inherit profileName; }).out;
|
||||||
garage_table = (rustPackages."unknown".garage_table."0.8.1" { inherit profileName; }).out;
|
garage_table = (rustPackages."unknown".garage_table."0.8.0" { inherit profileName; }).out;
|
||||||
garage_util = (rustPackages."unknown".garage_util."0.8.1" { inherit profileName; }).out;
|
garage_util = (rustPackages."unknown".garage_util."0.8.0" { inherit profileName; }).out;
|
||||||
hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out;
|
hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out;
|
||||||
opentelemetry = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry."0.17.0" { inherit profileName; }).out;
|
opentelemetry = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry."0.17.0" { inherit profileName; }).out;
|
||||||
rand = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand."0.8.5" { inherit profileName; }).out;
|
rand = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand."0.8.5" { inherit profileName; }).out;
|
||||||
|
rmp_serde = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rmp-serde."0.15.5" { inherit profileName; }).out;
|
||||||
serde = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde."1.0.137" { inherit profileName; }).out;
|
serde = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde."1.0.137" { inherit profileName; }).out;
|
||||||
serde_bytes = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde_bytes."0.11.5" { inherit profileName; }).out;
|
serde_bytes = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde_bytes."0.11.5" { inherit profileName; }).out;
|
||||||
tokio = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.17.0" { inherit profileName; }).out;
|
tokio = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.17.0" { inherit profileName; }).out;
|
||||||
@@ -1647,9 +1649,9 @@ in
|
|||||||
};
|
};
|
||||||
});
|
});
|
||||||
|
|
||||||
"unknown".garage_db."0.8.1" = overridableMkRustCrate (profileName: rec {
|
"unknown".garage_db."0.8.0" = overridableMkRustCrate (profileName: rec {
|
||||||
name = "garage_db";
|
name = "garage_db";
|
||||||
version = "0.8.1";
|
version = "0.8.0";
|
||||||
registry = "unknown";
|
registry = "unknown";
|
||||||
src = fetchCrateLocal (workspaceSrc + "/src/db");
|
src = fetchCrateLocal (workspaceSrc + "/src/db");
|
||||||
features = builtins.concatLists [
|
features = builtins.concatLists [
|
||||||
@@ -1679,14 +1681,14 @@ in
|
|||||||
};
|
};
|
||||||
});
|
});
|
||||||
|
|
||||||
"unknown".garage_model."0.8.1" = overridableMkRustCrate (profileName: rec {
|
"unknown".garage_model."0.8.0" = overridableMkRustCrate (profileName: rec {
|
||||||
name = "garage_model";
|
name = "garage_model";
|
||||||
version = "0.8.1";
|
version = "0.8.0";
|
||||||
registry = "unknown";
|
registry = "unknown";
|
||||||
src = fetchCrateLocal (workspaceSrc + "/src/model");
|
src = fetchCrateLocal (workspaceSrc + "/src/model");
|
||||||
features = builtins.concatLists [
|
features = builtins.concatLists [
|
||||||
[ "default" ]
|
[ "default" ]
|
||||||
(lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/k2v" || rootFeatures' ? "garage_api/k2v" || rootFeatures' ? "garage_model/k2v") "k2v")
|
(lib.optional (rootFeatures' ? "garage/k2v" || rootFeatures' ? "garage_api/k2v" || rootFeatures' ? "garage_model/k2v") "k2v")
|
||||||
(lib.optional (rootFeatures' ? "garage/lmdb" || rootFeatures' ? "garage_model/lmdb") "lmdb")
|
(lib.optional (rootFeatures' ? "garage/lmdb" || rootFeatures' ? "garage_model/lmdb") "lmdb")
|
||||||
[ "sled" ]
|
[ "sled" ]
|
||||||
(lib.optional (rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_model/sqlite") "sqlite")
|
(lib.optional (rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_model/sqlite") "sqlite")
|
||||||
@@ -1699,15 +1701,16 @@ in
|
|||||||
err_derive = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".err-derive."0.3.1" { profileName = "__noProfile"; }).out;
|
err_derive = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".err-derive."0.3.1" { profileName = "__noProfile"; }).out;
|
||||||
futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.21" { inherit profileName; }).out;
|
futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.21" { inherit profileName; }).out;
|
||||||
futures_util = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.21" { inherit profileName; }).out;
|
futures_util = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.21" { inherit profileName; }).out;
|
||||||
garage_block = (rustPackages."unknown".garage_block."0.8.1" { inherit profileName; }).out;
|
garage_block = (rustPackages."unknown".garage_block."0.8.0" { inherit profileName; }).out;
|
||||||
garage_db = (rustPackages."unknown".garage_db."0.8.1" { inherit profileName; }).out;
|
garage_db = (rustPackages."unknown".garage_db."0.8.0" { inherit profileName; }).out;
|
||||||
garage_rpc = (rustPackages."unknown".garage_rpc."0.8.1" { inherit profileName; }).out;
|
garage_rpc = (rustPackages."unknown".garage_rpc."0.8.0" { inherit profileName; }).out;
|
||||||
garage_table = (rustPackages."unknown".garage_table."0.8.1" { inherit profileName; }).out;
|
garage_table = (rustPackages."unknown".garage_table."0.8.0" { inherit profileName; }).out;
|
||||||
garage_util = (rustPackages."unknown".garage_util."0.8.1" { inherit profileName; }).out;
|
garage_util = (rustPackages."unknown".garage_util."0.8.0" { inherit profileName; }).out;
|
||||||
hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out;
|
hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out;
|
||||||
netapp = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".netapp."0.5.2" { inherit profileName; }).out;
|
netapp = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".netapp."0.5.2" { inherit profileName; }).out;
|
||||||
opentelemetry = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry."0.17.0" { inherit profileName; }).out;
|
opentelemetry = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry."0.17.0" { inherit profileName; }).out;
|
||||||
rand = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand."0.8.5" { inherit profileName; }).out;
|
rand = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand."0.8.5" { inherit profileName; }).out;
|
||||||
|
rmp_serde = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rmp-serde."0.15.5" { inherit profileName; }).out;
|
||||||
serde = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde."1.0.137" { inherit profileName; }).out;
|
serde = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde."1.0.137" { inherit profileName; }).out;
|
||||||
serde_bytes = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde_bytes."0.11.5" { inherit profileName; }).out;
|
serde_bytes = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde_bytes."0.11.5" { inherit profileName; }).out;
|
||||||
tokio = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.17.0" { inherit profileName; }).out;
|
tokio = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.17.0" { inherit profileName; }).out;
|
||||||
@@ -1716,9 +1719,9 @@ in
|
|||||||
};
|
};
|
||||||
});
|
});
|
||||||
|
|
||||||
"unknown".garage_rpc."0.8.1" = overridableMkRustCrate (profileName: rec {
|
"unknown".garage_rpc."0.8.0" = overridableMkRustCrate (profileName: rec {
|
||||||
name = "garage_rpc";
|
name = "garage_rpc";
|
||||||
version = "0.8.1";
|
version = "0.8.0";
|
||||||
registry = "unknown";
|
registry = "unknown";
|
||||||
src = fetchCrateLocal (workspaceSrc + "/src/rpc");
|
src = fetchCrateLocal (workspaceSrc + "/src/rpc");
|
||||||
features = builtins.concatLists [
|
features = builtins.concatLists [
|
||||||
@@ -1738,7 +1741,7 @@ in
|
|||||||
${ if rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/err-derive" then "err_derive" else null } = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".err-derive."0.3.1" { profileName = "__noProfile"; }).out;
|
${ if rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/err-derive" then "err_derive" else null } = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".err-derive."0.3.1" { profileName = "__noProfile"; }).out;
|
||||||
futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.21" { inherit profileName; }).out;
|
futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.21" { inherit profileName; }).out;
|
||||||
futures_util = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.21" { inherit profileName; }).out;
|
futures_util = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.21" { inherit profileName; }).out;
|
||||||
garage_util = (rustPackages."unknown".garage_util."0.8.1" { inherit profileName; }).out;
|
garage_util = (rustPackages."unknown".garage_util."0.8.0" { inherit profileName; }).out;
|
||||||
gethostname = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".gethostname."0.2.3" { inherit profileName; }).out;
|
gethostname = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".gethostname."0.2.3" { inherit profileName; }).out;
|
||||||
hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out;
|
hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out;
|
||||||
${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/k8s-openapi" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "k8s_openapi" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".k8s-openapi."0.16.0" { inherit profileName; }).out;
|
${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/k8s-openapi" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "k8s_openapi" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".k8s-openapi."0.16.0" { inherit profileName; }).out;
|
||||||
@@ -1749,6 +1752,7 @@ in
|
|||||||
pnet_datalink = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".pnet_datalink."0.28.0" { inherit profileName; }).out;
|
pnet_datalink = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".pnet_datalink."0.28.0" { inherit profileName; }).out;
|
||||||
rand = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand."0.8.5" { inherit profileName; }).out;
|
rand = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand."0.8.5" { inherit profileName; }).out;
|
||||||
${ if rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest" then "reqwest" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".reqwest."0.11.12" { inherit profileName; }).out;
|
${ if rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest" then "reqwest" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".reqwest."0.11.12" { inherit profileName; }).out;
|
||||||
|
rmp_serde = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rmp-serde."0.15.5" { inherit profileName; }).out;
|
||||||
${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kubernetes-discovery" || rootFeatures' ? "garage_rpc/schemars" then "schemars" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".schemars."0.8.8" { inherit profileName; }).out;
|
${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kubernetes-discovery" || rootFeatures' ? "garage_rpc/schemars" then "schemars" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".schemars."0.8.8" { inherit profileName; }).out;
|
||||||
serde = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde."1.0.137" { inherit profileName; }).out;
|
serde = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde."1.0.137" { inherit profileName; }).out;
|
||||||
serde_bytes = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde_bytes."0.11.5" { inherit profileName; }).out;
|
serde_bytes = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde_bytes."0.11.5" { inherit profileName; }).out;
|
||||||
@@ -1759,9 +1763,9 @@ in
|
|||||||
};
|
};
|
||||||
});
|
});
|
||||||
|
|
||||||
"unknown".garage_table."0.8.1" = overridableMkRustCrate (profileName: rec {
|
"unknown".garage_table."0.8.0" = overridableMkRustCrate (profileName: rec {
|
||||||
name = "garage_table";
|
name = "garage_table";
|
||||||
version = "0.8.1";
|
version = "0.8.0";
|
||||||
registry = "unknown";
|
registry = "unknown";
|
||||||
src = fetchCrateLocal (workspaceSrc + "/src/table");
|
src = fetchCrateLocal (workspaceSrc + "/src/table");
|
||||||
dependencies = {
|
dependencies = {
|
||||||
@@ -1770,13 +1774,14 @@ in
|
|||||||
bytes = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.2.0" { inherit profileName; }).out;
|
bytes = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.2.0" { inherit profileName; }).out;
|
||||||
futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.21" { inherit profileName; }).out;
|
futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.21" { inherit profileName; }).out;
|
||||||
futures_util = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.21" { inherit profileName; }).out;
|
futures_util = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.21" { inherit profileName; }).out;
|
||||||
garage_db = (rustPackages."unknown".garage_db."0.8.1" { inherit profileName; }).out;
|
garage_db = (rustPackages."unknown".garage_db."0.8.0" { inherit profileName; }).out;
|
||||||
garage_rpc = (rustPackages."unknown".garage_rpc."0.8.1" { inherit profileName; }).out;
|
garage_rpc = (rustPackages."unknown".garage_rpc."0.8.0" { inherit profileName; }).out;
|
||||||
garage_util = (rustPackages."unknown".garage_util."0.8.1" { inherit profileName; }).out;
|
garage_util = (rustPackages."unknown".garage_util."0.8.0" { inherit profileName; }).out;
|
||||||
hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out;
|
hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out;
|
||||||
hexdump = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hexdump."0.1.1" { inherit profileName; }).out;
|
hexdump = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hexdump."0.1.1" { inherit profileName; }).out;
|
||||||
opentelemetry = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry."0.17.0" { inherit profileName; }).out;
|
opentelemetry = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry."0.17.0" { inherit profileName; }).out;
|
||||||
rand = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand."0.8.5" { inherit profileName; }).out;
|
rand = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand."0.8.5" { inherit profileName; }).out;
|
||||||
|
rmp_serde = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rmp-serde."0.15.5" { inherit profileName; }).out;
|
||||||
serde = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde."1.0.137" { inherit profileName; }).out;
|
serde = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde."1.0.137" { inherit profileName; }).out;
|
||||||
serde_bytes = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde_bytes."0.11.5" { inherit profileName; }).out;
|
serde_bytes = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde_bytes."0.11.5" { inherit profileName; }).out;
|
||||||
tokio = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.17.0" { inherit profileName; }).out;
|
tokio = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.17.0" { inherit profileName; }).out;
|
||||||
@@ -1784,13 +1789,13 @@ in
|
|||||||
};
|
};
|
||||||
});
|
});
|
||||||
|
|
||||||
"unknown".garage_util."0.8.1" = overridableMkRustCrate (profileName: rec {
|
"unknown".garage_util."0.8.0" = overridableMkRustCrate (profileName: rec {
|
||||||
name = "garage_util";
|
name = "garage_util";
|
||||||
version = "0.8.1";
|
version = "0.8.0";
|
||||||
registry = "unknown";
|
registry = "unknown";
|
||||||
src = fetchCrateLocal (workspaceSrc + "/src/util");
|
src = fetchCrateLocal (workspaceSrc + "/src/util");
|
||||||
features = builtins.concatLists [
|
features = builtins.concatLists [
|
||||||
(lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/k2v" || rootFeatures' ? "garage_api/k2v" || rootFeatures' ? "garage_model/k2v" || rootFeatures' ? "garage_util/k2v") "k2v")
|
(lib.optional (rootFeatures' ? "garage/k2v" || rootFeatures' ? "garage_api/k2v" || rootFeatures' ? "garage_model/k2v" || rootFeatures' ? "garage_util/k2v") "k2v")
|
||||||
];
|
];
|
||||||
dependencies = {
|
dependencies = {
|
||||||
arc_swap = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".arc-swap."1.5.0" { inherit profileName; }).out;
|
arc_swap = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".arc-swap."1.5.0" { inherit profileName; }).out;
|
||||||
@@ -1801,10 +1806,9 @@ in
|
|||||||
digest = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".digest."0.10.3" { inherit profileName; }).out;
|
digest = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".digest."0.10.3" { inherit profileName; }).out;
|
||||||
err_derive = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".err-derive."0.3.1" { profileName = "__noProfile"; }).out;
|
err_derive = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".err-derive."0.3.1" { profileName = "__noProfile"; }).out;
|
||||||
futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.21" { inherit profileName; }).out;
|
futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.21" { inherit profileName; }).out;
|
||||||
garage_db = (rustPackages."unknown".garage_db."0.8.1" { inherit profileName; }).out;
|
garage_db = (rustPackages."unknown".garage_db."0.8.0" { inherit profileName; }).out;
|
||||||
git_version = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".git-version."0.3.5" { inherit profileName; }).out;
|
git_version = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".git-version."0.3.5" { inherit profileName; }).out;
|
||||||
hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out;
|
hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out;
|
||||||
hexdump = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hexdump."0.1.1" { inherit profileName; }).out;
|
|
||||||
http = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.8" { inherit profileName; }).out;
|
http = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.8" { inherit profileName; }).out;
|
||||||
hyper = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hyper."0.14.18" { inherit profileName; }).out;
|
hyper = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hyper."0.14.18" { inherit profileName; }).out;
|
||||||
lazy_static = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".lazy_static."1.4.0" { inherit profileName; }).out;
|
lazy_static = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".lazy_static."1.4.0" { inherit profileName; }).out;
|
||||||
@@ -1822,18 +1826,18 @@ in
|
|||||||
};
|
};
|
||||||
});
|
});
|
||||||
|
|
||||||
"unknown".garage_web."0.8.1" = overridableMkRustCrate (profileName: rec {
|
"unknown".garage_web."0.8.0" = overridableMkRustCrate (profileName: rec {
|
||||||
name = "garage_web";
|
name = "garage_web";
|
||||||
version = "0.8.1";
|
version = "0.8.0";
|
||||||
registry = "unknown";
|
registry = "unknown";
|
||||||
src = fetchCrateLocal (workspaceSrc + "/src/web");
|
src = fetchCrateLocal (workspaceSrc + "/src/web");
|
||||||
dependencies = {
|
dependencies = {
|
||||||
err_derive = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".err-derive."0.3.1" { profileName = "__noProfile"; }).out;
|
err_derive = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".err-derive."0.3.1" { profileName = "__noProfile"; }).out;
|
||||||
futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.21" { inherit profileName; }).out;
|
futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.21" { inherit profileName; }).out;
|
||||||
garage_api = (rustPackages."unknown".garage_api."0.8.1" { inherit profileName; }).out;
|
garage_api = (rustPackages."unknown".garage_api."0.8.0" { inherit profileName; }).out;
|
||||||
garage_model = (rustPackages."unknown".garage_model."0.8.1" { inherit profileName; }).out;
|
garage_model = (rustPackages."unknown".garage_model."0.8.0" { inherit profileName; }).out;
|
||||||
garage_table = (rustPackages."unknown".garage_table."0.8.1" { inherit profileName; }).out;
|
garage_table = (rustPackages."unknown".garage_table."0.8.0" { inherit profileName; }).out;
|
||||||
garage_util = (rustPackages."unknown".garage_util."0.8.1" { inherit profileName; }).out;
|
garage_util = (rustPackages."unknown".garage_util."0.8.0" { inherit profileName; }).out;
|
||||||
http = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.8" { inherit profileName; }).out;
|
http = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.8" { inherit profileName; }).out;
|
||||||
hyper = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hyper."0.14.18" { inherit profileName; }).out;
|
hyper = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hyper."0.14.18" { inherit profileName; }).out;
|
||||||
opentelemetry = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry."0.17.0" { inherit profileName; }).out;
|
opentelemetry = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry."0.17.0" { inherit profileName; }).out;
|
||||||
@@ -2447,7 +2451,7 @@ in
|
|||||||
dependencies = {
|
dependencies = {
|
||||||
base64 = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".base64."0.13.0" { inherit profileName; }).out;
|
base64 = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".base64."0.13.0" { inherit profileName; }).out;
|
||||||
${ if rootFeatures' ? "k2v-client/clap" || rootFeatures' ? "k2v-client/cli" then "clap" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".clap."3.1.18" { inherit profileName; }).out;
|
${ if rootFeatures' ? "k2v-client/clap" || rootFeatures' ? "k2v-client/cli" then "clap" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".clap."3.1.18" { inherit profileName; }).out;
|
||||||
${ if rootFeatures' ? "k2v-client/cli" || rootFeatures' ? "k2v-client/garage_util" then "garage_util" else null } = (rustPackages."unknown".garage_util."0.8.1" { inherit profileName; }).out;
|
${ if rootFeatures' ? "k2v-client/cli" || rootFeatures' ? "k2v-client/garage_util" then "garage_util" else null } = (rustPackages."unknown".garage_util."0.8.0" { inherit profileName; }).out;
|
||||||
http = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.8" { inherit profileName; }).out;
|
http = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.8" { inherit profileName; }).out;
|
||||||
log = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".log."0.4.16" { inherit profileName; }).out;
|
log = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".log."0.4.16" { inherit profileName; }).out;
|
||||||
rusoto_core = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rusoto_core."0.48.0" { inherit profileName; }).out;
|
rusoto_core = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rusoto_core."0.48.0" { inherit profileName; }).out;
|
||||||
|
|||||||
@@ -5,59 +5,16 @@ weight = 20
|
|||||||
|
|
||||||
## S3
|
## S3
|
||||||
|
|
||||||
### Using Minio SDK
|
|
||||||
|
|
||||||
First install the SDK:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
pip3 install minio
|
|
||||||
```
|
|
||||||
|
|
||||||
Then instantiate a client object using garage root domain, api key and secret:
|
|
||||||
|
|
||||||
```python
|
|
||||||
import minio
|
|
||||||
|
|
||||||
client = minio.Minio(
|
|
||||||
"your.domain.tld",
|
|
||||||
"GKyourapikey",
|
|
||||||
"abcd[...]1234",
|
|
||||||
# Force the region, this is specific to garage
|
|
||||||
region="region",
|
|
||||||
)
|
|
||||||
```
|
|
||||||
|
|
||||||
Then use all the standard S3 endpoints as implemented by the Minio SDK:
|
|
||||||
|
|
||||||
```
|
|
||||||
# List buckets
|
|
||||||
print(client.list_buckets())
|
|
||||||
|
|
||||||
# Put an object containing 'content' to /path in bucket named 'bucket':
|
|
||||||
content = b"content"
|
|
||||||
client.put_object(
|
|
||||||
"bucket",
|
|
||||||
"path",
|
|
||||||
io.BytesIO(content),
|
|
||||||
len(content),
|
|
||||||
)
|
|
||||||
|
|
||||||
# Read the object back and check contents
|
|
||||||
data = client.get_object("bucket", "path").read()
|
|
||||||
assert data == content
|
|
||||||
```
|
|
||||||
|
|
||||||
For further documentation, see the Minio SDK
|
|
||||||
[Reference](https://docs.min.io/docs/python-client-api-reference.html)
|
|
||||||
|
|
||||||
### Using Amazon boto3
|
|
||||||
|
|
||||||
*Coming soon*
|
*Coming soon*
|
||||||
|
|
||||||
See the official documentation:
|
Some refs:
|
||||||
- [Installation](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/quickstart.html)
|
- Minio SDK
|
||||||
- [Reference](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/s3.html)
|
- [Reference](https://docs.min.io/docs/python-client-api-reference.html)
|
||||||
- [Example](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/s3-uploading-files.html)
|
|
||||||
|
- Amazon boto3
|
||||||
|
- [Installation](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/quickstart.html)
|
||||||
|
- [Reference](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/s3.html)
|
||||||
|
- [Example](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/s3-uploading-files.html)
|
||||||
|
|
||||||
## K2V
|
## K2V
|
||||||
|
|
||||||
|
|||||||
@@ -8,7 +8,7 @@ In this section, we cover the following web applications:
|
|||||||
| Name | Status | Note |
|
| Name | Status | Note |
|
||||||
|------|--------|------|
|
|------|--------|------|
|
||||||
| [Nextcloud](#nextcloud) | ✅ | Both Primary Storage and External Storage are supported |
|
| [Nextcloud](#nextcloud) | ✅ | Both Primary Storage and External Storage are supported |
|
||||||
| [Peertube](#peertube) | ✅ | Supported with the website endpoint, proxifying private videos unsupported |
|
| [Peertube](#peertube) | ✅ | Must be configured with the website endpoint |
|
||||||
| [Mastodon](#mastodon) | ✅ | Natively supported |
|
| [Mastodon](#mastodon) | ✅ | Natively supported |
|
||||||
| [Matrix](#matrix) | ✅ | Tested with `synapse-s3-storage-provider` |
|
| [Matrix](#matrix) | ✅ | Tested with `synapse-s3-storage-provider` |
|
||||||
| [Pixelfed](#pixelfed) | ❓ | Not yet tested |
|
| [Pixelfed](#pixelfed) | ❓ | Not yet tested |
|
||||||
@@ -128,10 +128,6 @@ In other words, Peertube is only responsible of the "control plane" and offload
|
|||||||
In return, this system is a bit harder to configure.
|
In return, this system is a bit harder to configure.
|
||||||
We show how it is still possible to configure Garage with Peertube, allowing you to spread the load and the bandwidth usage on the Garage cluster.
|
We show how it is still possible to configure Garage with Peertube, allowing you to spread the load and the bandwidth usage on the Garage cluster.
|
||||||
|
|
||||||
Starting from version 5.0, Peertube also supports improving the security for private videos by not exposing them directly
|
|
||||||
but relying on a single control point in the Peertube instance. This is based on S3 per-object and prefix ACL, which are not currently supported
|
|
||||||
in Garage, so this feature is unsupported. While this technically impedes security for private videos, it is not a blocking issue and could be
|
|
||||||
a reasonable trade-off for some instances.
|
|
||||||
|
|
||||||
### Create resources in Garage
|
### Create resources in Garage
|
||||||
|
|
||||||
@@ -199,11 +195,6 @@ object_storage:
|
|||||||
|
|
||||||
max_upload_part: 2GB
|
max_upload_part: 2GB
|
||||||
|
|
||||||
proxy:
|
|
||||||
# You may enable this feature, yet it will not provide any security benefit, so
|
|
||||||
# you should rather benefit from Garage public endpoint for all videos
|
|
||||||
proxify_private_files: false
|
|
||||||
|
|
||||||
streaming_playlists:
|
streaming_playlists:
|
||||||
bucket_name: 'peertube-playlist'
|
bucket_name: 'peertube-playlist'
|
||||||
|
|
||||||
|
|||||||
Generated
+8
-8
@@ -10,17 +10,17 @@
|
|||||||
"rust-overlay": "rust-overlay"
|
"rust-overlay": "rust-overlay"
|
||||||
},
|
},
|
||||||
"locked": {
|
"locked": {
|
||||||
"lastModified": 1673262828,
|
"lastModified": 1666087781,
|
||||||
"narHash": "sha256-pDqno5/2ghQDt4LjVt5ZUMV9pTSA5rGGdz6Skf2rBwc=",
|
"narHash": "sha256-trKVdjMZ8mNkGfLcY5LsJJGtdV3xJDZnMVrkFjErlcs=",
|
||||||
"owner": "Alexis211",
|
"owner": "Alexis211",
|
||||||
"repo": "cargo2nix",
|
"repo": "cargo2nix",
|
||||||
"rev": "505caa32110d42ee03bd68b47031142eff9c827b",
|
"rev": "a7a61179b66054904ef6a195d8da736eaaa06c36",
|
||||||
"type": "github"
|
"type": "github"
|
||||||
},
|
},
|
||||||
"original": {
|
"original": {
|
||||||
"owner": "Alexis211",
|
"owner": "Alexis211",
|
||||||
"repo": "cargo2nix",
|
"repo": "cargo2nix",
|
||||||
"rev": "505caa32110d42ee03bd68b47031142eff9c827b",
|
"rev": "a7a61179b66054904ef6a195d8da736eaaa06c36",
|
||||||
"type": "github"
|
"type": "github"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
@@ -57,17 +57,17 @@
|
|||||||
},
|
},
|
||||||
"nixpkgs": {
|
"nixpkgs": {
|
||||||
"locked": {
|
"locked": {
|
||||||
"lastModified": 1673261889,
|
"lastModified": 1665657542,
|
||||||
"narHash": "sha256-7trMsi0z7EfYNC/Nc5EtulvChBHQAo376XRICyWr89Q=",
|
"narHash": "sha256-mojxNyzbvmp8NtVtxqiHGhRfjCALLfk9i/Uup68Y5q8=",
|
||||||
"owner": "NixOS",
|
"owner": "NixOS",
|
||||||
"repo": "nixpkgs",
|
"repo": "nixpkgs",
|
||||||
"rev": "baed728abe983508cabc99d05cccc164fe748744",
|
"rev": "a3073c49bc0163fea6a121c276f526837672b555",
|
||||||
"type": "github"
|
"type": "github"
|
||||||
},
|
},
|
||||||
"original": {
|
"original": {
|
||||||
"owner": "NixOS",
|
"owner": "NixOS",
|
||||||
"repo": "nixpkgs",
|
"repo": "nixpkgs",
|
||||||
"rev": "baed728abe983508cabc99d05cccc164fe748744",
|
"rev": "a3073c49bc0163fea6a121c276f526837672b555",
|
||||||
"type": "github"
|
"type": "github"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
|||||||
@@ -1,10 +1,10 @@
|
|||||||
{
|
{
|
||||||
description = "Garage, an S3-compatible distributed object store for self-hosted deployments";
|
description = "Garage, an S3-compatible distributed object store for self-hosted deployments";
|
||||||
|
|
||||||
inputs.nixpkgs.url = "github:NixOS/nixpkgs/baed728abe983508cabc99d05cccc164fe748744";
|
inputs.nixpkgs.url = "github:NixOS/nixpkgs/a3073c49bc0163fea6a121c276f526837672b555";
|
||||||
inputs.cargo2nix = {
|
inputs.cargo2nix = {
|
||||||
# As of 2022-10-18: two small patches over unstable branch, one for clippy and one to fix feature detection
|
# As of 2022-10-18: two small patches over unstable branch, one for clippy and one to fix feature detection
|
||||||
url = "github:Alexis211/cargo2nix/505caa32110d42ee03bd68b47031142eff9c827b";
|
url = "github:Alexis211/cargo2nix/a7a61179b66054904ef6a195d8da736eaaa06c36";
|
||||||
inputs.nixpkgs.follows = "nixpkgs";
|
inputs.nixpkgs.follows = "nixpkgs";
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|||||||
+4
-4
@@ -3,15 +3,15 @@ rec {
|
|||||||
* Fixed dependencies
|
* Fixed dependencies
|
||||||
*/
|
*/
|
||||||
pkgsSrc = fetchTarball {
|
pkgsSrc = fetchTarball {
|
||||||
# As of 2023-01-09
|
# As of 2022-10-13
|
||||||
url = "https://github.com/NixOS/nixpkgs/archive/baed728abe983508cabc99d05cccc164fe748744.zip";
|
url = "https://github.com/NixOS/nixpkgs/archive/a3073c49bc0163fea6a121c276f526837672b555.zip";
|
||||||
sha256 = "1m7kmcjhnj3lx7xqs0nh262c4nxs5n8p7k9g6kc4gv1k5nrcrnpf";
|
sha256 = "1bz632psfbpmicyzjb8b4265y50shylccvfm6ry6mgnv5hvz324s";
|
||||||
};
|
};
|
||||||
cargo2nixSrc = fetchGit {
|
cargo2nixSrc = fetchGit {
|
||||||
# As of 2022-10-18: two small patches over unstable branch, one for clippy and one to fix feature detection
|
# As of 2022-10-18: two small patches over unstable branch, one for clippy and one to fix feature detection
|
||||||
url = "https://github.com/Alexis211/cargo2nix";
|
url = "https://github.com/Alexis211/cargo2nix";
|
||||||
ref = "custom_unstable";
|
ref = "custom_unstable";
|
||||||
rev = "505caa32110d42ee03bd68b47031142eff9c827b";
|
rev = "a7a61179b66054904ef6a195d8da736eaaa06c36";
|
||||||
};
|
};
|
||||||
|
|
||||||
/*
|
/*
|
||||||
|
|||||||
+1
-1
@@ -42,7 +42,7 @@ let
|
|||||||
*/
|
*/
|
||||||
toolchainOptions =
|
toolchainOptions =
|
||||||
if target == null || target == "x86_64-unknown-linux-musl" || target == "aarch64-unknown-linux-musl" then {
|
if target == null || target == "x86_64-unknown-linux-musl" || target == "aarch64-unknown-linux-musl" then {
|
||||||
rustVersion = "1.65.0";
|
rustVersion = "1.63.0";
|
||||||
extraRustComponents = [ "clippy" ];
|
extraRustComponents = [ "clippy" ];
|
||||||
} else {
|
} else {
|
||||||
rustToolchain = pkgs.symlinkJoin {
|
rustToolchain = pkgs.symlinkJoin {
|
||||||
|
|||||||
+1
-1
@@ -7,7 +7,7 @@ pkgs.buildGoModule rec {
|
|||||||
owner = "GoogleContainerTools";
|
owner = "GoogleContainerTools";
|
||||||
repo = "kaniko";
|
repo = "kaniko";
|
||||||
rev = "v${version}";
|
rev = "v${version}";
|
||||||
sha256 = "TXgzO/NfLXVo5a7yyO3XYSk+9H1CwMF+vwbRx3kchQ8=";
|
sha256 = "1fnclr556avxay6pvgw5ya3xbxfnf2gv4njq2hr4fd6fcjyslq5h";
|
||||||
};
|
};
|
||||||
|
|
||||||
vendorSha256 = null;
|
vendorSha256 = null;
|
||||||
|
|||||||
+6
-6
@@ -1,6 +1,6 @@
|
|||||||
[package]
|
[package]
|
||||||
name = "garage_api"
|
name = "garage_api"
|
||||||
version = "0.8.1"
|
version = "0.8.0"
|
||||||
authors = ["Alex Auvolat <alex@adnab.me>"]
|
authors = ["Alex Auvolat <alex@adnab.me>"]
|
||||||
edition = "2018"
|
edition = "2018"
|
||||||
license = "AGPL-3.0"
|
license = "AGPL-3.0"
|
||||||
@@ -14,11 +14,11 @@ path = "lib.rs"
|
|||||||
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
|
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
|
||||||
|
|
||||||
[dependencies]
|
[dependencies]
|
||||||
garage_model = { version = "0.8.1", path = "../model" }
|
garage_model = { version = "0.8.0", path = "../model" }
|
||||||
garage_table = { version = "0.8.1", path = "../table" }
|
garage_table = { version = "0.8.0", path = "../table" }
|
||||||
garage_block = { version = "0.8.1", path = "../block" }
|
garage_block = { version = "0.8.0", path = "../block" }
|
||||||
garage_util = { version = "0.8.1", path = "../util" }
|
garage_util = { version = "0.8.0", path = "../util" }
|
||||||
garage_rpc = { version = "0.8.1", path = "../rpc" }
|
garage_rpc = { version = "0.8.0", path = "../rpc" }
|
||||||
|
|
||||||
async-trait = "0.1.7"
|
async-trait = "0.1.7"
|
||||||
base64 = "0.13"
|
base64 = "0.13"
|
||||||
|
|||||||
@@ -143,13 +143,10 @@ impl Endpoint {
|
|||||||
}
|
}
|
||||||
|
|
||||||
generateQueryParameters! {
|
generateQueryParameters! {
|
||||||
keywords: [],
|
"format" => format,
|
||||||
fields: [
|
"id" => id,
|
||||||
"format" => format,
|
"search" => search,
|
||||||
"id" => id,
|
"globalAlias" => global_alias,
|
||||||
"search" => search,
|
"alias" => alias,
|
||||||
"globalAlias" => global_alias,
|
"accessKeyId" => access_key_id
|
||||||
"alias" => alias,
|
|
||||||
"accessKeyId" => access_key_id
|
|
||||||
]
|
|
||||||
}
|
}
|
||||||
|
|||||||
+1
-1
@@ -211,7 +211,7 @@ pub async fn handle_poll_item(
|
|||||||
let item = garage
|
let item = garage
|
||||||
.k2v
|
.k2v
|
||||||
.rpc
|
.rpc
|
||||||
.poll_item(
|
.poll(
|
||||||
bucket_id,
|
bucket_id,
|
||||||
partition_key,
|
partition_key,
|
||||||
sort_key,
|
sort_key,
|
||||||
|
|||||||
+23
-20
@@ -96,7 +96,7 @@ impl Endpoint {
|
|||||||
fn from_get(partition_key: String, query: &mut QueryParameters<'_>) -> Result<Self, Error> {
|
fn from_get(partition_key: String, query: &mut QueryParameters<'_>) -> Result<Self, Error> {
|
||||||
router_match! {
|
router_match! {
|
||||||
@gen_parser
|
@gen_parser
|
||||||
(query.keyword.take().unwrap_or_default(), partition_key, query, None),
|
(query.keyword.take().unwrap_or_default().as_ref(), partition_key, query, None),
|
||||||
key: [
|
key: [
|
||||||
EMPTY if causality_token => PollItem (query::sort_key, query::causality_token, opt_parse::timeout),
|
EMPTY if causality_token => PollItem (query::sort_key, query::causality_token, opt_parse::timeout),
|
||||||
EMPTY => ReadItem (query::sort_key),
|
EMPTY => ReadItem (query::sort_key),
|
||||||
@@ -111,7 +111,7 @@ impl Endpoint {
|
|||||||
fn from_search(partition_key: String, query: &mut QueryParameters<'_>) -> Result<Self, Error> {
|
fn from_search(partition_key: String, query: &mut QueryParameters<'_>) -> Result<Self, Error> {
|
||||||
router_match! {
|
router_match! {
|
||||||
@gen_parser
|
@gen_parser
|
||||||
(query.keyword.take().unwrap_or_default(), partition_key, query, None),
|
(query.keyword.take().unwrap_or_default().as_ref(), partition_key, query, None),
|
||||||
key: [
|
key: [
|
||||||
],
|
],
|
||||||
no_key: [
|
no_key: [
|
||||||
@@ -125,7 +125,7 @@ impl Endpoint {
|
|||||||
fn from_head(partition_key: String, query: &mut QueryParameters<'_>) -> Result<Self, Error> {
|
fn from_head(partition_key: String, query: &mut QueryParameters<'_>) -> Result<Self, Error> {
|
||||||
router_match! {
|
router_match! {
|
||||||
@gen_parser
|
@gen_parser
|
||||||
(query.keyword.take().unwrap_or_default(), partition_key, query, None),
|
(query.keyword.take().unwrap_or_default().as_ref(), partition_key, query, None),
|
||||||
key: [
|
key: [
|
||||||
EMPTY => HeadObject(opt_parse::part_number, query_opt::version_id),
|
EMPTY => HeadObject(opt_parse::part_number, query_opt::version_id),
|
||||||
],
|
],
|
||||||
@@ -140,7 +140,7 @@ impl Endpoint {
|
|||||||
fn from_post(partition_key: String, query: &mut QueryParameters<'_>) -> Result<Self, Error> {
|
fn from_post(partition_key: String, query: &mut QueryParameters<'_>) -> Result<Self, Error> {
|
||||||
router_match! {
|
router_match! {
|
||||||
@gen_parser
|
@gen_parser
|
||||||
(query.keyword.take().unwrap_or_default(), partition_key, query, None),
|
(query.keyword.take().unwrap_or_default().as_ref(), partition_key, query, None),
|
||||||
key: [
|
key: [
|
||||||
],
|
],
|
||||||
no_key: [
|
no_key: [
|
||||||
@@ -155,7 +155,7 @@ impl Endpoint {
|
|||||||
fn from_put(partition_key: String, query: &mut QueryParameters<'_>) -> Result<Self, Error> {
|
fn from_put(partition_key: String, query: &mut QueryParameters<'_>) -> Result<Self, Error> {
|
||||||
router_match! {
|
router_match! {
|
||||||
@gen_parser
|
@gen_parser
|
||||||
(query.keyword.take().unwrap_or_default(), partition_key, query, None),
|
(query.keyword.take().unwrap_or_default().as_ref(), partition_key, query, None),
|
||||||
key: [
|
key: [
|
||||||
EMPTY => InsertItem (query::sort_key),
|
EMPTY => InsertItem (query::sort_key),
|
||||||
|
|
||||||
@@ -169,7 +169,7 @@ impl Endpoint {
|
|||||||
fn from_delete(partition_key: String, query: &mut QueryParameters<'_>) -> Result<Self, Error> {
|
fn from_delete(partition_key: String, query: &mut QueryParameters<'_>) -> Result<Self, Error> {
|
||||||
router_match! {
|
router_match! {
|
||||||
@gen_parser
|
@gen_parser
|
||||||
(query.keyword.take().unwrap_or_default(), partition_key, query, None),
|
(query.keyword.take().unwrap_or_default().as_ref(), partition_key, query, None),
|
||||||
key: [
|
key: [
|
||||||
EMPTY => DeleteItem (query::sort_key),
|
EMPTY => DeleteItem (query::sort_key),
|
||||||
],
|
],
|
||||||
@@ -232,18 +232,21 @@ impl Endpoint {
|
|||||||
|
|
||||||
// parameter name => struct field
|
// parameter name => struct field
|
||||||
generateQueryParameters! {
|
generateQueryParameters! {
|
||||||
keywords: [
|
"prefix" => prefix,
|
||||||
"delete" => DELETE,
|
"start" => start,
|
||||||
"search" => SEARCH
|
"causality_token" => causality_token,
|
||||||
],
|
"end" => end,
|
||||||
fields: [
|
"limit" => limit,
|
||||||
"prefix" => prefix,
|
"reverse" => reverse,
|
||||||
"start" => start,
|
"sort_key" => sort_key,
|
||||||
"causality_token" => causality_token,
|
"timeout" => timeout
|
||||||
"end" => end,
|
}
|
||||||
"limit" => limit,
|
|
||||||
"reverse" => reverse,
|
mod keywords {
|
||||||
"sort_key" => sort_key,
|
//! This module contain all query parameters with no associated value
|
||||||
"timeout" => timeout
|
//! used to differentiate endpoints.
|
||||||
]
|
pub const EMPTY: &str = "";
|
||||||
|
|
||||||
|
pub const DELETE: &str = "delete";
|
||||||
|
pub const SEARCH: &str = "search";
|
||||||
}
|
}
|
||||||
|
|||||||
+58
-74
@@ -4,9 +4,10 @@ macro_rules! router_match {
|
|||||||
(@match $enum:expr , [ $($endpoint:ident,)* ]) => {{
|
(@match $enum:expr , [ $($endpoint:ident,)* ]) => {{
|
||||||
// usage: router_match {@match my_enum, [ VariantWithField1, VariantWithField2 ..] }
|
// usage: router_match {@match my_enum, [ VariantWithField1, VariantWithField2 ..] }
|
||||||
// returns true if the variant was one of the listed variants, false otherwise.
|
// returns true if the variant was one of the listed variants, false otherwise.
|
||||||
|
use Endpoint::*;
|
||||||
match $enum {
|
match $enum {
|
||||||
$(
|
$(
|
||||||
Endpoint::$endpoint { .. } => true,
|
$endpoint { .. } => true,
|
||||||
)*
|
)*
|
||||||
_ => false
|
_ => false
|
||||||
}
|
}
|
||||||
@@ -14,35 +15,37 @@ macro_rules! router_match {
|
|||||||
(@extract $enum:expr , $param:ident, [ $($endpoint:ident,)* ]) => {{
|
(@extract $enum:expr , $param:ident, [ $($endpoint:ident,)* ]) => {{
|
||||||
// usage: router_match {@extract my_enum, field_name, [ VariantWithField1, VariantWithField2 ..] }
|
// usage: router_match {@extract my_enum, field_name, [ VariantWithField1, VariantWithField2 ..] }
|
||||||
// returns Some(field_value), or None if the variant was not one of the listed variants.
|
// returns Some(field_value), or None if the variant was not one of the listed variants.
|
||||||
|
use Endpoint::*;
|
||||||
match $enum {
|
match $enum {
|
||||||
$(
|
$(
|
||||||
Endpoint::$endpoint {$param, ..} => Some($param),
|
$endpoint {$param, ..} => Some($param),
|
||||||
)*
|
)*
|
||||||
_ => None
|
_ => None
|
||||||
}
|
}
|
||||||
}};
|
}};
|
||||||
(@gen_path_parser ($method:expr, $reqpath:expr, $query:expr)
|
(@gen_path_parser ($method:expr, $reqpath:expr, $query:expr)
|
||||||
[
|
[
|
||||||
$($meth:ident $path:pat $(if $required:ident)? => $api:ident $(($($conv:ident :: $param:ident),*))?,)*
|
$($meth:ident $path:pat $(if $required:ident)? => $api:ident $(($($conv:ident :: $param:ident),*))?,)*
|
||||||
]) => {{
|
]) => {{
|
||||||
{
|
{
|
||||||
match ($method, $reqpath) {
|
use Endpoint::*;
|
||||||
$(
|
match ($method, $reqpath) {
|
||||||
(&Method::$meth, $path) if true $(&& $query.$required.is_some())? => Endpoint::$api {
|
$(
|
||||||
$($(
|
(&Method::$meth, $path) if true $(&& $query.$required.is_some())? => $api {
|
||||||
$param: router_match!(@@parse_param $query, $conv, $param),
|
$($(
|
||||||
)*)?
|
$param: router_match!(@@parse_param $query, $conv, $param),
|
||||||
},
|
)*)?
|
||||||
)*
|
},
|
||||||
(m, p) => {
|
)*
|
||||||
return Err(Error::bad_request(format!(
|
(m, p) => {
|
||||||
"Unknown API endpoint: {} {}",
|
return Err(Error::bad_request(format!(
|
||||||
m, p
|
"Unknown API endpoint: {} {}",
|
||||||
)))
|
m, p
|
||||||
}
|
)))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}};
|
}
|
||||||
|
}};
|
||||||
(@gen_parser ($keyword:expr, $key:ident, $query:expr, $header:expr),
|
(@gen_parser ($keyword:expr, $key:ident, $query:expr, $header:expr),
|
||||||
key: [$($kw_k:ident $(if $required_k:ident)? $(header $header_k:expr)? => $api_k:ident $(($($conv_k:ident :: $param_k:ident),*))?,)*],
|
key: [$($kw_k:ident $(if $required_k:ident)? $(header $header_k:expr)? => $api_k:ident $(($($conv_k:ident :: $param_k:ident),*))?,)*],
|
||||||
no_key: [$($kw_nk:ident $(if $required_nk:ident)? $(if_header $header_nk:expr)? => $api_nk:ident $(($($conv_nk:ident :: $param_nk:ident),*))?,)*]) => {{
|
no_key: [$($kw_nk:ident $(if $required_nk:ident)? $(if_header $header_nk:expr)? => $api_nk:ident $(($($conv_nk:ident :: $param_nk:ident),*))?,)*]) => {{
|
||||||
@@ -57,9 +60,11 @@ macro_rules! router_match {
|
|||||||
// ]
|
// ]
|
||||||
// }
|
// }
|
||||||
// See in from_{method} for more detailed usage.
|
// See in from_{method} for more detailed usage.
|
||||||
|
use Endpoint::*;
|
||||||
|
use keywords::*;
|
||||||
match ($keyword, !$key.is_empty()){
|
match ($keyword, !$key.is_empty()){
|
||||||
$(
|
$(
|
||||||
(Keyword::$kw_k, true) if true $(&& $query.$required_k.is_some())? $(&& $header.contains_key($header_k))? => Ok(Endpoint::$api_k {
|
($kw_k, true) if true $(&& $query.$required_k.is_some())? $(&& $header.contains_key($header_k))? => Ok($api_k {
|
||||||
$key,
|
$key,
|
||||||
$($(
|
$($(
|
||||||
$param_k: router_match!(@@parse_param $query, $conv_k, $param_k),
|
$param_k: router_match!(@@parse_param $query, $conv_k, $param_k),
|
||||||
@@ -67,7 +72,7 @@ macro_rules! router_match {
|
|||||||
}),
|
}),
|
||||||
)*
|
)*
|
||||||
$(
|
$(
|
||||||
(Keyword::$kw_nk, false) $(if $query.$required_nk.is_some())? $(if $header.contains($header_nk))? => Ok(Endpoint::$api_nk {
|
($kw_nk, false) $(if $query.$required_nk.is_some())? $(if $header.contains($header_nk))? => Ok($api_nk {
|
||||||
$($(
|
$($(
|
||||||
$param_nk: router_match!(@@parse_param $query, $conv_nk, $param_nk),
|
$param_nk: router_match!(@@parse_param $query, $conv_nk, $param_nk),
|
||||||
)*)?
|
)*)?
|
||||||
@@ -79,7 +84,7 @@ macro_rules! router_match {
|
|||||||
|
|
||||||
(@@parse_param $query:expr, query_opt, $param:ident) => {{
|
(@@parse_param $query:expr, query_opt, $param:ident) => {{
|
||||||
// extract optional query parameter
|
// extract optional query parameter
|
||||||
$query.$param.take().map(|param| param.into_owned())
|
$query.$param.take().map(|param| param.into_owned())
|
||||||
}};
|
}};
|
||||||
(@@parse_param $query:expr, query, $param:ident) => {{
|
(@@parse_param $query:expr, query, $param:ident) => {{
|
||||||
// extract mendatory query parameter
|
// extract mendatory query parameter
|
||||||
@@ -88,7 +93,7 @@ macro_rules! router_match {
|
|||||||
(@@parse_param $query:expr, opt_parse, $param:ident) => {{
|
(@@parse_param $query:expr, opt_parse, $param:ident) => {{
|
||||||
// extract and parse optional query parameter
|
// extract and parse optional query parameter
|
||||||
// missing parameter is file, however parse error is reported as an error
|
// missing parameter is file, however parse error is reported as an error
|
||||||
$query.$param
|
$query.$param
|
||||||
.take()
|
.take()
|
||||||
.map(|param| param.parse())
|
.map(|param| param.parse())
|
||||||
.transpose()
|
.transpose()
|
||||||
@@ -139,40 +144,14 @@ macro_rules! router_match {
|
|||||||
/// This macro is used to generate part of the code in this module. It must be called only one, and
|
/// This macro is used to generate part of the code in this module. It must be called only one, and
|
||||||
/// is useless outside of this module.
|
/// is useless outside of this module.
|
||||||
macro_rules! generateQueryParameters {
|
macro_rules! generateQueryParameters {
|
||||||
(
|
( $($rest:expr => $name:ident),* ) => {
|
||||||
keywords: [ $($kw_param:expr => $kw_name: ident),* ],
|
|
||||||
fields: [ $($f_param:expr => $f_name:ident),* ]
|
|
||||||
) => {
|
|
||||||
#[derive(Debug)]
|
|
||||||
#[allow(non_camel_case_types)]
|
|
||||||
#[allow(clippy::upper_case_acronyms)]
|
|
||||||
enum Keyword {
|
|
||||||
EMPTY,
|
|
||||||
$( $kw_name, )*
|
|
||||||
}
|
|
||||||
|
|
||||||
impl std::fmt::Display for Keyword {
|
|
||||||
fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
|
|
||||||
match self {
|
|
||||||
Keyword::EMPTY => write!(f, "``"),
|
|
||||||
$( Keyword::$kw_name => write!(f, "`{}`", $kw_param), )*
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl Default for Keyword {
|
|
||||||
fn default() -> Self {
|
|
||||||
Keyword::EMPTY
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Struct containing all query parameters used in endpoints. Think of it as an HashMap,
|
/// Struct containing all query parameters used in endpoints. Think of it as an HashMap,
|
||||||
/// but with keys statically known.
|
/// but with keys statically known.
|
||||||
#[derive(Debug, Default)]
|
#[derive(Debug, Default)]
|
||||||
struct QueryParameters<'a> {
|
struct QueryParameters<'a> {
|
||||||
keyword: Option<Keyword>,
|
keyword: Option<Cow<'a, str>>,
|
||||||
$(
|
$(
|
||||||
$f_name: Option<Cow<'a, str>>,
|
$name: Option<Cow<'a, str>>,
|
||||||
)*
|
)*
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -181,29 +160,34 @@ macro_rules! generateQueryParameters {
|
|||||||
fn from_query(query: &'a str) -> Result<Self, Error> {
|
fn from_query(query: &'a str) -> Result<Self, Error> {
|
||||||
let mut res: Self = Default::default();
|
let mut res: Self = Default::default();
|
||||||
for (k, v) in url::form_urlencoded::parse(query.as_bytes()) {
|
for (k, v) in url::form_urlencoded::parse(query.as_bytes()) {
|
||||||
match k.as_ref() {
|
let repeated = match k.as_ref() {
|
||||||
$(
|
$(
|
||||||
$kw_param => if let Some(prev_kw) = res.keyword.replace(Keyword::$kw_name) {
|
$rest => if !v.is_empty() {
|
||||||
return Err(Error::bad_request(format!(
|
res.$name.replace(v).is_some()
|
||||||
"Multiple keywords: '{}' and '{}'", prev_kw, $kw_param
|
} else {
|
||||||
)));
|
false
|
||||||
},
|
|
||||||
)*
|
|
||||||
$(
|
|
||||||
$f_param => if !v.is_empty() {
|
|
||||||
if res.$f_name.replace(v).is_some() {
|
|
||||||
return Err(Error::bad_request(format!(
|
|
||||||
"Query parameter repeated: '{}'", k
|
|
||||||
)));
|
|
||||||
}
|
|
||||||
},
|
},
|
||||||
)*
|
)*
|
||||||
_ => {
|
_ => {
|
||||||
if !(k.starts_with("response-") || k.starts_with("X-Amz-")) {
|
if k.starts_with("response-") || k.starts_with("X-Amz-") {
|
||||||
|
false
|
||||||
|
} else if v.as_ref().is_empty() {
|
||||||
|
if res.keyword.replace(k).is_some() {
|
||||||
|
return Err(Error::bad_request("Multiple keywords"));
|
||||||
|
}
|
||||||
|
continue;
|
||||||
|
} else {
|
||||||
debug!("Received an unknown query parameter: '{}'", k);
|
debug!("Received an unknown query parameter: '{}'", k);
|
||||||
|
false
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
if repeated {
|
||||||
|
return Err(Error::bad_request(format!(
|
||||||
|
"Query parameter repeated: '{}'",
|
||||||
|
k
|
||||||
|
)));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
Ok(res)
|
Ok(res)
|
||||||
}
|
}
|
||||||
@@ -214,8 +198,8 @@ macro_rules! generateQueryParameters {
|
|||||||
if self.keyword.is_some() {
|
if self.keyword.is_some() {
|
||||||
Some("Keyword not used")
|
Some("Keyword not used")
|
||||||
} $(
|
} $(
|
||||||
else if self.$f_name.is_some() {
|
else if self.$name.is_some() {
|
||||||
Some(concat!("'", $f_param, "'"))
|
Some(concat!("'", $rest, "'"))
|
||||||
}
|
}
|
||||||
)* else {
|
)* else {
|
||||||
None
|
None
|
||||||
|
|||||||
+34
-59
@@ -119,17 +119,6 @@ pub(crate) async fn save_stream<S: Stream<Item = Result<Bytes, Error>> + Unpin>(
|
|||||||
return Ok((version_uuid, data_md5sum_hex));
|
return Ok((version_uuid, data_md5sum_hex));
|
||||||
}
|
}
|
||||||
|
|
||||||
// The following consists in many steps that can each fail.
|
|
||||||
// Keep track that some cleanup will be needed if things fail
|
|
||||||
// before everything is finished (cleanup is done using the Drop trait).
|
|
||||||
let mut interrupted_cleanup = InterruptedCleanup(Some((
|
|
||||||
garage.clone(),
|
|
||||||
bucket.id,
|
|
||||||
key.into(),
|
|
||||||
version_uuid,
|
|
||||||
version_timestamp,
|
|
||||||
)));
|
|
||||||
|
|
||||||
// Write version identifier in object table so that we have a trace
|
// Write version identifier in object table so that we have a trace
|
||||||
// that we are uploading something
|
// that we are uploading something
|
||||||
let mut object_version = ObjectVersion {
|
let mut object_version = ObjectVersion {
|
||||||
@@ -150,27 +139,44 @@ pub(crate) async fn save_stream<S: Stream<Item = Result<Bytes, Error>> + Unpin>(
|
|||||||
// Transfer data and verify checksum
|
// Transfer data and verify checksum
|
||||||
let first_block_hash = async_blake2sum(first_block.clone()).await;
|
let first_block_hash = async_blake2sum(first_block.clone()).await;
|
||||||
|
|
||||||
let (total_size, data_md5sum, data_sha256sum) = read_and_put_blocks(
|
let tx_result = (|| async {
|
||||||
&garage,
|
let (total_size, data_md5sum, data_sha256sum) = read_and_put_blocks(
|
||||||
&version,
|
&garage,
|
||||||
1,
|
&version,
|
||||||
first_block,
|
1,
|
||||||
first_block_hash,
|
first_block,
|
||||||
&mut chunker,
|
first_block_hash,
|
||||||
)
|
&mut chunker,
|
||||||
.await?;
|
)
|
||||||
|
.await?;
|
||||||
|
|
||||||
ensure_checksum_matches(
|
ensure_checksum_matches(
|
||||||
data_md5sum.as_slice(),
|
data_md5sum.as_slice(),
|
||||||
data_sha256sum,
|
data_sha256sum,
|
||||||
content_md5.as_deref(),
|
content_md5.as_deref(),
|
||||||
content_sha256,
|
content_sha256,
|
||||||
)?;
|
)?;
|
||||||
|
|
||||||
check_quotas(&garage, bucket, key, total_size).await?;
|
check_quotas(&garage, bucket, key, total_size).await?;
|
||||||
|
|
||||||
|
Ok((total_size, data_md5sum))
|
||||||
|
})()
|
||||||
|
.await;
|
||||||
|
|
||||||
|
// If something went wrong, clean up
|
||||||
|
let (total_size, md5sum_arr) = match tx_result {
|
||||||
|
Ok(rv) => rv,
|
||||||
|
Err(e) => {
|
||||||
|
// Mark object as aborted, this will free the blocks further down
|
||||||
|
object_version.state = ObjectVersionState::Aborted;
|
||||||
|
let object = Object::new(bucket.id, key.into(), vec![object_version.clone()]);
|
||||||
|
garage.object_table.insert(&object).await?;
|
||||||
|
return Err(e);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
// Save final object state, marked as Complete
|
// Save final object state, marked as Complete
|
||||||
let md5sum_hex = hex::encode(data_md5sum);
|
let md5sum_hex = hex::encode(md5sum_arr);
|
||||||
object_version.state = ObjectVersionState::Complete(ObjectVersionData::FirstBlock(
|
object_version.state = ObjectVersionState::Complete(ObjectVersionData::FirstBlock(
|
||||||
ObjectVersionMeta {
|
ObjectVersionMeta {
|
||||||
headers,
|
headers,
|
||||||
@@ -182,10 +188,6 @@ pub(crate) async fn save_stream<S: Stream<Item = Result<Bytes, Error>> + Unpin>(
|
|||||||
let object = Object::new(bucket.id, key.into(), vec![object_version]);
|
let object = Object::new(bucket.id, key.into(), vec![object_version]);
|
||||||
garage.object_table.insert(&object).await?;
|
garage.object_table.insert(&object).await?;
|
||||||
|
|
||||||
// We were not interrupted, everything went fine.
|
|
||||||
// We won't have to clean up on drop.
|
|
||||||
interrupted_cleanup.cancel();
|
|
||||||
|
|
||||||
Ok((version_uuid, md5sum_hex))
|
Ok((version_uuid, md5sum_hex))
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -424,33 +426,6 @@ pub fn put_response(version_uuid: Uuid, md5sum_hex: String) -> Response<Body> {
|
|||||||
.unwrap()
|
.unwrap()
|
||||||
}
|
}
|
||||||
|
|
||||||
struct InterruptedCleanup(Option<(Arc<Garage>, Uuid, String, Uuid, u64)>);
|
|
||||||
|
|
||||||
impl InterruptedCleanup {
|
|
||||||
fn cancel(&mut self) {
|
|
||||||
drop(self.0.take());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
impl Drop for InterruptedCleanup {
|
|
||||||
fn drop(&mut self) {
|
|
||||||
if let Some((garage, bucket_id, key, version_uuid, version_ts)) = self.0.take() {
|
|
||||||
tokio::spawn(async move {
|
|
||||||
let object_version = ObjectVersion {
|
|
||||||
uuid: version_uuid,
|
|
||||||
timestamp: version_ts,
|
|
||||||
state: ObjectVersionState::Aborted,
|
|
||||||
};
|
|
||||||
let object = Object::new(bucket_id, key, vec![object_version]);
|
|
||||||
if let Err(e) = garage.object_table.insert(&object).await {
|
|
||||||
warn!("Cannot cleanup after aborted PutObject: {}", e);
|
|
||||||
}
|
|
||||||
});
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// ----
|
|
||||||
|
|
||||||
pub async fn handle_create_multipart_upload(
|
pub async fn handle_create_multipart_upload(
|
||||||
garage: Arc<Garage>,
|
garage: Arc<Garage>,
|
||||||
req: &Request<Body>,
|
req: &Request<Body>,
|
||||||
|
|||||||
+62
-59
@@ -355,7 +355,7 @@ impl Endpoint {
|
|||||||
fn from_get(key: String, query: &mut QueryParameters<'_>) -> Result<Self, Error> {
|
fn from_get(key: String, query: &mut QueryParameters<'_>) -> Result<Self, Error> {
|
||||||
router_match! {
|
router_match! {
|
||||||
@gen_parser
|
@gen_parser
|
||||||
(query.keyword.take().unwrap_or_default(), key, query, None),
|
(query.keyword.take().unwrap_or_default().as_ref(), key, query, None),
|
||||||
key: [
|
key: [
|
||||||
EMPTY if upload_id => ListParts (query::upload_id, opt_parse::max_parts, opt_parse::part_number_marker),
|
EMPTY if upload_id => ListParts (query::upload_id, opt_parse::max_parts, opt_parse::part_number_marker),
|
||||||
EMPTY => GetObject (query_opt::version_id, opt_parse::part_number),
|
EMPTY => GetObject (query_opt::version_id, opt_parse::part_number),
|
||||||
@@ -412,7 +412,7 @@ impl Endpoint {
|
|||||||
fn from_head(key: String, query: &mut QueryParameters<'_>) -> Result<Self, Error> {
|
fn from_head(key: String, query: &mut QueryParameters<'_>) -> Result<Self, Error> {
|
||||||
router_match! {
|
router_match! {
|
||||||
@gen_parser
|
@gen_parser
|
||||||
(query.keyword.take().unwrap_or_default(), key, query, None),
|
(query.keyword.take().unwrap_or_default().as_ref(), key, query, None),
|
||||||
key: [
|
key: [
|
||||||
EMPTY => HeadObject(opt_parse::part_number, query_opt::version_id),
|
EMPTY => HeadObject(opt_parse::part_number, query_opt::version_id),
|
||||||
],
|
],
|
||||||
@@ -426,7 +426,7 @@ impl Endpoint {
|
|||||||
fn from_post(key: String, query: &mut QueryParameters<'_>) -> Result<Self, Error> {
|
fn from_post(key: String, query: &mut QueryParameters<'_>) -> Result<Self, Error> {
|
||||||
router_match! {
|
router_match! {
|
||||||
@gen_parser
|
@gen_parser
|
||||||
(query.keyword.take().unwrap_or_default(), key, query, None),
|
(query.keyword.take().unwrap_or_default().as_ref(), key, query, None),
|
||||||
key: [
|
key: [
|
||||||
EMPTY if upload_id => CompleteMultipartUpload (query::upload_id),
|
EMPTY if upload_id => CompleteMultipartUpload (query::upload_id),
|
||||||
RESTORE => RestoreObject (query_opt::version_id),
|
RESTORE => RestoreObject (query_opt::version_id),
|
||||||
@@ -448,7 +448,7 @@ impl Endpoint {
|
|||||||
) -> Result<Self, Error> {
|
) -> Result<Self, Error> {
|
||||||
router_match! {
|
router_match! {
|
||||||
@gen_parser
|
@gen_parser
|
||||||
(query.keyword.take().unwrap_or_default(), key, query, headers),
|
(query.keyword.take().unwrap_or_default().as_ref(), key, query, headers),
|
||||||
key: [
|
key: [
|
||||||
EMPTY if part_number header "x-amz-copy-source" => UploadPartCopy (parse::part_number, query::upload_id),
|
EMPTY if part_number header "x-amz-copy-source" => UploadPartCopy (parse::part_number, query::upload_id),
|
||||||
EMPTY header "x-amz-copy-source" => CopyObject,
|
EMPTY header "x-amz-copy-source" => CopyObject,
|
||||||
@@ -490,7 +490,7 @@ impl Endpoint {
|
|||||||
fn from_delete(key: String, query: &mut QueryParameters<'_>) -> Result<Self, Error> {
|
fn from_delete(key: String, query: &mut QueryParameters<'_>) -> Result<Self, Error> {
|
||||||
router_match! {
|
router_match! {
|
||||||
@gen_parser
|
@gen_parser
|
||||||
(query.keyword.take().unwrap_or_default(), key, query, None),
|
(query.keyword.take().unwrap_or_default().as_ref(), key, query, None),
|
||||||
key: [
|
key: [
|
||||||
EMPTY if upload_id => AbortMultipartUpload (query::upload_id),
|
EMPTY if upload_id => AbortMultipartUpload (query::upload_id),
|
||||||
EMPTY => DeleteObject (query_opt::version_id),
|
EMPTY => DeleteObject (query_opt::version_id),
|
||||||
@@ -624,60 +624,63 @@ impl Endpoint {
|
|||||||
|
|
||||||
// parameter name => struct field
|
// parameter name => struct field
|
||||||
generateQueryParameters! {
|
generateQueryParameters! {
|
||||||
keywords: [
|
"continuation-token" => continuation_token,
|
||||||
"accelerate" => ACCELERATE,
|
"delimiter" => delimiter,
|
||||||
"acl" => ACL,
|
"encoding-type" => encoding_type,
|
||||||
"analytics" => ANALYTICS,
|
"fetch-owner" => fetch_owner,
|
||||||
"cors" => CORS,
|
"id" => id,
|
||||||
"delete" => DELETE,
|
"key-marker" => key_marker,
|
||||||
"encryption" => ENCRYPTION,
|
"list-type" => list_type,
|
||||||
"intelligent-tiering" => INTELLIGENT_TIERING,
|
"marker" => marker,
|
||||||
"inventory" => INVENTORY,
|
"max-keys" => max_keys,
|
||||||
"legal-hold" => LEGAL_HOLD,
|
"max-parts" => max_parts,
|
||||||
"lifecycle" => LIFECYCLE,
|
"max-uploads" => max_uploads,
|
||||||
"location" => LOCATION,
|
"partNumber" => part_number,
|
||||||
"logging" => LOGGING,
|
"part-number-marker" => part_number_marker,
|
||||||
"metrics" => METRICS,
|
"prefix" => prefix,
|
||||||
"notification" => NOTIFICATION,
|
"select-type" => select_type,
|
||||||
"object-lock" => OBJECT_LOCK,
|
"start-after" => start_after,
|
||||||
"ownershipControls" => OWNERSHIP_CONTROLS,
|
"uploadId" => upload_id,
|
||||||
"policy" => POLICY,
|
"upload-id-marker" => upload_id_marker,
|
||||||
"policyStatus" => POLICY_STATUS,
|
"versionId" => version_id,
|
||||||
"publicAccessBlock" => PUBLIC_ACCESS_BLOCK,
|
"version-id-marker" => version_id_marker
|
||||||
"replication" => REPLICATION,
|
}
|
||||||
"requestPayment" => REQUEST_PAYMENT,
|
|
||||||
"restore" => RESTORE,
|
mod keywords {
|
||||||
"retention" => RETENTION,
|
//! This module contain all query parameters with no associated value S3 uses to differentiate
|
||||||
"select" => SELECT,
|
//! endpoints.
|
||||||
"tagging" => TAGGING,
|
pub const EMPTY: &str = "";
|
||||||
"torrent" => TORRENT,
|
|
||||||
"uploads" => UPLOADS,
|
pub const ACCELERATE: &str = "accelerate";
|
||||||
"versioning" => VERSIONING,
|
pub const ACL: &str = "acl";
|
||||||
"versions" => VERSIONS,
|
pub const ANALYTICS: &str = "analytics";
|
||||||
"website" => WEBSITE
|
pub const CORS: &str = "cors";
|
||||||
],
|
pub const DELETE: &str = "delete";
|
||||||
fields: [
|
pub const ENCRYPTION: &str = "encryption";
|
||||||
"continuation-token" => continuation_token,
|
pub const INTELLIGENT_TIERING: &str = "intelligent-tiering";
|
||||||
"delimiter" => delimiter,
|
pub const INVENTORY: &str = "inventory";
|
||||||
"encoding-type" => encoding_type,
|
pub const LEGAL_HOLD: &str = "legal-hold";
|
||||||
"fetch-owner" => fetch_owner,
|
pub const LIFECYCLE: &str = "lifecycle";
|
||||||
"id" => id,
|
pub const LOCATION: &str = "location";
|
||||||
"key-marker" => key_marker,
|
pub const LOGGING: &str = "logging";
|
||||||
"list-type" => list_type,
|
pub const METRICS: &str = "metrics";
|
||||||
"marker" => marker,
|
pub const NOTIFICATION: &str = "notification";
|
||||||
"max-keys" => max_keys,
|
pub const OBJECT_LOCK: &str = "object-lock";
|
||||||
"max-parts" => max_parts,
|
pub const OWNERSHIP_CONTROLS: &str = "ownershipControls";
|
||||||
"max-uploads" => max_uploads,
|
pub const POLICY: &str = "policy";
|
||||||
"partNumber" => part_number,
|
pub const POLICY_STATUS: &str = "policyStatus";
|
||||||
"part-number-marker" => part_number_marker,
|
pub const PUBLIC_ACCESS_BLOCK: &str = "publicAccessBlock";
|
||||||
"prefix" => prefix,
|
pub const REPLICATION: &str = "replication";
|
||||||
"select-type" => select_type,
|
pub const REQUEST_PAYMENT: &str = "requestPayment";
|
||||||
"start-after" => start_after,
|
pub const RESTORE: &str = "restore";
|
||||||
"uploadId" => upload_id,
|
pub const RETENTION: &str = "retention";
|
||||||
"upload-id-marker" => upload_id_marker,
|
pub const SELECT: &str = "select";
|
||||||
"versionId" => version_id,
|
pub const TAGGING: &str = "tagging";
|
||||||
"version-id-marker" => version_id_marker
|
pub const TORRENT: &str = "torrent";
|
||||||
]
|
pub const UPLOADS: &str = "uploads";
|
||||||
|
pub const VERSIONING: &str = "versioning";
|
||||||
|
pub const VERSIONS: &str = "versions";
|
||||||
|
pub const WEBSITE: &str = "website";
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
[package]
|
[package]
|
||||||
name = "garage_block"
|
name = "garage_block"
|
||||||
version = "0.8.1"
|
version = "0.8.0"
|
||||||
authors = ["Alex Auvolat <alex@adnab.me>"]
|
authors = ["Alex Auvolat <alex@adnab.me>"]
|
||||||
edition = "2018"
|
edition = "2018"
|
||||||
license = "AGPL-3.0"
|
license = "AGPL-3.0"
|
||||||
@@ -14,10 +14,10 @@ path = "lib.rs"
|
|||||||
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
|
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
|
||||||
|
|
||||||
[dependencies]
|
[dependencies]
|
||||||
garage_db = { version = "0.8.1", path = "../db" }
|
garage_db = { version = "0.8.0", path = "../db" }
|
||||||
garage_rpc = { version = "0.8.1", path = "../rpc" }
|
garage_rpc = { version = "0.8.0", path = "../rpc" }
|
||||||
garage_util = { version = "0.8.1", path = "../util" }
|
garage_util = { version = "0.8.0", path = "../util" }
|
||||||
garage_table = { version = "0.8.1", path = "../table" }
|
garage_table = { version = "0.8.0", path = "../table" }
|
||||||
|
|
||||||
opentelemetry = "0.17"
|
opentelemetry = "0.17"
|
||||||
|
|
||||||
@@ -31,6 +31,7 @@ rand = "0.8"
|
|||||||
async-compression = { version = "0.3", features = ["tokio", "zstd"] }
|
async-compression = { version = "0.3", features = ["tokio", "zstd"] }
|
||||||
zstd = { version = "0.9", default-features = false }
|
zstd = { version = "0.9", default-features = false }
|
||||||
|
|
||||||
|
rmp-serde = "0.15"
|
||||||
serde = { version = "1.0", default-features = false, features = ["derive", "rc"] }
|
serde = { version = "1.0", default-features = false, features = ["derive", "rc"] }
|
||||||
serde_bytes = "0.11"
|
serde_bytes = "0.11"
|
||||||
|
|
||||||
|
|||||||
+6
-61
@@ -6,7 +6,6 @@ use std::time::Duration;
|
|||||||
use arc_swap::ArcSwapOption;
|
use arc_swap::ArcSwapOption;
|
||||||
use async_trait::async_trait;
|
use async_trait::async_trait;
|
||||||
use bytes::Bytes;
|
use bytes::Bytes;
|
||||||
use rand::prelude::*;
|
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
|
|
||||||
use futures::Stream;
|
use futures::Stream;
|
||||||
@@ -24,12 +23,10 @@ use garage_rpc::rpc_helper::netapp::stream::{stream_asyncread, ByteStream};
|
|||||||
|
|
||||||
use garage_db as db;
|
use garage_db as db;
|
||||||
|
|
||||||
use garage_util::background::{vars, BackgroundRunner};
|
use garage_util::background::BackgroundRunner;
|
||||||
use garage_util::data::*;
|
use garage_util::data::*;
|
||||||
use garage_util::error::*;
|
use garage_util::error::*;
|
||||||
use garage_util::metrics::RecordDuration;
|
use garage_util::metrics::RecordDuration;
|
||||||
use garage_util::persister::PersisterShared;
|
|
||||||
use garage_util::time::msec_to_rfc3339;
|
|
||||||
|
|
||||||
use garage_rpc::rpc_helper::OrderTag;
|
use garage_rpc::rpc_helper::OrderTag;
|
||||||
use garage_rpc::system::System;
|
use garage_rpc::system::System;
|
||||||
@@ -92,7 +89,6 @@ pub struct BlockManager {
|
|||||||
|
|
||||||
pub(crate) metrics: BlockManagerMetrics,
|
pub(crate) metrics: BlockManagerMetrics,
|
||||||
|
|
||||||
pub scrub_persister: PersisterShared<ScrubWorkerPersisted>,
|
|
||||||
tx_scrub_command: ArcSwapOption<mpsc::Sender<ScrubWorkerCommand>>,
|
tx_scrub_command: ArcSwapOption<mpsc::Sender<ScrubWorkerCommand>>,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -132,8 +128,6 @@ impl BlockManager {
|
|||||||
let metrics =
|
let metrics =
|
||||||
BlockManagerMetrics::new(rc.rc.clone(), resync.queue.clone(), resync.errors.clone());
|
BlockManagerMetrics::new(rc.rc.clone(), resync.queue.clone(), resync.errors.clone());
|
||||||
|
|
||||||
let scrub_persister = PersisterShared::new(&system.metadata_dir, "scrub_info");
|
|
||||||
|
|
||||||
let block_manager = Arc::new(Self {
|
let block_manager = Arc::new(Self {
|
||||||
replication,
|
replication,
|
||||||
data_dir,
|
data_dir,
|
||||||
@@ -144,7 +138,6 @@ impl BlockManager {
|
|||||||
system,
|
system,
|
||||||
endpoint,
|
endpoint,
|
||||||
metrics,
|
metrics,
|
||||||
scrub_persister,
|
|
||||||
tx_scrub_command: ArcSwapOption::new(None),
|
tx_scrub_command: ArcSwapOption::new(None),
|
||||||
});
|
});
|
||||||
block_manager.endpoint.set_handler(block_manager.clone());
|
block_manager.endpoint.set_handler(block_manager.clone());
|
||||||
@@ -162,28 +155,7 @@ impl BlockManager {
|
|||||||
// Spawn scrub worker
|
// Spawn scrub worker
|
||||||
let (scrub_tx, scrub_rx) = mpsc::channel(1);
|
let (scrub_tx, scrub_rx) = mpsc::channel(1);
|
||||||
self.tx_scrub_command.store(Some(Arc::new(scrub_tx)));
|
self.tx_scrub_command.store(Some(Arc::new(scrub_tx)));
|
||||||
bg.spawn_worker(ScrubWorker::new(
|
bg.spawn_worker(ScrubWorker::new(self.clone(), scrub_rx));
|
||||||
self.clone(),
|
|
||||||
scrub_rx,
|
|
||||||
self.scrub_persister.clone(),
|
|
||||||
));
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn register_bg_vars(&self, vars: &mut vars::BgVars) {
|
|
||||||
self.resync.register_bg_vars(vars);
|
|
||||||
|
|
||||||
vars.register_rw(
|
|
||||||
&self.scrub_persister,
|
|
||||||
"scrub-tranquility",
|
|
||||||
|p| p.get_with(|x| x.tranquility),
|
|
||||||
|p, tranquility| p.set_with(|x| x.tranquility = tranquility),
|
|
||||||
);
|
|
||||||
vars.register_ro(&self.scrub_persister, "scrub-last-completed", |p| {
|
|
||||||
p.get_with(|x| msec_to_rfc3339(x.time_last_complete_scrub))
|
|
||||||
});
|
|
||||||
vars.register_ro(&self.scrub_persister, "scrub-corruptions_detected", |p| {
|
|
||||||
p.get_with(|x| x.corruptions_detected)
|
|
||||||
});
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Ask nodes that might have a (possibly compressed) block for it
|
/// Ask nodes that might have a (possibly compressed) block for it
|
||||||
@@ -677,21 +649,14 @@ impl BlockManagerLocked {
|
|||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
let mut path_tmp = path.clone();
|
let mut path2 = path.clone();
|
||||||
let tmp_extension = format!("tmp{}", hex::encode(thread_rng().gen::<[u8; 4]>()));
|
path2.set_extension("tmp");
|
||||||
path_tmp.set_extension(tmp_extension);
|
let mut f = fs::File::create(&path2).await?;
|
||||||
|
|
||||||
let mut delete_on_drop = DeleteOnDrop(Some(path_tmp.clone()));
|
|
||||||
|
|
||||||
let mut f = fs::File::create(&path_tmp).await?;
|
|
||||||
f.write_all(data).await?;
|
f.write_all(data).await?;
|
||||||
f.sync_all().await?;
|
f.sync_all().await?;
|
||||||
drop(f);
|
drop(f);
|
||||||
|
|
||||||
fs::rename(path_tmp, path).await?;
|
fs::rename(path2, path).await?;
|
||||||
|
|
||||||
delete_on_drop.cancel();
|
|
||||||
|
|
||||||
if let Some(to_delete) = to_delete {
|
if let Some(to_delete) = to_delete {
|
||||||
fs::remove_file(to_delete).await?;
|
fs::remove_file(to_delete).await?;
|
||||||
}
|
}
|
||||||
@@ -757,23 +722,3 @@ async fn read_stream_to_end(mut stream: ByteStream) -> Result<Bytes, Error> {
|
|||||||
.concat()
|
.concat()
|
||||||
.into())
|
.into())
|
||||||
}
|
}
|
||||||
|
|
||||||
struct DeleteOnDrop(Option<PathBuf>);
|
|
||||||
|
|
||||||
impl DeleteOnDrop {
|
|
||||||
fn cancel(&mut self) {
|
|
||||||
drop(self.0.take());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl Drop for DeleteOnDrop {
|
|
||||||
fn drop(&mut self) {
|
|
||||||
if let Some(path) = self.0.take() {
|
|
||||||
tokio::spawn(async move {
|
|
||||||
if let Err(e) = fs::remove_file(&path).await {
|
|
||||||
debug!("DeleteOnDrop failed for {}: {}", path.display(), e);
|
|
||||||
}
|
|
||||||
});
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|||||||
+34
-39
@@ -13,7 +13,7 @@ use tokio::sync::watch;
|
|||||||
use garage_util::background::*;
|
use garage_util::background::*;
|
||||||
use garage_util::data::*;
|
use garage_util::data::*;
|
||||||
use garage_util::error::*;
|
use garage_util::error::*;
|
||||||
use garage_util::persister::PersisterShared;
|
use garage_util::persister::Persister;
|
||||||
use garage_util::time::*;
|
use garage_util::time::*;
|
||||||
use garage_util::tranquilizer::Tranquilizer;
|
use garage_util::tranquilizer::Tranquilizer;
|
||||||
|
|
||||||
@@ -168,24 +168,15 @@ pub struct ScrubWorker {
|
|||||||
work: ScrubWorkerState,
|
work: ScrubWorkerState,
|
||||||
tranquilizer: Tranquilizer,
|
tranquilizer: Tranquilizer,
|
||||||
|
|
||||||
persister: PersisterShared<ScrubWorkerPersisted>,
|
persister: Persister<ScrubWorkerPersisted>,
|
||||||
|
persisted: ScrubWorkerPersisted,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Serialize, Deserialize)]
|
#[derive(Serialize, Deserialize)]
|
||||||
pub struct ScrubWorkerPersisted {
|
struct ScrubWorkerPersisted {
|
||||||
pub tranquility: u32,
|
tranquility: u32,
|
||||||
pub(crate) time_last_complete_scrub: u64,
|
time_last_complete_scrub: u64,
|
||||||
pub(crate) corruptions_detected: u64,
|
corruptions_detected: u64,
|
||||||
}
|
|
||||||
impl garage_util::migrate::InitialFormat for ScrubWorkerPersisted {}
|
|
||||||
impl Default for ScrubWorkerPersisted {
|
|
||||||
fn default() -> Self {
|
|
||||||
ScrubWorkerPersisted {
|
|
||||||
time_last_complete_scrub: 0,
|
|
||||||
tranquility: INITIAL_SCRUB_TRANQUILITY,
|
|
||||||
corruptions_detected: 0,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
enum ScrubWorkerState {
|
enum ScrubWorkerState {
|
||||||
@@ -206,20 +197,27 @@ pub enum ScrubWorkerCommand {
|
|||||||
Pause(Duration),
|
Pause(Duration),
|
||||||
Resume,
|
Resume,
|
||||||
Cancel,
|
Cancel,
|
||||||
|
SetTranquility(u32),
|
||||||
}
|
}
|
||||||
|
|
||||||
impl ScrubWorker {
|
impl ScrubWorker {
|
||||||
pub(crate) fn new(
|
pub fn new(manager: Arc<BlockManager>, rx_cmd: mpsc::Receiver<ScrubWorkerCommand>) -> Self {
|
||||||
manager: Arc<BlockManager>,
|
let persister = Persister::new(&manager.system.metadata_dir, "scrub_info");
|
||||||
rx_cmd: mpsc::Receiver<ScrubWorkerCommand>,
|
let persisted = match persister.load() {
|
||||||
persister: PersisterShared<ScrubWorkerPersisted>,
|
Ok(v) => v,
|
||||||
) -> Self {
|
Err(_) => ScrubWorkerPersisted {
|
||||||
|
time_last_complete_scrub: 0,
|
||||||
|
tranquility: INITIAL_SCRUB_TRANQUILITY,
|
||||||
|
corruptions_detected: 0,
|
||||||
|
},
|
||||||
|
};
|
||||||
Self {
|
Self {
|
||||||
manager,
|
manager,
|
||||||
rx_cmd,
|
rx_cmd,
|
||||||
work: ScrubWorkerState::Finished,
|
work: ScrubWorkerState::Finished,
|
||||||
tranquilizer: Tranquilizer::new(30),
|
tranquilizer: Tranquilizer::new(30),
|
||||||
persister,
|
persister,
|
||||||
|
persisted,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -268,6 +266,12 @@ impl ScrubWorker {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
ScrubWorkerCommand::SetTranquility(t) => {
|
||||||
|
self.persisted.tranquility = t;
|
||||||
|
if let Err(e) = self.persister.save_async(&self.persisted).await {
|
||||||
|
error!("Could not save new tranquilitiy value: {}", e);
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -279,18 +283,9 @@ impl Worker for ScrubWorker {
|
|||||||
}
|
}
|
||||||
|
|
||||||
fn status(&self) -> WorkerStatus {
|
fn status(&self) -> WorkerStatus {
|
||||||
let (corruptions_detected, tranquility, time_last_complete_scrub) =
|
|
||||||
self.persister.get_with(|p| {
|
|
||||||
(
|
|
||||||
p.corruptions_detected,
|
|
||||||
p.tranquility,
|
|
||||||
p.time_last_complete_scrub,
|
|
||||||
)
|
|
||||||
});
|
|
||||||
|
|
||||||
let mut s = WorkerStatus {
|
let mut s = WorkerStatus {
|
||||||
persistent_errors: Some(corruptions_detected),
|
persistent_errors: Some(self.persisted.corruptions_detected),
|
||||||
tranquility: Some(tranquility),
|
tranquility: Some(self.persisted.tranquility),
|
||||||
..Default::default()
|
..Default::default()
|
||||||
};
|
};
|
||||||
match &self.work {
|
match &self.work {
|
||||||
@@ -304,7 +299,7 @@ impl Worker for ScrubWorker {
|
|||||||
ScrubWorkerState::Finished => {
|
ScrubWorkerState::Finished => {
|
||||||
s.freeform = vec![format!(
|
s.freeform = vec![format!(
|
||||||
"Last scrub completed at {}",
|
"Last scrub completed at {}",
|
||||||
msec_to_rfc3339(time_last_complete_scrub)
|
msec_to_rfc3339(self.persisted.time_last_complete_scrub)
|
||||||
)];
|
)];
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -325,17 +320,18 @@ impl Worker for ScrubWorker {
|
|||||||
match self.manager.read_block(&hash).await {
|
match self.manager.read_block(&hash).await {
|
||||||
Err(Error::CorruptData(_)) => {
|
Err(Error::CorruptData(_)) => {
|
||||||
error!("Found corrupt data block during scrub: {:?}", hash);
|
error!("Found corrupt data block during scrub: {:?}", hash);
|
||||||
self.persister.set_with(|p| p.corruptions_detected += 1)?;
|
self.persisted.corruptions_detected += 1;
|
||||||
|
self.persister.save_async(&self.persisted).await?;
|
||||||
}
|
}
|
||||||
Err(e) => return Err(e),
|
Err(e) => return Err(e),
|
||||||
_ => (),
|
_ => (),
|
||||||
};
|
};
|
||||||
Ok(self
|
Ok(self
|
||||||
.tranquilizer
|
.tranquilizer
|
||||||
.tranquilize_worker(self.persister.get_with(|p| p.tranquility)))
|
.tranquilize_worker(self.persisted.tranquility))
|
||||||
} else {
|
} else {
|
||||||
self.persister
|
self.persisted.time_last_complete_scrub = now_msec();
|
||||||
.set_with(|p| p.time_last_complete_scrub = now_msec())?;
|
self.persister.save_async(&self.persisted).await?;
|
||||||
self.work = ScrubWorkerState::Finished;
|
self.work = ScrubWorkerState::Finished;
|
||||||
self.tranquilizer.clear();
|
self.tranquilizer.clear();
|
||||||
Ok(WorkerState::Idle)
|
Ok(WorkerState::Idle)
|
||||||
@@ -350,8 +346,7 @@ impl Worker for ScrubWorker {
|
|||||||
ScrubWorkerState::Running(_) => return WorkerState::Busy,
|
ScrubWorkerState::Running(_) => return WorkerState::Busy,
|
||||||
ScrubWorkerState::Paused(_, resume_time) => (*resume_time, ScrubWorkerCommand::Resume),
|
ScrubWorkerState::Paused(_, resume_time) => (*resume_time, ScrubWorkerCommand::Resume),
|
||||||
ScrubWorkerState::Finished => (
|
ScrubWorkerState::Finished => (
|
||||||
self.persister.get_with(|p| p.time_last_complete_scrub)
|
self.persisted.time_last_complete_scrub + SCRUB_INTERVAL.as_millis() as u64,
|
||||||
+ SCRUB_INTERVAL.as_millis() as u64,
|
|
||||||
ScrubWorkerCommand::Start,
|
ScrubWorkerCommand::Start,
|
||||||
),
|
),
|
||||||
};
|
};
|
||||||
|
|||||||
+50
-59
@@ -3,6 +3,7 @@ use std::convert::TryInto;
|
|||||||
use std::sync::{Arc, Mutex};
|
use std::sync::{Arc, Mutex};
|
||||||
use std::time::Duration;
|
use std::time::Duration;
|
||||||
|
|
||||||
|
use arc_swap::ArcSwap;
|
||||||
use async_trait::async_trait;
|
use async_trait::async_trait;
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
|
|
||||||
@@ -21,7 +22,7 @@ use garage_util::background::*;
|
|||||||
use garage_util::data::*;
|
use garage_util::data::*;
|
||||||
use garage_util::error::*;
|
use garage_util::error::*;
|
||||||
use garage_util::metrics::RecordDuration;
|
use garage_util::metrics::RecordDuration;
|
||||||
use garage_util::persister::PersisterShared;
|
use garage_util::persister::Persister;
|
||||||
use garage_util::time::*;
|
use garage_util::time::*;
|
||||||
use garage_util::tranquilizer::Tranquilizer;
|
use garage_util::tranquilizer::Tranquilizer;
|
||||||
|
|
||||||
@@ -48,12 +49,13 @@ const INITIAL_RESYNC_TRANQUILITY: u32 = 2;
|
|||||||
|
|
||||||
pub struct BlockResyncManager {
|
pub struct BlockResyncManager {
|
||||||
pub(crate) queue: CountedTree,
|
pub(crate) queue: CountedTree,
|
||||||
pub(crate) notify: Arc<Notify>,
|
pub(crate) notify: Notify,
|
||||||
pub(crate) errors: CountedTree,
|
pub(crate) errors: CountedTree,
|
||||||
|
|
||||||
busy_set: BusySet,
|
busy_set: BusySet,
|
||||||
|
|
||||||
persister: PersisterShared<ResyncPersistedConfig>,
|
persister: Persister<ResyncPersistedConfig>,
|
||||||
|
persisted: ArcSwap<ResyncPersistedConfig>,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Serialize, Deserialize, Clone, Copy)]
|
#[derive(Serialize, Deserialize, Clone, Copy)]
|
||||||
@@ -61,15 +63,6 @@ struct ResyncPersistedConfig {
|
|||||||
n_workers: usize,
|
n_workers: usize,
|
||||||
tranquility: u32,
|
tranquility: u32,
|
||||||
}
|
}
|
||||||
impl garage_util::migrate::InitialFormat for ResyncPersistedConfig {}
|
|
||||||
impl Default for ResyncPersistedConfig {
|
|
||||||
fn default() -> Self {
|
|
||||||
ResyncPersistedConfig {
|
|
||||||
n_workers: 1,
|
|
||||||
tranquility: INITIAL_RESYNC_TRANQUILITY,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
enum ResyncIterResult {
|
enum ResyncIterResult {
|
||||||
BusyDidSomething,
|
BusyDidSomething,
|
||||||
@@ -97,14 +90,22 @@ impl BlockResyncManager {
|
|||||||
.expect("Unable to open block_local_resync_errors tree");
|
.expect("Unable to open block_local_resync_errors tree");
|
||||||
let errors = CountedTree::new(errors).expect("Could not count block_local_resync_errors");
|
let errors = CountedTree::new(errors).expect("Could not count block_local_resync_errors");
|
||||||
|
|
||||||
let persister = PersisterShared::new(&system.metadata_dir, "resync_cfg");
|
let persister = Persister::new(&system.metadata_dir, "resync_cfg");
|
||||||
|
let persisted = match persister.load() {
|
||||||
|
Ok(v) => v,
|
||||||
|
Err(_) => ResyncPersistedConfig {
|
||||||
|
n_workers: 1,
|
||||||
|
tranquility: INITIAL_RESYNC_TRANQUILITY,
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
Self {
|
Self {
|
||||||
queue,
|
queue,
|
||||||
notify: Arc::new(Notify::new()),
|
notify: Notify::new(),
|
||||||
errors,
|
errors,
|
||||||
busy_set: Arc::new(Mutex::new(HashSet::new())),
|
busy_set: Arc::new(Mutex::new(HashSet::new())),
|
||||||
persister,
|
persister,
|
||||||
|
persisted: ArcSwap::new(Arc::new(persisted)),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -140,38 +141,6 @@ impl BlockResyncManager {
|
|||||||
)))
|
)))
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn register_bg_vars(&self, vars: &mut vars::BgVars) {
|
|
||||||
let notify = self.notify.clone();
|
|
||||||
vars.register_rw(
|
|
||||||
&self.persister,
|
|
||||||
"resync-worker-count",
|
|
||||||
|p| p.get_with(|x| x.n_workers),
|
|
||||||
move |p, n_workers| {
|
|
||||||
if !(1..=MAX_RESYNC_WORKERS).contains(&n_workers) {
|
|
||||||
return Err(Error::Message(format!(
|
|
||||||
"Invalid number of resync workers, must be between 1 and {}",
|
|
||||||
MAX_RESYNC_WORKERS
|
|
||||||
)));
|
|
||||||
}
|
|
||||||
p.set_with(|x| x.n_workers = n_workers)?;
|
|
||||||
notify.notify_waiters();
|
|
||||||
Ok(())
|
|
||||||
},
|
|
||||||
);
|
|
||||||
|
|
||||||
let notify = self.notify.clone();
|
|
||||||
vars.register_rw(
|
|
||||||
&self.persister,
|
|
||||||
"resync-tranquility",
|
|
||||||
|p| p.get_with(|x| x.tranquility),
|
|
||||||
move |p, tranquility| {
|
|
||||||
p.set_with(|x| x.tranquility = tranquility)?;
|
|
||||||
notify.notify_waiters();
|
|
||||||
Ok(())
|
|
||||||
},
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
// ---- Resync loop ----
|
// ---- Resync loop ----
|
||||||
|
|
||||||
// This part manages a queue of blocks that need to be
|
// This part manages a queue of blocks that need to be
|
||||||
@@ -466,6 +435,33 @@ impl BlockResyncManager {
|
|||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async fn update_persisted(
|
||||||
|
&self,
|
||||||
|
update: impl Fn(&mut ResyncPersistedConfig),
|
||||||
|
) -> Result<(), Error> {
|
||||||
|
let mut cfg: ResyncPersistedConfig = *self.persisted.load().as_ref();
|
||||||
|
update(&mut cfg);
|
||||||
|
self.persister.save_async(&cfg).await?;
|
||||||
|
self.persisted.store(Arc::new(cfg));
|
||||||
|
self.notify.notify_waiters();
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn set_n_workers(&self, n_workers: usize) -> Result<(), Error> {
|
||||||
|
if !(1..=MAX_RESYNC_WORKERS).contains(&n_workers) {
|
||||||
|
return Err(Error::Message(format!(
|
||||||
|
"Invalid number of resync workers, must be between 1 and {}",
|
||||||
|
MAX_RESYNC_WORKERS
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
self.update_persisted(|cfg| cfg.n_workers = n_workers).await
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn set_tranquility(&self, tranquility: u32) -> Result<(), Error> {
|
||||||
|
self.update_persisted(|cfg| cfg.tranquility = tranquility)
|
||||||
|
.await
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Drop for BusyBlock {
|
impl Drop for BusyBlock {
|
||||||
@@ -480,18 +476,15 @@ pub(crate) struct ResyncWorker {
|
|||||||
manager: Arc<BlockManager>,
|
manager: Arc<BlockManager>,
|
||||||
tranquilizer: Tranquilizer,
|
tranquilizer: Tranquilizer,
|
||||||
next_delay: Duration,
|
next_delay: Duration,
|
||||||
persister: PersisterShared<ResyncPersistedConfig>,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
impl ResyncWorker {
|
impl ResyncWorker {
|
||||||
pub(crate) fn new(index: usize, manager: Arc<BlockManager>) -> Self {
|
pub(crate) fn new(index: usize, manager: Arc<BlockManager>) -> Self {
|
||||||
let persister = manager.resync.persister.clone();
|
|
||||||
Self {
|
Self {
|
||||||
index,
|
index,
|
||||||
manager,
|
manager,
|
||||||
tranquilizer: Tranquilizer::new(30),
|
tranquilizer: Tranquilizer::new(30),
|
||||||
next_delay: Duration::from_secs(10),
|
next_delay: Duration::from_secs(10),
|
||||||
persister,
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -503,9 +496,9 @@ impl Worker for ResyncWorker {
|
|||||||
}
|
}
|
||||||
|
|
||||||
fn status(&self) -> WorkerStatus {
|
fn status(&self) -> WorkerStatus {
|
||||||
let (n_workers, tranquility) = self.persister.get_with(|x| (x.n_workers, x.tranquility));
|
let persisted = self.manager.resync.persisted.load();
|
||||||
|
|
||||||
if self.index >= n_workers {
|
if self.index >= persisted.n_workers {
|
||||||
return WorkerStatus {
|
return WorkerStatus {
|
||||||
freeform: vec!["This worker is currently disabled".into()],
|
freeform: vec!["This worker is currently disabled".into()],
|
||||||
..Default::default()
|
..Default::default()
|
||||||
@@ -514,24 +507,22 @@ impl Worker for ResyncWorker {
|
|||||||
|
|
||||||
WorkerStatus {
|
WorkerStatus {
|
||||||
queue_length: Some(self.manager.resync.queue_len().unwrap_or(0) as u64),
|
queue_length: Some(self.manager.resync.queue_len().unwrap_or(0) as u64),
|
||||||
tranquility: Some(tranquility),
|
tranquility: Some(persisted.tranquility),
|
||||||
persistent_errors: Some(self.manager.resync.errors_len().unwrap_or(0) as u64),
|
persistent_errors: Some(self.manager.resync.errors_len().unwrap_or(0) as u64),
|
||||||
..Default::default()
|
..Default::default()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn work(&mut self, _must_exit: &mut watch::Receiver<bool>) -> Result<WorkerState, Error> {
|
async fn work(&mut self, _must_exit: &mut watch::Receiver<bool>) -> Result<WorkerState, Error> {
|
||||||
let (n_workers, tranquility) = self.persister.get_with(|x| (x.n_workers, x.tranquility));
|
if self.index >= self.manager.resync.persisted.load().n_workers {
|
||||||
|
|
||||||
if self.index >= n_workers {
|
|
||||||
return Ok(WorkerState::Idle);
|
return Ok(WorkerState::Idle);
|
||||||
}
|
}
|
||||||
|
|
||||||
self.tranquilizer.reset();
|
self.tranquilizer.reset();
|
||||||
match self.manager.resync.resync_iter(&self.manager).await {
|
match self.manager.resync.resync_iter(&self.manager).await {
|
||||||
Ok(ResyncIterResult::BusyDidSomething) => {
|
Ok(ResyncIterResult::BusyDidSomething) => Ok(self
|
||||||
Ok(self.tranquilizer.tranquilize_worker(tranquility))
|
.tranquilizer
|
||||||
}
|
.tranquilize_worker(self.manager.resync.persisted.load().tranquility)),
|
||||||
Ok(ResyncIterResult::BusyDidNothing) => Ok(WorkerState::Busy),
|
Ok(ResyncIterResult::BusyDidNothing) => Ok(WorkerState::Busy),
|
||||||
Ok(ResyncIterResult::IdleFor(delay)) => {
|
Ok(ResyncIterResult::IdleFor(delay)) => {
|
||||||
self.next_delay = delay;
|
self.next_delay = delay;
|
||||||
@@ -550,7 +541,7 @@ impl Worker for ResyncWorker {
|
|||||||
}
|
}
|
||||||
|
|
||||||
async fn wait_for_work(&mut self) -> WorkerState {
|
async fn wait_for_work(&mut self) -> WorkerState {
|
||||||
while self.index >= self.persister.get_with(|x| x.n_workers) {
|
while self.index >= self.manager.resync.persisted.load().n_workers {
|
||||||
self.manager.resync.notify.notified().await
|
self.manager.resync.notify.notified().await
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+8
-1
@@ -1,6 +1,6 @@
|
|||||||
[package]
|
[package]
|
||||||
name = "garage_db"
|
name = "garage_db"
|
||||||
version = "0.8.1"
|
version = "0.8.0"
|
||||||
authors = ["Alex Auvolat <alex@adnab.me>"]
|
authors = ["Alex Auvolat <alex@adnab.me>"]
|
||||||
edition = "2018"
|
edition = "2018"
|
||||||
license = "AGPL-3.0"
|
license = "AGPL-3.0"
|
||||||
@@ -18,6 +18,7 @@ required-features = ["cli"]
|
|||||||
|
|
||||||
[dependencies]
|
[dependencies]
|
||||||
err-derive = "0.3"
|
err-derive = "0.3"
|
||||||
|
hex = "0.4"
|
||||||
hexdump = "0.1"
|
hexdump = "0.1"
|
||||||
tracing = "0.1.30"
|
tracing = "0.1.30"
|
||||||
|
|
||||||
@@ -25,6 +26,11 @@ heed = { version = "0.11", default-features = false, features = ["lmdb"], option
|
|||||||
rusqlite = { version = "0.27", optional = true }
|
rusqlite = { version = "0.27", optional = true }
|
||||||
sled = { version = "0.34", optional = true }
|
sled = { version = "0.34", optional = true }
|
||||||
|
|
||||||
|
rmp-serde = { version = "0.15", optional = true }
|
||||||
|
serde = { version = "1.0", default-features = false, features = ["derive", "rc"], optional = true }
|
||||||
|
serde-transcode = { version = "1.1", optional = true }
|
||||||
|
nettext = { version = "0.3.2", default-features = false, features = ["serde"], optional = true, path = "../../../nettext" }
|
||||||
|
|
||||||
# cli deps
|
# cli deps
|
||||||
clap = { version = "3.1.18", optional = true, features = ["derive", "env"] }
|
clap = { version = "3.1.18", optional = true, features = ["derive", "env"] }
|
||||||
pretty_env_logger = { version = "0.4", optional = true }
|
pretty_env_logger = { version = "0.4", optional = true }
|
||||||
@@ -38,3 +44,4 @@ bundled-libs = [ "rusqlite/bundled" ]
|
|||||||
cli = ["clap", "pretty_env_logger"]
|
cli = ["clap", "pretty_env_logger"]
|
||||||
lmdb = [ "heed" ]
|
lmdb = [ "heed" ]
|
||||||
sqlite = [ "rusqlite" ]
|
sqlite = [ "rusqlite" ]
|
||||||
|
debuglog = [ "serde", "rmp-serde", "nettext", "serde-transcode" ]
|
||||||
|
|||||||
+49
-3
@@ -173,9 +173,22 @@ impl Tree {
|
|||||||
Db(self.0.clone())
|
Db(self.0.clone())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[inline]
|
||||||
|
pub fn name(&self) -> Option<String> {
|
||||||
|
self.0.tree_name(self.1)
|
||||||
|
}
|
||||||
|
|
||||||
#[inline]
|
#[inline]
|
||||||
pub fn get<T: AsRef<[u8]>>(&self, key: T) -> Result<Option<Value>> {
|
pub fn get<T: AsRef<[u8]>>(&self, key: T) -> Result<Option<Value>> {
|
||||||
self.0.get(self.1, key.as_ref())
|
let res = self.0.get(self.1, key.as_ref())?;
|
||||||
|
#[cfg(feature = "debuglog")]
|
||||||
|
debuglog(
|
||||||
|
self.name(),
|
||||||
|
"GET",
|
||||||
|
key.as_ref(),
|
||||||
|
res.as_deref().unwrap_or(b"-"),
|
||||||
|
);
|
||||||
|
Ok(res)
|
||||||
}
|
}
|
||||||
#[inline]
|
#[inline]
|
||||||
pub fn len(&self) -> Result<usize> {
|
pub fn len(&self) -> Result<usize> {
|
||||||
@@ -204,7 +217,10 @@ impl Tree {
|
|||||||
key: T,
|
key: T,
|
||||||
value: U,
|
value: U,
|
||||||
) -> Result<Option<Value>> {
|
) -> Result<Option<Value>> {
|
||||||
self.0.insert(self.1, key.as_ref(), value.as_ref())
|
let res = self.0.insert(self.1, key.as_ref(), value.as_ref())?;
|
||||||
|
#[cfg(feature = "debuglog")]
|
||||||
|
debuglog(self.name(), "PUT", key.as_ref(), value.as_ref());
|
||||||
|
Ok(res)
|
||||||
}
|
}
|
||||||
/// Returns the old value if there was one
|
/// Returns the old value if there was one
|
||||||
#[inline]
|
#[inline]
|
||||||
@@ -267,7 +283,10 @@ impl<'a> Transaction<'a> {
|
|||||||
key: T,
|
key: T,
|
||||||
value: U,
|
value: U,
|
||||||
) -> TxOpResult<Option<Value>> {
|
) -> TxOpResult<Option<Value>> {
|
||||||
self.0.insert(tree.1, key.as_ref(), value.as_ref())
|
let res = self.0.insert(tree.1, key.as_ref(), value.as_ref())?;
|
||||||
|
#[cfg(feature = "debuglog")]
|
||||||
|
debuglog(tree.name(), "txPUT", key.as_ref(), value.as_ref());
|
||||||
|
Ok(res)
|
||||||
}
|
}
|
||||||
/// Returns the old value if there was one
|
/// Returns the old value if there was one
|
||||||
#[inline]
|
#[inline]
|
||||||
@@ -324,6 +343,7 @@ pub(crate) trait IDb: Send + Sync {
|
|||||||
fn engine(&self) -> String;
|
fn engine(&self) -> String;
|
||||||
fn open_tree(&self, name: &str) -> Result<usize>;
|
fn open_tree(&self, name: &str) -> Result<usize>;
|
||||||
fn list_trees(&self) -> Result<Vec<String>>;
|
fn list_trees(&self) -> Result<Vec<String>>;
|
||||||
|
fn tree_name(&self, tree: usize) -> Option<String>;
|
||||||
|
|
||||||
fn get(&self, tree: usize, key: &[u8]) -> Result<Option<Value>>;
|
fn get(&self, tree: usize, key: &[u8]) -> Result<Option<Value>>;
|
||||||
fn len(&self, tree: usize) -> Result<usize>;
|
fn len(&self, tree: usize) -> Result<usize>;
|
||||||
@@ -421,3 +441,29 @@ fn get_bound<K: AsRef<[u8]>>(b: Bound<&K>) -> Bound<&[u8]> {
|
|||||||
Bound::Unbounded => Bound::Unbounded,
|
Bound::Unbounded => Bound::Unbounded,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(feature = "debuglog")]
|
||||||
|
fn debuglog(tree: Option<String>, action: &str, k: &[u8], v: &[u8]) {
|
||||||
|
let key = String::from_utf8(nettext::switch64::encode(k, false)).unwrap();
|
||||||
|
let tree = tree.as_deref().unwrap_or("(?)");
|
||||||
|
if let Ok(vstr) = std::str::from_utf8(v) {
|
||||||
|
eprintln!("{} {} {} S:{}", tree, action, key, vstr);
|
||||||
|
} else {
|
||||||
|
let mut vread = &v[..];
|
||||||
|
let mut vder = rmp_serde::decode::Deserializer::new(&mut vread);
|
||||||
|
let mut vser = nettext::serde::Serializer {
|
||||||
|
string_format: nettext::BytesEncoding::Switch64 {
|
||||||
|
allow_whitespace: true,
|
||||||
|
},
|
||||||
|
bytes_format: nettext::BytesEncoding::Hex { split: true },
|
||||||
|
};
|
||||||
|
if let Some(venc) = serde_transcode::transcode(&mut vder, &mut vser)
|
||||||
|
.ok()
|
||||||
|
.and_then(|x| String::from_utf8(x.encode_concise()).ok())
|
||||||
|
{
|
||||||
|
eprintln!("{} {} {} N:{}", tree, action, key, venc);
|
||||||
|
} else {
|
||||||
|
eprintln!("{} {} {} X:{}", tree, action, key, hex::encode(v));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -88,6 +88,15 @@ impl IDb for SledDb {
|
|||||||
Ok(trees)
|
Ok(trees)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn tree_name(&self, tree: usize) -> Option<String> {
|
||||||
|
self.trees
|
||||||
|
.read()
|
||||||
|
.unwrap()
|
||||||
|
.0
|
||||||
|
.get(tree)
|
||||||
|
.and_then(|x| String::from_utf8(x.name().to_vec()).ok())
|
||||||
|
}
|
||||||
|
|
||||||
// ----
|
// ----
|
||||||
|
|
||||||
fn get(&self, tree: usize, key: &[u8]) -> Result<Option<Value>> {
|
fn get(&self, tree: usize, key: &[u8]) -> Result<Option<Value>> {
|
||||||
|
|||||||
+11
-10
@@ -1,6 +1,6 @@
|
|||||||
[package]
|
[package]
|
||||||
name = "garage"
|
name = "garage"
|
||||||
version = "0.8.1"
|
version = "0.8.0"
|
||||||
authors = ["Alex Auvolat <alex@adnab.me>"]
|
authors = ["Alex Auvolat <alex@adnab.me>"]
|
||||||
edition = "2018"
|
edition = "2018"
|
||||||
license = "AGPL-3.0"
|
license = "AGPL-3.0"
|
||||||
@@ -21,14 +21,14 @@ path = "tests/lib.rs"
|
|||||||
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
|
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
|
||||||
|
|
||||||
[dependencies]
|
[dependencies]
|
||||||
garage_db = { version = "0.8.1", path = "../db" }
|
garage_db = { version = "0.8.0", path = "../db" }
|
||||||
garage_api = { version = "0.8.1", path = "../api" }
|
garage_api = { version = "0.8.0", path = "../api" }
|
||||||
garage_block = { version = "0.8.1", path = "../block" }
|
garage_block = { version = "0.8.0", path = "../block" }
|
||||||
garage_model = { version = "0.8.1", path = "../model" }
|
garage_model = { version = "0.8.0", path = "../model" }
|
||||||
garage_rpc = { version = "0.8.1", path = "../rpc" }
|
garage_rpc = { version = "0.8.0", path = "../rpc" }
|
||||||
garage_table = { version = "0.8.1", path = "../table" }
|
garage_table = { version = "0.8.0", path = "../table" }
|
||||||
garage_util = { version = "0.8.1", path = "../util" }
|
garage_util = { version = "0.8.0", path = "../util" }
|
||||||
garage_web = { version = "0.8.1", path = "../web" }
|
garage_web = { version = "0.8.0", path = "../web" }
|
||||||
|
|
||||||
backtrace = "0.3"
|
backtrace = "0.3"
|
||||||
bytes = "1.0"
|
bytes = "1.0"
|
||||||
@@ -42,6 +42,7 @@ rand = "0.8"
|
|||||||
async-trait = "0.1.7"
|
async-trait = "0.1.7"
|
||||||
sodiumoxide = { version = "0.2.5-0", package = "kuska-sodiumoxide" }
|
sodiumoxide = { version = "0.2.5-0", package = "kuska-sodiumoxide" }
|
||||||
|
|
||||||
|
rmp-serde = "0.15"
|
||||||
serde = { version = "1.0", default-features = false, features = ["derive", "rc"] }
|
serde = { version = "1.0", default-features = false, features = ["derive", "rc"] }
|
||||||
serde_bytes = "0.11"
|
serde_bytes = "0.11"
|
||||||
structopt = { version = "0.3", default-features = false }
|
structopt = { version = "0.3", default-features = false }
|
||||||
@@ -73,7 +74,7 @@ base64 = "0.13"
|
|||||||
|
|
||||||
|
|
||||||
[features]
|
[features]
|
||||||
default = [ "bundled-libs", "metrics", "sled", "k2v" ]
|
default = [ "bundled-libs", "metrics", "sled" ]
|
||||||
|
|
||||||
k2v = [ "garage_util/k2v", "garage_api/k2v" ]
|
k2v = [ "garage_util/k2v", "garage_api/k2v" ]
|
||||||
|
|
||||||
|
|||||||
+25
-94
@@ -18,6 +18,7 @@ use garage_table::*;
|
|||||||
use garage_rpc::*;
|
use garage_rpc::*;
|
||||||
|
|
||||||
use garage_block::manager::BlockResyncErrorInfo;
|
use garage_block::manager::BlockResyncErrorInfo;
|
||||||
|
use garage_block::repair::ScrubWorkerCommand;
|
||||||
|
|
||||||
use garage_model::bucket_alias_table::*;
|
use garage_model::bucket_alias_table::*;
|
||||||
use garage_model::bucket_table::*;
|
use garage_model::bucket_table::*;
|
||||||
@@ -59,7 +60,6 @@ pub enum AdminRpc {
|
|||||||
HashMap<usize, garage_util::background::WorkerInfo>,
|
HashMap<usize, garage_util::background::WorkerInfo>,
|
||||||
WorkerListOpt,
|
WorkerListOpt,
|
||||||
),
|
),
|
||||||
WorkerVars(Vec<(Uuid, String, String)>),
|
|
||||||
WorkerInfo(usize, garage_util::background::WorkerInfo),
|
WorkerInfo(usize, garage_util::background::WorkerInfo),
|
||||||
BlockErrorList(Vec<BlockResyncErrorInfo>),
|
BlockErrorList(Vec<BlockResyncErrorInfo>),
|
||||||
BlockInfo {
|
BlockInfo {
|
||||||
@@ -943,101 +943,32 @@ impl AdminRpcHandler {
|
|||||||
.clone();
|
.clone();
|
||||||
Ok(AdminRpc::WorkerInfo(*tid, info))
|
Ok(AdminRpc::WorkerInfo(*tid, info))
|
||||||
}
|
}
|
||||||
WorkerOperation::Get {
|
WorkerOperation::Set { opt } => match opt {
|
||||||
all_nodes,
|
WorkerSetCmd::ScrubTranquility { tranquility } => {
|
||||||
variable,
|
let scrub_command = ScrubWorkerCommand::SetTranquility(*tranquility);
|
||||||
} => self.handle_get_var(*all_nodes, variable).await,
|
self.garage
|
||||||
WorkerOperation::Set {
|
.block_manager
|
||||||
all_nodes,
|
.send_scrub_command(scrub_command)
|
||||||
variable,
|
.await?;
|
||||||
value,
|
Ok(AdminRpc::Ok("Scrub tranquility updated".into()))
|
||||||
} => self.handle_set_var(*all_nodes, variable, value).await,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn handle_get_var(
|
|
||||||
&self,
|
|
||||||
all_nodes: bool,
|
|
||||||
variable: &Option<String>,
|
|
||||||
) -> Result<AdminRpc, Error> {
|
|
||||||
if all_nodes {
|
|
||||||
let mut ret = vec![];
|
|
||||||
let ring = self.garage.system.ring.borrow().clone();
|
|
||||||
for node in ring.layout.node_ids().iter() {
|
|
||||||
let node = (*node).into();
|
|
||||||
match self
|
|
||||||
.endpoint
|
|
||||||
.call(
|
|
||||||
&node,
|
|
||||||
AdminRpc::Worker(WorkerOperation::Get {
|
|
||||||
all_nodes: false,
|
|
||||||
variable: variable.clone(),
|
|
||||||
}),
|
|
||||||
PRIO_NORMAL,
|
|
||||||
)
|
|
||||||
.await??
|
|
||||||
{
|
|
||||||
AdminRpc::WorkerVars(v) => ret.extend(v),
|
|
||||||
m => return Err(GarageError::unexpected_rpc_message(m).into()),
|
|
||||||
}
|
}
|
||||||
}
|
WorkerSetCmd::ResyncWorkerCount { worker_count } => {
|
||||||
Ok(AdminRpc::WorkerVars(ret))
|
self.garage
|
||||||
} else {
|
.block_manager
|
||||||
#[allow(clippy::collapsible_else_if)]
|
.resync
|
||||||
if let Some(v) = variable {
|
.set_n_workers(*worker_count)
|
||||||
Ok(AdminRpc::WorkerVars(vec![(
|
.await?;
|
||||||
self.garage.system.id,
|
Ok(AdminRpc::Ok("Number of resync workers updated".into()))
|
||||||
v.clone(),
|
|
||||||
self.garage.bg_vars.get(v)?,
|
|
||||||
)]))
|
|
||||||
} else {
|
|
||||||
let mut vars = self.garage.bg_vars.get_all();
|
|
||||||
vars.sort();
|
|
||||||
Ok(AdminRpc::WorkerVars(
|
|
||||||
vars.into_iter()
|
|
||||||
.map(|(k, v)| (self.garage.system.id, k.to_string(), v))
|
|
||||||
.collect(),
|
|
||||||
))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn handle_set_var(
|
|
||||||
&self,
|
|
||||||
all_nodes: bool,
|
|
||||||
variable: &str,
|
|
||||||
value: &str,
|
|
||||||
) -> Result<AdminRpc, Error> {
|
|
||||||
if all_nodes {
|
|
||||||
let mut ret = vec![];
|
|
||||||
let ring = self.garage.system.ring.borrow().clone();
|
|
||||||
for node in ring.layout.node_ids().iter() {
|
|
||||||
let node = (*node).into();
|
|
||||||
match self
|
|
||||||
.endpoint
|
|
||||||
.call(
|
|
||||||
&node,
|
|
||||||
AdminRpc::Worker(WorkerOperation::Set {
|
|
||||||
all_nodes: false,
|
|
||||||
variable: variable.to_string(),
|
|
||||||
value: value.to_string(),
|
|
||||||
}),
|
|
||||||
PRIO_NORMAL,
|
|
||||||
)
|
|
||||||
.await??
|
|
||||||
{
|
|
||||||
AdminRpc::WorkerVars(v) => ret.extend(v),
|
|
||||||
m => return Err(GarageError::unexpected_rpc_message(m).into()),
|
|
||||||
}
|
}
|
||||||
}
|
WorkerSetCmd::ResyncTranquility { tranquility } => {
|
||||||
Ok(AdminRpc::WorkerVars(ret))
|
self.garage
|
||||||
} else {
|
.block_manager
|
||||||
self.garage.bg_vars.set(variable, value)?;
|
.resync
|
||||||
Ok(AdminRpc::WorkerVars(vec![(
|
.set_tranquility(*tranquility)
|
||||||
self.garage.system.id,
|
.await?;
|
||||||
variable.to_string(),
|
Ok(AdminRpc::Ok("Resync tranquility updated".into()))
|
||||||
value.to_string(),
|
}
|
||||||
)]))
|
},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -191,9 +191,6 @@ pub async fn cmd_admin(
|
|||||||
AdminRpc::WorkerList(wi, wlo) => {
|
AdminRpc::WorkerList(wi, wlo) => {
|
||||||
print_worker_list(wi, wlo);
|
print_worker_list(wi, wlo);
|
||||||
}
|
}
|
||||||
AdminRpc::WorkerVars(wv) => {
|
|
||||||
print_worker_vars(wv);
|
|
||||||
}
|
|
||||||
AdminRpc::WorkerInfo(tid, wi) => {
|
AdminRpc::WorkerInfo(tid, wi) => {
|
||||||
print_worker_info(tid, wi);
|
print_worker_info(tid, wi);
|
||||||
}
|
}
|
||||||
|
|||||||
+15
-16
@@ -517,25 +517,11 @@ pub enum WorkerOperation {
|
|||||||
/// Get detailed information about a worker
|
/// Get detailed information about a worker
|
||||||
#[structopt(name = "info", version = garage_version())]
|
#[structopt(name = "info", version = garage_version())]
|
||||||
Info { tid: usize },
|
Info { tid: usize },
|
||||||
/// Get worker parameter
|
|
||||||
#[structopt(name = "get", version = garage_version())]
|
|
||||||
Get {
|
|
||||||
/// Gather variable values from all nodes
|
|
||||||
#[structopt(short = "a", long = "all-nodes")]
|
|
||||||
all_nodes: bool,
|
|
||||||
/// Variable name to get, or none to get all variables
|
|
||||||
variable: Option<String>,
|
|
||||||
},
|
|
||||||
/// Set worker parameter
|
/// Set worker parameter
|
||||||
#[structopt(name = "set", version = garage_version())]
|
#[structopt(name = "set", version = garage_version())]
|
||||||
Set {
|
Set {
|
||||||
/// Set variable values on all nodes
|
#[structopt(subcommand)]
|
||||||
#[structopt(short = "a", long = "all-nodes")]
|
opt: WorkerSetCmd,
|
||||||
all_nodes: bool,
|
|
||||||
/// Variable node to set
|
|
||||||
variable: String,
|
|
||||||
/// Value to set the variable to
|
|
||||||
value: String,
|
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -549,6 +535,19 @@ pub struct WorkerListOpt {
|
|||||||
pub errors: bool,
|
pub errors: bool,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Serialize, Deserialize, StructOpt, Debug, Eq, PartialEq, Clone)]
|
||||||
|
pub enum WorkerSetCmd {
|
||||||
|
/// Set tranquility of scrub operations
|
||||||
|
#[structopt(name = "scrub-tranquility", version = garage_version())]
|
||||||
|
ScrubTranquility { tranquility: u32 },
|
||||||
|
/// Set number of concurrent block resync workers
|
||||||
|
#[structopt(name = "resync-worker-count", version = garage_version())]
|
||||||
|
ResyncWorkerCount { worker_count: usize },
|
||||||
|
/// Set tranquility of block resync operations
|
||||||
|
#[structopt(name = "resync-tranquility", version = garage_version())]
|
||||||
|
ResyncTranquility { tranquility: u32 },
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Serialize, Deserialize, StructOpt, Debug, Eq, PartialEq, Clone)]
|
#[derive(Serialize, Deserialize, StructOpt, Debug, Eq, PartialEq, Clone)]
|
||||||
pub enum BlockOperation {
|
pub enum BlockOperation {
|
||||||
/// List all blocks that currently have a resync error
|
/// List all blocks that currently have a resync error
|
||||||
|
|||||||
@@ -357,14 +357,6 @@ pub fn print_worker_info(tid: usize, info: WorkerInfo) {
|
|||||||
format_table(table);
|
format_table(table);
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn print_worker_vars(wv: Vec<(Uuid, String, String)>) {
|
|
||||||
let table = wv
|
|
||||||
.into_iter()
|
|
||||||
.map(|(n, k, v)| format!("{:?}\t{}\t{}", n, k, v))
|
|
||||||
.collect::<Vec<_>>();
|
|
||||||
format_table(table);
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn print_block_error_list(el: Vec<BlockResyncErrorInfo>) {
|
pub fn print_block_error_list(el: Vec<BlockResyncErrorInfo>) {
|
||||||
let now = now_msec();
|
let now = now_msec();
|
||||||
let tf = timeago::Formatter::new();
|
let tf = timeago::Formatter::new();
|
||||||
|
|||||||
@@ -12,7 +12,6 @@ use garage_model::s3::version_table::*;
|
|||||||
use garage_table::*;
|
use garage_table::*;
|
||||||
use garage_util::background::*;
|
use garage_util::background::*;
|
||||||
use garage_util::error::Error;
|
use garage_util::error::Error;
|
||||||
use garage_util::migrate::Migrate;
|
|
||||||
|
|
||||||
use crate::*;
|
use crate::*;
|
||||||
|
|
||||||
@@ -51,11 +50,7 @@ pub async fn launch_online_repair(
|
|||||||
ScrubCmd::Resume => ScrubWorkerCommand::Resume,
|
ScrubCmd::Resume => ScrubWorkerCommand::Resume,
|
||||||
ScrubCmd::Cancel => ScrubWorkerCommand::Cancel,
|
ScrubCmd::Cancel => ScrubWorkerCommand::Cancel,
|
||||||
ScrubCmd::SetTranquility { tranquility } => {
|
ScrubCmd::SetTranquility { tranquility } => {
|
||||||
garage
|
ScrubWorkerCommand::SetTranquility(tranquility)
|
||||||
.block_manager
|
|
||||||
.scrub_persister
|
|
||||||
.set_with(|x| x.tranquility = tranquility)?;
|
|
||||||
return Ok(());
|
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
info!("Sending command to scrub worker: {:?}", cmd);
|
info!("Sending command to scrub worker: {:?}", cmd);
|
||||||
@@ -105,7 +100,7 @@ impl Worker for RepairVersionsWorker {
|
|||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
let version = Version::decode(&item_bytes).ok_or_message("Cannot decode Version")?;
|
let version = rmp_serde::decode::from_read_ref::<_, Version>(&item_bytes)?;
|
||||||
if !version.deleted.get() {
|
if !version.deleted.get() {
|
||||||
let object = self
|
let object = self
|
||||||
.garage
|
.garage
|
||||||
@@ -185,7 +180,7 @@ impl Worker for RepairBlockrefsWorker {
|
|||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
let block_ref = BlockRef::decode(&item_bytes).ok_or_message("Cannot decode BlockRef")?;
|
let block_ref = rmp_serde::decode::from_read_ref::<_, BlockRef>(&item_bytes)?;
|
||||||
if !block_ref.deleted.get() {
|
if !block_ref.deleted.get() {
|
||||||
let version = self
|
let version = self
|
||||||
.garage
|
.garage
|
||||||
|
|||||||
@@ -22,7 +22,7 @@ tokio = "1.17.0"
|
|||||||
|
|
||||||
# cli deps
|
# cli deps
|
||||||
clap = { version = "3.1.18", optional = true, features = ["derive", "env"] }
|
clap = { version = "3.1.18", optional = true, features = ["derive", "env"] }
|
||||||
garage_util = { version = "0.8.1", path = "../util", optional = true }
|
garage_util = { version = "0.8.0", path = "../util", optional = true }
|
||||||
|
|
||||||
|
|
||||||
[features]
|
[features]
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
[package]
|
[package]
|
||||||
name = "garage_model"
|
name = "garage_model"
|
||||||
version = "0.8.1"
|
version = "0.8.0"
|
||||||
authors = ["Alex Auvolat <alex@adnab.me>"]
|
authors = ["Alex Auvolat <alex@adnab.me>"]
|
||||||
edition = "2018"
|
edition = "2018"
|
||||||
license = "AGPL-3.0"
|
license = "AGPL-3.0"
|
||||||
@@ -14,11 +14,11 @@ path = "lib.rs"
|
|||||||
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
|
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
|
||||||
|
|
||||||
[dependencies]
|
[dependencies]
|
||||||
garage_db = { version = "0.8.1", default-features = false, path = "../db" }
|
garage_db = { version = "0.8.0", default-features = false, path = "../db" }
|
||||||
garage_rpc = { version = "0.8.1", path = "../rpc" }
|
garage_rpc = { version = "0.8.0", path = "../rpc" }
|
||||||
garage_table = { version = "0.8.1", path = "../table" }
|
garage_table = { version = "0.8.0", path = "../table" }
|
||||||
garage_block = { version = "0.8.1", path = "../block" }
|
garage_block = { version = "0.8.0", path = "../block" }
|
||||||
garage_util = { version = "0.8.1", path = "../util" }
|
garage_util = { version = "0.8.0", path = "../util" }
|
||||||
|
|
||||||
async-trait = "0.1.7"
|
async-trait = "0.1.7"
|
||||||
arc-swap = "1.0"
|
arc-swap = "1.0"
|
||||||
@@ -30,6 +30,7 @@ tracing = "0.1.30"
|
|||||||
rand = "0.8"
|
rand = "0.8"
|
||||||
zstd = { version = "0.9", default-features = false }
|
zstd = { version = "0.9", default-features = false }
|
||||||
|
|
||||||
|
rmp-serde = "0.15"
|
||||||
serde = { version = "1.0", default-features = false, features = ["derive", "rc"] }
|
serde = { version = "1.0", default-features = false, features = ["derive", "rc"] }
|
||||||
serde_bytes = "0.11"
|
serde_bytes = "0.11"
|
||||||
|
|
||||||
|
|||||||
@@ -1,26 +1,18 @@
|
|||||||
|
use serde::{Deserialize, Serialize};
|
||||||
|
|
||||||
use garage_util::data::*;
|
use garage_util::data::*;
|
||||||
|
|
||||||
use garage_table::crdt::*;
|
use garage_table::crdt::*;
|
||||||
use garage_table::*;
|
use garage_table::*;
|
||||||
|
|
||||||
mod v08 {
|
/// The bucket alias table holds the names given to buckets
|
||||||
use garage_util::crdt;
|
/// in the global namespace.
|
||||||
use garage_util::data::Uuid;
|
#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
|
||||||
use serde::{Deserialize, Serialize};
|
pub struct BucketAlias {
|
||||||
|
name: String,
|
||||||
/// The bucket alias table holds the names given to buckets
|
pub state: crdt::Lww<Option<Uuid>>,
|
||||||
/// in the global namespace.
|
|
||||||
#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
|
|
||||||
pub struct BucketAlias {
|
|
||||||
pub(super) name: String,
|
|
||||||
pub state: crdt::Lww<Option<Uuid>>,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl garage_util::migrate::InitialFormat for BucketAlias {}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
pub use v08::*;
|
|
||||||
|
|
||||||
impl BucketAlias {
|
impl BucketAlias {
|
||||||
pub fn new(name: String, ts: u64, bucket_id: Option<Uuid>) -> Option<Self> {
|
pub fn new(name: String, ts: u64, bucket_id: Option<Uuid>) -> Option<Self> {
|
||||||
if !is_valid_bucket_name(&name) {
|
if !is_valid_bucket_name(&name) {
|
||||||
|
|||||||
+65
-74
@@ -1,3 +1,5 @@
|
|||||||
|
use serde::{Deserialize, Serialize};
|
||||||
|
|
||||||
use garage_table::crdt::*;
|
use garage_table::crdt::*;
|
||||||
use garage_table::*;
|
use garage_table::*;
|
||||||
use garage_util::data::*;
|
use garage_util::data::*;
|
||||||
@@ -5,82 +7,71 @@ use garage_util::time::*;
|
|||||||
|
|
||||||
use crate::permission::BucketKeyPerm;
|
use crate::permission::BucketKeyPerm;
|
||||||
|
|
||||||
mod v08 {
|
/// A bucket is a collection of objects
|
||||||
use crate::permission::BucketKeyPerm;
|
///
|
||||||
use garage_util::crdt;
|
/// Its parameters are not directly accessible as:
|
||||||
use garage_util::data::Uuid;
|
/// - It must be possible to merge paramaters, hence the use of a LWW CRDT.
|
||||||
use serde::{Deserialize, Serialize};
|
/// - A bucket has 2 states, Present or Deleted and parameters make sense only if present.
|
||||||
|
#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
|
||||||
/// A bucket is a collection of objects
|
pub struct Bucket {
|
||||||
///
|
/// ID of the bucket
|
||||||
/// Its parameters are not directly accessible as:
|
pub id: Uuid,
|
||||||
/// - It must be possible to merge paramaters, hence the use of a LWW CRDT.
|
/// State, and configuration if not deleted, of the bucket
|
||||||
/// - A bucket has 2 states, Present or Deleted and parameters make sense only if present.
|
pub state: crdt::Deletable<BucketParams>,
|
||||||
#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
|
|
||||||
pub struct Bucket {
|
|
||||||
/// ID of the bucket
|
|
||||||
pub id: Uuid,
|
|
||||||
/// State, and configuration if not deleted, of the bucket
|
|
||||||
pub state: crdt::Deletable<BucketParams>,
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Configuration for a bucket
|
|
||||||
#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
|
|
||||||
pub struct BucketParams {
|
|
||||||
/// Bucket's creation date
|
|
||||||
pub creation_date: u64,
|
|
||||||
/// Map of key with access to the bucket, and what kind of access they give
|
|
||||||
pub authorized_keys: crdt::Map<String, BucketKeyPerm>,
|
|
||||||
|
|
||||||
/// Map of aliases that are or have been given to this bucket
|
|
||||||
/// in the global namespace
|
|
||||||
/// (not authoritative: this is just used as an indication to
|
|
||||||
/// map back to aliases when doing ListBuckets)
|
|
||||||
pub aliases: crdt::LwwMap<String, bool>,
|
|
||||||
/// Map of aliases that are or have been given to this bucket
|
|
||||||
/// in namespaces local to keys
|
|
||||||
/// key = (access key id, alias name)
|
|
||||||
pub local_aliases: crdt::LwwMap<(String, String), bool>,
|
|
||||||
|
|
||||||
/// Whether this bucket is allowed for website access
|
|
||||||
/// (under all of its global alias names),
|
|
||||||
/// and if so, the website configuration XML document
|
|
||||||
pub website_config: crdt::Lww<Option<WebsiteConfig>>,
|
|
||||||
/// CORS rules
|
|
||||||
pub cors_config: crdt::Lww<Option<Vec<CorsRule>>>,
|
|
||||||
/// Bucket quotas
|
|
||||||
#[serde(default)]
|
|
||||||
pub quotas: crdt::Lww<BucketQuotas>,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
|
|
||||||
pub struct WebsiteConfig {
|
|
||||||
pub index_document: String,
|
|
||||||
pub error_document: Option<String>,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
|
|
||||||
pub struct CorsRule {
|
|
||||||
pub id: Option<String>,
|
|
||||||
pub max_age_seconds: Option<u64>,
|
|
||||||
pub allow_origins: Vec<String>,
|
|
||||||
pub allow_methods: Vec<String>,
|
|
||||||
pub allow_headers: Vec<String>,
|
|
||||||
pub expose_headers: Vec<String>,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Default, PartialEq, Eq, PartialOrd, Ord, Clone, Debug, Serialize, Deserialize)]
|
|
||||||
pub struct BucketQuotas {
|
|
||||||
/// Maximum size in bytes (bucket size = sum of sizes of objects in the bucket)
|
|
||||||
pub max_size: Option<u64>,
|
|
||||||
/// Maximum number of non-deleted objects in the bucket
|
|
||||||
pub max_objects: Option<u64>,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl garage_util::migrate::InitialFormat for Bucket {}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
pub use v08::*;
|
/// Configuration for a bucket
|
||||||
|
#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
|
||||||
|
pub struct BucketParams {
|
||||||
|
/// Bucket's creation date
|
||||||
|
pub creation_date: u64,
|
||||||
|
/// Map of key with access to the bucket, and what kind of access they give
|
||||||
|
pub authorized_keys: crdt::Map<String, BucketKeyPerm>,
|
||||||
|
|
||||||
|
/// Map of aliases that are or have been given to this bucket
|
||||||
|
/// in the global namespace
|
||||||
|
/// (not authoritative: this is just used as an indication to
|
||||||
|
/// map back to aliases when doing ListBuckets)
|
||||||
|
pub aliases: crdt::LwwMap<String, bool>,
|
||||||
|
/// Map of aliases that are or have been given to this bucket
|
||||||
|
/// in namespaces local to keys
|
||||||
|
/// key = (access key id, alias name)
|
||||||
|
pub local_aliases: crdt::LwwMap<(String, String), bool>,
|
||||||
|
|
||||||
|
/// Whether this bucket is allowed for website access
|
||||||
|
/// (under all of its global alias names),
|
||||||
|
/// and if so, the website configuration XML document
|
||||||
|
pub website_config: crdt::Lww<Option<WebsiteConfig>>,
|
||||||
|
/// CORS rules
|
||||||
|
pub cors_config: crdt::Lww<Option<Vec<CorsRule>>>,
|
||||||
|
/// Bucket quotas
|
||||||
|
#[serde(default)]
|
||||||
|
pub quotas: crdt::Lww<BucketQuotas>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
|
||||||
|
pub struct WebsiteConfig {
|
||||||
|
pub index_document: String,
|
||||||
|
pub error_document: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
|
||||||
|
pub struct CorsRule {
|
||||||
|
pub id: Option<String>,
|
||||||
|
pub max_age_seconds: Option<u64>,
|
||||||
|
pub allow_origins: Vec<String>,
|
||||||
|
pub allow_methods: Vec<String>,
|
||||||
|
pub allow_headers: Vec<String>,
|
||||||
|
pub expose_headers: Vec<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Default, PartialEq, Eq, PartialOrd, Ord, Clone, Debug, Serialize, Deserialize)]
|
||||||
|
pub struct BucketQuotas {
|
||||||
|
/// Maximum size in bytes (bucket size = sum of sizes of objects in the bucket)
|
||||||
|
pub max_size: Option<u64>,
|
||||||
|
/// Maximum number of non-deleted objects in the bucket
|
||||||
|
pub max_objects: Option<u64>,
|
||||||
|
}
|
||||||
|
|
||||||
impl AutoCrdt for BucketQuotas {
|
impl AutoCrdt for BucketQuotas {
|
||||||
const WARN_IF_DIFFERENT: bool = true;
|
const WARN_IF_DIFFERENT: bool = true;
|
||||||
|
|||||||
+2
-31
@@ -27,14 +27,12 @@ use crate::index_counter::*;
|
|||||||
use crate::key_table::*;
|
use crate::key_table::*;
|
||||||
|
|
||||||
#[cfg(feature = "k2v")]
|
#[cfg(feature = "k2v")]
|
||||||
use crate::k2v::{history_table::*, item_table::*, sub::*, rpc::*};
|
use crate::k2v::{item_table::*, poll::*, rpc::*};
|
||||||
|
|
||||||
/// An entire Garage full of data
|
/// An entire Garage full of data
|
||||||
pub struct Garage {
|
pub struct Garage {
|
||||||
/// The parsed configuration Garage is running
|
/// The parsed configuration Garage is running
|
||||||
pub config: Config,
|
pub config: Config,
|
||||||
/// The set of background variables that can be viewed/modified at runtime
|
|
||||||
pub bg_vars: vars::BgVars,
|
|
||||||
|
|
||||||
/// The replication mode of this cluster
|
/// The replication mode of this cluster
|
||||||
pub replication_mode: ReplicationMode,
|
pub replication_mode: ReplicationMode,
|
||||||
@@ -70,8 +68,6 @@ pub struct Garage {
|
|||||||
pub struct GarageK2V {
|
pub struct GarageK2V {
|
||||||
/// Table containing K2V items
|
/// Table containing K2V items
|
||||||
pub item_table: Arc<Table<K2VItemTable, TableShardedReplication>>,
|
pub item_table: Arc<Table<K2VItemTable, TableShardedReplication>>,
|
||||||
/// Table containing K2V modification history
|
|
||||||
pub history_table: Arc<Table<K2VHistoryTable, TableShardedReplication>>,
|
|
||||||
/// Indexing table containing K2V item counters
|
/// Indexing table containing K2V item counters
|
||||||
pub counter_table: Arc<IndexCounter<K2VItem>>,
|
pub counter_table: Arc<IndexCounter<K2VItem>>,
|
||||||
/// K2V RPC handler
|
/// K2V RPC handler
|
||||||
@@ -253,14 +249,9 @@ impl Garage {
|
|||||||
#[cfg(feature = "k2v")]
|
#[cfg(feature = "k2v")]
|
||||||
let k2v = GarageK2V::new(system.clone(), &db, meta_rep_param);
|
let k2v = GarageK2V::new(system.clone(), &db, meta_rep_param);
|
||||||
|
|
||||||
// Initialize bg vars
|
|
||||||
let mut bg_vars = vars::BgVars::new();
|
|
||||||
block_manager.register_bg_vars(&mut bg_vars);
|
|
||||||
|
|
||||||
// -- done --
|
// -- done --
|
||||||
Ok(Arc::new(Self {
|
Ok(Arc::new(Self {
|
||||||
config,
|
config,
|
||||||
bg_vars,
|
|
||||||
replication_mode,
|
replication_mode,
|
||||||
db,
|
db,
|
||||||
system,
|
system,
|
||||||
@@ -307,42 +298,22 @@ impl GarageK2V {
|
|||||||
fn new(system: Arc<System>, db: &db::Db, meta_rep_param: TableShardedReplication) -> Self {
|
fn new(system: Arc<System>, db: &db::Db, meta_rep_param: TableShardedReplication) -> Self {
|
||||||
info!("Initialize K2V counter table...");
|
info!("Initialize K2V counter table...");
|
||||||
let counter_table = IndexCounter::new(system.clone(), meta_rep_param.clone(), db);
|
let counter_table = IndexCounter::new(system.clone(), meta_rep_param.clone(), db);
|
||||||
|
|
||||||
info!("Initialize K2V subscription manager...");
|
info!("Initialize K2V subscription manager...");
|
||||||
let subscriptions = Arc::new(SubscriptionManager::new());
|
let subscriptions = Arc::new(SubscriptionManager::new());
|
||||||
|
|
||||||
info!("Initialize K2V item table...");
|
info!("Initialize K2V item table...");
|
||||||
let item_table = Table::new(
|
let item_table = Table::new(
|
||||||
K2VItemTable {
|
K2VItemTable {
|
||||||
counter_table: counter_table.clone(),
|
counter_table: counter_table.clone(),
|
||||||
subscriptions: subscriptions.clone(),
|
subscriptions: subscriptions.clone(),
|
||||||
},
|
},
|
||||||
meta_rep_param.clone(),
|
|
||||||
system.clone(),
|
|
||||||
db,
|
|
||||||
);
|
|
||||||
info!("Initialize K2V history table...");
|
|
||||||
let history_table = Table::new(
|
|
||||||
K2VHistoryTable {
|
|
||||||
subscriptions: subscriptions.clone(),
|
|
||||||
},
|
|
||||||
meta_rep_param,
|
meta_rep_param,
|
||||||
system.clone(),
|
system.clone(),
|
||||||
db,
|
db,
|
||||||
);
|
);
|
||||||
|
let rpc = K2VRpcHandler::new(system, item_table.clone(), subscriptions);
|
||||||
info!("Initialize K2V RPC handler...");
|
|
||||||
let rpc = K2VRpcHandler::new(
|
|
||||||
system,
|
|
||||||
db,
|
|
||||||
item_table.clone(),
|
|
||||||
history_table.clone(),
|
|
||||||
subscriptions,
|
|
||||||
);
|
|
||||||
|
|
||||||
Self {
|
Self {
|
||||||
item_table,
|
item_table,
|
||||||
history_table,
|
|
||||||
counter_table,
|
counter_table,
|
||||||
rpc,
|
rpc,
|
||||||
}
|
}
|
||||||
|
|||||||
+32
-48
@@ -12,7 +12,6 @@ use garage_rpc::system::System;
|
|||||||
use garage_util::background::BackgroundRunner;
|
use garage_util::background::BackgroundRunner;
|
||||||
use garage_util::data::*;
|
use garage_util::data::*;
|
||||||
use garage_util::error::*;
|
use garage_util::error::*;
|
||||||
use garage_util::migrate::Migrate;
|
|
||||||
use garage_util::time::*;
|
use garage_util::time::*;
|
||||||
|
|
||||||
use garage_table::crdt::*;
|
use garage_table::crdt::*;
|
||||||
@@ -30,44 +29,14 @@ pub trait CountedItem: Clone + PartialEq + Send + Sync + 'static {
|
|||||||
fn counts(&self) -> Vec<(&'static str, i64)>;
|
fn counts(&self) -> Vec<(&'static str, i64)>;
|
||||||
}
|
}
|
||||||
|
|
||||||
mod v08 {
|
/// A counter entry in the global table
|
||||||
use super::CountedItem;
|
#[derive(Clone, PartialEq, Debug, Serialize, Deserialize)]
|
||||||
use garage_util::data::Uuid;
|
pub struct CounterEntry<T: CountedItem> {
|
||||||
use serde::{Deserialize, Serialize};
|
pub pk: T::CP,
|
||||||
use std::collections::BTreeMap;
|
pub sk: T::CS,
|
||||||
|
pub values: BTreeMap<String, CounterValue>,
|
||||||
// ---- Global part (the table everyone queries) ----
|
|
||||||
|
|
||||||
/// A counter entry in the global table
|
|
||||||
#[derive(Clone, PartialEq, Debug, Serialize, Deserialize)]
|
|
||||||
pub struct CounterEntry<T: CountedItem> {
|
|
||||||
pub pk: T::CP,
|
|
||||||
pub sk: T::CS,
|
|
||||||
pub values: BTreeMap<String, CounterValue>,
|
|
||||||
}
|
|
||||||
|
|
||||||
/// A counter entry in the global table
|
|
||||||
#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
|
|
||||||
pub struct CounterValue {
|
|
||||||
pub node_values: BTreeMap<Uuid, (u64, i64)>,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl<T: CountedItem> garage_util::migrate::InitialFormat for CounterEntry<T> {}
|
|
||||||
|
|
||||||
// ---- Local part (the counter we maintain transactionnaly on each node) ----
|
|
||||||
|
|
||||||
#[derive(PartialEq, Clone, Debug, Serialize, Deserialize)]
|
|
||||||
pub(super) struct LocalCounterEntry<T: CountedItem> {
|
|
||||||
pub(super) pk: T::CP,
|
|
||||||
pub(super) sk: T::CS,
|
|
||||||
pub(super) values: BTreeMap<String, (u64, i64)>,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl<T: CountedItem> garage_util::migrate::InitialFormat for LocalCounterEntry<T> {}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
pub use v08::*;
|
|
||||||
|
|
||||||
impl<T: CountedItem> Entry<T::CP, T::CS> for CounterEntry<T> {
|
impl<T: CountedItem> Entry<T::CP, T::CS> for CounterEntry<T> {
|
||||||
fn partition_key(&self) -> &T::CP {
|
fn partition_key(&self) -> &T::CP {
|
||||||
&self.pk
|
&self.pk
|
||||||
@@ -109,6 +78,12 @@ impl<T: CountedItem> CounterEntry<T> {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// A counter entry in the global table
|
||||||
|
#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
|
||||||
|
pub struct CounterValue {
|
||||||
|
pub node_values: BTreeMap<Uuid, (u64, i64)>,
|
||||||
|
}
|
||||||
|
|
||||||
impl<T: CountedItem> Crdt for CounterEntry<T> {
|
impl<T: CountedItem> Crdt for CounterEntry<T> {
|
||||||
fn merge(&mut self, other: &Self) {
|
fn merge(&mut self, other: &Self) {
|
||||||
for (name, e2) in other.values.iter() {
|
for (name, e2) in other.values.iter() {
|
||||||
@@ -220,9 +195,11 @@ impl<T: CountedItem> IndexCounter<T> {
|
|||||||
let tree_key = self.table.data.tree_key(pk, sk);
|
let tree_key = self.table.data.tree_key(pk, sk);
|
||||||
|
|
||||||
let mut entry = match tx.get(&self.local_counter, &tree_key[..])? {
|
let mut entry = match tx.get(&self.local_counter, &tree_key[..])? {
|
||||||
Some(old_bytes) => LocalCounterEntry::<T>::decode(&old_bytes)
|
Some(old_bytes) => {
|
||||||
.ok_or_message("Cannot decode local counter entry")
|
rmp_serde::decode::from_read_ref::<_, LocalCounterEntry<T>>(&old_bytes)
|
||||||
.map_err(db::TxError::Abort)?,
|
.map_err(Error::RmpDecode)
|
||||||
|
.map_err(db::TxError::Abort)?
|
||||||
|
}
|
||||||
None => LocalCounterEntry {
|
None => LocalCounterEntry {
|
||||||
pk: pk.clone(),
|
pk: pk.clone(),
|
||||||
sk: sk.clone(),
|
sk: sk.clone(),
|
||||||
@@ -237,8 +214,7 @@ impl<T: CountedItem> IndexCounter<T> {
|
|||||||
ent.1 += *inc;
|
ent.1 += *inc;
|
||||||
}
|
}
|
||||||
|
|
||||||
let new_entry_bytes = entry
|
let new_entry_bytes = rmp_to_vec_all_named(&entry)
|
||||||
.encode()
|
|
||||||
.map_err(Error::RmpEncode)
|
.map_err(Error::RmpEncode)
|
||||||
.map_err(db::TxError::Abort)?;
|
.map_err(db::TxError::Abort)?;
|
||||||
tx.insert(&self.local_counter, &tree_key[..], new_entry_bytes)?;
|
tx.insert(&self.local_counter, &tree_key[..], new_entry_bytes)?;
|
||||||
@@ -279,15 +255,15 @@ impl<T: CountedItem> IndexCounter<T> {
|
|||||||
|
|
||||||
info!("zeroing old counters... ({})", hex::encode(&batch[0].0));
|
info!("zeroing old counters... ({})", hex::encode(&batch[0].0));
|
||||||
for (local_counter_k, local_counter) in batch {
|
for (local_counter_k, local_counter) in batch {
|
||||||
let mut local_counter = LocalCounterEntry::<T>::decode(&local_counter)
|
let mut local_counter =
|
||||||
.ok_or_message("Cannot decode local counter entry")?;
|
rmp_serde::decode::from_read_ref::<_, LocalCounterEntry<T>>(&local_counter)?;
|
||||||
|
|
||||||
for (_, tv) in local_counter.values.iter_mut() {
|
for (_, tv) in local_counter.values.iter_mut() {
|
||||||
tv.0 = std::cmp::max(tv.0 + 1, now);
|
tv.0 = std::cmp::max(tv.0 + 1, now);
|
||||||
tv.1 = 0;
|
tv.1 = 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
let local_counter_bytes = local_counter.encode()?;
|
let local_counter_bytes = rmp_to_vec_all_named(&local_counter)?;
|
||||||
self.local_counter
|
self.local_counter
|
||||||
.insert(&local_counter_k, &local_counter_bytes)?;
|
.insert(&local_counter_k, &local_counter_bytes)?;
|
||||||
|
|
||||||
@@ -335,8 +311,9 @@ impl<T: CountedItem> IndexCounter<T> {
|
|||||||
let local_counter_key = self.table.data.tree_key(pk, sk);
|
let local_counter_key = self.table.data.tree_key(pk, sk);
|
||||||
let mut local_counter = match self.local_counter.get(&local_counter_key)? {
|
let mut local_counter = match self.local_counter.get(&local_counter_key)? {
|
||||||
Some(old_bytes) => {
|
Some(old_bytes) => {
|
||||||
let ent = LocalCounterEntry::<T>::decode(&old_bytes)
|
let ent = rmp_serde::decode::from_read_ref::<_, LocalCounterEntry<T>>(
|
||||||
.ok_or_message("Cannot decode local counter entry")?;
|
&old_bytes,
|
||||||
|
)?;
|
||||||
assert!(ent.pk == *pk);
|
assert!(ent.pk == *pk);
|
||||||
assert!(ent.sk == *sk);
|
assert!(ent.sk == *sk);
|
||||||
ent
|
ent
|
||||||
@@ -353,7 +330,7 @@ impl<T: CountedItem> IndexCounter<T> {
|
|||||||
tv.1 += v;
|
tv.1 += v;
|
||||||
}
|
}
|
||||||
|
|
||||||
let local_counter_bytes = local_counter.encode()?;
|
let local_counter_bytes = rmp_to_vec_all_named(&local_counter)?;
|
||||||
self.local_counter
|
self.local_counter
|
||||||
.insert(&local_counter_key, local_counter_bytes)?;
|
.insert(&local_counter_key, local_counter_bytes)?;
|
||||||
|
|
||||||
@@ -373,6 +350,13 @@ impl<T: CountedItem> IndexCounter<T> {
|
|||||||
|
|
||||||
// ----
|
// ----
|
||||||
|
|
||||||
|
#[derive(PartialEq, Clone, Debug, Serialize, Deserialize)]
|
||||||
|
struct LocalCounterEntry<T: CountedItem> {
|
||||||
|
pk: T::CP,
|
||||||
|
sk: T::CS,
|
||||||
|
values: BTreeMap<String, (u64, i64)>,
|
||||||
|
}
|
||||||
|
|
||||||
impl<T: CountedItem> LocalCounterEntry<T> {
|
impl<T: CountedItem> LocalCounterEntry<T> {
|
||||||
fn into_counter_entry(self, this_node: Uuid) -> CounterEntry<T> {
|
fn into_counter_entry(self, this_node: Uuid) -> CounterEntry<T> {
|
||||||
CounterEntry {
|
CounterEntry {
|
||||||
|
|||||||
@@ -1,107 +0,0 @@
|
|||||||
use std::sync::Arc;
|
|
||||||
|
|
||||||
use garage_db as db;
|
|
||||||
|
|
||||||
use garage_table::crdt::*;
|
|
||||||
use garage_table::*;
|
|
||||||
|
|
||||||
use crate::k2v::sub::*;
|
|
||||||
|
|
||||||
mod v08 {
|
|
||||||
use crate::k2v::causality::K2VNodeId;
|
|
||||||
pub use crate::k2v::item_table::v08::{DvvsValue, K2VItem, K2VItemPartition};
|
|
||||||
use garage_util::crdt;
|
|
||||||
use serde::{Deserialize, Serialize};
|
|
||||||
|
|
||||||
#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
|
|
||||||
pub struct K2VHistoryEntry {
|
|
||||||
// Partition key: the partition key of ins_item
|
|
||||||
|
|
||||||
/// The inserted item
|
|
||||||
pub ins_item: K2VItem,
|
|
||||||
|
|
||||||
/// Sort key: the node ID and its local counter
|
|
||||||
pub node_counter: K2VHistorySortKey,
|
|
||||||
|
|
||||||
/// The value of the node's local counter before this entry was updated
|
|
||||||
pub prev_counter: u64,
|
|
||||||
/// The timesamp of the update (!= counter, counters are incremented
|
|
||||||
/// by one, timestamps are real clock timestamps)
|
|
||||||
pub timestamp: u64,
|
|
||||||
|
|
||||||
/// Mark this history entry for deletion
|
|
||||||
pub deleted: crdt::Bool,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
|
|
||||||
pub struct K2VHistorySortKey {
|
|
||||||
pub node: K2VNodeId,
|
|
||||||
pub counter: u64,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl garage_util::migrate::InitialFormat for K2VHistoryEntry {
|
|
||||||
const VERSION_MARKER: &'static [u8] = b"Gk2vhe08";
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
pub use v08::*;
|
|
||||||
|
|
||||||
impl Crdt for K2VHistoryEntry {
|
|
||||||
fn merge(&mut self, other: &Self) {
|
|
||||||
self.ins_item.merge(&other.ins_item);
|
|
||||||
self.deleted.merge(&other.deleted);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl SortKey for K2VHistorySortKey {
|
|
||||||
type B<'a> = [u8; 16];
|
|
||||||
|
|
||||||
fn sort_key(&self) -> [u8; 16] {
|
|
||||||
let mut ret = [0u8; 16];
|
|
||||||
ret[0..8].copy_from_slice(&u64::to_be_bytes(self.node));
|
|
||||||
ret[8..16].copy_from_slice(&u64::to_be_bytes(self.counter));
|
|
||||||
ret
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl Entry<K2VItemPartition, K2VHistorySortKey> for K2VHistoryEntry {
|
|
||||||
fn partition_key(&self) -> &K2VItemPartition {
|
|
||||||
&self.ins_item.partition
|
|
||||||
}
|
|
||||||
fn sort_key(&self) -> &K2VHistorySortKey {
|
|
||||||
&self.node_counter
|
|
||||||
}
|
|
||||||
fn is_tombstone(&self) -> bool {
|
|
||||||
self.deleted.get()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
pub struct K2VHistoryTable {
|
|
||||||
pub(crate) subscriptions: Arc<SubscriptionManager>,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl TableSchema for K2VHistoryTable {
|
|
||||||
const TABLE_NAME: &'static str = "k2v_history";
|
|
||||||
|
|
||||||
type P = K2VItemPartition;
|
|
||||||
type S = K2VHistorySortKey;
|
|
||||||
type E = K2VHistoryEntry;
|
|
||||||
type Filter = DeletedFilter;
|
|
||||||
|
|
||||||
fn updated(
|
|
||||||
&self,
|
|
||||||
_tx: &mut db::Transaction,
|
|
||||||
_old: Option<&Self::E>,
|
|
||||||
new: Option<&Self::E>,
|
|
||||||
) -> db::TxOpResult<()> {
|
|
||||||
if let Some(new_ent) = new {
|
|
||||||
self.subscriptions.notify_range(new_ent);
|
|
||||||
}
|
|
||||||
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
|
|
||||||
fn matches_filter(entry: &Self::E, filter: &Self::Filter) -> bool {
|
|
||||||
filter.apply(entry.deleted.get())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
+27
-42
@@ -1,8 +1,7 @@
|
|||||||
|
use serde::{Deserialize, Serialize};
|
||||||
use std::collections::BTreeMap;
|
use std::collections::BTreeMap;
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
|
|
||||||
use serde::{Deserialize, Serialize};
|
|
||||||
|
|
||||||
use garage_db as db;
|
use garage_db as db;
|
||||||
use garage_util::data::*;
|
use garage_util::data::*;
|
||||||
|
|
||||||
@@ -11,49 +10,38 @@ use garage_table::*;
|
|||||||
|
|
||||||
use crate::index_counter::*;
|
use crate::index_counter::*;
|
||||||
use crate::k2v::causality::*;
|
use crate::k2v::causality::*;
|
||||||
use crate::k2v::sub::*;
|
use crate::k2v::poll::*;
|
||||||
|
|
||||||
pub const ENTRIES: &str = "entries";
|
pub const ENTRIES: &str = "entries";
|
||||||
pub const CONFLICTS: &str = "conflicts";
|
pub const CONFLICTS: &str = "conflicts";
|
||||||
pub const VALUES: &str = "values";
|
pub const VALUES: &str = "values";
|
||||||
pub const BYTES: &str = "bytes";
|
pub const BYTES: &str = "bytes";
|
||||||
|
|
||||||
pub(super) mod v08 {
|
#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
|
||||||
use crate::k2v::causality::K2VNodeId;
|
pub struct K2VItem {
|
||||||
use garage_util::data::Uuid;
|
pub partition: K2VItemPartition,
|
||||||
use serde::{Deserialize, Serialize};
|
pub sort_key: String,
|
||||||
use std::collections::BTreeMap;
|
|
||||||
|
|
||||||
#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
|
items: BTreeMap<K2VNodeId, DvvsEntry>,
|
||||||
pub struct K2VItem {
|
|
||||||
pub partition: K2VItemPartition,
|
|
||||||
pub sort_key: String,
|
|
||||||
|
|
||||||
pub(super) items: BTreeMap<K2VNodeId, DvvsEntry>,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize, Hash)]
|
|
||||||
pub struct K2VItemPartition {
|
|
||||||
pub bucket_id: Uuid,
|
|
||||||
pub partition_key: String,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
|
|
||||||
pub struct DvvsEntry {
|
|
||||||
pub(super) t_discard: u64,
|
|
||||||
pub(super) values: Vec<(u64, DvvsValue)>,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
|
|
||||||
pub enum DvvsValue {
|
|
||||||
Value(#[serde(with = "serde_bytes")] Vec<u8>),
|
|
||||||
Deleted,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl garage_util::migrate::InitialFormat for K2VItem {}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
pub use v08::*;
|
#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize, Hash)]
|
||||||
|
pub struct K2VItemPartition {
|
||||||
|
pub bucket_id: Uuid,
|
||||||
|
pub partition_key: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
|
||||||
|
struct DvvsEntry {
|
||||||
|
t_discard: u64,
|
||||||
|
values: Vec<(u64, DvvsValue)>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
|
||||||
|
pub enum DvvsValue {
|
||||||
|
Value(#[serde(with = "serde_bytes")] Vec<u8>),
|
||||||
|
Deleted,
|
||||||
|
}
|
||||||
|
|
||||||
impl K2VItem {
|
impl K2VItem {
|
||||||
/// Creates a new K2VItem when no previous entry existed in the db
|
/// Creates a new K2VItem when no previous entry existed in the db
|
||||||
@@ -73,8 +61,7 @@ impl K2VItem {
|
|||||||
this_node: Uuid,
|
this_node: Uuid,
|
||||||
context: &Option<CausalContext>,
|
context: &Option<CausalContext>,
|
||||||
new_value: DvvsValue,
|
new_value: DvvsValue,
|
||||||
node_counter: u64,
|
) {
|
||||||
) -> u64 {
|
|
||||||
if let Some(context) = context {
|
if let Some(context) = context {
|
||||||
for (node, t_discard) in context.vector_clock.iter() {
|
for (node, t_discard) in context.vector_clock.iter() {
|
||||||
if let Some(e) = self.items.get_mut(node) {
|
if let Some(e) = self.items.get_mut(node) {
|
||||||
@@ -99,9 +86,7 @@ impl K2VItem {
|
|||||||
values: vec![],
|
values: vec![],
|
||||||
});
|
});
|
||||||
let t_prev = e.max_time();
|
let t_prev = e.max_time();
|
||||||
let t_new = std::cmp::max(node_counter + 1, t_prev + 1);
|
e.values.push((t_prev + 1, new_value));
|
||||||
e.values.push((t_new, new_value));
|
|
||||||
t_new
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Extract the causality context of a K2V Item
|
/// Extract the causality context of a K2V Item
|
||||||
@@ -240,7 +225,7 @@ impl TableSchema for K2VItemTable {
|
|||||||
|
|
||||||
// 2. Notify
|
// 2. Notify
|
||||||
if let Some(new_ent) = new {
|
if let Some(new_ent) = new {
|
||||||
self.subscriptions.notify_item(new_ent);
|
self.subscriptions.notify(new_ent);
|
||||||
}
|
}
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
|
|||||||
@@ -1,8 +1,6 @@
|
|||||||
pub mod causality;
|
pub mod causality;
|
||||||
|
|
||||||
pub mod history_table;
|
|
||||||
pub mod item_table;
|
pub mod item_table;
|
||||||
|
|
||||||
pub(crate) mod sub;
|
pub mod poll;
|
||||||
|
|
||||||
pub mod rpc;
|
pub mod rpc;
|
||||||
|
|||||||
@@ -0,0 +1,50 @@
|
|||||||
|
use std::collections::HashMap;
|
||||||
|
use std::sync::Mutex;
|
||||||
|
|
||||||
|
use serde::{Deserialize, Serialize};
|
||||||
|
use tokio::sync::broadcast;
|
||||||
|
|
||||||
|
use crate::k2v::item_table::*;
|
||||||
|
|
||||||
|
#[derive(Debug, Hash, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
|
pub struct PollKey {
|
||||||
|
pub partition: K2VItemPartition,
|
||||||
|
pub sort_key: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Default)]
|
||||||
|
pub struct SubscriptionManager {
|
||||||
|
subscriptions: Mutex<HashMap<PollKey, broadcast::Sender<K2VItem>>>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl SubscriptionManager {
|
||||||
|
pub fn new() -> Self {
|
||||||
|
Self::default()
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn subscribe(&self, key: &PollKey) -> broadcast::Receiver<K2VItem> {
|
||||||
|
let mut subs = self.subscriptions.lock().unwrap();
|
||||||
|
if let Some(s) = subs.get(key) {
|
||||||
|
s.subscribe()
|
||||||
|
} else {
|
||||||
|
let (tx, rx) = broadcast::channel(8);
|
||||||
|
subs.insert(key.clone(), tx);
|
||||||
|
rx
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn notify(&self, item: &K2VItem) {
|
||||||
|
let key = PollKey {
|
||||||
|
partition: item.partition.clone(),
|
||||||
|
sort_key: item.sort_key.clone(),
|
||||||
|
};
|
||||||
|
let mut subs = self.subscriptions.lock().unwrap();
|
||||||
|
if let Some(s) = subs.get(&key) {
|
||||||
|
if s.send(item.clone()).is_err() {
|
||||||
|
// no more subscribers, remove channel from here
|
||||||
|
// (we will re-create it later if we need to subscribe again)
|
||||||
|
subs.remove(&key);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
+8
-55
@@ -6,7 +6,6 @@
|
|||||||
//! mean the vector clock gets much larger than needed).
|
//! mean the vector clock gets much larger than needed).
|
||||||
|
|
||||||
use std::collections::HashMap;
|
use std::collections::HashMap;
|
||||||
use std::convert::TryInto;
|
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
use std::time::Duration;
|
use std::time::Duration;
|
||||||
|
|
||||||
@@ -16,12 +15,9 @@ use futures::StreamExt;
|
|||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
use tokio::select;
|
use tokio::select;
|
||||||
|
|
||||||
use garage_db as db;
|
|
||||||
|
|
||||||
use garage_util::crdt::*;
|
use garage_util::crdt::*;
|
||||||
use garage_util::data::*;
|
use garage_util::data::*;
|
||||||
use garage_util::error::*;
|
use garage_util::error::*;
|
||||||
use garage_util::time::*;
|
|
||||||
|
|
||||||
use garage_rpc::system::System;
|
use garage_rpc::system::System;
|
||||||
use garage_rpc::*;
|
use garage_rpc::*;
|
||||||
@@ -30,9 +26,8 @@ use garage_table::replication::{TableReplication, TableShardedReplication};
|
|||||||
use garage_table::{PartitionKey, Table};
|
use garage_table::{PartitionKey, Table};
|
||||||
|
|
||||||
use crate::k2v::causality::*;
|
use crate::k2v::causality::*;
|
||||||
use crate::k2v::history_table::*;
|
|
||||||
use crate::k2v::item_table::*;
|
use crate::k2v::item_table::*;
|
||||||
use crate::k2v::sub::*;
|
use crate::k2v::poll::*;
|
||||||
|
|
||||||
/// RPC messages for K2V
|
/// RPC messages for K2V
|
||||||
#[derive(Debug, Serialize, Deserialize)]
|
#[derive(Debug, Serialize, Deserialize)]
|
||||||
@@ -64,8 +59,6 @@ impl Rpc for K2VRpc {
|
|||||||
pub struct K2VRpcHandler {
|
pub struct K2VRpcHandler {
|
||||||
system: Arc<System>,
|
system: Arc<System>,
|
||||||
item_table: Arc<Table<K2VItemTable, TableShardedReplication>>,
|
item_table: Arc<Table<K2VItemTable, TableShardedReplication>>,
|
||||||
history_table: Arc<Table<K2VHistoryTable, TableShardedReplication>>,
|
|
||||||
local_counter_tree: db::Tree,
|
|
||||||
endpoint: Arc<Endpoint<K2VRpc, Self>>,
|
endpoint: Arc<Endpoint<K2VRpc, Self>>,
|
||||||
subscriptions: Arc<SubscriptionManager>,
|
subscriptions: Arc<SubscriptionManager>,
|
||||||
}
|
}
|
||||||
@@ -73,21 +66,14 @@ pub struct K2VRpcHandler {
|
|||||||
impl K2VRpcHandler {
|
impl K2VRpcHandler {
|
||||||
pub fn new(
|
pub fn new(
|
||||||
system: Arc<System>,
|
system: Arc<System>,
|
||||||
db: &db::Db,
|
|
||||||
item_table: Arc<Table<K2VItemTable, TableShardedReplication>>,
|
item_table: Arc<Table<K2VItemTable, TableShardedReplication>>,
|
||||||
history_table: Arc<Table<K2VHistoryTable, TableShardedReplication>>,
|
|
||||||
subscriptions: Arc<SubscriptionManager>,
|
subscriptions: Arc<SubscriptionManager>,
|
||||||
) -> Arc<Self> {
|
) -> Arc<Self> {
|
||||||
let local_counter_tree = db
|
|
||||||
.open_tree("k2v_local_counter")
|
|
||||||
.expect("Unable to open DB tree for k2v local counter");
|
|
||||||
let endpoint = system.netapp.endpoint("garage_model/k2v/Rpc".to_string());
|
let endpoint = system.netapp.endpoint("garage_model/k2v/Rpc".to_string());
|
||||||
|
|
||||||
let rpc_handler = Arc::new(Self {
|
let rpc_handler = Arc::new(Self {
|
||||||
system,
|
system,
|
||||||
item_table,
|
item_table,
|
||||||
history_table,
|
|
||||||
local_counter_tree,
|
|
||||||
endpoint,
|
endpoint,
|
||||||
subscriptions,
|
subscriptions,
|
||||||
});
|
});
|
||||||
@@ -195,7 +181,7 @@ impl K2VRpcHandler {
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn poll_item(
|
pub async fn poll(
|
||||||
&self,
|
&self,
|
||||||
bucket_id: Uuid,
|
bucket_id: Uuid,
|
||||||
partition_key: String,
|
partition_key: String,
|
||||||
@@ -287,18 +273,9 @@ impl K2VRpcHandler {
|
|||||||
}
|
}
|
||||||
|
|
||||||
fn local_insert(&self, item: &InsertedItem) -> Result<Option<K2VItem>, Error> {
|
fn local_insert(&self, item: &InsertedItem) -> Result<Option<K2VItem>, Error> {
|
||||||
let now = now_msec();
|
|
||||||
|
|
||||||
self.item_table
|
self.item_table
|
||||||
.data
|
.data
|
||||||
.update_entry_with(&item.partition, &item.sort_key, |tx, ent| {
|
.update_entry_with(&item.partition, &item.sort_key, |ent| {
|
||||||
let local_counter_key = item.partition.hash();
|
|
||||||
let old_local_counter = tx
|
|
||||||
.get(&self.local_counter_tree, &local_counter_key)?
|
|
||||||
.and_then(|x| x.try_into().ok())
|
|
||||||
.map(u64::from_be_bytes)
|
|
||||||
.unwrap_or_default();
|
|
||||||
|
|
||||||
let mut ent = ent.unwrap_or_else(|| {
|
let mut ent = ent.unwrap_or_else(|| {
|
||||||
K2VItem::new(
|
K2VItem::new(
|
||||||
item.partition.bucket_id,
|
item.partition.bucket_id,
|
||||||
@@ -306,37 +283,13 @@ impl K2VRpcHandler {
|
|||||||
item.sort_key.clone(),
|
item.sort_key.clone(),
|
||||||
)
|
)
|
||||||
});
|
});
|
||||||
let new_local_counter = ent.update(
|
ent.update(self.system.id, &item.causal_context, item.value.clone());
|
||||||
self.system.id,
|
ent
|
||||||
&item.causal_context,
|
|
||||||
item.value.clone(),
|
|
||||||
old_local_counter,
|
|
||||||
);
|
|
||||||
|
|
||||||
tx.insert(
|
|
||||||
&self.local_counter_tree,
|
|
||||||
&local_counter_key,
|
|
||||||
u64::to_be_bytes(new_local_counter),
|
|
||||||
)?;
|
|
||||||
|
|
||||||
let hist_entry = K2VHistoryEntry {
|
|
||||||
ins_item: ent.clone(),
|
|
||||||
node_counter: K2VHistorySortKey {
|
|
||||||
node: make_node_id(self.system.id),
|
|
||||||
counter: new_local_counter,
|
|
||||||
},
|
|
||||||
prev_counter: old_local_counter,
|
|
||||||
timestamp: now,
|
|
||||||
deleted: false.into(),
|
|
||||||
};
|
|
||||||
self.history_table.queue_insert(tx, &hist_entry)?;
|
|
||||||
|
|
||||||
Ok(ent)
|
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn handle_poll_item(&self, key: &PollKey, ct: &CausalContext) -> Result<K2VItem, Error> {
|
async fn handle_poll(&self, key: &PollKey, ct: &CausalContext) -> Result<K2VItem, Error> {
|
||||||
let mut chan = self.subscriptions.subscribe_item(key);
|
let mut chan = self.subscriptions.subscribe(key);
|
||||||
|
|
||||||
let mut value = self
|
let mut value = self
|
||||||
.item_table
|
.item_table
|
||||||
@@ -373,7 +326,7 @@ impl EndpointHandler<K2VRpc> for K2VRpcHandler {
|
|||||||
} => {
|
} => {
|
||||||
let delay = tokio::time::sleep(Duration::from_millis(*timeout_msec));
|
let delay = tokio::time::sleep(Duration::from_millis(*timeout_msec));
|
||||||
select! {
|
select! {
|
||||||
ret = self.handle_poll_item(key, causal_context) => ret.map(Some).map(K2VRpc::PollItemResponse),
|
ret = self.handle_poll(key, causal_context) => ret.map(Some).map(K2VRpc::PollItemResponse),
|
||||||
_ = delay => Ok(K2VRpc::PollItemResponse(None)),
|
_ = delay => Ok(K2VRpc::PollItemResponse(None)),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,115 +0,0 @@
|
|||||||
use std::collections::HashMap;
|
|
||||||
use std::sync::Mutex;
|
|
||||||
|
|
||||||
use serde::{Deserialize, Serialize};
|
|
||||||
use tokio::sync::broadcast;
|
|
||||||
|
|
||||||
use crate::k2v::history_table::*;
|
|
||||||
use crate::k2v::item_table::*;
|
|
||||||
|
|
||||||
#[derive(Debug, Hash, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
|
||||||
pub struct PollKey {
|
|
||||||
pub partition: K2VItemPartition,
|
|
||||||
pub sort_key: String,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Debug, Hash, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
|
||||||
pub struct PollRange {
|
|
||||||
pub partition: K2VItemPartition,
|
|
||||||
pub prefix: Option<String>,
|
|
||||||
pub start: Option<String>,
|
|
||||||
pub end: Option<String>,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Default)]
|
|
||||||
pub struct SubscriptionManager {
|
|
||||||
item_subscriptions: Mutex<HashMap<PollKey, broadcast::Sender<K2VItem>>>,
|
|
||||||
range_subscriptions: Mutex<HashMap<PollRange, broadcast::Sender<K2VHistoryEntry>>>,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl SubscriptionManager {
|
|
||||||
pub fn new() -> Self {
|
|
||||||
Self::default()
|
|
||||||
}
|
|
||||||
|
|
||||||
// ---- simple item polling ----
|
|
||||||
|
|
||||||
pub fn subscribe_item(&self, key: &PollKey) -> broadcast::Receiver<K2VItem> {
|
|
||||||
let mut subs = self.item_subscriptions.lock().unwrap();
|
|
||||||
if let Some(s) = subs.get(key) {
|
|
||||||
s.subscribe()
|
|
||||||
} else {
|
|
||||||
let (tx, rx) = broadcast::channel(8);
|
|
||||||
subs.insert(key.clone(), tx);
|
|
||||||
rx
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn notify_item(&self, item: &K2VItem) {
|
|
||||||
let key = PollKey {
|
|
||||||
partition: item.partition.clone(),
|
|
||||||
sort_key: item.sort_key.clone(),
|
|
||||||
};
|
|
||||||
let mut subs = self.item_subscriptions.lock().unwrap();
|
|
||||||
if let Some(s) = subs.get(&key) {
|
|
||||||
if s.send(item.clone()).is_err() {
|
|
||||||
// no more subscribers, remove channel from here
|
|
||||||
// (we will re-create it later if we need to subscribe again)
|
|
||||||
subs.remove(&key);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// ---- range polling ----
|
|
||||||
|
|
||||||
pub fn subscribe_range(&self, key: &PollRange) -> broadcast::Receiver<K2VHistoryEntry> {
|
|
||||||
let mut subs = self.range_subscriptions.lock().unwrap();
|
|
||||||
if let Some(s) = subs.get(key) {
|
|
||||||
s.subscribe()
|
|
||||||
} else {
|
|
||||||
let (tx, rx) = broadcast::channel(8);
|
|
||||||
subs.insert(key.clone(), tx);
|
|
||||||
rx
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn notify_range(&self, entry: &K2VHistoryEntry) {
|
|
||||||
let mut subs = self.range_subscriptions.lock().unwrap();
|
|
||||||
let mut dead_subs = vec![];
|
|
||||||
|
|
||||||
for (sub, chan) in subs.iter() {
|
|
||||||
if sub.matches(&entry) {
|
|
||||||
if chan.send(entry.clone()).is_err() {
|
|
||||||
dead_subs.push(sub.clone());
|
|
||||||
}
|
|
||||||
} else if chan.receiver_count() == 0 {
|
|
||||||
dead_subs.push(sub.clone());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
for sub in dead_subs.iter() {
|
|
||||||
subs.remove(sub);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl PollRange {
|
|
||||||
fn matches(&self, entry: &K2VHistoryEntry) -> bool {
|
|
||||||
entry.ins_item.partition == self.partition
|
|
||||||
&& self
|
|
||||||
.prefix
|
|
||||||
.as_ref()
|
|
||||||
.map(|x| entry.ins_item.sort_key.starts_with(x))
|
|
||||||
.unwrap_or(true)
|
|
||||||
&& self
|
|
||||||
.start
|
|
||||||
.as_ref()
|
|
||||||
.map(|x| entry.ins_item.sort_key >= *x)
|
|
||||||
.unwrap_or(true)
|
|
||||||
&& self
|
|
||||||
.end
|
|
||||||
.as_ref()
|
|
||||||
.map(|x| entry.ins_item.sort_key < *x)
|
|
||||||
.unwrap_or(true)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
+50
-102
@@ -1,121 +1,45 @@
|
|||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
|
|
||||||
use garage_util::crdt::{self, Crdt};
|
use garage_table::crdt::*;
|
||||||
|
use garage_table::*;
|
||||||
use garage_util::data::*;
|
use garage_util::data::*;
|
||||||
|
|
||||||
use garage_table::{DeletedFilter, EmptyKey, Entry, TableSchema};
|
|
||||||
|
|
||||||
use crate::permission::BucketKeyPerm;
|
use crate::permission::BucketKeyPerm;
|
||||||
|
|
||||||
pub(crate) mod v05 {
|
use crate::prev::v051::key_table as old;
|
||||||
use garage_util::crdt;
|
|
||||||
use serde::{Deserialize, Serialize};
|
|
||||||
|
|
||||||
/// An api key
|
/// An api key
|
||||||
#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
|
#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
|
||||||
pub struct Key {
|
pub struct Key {
|
||||||
/// The id of the key (immutable), used as partition key
|
/// The id of the key (immutable), used as partition key
|
||||||
pub key_id: String,
|
pub key_id: String,
|
||||||
|
|
||||||
/// The secret_key associated
|
/// Internal state of the key
|
||||||
pub secret_key: String,
|
pub state: crdt::Deletable<KeyParams>,
|
||||||
|
|
||||||
/// Name for the key
|
|
||||||
pub name: crdt::Lww<String>,
|
|
||||||
|
|
||||||
/// Is the key deleted
|
|
||||||
pub deleted: crdt::Bool,
|
|
||||||
|
|
||||||
/// Buckets in which the key is authorized. Empty if `Key` is deleted
|
|
||||||
// CRDT interaction: deleted implies authorized_buckets is empty
|
|
||||||
pub authorized_buckets: crdt::LwwMap<String, PermissionSet>,
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Permission given to a key in a bucket
|
|
||||||
#[derive(PartialOrd, Ord, PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
|
|
||||||
pub struct PermissionSet {
|
|
||||||
/// The key can be used to read the bucket
|
|
||||||
pub allow_read: bool,
|
|
||||||
/// The key can be used to write in the bucket
|
|
||||||
pub allow_write: bool,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl crdt::AutoCrdt for PermissionSet {
|
|
||||||
const WARN_IF_DIFFERENT: bool = true;
|
|
||||||
}
|
|
||||||
|
|
||||||
impl garage_util::migrate::InitialFormat for Key {}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
mod v08 {
|
/// Configuration for a key
|
||||||
use super::v05;
|
#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
|
||||||
use crate::permission::BucketKeyPerm;
|
pub struct KeyParams {
|
||||||
use garage_util::crdt;
|
/// The secret_key associated (immutable)
|
||||||
use garage_util::data::Uuid;
|
pub secret_key: String,
|
||||||
use serde::{Deserialize, Serialize};
|
|
||||||
|
|
||||||
/// An api key
|
/// Name for the key
|
||||||
#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
|
pub name: crdt::Lww<String>,
|
||||||
pub struct Key {
|
|
||||||
/// The id of the key (immutable), used as partition key
|
|
||||||
pub key_id: String,
|
|
||||||
|
|
||||||
/// Internal state of the key
|
/// Flag to allow users having this key to create buckets
|
||||||
pub state: crdt::Deletable<KeyParams>,
|
pub allow_create_bucket: crdt::Lww<bool>,
|
||||||
}
|
|
||||||
|
|
||||||
/// Configuration for a key
|
/// If the key is present: it gives some permissions,
|
||||||
#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
|
/// a map of bucket IDs (uuids) to permissions.
|
||||||
pub struct KeyParams {
|
/// Otherwise no permissions are granted to key
|
||||||
/// The secret_key associated (immutable)
|
pub authorized_buckets: crdt::Map<Uuid, BucketKeyPerm>,
|
||||||
pub secret_key: String,
|
|
||||||
|
|
||||||
/// Name for the key
|
/// A key can have a local view of buckets names it is
|
||||||
pub name: crdt::Lww<String>,
|
/// the only one to see, this is the namespace for these aliases
|
||||||
|
pub local_aliases: crdt::LwwMap<String, Option<Uuid>>,
|
||||||
/// Flag to allow users having this key to create buckets
|
|
||||||
pub allow_create_bucket: crdt::Lww<bool>,
|
|
||||||
|
|
||||||
/// If the key is present: it gives some permissions,
|
|
||||||
/// a map of bucket IDs (uuids) to permissions.
|
|
||||||
/// Otherwise no permissions are granted to key
|
|
||||||
pub authorized_buckets: crdt::Map<Uuid, BucketKeyPerm>,
|
|
||||||
|
|
||||||
/// A key can have a local view of buckets names it is
|
|
||||||
/// the only one to see, this is the namespace for these aliases
|
|
||||||
pub local_aliases: crdt::LwwMap<String, Option<Uuid>>,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl garage_util::migrate::Migrate for Key {
|
|
||||||
type Previous = v05::Key;
|
|
||||||
|
|
||||||
fn migrate(old_k: v05::Key) -> Key {
|
|
||||||
let name = crdt::Lww::raw(old_k.name.timestamp(), old_k.name.get().clone());
|
|
||||||
|
|
||||||
let state = if old_k.deleted.get() {
|
|
||||||
crdt::Deletable::Deleted
|
|
||||||
} else {
|
|
||||||
// Authorized buckets is ignored here,
|
|
||||||
// migration is performed in specific migration code in
|
|
||||||
// garage/migrate.rs
|
|
||||||
crdt::Deletable::Present(KeyParams {
|
|
||||||
secret_key: old_k.secret_key,
|
|
||||||
name,
|
|
||||||
allow_create_bucket: crdt::Lww::new(false),
|
|
||||||
authorized_buckets: crdt::Map::new(),
|
|
||||||
local_aliases: crdt::LwwMap::new(),
|
|
||||||
})
|
|
||||||
};
|
|
||||||
Key {
|
|
||||||
key_id: old_k.key_id,
|
|
||||||
state,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
pub use v08::*;
|
|
||||||
|
|
||||||
impl KeyParams {
|
impl KeyParams {
|
||||||
fn new(secret_key: &str, name: &str) -> Self {
|
fn new(secret_key: &str, name: &str) -> Self {
|
||||||
KeyParams {
|
KeyParams {
|
||||||
@@ -249,4 +173,28 @@ impl TableSchema for KeyTable {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn try_migrate(bytes: &[u8]) -> Option<Self::E> {
|
||||||
|
let old_k = rmp_serde::decode::from_read_ref::<_, old::Key>(bytes).ok()?;
|
||||||
|
let name = crdt::Lww::raw(old_k.name.timestamp(), old_k.name.get().clone());
|
||||||
|
|
||||||
|
let state = if old_k.deleted.get() {
|
||||||
|
crdt::Deletable::Deleted
|
||||||
|
} else {
|
||||||
|
// Authorized buckets is ignored here,
|
||||||
|
// migration is performed in specific migration code in
|
||||||
|
// garage/migrate.rs
|
||||||
|
crdt::Deletable::Present(KeyParams {
|
||||||
|
secret_key: old_k.secret_key,
|
||||||
|
name,
|
||||||
|
allow_create_bucket: crdt::Lww::new(false),
|
||||||
|
authorized_buckets: crdt::Map::new(),
|
||||||
|
local_aliases: crdt::LwwMap::new(),
|
||||||
|
})
|
||||||
|
};
|
||||||
|
Some(Key {
|
||||||
|
key_id: old_k.key_id,
|
||||||
|
state,
|
||||||
|
})
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -2,7 +2,6 @@ use std::sync::Arc;
|
|||||||
|
|
||||||
use garage_util::crdt::*;
|
use garage_util::crdt::*;
|
||||||
use garage_util::data::*;
|
use garage_util::data::*;
|
||||||
use garage_util::encode::nonversioned_decode;
|
|
||||||
use garage_util::error::Error as GarageError;
|
use garage_util::error::Error as GarageError;
|
||||||
use garage_util::time::*;
|
use garage_util::time::*;
|
||||||
|
|
||||||
@@ -29,8 +28,8 @@ impl Migrate {
|
|||||||
let mut old_buckets = vec![];
|
let mut old_buckets = vec![];
|
||||||
for res in tree.iter().map_err(GarageError::from)? {
|
for res in tree.iter().map_err(GarageError::from)? {
|
||||||
let (_k, v) = res.map_err(GarageError::from)?;
|
let (_k, v) = res.map_err(GarageError::from)?;
|
||||||
let bucket =
|
let bucket = rmp_serde::decode::from_read_ref::<_, old_bucket::Bucket>(&v[..])
|
||||||
nonversioned_decode::<old_bucket::Bucket>(&v[..]).map_err(GarageError::from)?;
|
.map_err(GarageError::from)?;
|
||||||
old_buckets.push(bucket);
|
old_buckets.push(bucket);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -3,7 +3,7 @@ use serde::{Deserialize, Serialize};
|
|||||||
use garage_table::crdt::Crdt;
|
use garage_table::crdt::Crdt;
|
||||||
use garage_table::*;
|
use garage_table::*;
|
||||||
|
|
||||||
use crate::key_table::v05::PermissionSet;
|
use super::key_table::PermissionSet;
|
||||||
|
|
||||||
/// A bucket is a collection of objects
|
/// A bucket is a collection of objects
|
||||||
///
|
///
|
||||||
|
|||||||
@@ -0,0 +1,50 @@
|
|||||||
|
use serde::{Deserialize, Serialize};
|
||||||
|
|
||||||
|
use garage_table::crdt::*;
|
||||||
|
use garage_table::*;
|
||||||
|
|
||||||
|
/// An api key
|
||||||
|
#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
|
||||||
|
pub struct Key {
|
||||||
|
/// The id of the key (immutable), used as partition key
|
||||||
|
pub key_id: String,
|
||||||
|
|
||||||
|
/// The secret_key associated
|
||||||
|
pub secret_key: String,
|
||||||
|
|
||||||
|
/// Name for the key
|
||||||
|
pub name: crdt::Lww<String>,
|
||||||
|
|
||||||
|
/// Is the key deleted
|
||||||
|
pub deleted: crdt::Bool,
|
||||||
|
|
||||||
|
/// Buckets in which the key is authorized. Empty if `Key` is deleted
|
||||||
|
// CRDT interaction: deleted implies authorized_buckets is empty
|
||||||
|
pub authorized_buckets: crdt::LwwMap<String, PermissionSet>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Permission given to a key in a bucket
|
||||||
|
#[derive(PartialOrd, Ord, PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
|
||||||
|
pub struct PermissionSet {
|
||||||
|
/// The key can be used to read the bucket
|
||||||
|
pub allow_read: bool,
|
||||||
|
/// The key can be used to write in the bucket
|
||||||
|
pub allow_write: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl AutoCrdt for PermissionSet {
|
||||||
|
const WARN_IF_DIFFERENT: bool = true;
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Crdt for Key {
|
||||||
|
fn merge(&mut self, other: &Self) {
|
||||||
|
self.name.merge(&other.name);
|
||||||
|
self.deleted.merge(&other.deleted);
|
||||||
|
|
||||||
|
if self.deleted.get() {
|
||||||
|
self.authorized_buckets.clear();
|
||||||
|
} else {
|
||||||
|
self.authorized_buckets.merge(&other.authorized_buckets);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1 +1,4 @@
|
|||||||
pub(crate) mod bucket_table;
|
pub(crate) mod bucket_table;
|
||||||
|
pub(crate) mod key_table;
|
||||||
|
pub(crate) mod object_table;
|
||||||
|
pub(crate) mod version_table;
|
||||||
|
|||||||
@@ -0,0 +1,149 @@
|
|||||||
|
use serde::{Deserialize, Serialize};
|
||||||
|
use std::collections::BTreeMap;
|
||||||
|
|
||||||
|
use garage_util::data::*;
|
||||||
|
|
||||||
|
use garage_table::crdt::*;
|
||||||
|
|
||||||
|
/// An object
|
||||||
|
#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
|
||||||
|
pub struct Object {
|
||||||
|
/// The bucket in which the object is stored, used as partition key
|
||||||
|
pub bucket: String,
|
||||||
|
|
||||||
|
/// The key at which the object is stored in its bucket, used as sorting key
|
||||||
|
pub key: String,
|
||||||
|
|
||||||
|
/// The list of currenty stored versions of the object
|
||||||
|
versions: Vec<ObjectVersion>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Object {
|
||||||
|
/// Get a list of currently stored versions of `Object`
|
||||||
|
pub fn versions(&self) -> &[ObjectVersion] {
|
||||||
|
&self.versions[..]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Informations about a version of an object
|
||||||
|
#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
|
||||||
|
pub struct ObjectVersion {
|
||||||
|
/// Id of the version
|
||||||
|
pub uuid: Uuid,
|
||||||
|
/// Timestamp of when the object was created
|
||||||
|
pub timestamp: u64,
|
||||||
|
/// State of the version
|
||||||
|
pub state: ObjectVersionState,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// State of an object version
|
||||||
|
#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
|
||||||
|
pub enum ObjectVersionState {
|
||||||
|
/// The version is being received
|
||||||
|
Uploading(ObjectVersionHeaders),
|
||||||
|
/// The version is fully received
|
||||||
|
Complete(ObjectVersionData),
|
||||||
|
/// The version uploaded containded errors or the upload was explicitly aborted
|
||||||
|
Aborted,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Crdt for ObjectVersionState {
|
||||||
|
fn merge(&mut self, other: &Self) {
|
||||||
|
use ObjectVersionState::*;
|
||||||
|
match other {
|
||||||
|
Aborted => {
|
||||||
|
*self = Aborted;
|
||||||
|
}
|
||||||
|
Complete(b) => match self {
|
||||||
|
Aborted => {}
|
||||||
|
Complete(a) => {
|
||||||
|
a.merge(b);
|
||||||
|
}
|
||||||
|
Uploading(_) => {
|
||||||
|
*self = Complete(b.clone());
|
||||||
|
}
|
||||||
|
},
|
||||||
|
Uploading(_) => {}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Data stored in object version
|
||||||
|
#[derive(PartialEq, Eq, PartialOrd, Ord, Clone, Debug, Serialize, Deserialize)]
|
||||||
|
pub enum ObjectVersionData {
|
||||||
|
/// The object was deleted, this Version is a tombstone to mark it as such
|
||||||
|
DeleteMarker,
|
||||||
|
/// The object is short, it's stored inlined
|
||||||
|
Inline(ObjectVersionMeta, #[serde(with = "serde_bytes")] Vec<u8>),
|
||||||
|
/// The object is not short, Hash of first block is stored here, next segments hashes are
|
||||||
|
/// stored in the version table
|
||||||
|
FirstBlock(ObjectVersionMeta, Hash),
|
||||||
|
}
|
||||||
|
|
||||||
|
impl AutoCrdt for ObjectVersionData {
|
||||||
|
const WARN_IF_DIFFERENT: bool = true;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Metadata about the object version
|
||||||
|
#[derive(PartialEq, Eq, PartialOrd, Ord, Clone, Debug, Serialize, Deserialize)]
|
||||||
|
pub struct ObjectVersionMeta {
|
||||||
|
/// Headers to send to the client
|
||||||
|
pub headers: ObjectVersionHeaders,
|
||||||
|
/// Size of the object
|
||||||
|
pub size: u64,
|
||||||
|
/// etag of the object
|
||||||
|
pub etag: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Additional headers for an object
|
||||||
|
#[derive(PartialEq, Eq, PartialOrd, Ord, Clone, Debug, Serialize, Deserialize)]
|
||||||
|
pub struct ObjectVersionHeaders {
|
||||||
|
/// Content type of the object
|
||||||
|
pub content_type: String,
|
||||||
|
/// Any other http headers to send
|
||||||
|
pub other: BTreeMap<String, String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl ObjectVersion {
|
||||||
|
fn cmp_key(&self) -> (u64, Uuid) {
|
||||||
|
(self.timestamp, self.uuid)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Is the object version completely received
|
||||||
|
pub fn is_complete(&self) -> bool {
|
||||||
|
matches!(self.state, ObjectVersionState::Complete(_))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Crdt for Object {
|
||||||
|
fn merge(&mut self, other: &Self) {
|
||||||
|
// Merge versions from other into here
|
||||||
|
for other_v in other.versions.iter() {
|
||||||
|
match self
|
||||||
|
.versions
|
||||||
|
.binary_search_by(|v| v.cmp_key().cmp(&other_v.cmp_key()))
|
||||||
|
{
|
||||||
|
Ok(i) => {
|
||||||
|
self.versions[i].state.merge(&other_v.state);
|
||||||
|
}
|
||||||
|
Err(i) => {
|
||||||
|
self.versions.insert(i, other_v.clone());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Remove versions which are obsolete, i.e. those that come
|
||||||
|
// before the last version which .is_complete().
|
||||||
|
let last_complete = self
|
||||||
|
.versions
|
||||||
|
.iter()
|
||||||
|
.enumerate()
|
||||||
|
.rev()
|
||||||
|
.find(|(_, v)| v.is_complete())
|
||||||
|
.map(|(vi, _)| vi);
|
||||||
|
|
||||||
|
if let Some(last_vi) = last_complete {
|
||||||
|
self.versions = self.versions.drain(last_vi..).collect::<Vec<_>>();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,79 @@
|
|||||||
|
use serde::{Deserialize, Serialize};
|
||||||
|
|
||||||
|
use garage_util::data::*;
|
||||||
|
|
||||||
|
use garage_table::crdt::*;
|
||||||
|
use garage_table::*;
|
||||||
|
|
||||||
|
/// A version of an object
|
||||||
|
#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
|
||||||
|
pub struct Version {
|
||||||
|
/// UUID of the version, used as partition key
|
||||||
|
pub uuid: Uuid,
|
||||||
|
|
||||||
|
// Actual data: the blocks for this version
|
||||||
|
// In the case of a multipart upload, also store the etags
|
||||||
|
// of individual parts and check them when doing CompleteMultipartUpload
|
||||||
|
/// Is this version deleted
|
||||||
|
pub deleted: crdt::Bool,
|
||||||
|
/// list of blocks of data composing the version
|
||||||
|
pub blocks: crdt::Map<VersionBlockKey, VersionBlock>,
|
||||||
|
/// Etag of each part in case of a multipart upload, empty otherwise
|
||||||
|
pub parts_etags: crdt::Map<u64, String>,
|
||||||
|
|
||||||
|
// Back link to bucket+key so that we can figure if
|
||||||
|
// this was deleted later on
|
||||||
|
/// Bucket in which the related object is stored
|
||||||
|
pub bucket: String,
|
||||||
|
/// Key in which the related object is stored
|
||||||
|
pub key: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(PartialEq, Eq, Clone, Copy, Debug, Serialize, Deserialize)]
|
||||||
|
pub struct VersionBlockKey {
|
||||||
|
/// Number of the part
|
||||||
|
pub part_number: u64,
|
||||||
|
/// Offset of this sub-segment in its part
|
||||||
|
pub offset: u64,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Ord for VersionBlockKey {
|
||||||
|
fn cmp(&self, other: &Self) -> std::cmp::Ordering {
|
||||||
|
self.part_number
|
||||||
|
.cmp(&other.part_number)
|
||||||
|
.then(self.offset.cmp(&other.offset))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl PartialOrd for VersionBlockKey {
|
||||||
|
fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
|
||||||
|
Some(self.cmp(other))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Informations about a single block
|
||||||
|
#[derive(PartialEq, Eq, Ord, PartialOrd, Clone, Copy, Debug, Serialize, Deserialize)]
|
||||||
|
pub struct VersionBlock {
|
||||||
|
/// Blake2 sum of the block
|
||||||
|
pub hash: Hash,
|
||||||
|
/// Size of the block
|
||||||
|
pub size: u64,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl AutoCrdt for VersionBlock {
|
||||||
|
const WARN_IF_DIFFERENT: bool = true;
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Crdt for Version {
|
||||||
|
fn merge(&mut self, other: &Self) {
|
||||||
|
self.deleted.merge(&other.deleted);
|
||||||
|
|
||||||
|
if self.deleted.get() {
|
||||||
|
self.blocks.clear();
|
||||||
|
self.parts_etags.clear();
|
||||||
|
} else {
|
||||||
|
self.blocks.merge(&other.blocks);
|
||||||
|
self.parts_etags.merge(&other.parts_etags);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,3 +1,4 @@
|
|||||||
|
use serde::{Deserialize, Serialize};
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
|
|
||||||
use garage_db as db;
|
use garage_db as db;
|
||||||
@@ -9,29 +10,19 @@ use garage_table::*;
|
|||||||
|
|
||||||
use garage_block::manager::*;
|
use garage_block::manager::*;
|
||||||
|
|
||||||
mod v08 {
|
#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
|
||||||
use garage_util::crdt;
|
pub struct BlockRef {
|
||||||
use garage_util::data::{Hash, Uuid};
|
/// Hash (blake2 sum) of the block, used as partition key
|
||||||
use serde::{Deserialize, Serialize};
|
pub block: Hash,
|
||||||
|
|
||||||
#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
|
/// Id of the Version for the object containing this block, used as sorting key
|
||||||
pub struct BlockRef {
|
pub version: Uuid,
|
||||||
/// Hash (blake2 sum) of the block, used as partition key
|
|
||||||
pub block: Hash,
|
|
||||||
|
|
||||||
/// Id of the Version for the object containing this block, used as sorting key
|
// Keep track of deleted status
|
||||||
pub version: Uuid,
|
/// Is the Version that contains this block deleted
|
||||||
|
pub deleted: crdt::Bool,
|
||||||
// Keep track of deleted status
|
|
||||||
/// Is the Version that contains this block deleted
|
|
||||||
pub deleted: crdt::Bool,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl garage_util::migrate::InitialFormat for BlockRef {}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
pub use v08::*;
|
|
||||||
|
|
||||||
impl Entry<Hash, Uuid> for BlockRef {
|
impl Entry<Hash, Uuid> for BlockRef {
|
||||||
fn partition_key(&self) -> &Hash {
|
fn partition_key(&self) -> &Hash {
|
||||||
&self.block
|
&self.block
|
||||||
|
|||||||
+132
-112
@@ -1,4 +1,5 @@
|
|||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
|
use std::collections::BTreeMap;
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
|
|
||||||
use garage_db as db;
|
use garage_db as db;
|
||||||
@@ -12,126 +13,25 @@ use garage_table::*;
|
|||||||
use crate::index_counter::*;
|
use crate::index_counter::*;
|
||||||
use crate::s3::version_table::*;
|
use crate::s3::version_table::*;
|
||||||
|
|
||||||
|
use crate::prev::v051::object_table as old;
|
||||||
|
|
||||||
pub const OBJECTS: &str = "objects";
|
pub const OBJECTS: &str = "objects";
|
||||||
pub const UNFINISHED_UPLOADS: &str = "unfinished_uploads";
|
pub const UNFINISHED_UPLOADS: &str = "unfinished_uploads";
|
||||||
pub const BYTES: &str = "bytes";
|
pub const BYTES: &str = "bytes";
|
||||||
|
|
||||||
mod v05 {
|
/// An object
|
||||||
use garage_util::data::{Hash, Uuid};
|
#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
|
||||||
use serde::{Deserialize, Serialize};
|
pub struct Object {
|
||||||
use std::collections::BTreeMap;
|
/// The bucket in which the object is stored, used as partition key
|
||||||
|
pub bucket_id: Uuid,
|
||||||
|
|
||||||
/// An object
|
/// The key at which the object is stored in its bucket, used as sorting key
|
||||||
#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
|
pub key: String,
|
||||||
pub struct Object {
|
|
||||||
/// The bucket in which the object is stored, used as partition key
|
|
||||||
pub bucket: String,
|
|
||||||
|
|
||||||
/// The key at which the object is stored in its bucket, used as sorting key
|
/// The list of currenty stored versions of the object
|
||||||
pub key: String,
|
versions: Vec<ObjectVersion>,
|
||||||
|
|
||||||
/// The list of currenty stored versions of the object
|
|
||||||
pub(super) versions: Vec<ObjectVersion>,
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Informations about a version of an object
|
|
||||||
#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
|
|
||||||
pub struct ObjectVersion {
|
|
||||||
/// Id of the version
|
|
||||||
pub uuid: Uuid,
|
|
||||||
/// Timestamp of when the object was created
|
|
||||||
pub timestamp: u64,
|
|
||||||
/// State of the version
|
|
||||||
pub state: ObjectVersionState,
|
|
||||||
}
|
|
||||||
|
|
||||||
/// State of an object version
|
|
||||||
#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
|
|
||||||
pub enum ObjectVersionState {
|
|
||||||
/// The version is being received
|
|
||||||
Uploading(ObjectVersionHeaders),
|
|
||||||
/// The version is fully received
|
|
||||||
Complete(ObjectVersionData),
|
|
||||||
/// The version uploaded containded errors or the upload was explicitly aborted
|
|
||||||
Aborted,
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Data stored in object version
|
|
||||||
#[derive(PartialEq, Eq, PartialOrd, Ord, Clone, Debug, Serialize, Deserialize)]
|
|
||||||
pub enum ObjectVersionData {
|
|
||||||
/// The object was deleted, this Version is a tombstone to mark it as such
|
|
||||||
DeleteMarker,
|
|
||||||
/// The object is short, it's stored inlined
|
|
||||||
Inline(ObjectVersionMeta, #[serde(with = "serde_bytes")] Vec<u8>),
|
|
||||||
/// The object is not short, Hash of first block is stored here, next segments hashes are
|
|
||||||
/// stored in the version table
|
|
||||||
FirstBlock(ObjectVersionMeta, Hash),
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Metadata about the object version
|
|
||||||
#[derive(PartialEq, Eq, PartialOrd, Ord, Clone, Debug, Serialize, Deserialize)]
|
|
||||||
pub struct ObjectVersionMeta {
|
|
||||||
/// Headers to send to the client
|
|
||||||
pub headers: ObjectVersionHeaders,
|
|
||||||
/// Size of the object
|
|
||||||
pub size: u64,
|
|
||||||
/// etag of the object
|
|
||||||
pub etag: String,
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Additional headers for an object
|
|
||||||
#[derive(PartialEq, Eq, PartialOrd, Ord, Clone, Debug, Serialize, Deserialize)]
|
|
||||||
pub struct ObjectVersionHeaders {
|
|
||||||
/// Content type of the object
|
|
||||||
pub content_type: String,
|
|
||||||
/// Any other http headers to send
|
|
||||||
pub other: BTreeMap<String, String>,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl garage_util::migrate::InitialFormat for Object {}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
mod v08 {
|
|
||||||
use garage_util::data::Uuid;
|
|
||||||
use serde::{Deserialize, Serialize};
|
|
||||||
|
|
||||||
use super::v05;
|
|
||||||
|
|
||||||
pub use v05::{
|
|
||||||
ObjectVersion, ObjectVersionData, ObjectVersionHeaders, ObjectVersionMeta,
|
|
||||||
ObjectVersionState,
|
|
||||||
};
|
|
||||||
|
|
||||||
/// An object
|
|
||||||
#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
|
|
||||||
pub struct Object {
|
|
||||||
/// The bucket in which the object is stored, used as partition key
|
|
||||||
pub bucket_id: Uuid,
|
|
||||||
|
|
||||||
/// The key at which the object is stored in its bucket, used as sorting key
|
|
||||||
pub key: String,
|
|
||||||
|
|
||||||
/// The list of currenty stored versions of the object
|
|
||||||
pub(super) versions: Vec<ObjectVersion>,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl garage_util::migrate::Migrate for Object {
|
|
||||||
type Previous = v05::Object;
|
|
||||||
|
|
||||||
fn migrate(old: v05::Object) -> Object {
|
|
||||||
use garage_util::data::blake2sum;
|
|
||||||
|
|
||||||
Object {
|
|
||||||
bucket_id: blake2sum(old.bucket.as_bytes()),
|
|
||||||
key: old.key,
|
|
||||||
versions: old.versions,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
pub use v08::*;
|
|
||||||
|
|
||||||
impl Object {
|
impl Object {
|
||||||
/// Initialize an Object struct from parts
|
/// Initialize an Object struct from parts
|
||||||
pub fn new(bucket_id: Uuid, key: String, versions: Vec<ObjectVersion>) -> Self {
|
pub fn new(bucket_id: Uuid, key: String, versions: Vec<ObjectVersion>) -> Self {
|
||||||
@@ -168,6 +68,28 @@ impl Object {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Informations about a version of an object
|
||||||
|
#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
|
||||||
|
pub struct ObjectVersion {
|
||||||
|
/// Id of the version
|
||||||
|
pub uuid: Uuid,
|
||||||
|
/// Timestamp of when the object was created
|
||||||
|
pub timestamp: u64,
|
||||||
|
/// State of the version
|
||||||
|
pub state: ObjectVersionState,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// State of an object version
|
||||||
|
#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
|
||||||
|
pub enum ObjectVersionState {
|
||||||
|
/// The version is being received
|
||||||
|
Uploading(ObjectVersionHeaders),
|
||||||
|
/// The version is fully received
|
||||||
|
Complete(ObjectVersionData),
|
||||||
|
/// The version uploaded containded errors or the upload was explicitly aborted
|
||||||
|
Aborted,
|
||||||
|
}
|
||||||
|
|
||||||
impl Crdt for ObjectVersionState {
|
impl Crdt for ObjectVersionState {
|
||||||
fn merge(&mut self, other: &Self) {
|
fn merge(&mut self, other: &Self) {
|
||||||
use ObjectVersionState::*;
|
use ObjectVersionState::*;
|
||||||
@@ -189,10 +111,42 @@ impl Crdt for ObjectVersionState {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Data stored in object version
|
||||||
|
#[derive(PartialEq, Eq, PartialOrd, Ord, Clone, Debug, Serialize, Deserialize)]
|
||||||
|
pub enum ObjectVersionData {
|
||||||
|
/// The object was deleted, this Version is a tombstone to mark it as such
|
||||||
|
DeleteMarker,
|
||||||
|
/// The object is short, it's stored inlined
|
||||||
|
Inline(ObjectVersionMeta, #[serde(with = "serde_bytes")] Vec<u8>),
|
||||||
|
/// The object is not short, Hash of first block is stored here, next segments hashes are
|
||||||
|
/// stored in the version table
|
||||||
|
FirstBlock(ObjectVersionMeta, Hash),
|
||||||
|
}
|
||||||
|
|
||||||
impl AutoCrdt for ObjectVersionData {
|
impl AutoCrdt for ObjectVersionData {
|
||||||
const WARN_IF_DIFFERENT: bool = true;
|
const WARN_IF_DIFFERENT: bool = true;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Metadata about the object version
|
||||||
|
#[derive(PartialEq, Eq, PartialOrd, Ord, Clone, Debug, Serialize, Deserialize)]
|
||||||
|
pub struct ObjectVersionMeta {
|
||||||
|
/// Headers to send to the client
|
||||||
|
pub headers: ObjectVersionHeaders,
|
||||||
|
/// Size of the object
|
||||||
|
pub size: u64,
|
||||||
|
/// etag of the object
|
||||||
|
pub etag: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Additional headers for an object
|
||||||
|
#[derive(PartialEq, Eq, PartialOrd, Ord, Clone, Debug, Serialize, Deserialize)]
|
||||||
|
pub struct ObjectVersionHeaders {
|
||||||
|
/// Content type of the object
|
||||||
|
pub content_type: String,
|
||||||
|
/// Any other http headers to send
|
||||||
|
pub other: BTreeMap<String, String>,
|
||||||
|
}
|
||||||
|
|
||||||
impl ObjectVersion {
|
impl ObjectVersion {
|
||||||
fn cmp_key(&self) -> (u64, Uuid) {
|
fn cmp_key(&self) -> (u64, Uuid) {
|
||||||
(self.timestamp, self.uuid)
|
(self.timestamp, self.uuid)
|
||||||
@@ -336,6 +290,11 @@ impl TableSchema for ObjectTable {
|
|||||||
ObjectFilter::IsUploading => entry.versions.iter().any(|v| v.is_uploading()),
|
ObjectFilter::IsUploading => entry.versions.iter().any(|v| v.is_uploading()),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn try_migrate(bytes: &[u8]) -> Option<Self::E> {
|
||||||
|
let old_obj = rmp_serde::decode::from_read_ref::<_, old::Object>(bytes).ok()?;
|
||||||
|
Some(migrate_object(old_obj))
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
impl CountedItem for Object {
|
impl CountedItem for Object {
|
||||||
@@ -380,3 +339,64 @@ impl CountedItem for Object {
|
|||||||
]
|
]
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// vvvvvvvv migration code, stupid stuff vvvvvvvvvvvv
|
||||||
|
// (we just want to change bucket into bucket_id by hashing it)
|
||||||
|
|
||||||
|
fn migrate_object(o: old::Object) -> Object {
|
||||||
|
let versions = o
|
||||||
|
.versions()
|
||||||
|
.iter()
|
||||||
|
.cloned()
|
||||||
|
.map(migrate_object_version)
|
||||||
|
.collect();
|
||||||
|
Object {
|
||||||
|
bucket_id: blake2sum(o.bucket.as_bytes()),
|
||||||
|
key: o.key,
|
||||||
|
versions,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn migrate_object_version(v: old::ObjectVersion) -> ObjectVersion {
|
||||||
|
ObjectVersion {
|
||||||
|
uuid: Uuid::try_from(v.uuid.as_slice()).unwrap(),
|
||||||
|
timestamp: v.timestamp,
|
||||||
|
state: match v.state {
|
||||||
|
old::ObjectVersionState::Uploading(h) => {
|
||||||
|
ObjectVersionState::Uploading(migrate_object_version_headers(h))
|
||||||
|
}
|
||||||
|
old::ObjectVersionState::Complete(d) => {
|
||||||
|
ObjectVersionState::Complete(migrate_object_version_data(d))
|
||||||
|
}
|
||||||
|
old::ObjectVersionState::Aborted => ObjectVersionState::Aborted,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn migrate_object_version_headers(h: old::ObjectVersionHeaders) -> ObjectVersionHeaders {
|
||||||
|
ObjectVersionHeaders {
|
||||||
|
content_type: h.content_type,
|
||||||
|
other: h.other,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn migrate_object_version_data(d: old::ObjectVersionData) -> ObjectVersionData {
|
||||||
|
match d {
|
||||||
|
old::ObjectVersionData::DeleteMarker => ObjectVersionData::DeleteMarker,
|
||||||
|
old::ObjectVersionData::Inline(m, b) => {
|
||||||
|
ObjectVersionData::Inline(migrate_object_version_meta(m), b)
|
||||||
|
}
|
||||||
|
old::ObjectVersionData::FirstBlock(m, h) => ObjectVersionData::FirstBlock(
|
||||||
|
migrate_object_version_meta(m),
|
||||||
|
Hash::try_from(h.as_slice()).unwrap(),
|
||||||
|
),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn migrate_object_version_meta(m: old::ObjectVersionMeta) -> ObjectVersionMeta {
|
||||||
|
ObjectVersionMeta {
|
||||||
|
headers: migrate_object_version_headers(m.headers),
|
||||||
|
size: m.size,
|
||||||
|
etag: m.etag,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -1,3 +1,4 @@
|
|||||||
|
use serde::{Deserialize, Serialize};
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
|
|
||||||
use garage_db as db;
|
use garage_db as db;
|
||||||
@@ -10,108 +11,32 @@ use garage_table::*;
|
|||||||
|
|
||||||
use crate::s3::block_ref_table::*;
|
use crate::s3::block_ref_table::*;
|
||||||
|
|
||||||
mod v05 {
|
use crate::prev::v051::version_table as old;
|
||||||
use garage_util::crdt;
|
|
||||||
use garage_util::data::{Hash, Uuid};
|
|
||||||
use serde::{Deserialize, Serialize};
|
|
||||||
|
|
||||||
/// A version of an object
|
/// A version of an object
|
||||||
#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
|
#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
|
||||||
pub struct Version {
|
pub struct Version {
|
||||||
/// UUID of the version, used as partition key
|
/// UUID of the version, used as partition key
|
||||||
pub uuid: Uuid,
|
pub uuid: Uuid,
|
||||||
|
|
||||||
// Actual data: the blocks for this version
|
// Actual data: the blocks for this version
|
||||||
// In the case of a multipart upload, also store the etags
|
// In the case of a multipart upload, also store the etags
|
||||||
// of individual parts and check them when doing CompleteMultipartUpload
|
// of individual parts and check them when doing CompleteMultipartUpload
|
||||||
/// Is this version deleted
|
/// Is this version deleted
|
||||||
pub deleted: crdt::Bool,
|
pub deleted: crdt::Bool,
|
||||||
/// list of blocks of data composing the version
|
/// list of blocks of data composing the version
|
||||||
pub blocks: crdt::Map<VersionBlockKey, VersionBlock>,
|
pub blocks: crdt::Map<VersionBlockKey, VersionBlock>,
|
||||||
/// Etag of each part in case of a multipart upload, empty otherwise
|
/// Etag of each part in case of a multipart upload, empty otherwise
|
||||||
pub parts_etags: crdt::Map<u64, String>,
|
pub parts_etags: crdt::Map<u64, String>,
|
||||||
|
|
||||||
// Back link to bucket+key so that we can figure if
|
// Back link to bucket+key so that we can figure if
|
||||||
// this was deleted later on
|
// this was deleted later on
|
||||||
/// Bucket in which the related object is stored
|
/// Bucket in which the related object is stored
|
||||||
pub bucket: String,
|
pub bucket_id: Uuid,
|
||||||
/// Key in which the related object is stored
|
/// Key in which the related object is stored
|
||||||
pub key: String,
|
pub key: String,
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(PartialEq, Eq, Clone, Copy, Debug, Serialize, Deserialize)]
|
|
||||||
pub struct VersionBlockKey {
|
|
||||||
/// Number of the part
|
|
||||||
pub part_number: u64,
|
|
||||||
/// Offset of this sub-segment in its part
|
|
||||||
pub offset: u64,
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Informations about a single block
|
|
||||||
#[derive(PartialEq, Eq, Ord, PartialOrd, Clone, Copy, Debug, Serialize, Deserialize)]
|
|
||||||
pub struct VersionBlock {
|
|
||||||
/// Blake2 sum of the block
|
|
||||||
pub hash: Hash,
|
|
||||||
/// Size of the block
|
|
||||||
pub size: u64,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl garage_util::migrate::InitialFormat for Version {}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
mod v08 {
|
|
||||||
use garage_util::crdt;
|
|
||||||
use garage_util::data::Uuid;
|
|
||||||
use serde::{Deserialize, Serialize};
|
|
||||||
|
|
||||||
use super::v05;
|
|
||||||
|
|
||||||
/// A version of an object
|
|
||||||
#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
|
|
||||||
pub struct Version {
|
|
||||||
/// UUID of the version, used as partition key
|
|
||||||
pub uuid: Uuid,
|
|
||||||
|
|
||||||
// Actual data: the blocks for this version
|
|
||||||
// In the case of a multipart upload, also store the etags
|
|
||||||
// of individual parts and check them when doing CompleteMultipartUpload
|
|
||||||
/// Is this version deleted
|
|
||||||
pub deleted: crdt::Bool,
|
|
||||||
/// list of blocks of data composing the version
|
|
||||||
pub blocks: crdt::Map<VersionBlockKey, VersionBlock>,
|
|
||||||
/// Etag of each part in case of a multipart upload, empty otherwise
|
|
||||||
pub parts_etags: crdt::Map<u64, String>,
|
|
||||||
|
|
||||||
// Back link to bucket+key so that we can figure if
|
|
||||||
// this was deleted later on
|
|
||||||
/// Bucket in which the related object is stored
|
|
||||||
pub bucket_id: Uuid,
|
|
||||||
/// Key in which the related object is stored
|
|
||||||
pub key: String,
|
|
||||||
}
|
|
||||||
|
|
||||||
pub use v05::{VersionBlock, VersionBlockKey};
|
|
||||||
|
|
||||||
impl garage_util::migrate::Migrate for Version {
|
|
||||||
type Previous = v05::Version;
|
|
||||||
|
|
||||||
fn migrate(old: v05::Version) -> Version {
|
|
||||||
use garage_util::data::blake2sum;
|
|
||||||
|
|
||||||
Version {
|
|
||||||
uuid: old.uuid,
|
|
||||||
deleted: old.deleted,
|
|
||||||
blocks: old.blocks,
|
|
||||||
parts_etags: old.parts_etags,
|
|
||||||
bucket_id: blake2sum(old.bucket.as_bytes()),
|
|
||||||
key: old.key,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
pub use v08::*;
|
|
||||||
|
|
||||||
impl Version {
|
impl Version {
|
||||||
pub fn new(uuid: Uuid, bucket_id: Uuid, key: String, deleted: bool) -> Self {
|
pub fn new(uuid: Uuid, bucket_id: Uuid, key: String, deleted: bool) -> Self {
|
||||||
Self {
|
Self {
|
||||||
@@ -139,6 +64,14 @@ impl Version {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(PartialEq, Eq, Clone, Copy, Debug, Serialize, Deserialize)]
|
||||||
|
pub struct VersionBlockKey {
|
||||||
|
/// Number of the part
|
||||||
|
pub part_number: u64,
|
||||||
|
/// Offset of this sub-segment in its part
|
||||||
|
pub offset: u64,
|
||||||
|
}
|
||||||
|
|
||||||
impl Ord for VersionBlockKey {
|
impl Ord for VersionBlockKey {
|
||||||
fn cmp(&self, other: &Self) -> std::cmp::Ordering {
|
fn cmp(&self, other: &Self) -> std::cmp::Ordering {
|
||||||
self.part_number
|
self.part_number
|
||||||
@@ -153,6 +86,15 @@ impl PartialOrd for VersionBlockKey {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Informations about a single block
|
||||||
|
#[derive(PartialEq, Eq, Ord, PartialOrd, Clone, Copy, Debug, Serialize, Deserialize)]
|
||||||
|
pub struct VersionBlock {
|
||||||
|
/// Blake2 sum of the block
|
||||||
|
pub hash: Hash,
|
||||||
|
/// Size of the block
|
||||||
|
pub size: u64,
|
||||||
|
}
|
||||||
|
|
||||||
impl AutoCrdt for VersionBlock {
|
impl AutoCrdt for VersionBlock {
|
||||||
const WARN_IF_DIFFERENT: bool = true;
|
const WARN_IF_DIFFERENT: bool = true;
|
||||||
}
|
}
|
||||||
@@ -224,4 +166,42 @@ impl TableSchema for VersionTable {
|
|||||||
fn matches_filter(entry: &Self::E, filter: &Self::Filter) -> bool {
|
fn matches_filter(entry: &Self::E, filter: &Self::Filter) -> bool {
|
||||||
filter.apply(entry.deleted.get())
|
filter.apply(entry.deleted.get())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn try_migrate(bytes: &[u8]) -> Option<Self::E> {
|
||||||
|
let old = rmp_serde::decode::from_read_ref::<_, old::Version>(bytes).ok()?;
|
||||||
|
|
||||||
|
let blocks = old
|
||||||
|
.blocks
|
||||||
|
.items()
|
||||||
|
.iter()
|
||||||
|
.map(|(k, v)| {
|
||||||
|
(
|
||||||
|
VersionBlockKey {
|
||||||
|
part_number: k.part_number,
|
||||||
|
offset: k.offset,
|
||||||
|
},
|
||||||
|
VersionBlock {
|
||||||
|
hash: Hash::try_from(v.hash.as_slice()).unwrap(),
|
||||||
|
size: v.size,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
})
|
||||||
|
.collect::<crdt::Map<_, _>>();
|
||||||
|
|
||||||
|
let parts_etags = old
|
||||||
|
.parts_etags
|
||||||
|
.items()
|
||||||
|
.iter()
|
||||||
|
.map(|(k, v)| (*k, v.clone()))
|
||||||
|
.collect::<crdt::Map<_, _>>();
|
||||||
|
|
||||||
|
Some(Version {
|
||||||
|
uuid: Hash::try_from(old.uuid.as_slice()).unwrap(),
|
||||||
|
deleted: crdt::Bool::new(old.deleted.get()),
|
||||||
|
blocks,
|
||||||
|
parts_etags,
|
||||||
|
bucket_id: blake2sum(old.bucket.as_bytes()),
|
||||||
|
key: old.key,
|
||||||
|
})
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
+3
-2
@@ -1,6 +1,6 @@
|
|||||||
[package]
|
[package]
|
||||||
name = "garage_rpc"
|
name = "garage_rpc"
|
||||||
version = "0.8.1"
|
version = "0.8.0"
|
||||||
authors = ["Alex Auvolat <alex@adnab.me>"]
|
authors = ["Alex Auvolat <alex@adnab.me>"]
|
||||||
edition = "2018"
|
edition = "2018"
|
||||||
license = "AGPL-3.0"
|
license = "AGPL-3.0"
|
||||||
@@ -14,7 +14,7 @@ path = "lib.rs"
|
|||||||
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
|
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
|
||||||
|
|
||||||
[dependencies]
|
[dependencies]
|
||||||
garage_util = { version = "0.8.1", path = "../util" }
|
garage_util = { version = "0.8.0", path = "../util" }
|
||||||
|
|
||||||
arc-swap = "1.0"
|
arc-swap = "1.0"
|
||||||
bytes = "1.0"
|
bytes = "1.0"
|
||||||
@@ -25,6 +25,7 @@ rand = "0.8"
|
|||||||
sodiumoxide = { version = "0.2.5-0", package = "kuska-sodiumoxide" }
|
sodiumoxide = { version = "0.2.5-0", package = "kuska-sodiumoxide" }
|
||||||
|
|
||||||
async-trait = "0.1.7"
|
async-trait = "0.1.7"
|
||||||
|
rmp-serde = "0.15"
|
||||||
serde = { version = "1.0", default-features = false, features = ["derive", "rc"] }
|
serde = { version = "1.0", default-features = false, features = ["derive", "rc"] }
|
||||||
serde_bytes = "0.11"
|
serde_bytes = "0.11"
|
||||||
serde_json = "1.0"
|
serde_json = "1.0"
|
||||||
|
|||||||
+5
-8
@@ -5,7 +5,6 @@ use serde::{Deserialize, Serialize};
|
|||||||
|
|
||||||
use garage_util::crdt::{AutoCrdt, Crdt, LwwMap};
|
use garage_util::crdt::{AutoCrdt, Crdt, LwwMap};
|
||||||
use garage_util::data::*;
|
use garage_util::data::*;
|
||||||
use garage_util::encode::nonversioned_encode;
|
|
||||||
use garage_util::error::*;
|
use garage_util::error::*;
|
||||||
|
|
||||||
use crate::ring::*;
|
use crate::ring::*;
|
||||||
@@ -36,8 +35,6 @@ pub struct ClusterLayout {
|
|||||||
pub staging_hash: Hash,
|
pub staging_hash: Hash,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl garage_util::migrate::InitialFormat for ClusterLayout {}
|
|
||||||
|
|
||||||
#[derive(PartialEq, Eq, PartialOrd, Ord, Clone, Debug, Serialize, Deserialize)]
|
#[derive(PartialEq, Eq, PartialOrd, Ord, Clone, Debug, Serialize, Deserialize)]
|
||||||
pub struct NodeRoleV(pub Option<NodeRole>);
|
pub struct NodeRoleV(pub Option<NodeRole>);
|
||||||
|
|
||||||
@@ -71,7 +68,7 @@ impl NodeRole {
|
|||||||
impl ClusterLayout {
|
impl ClusterLayout {
|
||||||
pub fn new(replication_factor: usize) -> Self {
|
pub fn new(replication_factor: usize) -> Self {
|
||||||
let empty_lwwmap = LwwMap::new();
|
let empty_lwwmap = LwwMap::new();
|
||||||
let empty_lwwmap_hash = blake2sum(&nonversioned_encode(&empty_lwwmap).unwrap()[..]);
|
let empty_lwwmap_hash = blake2sum(&rmp_to_vec_all_named(&empty_lwwmap).unwrap()[..]);
|
||||||
|
|
||||||
ClusterLayout {
|
ClusterLayout {
|
||||||
version: 0,
|
version: 0,
|
||||||
@@ -93,7 +90,7 @@ impl ClusterLayout {
|
|||||||
Ordering::Equal => {
|
Ordering::Equal => {
|
||||||
self.staging.merge(&other.staging);
|
self.staging.merge(&other.staging);
|
||||||
|
|
||||||
let new_staging_hash = blake2sum(&nonversioned_encode(&self.staging).unwrap()[..]);
|
let new_staging_hash = blake2sum(&rmp_to_vec_all_named(&self.staging).unwrap()[..]);
|
||||||
let changed = new_staging_hash != self.staging_hash;
|
let changed = new_staging_hash != self.staging_hash;
|
||||||
|
|
||||||
self.staging_hash = new_staging_hash;
|
self.staging_hash = new_staging_hash;
|
||||||
@@ -128,7 +125,7 @@ To know the correct value of the new layout version, invoke `garage layout show`
|
|||||||
}
|
}
|
||||||
|
|
||||||
self.staging.clear();
|
self.staging.clear();
|
||||||
self.staging_hash = blake2sum(&nonversioned_encode(&self.staging).unwrap()[..]);
|
self.staging_hash = blake2sum(&rmp_to_vec_all_named(&self.staging).unwrap()[..]);
|
||||||
|
|
||||||
self.version += 1;
|
self.version += 1;
|
||||||
|
|
||||||
@@ -152,7 +149,7 @@ To know the correct value of the new layout version, invoke `garage layout show`
|
|||||||
}
|
}
|
||||||
|
|
||||||
self.staging.clear();
|
self.staging.clear();
|
||||||
self.staging_hash = blake2sum(&nonversioned_encode(&self.staging).unwrap()[..]);
|
self.staging_hash = blake2sum(&rmp_to_vec_all_named(&self.staging).unwrap()[..]);
|
||||||
|
|
||||||
self.version += 1;
|
self.version += 1;
|
||||||
|
|
||||||
@@ -181,7 +178,7 @@ To know the correct value of the new layout version, invoke `garage layout show`
|
|||||||
/// returns true if consistent, false if error
|
/// returns true if consistent, false if error
|
||||||
pub fn check(&self) -> bool {
|
pub fn check(&self) -> bool {
|
||||||
// Check that the hash of the staging data is correct
|
// Check that the hash of the staging data is correct
|
||||||
let staging_hash = blake2sum(&nonversioned_encode(&self.staging).unwrap()[..]);
|
let staging_hash = blake2sum(&rmp_to_vec_all_named(&self.staging).unwrap()[..]);
|
||||||
if staging_hash != self.staging_hash {
|
if staging_hash != self.staging_hash {
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|||||||
+22
-35
@@ -73,17 +73,13 @@ impl Rpc for SystemRpc {
|
|||||||
type Response = Result<SystemRpc, Error>;
|
type Response = Result<SystemRpc, Error>;
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Serialize, Deserialize)]
|
|
||||||
pub struct PeerList(Vec<(Uuid, SocketAddr)>);
|
|
||||||
impl garage_util::migrate::InitialFormat for PeerList {}
|
|
||||||
|
|
||||||
/// This node's membership manager
|
/// This node's membership manager
|
||||||
pub struct System {
|
pub struct System {
|
||||||
/// The id of this node
|
/// The id of this node
|
||||||
pub id: Uuid,
|
pub id: Uuid,
|
||||||
|
|
||||||
persist_cluster_layout: Persister<ClusterLayout>,
|
persist_cluster_layout: Persister<ClusterLayout>,
|
||||||
persist_peer_list: Persister<PeerList>,
|
persist_peer_list: Persister<Vec<(Uuid, SocketAddr)>>,
|
||||||
|
|
||||||
local_status: ArcSwap<NodeStatus>,
|
local_status: ArcSwap<NodeStatus>,
|
||||||
node_status: RwLock<HashMap<Uuid, (u64, NodeStatus)>>,
|
node_status: RwLock<HashMap<Uuid, (u64, NodeStatus)>>,
|
||||||
@@ -521,57 +517,52 @@ impl System {
|
|||||||
// ---- INTERNALS ----
|
// ---- INTERNALS ----
|
||||||
|
|
||||||
#[cfg(feature = "consul-discovery")]
|
#[cfg(feature = "consul-discovery")]
|
||||||
async fn advertise_to_consul(self: Arc<Self>) {
|
async fn advertise_to_consul(self: Arc<Self>) -> Result<(), Error> {
|
||||||
let c = match &self.consul_discovery {
|
let c = match &self.consul_discovery {
|
||||||
Some(c) => c,
|
Some(c) => c,
|
||||||
_ => return,
|
_ => return Ok(()),
|
||||||
};
|
};
|
||||||
|
|
||||||
let rpc_public_addr = match self.rpc_public_addr {
|
let rpc_public_addr = match self.rpc_public_addr {
|
||||||
Some(addr) => addr,
|
Some(addr) => addr,
|
||||||
None => {
|
None => {
|
||||||
warn!("Not advertising to Consul because rpc_public_addr is not defined in config file and could not be autodetected.");
|
warn!("Not advertising to Consul because rpc_public_addr is not defined in config file and could not be autodetected.");
|
||||||
return;
|
return Ok(());
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
if let Err(e) = c
|
c.publish_consul_service(
|
||||||
.publish_consul_service(
|
self.netapp.id,
|
||||||
self.netapp.id,
|
&self.local_status.load_full().hostname,
|
||||||
&self.local_status.load_full().hostname,
|
rpc_public_addr,
|
||||||
rpc_public_addr,
|
)
|
||||||
)
|
.await
|
||||||
.await
|
.err_context("Error while publishing Consul service")
|
||||||
{
|
|
||||||
error!("Error while publishing Consul service: {}", e);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(feature = "kubernetes-discovery")]
|
#[cfg(feature = "kubernetes-discovery")]
|
||||||
async fn advertise_to_kubernetes(self: Arc<Self>) {
|
async fn advertise_to_kubernetes(self: Arc<Self>) -> Result<(), Error> {
|
||||||
let k = match &self.kubernetes_discovery {
|
let k = match &self.kubernetes_discovery {
|
||||||
Some(k) => k,
|
Some(k) => k,
|
||||||
_ => return,
|
_ => return Ok(()),
|
||||||
};
|
};
|
||||||
|
|
||||||
let rpc_public_addr = match self.rpc_public_addr {
|
let rpc_public_addr = match self.rpc_public_addr {
|
||||||
Some(addr) => addr,
|
Some(addr) => addr,
|
||||||
None => {
|
None => {
|
||||||
warn!("Not advertising to Kubernetes because rpc_public_addr is not defined in config file and could not be autodetected.");
|
warn!("Not advertising to Kubernetes because rpc_public_addr is not defined in config file and could not be autodetected.");
|
||||||
return;
|
return Ok(());
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
if let Err(e) = publish_kubernetes_node(
|
publish_kubernetes_node(
|
||||||
k,
|
k,
|
||||||
self.netapp.id,
|
self.netapp.id,
|
||||||
&self.local_status.load_full().hostname,
|
&self.local_status.load_full().hostname,
|
||||||
rpc_public_addr,
|
rpc_public_addr,
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
{
|
.err_context("Error while publishing node to kubernetes")
|
||||||
error!("Error while publishing node to Kubernetes: {}", e);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Save network configuration to disc
|
/// Save network configuration to disc
|
||||||
@@ -730,7 +721,7 @@ impl System {
|
|||||||
|
|
||||||
// Add peer list from list stored on disk
|
// Add peer list from list stored on disk
|
||||||
if let Ok(peers) = self.persist_peer_list.load_async().await {
|
if let Ok(peers) = self.persist_peer_list.load_async().await {
|
||||||
ping_list.extend(peers.0.iter().map(|(id, addr)| ((*id).into(), *addr)))
|
ping_list.extend(peers.iter().map(|(id, addr)| ((*id).into(), *addr)))
|
||||||
}
|
}
|
||||||
|
|
||||||
// Fetch peer list from Consul
|
// Fetch peer list from Consul
|
||||||
@@ -783,10 +774,10 @@ impl System {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(feature = "consul-discovery")]
|
#[cfg(feature = "consul-discovery")]
|
||||||
tokio::spawn(self.clone().advertise_to_consul());
|
background::spawn(self.clone().advertise_to_consul());
|
||||||
|
|
||||||
#[cfg(feature = "kubernetes-discovery")]
|
#[cfg(feature = "kubernetes-discovery")]
|
||||||
tokio::spawn(self.clone().advertise_to_kubernetes());
|
background::spawn(self.clone().advertise_to_kubernetes());
|
||||||
|
|
||||||
let restart_at = tokio::time::sleep(DISCOVERY_INTERVAL);
|
let restart_at = tokio::time::sleep(DISCOVERY_INTERVAL);
|
||||||
select! {
|
select! {
|
||||||
@@ -810,16 +801,12 @@ impl System {
|
|||||||
// and append it to the list we are about to save,
|
// and append it to the list we are about to save,
|
||||||
// so that no peer ID gets lost in the process.
|
// so that no peer ID gets lost in the process.
|
||||||
if let Ok(mut prev_peer_list) = self.persist_peer_list.load_async().await {
|
if let Ok(mut prev_peer_list) = self.persist_peer_list.load_async().await {
|
||||||
prev_peer_list
|
prev_peer_list.retain(|(id, _ip)| peer_list.iter().all(|(id2, _ip2)| id2 != id));
|
||||||
.0
|
peer_list.extend(prev_peer_list);
|
||||||
.retain(|(id, _ip)| peer_list.iter().all(|(id2, _ip2)| id2 != id));
|
|
||||||
peer_list.extend(prev_peer_list.0);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Save new peer list to file
|
// Save new peer list to file
|
||||||
self.persist_peer_list
|
self.persist_peer_list.save_async(&peer_list).await
|
||||||
.save_async(&PeerList(peer_list))
|
|
||||||
.await
|
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn pull_cluster_layout(self: Arc<Self>, peer: Uuid) {
|
async fn pull_cluster_layout(self: Arc<Self>, peer: Uuid) {
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
[package]
|
[package]
|
||||||
name = "garage_table"
|
name = "garage_table"
|
||||||
version = "0.8.1"
|
version = "0.8.0"
|
||||||
authors = ["Alex Auvolat <alex@adnab.me>"]
|
authors = ["Alex Auvolat <alex@adnab.me>"]
|
||||||
edition = "2018"
|
edition = "2018"
|
||||||
license = "AGPL-3.0"
|
license = "AGPL-3.0"
|
||||||
@@ -14,9 +14,9 @@ path = "lib.rs"
|
|||||||
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
|
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
|
||||||
|
|
||||||
[dependencies]
|
[dependencies]
|
||||||
garage_db = { version = "0.8.1", path = "../db" }
|
garage_db = { version = "0.8.0", path = "../db" }
|
||||||
garage_rpc = { version = "0.8.1", path = "../rpc" }
|
garage_rpc = { version = "0.8.0", path = "../rpc" }
|
||||||
garage_util = { version = "0.8.1", path = "../util" }
|
garage_util = { version = "0.8.0", path = "../util" }
|
||||||
|
|
||||||
opentelemetry = "0.17"
|
opentelemetry = "0.17"
|
||||||
|
|
||||||
@@ -28,6 +28,7 @@ hexdump = "0.1"
|
|||||||
tracing = "0.1.30"
|
tracing = "0.1.30"
|
||||||
rand = "0.8"
|
rand = "0.8"
|
||||||
|
|
||||||
|
rmp-serde = "0.15"
|
||||||
serde = { version = "1.0", default-features = false, features = ["derive", "rc"] }
|
serde = { version = "1.0", default-features = false, features = ["derive", "rc"] }
|
||||||
serde_bytes = "0.11"
|
serde_bytes = "0.11"
|
||||||
|
|
||||||
|
|||||||
+30
-33
@@ -10,7 +10,6 @@ use garage_db::counted_tree_hack::CountedTree;
|
|||||||
|
|
||||||
use garage_util::data::*;
|
use garage_util::data::*;
|
||||||
use garage_util::error::*;
|
use garage_util::error::*;
|
||||||
use garage_util::migrate::Migrate;
|
|
||||||
|
|
||||||
use garage_rpc::system::System;
|
use garage_rpc::system::System;
|
||||||
|
|
||||||
@@ -41,7 +40,11 @@ pub struct TableData<F: TableSchema, R: TableReplication> {
|
|||||||
pub(crate) metrics: TableMetrics,
|
pub(crate) metrics: TableMetrics,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl<F: TableSchema, R: TableReplication> TableData<F, R> {
|
impl<F, R> TableData<F, R>
|
||||||
|
where
|
||||||
|
F: TableSchema,
|
||||||
|
R: TableReplication,
|
||||||
|
{
|
||||||
pub fn new(system: Arc<System>, instance: F, replication: R, db: &db::Db) -> Arc<Self> {
|
pub fn new(system: Arc<System>, instance: F, replication: R, db: &db::Db) -> Arc<Self> {
|
||||||
let store = db
|
let store = db
|
||||||
.open_tree(&format!("{}:table", F::TABLE_NAME))
|
.open_tree(&format!("{}:table", F::TABLE_NAME))
|
||||||
@@ -181,17 +184,13 @@ impl<F: TableSchema, R: TableReplication> TableData<F, R> {
|
|||||||
pub(crate) fn update_entry(&self, update_bytes: &[u8]) -> Result<(), Error> {
|
pub(crate) fn update_entry(&self, update_bytes: &[u8]) -> Result<(), Error> {
|
||||||
let update = self.decode_entry(update_bytes)?;
|
let update = self.decode_entry(update_bytes)?;
|
||||||
|
|
||||||
self.update_entry_with(
|
self.update_entry_with(update.partition_key(), update.sort_key(), |ent| match ent {
|
||||||
update.partition_key(),
|
Some(mut ent) => {
|
||||||
update.sort_key(),
|
ent.merge(&update);
|
||||||
|_tx, ent| match ent {
|
ent
|
||||||
Some(mut ent) => {
|
}
|
||||||
ent.merge(&update);
|
None => update.clone(),
|
||||||
Ok(ent)
|
})?;
|
||||||
}
|
|
||||||
None => Ok(update.clone()),
|
|
||||||
},
|
|
||||||
)?;
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -199,7 +198,7 @@ impl<F: TableSchema, R: TableReplication> TableData<F, R> {
|
|||||||
&self,
|
&self,
|
||||||
partition_key: &F::P,
|
partition_key: &F::P,
|
||||||
sort_key: &F::S,
|
sort_key: &F::S,
|
||||||
update_fn: impl Fn(&mut db::Transaction, Option<F::E>) -> db::TxResult<F::E, Error>,
|
f: impl Fn(Option<F::E>) -> F::E,
|
||||||
) -> Result<Option<F::E>, Error> {
|
) -> Result<Option<F::E>, Error> {
|
||||||
let tree_key = self.tree_key(partition_key, sort_key);
|
let tree_key = self.tree_key(partition_key, sort_key);
|
||||||
|
|
||||||
@@ -207,10 +206,10 @@ impl<F: TableSchema, R: TableReplication> TableData<F, R> {
|
|||||||
let (old_entry, old_bytes, new_entry) = match tx.get(&self.store, &tree_key)? {
|
let (old_entry, old_bytes, new_entry) = match tx.get(&self.store, &tree_key)? {
|
||||||
Some(old_bytes) => {
|
Some(old_bytes) => {
|
||||||
let old_entry = self.decode_entry(&old_bytes).map_err(db::TxError::Abort)?;
|
let old_entry = self.decode_entry(&old_bytes).map_err(db::TxError::Abort)?;
|
||||||
let new_entry = update_fn(&mut tx, Some(old_entry.clone()))?;
|
let new_entry = f(Some(old_entry.clone()));
|
||||||
(Some(old_entry), Some(old_bytes), new_entry)
|
(Some(old_entry), Some(old_bytes), new_entry)
|
||||||
}
|
}
|
||||||
None => (None, None, update_fn(&mut tx, None)?),
|
None => (None, None, f(None)),
|
||||||
};
|
};
|
||||||
|
|
||||||
// Changed can be true in two scenarios
|
// Changed can be true in two scenarios
|
||||||
@@ -220,8 +219,7 @@ impl<F: TableSchema, R: TableReplication> TableData<F, R> {
|
|||||||
// data format, the messagepack encoding changed. In this case,
|
// data format, the messagepack encoding changed. In this case,
|
||||||
// we also have to write the migrated value in the table and update
|
// we also have to write the migrated value in the table and update
|
||||||
// the associated Merkle tree entry.
|
// the associated Merkle tree entry.
|
||||||
let new_bytes = new_entry
|
let new_bytes = rmp_to_vec_all_named(&new_entry)
|
||||||
.encode()
|
|
||||||
.map_err(Error::RmpEncode)
|
.map_err(Error::RmpEncode)
|
||||||
.map_err(db::TxError::Abort)?;
|
.map_err(db::TxError::Abort)?;
|
||||||
let changed = Some(&new_bytes[..]) != old_bytes.as_deref();
|
let changed = Some(&new_bytes[..]) != old_bytes.as_deref();
|
||||||
@@ -331,15 +329,14 @@ impl<F: TableSchema, R: TableReplication> TableData<F, R> {
|
|||||||
Some(old_v) => {
|
Some(old_v) => {
|
||||||
let mut entry = self.decode_entry(&old_v).map_err(db::TxError::Abort)?;
|
let mut entry = self.decode_entry(&old_v).map_err(db::TxError::Abort)?;
|
||||||
entry.merge(ins);
|
entry.merge(ins);
|
||||||
entry.encode()
|
rmp_to_vec_all_named(&entry)
|
||||||
}
|
}
|
||||||
None => ins.encode(),
|
None => rmp_to_vec_all_named(ins),
|
||||||
};
|
};
|
||||||
let new_entry = new_entry
|
let new_entry = new_entry
|
||||||
.map_err(Error::RmpEncode)
|
.map_err(Error::RmpEncode)
|
||||||
.map_err(db::TxError::Abort)?;
|
.map_err(db::TxError::Abort)?;
|
||||||
tx.insert(&self.insert_queue, &tree_key, new_entry)?;
|
tx.insert(&self.insert_queue, &tree_key, new_entry)?;
|
||||||
|
|
||||||
self.insert_queue_notify.notify_one();
|
self.insert_queue_notify.notify_one();
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
@@ -349,23 +346,23 @@ impl<F: TableSchema, R: TableReplication> TableData<F, R> {
|
|||||||
|
|
||||||
pub fn tree_key(&self, p: &F::P, s: &F::S) -> Vec<u8> {
|
pub fn tree_key(&self, p: &F::P, s: &F::S) -> Vec<u8> {
|
||||||
let mut ret = p.hash().to_vec();
|
let mut ret = p.hash().to_vec();
|
||||||
ret.extend(s.sort_key().borrow());
|
ret.extend(s.sort_key());
|
||||||
ret
|
ret
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn decode_entry(&self, bytes: &[u8]) -> Result<F::E, Error> {
|
pub fn decode_entry(&self, bytes: &[u8]) -> Result<F::E, Error> {
|
||||||
match F::E::decode(bytes) {
|
match rmp_serde::decode::from_read_ref::<_, F::E>(bytes) {
|
||||||
Some(x) => Ok(x),
|
Ok(x) => Ok(x),
|
||||||
None => {
|
Err(e) => match F::try_migrate(bytes) {
|
||||||
error!("Unable to decode entry of {}", F::TABLE_NAME);
|
Some(x) => Ok(x),
|
||||||
for line in hexdump::hexdump_iter(bytes) {
|
None => {
|
||||||
debug!("{}", line);
|
warn!("Unable to decode entry of {}: {}", F::TABLE_NAME, e);
|
||||||
|
for line in hexdump::hexdump_iter(bytes) {
|
||||||
|
debug!("{}", line);
|
||||||
|
}
|
||||||
|
Err(e.into())
|
||||||
}
|
}
|
||||||
Err(Error::Message(format!(
|
},
|
||||||
"Unable to decode entry of {}",
|
|
||||||
F::TABLE_NAME
|
|
||||||
)))
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+26
-6
@@ -31,7 +31,7 @@ const TABLE_GC_BATCH_SIZE: usize = 1024;
|
|||||||
// and the moment the garbage collection actually happens)
|
// and the moment the garbage collection actually happens)
|
||||||
const TABLE_GC_DELAY: Duration = Duration::from_secs(24 * 3600);
|
const TABLE_GC_DELAY: Duration = Duration::from_secs(24 * 3600);
|
||||||
|
|
||||||
pub(crate) struct TableGc<F: TableSchema, R: TableReplication> {
|
pub(crate) struct TableGc<F: TableSchema + 'static, R: TableReplication + 'static> {
|
||||||
system: Arc<System>,
|
system: Arc<System>,
|
||||||
data: Arc<TableData<F, R>>,
|
data: Arc<TableData<F, R>>,
|
||||||
|
|
||||||
@@ -49,7 +49,11 @@ impl Rpc for GcRpc {
|
|||||||
type Response = Result<GcRpc, Error>;
|
type Response = Result<GcRpc, Error>;
|
||||||
}
|
}
|
||||||
|
|
||||||
impl<F: TableSchema, R: TableReplication> TableGc<F, R> {
|
impl<F, R> TableGc<F, R>
|
||||||
|
where
|
||||||
|
F: TableSchema + 'static,
|
||||||
|
R: TableReplication + 'static,
|
||||||
|
{
|
||||||
pub(crate) fn new(system: Arc<System>, data: Arc<TableData<F, R>>) -> Arc<Self> {
|
pub(crate) fn new(system: Arc<System>, data: Arc<TableData<F, R>>) -> Arc<Self> {
|
||||||
let endpoint = system
|
let endpoint = system
|
||||||
.netapp
|
.netapp
|
||||||
@@ -273,7 +277,11 @@ impl<F: TableSchema, R: TableReplication> TableGc<F, R> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[async_trait]
|
#[async_trait]
|
||||||
impl<F: TableSchema, R: TableReplication> EndpointHandler<GcRpc> for TableGc<F, R> {
|
impl<F, R> EndpointHandler<GcRpc> for TableGc<F, R>
|
||||||
|
where
|
||||||
|
F: TableSchema + 'static,
|
||||||
|
R: TableReplication + 'static,
|
||||||
|
{
|
||||||
async fn handle(self: &Arc<Self>, message: &GcRpc, _from: NodeID) -> Result<GcRpc, Error> {
|
async fn handle(self: &Arc<Self>, message: &GcRpc, _from: NodeID) -> Result<GcRpc, Error> {
|
||||||
match message {
|
match message {
|
||||||
GcRpc::Update(items) => {
|
GcRpc::Update(items) => {
|
||||||
@@ -291,12 +299,20 @@ impl<F: TableSchema, R: TableReplication> EndpointHandler<GcRpc> for TableGc<F,
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
struct GcWorker<F: TableSchema, R: TableReplication> {
|
struct GcWorker<F, R>
|
||||||
|
where
|
||||||
|
F: TableSchema + 'static,
|
||||||
|
R: TableReplication + 'static,
|
||||||
|
{
|
||||||
gc: Arc<TableGc<F, R>>,
|
gc: Arc<TableGc<F, R>>,
|
||||||
wait_delay: Duration,
|
wait_delay: Duration,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl<F: TableSchema, R: TableReplication> GcWorker<F, R> {
|
impl<F, R> GcWorker<F, R>
|
||||||
|
where
|
||||||
|
F: TableSchema + 'static,
|
||||||
|
R: TableReplication + 'static,
|
||||||
|
{
|
||||||
fn new(gc: Arc<TableGc<F, R>>) -> Self {
|
fn new(gc: Arc<TableGc<F, R>>) -> Self {
|
||||||
Self {
|
Self {
|
||||||
gc,
|
gc,
|
||||||
@@ -306,7 +322,11 @@ impl<F: TableSchema, R: TableReplication> GcWorker<F, R> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[async_trait]
|
#[async_trait]
|
||||||
impl<F: TableSchema, R: TableReplication> Worker for GcWorker<F, R> {
|
impl<F, R> Worker for GcWorker<F, R>
|
||||||
|
where
|
||||||
|
F: TableSchema + 'static,
|
||||||
|
R: TableReplication + 'static,
|
||||||
|
{
|
||||||
fn name(&self) -> String {
|
fn name(&self) -> String {
|
||||||
format!("{} GC", F::TABLE_NAME)
|
format!("{} GC", F::TABLE_NAME)
|
||||||
}
|
}
|
||||||
|
|||||||
+17
-7
@@ -10,7 +10,6 @@ use garage_db as db;
|
|||||||
|
|
||||||
use garage_util::background::*;
|
use garage_util::background::*;
|
||||||
use garage_util::data::*;
|
use garage_util::data::*;
|
||||||
use garage_util::encode::{nonversioned_decode, nonversioned_encode};
|
|
||||||
use garage_util::error::Error;
|
use garage_util::error::Error;
|
||||||
|
|
||||||
use garage_rpc::ring::*;
|
use garage_rpc::ring::*;
|
||||||
@@ -66,9 +65,13 @@ pub enum MerkleNode {
|
|||||||
Leaf(Vec<u8>, Hash),
|
Leaf(Vec<u8>, Hash),
|
||||||
}
|
}
|
||||||
|
|
||||||
impl<F: TableSchema, R: TableReplication> MerkleUpdater<F, R> {
|
impl<F, R> MerkleUpdater<F, R>
|
||||||
|
where
|
||||||
|
F: TableSchema + 'static,
|
||||||
|
R: TableReplication + 'static,
|
||||||
|
{
|
||||||
pub(crate) fn new(data: Arc<TableData<F, R>>) -> Arc<Self> {
|
pub(crate) fn new(data: Arc<TableData<F, R>>) -> Arc<Self> {
|
||||||
let empty_node_hash = blake2sum(&nonversioned_encode(&MerkleNode::Empty).unwrap()[..]);
|
let empty_node_hash = blake2sum(&rmp_to_vec_all_named(&MerkleNode::Empty).unwrap()[..]);
|
||||||
|
|
||||||
Arc::new(Self {
|
Arc::new(Self {
|
||||||
data,
|
data,
|
||||||
@@ -274,7 +277,7 @@ impl<F: TableSchema, R: TableReplication> MerkleUpdater<F, R> {
|
|||||||
tx.remove(&self.data.merkle_tree, k.encode())?;
|
tx.remove(&self.data.merkle_tree, k.encode())?;
|
||||||
Ok(self.empty_node_hash)
|
Ok(self.empty_node_hash)
|
||||||
} else {
|
} else {
|
||||||
let vby = nonversioned_encode(v).map_err(|e| db::TxError::Abort(e.into()))?;
|
let vby = rmp_to_vec_all_named(v).map_err(|e| db::TxError::Abort(e.into()))?;
|
||||||
let rethash = blake2sum(&vby[..]);
|
let rethash = blake2sum(&vby[..]);
|
||||||
tx.insert(&self.data.merkle_tree, k.encode(), vby)?;
|
tx.insert(&self.data.merkle_tree, k.encode(), vby)?;
|
||||||
Ok(rethash)
|
Ok(rethash)
|
||||||
@@ -300,10 +303,17 @@ impl<F: TableSchema, R: TableReplication> MerkleUpdater<F, R> {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
struct MerkleWorker<F: TableSchema, R: TableReplication>(Arc<MerkleUpdater<F, R>>);
|
struct MerkleWorker<F, R>(Arc<MerkleUpdater<F, R>>)
|
||||||
|
where
|
||||||
|
F: TableSchema + 'static,
|
||||||
|
R: TableReplication + 'static;
|
||||||
|
|
||||||
#[async_trait]
|
#[async_trait]
|
||||||
impl<F: TableSchema, R: TableReplication> Worker for MerkleWorker<F, R> {
|
impl<F, R> Worker for MerkleWorker<F, R>
|
||||||
|
where
|
||||||
|
F: TableSchema + 'static,
|
||||||
|
R: TableReplication + 'static,
|
||||||
|
{
|
||||||
fn name(&self) -> String {
|
fn name(&self) -> String {
|
||||||
format!("{} Merkle", F::TABLE_NAME)
|
format!("{} Merkle", F::TABLE_NAME)
|
||||||
}
|
}
|
||||||
@@ -365,7 +375,7 @@ impl MerkleNode {
|
|||||||
fn decode_opt(ent: &Option<db::Value>) -> Result<Self, Error> {
|
fn decode_opt(ent: &Option<db::Value>) -> Result<Self, Error> {
|
||||||
match ent {
|
match ent {
|
||||||
None => Ok(MerkleNode::Empty),
|
None => Ok(MerkleNode::Empty),
|
||||||
Some(v) => Ok(nonversioned_decode::<MerkleNode>(&v[..])?),
|
Some(v) => Ok(rmp_serde::decode::from_read_ref::<_, MerkleNode>(&v[..])?),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+7
-3
@@ -16,11 +16,15 @@ const BATCH_SIZE: usize = 100;
|
|||||||
|
|
||||||
pub(crate) struct InsertQueueWorker<F, R>(pub(crate) Arc<Table<F, R>>)
|
pub(crate) struct InsertQueueWorker<F, R>(pub(crate) Arc<Table<F, R>>)
|
||||||
where
|
where
|
||||||
F: TableSchema,
|
F: TableSchema + 'static,
|
||||||
R: TableReplication;
|
R: TableReplication + 'static;
|
||||||
|
|
||||||
#[async_trait]
|
#[async_trait]
|
||||||
impl<F: TableSchema, R: TableReplication> Worker for InsertQueueWorker<F, R> {
|
impl<F, R> Worker for InsertQueueWorker<F, R>
|
||||||
|
where
|
||||||
|
F: TableSchema + 'static,
|
||||||
|
R: TableReplication + 'static,
|
||||||
|
{
|
||||||
fn name(&self) -> String {
|
fn name(&self) -> String {
|
||||||
format!("{} queue", F::TABLE_NAME)
|
format!("{} queue", F::TABLE_NAME)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -2,7 +2,7 @@ use garage_rpc::ring::*;
|
|||||||
use garage_util::data::*;
|
use garage_util::data::*;
|
||||||
|
|
||||||
/// Trait to describe how a table shall be replicated
|
/// Trait to describe how a table shall be replicated
|
||||||
pub trait TableReplication: Send + Sync + 'static {
|
pub trait TableReplication: Send + Sync {
|
||||||
// See examples in table_sharded.rs and table_fullcopy.rs
|
// See examples in table_sharded.rs and table_fullcopy.rs
|
||||||
// To understand various replication methods
|
// To understand various replication methods
|
||||||
|
|
||||||
|
|||||||
+14
-16
@@ -2,14 +2,11 @@ use serde::{Deserialize, Serialize};
|
|||||||
|
|
||||||
use garage_db as db;
|
use garage_db as db;
|
||||||
use garage_util::data::*;
|
use garage_util::data::*;
|
||||||
use garage_util::migrate::Migrate;
|
|
||||||
|
|
||||||
use crate::crdt::Crdt;
|
use crate::crdt::Crdt;
|
||||||
|
|
||||||
/// Trait for field used to partition data
|
/// Trait for field used to partition data
|
||||||
pub trait PartitionKey:
|
pub trait PartitionKey {
|
||||||
Clone + PartialEq + Serialize + for<'de> Deserialize<'de> + Send + Sync + 'static
|
|
||||||
{
|
|
||||||
/// Get the key used to partition
|
/// Get the key used to partition
|
||||||
fn hash(&self) -> Hash;
|
fn hash(&self) -> Hash;
|
||||||
}
|
}
|
||||||
@@ -30,24 +27,18 @@ impl PartitionKey for FixedBytes32 {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Trait for field used to sort data
|
/// Trait for field used to sort data
|
||||||
pub trait SortKey: Clone + Serialize + for<'de> Deserialize<'de> + Send + Sync + 'static {
|
pub trait SortKey {
|
||||||
type B<'a>: std::borrow::Borrow<[u8]>;
|
|
||||||
|
|
||||||
/// Get the key used to sort
|
/// Get the key used to sort
|
||||||
fn sort_key(&self) -> Self::B<'_>;
|
fn sort_key(&self) -> &[u8];
|
||||||
}
|
}
|
||||||
|
|
||||||
impl SortKey for String {
|
impl SortKey for String {
|
||||||
type B<'a> = &'a [u8];
|
|
||||||
|
|
||||||
fn sort_key(&self) -> &[u8] {
|
fn sort_key(&self) -> &[u8] {
|
||||||
self.as_bytes()
|
self.as_bytes()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
impl SortKey for FixedBytes32 {
|
impl SortKey for FixedBytes32 {
|
||||||
type B<'a> = &'a [u8];
|
|
||||||
|
|
||||||
fn sort_key(&self) -> &[u8] {
|
fn sort_key(&self) -> &[u8] {
|
||||||
self.as_slice()
|
self.as_slice()
|
||||||
}
|
}
|
||||||
@@ -55,7 +46,7 @@ impl SortKey for FixedBytes32 {
|
|||||||
|
|
||||||
/// Trait for an entry in a table. It must be sortable and partitionnable.
|
/// Trait for an entry in a table. It must be sortable and partitionnable.
|
||||||
pub trait Entry<P: PartitionKey, S: SortKey>:
|
pub trait Entry<P: PartitionKey, S: SortKey>:
|
||||||
Crdt + PartialEq + Clone + Migrate + Send + Sync + 'static
|
Crdt + PartialEq + Clone + Serialize + for<'de> Deserialize<'de> + Send + Sync
|
||||||
{
|
{
|
||||||
/// Get the key used to partition
|
/// Get the key used to partition
|
||||||
fn partition_key(&self) -> &P;
|
fn partition_key(&self) -> &P;
|
||||||
@@ -74,16 +65,23 @@ pub trait TableSchema: Send + Sync + 'static {
|
|||||||
const TABLE_NAME: &'static str;
|
const TABLE_NAME: &'static str;
|
||||||
|
|
||||||
/// The partition key used in that table
|
/// The partition key used in that table
|
||||||
type P: PartitionKey;
|
type P: PartitionKey + Clone + PartialEq + Serialize + for<'de> Deserialize<'de> + Send + Sync;
|
||||||
/// The sort key used int that table
|
/// The sort key used int that table
|
||||||
type S: SortKey;
|
type S: SortKey + Clone + Serialize + for<'de> Deserialize<'de> + Send + Sync;
|
||||||
|
|
||||||
/// They type for an entry in that table
|
/// They type for an entry in that table
|
||||||
type E: Entry<Self::P, Self::S>;
|
type E: Entry<Self::P, Self::S>;
|
||||||
|
|
||||||
/// The type for a filter that can be applied to select entries
|
/// The type for a filter that can be applied to select entries
|
||||||
/// (e.g. filter out deleted entries)
|
/// (e.g. filter out deleted entries)
|
||||||
type Filter: Clone + Serialize + for<'de> Deserialize<'de> + Send + Sync + 'static;
|
type Filter: Clone + Serialize + for<'de> Deserialize<'de> + Send + Sync;
|
||||||
|
|
||||||
|
// Action to take if not able to decode current version:
|
||||||
|
// try loading from an older version
|
||||||
|
/// Try migrating an entry from an older version
|
||||||
|
fn try_migrate(_bytes: &[u8]) -> Option<Self::E> {
|
||||||
|
None
|
||||||
|
}
|
||||||
|
|
||||||
/// Actions triggered by data changing in a table. If such actions
|
/// Actions triggered by data changing in a table. If such actions
|
||||||
/// include updates to the local database that should be applied
|
/// include updates to the local database that should be applied
|
||||||
|
|||||||
+18
-11
@@ -14,7 +14,6 @@ use tokio::sync::{mpsc, watch};
|
|||||||
|
|
||||||
use garage_util::background::*;
|
use garage_util::background::*;
|
||||||
use garage_util::data::*;
|
use garage_util::data::*;
|
||||||
use garage_util::encode::{debug_serialize, nonversioned_encode};
|
|
||||||
use garage_util::error::{Error, OkOrMessage};
|
use garage_util::error::{Error, OkOrMessage};
|
||||||
|
|
||||||
use garage_rpc::ring::*;
|
use garage_rpc::ring::*;
|
||||||
@@ -29,7 +28,7 @@ use crate::*;
|
|||||||
// Do anti-entropy every 10 minutes
|
// Do anti-entropy every 10 minutes
|
||||||
const ANTI_ENTROPY_INTERVAL: Duration = Duration::from_secs(10 * 60);
|
const ANTI_ENTROPY_INTERVAL: Duration = Duration::from_secs(10 * 60);
|
||||||
|
|
||||||
pub struct TableSyncer<F: TableSchema, R: TableReplication> {
|
pub struct TableSyncer<F: TableSchema + 'static, R: TableReplication + 'static> {
|
||||||
system: Arc<System>,
|
system: Arc<System>,
|
||||||
data: Arc<TableData<F, R>>,
|
data: Arc<TableData<F, R>>,
|
||||||
merkle: Arc<MerkleUpdater<F, R>>,
|
merkle: Arc<MerkleUpdater<F, R>>,
|
||||||
@@ -62,7 +61,11 @@ struct TodoPartition {
|
|||||||
retain: bool,
|
retain: bool,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl<F: TableSchema, R: TableReplication> TableSyncer<F, R> {
|
impl<F, R> TableSyncer<F, R>
|
||||||
|
where
|
||||||
|
F: TableSchema + 'static,
|
||||||
|
R: TableReplication + 'static,
|
||||||
|
{
|
||||||
pub(crate) fn new(
|
pub(crate) fn new(
|
||||||
system: Arc<System>,
|
system: Arc<System>,
|
||||||
data: Arc<TableData<F, R>>,
|
data: Arc<TableData<F, R>>,
|
||||||
@@ -299,7 +302,7 @@ impl<F: TableSchema, R: TableReplication> TableSyncer<F, R> {
|
|||||||
);
|
);
|
||||||
return Ok(());
|
return Ok(());
|
||||||
}
|
}
|
||||||
let root_ck_hash = hash_of_merkle_node(&root_ck)?;
|
let root_ck_hash = hash_of::<MerkleNode>(&root_ck)?;
|
||||||
|
|
||||||
// Check if they have the same root checksum
|
// Check if they have the same root checksum
|
||||||
// If so, do nothing.
|
// If so, do nothing.
|
||||||
@@ -456,12 +459,16 @@ impl<F: TableSchema, R: TableReplication> TableSyncer<F, R> {
|
|||||||
// ======= SYNCHRONIZATION PROCEDURE -- RECEIVER SIDE ======
|
// ======= SYNCHRONIZATION PROCEDURE -- RECEIVER SIDE ======
|
||||||
|
|
||||||
#[async_trait]
|
#[async_trait]
|
||||||
impl<F: TableSchema, R: TableReplication> EndpointHandler<SyncRpc> for TableSyncer<F, R> {
|
impl<F, R> EndpointHandler<SyncRpc> for TableSyncer<F, R>
|
||||||
|
where
|
||||||
|
F: TableSchema + 'static,
|
||||||
|
R: TableReplication + 'static,
|
||||||
|
{
|
||||||
async fn handle(self: &Arc<Self>, message: &SyncRpc, from: NodeID) -> Result<SyncRpc, Error> {
|
async fn handle(self: &Arc<Self>, message: &SyncRpc, from: NodeID) -> Result<SyncRpc, Error> {
|
||||||
match message {
|
match message {
|
||||||
SyncRpc::RootCkHash(range, h) => {
|
SyncRpc::RootCkHash(range, h) => {
|
||||||
let (_root_ck_key, root_ck) = self.get_root_ck(*range)?;
|
let (_root_ck_key, root_ck) = self.get_root_ck(*range)?;
|
||||||
let hash = hash_of_merkle_node(&root_ck)?;
|
let hash = hash_of::<MerkleNode>(&root_ck)?;
|
||||||
Ok(SyncRpc::RootCkDifferent(hash != *h))
|
Ok(SyncRpc::RootCkDifferent(hash != *h))
|
||||||
}
|
}
|
||||||
SyncRpc::GetNode(k) => {
|
SyncRpc::GetNode(k) => {
|
||||||
@@ -490,7 +497,7 @@ impl<F: TableSchema, R: TableReplication> EndpointHandler<SyncRpc> for TableSync
|
|||||||
|
|
||||||
// -------- Sync Worker ---------
|
// -------- Sync Worker ---------
|
||||||
|
|
||||||
struct SyncWorker<F: TableSchema, R: TableReplication> {
|
struct SyncWorker<F: TableSchema + 'static, R: TableReplication + 'static> {
|
||||||
syncer: Arc<TableSyncer<F, R>>,
|
syncer: Arc<TableSyncer<F, R>>,
|
||||||
ring_recv: watch::Receiver<Arc<Ring>>,
|
ring_recv: watch::Receiver<Arc<Ring>>,
|
||||||
ring: Arc<Ring>,
|
ring: Arc<Ring>,
|
||||||
@@ -499,7 +506,7 @@ struct SyncWorker<F: TableSchema, R: TableReplication> {
|
|||||||
next_full_sync: Instant,
|
next_full_sync: Instant,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl<F: TableSchema, R: TableReplication> SyncWorker<F, R> {
|
impl<F: TableSchema + 'static, R: TableReplication + 'static> SyncWorker<F, R> {
|
||||||
fn add_full_sync(&mut self) {
|
fn add_full_sync(&mut self) {
|
||||||
let system = &self.syncer.system;
|
let system = &self.syncer.system;
|
||||||
let data = &self.syncer.data;
|
let data = &self.syncer.data;
|
||||||
@@ -565,7 +572,7 @@ impl<F: TableSchema, R: TableReplication> SyncWorker<F, R> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[async_trait]
|
#[async_trait]
|
||||||
impl<F: TableSchema, R: TableReplication> Worker for SyncWorker<F, R> {
|
impl<F: TableSchema + 'static, R: TableReplication + 'static> Worker for SyncWorker<F, R> {
|
||||||
fn name(&self) -> String {
|
fn name(&self) -> String {
|
||||||
format!("{} sync", F::TABLE_NAME)
|
format!("{} sync", F::TABLE_NAME)
|
||||||
}
|
}
|
||||||
@@ -615,8 +622,8 @@ impl<F: TableSchema, R: TableReplication> Worker for SyncWorker<F, R> {
|
|||||||
|
|
||||||
// ---- UTIL ----
|
// ---- UTIL ----
|
||||||
|
|
||||||
fn hash_of_merkle_node(x: &MerkleNode) -> Result<Hash, Error> {
|
fn hash_of<T: Serialize>(x: &T) -> Result<Hash, Error> {
|
||||||
Ok(blake2sum(&nonversioned_encode(x)?[..]))
|
Ok(blake2sum(&rmp_to_vec_all_named(x)?[..]))
|
||||||
}
|
}
|
||||||
|
|
||||||
fn join_ordered<'a, K: Ord + Eq, V1, V2>(
|
fn join_ordered<'a, K: Ord + Eq, V1, V2>(
|
||||||
|
|||||||
+14
-7
@@ -18,7 +18,6 @@ use garage_util::background::BackgroundRunner;
|
|||||||
use garage_util::data::*;
|
use garage_util::data::*;
|
||||||
use garage_util::error::Error;
|
use garage_util::error::Error;
|
||||||
use garage_util::metrics::RecordDuration;
|
use garage_util::metrics::RecordDuration;
|
||||||
use garage_util::migrate::Migrate;
|
|
||||||
|
|
||||||
use garage_rpc::system::System;
|
use garage_rpc::system::System;
|
||||||
use garage_rpc::*;
|
use garage_rpc::*;
|
||||||
@@ -33,7 +32,7 @@ use crate::schema::*;
|
|||||||
use crate::sync::*;
|
use crate::sync::*;
|
||||||
use crate::util::*;
|
use crate::util::*;
|
||||||
|
|
||||||
pub struct Table<F: TableSchema, R: TableReplication> {
|
pub struct Table<F: TableSchema + 'static, R: TableReplication + 'static> {
|
||||||
pub system: Arc<System>,
|
pub system: Arc<System>,
|
||||||
pub data: Arc<TableData<F, R>>,
|
pub data: Arc<TableData<F, R>>,
|
||||||
pub merkle_updater: Arc<MerkleUpdater<F, R>>,
|
pub merkle_updater: Arc<MerkleUpdater<F, R>>,
|
||||||
@@ -65,7 +64,11 @@ impl<F: TableSchema> Rpc for TableRpc<F> {
|
|||||||
type Response = Result<TableRpc<F>, Error>;
|
type Response = Result<TableRpc<F>, Error>;
|
||||||
}
|
}
|
||||||
|
|
||||||
impl<F: TableSchema, R: TableReplication> Table<F, R> {
|
impl<F, R> Table<F, R>
|
||||||
|
where
|
||||||
|
F: TableSchema + 'static,
|
||||||
|
R: TableReplication + 'static,
|
||||||
|
{
|
||||||
// =============== PUBLIC INTERFACE FUNCTIONS (new, insert, get, etc) ===============
|
// =============== PUBLIC INTERFACE FUNCTIONS (new, insert, get, etc) ===============
|
||||||
|
|
||||||
pub fn new(instance: F, replication: R, system: Arc<System>, db: &db::Db) -> Arc<Self> {
|
pub fn new(instance: F, replication: R, system: Arc<System>, db: &db::Db) -> Arc<Self> {
|
||||||
@@ -119,7 +122,7 @@ impl<F: TableSchema, R: TableReplication> Table<F, R> {
|
|||||||
let hash = e.partition_key().hash();
|
let hash = e.partition_key().hash();
|
||||||
let who = self.data.replication.write_nodes(&hash);
|
let who = self.data.replication.write_nodes(&hash);
|
||||||
|
|
||||||
let e_enc = Arc::new(ByteBuf::from(e.encode()?));
|
let e_enc = Arc::new(ByteBuf::from(rmp_to_vec_all_named(e)?));
|
||||||
let rpc = TableRpc::<F>::Update(vec![e_enc]);
|
let rpc = TableRpc::<F>::Update(vec![e_enc]);
|
||||||
|
|
||||||
self.system
|
self.system
|
||||||
@@ -170,7 +173,7 @@ impl<F: TableSchema, R: TableReplication> Table<F, R> {
|
|||||||
let entry = entry.borrow();
|
let entry = entry.borrow();
|
||||||
let hash = entry.partition_key().hash();
|
let hash = entry.partition_key().hash();
|
||||||
let who = self.data.replication.write_nodes(&hash);
|
let who = self.data.replication.write_nodes(&hash);
|
||||||
let e_enc = Arc::new(ByteBuf::from(entry.encode()?));
|
let e_enc = Arc::new(ByteBuf::from(rmp_to_vec_all_named(entry)?));
|
||||||
for node in who {
|
for node in who {
|
||||||
call_list.entry(node).or_default().push(e_enc.clone());
|
call_list.entry(node).or_default().push(e_enc.clone());
|
||||||
}
|
}
|
||||||
@@ -409,7 +412,7 @@ impl<F: TableSchema, R: TableReplication> Table<F, R> {
|
|||||||
// =============== UTILITY FUNCTION FOR CLIENT OPERATIONS ===============
|
// =============== UTILITY FUNCTION FOR CLIENT OPERATIONS ===============
|
||||||
|
|
||||||
async fn repair_on_read(&self, who: &[Uuid], what: F::E) -> Result<(), Error> {
|
async fn repair_on_read(&self, who: &[Uuid], what: F::E) -> Result<(), Error> {
|
||||||
let what_enc = Arc::new(ByteBuf::from(what.encode()?));
|
let what_enc = Arc::new(ByteBuf::from(rmp_to_vec_all_named(&what)?));
|
||||||
self.system
|
self.system
|
||||||
.rpc
|
.rpc
|
||||||
.try_call_many(
|
.try_call_many(
|
||||||
@@ -424,7 +427,11 @@ impl<F: TableSchema, R: TableReplication> Table<F, R> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[async_trait]
|
#[async_trait]
|
||||||
impl<F: TableSchema, R: TableReplication> EndpointHandler<TableRpc<F>> for Table<F, R> {
|
impl<F, R> EndpointHandler<TableRpc<F>> for Table<F, R>
|
||||||
|
where
|
||||||
|
F: TableSchema + 'static,
|
||||||
|
R: TableReplication + 'static,
|
||||||
|
{
|
||||||
async fn handle(
|
async fn handle(
|
||||||
self: &Arc<Self>,
|
self: &Arc<Self>,
|
||||||
msg: &TableRpc<F>,
|
msg: &TableRpc<F>,
|
||||||
|
|||||||
@@ -7,8 +7,6 @@ use crate::schema::*;
|
|||||||
#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)]
|
#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
pub struct EmptyKey;
|
pub struct EmptyKey;
|
||||||
impl SortKey for EmptyKey {
|
impl SortKey for EmptyKey {
|
||||||
type B<'a> = &'a [u8];
|
|
||||||
|
|
||||||
fn sort_key(&self) -> &[u8] {
|
fn sort_key(&self) -> &[u8] {
|
||||||
&[]
|
&[]
|
||||||
}
|
}
|
||||||
|
|||||||
+2
-3
@@ -1,6 +1,6 @@
|
|||||||
[package]
|
[package]
|
||||||
name = "garage_util"
|
name = "garage_util"
|
||||||
version = "0.8.1"
|
version = "0.8.0"
|
||||||
authors = ["Alex Auvolat <alex@adnab.me>"]
|
authors = ["Alex Auvolat <alex@adnab.me>"]
|
||||||
edition = "2018"
|
edition = "2018"
|
||||||
license = "AGPL-3.0"
|
license = "AGPL-3.0"
|
||||||
@@ -14,7 +14,7 @@ path = "lib.rs"
|
|||||||
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
|
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
|
||||||
|
|
||||||
[dependencies]
|
[dependencies]
|
||||||
garage_db = { version = "0.8.1", path = "../db" }
|
garage_db = { version = "0.8.0", path = "../db" }
|
||||||
|
|
||||||
arc-swap = "1.0"
|
arc-swap = "1.0"
|
||||||
async-trait = "0.1"
|
async-trait = "0.1"
|
||||||
@@ -23,7 +23,6 @@ bytes = "1.0"
|
|||||||
digest = "0.10"
|
digest = "0.10"
|
||||||
err-derive = "0.3"
|
err-derive = "0.3"
|
||||||
git-version = "0.3.4"
|
git-version = "0.3.4"
|
||||||
hexdump = "0.1"
|
|
||||||
xxhash-rust = { version = "0.8", default-features = false, features = ["xxh3"] }
|
xxhash-rust = { version = "0.8", default-features = false, features = ["xxh3"] }
|
||||||
hex = "0.4"
|
hex = "0.4"
|
||||||
lazy_static = "1.4"
|
lazy_static = "1.4"
|
||||||
|
|||||||
@@ -1,6 +1,5 @@
|
|||||||
//! Job runner for futures and async functions
|
//! Job runner for futures and async functions
|
||||||
|
|
||||||
pub mod vars;
|
|
||||||
pub mod worker;
|
pub mod worker;
|
||||||
|
|
||||||
use std::collections::HashMap;
|
use std::collections::HashMap;
|
||||||
|
|||||||
@@ -1,113 +0,0 @@
|
|||||||
use std::collections::HashMap;
|
|
||||||
use std::str::FromStr;
|
|
||||||
|
|
||||||
use crate::error::{Error, OkOrMessage};
|
|
||||||
use crate::migrate::Migrate;
|
|
||||||
use crate::persister::PersisterShared;
|
|
||||||
|
|
||||||
pub struct BgVars {
|
|
||||||
vars: HashMap<&'static str, Box<dyn BgVarTrait>>,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl BgVars {
|
|
||||||
pub fn new() -> Self {
|
|
||||||
Self {
|
|
||||||
vars: HashMap::new(),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn register_rw<V, T, GF, SF>(
|
|
||||||
&mut self,
|
|
||||||
p: &PersisterShared<V>,
|
|
||||||
name: &'static str,
|
|
||||||
get_fn: GF,
|
|
||||||
set_fn: SF,
|
|
||||||
) where
|
|
||||||
V: Migrate + Default + Send + Sync,
|
|
||||||
T: FromStr + ToString + Send + Sync + 'static,
|
|
||||||
GF: Fn(&PersisterShared<V>) -> T + Send + Sync + 'static,
|
|
||||||
SF: Fn(&PersisterShared<V>, T) -> Result<(), Error> + Send + Sync + 'static,
|
|
||||||
{
|
|
||||||
let p1 = p.clone();
|
|
||||||
let get_fn = move || get_fn(&p1);
|
|
||||||
|
|
||||||
let p2 = p.clone();
|
|
||||||
let set_fn = move |v| set_fn(&p2, v);
|
|
||||||
|
|
||||||
self.vars.insert(name, Box::new(BgVar { get_fn, set_fn }));
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn register_ro<V, T, GF>(&mut self, p: &PersisterShared<V>, name: &'static str, get_fn: GF)
|
|
||||||
where
|
|
||||||
V: Migrate + Default + Send + Sync,
|
|
||||||
T: FromStr + ToString + Send + Sync + 'static,
|
|
||||||
GF: Fn(&PersisterShared<V>) -> T + Send + Sync + 'static,
|
|
||||||
{
|
|
||||||
let p1 = p.clone();
|
|
||||||
let get_fn = move || get_fn(&p1);
|
|
||||||
|
|
||||||
let set_fn = move |_| Err(Error::Message(format!("Cannot set value of {}", name)));
|
|
||||||
|
|
||||||
self.vars.insert(name, Box::new(BgVar { get_fn, set_fn }));
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn get(&self, var: &str) -> Result<String, Error> {
|
|
||||||
Ok(self
|
|
||||||
.vars
|
|
||||||
.get(var)
|
|
||||||
.ok_or_message("variable does not exist")?
|
|
||||||
.get())
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn get_all(&self) -> Vec<(&'static str, String)> {
|
|
||||||
self.vars.iter().map(|(k, v)| (*k, v.get())).collect()
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn set(&self, var: &str, val: &str) -> Result<(), Error> {
|
|
||||||
self.vars
|
|
||||||
.get(var)
|
|
||||||
.ok_or_message("variable does not exist")?
|
|
||||||
.set(val)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl Default for BgVars {
|
|
||||||
fn default() -> Self {
|
|
||||||
Self::new()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// ----
|
|
||||||
|
|
||||||
trait BgVarTrait: Send + Sync + 'static {
|
|
||||||
fn get(&self) -> String;
|
|
||||||
fn set(&self, v: &str) -> Result<(), Error>;
|
|
||||||
}
|
|
||||||
|
|
||||||
struct BgVar<T, GF, SF>
|
|
||||||
where
|
|
||||||
T: FromStr + ToString + Send + Sync + 'static,
|
|
||||||
GF: Fn() -> T + Send + Sync + 'static,
|
|
||||||
SF: Fn(T) -> Result<(), Error> + Sync + Send + 'static,
|
|
||||||
{
|
|
||||||
get_fn: GF,
|
|
||||||
set_fn: SF,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl<T, GF, SF> BgVarTrait for BgVar<T, GF, SF>
|
|
||||||
where
|
|
||||||
T: FromStr + ToString + Sync + Send + 'static,
|
|
||||||
GF: Fn() -> T + Sync + Send + 'static,
|
|
||||||
SF: Fn(T) -> Result<(), Error> + Sync + Send + 'static,
|
|
||||||
{
|
|
||||||
fn get(&self) -> String {
|
|
||||||
(self.get_fn)().to_string()
|
|
||||||
}
|
|
||||||
|
|
||||||
fn set(&self, vstr: &str) -> Result<(), Error> {
|
|
||||||
let value = vstr
|
|
||||||
.parse()
|
|
||||||
.map_err(|_| Error::Message(format!("invalid value: {}", vstr)))?;
|
|
||||||
(self.set_fn)(value)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -140,3 +140,34 @@ pub fn fasthash(data: &[u8]) -> FastHash {
|
|||||||
pub fn gen_uuid() -> Uuid {
|
pub fn gen_uuid() -> Uuid {
|
||||||
rand::thread_rng().gen::<[u8; 32]>().into()
|
rand::thread_rng().gen::<[u8; 32]>().into()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// RMP serialization with names of fields and variants
|
||||||
|
|
||||||
|
/// Serialize to MessagePack
|
||||||
|
pub fn rmp_to_vec_all_named<T>(val: &T) -> Result<Vec<u8>, rmp_serde::encode::Error>
|
||||||
|
where
|
||||||
|
T: Serialize + ?Sized,
|
||||||
|
{
|
||||||
|
let mut wr = Vec::with_capacity(128);
|
||||||
|
let mut se = rmp_serde::Serializer::new(&mut wr)
|
||||||
|
.with_struct_map()
|
||||||
|
.with_string_variants();
|
||||||
|
val.serialize(&mut se)?;
|
||||||
|
Ok(wr)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Serialize to JSON, truncating long result
|
||||||
|
pub fn debug_serialize<T: Serialize>(x: T) -> String {
|
||||||
|
match serde_json::to_string(&x) {
|
||||||
|
Ok(ss) => {
|
||||||
|
if ss.len() > 100 {
|
||||||
|
// TODO this can panic if 100 is not a codepoint boundary, but inside a 2 Bytes
|
||||||
|
// (or more) codepoint
|
||||||
|
ss[..100].to_string()
|
||||||
|
} else {
|
||||||
|
ss
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Err(e) => format!("<JSON serialization error: {}>", e),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -1,42 +0,0 @@
|
|||||||
use serde::{Deserialize, Serialize};
|
|
||||||
|
|
||||||
/// Serialize to MessagePacki, without versionning
|
|
||||||
/// (see garage_util::migrate for functions that manage versionned
|
|
||||||
/// data formats)
|
|
||||||
pub fn nonversioned_encode<T>(val: &T) -> Result<Vec<u8>, rmp_serde::encode::Error>
|
|
||||||
where
|
|
||||||
T: Serialize + ?Sized,
|
|
||||||
{
|
|
||||||
let mut wr = Vec::with_capacity(128);
|
|
||||||
let mut se = rmp_serde::Serializer::new(&mut wr)
|
|
||||||
.with_struct_map()
|
|
||||||
.with_string_variants();
|
|
||||||
val.serialize(&mut se)?;
|
|
||||||
Ok(wr)
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Deserialize from MessagePacki, without versionning
|
|
||||||
/// (see garage_util::migrate for functions that manage versionned
|
|
||||||
/// data formats)
|
|
||||||
pub fn nonversioned_decode<T>(bytes: &[u8]) -> Result<T, rmp_serde::decode::Error>
|
|
||||||
where
|
|
||||||
T: for<'de> Deserialize<'de> + ?Sized,
|
|
||||||
{
|
|
||||||
rmp_serde::decode::from_read_ref::<_, T>(bytes)
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Serialize to JSON, truncating long result
|
|
||||||
pub fn debug_serialize<T: Serialize>(x: T) -> String {
|
|
||||||
match serde_json::to_string(&x) {
|
|
||||||
Ok(ss) => {
|
|
||||||
if ss.len() > 100 {
|
|
||||||
// TODO this can panic if 100 is not a codepoint boundary, but inside a 2 Bytes
|
|
||||||
// (or more) codepoint
|
|
||||||
ss[..100].to_string()
|
|
||||||
} else {
|
|
||||||
ss
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Err(e) => format!("<JSON serialization error: {}>", e),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -7,7 +7,6 @@ use err_derive::Error;
|
|||||||
use serde::{de::Visitor, Deserialize, Deserializer, Serialize, Serializer};
|
use serde::{de::Visitor, Deserialize, Deserializer, Serialize, Serializer};
|
||||||
|
|
||||||
use crate::data::*;
|
use crate::data::*;
|
||||||
use crate::encode::debug_serialize;
|
|
||||||
|
|
||||||
/// Regroup all Garage errors
|
/// Regroup all Garage errors
|
||||||
#[derive(Debug, Error)]
|
#[derive(Debug, Error)]
|
||||||
|
|||||||
+1
-2
@@ -8,12 +8,11 @@ pub mod background;
|
|||||||
pub mod config;
|
pub mod config;
|
||||||
pub mod crdt;
|
pub mod crdt;
|
||||||
pub mod data;
|
pub mod data;
|
||||||
pub mod encode;
|
|
||||||
pub mod error;
|
pub mod error;
|
||||||
pub mod formater;
|
pub mod formater;
|
||||||
pub mod metrics;
|
pub mod metrics;
|
||||||
pub mod migrate;
|
|
||||||
pub mod persister;
|
pub mod persister;
|
||||||
pub mod time;
|
pub mod time;
|
||||||
|
pub mod token_bucket;
|
||||||
pub mod tranquilizer;
|
pub mod tranquilizer;
|
||||||
pub mod version;
|
pub mod version;
|
||||||
|
|||||||
@@ -1,159 +0,0 @@
|
|||||||
use serde::{Deserialize, Serialize};
|
|
||||||
|
|
||||||
/// Indicates that this type has an encoding that can be migrated from
|
|
||||||
/// a previous version upon upgrades of Garage.
|
|
||||||
pub trait Migrate: Serialize + for<'de> Deserialize<'de> + 'static {
|
|
||||||
/// A sequence of bytes to add at the beginning of the serialized
|
|
||||||
/// string, to identify that the data is of this version.
|
|
||||||
const VERSION_MARKER: &'static [u8] = b"";
|
|
||||||
|
|
||||||
/// The previous version of this data type, from which items of this version
|
|
||||||
/// can be migrated.
|
|
||||||
type Previous: Migrate;
|
|
||||||
|
|
||||||
/// The migration function that transforms a value decoded in the old format
|
|
||||||
/// to an up-to-date value.
|
|
||||||
fn migrate(previous: Self::Previous) -> Self;
|
|
||||||
|
|
||||||
/// Decode an encoded version of this type, going through a migration if necessary.
|
|
||||||
fn decode(bytes: &[u8]) -> Option<Self> {
|
|
||||||
let marker_len = Self::VERSION_MARKER.len();
|
|
||||||
if bytes.get(..marker_len) == Some(Self::VERSION_MARKER) {
|
|
||||||
if let Ok(value) = rmp_serde::decode::from_read_ref::<_, Self>(&bytes[marker_len..]) {
|
|
||||||
return Some(value);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
Self::Previous::decode(bytes).map(Self::migrate)
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Encode this type with optionnal version marker
|
|
||||||
fn encode(&self) -> Result<Vec<u8>, rmp_serde::encode::Error> {
|
|
||||||
let mut wr = Vec::with_capacity(128);
|
|
||||||
wr.extend_from_slice(Self::VERSION_MARKER);
|
|
||||||
let mut se = rmp_serde::Serializer::new(&mut wr)
|
|
||||||
.with_struct_map()
|
|
||||||
.with_string_variants();
|
|
||||||
self.serialize(&mut se)?;
|
|
||||||
Ok(wr)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Indicates that this type has no previous encoding version to be migrated from.
|
|
||||||
pub trait InitialFormat: Serialize + for<'de> Deserialize<'de> + 'static {
|
|
||||||
/// A sequence of bytes to add at the beginning of the serialized
|
|
||||||
/// string, to identify that the data is of this version.
|
|
||||||
const VERSION_MARKER: &'static [u8] = b"";
|
|
||||||
}
|
|
||||||
|
|
||||||
impl<T: InitialFormat> Migrate for T {
|
|
||||||
const VERSION_MARKER: &'static [u8] = <T as InitialFormat>::VERSION_MARKER;
|
|
||||||
|
|
||||||
type Previous = NoPrevious;
|
|
||||||
|
|
||||||
fn migrate(_previous: Self::Previous) -> Self {
|
|
||||||
unreachable!();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Internal type used by InitialFormat, not meant for general use.
|
|
||||||
#[derive(Serialize, Deserialize)]
|
|
||||||
pub enum NoPrevious {}
|
|
||||||
|
|
||||||
impl Migrate for NoPrevious {
|
|
||||||
type Previous = NoPrevious;
|
|
||||||
|
|
||||||
fn migrate(_previous: Self::Previous) -> Self {
|
|
||||||
unreachable!();
|
|
||||||
}
|
|
||||||
|
|
||||||
fn decode(_bytes: &[u8]) -> Option<Self> {
|
|
||||||
None
|
|
||||||
}
|
|
||||||
|
|
||||||
fn encode(&self) -> Result<Vec<u8>, rmp_serde::encode::Error> {
|
|
||||||
unreachable!()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[cfg(test)]
|
|
||||||
mod test {
|
|
||||||
use super::*;
|
|
||||||
|
|
||||||
#[derive(Serialize, Deserialize, PartialEq, Eq, Debug)]
|
|
||||||
struct V1 {
|
|
||||||
a: usize,
|
|
||||||
b: String,
|
|
||||||
}
|
|
||||||
impl InitialFormat for V1 {}
|
|
||||||
|
|
||||||
#[derive(Serialize, Deserialize, PartialEq, Eq, Debug)]
|
|
||||||
struct V2 {
|
|
||||||
a: usize,
|
|
||||||
b: Vec<String>,
|
|
||||||
c: String,
|
|
||||||
}
|
|
||||||
impl Migrate for V2 {
|
|
||||||
const VERSION_MARKER: &'static [u8] = b"GtestV2";
|
|
||||||
type Previous = V1;
|
|
||||||
fn migrate(prev: V1) -> V2 {
|
|
||||||
V2 {
|
|
||||||
a: prev.a,
|
|
||||||
b: vec![prev.b],
|
|
||||||
c: String::new(),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn test_v1() {
|
|
||||||
let x = V1 {
|
|
||||||
a: 12,
|
|
||||||
b: "hello".into(),
|
|
||||||
};
|
|
||||||
let x_enc = x.encode().unwrap();
|
|
||||||
let y = V1::decode(&x_enc).unwrap();
|
|
||||||
assert_eq!(x, y);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn test_v2() {
|
|
||||||
let x = V2 {
|
|
||||||
a: 12,
|
|
||||||
b: vec!["hello".into(), "world".into()],
|
|
||||||
c: "plop".into(),
|
|
||||||
};
|
|
||||||
let x_enc = x.encode().unwrap();
|
|
||||||
assert_eq!(&x_enc[..V2::VERSION_MARKER.len()], V2::VERSION_MARKER);
|
|
||||||
let y = V2::decode(&x_enc).unwrap();
|
|
||||||
assert_eq!(x, y);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn test_migrate() {
|
|
||||||
let x = V1 {
|
|
||||||
a: 12,
|
|
||||||
b: "hello".into(),
|
|
||||||
};
|
|
||||||
let x_enc = x.encode().unwrap();
|
|
||||||
|
|
||||||
let xx = V1::decode(&x_enc).unwrap();
|
|
||||||
assert_eq!(x, xx);
|
|
||||||
|
|
||||||
let y = V2::decode(&x_enc).unwrap();
|
|
||||||
assert_eq!(
|
|
||||||
y,
|
|
||||||
V2 {
|
|
||||||
a: 12,
|
|
||||||
b: vec!["hello".into()],
|
|
||||||
c: "".into(),
|
|
||||||
}
|
|
||||||
);
|
|
||||||
|
|
||||||
let y_enc = y.encode().unwrap();
|
|
||||||
assert_eq!(&y_enc[..V2::VERSION_MARKER.len()], V2::VERSION_MARKER);
|
|
||||||
|
|
||||||
let z = V2::decode(&y_enc).unwrap();
|
|
||||||
assert_eq!(y, z);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
+13
-61
@@ -1,19 +1,23 @@
|
|||||||
use std::io::{Read, Write};
|
use std::io::{Read, Write};
|
||||||
use std::path::{Path, PathBuf};
|
use std::path::{Path, PathBuf};
|
||||||
use std::sync::{Arc, RwLock};
|
|
||||||
|
|
||||||
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
||||||
|
|
||||||
use crate::error::Error;
|
use serde::{Deserialize, Serialize};
|
||||||
use crate::migrate::Migrate;
|
|
||||||
|
|
||||||
pub struct Persister<T: Migrate> {
|
use crate::data::*;
|
||||||
|
use crate::error::Error;
|
||||||
|
|
||||||
|
pub struct Persister<T: Serialize + for<'de> Deserialize<'de>> {
|
||||||
path: PathBuf,
|
path: PathBuf,
|
||||||
|
|
||||||
_marker: std::marker::PhantomData<T>,
|
_marker: std::marker::PhantomData<T>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl<T: Migrate> Persister<T> {
|
impl<T> Persister<T>
|
||||||
|
where
|
||||||
|
T: Serialize + for<'de> Deserialize<'de>,
|
||||||
|
{
|
||||||
pub fn new(base_dir: &Path, file_name: &str) -> Self {
|
pub fn new(base_dir: &Path, file_name: &str) -> Self {
|
||||||
let mut path = base_dir.to_path_buf();
|
let mut path = base_dir.to_path_buf();
|
||||||
path.push(file_name);
|
path.push(file_name);
|
||||||
@@ -23,37 +27,18 @@ impl<T: Migrate> Persister<T> {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn decode(&self, bytes: &[u8]) -> Result<T, Error> {
|
|
||||||
match T::decode(bytes) {
|
|
||||||
Some(v) => Ok(v),
|
|
||||||
None => {
|
|
||||||
error!(
|
|
||||||
"Unable to decode persisted data file {}",
|
|
||||||
self.path.display()
|
|
||||||
);
|
|
||||||
for line in hexdump::hexdump_iter(bytes) {
|
|
||||||
debug!("{}", line);
|
|
||||||
}
|
|
||||||
Err(Error::Message(format!(
|
|
||||||
"Unable to decode persisted data file {}",
|
|
||||||
self.path.display()
|
|
||||||
)))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn load(&self) -> Result<T, Error> {
|
pub fn load(&self) -> Result<T, Error> {
|
||||||
let mut file = std::fs::OpenOptions::new().read(true).open(&self.path)?;
|
let mut file = std::fs::OpenOptions::new().read(true).open(&self.path)?;
|
||||||
|
|
||||||
let mut bytes = vec![];
|
let mut bytes = vec![];
|
||||||
file.read_to_end(&mut bytes)?;
|
file.read_to_end(&mut bytes)?;
|
||||||
|
|
||||||
let value = self.decode(&bytes[..])?;
|
let value = rmp_serde::decode::from_read_ref(&bytes[..])?;
|
||||||
Ok(value)
|
Ok(value)
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn save(&self, t: &T) -> Result<(), Error> {
|
pub fn save(&self, t: &T) -> Result<(), Error> {
|
||||||
let bytes = t.encode()?;
|
let bytes = rmp_to_vec_all_named(t)?;
|
||||||
|
|
||||||
let mut file = std::fs::OpenOptions::new()
|
let mut file = std::fs::OpenOptions::new()
|
||||||
.write(true)
|
.write(true)
|
||||||
@@ -72,12 +57,12 @@ impl<T: Migrate> Persister<T> {
|
|||||||
let mut bytes = vec![];
|
let mut bytes = vec![];
|
||||||
file.read_to_end(&mut bytes).await?;
|
file.read_to_end(&mut bytes).await?;
|
||||||
|
|
||||||
let value = self.decode(&bytes[..])?;
|
let value = rmp_serde::decode::from_read_ref(&bytes[..])?;
|
||||||
Ok(value)
|
Ok(value)
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn save_async(&self, t: &T) -> Result<(), Error> {
|
pub async fn save_async(&self, t: &T) -> Result<(), Error> {
|
||||||
let bytes = t.encode()?;
|
let bytes = rmp_to_vec_all_named(t)?;
|
||||||
|
|
||||||
let mut file = tokio::fs::File::create(&self.path).await?;
|
let mut file = tokio::fs::File::create(&self.path).await?;
|
||||||
file.write_all(&bytes[..]).await?;
|
file.write_all(&bytes[..]).await?;
|
||||||
@@ -85,36 +70,3 @@ impl<T: Migrate> Persister<T> {
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub struct PersisterShared<V: Migrate + Default>(Arc<(Persister<V>, RwLock<V>)>);
|
|
||||||
|
|
||||||
impl<V: Migrate + Default> Clone for PersisterShared<V> {
|
|
||||||
fn clone(&self) -> PersisterShared<V> {
|
|
||||||
PersisterShared(self.0.clone())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl<V: Migrate + Default> PersisterShared<V> {
|
|
||||||
pub fn new(base_dir: &Path, file_name: &str) -> Self {
|
|
||||||
let persister = Persister::new(base_dir, file_name);
|
|
||||||
let value = persister.load().unwrap_or_default();
|
|
||||||
Self(Arc::new((persister, RwLock::new(value))))
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn get_with<F, R>(&self, f: F) -> R
|
|
||||||
where
|
|
||||||
F: FnOnce(&V) -> R,
|
|
||||||
{
|
|
||||||
let value = self.0 .1.read().unwrap();
|
|
||||||
f(&value)
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn set_with<F>(&self, f: F) -> Result<(), Error>
|
|
||||||
where
|
|
||||||
F: FnOnce(&mut V),
|
|
||||||
{
|
|
||||||
let mut value = self.0 .1.write().unwrap();
|
|
||||||
f(&mut value);
|
|
||||||
self.0 .0.save(&value)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -0,0 +1,40 @@
|
|||||||
|
use std::time::{Duration, Instant};
|
||||||
|
|
||||||
|
use tokio::time::sleep;
|
||||||
|
|
||||||
|
pub struct TokenBucket {
|
||||||
|
// Replenish rate: number of tokens per second
|
||||||
|
replenish_rate: u64,
|
||||||
|
// Current number of tokens
|
||||||
|
tokens: u64,
|
||||||
|
// Last replenish time
|
||||||
|
last_replenish: Instant,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl TokenBucket {
|
||||||
|
pub fn new(replenish_rate: u64) -> Self {
|
||||||
|
Self {
|
||||||
|
replenish_rate,
|
||||||
|
tokens: 0,
|
||||||
|
last_replenish: Instant::now(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn take(&mut self, tokens: u64) {
|
||||||
|
while self.tokens < tokens {
|
||||||
|
let needed = tokens - self.tokens;
|
||||||
|
let delay = (needed as f64) / (self.replenish_rate as f64);
|
||||||
|
sleep(Duration::from_secs_f64(delay)).await;
|
||||||
|
self.replenish();
|
||||||
|
}
|
||||||
|
self.tokens -= tokens;
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn replenish(&mut self) {
|
||||||
|
let now = Instant::now();
|
||||||
|
let new_tokens =
|
||||||
|
((now - self.last_replenish).as_secs_f64() * (self.replenish_rate as f64)) as u64;
|
||||||
|
self.tokens += new_tokens;
|
||||||
|
self.last_replenish = now;
|
||||||
|
}
|
||||||
|
}
|
||||||
+5
-5
@@ -1,6 +1,6 @@
|
|||||||
[package]
|
[package]
|
||||||
name = "garage_web"
|
name = "garage_web"
|
||||||
version = "0.8.1"
|
version = "0.8.0"
|
||||||
authors = ["Alex Auvolat <alex@adnab.me>", "Quentin Dufour <quentin@dufour.io>"]
|
authors = ["Alex Auvolat <alex@adnab.me>", "Quentin Dufour <quentin@dufour.io>"]
|
||||||
edition = "2018"
|
edition = "2018"
|
||||||
license = "AGPL-3.0"
|
license = "AGPL-3.0"
|
||||||
@@ -14,10 +14,10 @@ path = "lib.rs"
|
|||||||
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
|
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
|
||||||
|
|
||||||
[dependencies]
|
[dependencies]
|
||||||
garage_api = { version = "0.8.1", path = "../api" }
|
garage_api = { version = "0.8.0", path = "../api" }
|
||||||
garage_model = { version = "0.8.1", path = "../model" }
|
garage_model = { version = "0.8.0", path = "../model" }
|
||||||
garage_util = { version = "0.8.1", path = "../util" }
|
garage_util = { version = "0.8.0", path = "../util" }
|
||||||
garage_table = { version = "0.8.1", path = "../table" }
|
garage_table = { version = "0.8.0", path = "../table" }
|
||||||
|
|
||||||
err-derive = "0.3"
|
err-derive = "0.3"
|
||||||
tracing = "0.1.30"
|
tracing = "0.1.30"
|
||||||
|
|||||||
Reference in New Issue
Block a user