mirror of
https://github.com/deuxfleurs-org/garage.git
synced 2026-07-26 07:58:14 +00:00
upgrade heed to version 0.22 (#1318)
- migration from `ByteSlice` to `Bytes` heed type. - not sure about the impact of only one `read_txn` for the entire function `list_trees`, whereas before `open_database` call uses their own access controller. Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/1318 Co-authored-by: Gwen Lg <me@gwenlg.fr> Co-committed-by: Gwen Lg <me@gwenlg.fr>
This commit is contained in:
Gwen Lg
Generated
+74
-33
@@ -610,15 +610,6 @@ version = "1.8.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06"
|
||||
|
||||
[[package]]
|
||||
name = "bincode"
|
||||
version = "1.3.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad"
|
||||
dependencies = [
|
||||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "bitflags"
|
||||
version = "1.3.2"
|
||||
@@ -630,6 +621,9 @@ name = "bitflags"
|
||||
version = "2.10.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "812e12b5285cc515a9c72a5c1d3b6d46a19dac5acfef5265968c166106e31dd3"
|
||||
dependencies = [
|
||||
"serde_core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "blake2"
|
||||
@@ -655,12 +649,6 @@ version = "3.19.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5dd9dc738b7a8311c7ade152424974d8115f2cdad61e8dab8dac9f2362298510"
|
||||
|
||||
[[package]]
|
||||
name = "bytemuck"
|
||||
version = "1.24.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1fbdf580320f38b612e485521afda1ee26d10cc9884efaaa750d383e13e3c5f4"
|
||||
|
||||
[[package]]
|
||||
name = "byteorder"
|
||||
version = "1.5.0"
|
||||
@@ -1053,6 +1041,15 @@ version = "0.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c0d05e1c0dbad51b52c38bda7adceef61b9efc2baf04acfe8726a8c4630a6f57"
|
||||
|
||||
[[package]]
|
||||
name = "doxygen-rs"
|
||||
version = "0.4.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "415b6ec780d34dcf624666747194393603d0373b7141eef01d12ee58881507d9"
|
||||
dependencies = [
|
||||
"phf",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "dyn-clone"
|
||||
version = "1.0.20"
|
||||
@@ -1843,16 +1840,16 @@ checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
|
||||
|
||||
[[package]]
|
||||
name = "heed"
|
||||
version = "0.11.0"
|
||||
version = "0.22.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "269c7486ed6def5d7b59a427cec3e87b4d4dd4381d01e21c8c9f2d3985688392"
|
||||
checksum = "6a56c94661ddfb51aa9cdfbf102cfcc340aa69267f95ebccc4af08d7c530d393"
|
||||
dependencies = [
|
||||
"bytemuck",
|
||||
"bitflags 2.10.0",
|
||||
"byteorder",
|
||||
"heed-traits",
|
||||
"heed-types",
|
||||
"libc",
|
||||
"lmdb-rkv-sys",
|
||||
"lmdb-master-sys",
|
||||
"once_cell",
|
||||
"page_size",
|
||||
"synchronoise",
|
||||
@@ -1861,22 +1858,18 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "heed-traits"
|
||||
version = "0.8.0"
|
||||
version = "0.20.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a53a94e5b2fd60417e83ffdfe136c39afacff0d4ac1d8d01cd66928ac610e1a2"
|
||||
checksum = "eb3130048d404c57ce5a1ac61a903696e8fcde7e8c2991e9fcfc1f27c3ef74ff"
|
||||
|
||||
[[package]]
|
||||
name = "heed-types"
|
||||
version = "0.8.0"
|
||||
version = "0.21.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9a6cf0a6952fcedc992602d5cddd1e3fff091fbe87d38636e3ec23a31f32acbd"
|
||||
checksum = "13c255bdf46e07fb840d120a36dcc81f385140d7191c76a7391672675c01a55d"
|
||||
dependencies = [
|
||||
"bincode",
|
||||
"bytemuck",
|
||||
"byteorder",
|
||||
"heed-traits",
|
||||
"serde",
|
||||
"serde_json",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -2626,14 +2619,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6373607a59f0be73a39b6fe456b8192fcc3585f602af20751600e974dd455e77"
|
||||
|
||||
[[package]]
|
||||
name = "lmdb-rkv-sys"
|
||||
version = "0.11.2"
|
||||
name = "lmdb-master-sys"
|
||||
version = "0.2.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "61b9ce6b3be08acefa3003c57b7565377432a89ec24476bbe72e11d101f852fe"
|
||||
checksum = "864808e0b19fb6dd3b70ba94ee671b82fce17554cf80aeb0a155c65bb08027df"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"doxygen-rs",
|
||||
"libc",
|
||||
"pkg-config",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -3026,9 +3019,9 @@ checksum = "1a80800c0488c3a21695ea981a54918fbb37abf04f4d0720c453632255e2ff0e"
|
||||
|
||||
[[package]]
|
||||
name = "page_size"
|
||||
version = "0.4.2"
|
||||
version = "0.6.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "eebde548fbbf1ea81a99b128872779c437752fb99f217c45245e1a61dcd9edcd"
|
||||
checksum = "30d5b2194ed13191c1999ae0704b7839fb18384fa22e49b57eeaa97d79ce40da"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"winapi",
|
||||
@@ -3172,6 +3165,48 @@ dependencies = [
|
||||
"indexmap 2.13.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "phf"
|
||||
version = "0.11.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1fd6780a80ae0c52cc120a26a1a42c1ae51b247a253e4e06113d23d2c2edd078"
|
||||
dependencies = [
|
||||
"phf_macros",
|
||||
"phf_shared",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "phf_generator"
|
||||
version = "0.11.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3c80231409c20246a13fddb31776fb942c38553c51e871f8cbd687a4cfb5843d"
|
||||
dependencies = [
|
||||
"phf_shared",
|
||||
"rand",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "phf_macros"
|
||||
version = "0.11.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f84ac04429c13a7ff43785d75ad27569f2951ce0ffd30a3321230db2fc727216"
|
||||
dependencies = [
|
||||
"phf_generator",
|
||||
"phf_shared",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.114",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "phf_shared"
|
||||
version = "0.11.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "67eabc2ef2a60eb7faa00097bd1ffdb5bd28e62bf39990626a582201b7a754e5"
|
||||
dependencies = [
|
||||
"siphasher",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pin-project"
|
||||
version = "1.1.10"
|
||||
@@ -4068,6 +4103,12 @@ dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "siphasher"
|
||||
version = "1.0.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b2aa850e253778c88a04c3d7323b043aeda9d3e30d5971937c1855769763678e"
|
||||
|
||||
[[package]]
|
||||
name = "slab"
|
||||
version = "0.4.11"
|
||||
|
||||
+1
-1
@@ -86,7 +86,7 @@ tracing = "0.1"
|
||||
tracing-journald = "0.3.1"
|
||||
tracing-subscriber = { version = "0.3", features = ["env-filter"] }
|
||||
|
||||
heed = { version = "0.11", default-features = false, features = ["lmdb"] }
|
||||
heed = { version = "0.22", default-features = false, features = [] }
|
||||
rusqlite = "0.37"
|
||||
r2d2 = "0.8"
|
||||
r2d2_sqlite = "0.31"
|
||||
|
||||
+30
-22
@@ -7,8 +7,10 @@ use std::path::{Path, PathBuf};
|
||||
use std::pin::Pin;
|
||||
use std::sync::{Arc, RwLock};
|
||||
|
||||
use heed::types::ByteSlice;
|
||||
use heed::{BytesDecode, Env, RoTxn, RwTxn, UntypedDatabase as Database};
|
||||
use heed::types::Bytes;
|
||||
use heed::{BytesDecode, Env, EnvFlags, RoTxn, RwTxn, WithTls};
|
||||
|
||||
type Database = heed::Database<Bytes, Bytes>;
|
||||
|
||||
use crate::{
|
||||
open::{Engine, OpenOpt},
|
||||
@@ -37,14 +39,15 @@ pub(crate) fn open_db(path: &PathBuf, opt: &OpenOpt) -> Result<Db> {
|
||||
env_builder.max_dbs(100);
|
||||
env_builder.map_size(map_size);
|
||||
env_builder.max_readers(2048);
|
||||
unsafe {
|
||||
env_builder.flag(heed::flags::Flags::MdbNoRdAhead);
|
||||
env_builder.flag(heed::flags::Flags::MdbNoMetaSync);
|
||||
if !opt.fsync {
|
||||
env_builder.flag(heed::flags::Flags::MdbNoSync);
|
||||
}
|
||||
let mut env_flags = EnvFlags::NO_READ_AHEAD | EnvFlags::NO_META_SYNC;
|
||||
if !opt.fsync {
|
||||
env_flags |= EnvFlags::NO_SYNC;
|
||||
}
|
||||
match env_builder.open(path) {
|
||||
let open_res = unsafe {
|
||||
env_builder.flags(env_flags);
|
||||
env_builder.open(path)
|
||||
};
|
||||
match open_res {
|
||||
Err(heed::Error::Io(e)) if e.kind() == std::io::ErrorKind::OutOfMemory => Err(Error(
|
||||
"OutOfMemory error while trying to open LMDB database. This can happen \
|
||||
if your operating system is not allowing you to use sufficient virtual \
|
||||
@@ -109,7 +112,9 @@ impl IDb for LmdbDb {
|
||||
if let Some(i) = trees.1.get(name) {
|
||||
Ok(*i)
|
||||
} else {
|
||||
let tree = self.db.create_database(Some(name))?;
|
||||
let mut wtxn = self.db.write_txn()?;
|
||||
let tree = self.db.create_database(&mut wtxn, Some(name))?;
|
||||
wtxn.commit()?;
|
||||
let i = trees.0.len();
|
||||
trees.0.push(tree);
|
||||
trees.1.insert(name.to_string(), i);
|
||||
@@ -118,29 +123,32 @@ impl IDb for LmdbDb {
|
||||
}
|
||||
|
||||
fn list_trees(&self) -> Result<Vec<String>> {
|
||||
let tree0 = match self.db.open_database::<heed::types::Str, ByteSlice>(None)? {
|
||||
let rtxn = self.db.read_txn()?;
|
||||
let tree0 = match self
|
||||
.db
|
||||
.open_database::<heed::types::Str, Bytes>(&rtxn, None)?
|
||||
{
|
||||
Some(x) => x,
|
||||
None => return Ok(vec![]),
|
||||
};
|
||||
|
||||
let mut ret = vec![];
|
||||
let tx = self.db.read_txn()?;
|
||||
for item in tree0.iter(&tx)? {
|
||||
for item in tree0.iter(&rtxn)? {
|
||||
let (tree_name, _) = item?;
|
||||
ret.push(tree_name.to_string());
|
||||
}
|
||||
drop(tx);
|
||||
|
||||
let mut ret2 = vec![];
|
||||
for tree_name in ret {
|
||||
if self
|
||||
.db
|
||||
.open_database::<ByteSlice, ByteSlice>(Some(&tree_name))?
|
||||
.open_database::<Bytes, Bytes>(&rtxn, Some(&tree_name))?
|
||||
.is_some()
|
||||
{
|
||||
ret2.push(tree_name);
|
||||
}
|
||||
}
|
||||
drop(rtxn);
|
||||
|
||||
Ok(ret2)
|
||||
}
|
||||
@@ -258,11 +266,11 @@ impl IDb for LmdbDb {
|
||||
Ok(on_commit)
|
||||
}
|
||||
TxFnResult::Abort => {
|
||||
tx.tx.abort().map_err(Error::from).map_err(TxError::Db)?;
|
||||
tx.tx.abort();
|
||||
Err(TxError::Abort(()))
|
||||
}
|
||||
TxFnResult::DbErr => {
|
||||
tx.tx.abort().map_err(Error::from).map_err(TxError::Db)?;
|
||||
tx.tx.abort();
|
||||
Err(TxError::Db(Error(
|
||||
"(this message will be discarded)".into(),
|
||||
)))
|
||||
@@ -275,7 +283,7 @@ impl IDb for LmdbDb {
|
||||
|
||||
struct LmdbTx<'a> {
|
||||
trees: &'a [Database],
|
||||
tx: RwTxn<'a, 'a>,
|
||||
tx: RwTxn<'a>,
|
||||
}
|
||||
|
||||
impl<'a> LmdbTx<'a> {
|
||||
@@ -355,15 +363,15 @@ impl<'a> ITx for LmdbTx<'a> {
|
||||
// therefore a bit of unsafe code (it is a self-referential struct)
|
||||
|
||||
type IteratorItem<'a> = heed::Result<(
|
||||
<ByteSlice as BytesDecode<'a>>::DItem,
|
||||
<ByteSlice as BytesDecode<'a>>::DItem,
|
||||
<Bytes as BytesDecode<'a>>::DItem,
|
||||
<Bytes as BytesDecode<'a>>::DItem,
|
||||
)>;
|
||||
|
||||
struct TxAndIterator<'a, I>
|
||||
where
|
||||
I: Iterator<Item = IteratorItem<'a>> + 'a,
|
||||
{
|
||||
tx: RoTxn<'a>,
|
||||
tx: RoTxn<'a, WithTls>,
|
||||
iter: Option<I>,
|
||||
_pin: PhantomPinned,
|
||||
}
|
||||
@@ -378,7 +386,7 @@ where
|
||||
}
|
||||
|
||||
/// Safety: iterfun must not store its argument anywhere but in its result.
|
||||
unsafe fn make<F>(tx: RoTxn<'a>, iterfun: F) -> Result<ValueIter<'a>>
|
||||
unsafe fn make<F>(tx: RoTxn<'a, WithTls>, iterfun: F) -> Result<ValueIter<'a>>
|
||||
where
|
||||
F: FnOnce(&'a RoTxn<'a>) -> Result<I>,
|
||||
{
|
||||
|
||||
+6
-4
@@ -130,10 +130,12 @@ fn test_lmdb_db() {
|
||||
use crate::lmdb_adapter::LmdbDb;
|
||||
|
||||
let path = mktemp::Temp::new_dir().unwrap();
|
||||
let db = heed::EnvOpenOptions::new()
|
||||
.max_dbs(100)
|
||||
.open(&path)
|
||||
.unwrap();
|
||||
let db = unsafe {
|
||||
heed::EnvOpenOptions::new()
|
||||
.max_dbs(100)
|
||||
.open(&path)
|
||||
.unwrap()
|
||||
};
|
||||
let db = LmdbDb::init(db);
|
||||
test_suite(db);
|
||||
drop(path);
|
||||
|
||||
Reference in New Issue
Block a user