diff --git a/Cargo.lock b/Cargo.lock index 968126e0..661bb2db 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -610,15 +610,6 @@ version = "1.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" -[[package]] -name = "bincode" -version = "1.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad" -dependencies = [ - "serde", -] - [[package]] name = "bitflags" version = "1.3.2" @@ -630,6 +621,9 @@ name = "bitflags" version = "2.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "812e12b5285cc515a9c72a5c1d3b6d46a19dac5acfef5265968c166106e31dd3" +dependencies = [ + "serde_core", +] [[package]] name = "blake2" @@ -655,12 +649,6 @@ version = "3.19.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5dd9dc738b7a8311c7ade152424974d8115f2cdad61e8dab8dac9f2362298510" -[[package]] -name = "bytemuck" -version = "1.24.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fbdf580320f38b612e485521afda1ee26d10cc9884efaaa750d383e13e3c5f4" - [[package]] name = "byteorder" version = "1.5.0" @@ -1053,6 +1041,15 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c0d05e1c0dbad51b52c38bda7adceef61b9efc2baf04acfe8726a8c4630a6f57" +[[package]] +name = "doxygen-rs" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "415b6ec780d34dcf624666747194393603d0373b7141eef01d12ee58881507d9" +dependencies = [ + "phf", +] + [[package]] name = "dyn-clone" version = "1.0.20" @@ -1843,16 +1840,16 @@ checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" [[package]] name = "heed" -version = "0.11.0" +version = "0.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "269c7486ed6def5d7b59a427cec3e87b4d4dd4381d01e21c8c9f2d3985688392" +checksum = "6a56c94661ddfb51aa9cdfbf102cfcc340aa69267f95ebccc4af08d7c530d393" dependencies = [ - "bytemuck", + "bitflags 2.10.0", "byteorder", "heed-traits", "heed-types", "libc", - "lmdb-rkv-sys", + "lmdb-master-sys", "once_cell", "page_size", "synchronoise", @@ -1861,22 +1858,18 @@ dependencies = [ [[package]] name = "heed-traits" -version = "0.8.0" +version = "0.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a53a94e5b2fd60417e83ffdfe136c39afacff0d4ac1d8d01cd66928ac610e1a2" +checksum = "eb3130048d404c57ce5a1ac61a903696e8fcde7e8c2991e9fcfc1f27c3ef74ff" [[package]] name = "heed-types" -version = "0.8.0" +version = "0.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a6cf0a6952fcedc992602d5cddd1e3fff091fbe87d38636e3ec23a31f32acbd" +checksum = "13c255bdf46e07fb840d120a36dcc81f385140d7191c76a7391672675c01a55d" dependencies = [ - "bincode", - "bytemuck", "byteorder", "heed-traits", - "serde", - "serde_json", ] [[package]] @@ -2626,14 +2619,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6373607a59f0be73a39b6fe456b8192fcc3585f602af20751600e974dd455e77" [[package]] -name = "lmdb-rkv-sys" -version = "0.11.2" +name = "lmdb-master-sys" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61b9ce6b3be08acefa3003c57b7565377432a89ec24476bbe72e11d101f852fe" +checksum = "864808e0b19fb6dd3b70ba94ee671b82fce17554cf80aeb0a155c65bb08027df" dependencies = [ "cc", + "doxygen-rs", "libc", - "pkg-config", ] [[package]] @@ -3026,9 +3019,9 @@ checksum = "1a80800c0488c3a21695ea981a54918fbb37abf04f4d0720c453632255e2ff0e" [[package]] name = "page_size" -version = "0.4.2" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eebde548fbbf1ea81a99b128872779c437752fb99f217c45245e1a61dcd9edcd" +checksum = "30d5b2194ed13191c1999ae0704b7839fb18384fa22e49b57eeaa97d79ce40da" dependencies = [ "libc", "winapi", @@ -3172,6 +3165,48 @@ dependencies = [ "indexmap 2.13.0", ] +[[package]] +name = "phf" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd6780a80ae0c52cc120a26a1a42c1ae51b247a253e4e06113d23d2c2edd078" +dependencies = [ + "phf_macros", + "phf_shared", +] + +[[package]] +name = "phf_generator" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c80231409c20246a13fddb31776fb942c38553c51e871f8cbd687a4cfb5843d" +dependencies = [ + "phf_shared", + "rand", +] + +[[package]] +name = "phf_macros" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f84ac04429c13a7ff43785d75ad27569f2951ce0ffd30a3321230db2fc727216" +dependencies = [ + "phf_generator", + "phf_shared", + "proc-macro2", + "quote", + "syn 2.0.114", +] + +[[package]] +name = "phf_shared" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67eabc2ef2a60eb7faa00097bd1ffdb5bd28e62bf39990626a582201b7a754e5" +dependencies = [ + "siphasher", +] + [[package]] name = "pin-project" version = "1.1.10" @@ -4068,6 +4103,12 @@ dependencies = [ "libc", ] +[[package]] +name = "siphasher" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2aa850e253778c88a04c3d7323b043aeda9d3e30d5971937c1855769763678e" + [[package]] name = "slab" version = "0.4.11" diff --git a/Cargo.toml b/Cargo.toml index 7e4a3f64..e0e2bef3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -86,7 +86,7 @@ tracing = "0.1" tracing-journald = "0.3.1" tracing-subscriber = { version = "0.3", features = ["env-filter"] } -heed = { version = "0.11", default-features = false, features = ["lmdb"] } +heed = { version = "0.22", default-features = false, features = [] } rusqlite = "0.37" r2d2 = "0.8" r2d2_sqlite = "0.31" diff --git a/src/db/lmdb_adapter.rs b/src/db/lmdb_adapter.rs index dca121fd..c5360d04 100644 --- a/src/db/lmdb_adapter.rs +++ b/src/db/lmdb_adapter.rs @@ -7,8 +7,10 @@ use std::path::{Path, PathBuf}; use std::pin::Pin; use std::sync::{Arc, RwLock}; -use heed::types::ByteSlice; -use heed::{BytesDecode, Env, RoTxn, RwTxn, UntypedDatabase as Database}; +use heed::types::Bytes; +use heed::{BytesDecode, Env, EnvFlags, RoTxn, RwTxn, WithTls}; + +type Database = heed::Database; use crate::{ open::{Engine, OpenOpt}, @@ -37,14 +39,15 @@ pub(crate) fn open_db(path: &PathBuf, opt: &OpenOpt) -> Result { env_builder.max_dbs(100); env_builder.map_size(map_size); env_builder.max_readers(2048); - unsafe { - env_builder.flag(heed::flags::Flags::MdbNoRdAhead); - env_builder.flag(heed::flags::Flags::MdbNoMetaSync); - if !opt.fsync { - env_builder.flag(heed::flags::Flags::MdbNoSync); - } + let mut env_flags = EnvFlags::NO_READ_AHEAD | EnvFlags::NO_META_SYNC; + if !opt.fsync { + env_flags |= EnvFlags::NO_SYNC; } - match env_builder.open(path) { + let open_res = unsafe { + env_builder.flags(env_flags); + env_builder.open(path) + }; + match open_res { Err(heed::Error::Io(e)) if e.kind() == std::io::ErrorKind::OutOfMemory => Err(Error( "OutOfMemory error while trying to open LMDB database. This can happen \ if your operating system is not allowing you to use sufficient virtual \ @@ -109,7 +112,9 @@ impl IDb for LmdbDb { if let Some(i) = trees.1.get(name) { Ok(*i) } else { - let tree = self.db.create_database(Some(name))?; + let mut wtxn = self.db.write_txn()?; + let tree = self.db.create_database(&mut wtxn, Some(name))?; + wtxn.commit()?; let i = trees.0.len(); trees.0.push(tree); trees.1.insert(name.to_string(), i); @@ -118,29 +123,32 @@ impl IDb for LmdbDb { } fn list_trees(&self) -> Result> { - let tree0 = match self.db.open_database::(None)? { + let rtxn = self.db.read_txn()?; + let tree0 = match self + .db + .open_database::(&rtxn, None)? + { Some(x) => x, None => return Ok(vec![]), }; let mut ret = vec![]; - let tx = self.db.read_txn()?; - for item in tree0.iter(&tx)? { + for item in tree0.iter(&rtxn)? { let (tree_name, _) = item?; ret.push(tree_name.to_string()); } - drop(tx); let mut ret2 = vec![]; for tree_name in ret { if self .db - .open_database::(Some(&tree_name))? + .open_database::(&rtxn, Some(&tree_name))? .is_some() { ret2.push(tree_name); } } + drop(rtxn); Ok(ret2) } @@ -258,11 +266,11 @@ impl IDb for LmdbDb { Ok(on_commit) } TxFnResult::Abort => { - tx.tx.abort().map_err(Error::from).map_err(TxError::Db)?; + tx.tx.abort(); Err(TxError::Abort(())) } TxFnResult::DbErr => { - tx.tx.abort().map_err(Error::from).map_err(TxError::Db)?; + tx.tx.abort(); Err(TxError::Db(Error( "(this message will be discarded)".into(), ))) @@ -275,7 +283,7 @@ impl IDb for LmdbDb { struct LmdbTx<'a> { trees: &'a [Database], - tx: RwTxn<'a, 'a>, + tx: RwTxn<'a>, } impl<'a> LmdbTx<'a> { @@ -355,15 +363,15 @@ impl<'a> ITx for LmdbTx<'a> { // therefore a bit of unsafe code (it is a self-referential struct) type IteratorItem<'a> = heed::Result<( - >::DItem, - >::DItem, + >::DItem, + >::DItem, )>; struct TxAndIterator<'a, I> where I: Iterator> + 'a, { - tx: RoTxn<'a>, + tx: RoTxn<'a, WithTls>, iter: Option, _pin: PhantomPinned, } @@ -378,7 +386,7 @@ where } /// Safety: iterfun must not store its argument anywhere but in its result. - unsafe fn make(tx: RoTxn<'a>, iterfun: F) -> Result> + unsafe fn make(tx: RoTxn<'a, WithTls>, iterfun: F) -> Result> where F: FnOnce(&'a RoTxn<'a>) -> Result, { diff --git a/src/db/test.rs b/src/db/test.rs index 91ac3522..f5f62b30 100644 --- a/src/db/test.rs +++ b/src/db/test.rs @@ -130,10 +130,12 @@ fn test_lmdb_db() { use crate::lmdb_adapter::LmdbDb; let path = mktemp::Temp::new_dir().unwrap(); - let db = heed::EnvOpenOptions::new() - .max_dbs(100) - .open(&path) - .unwrap(); + let db = unsafe { + heed::EnvOpenOptions::new() + .max_dbs(100) + .open(&path) + .unwrap() + }; let db = LmdbDb::init(db); test_suite(db); drop(path);