diff --git a/src/table/data.rs b/src/table/data.rs index 0cd5a05a..82c1575d 100644 --- a/src/table/data.rs +++ b/src/table/data.rs @@ -21,9 +21,9 @@ use crate::replication::*; use crate::schema::*; use crate::util::*; -pub(crate) const MERKLE_SLEEP_INITIAL: Duration = Duration::from_micros(100); -pub(crate) const MERKLE_SLEEP_MAX: Duration = Duration::from_secs(30); -pub(crate) const MERKLE_SLEEP_ADD_DECREASE: Duration = Duration::from_micros(100); +pub(crate) const MERKLE_SLEEP_INITIAL: Duration = Duration::from_micros(10); +pub(crate) const MERKLE_SLEEP_MAX: Duration = Duration::from_secs(2); +pub(crate) const MERKLE_SLEEP_ADD_DECREASE: Duration = Duration::from_micros(10); pub(crate) const MERKLE_SLEEP_MULT_INCREASE: f32 = 1.2; pub struct TableData { @@ -75,6 +75,7 @@ impl TableData { store.clone(), merkle_tree.clone(), merkle_todo.clone(), + merkle_todo_sleep.clone(), gc_todo.clone(), ); diff --git a/src/table/metrics.rs b/src/table/metrics.rs index 7bb0959a..aad962fd 100644 --- a/src/table/metrics.rs +++ b/src/table/metrics.rs @@ -1,4 +1,5 @@ use opentelemetry::{global, metrics::*, KeyValue}; +use std::sync::{Arc, Mutex}; use garage_db as db; @@ -7,6 +8,7 @@ pub struct TableMetrics { pub(crate) _table_size: ValueObserver, pub(crate) _merkle_tree_size: ValueObserver, pub(crate) _merkle_todo_len: ValueObserver, + pub(crate) _merkle_todo_sleep_ms: ValueObserver, pub(crate) _gc_todo_len: ValueObserver, pub(crate) get_request_counter: BoundCounter, @@ -26,6 +28,7 @@ impl TableMetrics { store: db::Tree, merkle_tree: db::Tree, merkle_todo: db::Tree, + merkle_todo_sleep: Arc>, gc_todo: db::Tree, ) -> Self { let meter = global::meter(table_name); @@ -72,6 +75,18 @@ impl TableMetrics { ) .with_description("Merkle tree updater TODO queue length") .init(), + _merkle_todo_sleep_ms: meter + .f64_value_observer( + "table.merkle_updater_todo_queue_backpressure_ms", + move |observer| { + let bp_ref = merkle_todo_sleep.clone(); + let bp_val = bp_ref.lock().unwrap(); + let bp_millis: f64 = bp_val.as_micros() as f64 / 1000.0f64; + observer.observe(bp_millis, &[KeyValue::new("table_name", table_name)]) + } + ) + .with_description("Merkle tree updater TODO sleep backpressure to apply in ms") + .init(), _gc_todo_len: meter .u64_value_observer( "table.gc_todo_queue_length",