add error case for layout not ready, and fail earlier in many places

src/rpc/layout/helper.rs

@@ -4,6 +4,7 @@ use std::sync::atomic::{AtomicUsize, Ordering};
 use serde::{Deserialize, Serialize};
 
 use garage_util::data::*;
+use garage_util::error::Error;
 
 use super::*;
 use crate::replication_mode::*;
@@ -145,24 +146,31 @@ impl LayoutHelper {
 	}
 
 	/// Returns the current layout version
-	pub fn current(&self) -> &LayoutVersion {
-		self.inner().current()
+	pub fn current(&self) -> Result<&LayoutVersion, Error> {
+		if !self.is_check_ok {
+			return Err(Error::LayoutNotReady);
+		}
+		Ok(self.inner().current())
 	}
 
 	/// Returns all layout versions currently active in the cluster
-	pub fn versions(&self) -> &[LayoutVersion] {
-		&self.inner().versions
+	pub fn versions(&self) -> Result<&[LayoutVersion], Error> {
+		if !self.is_check_ok {
+			return Err(Error::LayoutNotReady);
+		}
+		Ok(&self.inner().versions)
 	}
 
 	/// Returns the latest layout version for which it is safe to read data from,
 	/// i.e. the version whose version number is sync_map_min
-	pub fn read_version(&self) -> &LayoutVersion {
+	pub fn read_version(&self) -> Result<&LayoutVersion, Error> {
 		let sync_min = self.sync_map_min;
-		self.versions()
+		let versions = self.versions()?;
+		Ok(versions
 			.iter()
 			.find(|x| x.version == sync_min)
-			.or(self.versions().last())
-			.unwrap()
+			.or(versions.last())
+			.unwrap())
 	}
 
 	pub fn is_check_ok(&self) -> bool {
@@ -171,14 +179,20 @@ impl LayoutHelper {
 
 	/// Return all nodes that have a role (gateway or storage)
 	/// in one of the currently active layout versions
-	pub fn all_nodes(&self) -> &[Uuid] {
-		&self.all_nodes
+	pub fn all_nodes(&self) -> Result<&[Uuid], Error> {
+		if !self.is_check_ok {
+			return Err(Error::LayoutNotReady);
+		}
+		Ok(&self.all_nodes)
 	}
 
 	/// Return all nodes that are configured to store data
 	/// in one of the currently active layout versions
-	pub fn all_nongateway_nodes(&self) -> &[Uuid] {
-		&self.all_nongateway_nodes
+	pub fn all_nongateway_nodes(&self) -> Result<&[Uuid], Error> {
+		if !self.is_check_ok {
+			return Err(Error::LayoutNotReady);
+		}
+		Ok(&self.all_nongateway_nodes)
 	}
 
 	pub fn ack_map_min(&self) -> u64 {
@@ -193,7 +207,7 @@ impl LayoutHelper {
 
 	pub fn sync_digest(&self) -> SyncLayoutDigest {
 		SyncLayoutDigest {
-			current: self.current().version,
+			current: self.inner().current().version,
 			ack_map_min: self.ack_map_min(),
 			min_stored: self.inner().min_stored(),
 		}
@@ -201,8 +215,8 @@ impl LayoutHelper {
 
 	pub(crate) fn digest(&self) -> RpcLayoutDigest {
 		RpcLayoutDigest {
-			current_version: self.current().version,
-			active_versions: self.versions().len(),
+			current_version: self.inner().current().version,
+			active_versions: self.inner().versions.len(),
 			trackers_hash: self.trackers_hash,
 			staging_hash: self.staging_hash,
 		}
@@ -246,7 +260,8 @@ impl LayoutHelper {
 
 	pub(crate) fn update_ack_to_max_free(&mut self, local_node_id: Uuid) -> bool {
 		let max_free = self
-			.versions()
+			.inner()
+			.versions
 			.iter()
 			.map(|x| x.version)
 			.skip_while(|v| {
@@ -256,7 +271,7 @@ impl LayoutHelper {
 					.unwrap_or(true)
 			})
 			.next()
-			.unwrap_or(self.current().version);
+			.unwrap_or(self.inner().current().version);
 		let changed = self.update(|layout| {
 			layout
 				.update_trackers

src/rpc/layout/manager.rs

@@ -105,7 +105,7 @@ impl LayoutManager {
 	}
 
 	pub fn add_table(&self, table_name: &'static str) {
-		let first_version = self.layout().versions().first().unwrap().version;
+		let first_version = self.layout().inner().versions.first().unwrap().version;
 
 		self.table_sync_version
 			.lock()
@@ -139,19 +139,20 @@ impl LayoutManager {
 
 	// ---- ACK LOCKING ----
 
-	pub fn write_lock_with<T, F>(self: &Arc<Self>, f: F) -> WriteLock<T>
+	pub fn write_lock_with<T, F>(self: &Arc<Self>, f: F) -> Result<WriteLock<T>, Error>
 	where
-		F: FnOnce(&LayoutHelper) -> T,
+		F: FnOnce(&[LayoutVersion]) -> T,
 	{
 		let layout = self.layout();
-		let version = layout.current().version;
-		let value = f(&layout);
+		let current_version = layout.current()?.version;
+		let versions = layout.versions()?;
+		let value = f(versions);
 		layout
 			.ack_lock
-			.get(&version)
+			.get(&current_version)
 			.unwrap()
 			.fetch_add(1, Ordering::Relaxed);
-		WriteLock::new(version, self, value)
+		Ok(WriteLock::new(current_version, self, value))
 	}
 
 	// ---- INTERNALS ---
@@ -369,7 +370,7 @@ impl<T> Drop for WriteLock<T> {
 		let layout = self.layout_manager.layout(); // acquire read lock
 		if let Some(counter) = layout.ack_lock.get(&self.layout_version) {
 			let prev_lock = counter.fetch_sub(1, Ordering::Relaxed);
-			if prev_lock == 1 && layout.current().version > self.layout_version {
+			if prev_lock == 1 && layout.current().unwrap().version > self.layout_version {
 				drop(layout); // release read lock, write lock will be acquired
 				self.layout_manager.ack_new_version();
 			}

src/rpc/layout/version.rs

@@ -118,15 +118,14 @@ impl LayoutVersion {
 	pub fn nodes_of(&self, position: &Hash) -> impl Iterator<Item = Uuid> + '_ {
 		let data = &self.ring_assignment_data;
 
-		let partition_nodes = if data.len() == self.replication_factor * (1 << PARTITION_BITS) {
-			let partition_idx = self.partition_of(position) as usize;
-			let partition_start = partition_idx * self.replication_factor;
-			let partition_end = (partition_idx + 1) * self.replication_factor;
-			&data[partition_start..partition_end]
-		} else {
-			warn!("Ring not yet ready, read/writes will be lost!");
-			&[]
-		};
+		if data.len() != self.replication_factor * (1 << PARTITION_BITS) {
+			panic!(".nodes_of() called on invalid LayoutVersion (this is a bug)");
+		}
+
+		let partition_idx = self.partition_of(position) as usize;
+		let partition_start = partition_idx * self.replication_factor;
+		let partition_end = (partition_idx + 1) * self.replication_factor;
+		let partition_nodes = &data[partition_start..partition_end];
 
 		partition_nodes
 			.iter()