Merge commit 'ec12d6c' into next

src/api/admin/cluster.rs

@@ -91,7 +91,7 @@ fn get_cluster_layout(garage: &Arc<Garage>) -> GetClusterLayoutResponse {
 			.map(|(k, _, v)| (hex::encode(k), v.0.clone()))
 			.collect(),
 		staged_role_changes: layout
-			.staging
+			.staging_roles
 			.items()
 			.iter()
 			.filter(|(k, _, v)| layout.roles.get(k) != Some(v))
@@ -142,14 +142,14 @@ pub async fn handle_update_cluster_layout(
 	let mut layout = garage.system.get_cluster_layout();
 
 	let mut roles = layout.roles.clone();
-	roles.merge(&layout.staging);
+	roles.merge(&layout.staging_roles);
 
 	for (node, role) in updates {
 		let node = hex::decode(node).ok_or_bad_request("Invalid node identifier")?;
 		let node = Uuid::try_from(&node).ok_or_bad_request("Invalid node identifier")?;
 
 		layout
-			.staging
+			.staging_roles
 			.merge(&roles.update_mutator(node, NodeRoleV(role)));
 	}
 
@@ -167,12 +167,14 @@ pub async fn handle_apply_cluster_layout(
 	let param = parse_json_body::<ApplyRevertLayoutRequest>(req).await?;
 
 	let layout = garage.system.get_cluster_layout();
-	let layout = layout.apply_staged_changes(Some(param.version))?;
+	let (layout, msg) = layout.apply_staged_changes(Some(param.version))?;
+
 	garage.system.update_cluster_layout(&layout).await?;
 
 	Ok(Response::builder()
-		.status(StatusCode::NO_CONTENT)
-		.body(Body::empty())?)
+		.status(StatusCode::OK)
+		.header(http::header::CONTENT_TYPE, "text/plain")
+		.body(Body::from(msg.join("\n")))?)
 }
 
 pub async fn handle_revert_cluster_layout(

src/db/lib.rs

@@ -2,9 +2,6 @@
 #[cfg(feature = "sqlite")]
 extern crate tracing;
 
-#[cfg(not(any(feature = "lmdb", feature = "sled", feature = "sqlite")))]
-compile_error!("Must activate the Cargo feature for at least one DB engine: lmdb, sled or sqlite.");
-
 #[cfg(feature = "lmdb")]
 pub mod lmdb_adapter;
 #[cfg(feature = "sled")]

src/garage/cli/cmd.rs

@@ -71,7 +71,7 @@ pub async fn cmd_status(rpc_cli: &Endpoint<SystemRpc, ()>, rpc_host: NodeID) ->
 				));
 			}
 			_ => {
-				let new_role = match layout.staging.get(&adv.id) {
+				let new_role = match layout.staging_roles.get(&adv.id) {
 					Some(NodeRoleV(Some(_))) => "(pending)",
 					_ => "NO ROLE ASSIGNED",
 				};

src/garage/cli/layout.rs

@@ -1,3 +1,5 @@
+use bytesize::ByteSize;
+
 use garage_util::crdt::Crdt;
 use garage_util::error::*;
 use garage_util::formater::format_table;
@@ -14,8 +16,8 @@ pub async fn cli_layout_command_dispatch(
 	rpc_host: NodeID,
 ) -> Result<(), Error> {
 	match cmd {
-		LayoutOperation::Assign(configure_opt) => {
-			cmd_assign_role(system_rpc_endpoint, rpc_host, configure_opt).await
+		LayoutOperation::Assign(assign_opt) => {
+			cmd_assign_role(system_rpc_endpoint, rpc_host, assign_opt).await
 		}
 		LayoutOperation::Remove(remove_opt) => {
 			cmd_remove_role(system_rpc_endpoint, rpc_host, remove_opt).await
@@ -27,6 +29,9 @@ pub async fn cli_layout_command_dispatch(
 		LayoutOperation::Revert(revert_opt) => {
 			cmd_revert_layout(system_rpc_endpoint, rpc_host, revert_opt).await
 		}
+		LayoutOperation::Config(config_opt) => {
+			cmd_config_layout(system_rpc_endpoint, rpc_host, config_opt).await
+		}
 	}
 }
 
@@ -60,14 +65,14 @@ pub async fn cmd_assign_role(
 		.collect::<Result<Vec<_>, _>>()?;
 
 	let mut roles = layout.roles.clone();
-	roles.merge(&layout.staging);
+	roles.merge(&layout.staging_roles);
 
 	for replaced in args.replace.iter() {
 		let replaced_node = find_matching_node(layout.node_ids().iter().cloned(), replaced)?;
 		match roles.get(&replaced_node) {
 			Some(NodeRoleV(Some(_))) => {
 				layout
-					.staging
+					.staging_roles
 					.merge(&roles.update_mutator(replaced_node, NodeRoleV(None)));
 			}
 			_ => {
@@ -83,7 +88,7 @@ pub async fn cmd_assign_role(
 		return Err(Error::Message(
 				"-c and -g are mutually exclusive, please configure node either with c>0 to act as a storage node or with -g to act as a gateway node".into()));
 	}
-	if args.capacity == Some(0) {
+	if args.capacity == Some(ByteSize::b(0)) {
 		return Err(Error::Message("Invalid capacity value: 0".into()));
 	}
 
@@ -91,7 +96,7 @@ pub async fn cmd_assign_role(
 		let new_entry = match roles.get(&added_node) {
 			Some(NodeRoleV(Some(old))) => {
 				let capacity = match args.capacity {
-					Some(c) => Some(c),
+					Some(c) => Some(c.as_u64()),
 					None if args.gateway => None,
 					None => old.capacity,
 				};
@@ -108,7 +113,7 @@ pub async fn cmd_assign_role(
 			}
 			_ => {
 				let capacity = match args.capacity {
-					Some(c) => Some(c),
+					Some(c) => Some(c.as_u64()),
 					None if args.gateway => None,
 					None => return Err(Error::Message(
 							"Please specify a capacity with the -c flag, or set node explicitly as gateway with -g".into())),
@@ -125,7 +130,7 @@ pub async fn cmd_assign_role(
 		};
 
 		layout
-			.staging
+			.staging_roles
 			.merge(&roles.update_mutator(added_node, NodeRoleV(Some(new_entry))));
 	}
 
@@ -145,13 +150,13 @@ pub async fn cmd_remove_role(
 	let mut layout = fetch_layout(rpc_cli, rpc_host).await?;
 
 	let mut roles = layout.roles.clone();
-	roles.merge(&layout.staging);
+	roles.merge(&layout.staging_roles);
 
 	let deleted_node =
 		find_matching_node(roles.items().iter().map(|(id, _, _)| *id), &args.node_id)?;
 
 	layout
-		.staging
+		.staging_roles
 		.merge(&roles.update_mutator(deleted_node, NodeRoleV(None)));
 
 	send_layout(rpc_cli, rpc_host, layout).await?;
@@ -166,7 +171,7 @@ pub async fn cmd_show_layout(
 	rpc_cli: &Endpoint<SystemRpc, ()>,
 	rpc_host: NodeID,
 ) -> Result<(), Error> {
-	let mut layout = fetch_layout(rpc_cli, rpc_host).await?;
+	let layout = fetch_layout(rpc_cli, rpc_host).await?;
 
 	println!("==== CURRENT CLUSTER LAYOUT ====");
 	if !print_cluster_layout(&layout) {
@@ -176,30 +181,41 @@ pub async fn cmd_show_layout(
 	println!();
 	println!("Current cluster layout version: {}", layout.version);
 
-	if print_staging_role_changes(&layout) {
-		layout.roles.merge(&layout.staging);
-
-		println!();
-		println!("==== NEW CLUSTER LAYOUT AFTER APPLYING CHANGES ====");
-		if !print_cluster_layout(&layout) {
-			println!("No nodes have a role in the new layout.");
-		}
-		println!();
+	let has_role_changes = print_staging_role_changes(&layout);
+	let has_param_changes = print_staging_parameters_changes(&layout);
+	if has_role_changes || has_param_changes {
+		let v = layout.version;
+		let res_apply = layout.apply_staged_changes(Some(v + 1));
 
 		// this will print the stats of what partitions
 		// will move around when we apply
-		if layout.calculate_partition_assignation() {
-			println!("To enact the staged role changes, type:");
-			println!();
-			println!("    garage layout apply --version {}", layout.version + 1);
-			println!();
-			println!(
-				"You can also revert all proposed changes with: garage layout revert --version {}",
-				layout.version + 1
-			);
-		} else {
-			println!("Not enough nodes have an assigned role to maintain enough copies of data.");
-			println!("This new layout cannot yet be applied.");
+		match res_apply {
+			Ok((layout, msg)) => {
+				println!();
+				println!("==== NEW CLUSTER LAYOUT AFTER APPLYING CHANGES ====");
+				if !print_cluster_layout(&layout) {
+					println!("No nodes have a role in the new layout.");
+				}
+				println!();
+
+				for line in msg.iter() {
+					println!("{}", line);
+				}
+				println!("To enact the staged role changes, type:");
+				println!();
+				println!("    garage layout apply --version {}", v + 1);
+				println!();
+				println!(
+                    "You can also revert all proposed changes with: garage layout revert --version {}",
+                    v + 1)
+			}
+			Err(e) => {
+				println!("Error while trying to compute the assignation: {}", e);
+				println!("This new layout cannot yet be applied.");
+				println!(
+                    "You can also revert all proposed changes with: garage layout revert --version {}",
+                    v + 1)
+			}
 		}
 	}
 
@@ -213,7 +229,10 @@ pub async fn cmd_apply_layout(
 ) -> Result<(), Error> {
 	let layout = fetch_layout(rpc_cli, rpc_host).await?;
 
-	let layout = layout.apply_staged_changes(apply_opt.version)?;
+	let (layout, msg) = layout.apply_staged_changes(apply_opt.version)?;
+	for line in msg.iter() {
+		println!("{}", line);
+	}
 
 	send_layout(rpc_cli, rpc_host, layout).await?;
 
@@ -238,6 +257,45 @@ pub async fn cmd_revert_layout(
 	Ok(())
 }
 
+pub async fn cmd_config_layout(
+	rpc_cli: &Endpoint<SystemRpc, ()>,
+	rpc_host: NodeID,
+	config_opt: ConfigLayoutOpt,
+) -> Result<(), Error> {
+	let mut layout = fetch_layout(rpc_cli, rpc_host).await?;
+
+	let mut did_something = false;
+	match config_opt.redundancy {
+		None => (),
+		Some(r) => {
+			if r > layout.replication_factor {
+				println!(
+					"The zone redundancy must be smaller or equal to the \
+                replication factor ({}).",
+					layout.replication_factor
+				);
+			} else if r < 1 {
+				println!("The zone redundancy must be at least 1.");
+			} else {
+				layout
+					.staging_parameters
+					.update(LayoutParameters { zone_redundancy: r });
+				println!("The new zone redundancy has been saved ({}).", r);
+			}
+			did_something = true;
+		}
+	}
+
+	if !did_something {
+		return Err(Error::Message(
+			"Please specify an action for `garage layout config` to do".into(),
+		));
+	}
+
+	send_layout(rpc_cli, rpc_host, layout).await?;
+	Ok(())
+}
+
 // --- utility ---
 
 pub async fn fetch_layout(
@@ -269,21 +327,39 @@ pub async fn send_layout(
 }
 
 pub fn print_cluster_layout(layout: &ClusterLayout) -> bool {
-	let mut table = vec!["ID\tTags\tZone\tCapacity".to_string()];
+	let mut table = vec!["ID\tTags\tZone\tCapacity\tUsable capacity".to_string()];
 	for (id, _, role) in layout.roles.items().iter() {
 		let role = match &role.0 {
 			Some(r) => r,
 			_ => continue,
 		};
 		let tags = role.tags.join(",");
-		table.push(format!(
-			"{:?}\t{}\t{}\t{}",
-			id,
-			tags,
-			role.zone,
-			role.capacity_string()
-		));
+		let usage = layout.get_node_usage(id).unwrap_or(0);
+		let capacity = layout.get_node_capacity(id).unwrap_or(0);
+		if capacity > 0 {
+			table.push(format!(
+				"{:?}\t{}\t{}\t{}\t{} ({:.1}%)",
+				id,
+				tags,
+				role.zone,
+				role.capacity_string(),
+				ByteSize::b(usage as u64 * layout.partition_size).to_string_as(false),
+				(100.0 * usage as f32 * layout.partition_size as f32) / (capacity as f32)
+			));
+		} else {
+			table.push(format!(
+				"{:?}\t{}\t{}\t{}",
+				id,
+				tags,
+				role.zone,
+				role.capacity_string()
+			));
+		};
 	}
+	println!();
+	println!("Parameters of the layout computation:");
+	println!("Zone redundancy: {}", layout.parameters.zone_redundancy);
+	println!();
 	if table.len() == 1 {
 		false
 	} else {
@@ -292,9 +368,23 @@ pub fn print_cluster_layout(layout: &ClusterLayout) -> bool {
 	}
 }
 
+pub fn print_staging_parameters_changes(layout: &ClusterLayout) -> bool {
+	let has_changes = *layout.staging_parameters.get() != layout.parameters;
+	if has_changes {
+		println!();
+		println!("==== NEW LAYOUT PARAMETERS ====");
+		println!(
+			"Zone redundancy: {}",
+			layout.staging_parameters.get().zone_redundancy
+		);
+		println!();
+	}
+	has_changes
+}
+
 pub fn print_staging_role_changes(layout: &ClusterLayout) -> bool {
 	let has_changes = layout
-		.staging
+		.staging_roles
 		.items()
 		.iter()
 		.any(|(k, _, v)| layout.roles.get(k) != Some(v));
@@ -303,7 +393,7 @@ pub fn print_staging_role_changes(layout: &ClusterLayout) -> bool {
 		println!();
 		println!("==== STAGED ROLE CHANGES ====");
 		let mut table = vec!["ID\tTags\tZone\tCapacity".to_string()];
-		for (id, _, role) in layout.staging.items().iter() {
+		for (id, _, role) in layout.staging_roles.items().iter() {
 			if layout.roles.get(id) == Some(role) {
 				continue;
 			}

src/garage/cli/structs.rs

@@ -87,6 +87,10 @@ pub enum LayoutOperation {
 	#[structopt(name = "remove", version = garage_version())]
 	Remove(RemoveRoleOpt),
 
+	/// Configure parameters value for the layout computation
+	#[structopt(name = "config", version = garage_version())]
+	Config(ConfigLayoutOpt),
+
 	/// Show roles currently assigned to nodes and changes staged for commit
 	#[structopt(name = "show", version = garage_version())]
 	Show,
@@ -110,9 +114,9 @@ pub struct AssignRoleOpt {
 	#[structopt(short = "z", long = "zone")]
 	pub(crate) zone: Option<String>,
 
-	/// Capacity (in relative terms, use 1 to represent your smallest server)
+	/// Storage capacity, in bytes (supported suffixes: B, KB, MB, GB, TB, PB)
 	#[structopt(short = "c", long = "capacity")]
-	pub(crate) capacity: Option<u32>,
+	pub(crate) capacity: Option<bytesize::ByteSize>,
 
 	/// Gateway-only node
 	#[structopt(short = "g", long = "gateway")]
@@ -133,6 +137,13 @@ pub struct RemoveRoleOpt {
 	pub(crate) node_id: String,
 }
 
+#[derive(StructOpt, Debug)]
+pub struct ConfigLayoutOpt {
+	/// Zone redundancy parameter
+	#[structopt(short = "r", long = "redundancy")]
+	pub(crate) redundancy: Option<usize>,
+}
+
 #[derive(StructOpt, Debug)]
 pub struct ApplyLayoutOpt {
 	/// Version number of new configuration: this command will fail if

src/garage/main.rs

@@ -17,6 +17,9 @@ compile_error!("Either bundled-libs or system-libs Cargo feature must be enabled
 #[cfg(all(feature = "bundled-libs", feature = "system-libs"))]
 compile_error!("Only one of bundled-libs and system-libs Cargo features must be enabled");
 
+#[cfg(not(any(feature = "lmdb", feature = "sled", feature = "sqlite")))]
+compile_error!("Must activate the Cargo feature for at least one DB engine: lmdb, sled or sqlite.");
+
 use std::net::SocketAddr;
 use std::path::PathBuf;
 

src/garage/tests/common/garage.rs

@@ -126,7 +126,7 @@ api_bind_addr = "127.0.0.1:{admin_port}"
 		self.command()
 			.args(["layout", "assign"])
 			.arg(node_short_id)
-			.args(["-c", "1", "-z", "unzonned"])
+			.args(["-c", "1G", "-z", "unzonned"])
 			.quiet()
 			.expect_success_status("Could not assign garage node layout");
 		self.command()

src/rpc/Cargo.toml

@@ -18,10 +18,12 @@ garage_util = { version = "0.8.0", path = "../util" }
 
 arc-swap = "1.0"
 bytes = "1.0"
+bytesize = "1.1"
 gethostname = "0.2"
 hex = "0.4"
 tracing = "0.1.30"
 rand = "0.8"
+itertools="0.10"
 sodiumoxide = { version = "0.2.5-0", package = "kuska-sodiumoxide" }
 
 async-trait = "0.1.7"

src/rpc/graph_algo.rs

@@ -0,0 +1,411 @@
+//! This module deals with graph algorithms.
+//! It is used in layout.rs to build the partition to node assignation.
+
+use rand::prelude::SliceRandom;
+use std::cmp::{max, min};
+use std::collections::HashMap;
+use std::collections::VecDeque;
+
+/// Vertex data structures used in all the graphs used in layout.rs.
+/// usize parameters correspond to node/zone/partitions ids.
+/// To understand the vertex roles below, please refer to the formal description
+/// of the layout computation algorithm.
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
+pub enum Vertex {
+	Source,
+	Pup(usize),       // The vertex p+ of partition p
+	Pdown(usize),     // The vertex p- of partition p
+	PZ(usize, usize), // The vertex corresponding to x_(partition p, zone z)
+	N(usize),         // The vertex corresponding to node n
+	Sink,
+}
+
+/// Edge data structure for the flow algorithm.
+#[derive(Clone, Copy, Debug)]
+pub struct FlowEdge {
+	cap: u64,    // flow maximal capacity of the edge
+	flow: i64,   // flow value on the edge
+	dest: usize, // destination vertex id
+	rev: usize,  // index of the reversed edge (v, self) in the edge list of vertex v
+}
+
+/// Edge data structure for the detection of negative cycles.
+#[derive(Clone, Copy, Debug)]
+pub struct WeightedEdge {
+	w: i64, // weight of the edge
+	dest: usize,
+}
+
+pub trait Edge: Clone + Copy {}
+impl Edge for FlowEdge {}
+impl Edge for WeightedEdge {}
+
+/// Struct for the graph structure. We do encapsulation here to be able to both
+/// provide user friendly Vertex enum to address vertices, and to use internally usize
+/// indices and Vec instead of HashMap in the graph algorithm to optimize execution speed.
+pub struct Graph<E: Edge> {
+	vertex_to_id: HashMap<Vertex, usize>,
+	id_to_vertex: Vec<Vertex>,
+
+	// The graph is stored as an adjacency list
+	graph: Vec<Vec<E>>,
+}
+
+pub type CostFunction = HashMap<(Vertex, Vertex), i64>;
+
+impl<E: Edge> Graph<E> {
+	pub fn new(vertices: &[Vertex]) -> Self {
+		let mut map = HashMap::<Vertex, usize>::new();
+		for (i, vert) in vertices.iter().enumerate() {
+			map.insert(*vert, i);
+		}
+		Graph::<E> {
+			vertex_to_id: map,
+			id_to_vertex: vertices.to_vec(),
+			graph: vec![Vec::<E>::new(); vertices.len()],
+		}
+	}
+
+	fn get_vertex_id(&self, v: &Vertex) -> Result<usize, String> {
+		self.vertex_to_id
+			.get(v)
+			.cloned()
+			.ok_or_else(|| format!("The graph does not contain vertex {:?}", v))
+	}
+}
+
+impl Graph<FlowEdge> {
+	/// This function adds a directed edge to the graph with capacity c, and the
+	/// corresponding reversed edge with capacity 0.
+	pub fn add_edge(&mut self, u: Vertex, v: Vertex, c: u64) -> Result<(), String> {
+		let idu = self.get_vertex_id(&u)?;
+		let idv = self.get_vertex_id(&v)?;
+		if idu == idv {
+			return Err("Cannot add edge from vertex to itself in flow graph".into());
+		}
+
+		let rev_u = self.graph[idu].len();
+		let rev_v = self.graph[idv].len();
+		self.graph[idu].push(FlowEdge {
+			cap: c,
+			dest: idv,
+			flow: 0,
+			rev: rev_v,
+		});
+		self.graph[idv].push(FlowEdge {
+			cap: 0,
+			dest: idu,
+			flow: 0,
+			rev: rev_u,
+		});
+		Ok(())
+	}
+
+	/// This function returns the list of vertices that receive a positive flow from
+	/// vertex v.
+	pub fn get_positive_flow_from(&self, v: Vertex) -> Result<Vec<Vertex>, String> {
+		let idv = self.get_vertex_id(&v)?;
+		let mut result = Vec::<Vertex>::new();
+		for edge in self.graph[idv].iter() {
+			if edge.flow > 0 {
+				result.push(self.id_to_vertex[edge.dest]);
+			}
+		}
+		Ok(result)
+	}
+
+	/// This function returns the value of the flow incoming to v.
+	pub fn get_inflow(&self, v: Vertex) -> Result<i64, String> {
+		let idv = self.get_vertex_id(&v)?;
+		let mut result = 0;
+		for edge in self.graph[idv].iter() {
+			result += max(0, self.graph[edge.dest][edge.rev].flow);
+		}
+		Ok(result)
+	}
+
+	/// This function returns the value of the flow outgoing from v.
+	pub fn get_outflow(&self, v: Vertex) -> Result<i64, String> {
+		let idv = self.get_vertex_id(&v)?;
+		let mut result = 0;
+		for edge in self.graph[idv].iter() {
+			result += max(0, edge.flow);
+		}
+		Ok(result)
+	}
+
+	/// This function computes the flow total value by computing the outgoing flow
+	/// from the source.
+	pub fn get_flow_value(&mut self) -> Result<i64, String> {
+		self.get_outflow(Vertex::Source)
+	}
+
+	/// This function shuffles the order of the edge lists. It keeps the ids of the
+	/// reversed edges consistent.
+	fn shuffle_edges(&mut self) {
+		let mut rng = rand::thread_rng();
+		for i in 0..self.graph.len() {
+			self.graph[i].shuffle(&mut rng);
+			// We need to update the ids of the reverse edges.
+			for j in 0..self.graph[i].len() {
+				let target_v = self.graph[i][j].dest;
+				let target_rev = self.graph[i][j].rev;
+				self.graph[target_v][target_rev].rev = j;
+			}
+		}
+	}
+
+	/// Computes an upper bound of the flow on the graph
+	pub fn flow_upper_bound(&self) -> Result<u64, String> {
+		let idsource = self.get_vertex_id(&Vertex::Source)?;
+		let mut flow_upper_bound = 0;
+		for edge in self.graph[idsource].iter() {
+			flow_upper_bound += edge.cap;
+		}
+		Ok(flow_upper_bound)
+	}
+
+	/// This function computes the maximal flow using Dinic's algorithm. It starts with
+	/// the flow values already present in the graph. So it is possible to add some edge to
+	/// the graph, compute a flow, add other edges, update the flow.
+	pub fn compute_maximal_flow(&mut self) -> Result<(), String> {
+		let idsource = self.get_vertex_id(&Vertex::Source)?;
+		let idsink = self.get_vertex_id(&Vertex::Sink)?;
+
+		let nb_vertices = self.graph.len();
+
+		let flow_upper_bound = self.flow_upper_bound()?;
+
+		// To ensure the dispersion of the associations generated by the
+		// assignation, we shuffle the neighbours of the nodes. Hence,
+		// the vertices do not consider their neighbours in the same order.
+		self.shuffle_edges();
+
+		// We run Dinic's max flow algorithm
+		loop {
+			// We build the level array from Dinic's algorithm.
+			let mut level = vec![None; nb_vertices];
+
+			let mut fifo = VecDeque::new();
+			fifo.push_back((idsource, 0));
+			while let Some((id, lvl)) = fifo.pop_front() {
+				if level[id] == None {
+					// it means id has not yet been reached
+					level[id] = Some(lvl);
+					for edge in self.graph[id].iter() {
+						if edge.cap as i64 - edge.flow > 0 {
+							fifo.push_back((edge.dest, lvl + 1));
+						}
+					}
+				}
+			}
+			if level[idsink] == None {
+				// There is no residual flow
+				break;
+			}
+			// Now we run DFS respecting the level array
+			let mut next_nbd = vec![0; nb_vertices];
+			let mut lifo = Vec::new();
+
+			lifo.push((idsource, flow_upper_bound));
+
+			while let Some((id, f)) = lifo.last().cloned() {
+				if id == idsink {
+					// The DFS reached the sink, we can add a
+					// residual flow.
+					lifo.pop();
+					while let Some((id, _)) = lifo.pop() {
+						let nbd = next_nbd[id];
+						self.graph[id][nbd].flow += f as i64;
+						let id_rev = self.graph[id][nbd].dest;
+						let nbd_rev = self.graph[id][nbd].rev;
+						self.graph[id_rev][nbd_rev].flow -= f as i64;
+					}
+					lifo.push((idsource, flow_upper_bound));
+					continue;
+				}
+				// else we did not reach the sink
+				let nbd = next_nbd[id];
+				if nbd >= self.graph[id].len() {
+					// There is nothing to explore from id anymore
+					lifo.pop();
+					if let Some((parent, _)) = lifo.last() {
+						next_nbd[*parent] += 1;
+					}
+					continue;
+				}
+				// else we can try to send flow from id to its nbd
+				let new_flow = min(
+					f as i64,
+					self.graph[id][nbd].cap as i64 - self.graph[id][nbd].flow,
+				) as u64;
+				if new_flow == 0 {
+					next_nbd[id] += 1;
+					continue;
+				}
+				if let (Some(lvldest), Some(lvlid)) = (level[self.graph[id][nbd].dest], level[id]) {
+					if lvldest <= lvlid {
+						// We cannot send flow to nbd.
+						next_nbd[id] += 1;
+						continue;
+					}
+				}
+				// otherwise, we send flow to nbd.
+				lifo.push((self.graph[id][nbd].dest, new_flow));
+			}
+		}
+		Ok(())
+	}
+
+	/// This function takes a flow, and a cost function on the edges, and tries to find an
+	/// equivalent flow with a better cost, by finding improving overflow cycles. It uses
+	/// as subroutine the Bellman Ford algorithm run up to path_length.
+	/// We assume that the cost of edge (u,v) is the opposite of the cost of (v,u), and
+	/// only one needs to be present in the cost function.
+	pub fn optimize_flow_with_cost(
+		&mut self,
+		cost: &CostFunction,
+		path_length: usize,
+	) -> Result<(), String> {
+		// We build the weighted graph g where we will look for negative cycle
+		let mut gf = self.build_cost_graph(cost)?;
+		let mut cycles = gf.list_negative_cycles(path_length);
+		while !cycles.is_empty() {
+			// we enumerate negative cycles
+			for c in cycles.iter() {
+				for i in 0..c.len() {
+					// We add one flow unit to the edge (u,v) of cycle c
+					let idu = self.vertex_to_id[&c[i]];
+					let idv = self.vertex_to_id[&c[(i + 1) % c.len()]];
+					for j in 0..self.graph[idu].len() {
+						// since idu appears at most once in the cycles, we enumerate every
+						// edge at most once.
+						let edge = self.graph[idu][j];
+						if edge.dest == idv {
+							self.graph[idu][j].flow += 1;
+							self.graph[idv][edge.rev].flow -= 1;
+							break;
+						}
+					}
+				}
+			}
+
+			gf = self.build_cost_graph(cost)?;
+			cycles = gf.list_negative_cycles(path_length);
+		}
+		Ok(())
+	}
+
+	/// Construct the weighted graph G_f from the flow and the cost function
+	fn build_cost_graph(&self, cost: &CostFunction) -> Result<Graph<WeightedEdge>, String> {
+		let mut g = Graph::<WeightedEdge>::new(&self.id_to_vertex);
+		let nb_vertices = self.id_to_vertex.len();
+		for i in 0..nb_vertices {
+			for edge in self.graph[i].iter() {
+				if edge.cap as i64 - edge.flow > 0 {
+					// It is possible to send overflow through this edge
+					let u = self.id_to_vertex[i];
+					let v = self.id_to_vertex[edge.dest];
+					if cost.contains_key(&(u, v)) {
+						g.add_edge(u, v, cost[&(u, v)])?;
+					} else if cost.contains_key(&(v, u)) {
+						g.add_edge(u, v, -cost[&(v, u)])?;
+					} else {
+						g.add_edge(u, v, 0)?;
+					}
+				}
+			}
+		}
+		Ok(g)
+	}
+}
+
+impl Graph<WeightedEdge> {
+	/// This function adds a single directed weighted edge to the graph.
+	pub fn add_edge(&mut self, u: Vertex, v: Vertex, w: i64) -> Result<(), String> {
+		let idu = self.get_vertex_id(&u)?;
+		let idv = self.get_vertex_id(&v)?;
+		self.graph[idu].push(WeightedEdge { w, dest: idv });
+		Ok(())
+	}
+
+	/// This function lists the negative cycles it manages to find after path_length
+	/// iterations of the main loop of the Bellman-Ford algorithm. For the classical
+	/// algorithm, path_length needs to be equal to the number of vertices. However,
+	/// for particular graph structures like in our case, the algorithm is still correct
+	/// when path_length is the length of the longest possible simple path.
+	/// See the formal description of the algorithm for more details.
+	fn list_negative_cycles(&self, path_length: usize) -> Vec<Vec<Vertex>> {
+		let nb_vertices = self.graph.len();
+
+		// We start with every vertex at distance 0 of some imaginary extra -1 vertex.
+		let mut distance = vec![0; nb_vertices];
+		// The prev vector collects for every vertex from where does the shortest path come
+		let mut prev = vec![None; nb_vertices];
+
+		for _ in 0..path_length + 1 {
+			for id in 0..nb_vertices {
+				for e in self.graph[id].iter() {
+					if distance[id] + e.w < distance[e.dest] {
+						distance[e.dest] = distance[id] + e.w;
+						prev[e.dest] = Some(id);
+					}
+				}
+			}
+		}
+
+		// If self.graph contains a negative cycle, then at this point the graph described
+		// by prev (which is a directed 1-forest/functional graph)
+		// must contain a cycle. We list the cycles of prev.
+		let cycles_prev = cycles_of_1_forest(&prev);
+
+		// Remark that the cycle in prev is in the reverse order compared to the cycle
+		// in the graph. Thus the .rev().
+		return cycles_prev
+			.iter()
+			.map(|cycle| {
+				cycle
+					.iter()
+					.rev()
+					.map(|id| self.id_to_vertex[*id])
+					.collect()
+			})
+			.collect();
+	}
+}
+
+/// This function returns the list of cycles of a directed 1 forest. It does not
+/// check for the consistency of the input.
+fn cycles_of_1_forest(forest: &[Option<usize>]) -> Vec<Vec<usize>> {
+	let mut cycles = Vec::<Vec<usize>>::new();
+	let mut time_of_discovery = vec![None; forest.len()];
+
+	for t in 0..forest.len() {
+		let mut id = t;
+		// while we are on a valid undiscovered node
+		while time_of_discovery[id] == None {
+			time_of_discovery[id] = Some(t);
+			if let Some(i) = forest[id] {
+				id = i;
+			} else {
+				break;
+			}
+		}
+		if forest[id] != None && time_of_discovery[id] == Some(t) {
+			// We discovered an id that we explored at this iteration t.
+			// It means we are on a cycle
+			let mut cy = vec![id; 1];
+			let mut id2 = id;
+			while let Some(id_next) = forest[id2] {
+				id2 = id_next;
+				if id2 != id {
+					cy.push(id2);
+				} else {
+					break;
+				}
+			}
+			cycles.push(cy);
+		}
+	}
+	cycles
+}

src/rpc/lib.rs

@@ -8,6 +8,7 @@ mod consul;
 #[cfg(feature = "kubernetes-discovery")]
 mod kubernetes;
 
+pub mod graph_algo;
 pub mod layout;
 pub mod replication_mode;
 pub mod ring;

src/rpc/ring.rs

@@ -40,6 +40,7 @@ pub struct Ring {
 // Type to store compactly the id of a node in the system
 // Change this to u16 the day we want to have more than 256 nodes in a cluster
 pub type CompactNodeType = u8;
+pub const MAX_NODE_NUMBER: usize = 256;
 
 // The maximum number of times an object might get replicated
 // This must be at least 3 because Garage supports 3-way replication

src/rpc/system.rs

@@ -662,9 +662,9 @@ impl System {
 		let update_ring = self.update_ring.lock().await;
 		let mut layout: ClusterLayout = self.ring.borrow().layout.clone();
 
-		let prev_layout_check = layout.check();
+		let prev_layout_check = layout.check().is_ok();
 		if layout.merge(adv) {
-			if prev_layout_check && !layout.check() {
+			if prev_layout_check && !layout.check().is_ok() {
 				error!("New cluster layout is invalid, discarding.");
 				return Err(Error::Message(
 					"New cluster layout is invalid, discarding.".into(),
@@ -717,7 +717,7 @@ impl System {
 
 	async fn discovery_loop(self: &Arc<Self>, mut stop_signal: watch::Receiver<bool>) {
 		while !*stop_signal.borrow() {
-			let not_configured = !self.ring.borrow().layout.check();
+			let not_configured = !self.ring.borrow().layout.check().is_ok();
 			let no_peers = self.fullmesh.get_peer_list().len() < self.replication_factor;
 			let expected_n_nodes = self.ring.borrow().layout.num_nodes();
 			let bad_peers = self