Merge pull request 'admin refactoring, step 4' (#980) from refactor-admin into next-v2

Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/980

src/api/admin/api.rs

@@ -51,9 +51,12 @@ admin_endpoints![
 
 	// Layout operations
 	GetClusterLayout,
+	GetClusterLayoutHistory,
 	UpdateClusterLayout,
+	PreviewClusterLayoutChanges,
 	ApplyClusterLayout,
 	RevertClusterLayout,
+	ClusterLayoutSkipDeadNodes,
 
 	// Access key operations
 	ListKeys,
@@ -165,40 +168,61 @@ pub struct GetClusterStatusRequest;
 #[derive(Debug, Clone, Serialize, Deserialize, ToSchema)]
 #[serde(rename_all = "camelCase")]
 pub struct GetClusterStatusResponse {
+	/// Current version number of the cluster layout
 	pub layout_version: u64,
+	/// List of nodes that are either currently connected, part of the
+	/// current cluster layout, or part of an older cluster layout that
+	/// is still active in the cluster (being drained).
 	pub nodes: Vec<NodeResp>,
 }
 
 #[derive(Debug, Clone, Serialize, Deserialize, Default, ToSchema)]
 #[serde(rename_all = "camelCase")]
 pub struct NodeResp {
+	/// Full-length node identifier
 	pub id: String,
-	pub role: Option<NodeRoleResp>,
-	#[schema(value_type = Option<String> )]
+	/// Role assigned to this node in the current cluster layout
+	pub role: Option<NodeAssignedRole>,
+	/// Socket address used by other nodes to connect to this node for RPC
+	#[schema(value_type = Option<String>)]
 	pub addr: Option<SocketAddr>,
+	/// Hostname of the node
 	pub hostname: Option<String>,
+	/// Whether this node is connected in the cluster
 	pub is_up: bool,
+	/// For disconnected nodes, the number of seconds since last contact,
+	/// or `null` if no contact was established since Garage restarted.
 	pub last_seen_secs_ago: Option<u64>,
+	/// Whether this node is part of an older layout version and is draining data.
 	pub draining: bool,
-	#[serde(skip_serializing_if = "Option::is_none")]
+	/// Total and available space on the disk partition(s) containing the data
+	/// directory(ies)
+	#[serde(default, skip_serializing_if = "Option::is_none")]
 	pub data_partition: Option<FreeSpaceResp>,
-	#[serde(skip_serializing_if = "Option::is_none")]
+	/// Total and available space on the disk partition containing the
+	/// metadata directory
+	#[serde(default, skip_serializing_if = "Option::is_none")]
 	pub metadata_partition: Option<FreeSpaceResp>,
 }
 
 #[derive(Debug, Clone, Serialize, Deserialize, ToSchema)]
 #[serde(rename_all = "camelCase")]
-pub struct NodeRoleResp {
-	pub id: String,
+pub struct NodeAssignedRole {
+	/// Zone name assigned by the cluster administrator
 	pub zone: String,
-	pub capacity: Option<u64>,
+	/// List of tags assigned by the cluster administrator
 	pub tags: Vec<String>,
+	/// Capacity (in bytes) assigned by the cluster administrator,
+	/// absent for gateway nodes
+	pub capacity: Option<u64>,
 }
 
 #[derive(Debug, Clone, Serialize, Deserialize, ToSchema)]
 #[serde(rename_all = "camelCase")]
 pub struct FreeSpaceResp {
+	/// Number of bytes available
 	pub available: u64,
+	/// Total number of bytes
 	pub total: u64,
 }
 
@@ -270,9 +294,40 @@ pub struct GetClusterLayoutRequest;
 #[derive(Debug, Clone, Serialize, Deserialize, ToSchema)]
 #[serde(rename_all = "camelCase")]
 pub struct GetClusterLayoutResponse {
+	/// The current version number of the cluster layout
 	pub version: u64,
-	pub roles: Vec<NodeRoleResp>,
+	/// List of nodes that currently have a role in the cluster layout
+	pub roles: Vec<LayoutNodeRole>,
+	/// Layout parameters used when the current layout was computed
+	pub parameters: LayoutParameters,
+	/// The size, in bytes, of one Garage partition (= a shard)
+	pub partition_size: u64,
+	/// List of nodes that will have a new role or whose role will be
+	/// removed in the next version of the cluster layout
 	pub staged_role_changes: Vec<NodeRoleChange>,
+	/// Layout parameters to use when computing the next version of
+	/// the cluster layout
+	pub staged_parameters: Option<LayoutParameters>,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)]
+#[serde(rename_all = "camelCase")]
+pub struct LayoutNodeRole {
+	/// Identifier of the node
+	pub id: String,
+	/// Zone name assigned by the cluster administrator
+	pub zone: String,
+	/// List of tags assigned by the cluster administrator
+	pub tags: Vec<String>,
+	/// Capacity (in bytes) assigned by the cluster administrator,
+	/// absent for gateway nodes
+	pub capacity: Option<u64>,
+	/// Number of partitions stored on this node
+	/// (a result of the layout computation)
+	pub stored_partitions: Option<u64>,
+	/// Capacity (in bytes) that is actually usable on this node in the current
+	/// layout, which is equal to `stored_partitions` × `partition_size`
+	pub usable_capacity: Option<u64>,
 }
 
 #[derive(Debug, Clone, Serialize, Deserialize, ToSchema)]
@@ -293,36 +348,139 @@ pub enum NodeRoleChangeEnum {
 		remove: bool,
 	},
 	#[serde(rename_all = "camelCase")]
-	Update {
-		/// New zone of the node
-		zone: String,
-		/// New capacity (in bytes) of the node
-		capacity: Option<u64>,
-		/// New tags of the node
-		tags: Vec<String>,
-	},
+	Update(NodeAssignedRole),
+}
+
+#[derive(Copy, Debug, Clone, Serialize, Deserialize, ToSchema)]
+#[serde(rename_all = "camelCase")]
+pub struct LayoutParameters {
+	/// Minimum number of zones in which a data partition must be replicated
+	pub zone_redundancy: ZoneRedundancy,
+}
+
+#[derive(Copy, Debug, Clone, Serialize, Deserialize, ToSchema)]
+#[serde(rename_all = "camelCase")]
+pub enum ZoneRedundancy {
+	/// Partitions must be replicated in at least this number of
+	/// distinct zones.
+	AtLeast(usize),
+	/// Partitions must be replicated in as many zones as possible:
+	/// as many zones as there are replicas, if there are enough distinct
+	/// zones, or at least one in each zone otherwise.
+	Maximum,
+}
+
+// ---- GetClusterLayoutHistory ----
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct GetClusterLayoutHistoryRequest;
+
+#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)]
+#[serde(rename_all = "camelCase")]
+pub struct GetClusterLayoutHistoryResponse {
+	/// The current version number of the cluster layout
+	pub current_version: u64,
+	/// All nodes in the cluster are aware of layout versions up to
+	/// this version number (at least)
+	pub min_ack: u64,
+	/// Layout version history
+	pub versions: Vec<ClusterLayoutVersion>,
+	/// Detailed update trackers for nodes (see
+	/// `https://garagehq.deuxfleurs.fr/blog/2023-12-preserving-read-after-write-consistency/`)
+	pub update_trackers: Option<HashMap<String, NodeUpdateTrackers>>,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)]
+#[serde(rename_all = "camelCase")]
+pub struct ClusterLayoutVersion {
+	/// Version number of this layout version
+	pub version: u64,
+	/// Status of this layout version
+	pub status: ClusterLayoutVersionStatus,
+	/// Number of nodes with an assigned storage capacity in this layout version
+	pub storage_nodes: u64,
+	/// Number of nodes with a gateway role in this layout version
+	pub gateway_nodes: u64,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)]
+pub enum ClusterLayoutVersionStatus {
+	/// This is the most up-to-date layout version
+	Current,
+	/// This version is still active in the cluster because metadata
+	/// is being rebalanced or migrated from old nodes
+	Draining,
+	/// This version is no longer active in the cluster for metadata
+	/// reads and writes. Note that there is still the possibility
+	/// that data blocks are being migrated away from nodes in this
+	/// layout version.
+	Historical,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)]
+#[serde(rename_all = "camelCase")]
+pub struct NodeUpdateTrackers {
+	pub ack: u64,
+	pub sync: u64,
+	pub sync_ack: u64,
 }
 
 // ---- UpdateClusterLayout ----
 
 #[derive(Debug, Clone, Serialize, Deserialize, ToSchema)]
-pub struct UpdateClusterLayoutRequest(pub Vec<NodeRoleChange>);
+pub struct UpdateClusterLayoutRequest {
+	/// New node roles to assign or remove in the cluster layout
+	#[serde(default)]
+	pub roles: Vec<NodeRoleChange>,
+	/// New layout computation parameters to use
+	#[serde(default)]
+	pub parameters: Option<LayoutParameters>,
+}
 
 #[derive(Debug, Clone, Serialize, Deserialize, ToSchema)]
 pub struct UpdateClusterLayoutResponse(pub GetClusterLayoutResponse);
 
+// ---- PreviewClusterLayoutChanges ----
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct PreviewClusterLayoutChangesRequest;
+
+#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)]
+#[serde(untagged)]
+pub enum PreviewClusterLayoutChangesResponse {
+	#[serde(rename_all = "camelCase")]
+	Error {
+		/// Error message indicating that the layout could not be computed
+		/// with the provided configuration
+		error: String,
+	},
+	#[serde(rename_all = "camelCase")]
+	Success {
+		/// Plain-text information about the layout computation
+		/// (do not try to parse this)
+		message: Vec<String>,
+		/// Details about the new cluster layout
+		new_layout: GetClusterLayoutResponse,
+	},
+}
+
 // ---- ApplyClusterLayout ----
 
 #[derive(Debug, Clone, Serialize, Deserialize, ToSchema)]
 #[serde(rename_all = "camelCase")]
 pub struct ApplyClusterLayoutRequest {
+	/// As a safety measure, the new version number of the layout must
+	/// be specified here
 	pub version: u64,
 }
 
 #[derive(Debug, Clone, Serialize, Deserialize, ToSchema)]
 #[serde(rename_all = "camelCase")]
 pub struct ApplyClusterLayoutResponse {
+	/// Plain-text information about the layout computation
+	/// (do not try to parse this)
 	pub message: Vec<String>,
+	/// Details about the new cluster layout
 	pub layout: GetClusterLayoutResponse,
 }
 
@@ -334,6 +492,29 @@ pub struct RevertClusterLayoutRequest;
 #[derive(Debug, Clone, Serialize, Deserialize, ToSchema)]
 pub struct RevertClusterLayoutResponse(pub GetClusterLayoutResponse);
 
+// ---- ClusterLayoutSkipDeadNodes ----
+
+#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)]
+#[serde(rename_all = "camelCase")]
+pub struct ClusterLayoutSkipDeadNodesRequest {
+	/// Version number of the layout to assume is currently up-to-date.
+	/// This will generally be the current layout version.
+	pub version: u64,
+	/// Allow the skip even if a quorum of nodes could not be found for
+	/// the data among the remaining nodes
+	pub allow_missing_data: bool,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)]
+#[serde(rename_all = "camelCase")]
+pub struct ClusterLayoutSkipDeadNodesResponse {
+	/// Nodes for which the ACK update tracker has been updated to `version`
+	pub ack_updated: Vec<String>,
+	/// If `allow_missing_data` is set,
+	/// nodes for which the SYNC update tracker has been updated to `version`
+	pub sync_updated: Vec<String>,
+}
+
 // **********************************************
 //      Access key operations
 // **********************************************
@@ -367,7 +548,7 @@ pub struct GetKeyInfoRequest {
 pub struct GetKeyInfoResponse {
 	pub name: String,
 	pub access_key_id: String,
-	#[serde(skip_serializing_if = "is_default")]
+	#[serde(default, skip_serializing_if = "is_default")]
 	pub secret_access_key: Option<String>,
 	pub permissions: KeyPerm,
 	pub buckets: Vec<KeyInfoBucketResponse>,

src/api/admin/cluster.rs

@@ -3,6 +3,7 @@ use std::sync::Arc;
 
 use garage_util::crdt::*;
 use garage_util::data::*;
+use garage_util::error::Error as GarageError;
 
 use garage_rpc::layout;
 
@@ -54,8 +55,7 @@ impl RequestHandler for GetClusterStatusRequest {
 
 		for (id, _, role) in layout.current().roles.items().iter() {
 			if let layout::NodeRoleV(Some(r)) = role {
-				let role = NodeRoleResp {
-					id: hex::encode(id),
+				let role = NodeAssignedRole {
 					zone: r.zone.to_string(),
 					capacity: r.capacity,
 					tags: r.tags.clone(),
@@ -181,17 +181,23 @@ impl RequestHandler for GetClusterLayoutRequest {
 }
 
 fn format_cluster_layout(layout: &layout::LayoutHistory) -> GetClusterLayoutResponse {
-	let roles = layout
-		.current()
+	let current = layout.current();
+
+	let roles = current
 		.roles
 		.items()
 		.iter()
 		.filter_map(|(k, _, v)| v.0.clone().map(|x| (k, x)))
-		.map(|(k, v)| NodeRoleResp {
-			id: hex::encode(k),
-			zone: v.zone.clone(),
-			capacity: v.capacity,
-			tags: v.tags.clone(),
+		.map(|(k, v)| {
+			let stored_partitions = current.get_node_usage(k).ok().map(|x| x as u64);
+			LayoutNodeRole {
+				id: hex::encode(k),
+				zone: v.zone.clone(),
+				capacity: v.capacity,
+				stored_partitions,
+				usable_capacity: stored_partitions.map(|x| x * current.partition_size),
+				tags: v.tags.clone(),
+			}
 		})
 		.collect::<Vec<_>>();
 
@@ -201,7 +207,7 @@ fn format_cluster_layout(layout: &layout::LayoutHistory) -> GetClusterLayoutResp
 		.roles
 		.items()
 		.iter()
-		.filter(|(k, _, v)| layout.current().roles.get(k) != Some(v))
+		.filter(|(k, _, v)| current.roles.get(k) != Some(v))
 		.map(|(k, _, v)| match &v.0 {
 			None => NodeRoleChange {
 				id: hex::encode(k),
@@ -209,19 +215,108 @@ fn format_cluster_layout(layout: &layout::LayoutHistory) -> GetClusterLayoutResp
 			},
 			Some(r) => NodeRoleChange {
 				id: hex::encode(k),
-				action: NodeRoleChangeEnum::Update {
+				action: NodeRoleChangeEnum::Update(NodeAssignedRole {
 					zone: r.zone.clone(),
 					capacity: r.capacity,
 					tags: r.tags.clone(),
-				},
+				}),
 			},
 		})
 		.collect::<Vec<_>>();
 
+	let staged_parameters = if *layout.staging.get().parameters.get() != current.parameters {
+		Some((*layout.staging.get().parameters.get()).into())
+	} else {
+		None
+	};
+
 	GetClusterLayoutResponse {
-		version: layout.current().version,
+		version: current.version,
 		roles,
+		partition_size: current.partition_size,
+		parameters: current.parameters.into(),
 		staged_role_changes,
+		staged_parameters,
+	}
+}
+
+impl RequestHandler for GetClusterLayoutHistoryRequest {
+	type Response = GetClusterLayoutHistoryResponse;
+
+	async fn handle(
+		self,
+		garage: &Arc<Garage>,
+		_admin: &Admin,
+	) -> Result<GetClusterLayoutHistoryResponse, Error> {
+		let layout_helper = garage.system.cluster_layout();
+		let layout = layout_helper.inner();
+		let min_stored = layout.min_stored();
+
+		let versions = layout
+			.versions
+			.iter()
+			.rev()
+			.chain(layout.old_versions.iter().rev())
+			.map(|ver| {
+				let status = if ver.version == layout.current().version {
+					ClusterLayoutVersionStatus::Current
+				} else if ver.version >= min_stored {
+					ClusterLayoutVersionStatus::Draining
+				} else {
+					ClusterLayoutVersionStatus::Historical
+				};
+				ClusterLayoutVersion {
+					version: ver.version,
+					status,
+					storage_nodes: ver
+						.roles
+						.items()
+						.iter()
+						.filter(
+							|(_, _, x)| matches!(x, layout::NodeRoleV(Some(c)) if c.capacity.is_some()),
+						)
+						.count() as u64,
+					gateway_nodes: ver
+						.roles
+						.items()
+						.iter()
+						.filter(
+							|(_, _, x)| matches!(x, layout::NodeRoleV(Some(c)) if c.capacity.is_none()),
+						)
+						.count() as u64,
+				}
+			})
+			.collect::<Vec<_>>();
+
+		let all_nodes = layout.get_all_nodes();
+		let min_ack = layout_helper.ack_map_min();
+
+		let update_trackers = if layout.versions.len() > 1 {
+			Some(
+				all_nodes
+					.iter()
+					.map(|node| {
+						(
+							hex::encode(&node),
+							NodeUpdateTrackers {
+								ack: layout.update_trackers.ack_map.get(node, min_stored),
+								sync: layout.update_trackers.sync_map.get(node, min_stored),
+								sync_ack: layout.update_trackers.sync_ack_map.get(node, min_stored),
+							},
+						)
+					})
+					.collect(),
+			)
+		} else {
+			None
+		};
+
+		Ok(GetClusterLayoutHistoryResponse {
+			current_version: layout.current().version,
+			min_ack,
+			versions,
+			update_trackers,
+		})
 	}
 }
 
@@ -242,21 +337,26 @@ impl RequestHandler for UpdateClusterLayoutRequest {
 		let mut roles = layout.current().roles.clone();
 		roles.merge(&layout.staging.get().roles);
 
-		for change in self.0 {
+		for change in self.roles {
 			let node = hex::decode(&change.id).ok_or_bad_request("Invalid node identifier")?;
 			let node = Uuid::try_from(&node).ok_or_bad_request("Invalid node identifier")?;
 
 			let new_role = match change.action {
 				NodeRoleChangeEnum::Remove { remove: true } => None,
-				NodeRoleChangeEnum::Update {
+				NodeRoleChangeEnum::Update(NodeAssignedRole {
 					zone,
 					capacity,
 					tags,
-				} => Some(layout::NodeRole {
-					zone,
-					capacity,
-					tags,
-				}),
+				}) => {
+					if matches!(capacity, Some(cap) if cap < 1024) {
+						return Err(Error::bad_request("Capacity should be at least 1K (1024)"));
+					}
+					Some(layout::NodeRole {
+						zone,
+						capacity,
+						tags,
+					})
+				}
 				_ => return Err(Error::bad_request("Invalid layout change")),
 			};
 
@@ -267,6 +367,22 @@ impl RequestHandler for UpdateClusterLayoutRequest {
 				.merge(&roles.update_mutator(node, layout::NodeRoleV(new_role)));
 		}
 
+		if let Some(param) = self.parameters {
+			if let ZoneRedundancy::AtLeast(r_int) = param.zone_redundancy {
+				if r_int > layout.current().replication_factor {
+					return Err(Error::bad_request(format!(
+						"The zone redundancy must be smaller or equal to the replication factor ({}).",
+						layout.current().replication_factor
+					)));
+				} else if r_int < 1 {
+					return Err(Error::bad_request(
+						"The zone redundancy must be at least 1.",
+					));
+				}
+			}
+			layout.staging.get_mut().parameters.update(param.into());
+		}
+
 		garage
 			.system
 			.layout_manager
@@ -278,6 +394,29 @@ impl RequestHandler for UpdateClusterLayoutRequest {
 	}
 }
 
+impl RequestHandler for PreviewClusterLayoutChangesRequest {
+	type Response = PreviewClusterLayoutChangesResponse;
+
+	async fn handle(
+		self,
+		garage: &Arc<Garage>,
+		_admin: &Admin,
+	) -> Result<PreviewClusterLayoutChangesResponse, Error> {
+		let layout = garage.system.cluster_layout().inner().clone();
+		let new_ver = layout.current().version + 1;
+		match layout.apply_staged_changes(new_ver) {
+			Err(GarageError::Message(error)) => {
+				Ok(PreviewClusterLayoutChangesResponse::Error { error })
+			}
+			Err(e) => Err(e.into()),
+			Ok((new_layout, msg)) => Ok(PreviewClusterLayoutChangesResponse::Success {
+				message: msg,
+				new_layout: format_cluster_layout(&new_layout),
+			}),
+		}
+	}
+}
+
 impl RequestHandler for ApplyClusterLayoutRequest {
 	type Response = ApplyClusterLayoutResponse;
 
@@ -287,7 +426,7 @@ impl RequestHandler for ApplyClusterLayoutRequest {
 		_admin: &Admin,
 	) -> Result<ApplyClusterLayoutResponse, Error> {
 		let layout = garage.system.cluster_layout().inner().clone();
-		let (layout, msg) = layout.apply_staged_changes(Some(self.version))?;
+		let (layout, msg) = layout.apply_staged_changes(self.version)?;
 
 		garage
 			.system
@@ -322,3 +461,100 @@ impl RequestHandler for RevertClusterLayoutRequest {
 		Ok(RevertClusterLayoutResponse(res))
 	}
 }
+
+impl RequestHandler for ClusterLayoutSkipDeadNodesRequest {
+	type Response = ClusterLayoutSkipDeadNodesResponse;
+
+	async fn handle(
+		self,
+		garage: &Arc<Garage>,
+		_admin: &Admin,
+	) -> Result<ClusterLayoutSkipDeadNodesResponse, Error> {
+		let status = garage.system.get_known_nodes();
+
+		let mut layout = garage.system.cluster_layout().inner().clone();
+		let mut ack_updated = vec![];
+		let mut sync_updated = vec![];
+
+		if layout.versions.len() == 1 {
+			return Err(Error::bad_request(
+				"This command cannot be called when there is only one live cluster layout version",
+			));
+		}
+
+		let min_v = layout.min_stored();
+		if self.version <= min_v || self.version > layout.current().version {
+			return Err(Error::bad_request(format!(
+				"Invalid version, you may use the following version numbers: {}",
+				(min_v + 1..=layout.current().version)
+					.map(|x| x.to_string())
+					.collect::<Vec<_>>()
+					.join(" ")
+			)));
+		}
+
+		let all_nodes = layout.get_all_nodes();
+		for node in all_nodes.iter() {
+			// Update ACK tracker for dead nodes or for all nodes if --allow-missing-data
+			if self.allow_missing_data || !status.iter().any(|x| x.id == *node && x.is_up) {
+				if layout.update_trackers.ack_map.set_max(*node, self.version) {
+					ack_updated.push(hex::encode(node));
+				}
+			}
+
+			// If --allow-missing-data, update SYNC tracker for all nodes.
+			if self.allow_missing_data {
+				if layout.update_trackers.sync_map.set_max(*node, self.version) {
+					sync_updated.push(hex::encode(node));
+				}
+			}
+		}
+
+		garage
+			.system
+			.layout_manager
+			.update_cluster_layout(&layout)
+			.await?;
+
+		Ok(ClusterLayoutSkipDeadNodesResponse {
+			ack_updated,
+			sync_updated,
+		})
+	}
+}
+
+// ----
+
+impl From<layout::ZoneRedundancy> for ZoneRedundancy {
+	fn from(x: layout::ZoneRedundancy) -> Self {
+		match x {
+			layout::ZoneRedundancy::Maximum => ZoneRedundancy::Maximum,
+			layout::ZoneRedundancy::AtLeast(x) => ZoneRedundancy::AtLeast(x),
+		}
+	}
+}
+
+impl Into<layout::ZoneRedundancy> for ZoneRedundancy {
+	fn into(self) -> layout::ZoneRedundancy {
+		match self {
+			ZoneRedundancy::Maximum => layout::ZoneRedundancy::Maximum,
+			ZoneRedundancy::AtLeast(x) => layout::ZoneRedundancy::AtLeast(x),
+		}
+	}
+}
+
+impl From<layout::LayoutParameters> for LayoutParameters {
+	fn from(x: layout::LayoutParameters) -> Self {
+		LayoutParameters {
+			zone_redundancy: x.zone_redundancy.into(),
+		}
+	}
+}
+
+impl Into<layout::LayoutParameters> for LayoutParameters {
+	fn into(self) -> layout::LayoutParameters {
+		layout::LayoutParameters {
+			zone_redundancy: self.zone_redundancy.into(),
+		}
+	}
+}

src/api/admin/openapi.rs

@@ -88,6 +88,19 @@ Returns the cluster's current layout, including:
 )]
 fn GetClusterLayout() -> () {}
 
+#[utoipa::path(get,
+    path = "/v2/GetClusterLayoutHistory",
+    tag = "Cluster layout",
+    description = "
+Returns the history of layouts in the cluster
+    ",
+	responses(
+            (status = 200, description = "Cluster layout history", body = GetClusterLayoutHistoryResponse),
+            (status = 500, description = "Internal server error")
+        ),
+)]
+fn GetClusterLayoutHistory() -> () {}
+
 #[utoipa::path(post,
     path = "/v2/UpdateClusterLayout",
     tag = "Cluster layout",
@@ -117,6 +130,21 @@ Contrary to the CLI that may update only a subset of the fields capacity, zone a
 )]
 fn UpdateClusterLayout() -> () {}
 
+#[utoipa::path(post,
+    path = "/v2/PreviewClusterLayoutChanges",
+    tag = "Cluster layout",
+    description = "
+Computes a new layout taking into account the staged parameters, and returns it with detailed statistics. The new layout is not applied in the cluster.
+
+*Note: do not try to parse the `message` field of the response, it is given as an array of string specifically because its format is not stable.*
+    ",
+	responses(
+            (status = 200, description = "Information about the new layout", body = PreviewClusterLayoutChangesResponse),
+            (status = 500, description = "Internal server error")
+        ),
+)]
+fn PreviewClusterLayoutChanges() -> () {}
+
 #[utoipa::path(post,
     path = "/v2/ApplyClusterLayout",
     tag = "Cluster layout",
@@ -144,6 +172,18 @@ fn ApplyClusterLayout() -> () {}
 )]
 fn RevertClusterLayout() -> () {}
 
+#[utoipa::path(post,
+    path = "/v2/ClusterLayoutSkipDeadNodes",
+    tag = "Cluster layout",
+    description = "Force progress in layout update trackers",
+    request_body = ClusterLayoutSkipDeadNodesRequest,
+	responses(
+            (status = 200, description = "Request has been taken into account", body = ClusterLayoutSkipDeadNodesResponse),
+            (status = 500, description = "Internal server error")
+        ),
+)]
+fn ClusterLayoutSkipDeadNodes() -> () {}
+
 // **********************************************
 //      Access key operations
 // **********************************************
@@ -685,9 +725,12 @@ impl Modify for SecurityAddon {
         ConnectClusterNodes,
         // Layout operations
         GetClusterLayout,
+        GetClusterLayoutHistory,
         UpdateClusterLayout,
+        PreviewClusterLayoutChanges,
         ApplyClusterLayout,
         RevertClusterLayout,
+        ClusterLayoutSkipDeadNodes,
         // Key operations
         ListKeys,
         GetKeyInfo,

src/api/admin/router_v2.rs

@@ -36,9 +36,12 @@ impl AdminApiRequest {
 			POST ConnectClusterNodes (body),
 			// Layout endpoints
 			GET GetClusterLayout (),
+			GET GetClusterLayoutHistory (),
 			POST UpdateClusterLayout (body),
+			POST PreviewClusterLayoutChanges (),
 			POST ApplyClusterLayout (body),
 			POST RevertClusterLayout (),
+			POST ClusterLayoutSkipDeadNodes (body),
 			// API key endpoints
 			GET GetKeyInfo (query_opt::id, query_opt::search, parse_default(false)::show_secret_key),
 			POST UpdateKey (body_field, query::id),
@@ -108,10 +111,7 @@ impl AdminApiRequest {
 			Endpoint::GetClusterLayout => {
 				Ok(AdminApiRequest::GetClusterLayout(GetClusterLayoutRequest))
 			}
-			Endpoint::UpdateClusterLayout => {
-				let updates = parse_json_body::<UpdateClusterLayoutRequest, _, Error>(req).await?;
-				Ok(AdminApiRequest::UpdateClusterLayout(updates))
-			}
+			// UpdateClusterLayout semantics changed
 			Endpoint::ApplyClusterLayout => {
 				let param = parse_json_body::<ApplyClusterLayoutRequest, _, Error>(req).await?;
 				Ok(AdminApiRequest::ApplyClusterLayout(param))