From 03e6020c6bc147a1f3f16137377730ddc767a391 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Fri, 6 Mar 2026 09:49:00 +0100 Subject: [PATCH 01/11] admin api: report avilable space numerically in GetClusterStatistics --- doc/api/garage-admin-v2.json | 21 +++++++++++-- src/api/admin/api.rs | 14 +++++++++ src/api/admin/cluster.rs | 58 ++++++++++++++++++++---------------- 3 files changed, 66 insertions(+), 27 deletions(-) diff --git a/doc/api/garage-admin-v2.json b/doc/api/garage-admin-v2.json index 15059ce4..82796e3a 100644 --- a/doc/api/garage-admin-v2.json +++ b/doc/api/garage-admin-v2.json @@ -2581,8 +2581,25 @@ "freeform" ], "properties": { + "dataAvail": { + "type": "integer", + "format": "int64", + "description": "available storage space for object data in the entire cluster, in bytes", + "minimum": 0 + }, "freeform": { - "type": "string" + "type": "string", + "description": "cluster statistics as a free-form string, kept for compatibility with nodes\nrunning older v2.x versions of garage" + }, + "incompleteInfo": { + "type": "boolean", + "description": "true if the available storage space statistics are imprecise due to missing\ninformation of disconnected nodes. When this is the case, the actual\nspace available in the cluster might be lower than the reported values." + }, + "metadataAvail": { + "type": "integer", + "format": "int64", + "description": "available storage space for object metadata in the entire cluster, in bytes", + "minimum": 0 } } }, @@ -4117,7 +4134,7 @@ "items": { "type": "string" }, - "description": "Scope of the admin API token, a list of admin endpoint names (such as\n`GetClusterStatus`, etc), or the special value `*` to allow all\nadmin endpoints. **WARNING:** Granting a scope of `CreateAdminToken` or\n`UpdateAdminToken` trivially allows for privilege escalation, and is thus\nfunctionnally equivalent to granting a scope of `*`." + "description": "Scope of the admin API token, a list of admin endpoint names (such as\n`GetClusterStatus`, etc), or the special value `*` to allow all\nadmin endpoints. **WARNING:** Granting a scope of `CreateAdminToken` or\n`UpdateAdminToken` trivially allows for privilege escalation, and is thus\nfunctionally equivalent to granting a scope of `*`." } } }, diff --git a/src/api/admin/api.rs b/src/api/admin/api.rs index 83d456a4..fb91354c 100644 --- a/src/api/admin/api.rs +++ b/src/api/admin/api.rs @@ -282,8 +282,22 @@ pub struct GetClusterHealthResponse { pub struct GetClusterStatisticsRequest; #[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "camelCase")] pub struct GetClusterStatisticsResponse { + /// cluster statistics as a free-form string, kept for compatibility with nodes + /// running older v2.x versions of garage pub freeform: String, + /// available storage space for object data in the entire cluster, in bytes + #[serde(default)] + pub data_avail: u64, + /// available storage space for object metadata in the entire cluster, in bytes + #[serde(default)] + pub metadata_avail: u64, + /// true if the available storage space statistics are imprecise due to missing + /// information of disconnected nodes. When this is the case, the actual + /// space available in the cluster might be lower than the reported values. + #[serde(default)] + pub incomplete_info: bool, } // ---- ConnectClusterNodes ---- diff --git a/src/api/admin/cluster.rs b/src/api/admin/cluster.rs index 6a97c471..96efa7f3 100644 --- a/src/api/admin/cluster.rs +++ b/src/api/admin/cluster.rs @@ -231,33 +231,41 @@ impl RequestHandler for GetClusterStatisticsRequest { .map(|c| c.0 / *parts) }) .collect::>(); - if !meta_part_avail.is_empty() && !data_part_avail.is_empty() { - let meta_avail = - bytesize::ByteSize(meta_part_avail.iter().min().unwrap() * (1 << PARTITION_BITS)); - let data_avail = - bytesize::ByteSize(data_part_avail.iter().min().unwrap() * (1 << PARTITION_BITS)); - writeln!( - &mut ret, - "\nEstimated available storage space cluster-wide (might be lower in practice):" - ) - .unwrap(); - if meta_part_avail.len() < node_partition_count.len() - || data_part_avail.len() < node_partition_count.len() - { - ret += &format_table_to_string(vec![ - format!(" data: < {}", data_avail), - format!(" metadata: < {}", meta_avail), - ]); - writeln!(&mut ret, "A precise estimate could not be given as information is missing for some storage nodes.").unwrap(); - } else { - ret += &format_table_to_string(vec![ - format!(" data: {}", data_avail), - format!(" metadata: {}", meta_avail), - ]); - } + + let metadata_avail: u64 = + meta_part_avail.iter().min().unwrap_or(&0) * (1 << PARTITION_BITS); + let data_avail: u64 = data_part_avail.iter().min().unwrap_or(&0) * (1 << PARTITION_BITS); + + let metadata_avail_str = bytesize::ByteSize(metadata_avail); + let data_avail_str = bytesize::ByteSize(data_avail); + + let incomplete_info = meta_part_avail.len() < node_partition_count.len() + || data_part_avail.len() < node_partition_count.len(); + + writeln!( + &mut ret, + "\nEstimated available storage space cluster-wide (might be lower in practice):" + ) + .unwrap(); + if incomplete_info { + ret += &format_table_to_string(vec![ + format!(" data: < {}", data_avail_str), + format!(" metadata: < {}", metadata_avail_str), + ]); + writeln!(&mut ret, "A precise estimate could not be given as information is missing for some storage nodes.").unwrap(); + } else { + ret += &format_table_to_string(vec![ + format!(" data: {}", data_avail_str), + format!(" metadata: {}", metadata_avail_str), + ]); } - Ok(GetClusterStatisticsResponse { freeform: ret }) + Ok(GetClusterStatisticsResponse { + freeform: ret, + metadata_avail, + data_avail, + incomplete_info, + }) } } From 124a9eb5218635ea6e5f5ff7d591cae4b34ea905 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Fri, 6 Mar 2026 10:08:12 +0100 Subject: [PATCH 02/11] admin api: export node statistics as structured json --- doc/api/garage-admin-v2.json | 135 ++++++++++++++++++++++++++++++++--- src/api/admin/api.rs | 44 ++++++++++++ src/api/admin/node.rs | 128 +++++++++++++++++++-------------- 3 files changed, 244 insertions(+), 63 deletions(-) diff --git a/doc/api/garage-admin-v2.json b/doc/api/garage-admin-v2.json index 82796e3a..a5ae37f8 100644 --- a/doc/api/garage-admin-v2.json +++ b/doc/api/garage-admin-v2.json @@ -3072,7 +3072,8 @@ ], "properties": { "dbEngine": { - "type": "string" + "type": "string", + "description": "database engine used for metadata" }, "garageFeatures": { "type": [ @@ -3081,16 +3082,23 @@ ], "items": { "type": "string" - } + }, + "description": "build-time features enabled for this garage release" }, "garageVersion": { - "type": "string" + "type": "string", + "description": "garage version running on this node" + }, + "hostname": { + "type": "string", + "description": "hostname of this node" }, "nodeId": { "type": "string" }, "rustVersion": { - "type": "string" + "type": "string", + "description": "rustc version with which this garage release was compiled" } } }, @@ -3100,8 +3108,20 @@ "freeform" ], "properties": { + "blockManagerStats": { + "$ref": "#/components/schemas/NodeBlockManagerStats", + "description": "block manager statistics" + }, "freeform": { - "type": "string" + "type": "string", + "description": "node statistics as a free-form string, kept for compatibility with nodes\nrunning older v2.x versions of garage" + }, + "tableStats": { + "type": "array", + "items": { + "$ref": "#/components/schemas/NodeTableStats" + }, + "description": "metadata table statistics" } } }, @@ -3402,7 +3422,8 @@ ], "properties": { "dbEngine": { - "type": "string" + "type": "string", + "description": "database engine used for metadata" }, "garageFeatures": { "type": [ @@ -3411,16 +3432,23 @@ ], "items": { "type": "string" - } + }, + "description": "build-time features enabled for this garage release" }, "garageVersion": { - "type": "string" + "type": "string", + "description": "garage version running on this node" + }, + "hostname": { + "type": "string", + "description": "hostname of this node" }, "nodeId": { "type": "string" }, "rustVersion": { - "type": "string" + "type": "string", + "description": "rustc version with which this garage release was compiled" } } }, @@ -3456,8 +3484,20 @@ "freeform" ], "properties": { + "blockManagerStats": { + "$ref": "#/components/schemas/NodeBlockManagerStats", + "description": "block manager statistics" + }, "freeform": { - "type": "string" + "type": "string", + "description": "node statistics as a free-form string, kept for compatibility with nodes\nrunning older v2.x versions of garage" + }, + "tableStats": { + "type": "array", + "items": { + "$ref": "#/components/schemas/NodeTableStats" + }, + "description": "metadata table statistics" } } }, @@ -3796,6 +3836,34 @@ } } }, + "NodeBlockManagerStats": { + "type": "object", + "required": [ + "rcEntries", + "resyncQueueLen", + "resyncErrors" + ], + "properties": { + "rcEntries": { + "type": "integer", + "format": "int64", + "description": "number of reference counter entries", + "minimum": 0 + }, + "resyncErrors": { + "type": "integer", + "format": "int64", + "description": "number of blocks with resync errors", + "minimum": 0 + }, + "resyncQueueLen": { + "type": "integer", + "format": "int64", + "description": "number of blocks in the resync queue", + "minimum": 0 + } + } + }, "NodeResp": { "type": "object", "required": [ @@ -3959,6 +4027,53 @@ } ] }, + "NodeTableStats": { + "type": "object", + "required": [ + "tableName", + "items", + "merkleItems", + "merkleQueueLen", + "insertQueueLen", + "gcQueueLen" + ], + "properties": { + "gcQueueLen": { + "type": "integer", + "format": "int64", + "description": "number of items in the garbage collection queue", + "minimum": 0 + }, + "insertQueueLen": { + "type": "integer", + "format": "int64", + "description": "number of items in the remote insert queue", + "minimum": 0 + }, + "items": { + "type": "integer", + "format": "int64", + "description": "number of items stored in metadata table", + "minimum": 0 + }, + "merkleItems": { + "type": "integer", + "format": "int64", + "description": "size of the merkle tree representing all items in the table", + "minimum": 0 + }, + "merkleQueueLen": { + "type": "integer", + "format": "int64", + "description": "number of items in the merkle tree update queue", + "minimum": 0 + }, + "tableName": { + "type": "string", + "description": "name of metadata table" + } + } + }, "NodeUpdateTrackers": { "type": "object", "required": [ diff --git a/src/api/admin/api.rs b/src/api/admin/api.rs index fb91354c..b375ecb9 100644 --- a/src/api/admin/api.rs +++ b/src/api/admin/api.rs @@ -1123,9 +1123,16 @@ pub struct LocalGetNodeInfoRequest; #[serde(rename_all = "camelCase")] pub struct LocalGetNodeInfoResponse { pub node_id: String, + /// hostname of this node + #[serde(default)] + pub hostname: String, + /// garage version running on this node pub garage_version: String, + /// build-time features enabled for this garage release pub garage_features: Option>, + /// rustc version with which this garage release was compiled pub rust_version: String, + /// database engine used for metadata pub db_engine: String, } @@ -1135,8 +1142,45 @@ pub struct LocalGetNodeInfoResponse { pub struct LocalGetNodeStatisticsRequest; #[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "camelCase")] pub struct LocalGetNodeStatisticsResponse { + /// node statistics as a free-form string, kept for compatibility with nodes + /// running older v2.x versions of garage pub freeform: String, + /// metadata table statistics + #[serde(default)] + pub table_stats: Vec, + /// block manager statistics + #[serde(default)] + pub block_manager_stats: NodeBlockManagerStats, +} + +#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "camelCase")] +pub struct NodeTableStats { + /// name of metadata table + pub table_name: String, + /// number of items stored in metadata table + pub items: u64, + /// size of the merkle tree representing all items in the table + pub merkle_items: u64, + /// number of items in the merkle tree update queue + pub merkle_queue_len: u64, + /// number of items in the remote insert queue + pub insert_queue_len: u64, + /// number of items in the garbage collection queue + pub gc_queue_len: u64, +} + +#[derive(Debug, Clone, Serialize, Deserialize, ToSchema, Default)] +#[serde(rename_all = "camelCase")] +pub struct NodeBlockManagerStats { + /// number of reference counter entries + pub rc_entries: u64, + /// number of blocks in the resync queue + pub resync_queue_len: u64, + /// number of blocks with resync errors + pub resync_errors: u64, } // ---- CreateMetadataSnapshot ---- diff --git a/src/api/admin/node.rs b/src/api/admin/node.rs index 12963aab..6e46a46d 100644 --- a/src/api/admin/node.rs +++ b/src/api/admin/node.rs @@ -22,8 +22,12 @@ impl RequestHandler for LocalGetNodeInfoRequest { garage: &Arc, _admin: &Admin, ) -> Result { + let sys_status = garage.system.local_status(); + let hostname = sys_status.hostname.unwrap_or_default().to_string(); + Ok(LocalGetNodeInfoResponse { node_id: hex::encode(garage.system.id), + hostname, garage_version: garage_util::version::garage_version().to_string(), garage_features: garage_util::version::garage_features() .map(|features| features.iter().map(ToString::to_string).collect()), @@ -57,46 +61,58 @@ impl RequestHandler for LocalGetNodeStatisticsRequest { ) -> Result { let sys_status = garage.system.local_status(); + let hostname = sys_status.hostname.unwrap_or_default().to_string(); + let garage_version = garage_util::version::garage_version().to_string(); + let garage_features = garage_util::version::garage_features() + .unwrap() + .iter() + .map(ToString::to_string) + .collect::>(); + let rustc_version = garage_util::version::rust_version().to_string(); + let db_engine_descr = garage.db.engine(); + let mut ret = format_table_to_string(vec![ format!("Node ID:\t{:?}", garage.system.id), - format!("Hostname:\t{}", sys_status.hostname.unwrap_or_default(),), - format!( - "Garage version:\t{}", - garage_util::version::garage_version(), - ), - format!( - "Garage features:\t{}", - garage_util::version::garage_features() - .map(|list| list.join(", ")) - .unwrap_or_else(|| "(unknown)".into()), - ), - format!( - "Rust compiler version:\t{}", - garage_util::version::rust_version(), - ), - format!("Database engine:\t{}", garage.db.engine()), + format!("Hostname:\t{}", hostname), + format!("Garage version:\t{}", garage_version), + format!("Garage features:\t{}", garage_features.join(", ")), + format!("Rust compiler version:\t{}", rustc_version), + format!("Database engine:\t{}", db_engine_descr), ]); - // Gather table statistics - let mut table = vec![" Table\tItems\tMklItems\tMklTodo\tInsQueue\tGcTodo".into()]; - table.push(gather_table_stats(&garage.admin_token_table)?); - table.push(gather_table_stats(&garage.bucket_table)?); - table.push(gather_table_stats(&garage.bucket_alias_table)?); - table.push(gather_table_stats(&garage.key_table)?); - - table.push(gather_table_stats(&garage.object_table)?); - table.push(gather_table_stats(&garage.object_counter_table.table)?); - table.push(gather_table_stats(&garage.mpu_table)?); - table.push(gather_table_stats(&garage.mpu_counter_table.table)?); - table.push(gather_table_stats(&garage.version_table)?); - table.push(gather_table_stats(&garage.block_ref_table)?); + let mut table_stats = vec![ + gather_table_stats(&garage.admin_token_table)?, + gather_table_stats(&garage.bucket_table)?, + gather_table_stats(&garage.bucket_alias_table)?, + gather_table_stats(&garage.key_table)?, + gather_table_stats(&garage.object_table)?, + gather_table_stats(&garage.object_counter_table.table)?, + gather_table_stats(&garage.mpu_table)?, + gather_table_stats(&garage.mpu_counter_table.table)?, + gather_table_stats(&garage.version_table)?, + gather_table_stats(&garage.block_ref_table)?, + ]; #[cfg(feature = "k2v")] { - table.push(gather_table_stats(&garage.k2v.item_table)?); - table.push(gather_table_stats(&garage.k2v.counter_table.table)?); + table_stats.push(gather_table_stats(&garage.k2v.item_table)?); + table_stats.push(gather_table_stats(&garage.k2v.counter_table.table)?); } + // Gather table statistics + let mut table = vec![" Table\tItems\tMklItems\tMklTodo\tInsQueue\tGcTodo".into()]; + table.extend(table_stats.iter().map(|ts| { + format!( + " {}\t{}\t{}\t{}\t{}\t{}", + ts.table_name, + ts.items, + ts.merkle_items, + ts.merkle_queue_len, + ts.insert_queue_len, + ts.gc_queue_len, + ) + })); + write!( &mut ret, "\nTable stats:\n{}", @@ -104,46 +120,52 @@ impl RequestHandler for LocalGetNodeStatisticsRequest { ) .unwrap(); + let block_manager_stats = NodeBlockManagerStats { + rc_entries: garage.block_manager.rc_approximate_len()? as u64, + resync_queue_len: garage.block_manager.resync.queue_approximate_len()? as u64, + resync_errors: garage.block_manager.resync.errors_approximate_len()? as u64, + }; + // Gather block manager statistics writeln!(&mut ret, "\nBlock manager stats:").unwrap(); - let rc_len = garage.block_manager.rc_approximate_len()?.to_string(); ret += &format_table_to_string(vec![ - format!(" number of RC entries:\t{} (~= number of blocks)", rc_len), + format!( + " number of RC entries:\t{} (~= number of blocks)", + block_manager_stats.rc_entries + ), format!( " resync queue length:\t{}", - garage.block_manager.resync.queue_approximate_len()? + block_manager_stats.resync_queue_len, ), format!( " blocks with resync errors:\t{}", - garage.block_manager.resync.errors_approximate_len()? + block_manager_stats.resync_errors ), ]); - Ok(LocalGetNodeStatisticsResponse { freeform: ret }) + Ok(LocalGetNodeStatisticsResponse { + freeform: ret, + table_stats, + block_manager_stats, + }) } } -fn gather_table_stats(t: &Arc>) -> Result +fn gather_table_stats(t: &Arc>) -> Result where F: TableSchema + 'static, R: TableReplication + 'static, { - let data_len = t - .data - .store - .approximate_len() - .map_err(GarageError::from)? - .to_string(); - let mkl_len = t.merkle_updater.merkle_tree_approximate_len()?.to_string(); + let data_len = t.data.store.approximate_len().map_err(GarageError::from)?; + let mkl_len = t.merkle_updater.merkle_tree_approximate_len()?; - Ok(format!( - " {}\t{}\t{}\t{}\t{}\t{}", - F::TABLE_NAME, - data_len, - mkl_len, - t.merkle_updater.todo_approximate_len()?, - t.data.insert_queue_approximate_len()?, - t.data.gc_todo_approximate_len()? - )) + Ok(NodeTableStats { + table_name: F::TABLE_NAME.to_string(), + items: data_len as u64, + merkle_items: mkl_len as u64, + merkle_queue_len: t.merkle_updater.todo_approximate_len()? as u64, + insert_queue_len: t.data.insert_queue_approximate_len()? as u64, + gc_queue_len: t.data.gc_todo_approximate_len()? as u64, + }) } From 6c0bb1c9b62d5cd1bf95bf7e421e1411421299ed Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Fri, 6 Mar 2026 10:33:16 +0100 Subject: [PATCH 03/11] refactoring: move xml definitions for bucket cors/lifecycle/website config move these defnitions to garage_api_common so that they can also be used in admin api --- Cargo.lock | 1 + src/api/common/Cargo.toml | 1 + src/api/common/common_error.rs | 6 + src/api/common/lib.rs | 1 + src/api/common/xml/cors.rs | 197 +++++++++++++++++ src/api/common/xml/lifecycle.rs | 337 ++++++++++++++++++++++++++++ src/api/common/xml/mod.rs | 45 ++++ src/api/common/xml/website.rs | 378 +++++++++++++++++++++++++++++++ src/api/s3/api_server.rs | 5 +- src/api/s3/cors.rs | 202 +---------------- src/api/s3/error.rs | 6 - src/api/s3/lib.rs | 8 - src/api/s3/lifecycle.rs | 342 +--------------------------- src/api/s3/website.rs | 380 +------------------------------- src/api/s3/xml.rs | 36 +-- 15 files changed, 982 insertions(+), 963 deletions(-) create mode 100644 src/api/common/xml/cors.rs create mode 100644 src/api/common/xml/lifecycle.rs create mode 100644 src/api/common/xml/mod.rs create mode 100644 src/api/common/xml/website.rs diff --git a/Cargo.lock b/Cargo.lock index 5780ca65..b60ae3d7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1549,6 +1549,7 @@ dependencies = [ "nom", "opentelemetry", "pin-project", + "quick-xml", "serde", "serde_json", "sha1", diff --git a/src/api/common/Cargo.toml b/src/api/common/Cargo.toml index dba58202..05c640db 100644 --- a/src/api/common/Cargo.toml +++ b/src/api/common/Cargo.toml @@ -41,6 +41,7 @@ hyper = { workspace = true, default-features = false, features = ["server", "htt hyper-util.workspace = true url.workspace = true +quick-xml.workspace = true serde.workspace = true serde_json.workspace = true diff --git a/src/api/common/common_error.rs b/src/api/common/common_error.rs index 0de490c2..aa4c36fa 100644 --- a/src/api/common/common_error.rs +++ b/src/api/common/common_error.rs @@ -36,6 +36,10 @@ pub enum CommonError { #[error("Invalid header value: {0}")] InvalidHeader(#[from] hyper::header::ToStrError), + /// The client sent a request for an action not supported by garage + #[error("Unimplemented action: {0}")] + NotImplemented(String), + // ---- SPECIFIC ERROR CONDITIONS ---- // These have to be error codes referenced in the S3 spec here: // https://docs.aws.amazon.com/AmazonS3/latest/API/ErrorResponses.html#ErrorCodeList @@ -101,6 +105,7 @@ impl CommonError { } CommonError::BadRequest(_) => StatusCode::BAD_REQUEST, CommonError::Forbidden(_) => StatusCode::FORBIDDEN, + CommonError::NotImplemented(_) => StatusCode::NOT_IMPLEMENTED, CommonError::NoSuchBucket(_) => StatusCode::NOT_FOUND, CommonError::BucketNotEmpty | CommonError::BucketAlreadyExists @@ -127,6 +132,7 @@ impl CommonError { CommonError::InvalidBucketName(_) => "InvalidBucketName", CommonError::InvalidHeader(_) => "InvalidHeaderValue", CommonError::BucketAlreadyOwnedByYou => "BucketAlreadyOwnedByYou", + CommonError::NotImplemented(_) => "NotImplemented", } } diff --git a/src/api/common/lib.rs b/src/api/common/lib.rs index 0e655a53..219f3624 100644 --- a/src/api/common/lib.rs +++ b/src/api/common/lib.rs @@ -10,3 +10,4 @@ pub mod generic_server; pub mod helpers; pub mod router_macros; pub mod signature; +pub mod xml; diff --git a/src/api/common/xml/cors.rs b/src/api/common/xml/cors.rs new file mode 100644 index 00000000..63e3e618 --- /dev/null +++ b/src/api/common/xml/cors.rs @@ -0,0 +1,197 @@ +use serde::{Deserialize, Serialize}; + +use hyper::{header::HeaderName, Method}; + +use garage_model::bucket_table::CorsRule as GarageCorsRule; + +use super::{xmlns_tag, IntValue, Value}; +use crate::common_error::{CommonError as Error, OkOrBadRequest}; + +#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] +#[serde(rename = "CORSConfiguration")] +pub struct CorsConfiguration { + #[serde(rename = "@xmlns", serialize_with = "xmlns_tag", skip_deserializing)] + pub xmlns: (), + #[serde(rename = "CORSRule")] + pub cors_rules: Vec, +} + +#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] +pub struct CorsRule { + #[serde(rename = "ID", skip_serializing_if = "Option::is_none")] + pub id: Option, + #[serde(rename = "MaxAgeSeconds", skip_serializing_if = "Option::is_none")] + pub max_age_seconds: Option, + #[serde(rename = "AllowedOrigin")] + pub allowed_origins: Vec, + #[serde(rename = "AllowedMethod")] + pub allowed_methods: Vec, + #[serde(rename = "AllowedHeader", default)] + pub allowed_headers: Vec, + #[serde(rename = "ExposeHeader", default)] + pub expose_headers: Vec, +} + +#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] +pub struct AllowedMethod { + #[serde(rename = "AllowedMethod")] + pub allowed_method: Value, +} + +#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] +pub struct AllowedHeader { + #[serde(rename = "AllowedHeader")] + pub allowed_header: Value, +} + +#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] +pub struct ExposeHeader { + #[serde(rename = "ExposeHeader")] + pub expose_header: Value, +} + +impl CorsConfiguration { + pub fn validate(&self) -> Result<(), Error> { + for r in self.cors_rules.iter() { + r.validate()?; + } + Ok(()) + } + + pub fn into_garage_cors_config(self) -> Result, Error> { + Ok(self + .cors_rules + .iter() + .map(CorsRule::to_garage_cors_rule) + .collect()) + } +} + +impl CorsRule { + pub fn validate(&self) -> Result<(), Error> { + for method in self.allowed_methods.iter() { + method + .0 + .parse::() + .ok_or_bad_request("Invalid CORSRule method")?; + } + for header in self + .allowed_headers + .iter() + .chain(self.expose_headers.iter()) + { + header + .0 + .parse::() + .ok_or_bad_request("Invalid HTTP header name")?; + } + Ok(()) + } + + pub fn to_garage_cors_rule(&self) -> GarageCorsRule { + let convert_vec = + |vval: &[Value]| vval.iter().map(|x| x.0.to_owned()).collect::>(); + GarageCorsRule { + id: self.id.as_ref().map(|x| x.0.to_owned()), + max_age_seconds: self.max_age_seconds.as_ref().map(|x| x.0 as u64), + allow_origins: convert_vec(&self.allowed_origins), + allow_methods: convert_vec(&self.allowed_methods), + allow_headers: convert_vec(&self.allowed_headers), + expose_headers: convert_vec(&self.expose_headers), + } + } + + pub fn from_garage_cors_rule(rule: &GarageCorsRule) -> Self { + let convert_vec = |vval: &[String]| { + vval.iter() + .map(|x| Value(x.clone())) + .collect::>() + }; + Self { + id: rule.id.as_ref().map(|x| Value(x.clone())), + max_age_seconds: rule.max_age_seconds.map(|x| IntValue(x as i64)), + allowed_origins: convert_vec(&rule.allow_origins), + allowed_methods: convert_vec(&rule.allow_methods), + allowed_headers: convert_vec(&rule.allow_headers), + expose_headers: convert_vec(&rule.expose_headers), + } + } +} + +#[cfg(test)] +mod tests { + use crate::xml::{to_xml_with_header, unprettify_xml}; + + use super::*; + + use quick_xml::de::from_str; + + #[test] + fn test_deserialize() { + let message = r#" + + + http://www.example.com + + PUT + POST + DELETE + + * + + + * + GET + + + qsdfjklm + 12345 + https://perdu.com + + GET + DELETE + * + * + +"#; + let conf: CorsConfiguration = + from_str(message).expect("failed to deserialize xml into `CorsConfiguration` struct"); + let ref_value = CorsConfiguration { + xmlns: (), + cors_rules: vec![ + CorsRule { + id: None, + max_age_seconds: None, + allowed_origins: vec!["http://www.example.com".into()], + allowed_methods: vec!["PUT".into(), "POST".into(), "DELETE".into()], + allowed_headers: vec!["*".into()], + expose_headers: vec![], + }, + CorsRule { + id: None, + max_age_seconds: None, + allowed_origins: vec!["*".into()], + allowed_methods: vec!["GET".into()], + allowed_headers: vec![], + expose_headers: vec![], + }, + CorsRule { + id: Some("qsdfjklm".into()), + max_age_seconds: Some(IntValue(12345)), + allowed_origins: vec!["https://perdu.com".into()], + allowed_methods: vec!["GET".into(), "DELETE".into()], + allowed_headers: vec!["*".into()], + expose_headers: vec!["*".into()], + }, + ], + }; + assert_eq! { + ref_value, + conf + }; + + let message2 = to_xml_with_header(&ref_value).expect("xml serialization"); + + assert_eq!(unprettify_xml(message), unprettify_xml(&message2)); + } +} diff --git a/src/api/common/xml/lifecycle.rs b/src/api/common/xml/lifecycle.rs new file mode 100644 index 00000000..2997d070 --- /dev/null +++ b/src/api/common/xml/lifecycle.rs @@ -0,0 +1,337 @@ +use serde::{Deserialize, Serialize}; + +use garage_model::bucket_table::{ + parse_lifecycle_date, LifecycleExpiration as GarageLifecycleExpiration, + LifecycleFilter as GarageLifecycleFilter, LifecycleRule as GarageLifecycleRule, +}; + +use super::{xmlns_tag, IntValue, Value}; + +#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] +pub struct LifecycleConfiguration { + #[serde(rename = "@xmlns", serialize_with = "xmlns_tag", skip_deserializing)] + pub xmlns: (), + #[serde(rename = "Rule")] + pub lifecycle_rules: Vec, +} + +#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] +pub struct LifecycleRule { + #[serde(rename = "ID", skip_serializing_if = "Option::is_none")] + pub id: Option, + #[serde(rename = "Status")] + pub status: Value, + #[serde(rename = "Filter", default, skip_serializing_if = "Option::is_none")] + pub filter: Option, + #[serde( + rename = "Expiration", + default, + skip_serializing_if = "Option::is_none" + )] + pub expiration: Option, + #[serde( + rename = "AbortIncompleteMultipartUpload", + default, + skip_serializing_if = "Option::is_none" + )] + pub abort_incomplete_mpu: Option, +} + +#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord, Default)] +pub struct Filter { + #[serde(rename = "And", skip_serializing_if = "Option::is_none")] + pub and: Option>, + #[serde(rename = "Prefix", skip_serializing_if = "Option::is_none")] + pub prefix: Option, + #[serde( + rename = "ObjectSizeGreaterThan", + skip_serializing_if = "Option::is_none" + )] + pub size_gt: Option, + #[serde(rename = "ObjectSizeLessThan", skip_serializing_if = "Option::is_none")] + pub size_lt: Option, +} + +#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] +pub struct Expiration { + #[serde(rename = "Days", skip_serializing_if = "Option::is_none")] + pub days: Option, + #[serde(rename = "Date", skip_serializing_if = "Option::is_none")] + pub at_date: Option, +} + +#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] +pub struct AbortIncompleteMpu { + #[serde(rename = "DaysAfterInitiation")] + pub days: IntValue, +} + +impl LifecycleConfiguration { + pub fn validate_into_garage_lifecycle_config( + self, + ) -> Result, &'static str> { + let mut ret = vec![]; + for rule in self.lifecycle_rules { + ret.push(rule.validate_into_garage_lifecycle_rule()?); + } + Ok(ret) + } + + pub fn from_garage_lifecycle_config(config: &[GarageLifecycleRule]) -> Self { + Self { + xmlns: (), + lifecycle_rules: config + .iter() + .map(LifecycleRule::from_garage_lifecycle_rule) + .collect(), + } + } +} + +impl LifecycleRule { + pub fn validate_into_garage_lifecycle_rule(self) -> Result { + let enabled = match self.status.0.as_str() { + "Enabled" => true, + "Disabled" => false, + _ => return Err("invalid value for "), + }; + + let filter = self + .filter + .map(Filter::validate_into_garage_lifecycle_filter) + .transpose()? + .unwrap_or_default(); + + let abort_incomplete_mpu_days = self.abort_incomplete_mpu.map(|x| x.days.0 as usize); + + let expiration = self + .expiration + .map(Expiration::validate_into_garage_lifecycle_expiration) + .transpose()?; + + Ok(GarageLifecycleRule { + id: self.id.map(|x| x.0), + enabled, + filter, + abort_incomplete_mpu_days, + expiration, + }) + } + + pub fn from_garage_lifecycle_rule(rule: &GarageLifecycleRule) -> Self { + Self { + id: rule.id.as_deref().map(Value::from), + status: if rule.enabled { + Value::from("Enabled") + } else { + Value::from("Disabled") + }, + filter: Filter::from_garage_lifecycle_filter(&rule.filter), + abort_incomplete_mpu: rule + .abort_incomplete_mpu_days + .map(|days| AbortIncompleteMpu { + days: IntValue(days as i64), + }), + expiration: rule + .expiration + .as_ref() + .map(Expiration::from_garage_lifecycle_expiration), + } + } +} + +impl Filter { + pub fn count(&self) -> i32 { + fn count(x: &Option) -> i32 { + x.as_ref().map(|_| 1).unwrap_or(0) + } + count(&self.prefix) + count(&self.size_gt) + count(&self.size_lt) + } + + pub fn validate_into_garage_lifecycle_filter( + self, + ) -> Result { + if self.count() > 0 && self.and.is_some() { + Err("Filter tag cannot contain both and another condition") + } else if let Some(and) = self.and { + if and.and.is_some() { + return Err("Nested tags"); + } + Ok(and.internal_into_garage_lifecycle_filter()) + } else if self.count() > 1 { + Err("Multiple Filter conditions must be wrapped in an tag") + } else { + Ok(self.internal_into_garage_lifecycle_filter()) + } + } + + fn internal_into_garage_lifecycle_filter(self) -> GarageLifecycleFilter { + GarageLifecycleFilter { + prefix: self.prefix.map(|x| x.0), + size_gt: self.size_gt.map(|x| x.0 as u64), + size_lt: self.size_lt.map(|x| x.0 as u64), + } + } + + pub fn from_garage_lifecycle_filter(rule: &GarageLifecycleFilter) -> Option { + let filter = Filter { + and: None, + prefix: rule.prefix.as_deref().map(Value::from), + size_gt: rule.size_gt.map(|x| IntValue(x as i64)), + size_lt: rule.size_lt.map(|x| IntValue(x as i64)), + }; + match filter.count() { + 0 => None, + 1 => Some(filter), + _ => Some(Filter { + and: Some(Box::new(filter)), + ..Default::default() + }), + } + } +} + +impl Expiration { + pub fn validate_into_garage_lifecycle_expiration( + self, + ) -> Result { + match (self.days, self.at_date) { + (Some(_), Some(_)) => Err("cannot have both and in "), + (None, None) => Err(" must contain either or "), + (Some(days), None) => Ok(GarageLifecycleExpiration::AfterDays(days.0 as usize)), + (None, Some(date)) => { + parse_lifecycle_date(&date.0)?; + Ok(GarageLifecycleExpiration::AtDate(date.0)) + } + } + } + + pub fn from_garage_lifecycle_expiration(exp: &GarageLifecycleExpiration) -> Self { + match exp { + GarageLifecycleExpiration::AfterDays(days) => Expiration { + days: Some(IntValue(*days as i64)), + at_date: None, + }, + GarageLifecycleExpiration::AtDate(date) => Expiration { + days: None, + at_date: Some(Value(date.to_string())), + }, + } + } +} + +#[cfg(test)] +mod tests { + use crate::xml::{to_xml_with_header, unprettify_xml}; + + use super::*; + + use quick_xml::de::from_str; + + #[test] + fn test_deserialize_lifecycle_config() { + let message = r#" + + + id1 + Enabled + + documents/ + + + 7 + + + + id2 + Enabled + + + logs/ + 1000000 + + + + 365 + + +"#; + let conf: LifecycleConfiguration = from_str(message).unwrap(); + let ref_value = LifecycleConfiguration { + xmlns: (), + lifecycle_rules: vec![ + LifecycleRule { + id: Some("id1".into()), + status: "Enabled".into(), + filter: Some(Filter { + prefix: Some("documents/".into()), + ..Default::default() + }), + expiration: None, + abort_incomplete_mpu: Some(AbortIncompleteMpu { days: IntValue(7) }), + }, + LifecycleRule { + id: Some("id2".into()), + status: "Enabled".into(), + filter: Some(Filter { + and: Some(Box::new(Filter { + prefix: Some("logs/".into()), + size_gt: Some(IntValue(1000000)), + ..Default::default() + })), + ..Default::default() + }), + expiration: Some(Expiration { + days: Some(IntValue(365)), + at_date: None, + }), + abort_incomplete_mpu: None, + }, + ], + }; + assert_eq! { + ref_value, + conf + }; + + let message2 = to_xml_with_header(&ref_value).expect("serialize xml"); + + assert_eq!(unprettify_xml(message), unprettify_xml(&message2)); + + // Check validation + let validated = ref_value + .validate_into_garage_lifecycle_config() + .expect("invalid xml config"); + + let ref_config = vec![ + GarageLifecycleRule { + id: Some("id1".into()), + enabled: true, + filter: GarageLifecycleFilter { + prefix: Some("documents/".into()), + ..Default::default() + }, + expiration: None, + abort_incomplete_mpu_days: Some(7), + }, + GarageLifecycleRule { + id: Some("id2".into()), + enabled: true, + filter: GarageLifecycleFilter { + prefix: Some("logs/".into()), + size_gt: Some(1000000), + ..Default::default() + }, + expiration: Some(GarageLifecycleExpiration::AfterDays(365)), + abort_incomplete_mpu_days: None, + }, + ]; + assert_eq!(validated, ref_config); + + let message3 = to_xml_with_header(&LifecycleConfiguration::from_garage_lifecycle_config( + &validated, + )) + .expect("serialize xml"); + assert_eq!(unprettify_xml(message), unprettify_xml(&message3)); + } +} diff --git a/src/api/common/xml/mod.rs b/src/api/common/xml/mod.rs new file mode 100644 index 00000000..701e2834 --- /dev/null +++ b/src/api/common/xml/mod.rs @@ -0,0 +1,45 @@ +pub mod cors; +pub mod lifecycle; +pub mod website; + +use serde::{Deserialize, Serialize, Serializer}; + +pub fn to_xml_with_header(x: &T) -> Result { + use quick_xml::se::{self, EmptyElementHandling, QuoteLevel}; + + let mut xml = r#""#.to_string(); + + let mut ser = se::Serializer::new(&mut xml); + ser.set_quote_level(QuoteLevel::Full) + .empty_element_handling(EmptyElementHandling::Expanded); + let _serialized = x.serialize(ser)?; + Ok(xml) +} + +#[cfg(test)] +pub fn unprettify_xml(xml_in: &str) -> String { + xml_in.trim().lines().fold(String::new(), |mut val, line| { + val.push_str(line.trim()); + val + }) +} + +pub fn xmlns_tag(_v: &(), s: S) -> Result { + s.serialize_str("http://s3.amazonaws.com/doc/2006-03-01/") +} + +pub fn xmlns_xsi_tag(_v: &(), s: S) -> Result { + s.serialize_str("http://www.w3.org/2001/XMLSchema-instance") +} + +#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] +pub struct Value(#[serde(rename = "$value")] pub String); + +impl From<&str> for Value { + fn from(s: &str) -> Value { + Value(s.to_string()) + } +} + +#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] +pub struct IntValue(#[serde(rename = "$value")] pub i64); diff --git a/src/api/common/xml/website.rs b/src/api/common/xml/website.rs new file mode 100644 index 00000000..fc9d0b27 --- /dev/null +++ b/src/api/common/xml/website.rs @@ -0,0 +1,378 @@ +use serde::{Deserialize, Serialize}; + +use garage_model::bucket_table::{self, WebsiteConfig}; + +use crate::common_error::CommonError as Error; +use crate::xml::{xmlns_tag, IntValue, Value}; + +#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] +pub struct WebsiteConfiguration { + #[serde(rename = "@xmlns", serialize_with = "xmlns_tag", skip_deserializing)] + pub xmlns: (), + #[serde(rename = "ErrorDocument")] + pub error_document: Option, + #[serde(rename = "IndexDocument")] + pub index_document: Option, + #[serde(rename = "RedirectAllRequestsTo")] + pub redirect_all_requests_to: Option, + #[serde( + rename = "RoutingRules", + default, + skip_serializing_if = "RoutingRules::is_empty" + )] + pub routing_rules: RoutingRules, +} + +#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord, Default)] +pub struct RoutingRules { + #[serde(rename = "RoutingRule")] + pub rules: Vec, +} + +impl RoutingRules { + fn is_empty(&self) -> bool { + self.rules.is_empty() + } +} + +#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] +pub struct RoutingRule { + #[serde(rename = "Condition")] + pub condition: Option, + #[serde(rename = "Redirect")] + pub redirect: Redirect, +} + +#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] +pub struct Key { + #[serde(rename = "Key")] + pub key: Value, +} + +#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] +pub struct Suffix { + #[serde(rename = "Suffix")] + pub suffix: Value, +} + +#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] +pub struct Target { + #[serde(rename = "HostName")] + pub hostname: Value, + #[serde(rename = "Protocol")] + pub protocol: Option, +} + +#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] +pub struct Condition { + #[serde( + rename = "HttpErrorCodeReturnedEquals", + skip_serializing_if = "Option::is_none" + )] + pub http_error_code: Option, + #[serde(rename = "KeyPrefixEquals", skip_serializing_if = "Option::is_none")] + pub prefix: Option, +} + +#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] +pub struct Redirect { + #[serde(rename = "HostName", skip_serializing_if = "Option::is_none")] + pub hostname: Option, + #[serde(rename = "Protocol", skip_serializing_if = "Option::is_none")] + pub protocol: Option, + #[serde(rename = "HttpRedirectCode", skip_serializing_if = "Option::is_none")] + pub http_redirect_code: Option, + #[serde( + rename = "ReplaceKeyPrefixWith", + skip_serializing_if = "Option::is_none" + )] + pub replace_prefix: Option, + #[serde(rename = "ReplaceKeyWith", skip_serializing_if = "Option::is_none")] + pub replace_full: Option, +} + +impl WebsiteConfiguration { + pub fn validate(&self) -> Result<(), Error> { + if self.redirect_all_requests_to.is_some() + && (self.error_document.is_some() + || self.index_document.is_some() + || !self.routing_rules.is_empty()) + { + return Err(Error::bad_request( + "Bad XML: can't have RedirectAllRequestsTo and other fields", + )); + } + if let Some(ref ed) = self.error_document { + ed.validate()?; + } + if let Some(ref id) = self.index_document { + id.validate()?; + } + if let Some(ref rart) = self.redirect_all_requests_to { + rart.validate()?; + } + for rr in &self.routing_rules.rules { + rr.validate()?; + } + if self.routing_rules.rules.len() > 1000 { + // we will do linear scans, best to avoid overly long configuration. The + // limit was chosen arbitrarily + return Err(Error::bad_request( + "Bad XML: RoutingRules can't have more than 1000 child elements", + )); + } + + Ok(()) + } + + pub fn into_garage_website_config(self) -> Result { + if self.redirect_all_requests_to.is_some() { + Err(Error::NotImplemented( + "RedirectAllRequestsTo is not currently implemented in Garage, however its effect can be emulated using a single unconditional RoutingRule.".into(), + )) + } else { + Ok(WebsiteConfig { + index_document: self + .index_document + .map(|x| x.suffix.0) + .unwrap_or_else(|| "index.html".to_string()), + error_document: self.error_document.map(|x| x.key.0), + redirect_all: None, + routing_rules: self + .routing_rules + .rules + .into_iter() + .map(|rule| { + bucket_table::RoutingRule { + condition: rule.condition.map(|condition| { + bucket_table::RedirectCondition { + http_error_code: condition.http_error_code.map(|c| c.0 as u16), + prefix: condition.prefix.map(|p| p.0), + } + }), + redirect: bucket_table::Redirect { + hostname: rule.redirect.hostname.map(|h| h.0), + protocol: rule.redirect.protocol.map(|p| p.0), + // aws default to 301, which i find punitive in case of + // misconfiguration (can be permanently cached on the + // user agent) + http_redirect_code: rule + .redirect + .http_redirect_code + .map(|c| c.0 as u16) + .unwrap_or(302), + replace_key_prefix: rule.redirect.replace_prefix.map(|k| k.0), + replace_key: rule.redirect.replace_full.map(|k| k.0), + }, + } + }) + .collect(), + }) + } + } +} + +impl Key { + pub fn validate(&self) -> Result<(), Error> { + if self.key.0.is_empty() { + Err(Error::bad_request( + "Bad XML: error document specified but empty", + )) + } else { + Ok(()) + } + } +} + +impl Suffix { + pub fn validate(&self) -> Result<(), Error> { + if self.suffix.0.is_empty() | self.suffix.0.contains('/') { + Err(Error::bad_request( + "Bad XML: index document is empty or contains /", + )) + } else { + Ok(()) + } + } +} + +impl Target { + pub fn validate(&self) -> Result<(), Error> { + if let Some(ref protocol) = self.protocol { + if protocol.0 != "http" && protocol.0 != "https" { + return Err(Error::bad_request("Bad XML: invalid protocol")); + } + } + Ok(()) + } +} + +impl RoutingRule { + pub fn validate(&self) -> Result<(), Error> { + if let Some(condition) = &self.condition { + condition.validate()?; + } + self.redirect.validate() + } +} + +impl Condition { + pub fn validate(&self) -> Result { + if let Some(ref error_code) = self.http_error_code { + // TODO do other error codes make sense? Aws only allows 4xx and 5xx + if error_code.0 != 404 { + return Err(Error::bad_request( + "Bad XML: HttpErrorCodeReturnedEquals must be 404 or absent", + )); + } + } + Ok(self.prefix.is_some()) + } +} + +impl Redirect { + pub fn validate(&self) -> Result<(), Error> { + if self.replace_prefix.is_some() && self.replace_full.is_some() { + return Err(Error::bad_request( + "Bad XML: both ReplaceKeyPrefixWith and ReplaceKeyWith are set", + )); + } + if let Some(ref protocol) = self.protocol { + if protocol.0 != "http" && protocol.0 != "https" { + return Err(Error::bad_request("Bad XML: invalid protocol")); + } + } + if let Some(ref http_redirect_code) = self.http_redirect_code { + match http_redirect_code.0 { + // aws allows all 3xx except 300, but some are non-sensical (not modified, + // use proxy...) + 301 | 302 | 303 | 307 | 308 => { + if self.hostname.is_none() && self.protocol.is_some() { + return Err(Error::bad_request( + "Bad XML: HostName must be set if Protocol is set", + )); + } + } + // aws doesn't allow these codes, but netlify does, and it seems like a + // cool feature (change the page seen without changing the url shown by the + // user agent) + 200 | 404 => { + if self.hostname.is_some() || self.protocol.is_some() { + // hostname would mean different bucket, protocol doesn't make + // sense + return Err(Error::bad_request( + "Bad XML: an HttpRedirectCode of 200 is not acceptable alongside HostName or Protocol", + )); + } + } + _ => { + return Err(Error::bad_request("Bad XML: invalid HttpRedirectCode")); + } + } + } + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use crate::xml::{to_xml_with_header, unprettify_xml}; + + use super::*; + + use quick_xml::de::from_str; + + #[test] + fn test_deserialize() { + let message = r#" + + + my-error-doc + + + my-index + + + garage.tld + https + + + + + 404 + prefix1 + + + gara.ge + http + 303 + prefix2 + fullkey + + + + + + + + 404 + missing + + + +"#; + let conf: WebsiteConfiguration = + from_str(message).expect("failed to deserialize xml in `WebsiteConfiguration`"); + let ref_value = WebsiteConfiguration { + xmlns: (), + error_document: Some(Key { + key: Value("my-error-doc".to_owned()), + }), + index_document: Some(Suffix { + suffix: Value("my-index".to_owned()), + }), + redirect_all_requests_to: Some(Target { + hostname: Value("garage.tld".to_owned()), + protocol: Some(Value("https".to_owned())), + }), + routing_rules: RoutingRules { + rules: vec![ + RoutingRule { + condition: Some(Condition { + http_error_code: Some(IntValue(404)), + prefix: Some(Value("prefix1".to_owned())), + }), + redirect: Redirect { + hostname: Some(Value("gara.ge".to_owned())), + protocol: Some(Value("http".to_owned())), + http_redirect_code: Some(IntValue(303)), + replace_prefix: Some(Value("prefix2".to_owned())), + replace_full: Some(Value("fullkey".to_owned())), + }, + }, + RoutingRule { + condition: Some(Condition { + http_error_code: None, + prefix: Some(Value("".to_owned())), + }), + redirect: Redirect { + hostname: None, + protocol: None, + http_redirect_code: Some(IntValue(404)), + replace_prefix: None, + replace_full: Some(Value("missing".to_owned())), + }, + }, + ], + }, + }; + assert_eq! { + ref_value, + conf + } + + let message2 = to_xml_with_header(&ref_value).expect("xml serialization"); + + assert_eq!(unprettify_xml(message), unprettify_xml(&message2)); + } +} diff --git a/src/api/s3/api_server.rs b/src/api/s3/api_server.rs index fba32ec9..689d174d 100644 --- a/src/api/s3/api_server.rs +++ b/src/api/s3/api_server.rs @@ -13,6 +13,7 @@ use garage_util::socket_address::UnixOrTCPSocketAddress; use garage_model::garage::Garage; use garage_model::key_table::Key; +use garage_api_common::common_error::CommonError; use garage_api_common::cors::*; use garage_api_common::generic_server::*; use garage_api_common::helpers::*; @@ -64,7 +65,7 @@ impl S3ApiServer { ) -> Result, Error> { match endpoint { Endpoint::ListBuckets => handle_list_buckets(&self.garage, &api_key).await, - endpoint => Err(Error::NotImplemented(endpoint.name().to_owned())), + endpoint => Err(CommonError::NotImplemented(endpoint.name().to_owned()).into()), } } } @@ -327,7 +328,7 @@ impl ApiHandler for S3ApiServer { Endpoint::GetBucketLifecycleConfiguration {} => handle_get_lifecycle(ctx).await, Endpoint::PutBucketLifecycleConfiguration {} => handle_put_lifecycle(ctx, req).await, Endpoint::DeleteBucketLifecycle {} => handle_delete_lifecycle(ctx).await, - endpoint => Err(Error::NotImplemented(endpoint.name().to_owned())), + endpoint => Err(CommonError::NotImplemented(endpoint.name().to_owned()).into()), }; // If request was a success and we have a CORS rule that applies to it, diff --git a/src/api/s3/cors.rs b/src/api/s3/cors.rs index 3318a3ea..9356983f 100644 --- a/src/api/s3/cors.rs +++ b/src/api/s3/cors.rs @@ -1,16 +1,15 @@ use quick_xml::de::from_reader; -use hyper::{header::HeaderName, Method, Request, Response, StatusCode}; +use hyper::{Request, Response, StatusCode}; -use serde::{Deserialize, Serialize}; - -use garage_model::bucket_table::{Bucket, CorsRule as GarageCorsRule}; +use garage_model::bucket_table::Bucket; use garage_api_common::helpers::*; +use garage_api_common::xml::cors::*; use crate::api_server::{ReqBody, ResBody}; use crate::error::*; -use crate::xml::{to_xml_with_header, xmlns_tag, IntValue, Value}; +use crate::xml::to_xml_with_header; pub async fn handle_get_cors(ctx: ReqCtx) -> Result, Error> { let ReqCtx { bucket_params, .. } = ctx; @@ -78,196 +77,3 @@ pub async fn handle_put_cors( .status(StatusCode::OK) .body(empty_body())?) } - -// ---- SERIALIZATION AND DESERIALIZATION TO/FROM S3 XML ---- - -#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] -#[serde(rename = "CORSConfiguration")] -pub struct CorsConfiguration { - #[serde(rename = "@xmlns", serialize_with = "xmlns_tag", skip_deserializing)] - pub xmlns: (), - #[serde(rename = "CORSRule")] - pub cors_rules: Vec, -} - -#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] -pub struct CorsRule { - #[serde(rename = "ID", skip_serializing_if = "Option::is_none")] - pub id: Option, - #[serde(rename = "MaxAgeSeconds", skip_serializing_if = "Option::is_none")] - pub max_age_seconds: Option, - #[serde(rename = "AllowedOrigin")] - pub allowed_origins: Vec, - #[serde(rename = "AllowedMethod")] - pub allowed_methods: Vec, - #[serde(rename = "AllowedHeader", default)] - pub allowed_headers: Vec, - #[serde(rename = "ExposeHeader", default)] - pub expose_headers: Vec, -} - -#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] -pub struct AllowedMethod { - #[serde(rename = "AllowedMethod")] - pub allowed_method: Value, -} - -#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] -pub struct AllowedHeader { - #[serde(rename = "AllowedHeader")] - pub allowed_header: Value, -} - -#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] -pub struct ExposeHeader { - #[serde(rename = "ExposeHeader")] - pub expose_header: Value, -} - -impl CorsConfiguration { - pub fn validate(&self) -> Result<(), Error> { - for r in self.cors_rules.iter() { - r.validate()?; - } - Ok(()) - } - - pub fn into_garage_cors_config(self) -> Result, Error> { - Ok(self - .cors_rules - .iter() - .map(CorsRule::to_garage_cors_rule) - .collect()) - } -} - -impl CorsRule { - pub fn validate(&self) -> Result<(), Error> { - for method in self.allowed_methods.iter() { - method - .0 - .parse::() - .ok_or_bad_request("Invalid CORSRule method")?; - } - for header in self - .allowed_headers - .iter() - .chain(self.expose_headers.iter()) - { - header - .0 - .parse::() - .ok_or_bad_request("Invalid HTTP header name")?; - } - Ok(()) - } - - pub fn to_garage_cors_rule(&self) -> GarageCorsRule { - let convert_vec = - |vval: &[Value]| vval.iter().map(|x| x.0.to_owned()).collect::>(); - GarageCorsRule { - id: self.id.as_ref().map(|x| x.0.to_owned()), - max_age_seconds: self.max_age_seconds.as_ref().map(|x| x.0 as u64), - allow_origins: convert_vec(&self.allowed_origins), - allow_methods: convert_vec(&self.allowed_methods), - allow_headers: convert_vec(&self.allowed_headers), - expose_headers: convert_vec(&self.expose_headers), - } - } - - pub fn from_garage_cors_rule(rule: &GarageCorsRule) -> Self { - let convert_vec = |vval: &[String]| { - vval.iter() - .map(|x| Value(x.clone())) - .collect::>() - }; - Self { - id: rule.id.as_ref().map(|x| Value(x.clone())), - max_age_seconds: rule.max_age_seconds.map(|x| IntValue(x as i64)), - allowed_origins: convert_vec(&rule.allow_origins), - allowed_methods: convert_vec(&rule.allow_methods), - allowed_headers: convert_vec(&rule.allow_headers), - expose_headers: convert_vec(&rule.expose_headers), - } - } -} - -#[cfg(test)] -mod tests { - use crate::unprettify_xml; - - use super::*; - - use quick_xml::de::from_str; - - #[test] - fn test_deserialize() -> Result<(), Error> { - let message = r#" - - - http://www.example.com - - PUT - POST - DELETE - - * - - - * - GET - - - qsdfjklm - 12345 - https://perdu.com - - GET - DELETE - * - * - -"#; - let conf: CorsConfiguration = - from_str(message).expect("failed to deserialize xml into `CorsConfiguration` struct"); - let ref_value = CorsConfiguration { - xmlns: (), - cors_rules: vec![ - CorsRule { - id: None, - max_age_seconds: None, - allowed_origins: vec!["http://www.example.com".into()], - allowed_methods: vec!["PUT".into(), "POST".into(), "DELETE".into()], - allowed_headers: vec!["*".into()], - expose_headers: vec![], - }, - CorsRule { - id: None, - max_age_seconds: None, - allowed_origins: vec!["*".into()], - allowed_methods: vec!["GET".into()], - allowed_headers: vec![], - expose_headers: vec![], - }, - CorsRule { - id: Some("qsdfjklm".into()), - max_age_seconds: Some(IntValue(12345)), - allowed_origins: vec!["https://perdu.com".into()], - allowed_methods: vec!["GET".into(), "DELETE".into()], - allowed_headers: vec!["*".into()], - expose_headers: vec!["*".into()], - }, - ], - }; - assert_eq! { - ref_value, - conf - }; - - let message2 = to_xml_with_header(&ref_value)?; - - assert_eq!(unprettify_xml(message), unprettify_xml(&message2)); - - Ok(()) - } -} diff --git a/src/api/s3/error.rs b/src/api/s3/error.rs index b316b095..74ef6692 100644 --- a/src/api/s3/error.rs +++ b/src/api/s3/error.rs @@ -99,10 +99,6 @@ pub enum Error { /// The provided digest (checksum) value was invalid #[error("Invalid digest: {0}")] InvalidDigest(String), - - /// The client sent a request for an action not supported by garage - #[error("Unimplemented action: {0}")] - NotImplemented(String), } commonErrorDerivative!(Error); @@ -151,7 +147,6 @@ impl Error { Error::InvalidPartOrder => "InvalidPartOrder", Error::EntityTooSmall => "EntityTooSmall", Error::AuthorizationHeaderMalformed(_) => "AuthorizationHeaderMalformed", - Error::NotImplemented(_) => "NotImplemented", Error::InvalidXml(_) => "MalformedXML", Error::InvalidXmlDe(_) => "MalformedXML", Error::InvalidXmlSe(_) => "InternalError", @@ -176,7 +171,6 @@ impl ApiError for Error { | Error::NoSuchLifecycleConfiguration => StatusCode::NOT_FOUND, Error::PreconditionFailed => StatusCode::PRECONDITION_FAILED, Error::InvalidRange(_) => StatusCode::RANGE_NOT_SATISFIABLE, - Error::NotImplemented(_) => StatusCode::NOT_IMPLEMENTED, Error::InvalidXmlSe(_) => StatusCode::INTERNAL_SERVER_ERROR, Error::AuthorizationHeaderMalformed(_) | Error::InvalidPart diff --git a/src/api/s3/lib.rs b/src/api/s3/lib.rs index f5fab57a..83f684f8 100644 --- a/src/api/s3/lib.rs +++ b/src/api/s3/lib.rs @@ -19,11 +19,3 @@ pub mod website; mod encryption; mod router; pub mod xml; - -#[cfg(test)] -pub(crate) fn unprettify_xml(xml_in: &str) -> String { - xml_in.trim().lines().fold(String::new(), |mut val, line| { - val.push_str(line.trim()); - val - }) -} diff --git a/src/api/s3/lifecycle.rs b/src/api/s3/lifecycle.rs index 1e99fcc1..83064bca 100644 --- a/src/api/s3/lifecycle.rs +++ b/src/api/s3/lifecycle.rs @@ -2,18 +2,14 @@ use quick_xml::de::from_reader; use hyper::{Request, Response, StatusCode}; -use serde::{Deserialize, Serialize}; - use garage_api_common::helpers::*; +use garage_api_common::xml::lifecycle::*; use crate::api_server::{ReqBody, ResBody}; use crate::error::*; -use crate::xml::{to_xml_with_header, xmlns_tag, IntValue, Value}; +use crate::xml::to_xml_with_header; -use garage_model::bucket_table::{ - parse_lifecycle_date, Bucket, LifecycleExpiration as GarageLifecycleExpiration, - LifecycleFilter as GarageLifecycleFilter, LifecycleRule as GarageLifecycleRule, -}; +use garage_model::bucket_table::Bucket; pub async fn handle_get_lifecycle(ctx: ReqCtx) -> Result, Error> { let ReqCtx { bucket_params, .. } = ctx; @@ -76,335 +72,3 @@ pub async fn handle_put_lifecycle( .status(StatusCode::OK) .body(empty_body())?) } - -// ---- SERIALIZATION AND DESERIALIZATION TO/FROM S3 XML ---- - -#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] -pub struct LifecycleConfiguration { - #[serde(rename = "@xmlns", serialize_with = "xmlns_tag", skip_deserializing)] - pub xmlns: (), - #[serde(rename = "Rule")] - pub lifecycle_rules: Vec, -} - -#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] -pub struct LifecycleRule { - #[serde(rename = "ID", skip_serializing_if = "Option::is_none")] - pub id: Option, - #[serde(rename = "Status")] - pub status: Value, - #[serde(rename = "Filter", default, skip_serializing_if = "Option::is_none")] - pub filter: Option, - #[serde( - rename = "Expiration", - default, - skip_serializing_if = "Option::is_none" - )] - pub expiration: Option, - #[serde( - rename = "AbortIncompleteMultipartUpload", - default, - skip_serializing_if = "Option::is_none" - )] - pub abort_incomplete_mpu: Option, -} - -#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord, Default)] -pub struct Filter { - #[serde(rename = "And", skip_serializing_if = "Option::is_none")] - pub and: Option>, - #[serde(rename = "Prefix", skip_serializing_if = "Option::is_none")] - pub prefix: Option, - #[serde( - rename = "ObjectSizeGreaterThan", - skip_serializing_if = "Option::is_none" - )] - pub size_gt: Option, - #[serde(rename = "ObjectSizeLessThan", skip_serializing_if = "Option::is_none")] - pub size_lt: Option, -} - -#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] -pub struct Expiration { - #[serde(rename = "Days", skip_serializing_if = "Option::is_none")] - pub days: Option, - #[serde(rename = "Date", skip_serializing_if = "Option::is_none")] - pub at_date: Option, -} - -#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] -pub struct AbortIncompleteMpu { - #[serde(rename = "DaysAfterInitiation")] - pub days: IntValue, -} - -impl LifecycleConfiguration { - pub fn validate_into_garage_lifecycle_config( - self, - ) -> Result, &'static str> { - let mut ret = vec![]; - for rule in self.lifecycle_rules { - ret.push(rule.validate_into_garage_lifecycle_rule()?); - } - Ok(ret) - } - - pub fn from_garage_lifecycle_config(config: &[GarageLifecycleRule]) -> Self { - Self { - xmlns: (), - lifecycle_rules: config - .iter() - .map(LifecycleRule::from_garage_lifecycle_rule) - .collect(), - } - } -} - -impl LifecycleRule { - pub fn validate_into_garage_lifecycle_rule(self) -> Result { - let enabled = match self.status.0.as_str() { - "Enabled" => true, - "Disabled" => false, - _ => return Err("invalid value for "), - }; - - let filter = self - .filter - .map(Filter::validate_into_garage_lifecycle_filter) - .transpose()? - .unwrap_or_default(); - - let abort_incomplete_mpu_days = self.abort_incomplete_mpu.map(|x| x.days.0 as usize); - - let expiration = self - .expiration - .map(Expiration::validate_into_garage_lifecycle_expiration) - .transpose()?; - - Ok(GarageLifecycleRule { - id: self.id.map(|x| x.0), - enabled, - filter, - abort_incomplete_mpu_days, - expiration, - }) - } - - pub fn from_garage_lifecycle_rule(rule: &GarageLifecycleRule) -> Self { - Self { - id: rule.id.as_deref().map(Value::from), - status: if rule.enabled { - Value::from("Enabled") - } else { - Value::from("Disabled") - }, - filter: Filter::from_garage_lifecycle_filter(&rule.filter), - abort_incomplete_mpu: rule - .abort_incomplete_mpu_days - .map(|days| AbortIncompleteMpu { - days: IntValue(days as i64), - }), - expiration: rule - .expiration - .as_ref() - .map(Expiration::from_garage_lifecycle_expiration), - } - } -} - -impl Filter { - pub fn count(&self) -> i32 { - fn count(x: &Option) -> i32 { - x.as_ref().map(|_| 1).unwrap_or(0) - } - count(&self.prefix) + count(&self.size_gt) + count(&self.size_lt) - } - - pub fn validate_into_garage_lifecycle_filter( - self, - ) -> Result { - if self.count() > 0 && self.and.is_some() { - Err("Filter tag cannot contain both and another condition") - } else if let Some(and) = self.and { - if and.and.is_some() { - return Err("Nested tags"); - } - Ok(and.internal_into_garage_lifecycle_filter()) - } else if self.count() > 1 { - Err("Multiple Filter conditions must be wrapped in an tag") - } else { - Ok(self.internal_into_garage_lifecycle_filter()) - } - } - - fn internal_into_garage_lifecycle_filter(self) -> GarageLifecycleFilter { - GarageLifecycleFilter { - prefix: self.prefix.map(|x| x.0), - size_gt: self.size_gt.map(|x| x.0 as u64), - size_lt: self.size_lt.map(|x| x.0 as u64), - } - } - - pub fn from_garage_lifecycle_filter(rule: &GarageLifecycleFilter) -> Option { - let filter = Filter { - and: None, - prefix: rule.prefix.as_deref().map(Value::from), - size_gt: rule.size_gt.map(|x| IntValue(x as i64)), - size_lt: rule.size_lt.map(|x| IntValue(x as i64)), - }; - match filter.count() { - 0 => None, - 1 => Some(filter), - _ => Some(Filter { - and: Some(Box::new(filter)), - ..Default::default() - }), - } - } -} - -impl Expiration { - pub fn validate_into_garage_lifecycle_expiration( - self, - ) -> Result { - match (self.days, self.at_date) { - (Some(_), Some(_)) => Err("cannot have both and in "), - (None, None) => Err(" must contain either or "), - (Some(days), None) => Ok(GarageLifecycleExpiration::AfterDays(days.0 as usize)), - (None, Some(date)) => { - parse_lifecycle_date(&date.0)?; - Ok(GarageLifecycleExpiration::AtDate(date.0)) - } - } - } - - pub fn from_garage_lifecycle_expiration(exp: &GarageLifecycleExpiration) -> Self { - match exp { - GarageLifecycleExpiration::AfterDays(days) => Expiration { - days: Some(IntValue(*days as i64)), - at_date: None, - }, - GarageLifecycleExpiration::AtDate(date) => Expiration { - days: None, - at_date: Some(Value(date.to_string())), - }, - } - } -} - -#[cfg(test)] -mod tests { - use crate::unprettify_xml; - - use super::*; - - use quick_xml::de::from_str; - - #[test] - fn test_deserialize_lifecycle_config() -> Result<(), Error> { - let message = r#" - - - id1 - Enabled - - documents/ - - - 7 - - - - id2 - Enabled - - - logs/ - 1000000 - - - - 365 - - -"#; - let conf: LifecycleConfiguration = from_str(message).unwrap(); - let ref_value = LifecycleConfiguration { - xmlns: (), - lifecycle_rules: vec![ - LifecycleRule { - id: Some("id1".into()), - status: "Enabled".into(), - filter: Some(Filter { - prefix: Some("documents/".into()), - ..Default::default() - }), - expiration: None, - abort_incomplete_mpu: Some(AbortIncompleteMpu { days: IntValue(7) }), - }, - LifecycleRule { - id: Some("id2".into()), - status: "Enabled".into(), - filter: Some(Filter { - and: Some(Box::new(Filter { - prefix: Some("logs/".into()), - size_gt: Some(IntValue(1000000)), - ..Default::default() - })), - ..Default::default() - }), - expiration: Some(Expiration { - days: Some(IntValue(365)), - at_date: None, - }), - abort_incomplete_mpu: None, - }, - ], - }; - assert_eq! { - ref_value, - conf - }; - - let message2 = to_xml_with_header(&ref_value)?; - - assert_eq!(unprettify_xml(message), unprettify_xml(&message2)); - - // Check validation - let validated = ref_value - .validate_into_garage_lifecycle_config() - .ok_or_bad_request("invalid xml config")?; - - let ref_config = vec![ - GarageLifecycleRule { - id: Some("id1".into()), - enabled: true, - filter: GarageLifecycleFilter { - prefix: Some("documents/".into()), - ..Default::default() - }, - expiration: None, - abort_incomplete_mpu_days: Some(7), - }, - GarageLifecycleRule { - id: Some("id2".into()), - enabled: true, - filter: GarageLifecycleFilter { - prefix: Some("logs/".into()), - size_gt: Some(1000000), - ..Default::default() - }, - expiration: Some(GarageLifecycleExpiration::AfterDays(365)), - abort_incomplete_mpu_days: None, - }, - ]; - assert_eq!(validated, ref_config); - - let message3 = to_xml_with_header(&LifecycleConfiguration::from_garage_lifecycle_config( - &validated, - ))?; - assert_eq!(unprettify_xml(message), unprettify_xml(&message3)); - - Ok(()) - } -} diff --git a/src/api/s3/website.rs b/src/api/s3/website.rs index 4c14f922..d1241805 100644 --- a/src/api/s3/website.rs +++ b/src/api/s3/website.rs @@ -1,15 +1,15 @@ use quick_xml::de::from_reader; use hyper::{header::HeaderName, Request, Response, StatusCode}; -use serde::{Deserialize, Serialize}; -use garage_model::bucket_table::{self, Bucket, WebsiteConfig}; +use garage_model::bucket_table::Bucket; use garage_api_common::helpers::*; +use garage_api_common::xml::website::*; use crate::api_server::{ReqBody, ResBody}; use crate::error::*; -use crate::xml::{to_xml_with_header, xmlns_tag, IntValue, Value}; +use crate::xml::{to_xml_with_header, IntValue, Value}; pub const X_AMZ_WEBSITE_REDIRECT_LOCATION: HeaderName = HeaderName::from_static("x-amz-website-redirect-location"); @@ -107,377 +107,3 @@ pub async fn handle_put_website( .status(StatusCode::OK) .body(empty_body())?) } - -#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] -pub struct WebsiteConfiguration { - #[serde(rename = "@xmlns", serialize_with = "xmlns_tag", skip_deserializing)] - pub xmlns: (), - #[serde(rename = "ErrorDocument")] - pub error_document: Option, - #[serde(rename = "IndexDocument")] - pub index_document: Option, - #[serde(rename = "RedirectAllRequestsTo")] - pub redirect_all_requests_to: Option, - #[serde( - rename = "RoutingRules", - default, - skip_serializing_if = "RoutingRules::is_empty" - )] - pub routing_rules: RoutingRules, -} - -#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord, Default)] -pub struct RoutingRules { - #[serde(rename = "RoutingRule")] - pub rules: Vec, -} - -impl RoutingRules { - fn is_empty(&self) -> bool { - self.rules.is_empty() - } -} - -#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] -pub struct RoutingRule { - #[serde(rename = "Condition")] - pub condition: Option, - #[serde(rename = "Redirect")] - pub redirect: Redirect, -} - -#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] -pub struct Key { - #[serde(rename = "Key")] - pub key: Value, -} - -#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] -pub struct Suffix { - #[serde(rename = "Suffix")] - pub suffix: Value, -} - -#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] -pub struct Target { - #[serde(rename = "HostName")] - pub hostname: Value, - #[serde(rename = "Protocol")] - pub protocol: Option, -} - -#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] -pub struct Condition { - #[serde( - rename = "HttpErrorCodeReturnedEquals", - skip_serializing_if = "Option::is_none" - )] - pub http_error_code: Option, - #[serde(rename = "KeyPrefixEquals", skip_serializing_if = "Option::is_none")] - pub prefix: Option, -} - -#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] -pub struct Redirect { - #[serde(rename = "HostName", skip_serializing_if = "Option::is_none")] - pub hostname: Option, - #[serde(rename = "Protocol", skip_serializing_if = "Option::is_none")] - pub protocol: Option, - #[serde(rename = "HttpRedirectCode", skip_serializing_if = "Option::is_none")] - pub http_redirect_code: Option, - #[serde( - rename = "ReplaceKeyPrefixWith", - skip_serializing_if = "Option::is_none" - )] - pub replace_prefix: Option, - #[serde(rename = "ReplaceKeyWith", skip_serializing_if = "Option::is_none")] - pub replace_full: Option, -} - -impl WebsiteConfiguration { - pub fn validate(&self) -> Result<(), Error> { - if self.redirect_all_requests_to.is_some() - && (self.error_document.is_some() - || self.index_document.is_some() - || !self.routing_rules.is_empty()) - { - return Err(Error::bad_request( - "Bad XML: can't have RedirectAllRequestsTo and other fields", - )); - } - if let Some(ref ed) = self.error_document { - ed.validate()?; - } - if let Some(ref id) = self.index_document { - id.validate()?; - } - if let Some(ref rart) = self.redirect_all_requests_to { - rart.validate()?; - } - for rr in &self.routing_rules.rules { - rr.validate()?; - } - if self.routing_rules.rules.len() > 1000 { - // we will do linear scans, best to avoid overly long configuration. The - // limit was chosen arbitrarily - return Err(Error::bad_request( - "Bad XML: RoutingRules can't have more than 1000 child elements", - )); - } - - Ok(()) - } - - pub fn into_garage_website_config(self) -> Result { - if self.redirect_all_requests_to.is_some() { - Err(Error::NotImplemented( - "RedirectAllRequestsTo is not currently implemented in Garage, however its effect can be emulated using a single unconditional RoutingRule.".into(), - )) - } else { - Ok(WebsiteConfig { - index_document: self - .index_document - .map(|x| x.suffix.0) - .unwrap_or_else(|| "index.html".to_string()), - error_document: self.error_document.map(|x| x.key.0), - redirect_all: None, - routing_rules: self - .routing_rules - .rules - .into_iter() - .map(|rule| { - bucket_table::RoutingRule { - condition: rule.condition.map(|condition| { - bucket_table::RedirectCondition { - http_error_code: condition.http_error_code.map(|c| c.0 as u16), - prefix: condition.prefix.map(|p| p.0), - } - }), - redirect: bucket_table::Redirect { - hostname: rule.redirect.hostname.map(|h| h.0), - protocol: rule.redirect.protocol.map(|p| p.0), - // aws default to 301, which i find punitive in case of - // misconfiguration (can be permanently cached on the - // user agent) - http_redirect_code: rule - .redirect - .http_redirect_code - .map(|c| c.0 as u16) - .unwrap_or(302), - replace_key_prefix: rule.redirect.replace_prefix.map(|k| k.0), - replace_key: rule.redirect.replace_full.map(|k| k.0), - }, - } - }) - .collect(), - }) - } - } -} - -impl Key { - pub fn validate(&self) -> Result<(), Error> { - if self.key.0.is_empty() { - Err(Error::bad_request( - "Bad XML: error document specified but empty", - )) - } else { - Ok(()) - } - } -} - -impl Suffix { - pub fn validate(&self) -> Result<(), Error> { - if self.suffix.0.is_empty() | self.suffix.0.contains('/') { - Err(Error::bad_request( - "Bad XML: index document is empty or contains /", - )) - } else { - Ok(()) - } - } -} - -impl Target { - pub fn validate(&self) -> Result<(), Error> { - if let Some(ref protocol) = self.protocol { - if protocol.0 != "http" && protocol.0 != "https" { - return Err(Error::bad_request("Bad XML: invalid protocol")); - } - } - Ok(()) - } -} - -impl RoutingRule { - pub fn validate(&self) -> Result<(), Error> { - if let Some(condition) = &self.condition { - condition.validate()?; - } - self.redirect.validate() - } -} - -impl Condition { - pub fn validate(&self) -> Result { - if let Some(ref error_code) = self.http_error_code { - // TODO do other error codes make sense? Aws only allows 4xx and 5xx - if error_code.0 != 404 { - return Err(Error::bad_request( - "Bad XML: HttpErrorCodeReturnedEquals must be 404 or absent", - )); - } - } - Ok(self.prefix.is_some()) - } -} - -impl Redirect { - pub fn validate(&self) -> Result<(), Error> { - if self.replace_prefix.is_some() && self.replace_full.is_some() { - return Err(Error::bad_request( - "Bad XML: both ReplaceKeyPrefixWith and ReplaceKeyWith are set", - )); - } - if let Some(ref protocol) = self.protocol { - if protocol.0 != "http" && protocol.0 != "https" { - return Err(Error::bad_request("Bad XML: invalid protocol")); - } - } - if let Some(ref http_redirect_code) = self.http_redirect_code { - match http_redirect_code.0 { - // aws allows all 3xx except 300, but some are non-sensical (not modified, - // use proxy...) - 301 | 302 | 303 | 307 | 308 => { - if self.hostname.is_none() && self.protocol.is_some() { - return Err(Error::bad_request( - "Bad XML: HostName must be set if Protocol is set", - )); - } - } - // aws doesn't allow these codes, but netlify does, and it seems like a - // cool feature (change the page seen without changing the url shown by the - // user agent) - 200 | 404 => { - if self.hostname.is_some() || self.protocol.is_some() { - // hostname would mean different bucket, protocol doesn't make - // sense - return Err(Error::bad_request( - "Bad XML: an HttpRedirectCode of 200 is not acceptable alongside HostName or Protocol", - )); - } - } - _ => { - return Err(Error::bad_request("Bad XML: invalid HttpRedirectCode")); - } - } - } - Ok(()) - } -} - -#[cfg(test)] -mod tests { - use crate::unprettify_xml; - - use super::*; - - use quick_xml::de::from_str; - - #[test] - fn test_deserialize() -> Result<(), Error> { - let message = r#" - - - my-error-doc - - - my-index - - - garage.tld - https - - - - - 404 - prefix1 - - - gara.ge - http - 303 - prefix2 - fullkey - - - - - - - - 404 - missing - - - -"#; - let conf: WebsiteConfiguration = - from_str(message).expect("failed to deserialize xml in `WebsiteConfiguration`"); - let ref_value = WebsiteConfiguration { - xmlns: (), - error_document: Some(Key { - key: Value("my-error-doc".to_owned()), - }), - index_document: Some(Suffix { - suffix: Value("my-index".to_owned()), - }), - redirect_all_requests_to: Some(Target { - hostname: Value("garage.tld".to_owned()), - protocol: Some(Value("https".to_owned())), - }), - routing_rules: RoutingRules { - rules: vec![ - RoutingRule { - condition: Some(Condition { - http_error_code: Some(IntValue(404)), - prefix: Some(Value("prefix1".to_owned())), - }), - redirect: Redirect { - hostname: Some(Value("gara.ge".to_owned())), - protocol: Some(Value("http".to_owned())), - http_redirect_code: Some(IntValue(303)), - replace_prefix: Some(Value("prefix2".to_owned())), - replace_full: Some(Value("fullkey".to_owned())), - }, - }, - RoutingRule { - condition: Some(Condition { - http_error_code: None, - prefix: Some(Value("".to_owned())), - }), - redirect: Redirect { - hostname: None, - protocol: None, - http_redirect_code: Some(IntValue(404)), - replace_prefix: None, - replace_full: Some(Value("missing".to_owned())), - }, - }, - ], - }, - }; - assert_eq! { - ref_value, - conf - } - - let message2 = to_xml_with_header(&ref_value)?; - - assert_eq!(unprettify_xml(message), unprettify_xml(&message2)); - - Ok(()) - } -} diff --git a/src/api/s3/xml.rs b/src/api/s3/xml.rs index 952a46ee..5970c964 100644 --- a/src/api/s3/xml.rs +++ b/src/api/s3/xml.rs @@ -1,37 +1,6 @@ -use quick_xml::se::{self, EmptyElementHandling, QuoteLevel}; -use serde::{Deserialize, Serialize, Serializer}; +use serde::Serialize; -use crate::error::Error as ApiError; - -pub fn to_xml_with_header(x: &T) -> Result { - let mut xml = r#""#.to_string(); - - let mut ser = se::Serializer::new(&mut xml); - ser.set_quote_level(QuoteLevel::Full) - .empty_element_handling(EmptyElementHandling::Expanded); - let _serialized = x.serialize(ser)?; - Ok(xml) -} - -pub fn xmlns_tag(_v: &(), s: S) -> Result { - s.serialize_str("http://s3.amazonaws.com/doc/2006-03-01/") -} - -pub fn xmlns_xsi_tag(_v: &(), s: S) -> Result { - s.serialize_str("http://www.w3.org/2001/XMLSchema-instance") -} - -#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] -pub struct Value(#[serde(rename = "$value")] pub String); - -impl From<&str> for Value { - fn from(s: &str) -> Value { - Value(s.to_string()) - } -} - -#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] -pub struct IntValue(#[serde(rename = "$value")] pub i64); +pub use garage_api_common::xml::{to_xml_with_header, xmlns_tag, xmlns_xsi_tag, IntValue, Value}; #[derive(Debug, Serialize, PartialEq, Eq)] pub struct Bucket { @@ -378,6 +347,7 @@ pub struct AccessControlPolicy { #[cfg(test)] mod tests { use super::*; + use crate::error::Error as ApiError; use garage_util::time::*; From 64087172ff9dafe50fbff015290116ed0cab7fd6 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Fri, 6 Mar 2026 11:10:30 +0100 Subject: [PATCH 04/11] admin api: expose routing rules, cors rules and lifecycle rules --- Cargo.lock | 1 + doc/api/garage-admin-v2.json | 225 ++++++++++++++++++++++++++++++++ src/api/admin/api.rs | 12 +- src/api/admin/bucket.rs | 17 +++ src/api/common/Cargo.toml | 1 + src/api/common/xml/cors.rs | 13 +- src/api/common/xml/lifecycle.rs | 16 ++- src/api/common/xml/mod.rs | 7 +- src/api/common/xml/website.rs | 37 +++++- src/api/s3/website.rs | 18 +-- 10 files changed, 312 insertions(+), 35 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b60ae3d7..a1be0ae2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1558,6 +1558,7 @@ dependencies = [ "tokio", "tracing", "url", + "utoipa", ] [[package]] diff --git a/doc/api/garage-admin-v2.json b/doc/api/garage-admin-v2.json index a5ae37f8..23a6971e 100644 --- a/doc/api/garage-admin-v2.json +++ b/doc/api/garage-admin-v2.json @@ -2340,6 +2340,16 @@ "format": "int64", "description": "Total number of bytes used by objects in this bucket" }, + "corsRules": { + "type": [ + "array", + "null" + ], + "items": { + "$ref": "#/components/schemas/cors.Rule" + }, + "description": "CORS rules for this bucket" + }, "created": { "type": "string", "format": "date-time", @@ -2363,6 +2373,16 @@ }, "description": "List of access keys that have permissions granted on this bucket" }, + "lifecycleRules": { + "type": [ + "array", + "null" + ], + "items": { + "$ref": "#/components/schemas/lifecycle.Rule" + }, + "description": "Object lifecycle rules for this bucket" + }, "objects": { "type": "integer", "format": "int64", @@ -2423,6 +2443,12 @@ }, "indexDocument": { "type": "string" + }, + "routingRules": { + "type": "array", + "items": { + "$ref": "#/components/schemas/website.RoutingRule" + } } } }, @@ -4545,6 +4571,205 @@ ] } ] + }, + "cors.Rule": { + "type": "object", + "required": [ + "AllowedOrigin", + "AllowedMethod" + ], + "properties": { + "AllowedHeader": { + "type": "array", + "items": {} + }, + "AllowedMethod": { + "type": "array", + "items": {} + }, + "AllowedOrigin": { + "type": "array", + "items": {} + }, + "ExposeHeader": { + "type": "array", + "items": {} + }, + "ID": {}, + "MaxAgeSeconds": { + "oneOf": [ + { + "type": "null" + }, + { + "$ref": "#/components/schemas/xml.IntValue" + } + ] + } + } + }, + "lifecycle.AbortIncompleteMpu": { + "type": "object", + "required": [ + "DaysAfterInitiation" + ], + "properties": { + "DaysAfterInitiation": { + "$ref": "#/components/schemas/xml.IntValue" + } + } + }, + "lifecycle.Expiration": { + "type": "object", + "properties": { + "Date": {}, + "Days": { + "oneOf": [ + { + "type": "null" + }, + { + "$ref": "#/components/schemas/xml.IntValue" + } + ] + } + } + }, + "lifecycle.Filter": { + "type": "object", + "properties": { + "And": { + "oneOf": [ + { + "type": "null" + }, + { + "$ref": "#/components/schemas/lifecycle.Filter" + } + ] + }, + "ObjectSizeGreaterThan": { + "oneOf": [ + { + "type": "null" + }, + { + "$ref": "#/components/schemas/xml.IntValue" + } + ] + }, + "ObjectSizeLessThan": { + "oneOf": [ + { + "type": "null" + }, + { + "$ref": "#/components/schemas/xml.IntValue" + } + ] + }, + "Prefix": {} + } + }, + "lifecycle.Rule": { + "type": "object", + "required": [ + "Status" + ], + "properties": { + "AbortIncompleteMultipartUpload": { + "oneOf": [ + { + "type": "null" + }, + { + "$ref": "#/components/schemas/lifecycle.AbortIncompleteMpu" + } + ] + }, + "Expiration": { + "oneOf": [ + { + "type": "null" + }, + { + "$ref": "#/components/schemas/lifecycle.Expiration" + } + ] + }, + "Filter": { + "oneOf": [ + { + "type": "null" + }, + { + "$ref": "#/components/schemas/lifecycle.Filter" + } + ] + }, + "ID": {}, + "Status": {} + } + }, + "website.Condition": { + "type": "object", + "properties": { + "HttpErrorCodeReturnedEquals": { + "oneOf": [ + { + "type": "null" + }, + { + "$ref": "#/components/schemas/xml.IntValue" + } + ] + }, + "KeyPrefixEquals": {} + } + }, + "website.Redirect": { + "type": "object", + "properties": { + "HostName": {}, + "HttpRedirectCode": { + "oneOf": [ + { + "type": "null" + }, + { + "$ref": "#/components/schemas/xml.IntValue" + } + ] + }, + "Protocol": {}, + "ReplaceKeyPrefixWith": {}, + "ReplaceKeyWith": {} + } + }, + "website.RoutingRule": { + "type": "object", + "required": [ + "Redirect" + ], + "properties": { + "Condition": { + "oneOf": [ + { + "type": "null" + }, + { + "$ref": "#/components/schemas/website.Condition" + } + ] + }, + "Redirect": { + "$ref": "#/components/schemas/website.Redirect" + } + } + }, + "xml.IntValue": { + "type": "integer", + "format": "int64" } }, "securitySchemes": { diff --git a/src/api/admin/api.rs b/src/api/admin/api.rs index b375ecb9..5d751fbe 100644 --- a/src/api/admin/api.rs +++ b/src/api/admin/api.rs @@ -12,7 +12,7 @@ use garage_rpc::*; use garage_model::garage::Garage; -use garage_api_common::{common_error::CommonError, helpers::is_default}; +use garage_api_common::{common_error::CommonError, helpers::is_default, xml}; use crate::api_server::{find_matching_nodes, AdminRpc, AdminRpcResponse}; use crate::error::Error; @@ -857,9 +857,15 @@ pub struct GetBucketInfoResponse { pub global_aliases: Vec, /// Whether website access is enabled for this bucket pub website_access: bool, - #[serde(default)] /// Website configuration for this bucket pub website_config: Option, + // FIXME for v3: remove serde(default) for the two fields below + /// CORS rules for this bucket + #[serde(default)] + pub cors_rules: Option>, + /// Object lifecycle rules for this bucket + #[serde(default)] + pub lifecycle_rules: Option>, /// List of access keys that have permissions granted on this bucket pub keys: Vec, /// Number of objects in this bucket @@ -883,6 +889,8 @@ pub struct GetBucketInfoResponse { pub struct GetBucketInfoWebsiteResponse { pub index_document: String, pub error_document: Option, + #[serde(default)] + pub routing_rules: Vec, } #[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] diff --git a/src/api/admin/bucket.rs b/src/api/admin/bucket.rs index 61be02d9..2fc15f46 100644 --- a/src/api/admin/bucket.rs +++ b/src/api/admin/bucket.rs @@ -18,6 +18,7 @@ use garage_model::s3::mpu_table; use garage_model::s3::object_table::*; use garage_api_common::common_error::CommonError; +use garage_api_common::xml; use crate::api::*; use crate::error::*; @@ -693,8 +694,24 @@ async fn bucket_info_results( GetBucketInfoWebsiteResponse { index_document: wsc.index_document, error_document: wsc.error_document, + routing_rules: wsc + .routing_rules + .into_iter() + .map(xml::website::RoutingRule::from_garage_routing_rule) + .collect::>(), } }), + cors_rules: state.cors_config.get().as_ref().map(|rules| { + rules + .iter() + .map(xml::cors::CorsRule::from_garage_cors_rule) + .collect::>() + }), + lifecycle_rules: state.lifecycle_config.get().as_ref().map(|lc| { + lc.iter() + .map(xml::lifecycle::LifecycleRule::from_garage_lifecycle_rule) + .collect::>() + }), keys: relevant_keys .into_values() .filter_map(|key| { diff --git a/src/api/common/Cargo.toml b/src/api/common/Cargo.toml index 05c640db..8e9d074f 100644 --- a/src/api/common/Cargo.toml +++ b/src/api/common/Cargo.toml @@ -44,6 +44,7 @@ url.workspace = true quick-xml.workspace = true serde.workspace = true serde_json.workspace = true +utoipa.workspace = true opentelemetry.workspace = true diff --git a/src/api/common/xml/cors.rs b/src/api/common/xml/cors.rs index 63e3e618..ae1f9b72 100644 --- a/src/api/common/xml/cors.rs +++ b/src/api/common/xml/cors.rs @@ -1,4 +1,5 @@ use serde::{Deserialize, Serialize}; +use utoipa::ToSchema; use hyper::{header::HeaderName, Method}; @@ -16,7 +17,8 @@ pub struct CorsConfiguration { pub cors_rules: Vec, } -#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] +#[derive(Debug, ToSchema, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord, Clone)] +#[schema(as = cors::Rule)] pub struct CorsRule { #[serde(rename = "ID", skip_serializing_if = "Option::is_none")] pub id: Option, @@ -32,19 +34,22 @@ pub struct CorsRule { pub expose_headers: Vec, } -#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] +#[derive(Debug, ToSchema, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord, Clone)] +#[schema(as = cors::AllowedMethod)] pub struct AllowedMethod { #[serde(rename = "AllowedMethod")] pub allowed_method: Value, } -#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] +#[derive(Debug, ToSchema, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord, Clone)] +#[schema(as = cors::AllowedHeader)] pub struct AllowedHeader { #[serde(rename = "AllowedHeader")] pub allowed_header: Value, } -#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] +#[derive(Debug, ToSchema, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord, Clone)] +#[schema(as = cors::ExposedHeader)] pub struct ExposeHeader { #[serde(rename = "ExposeHeader")] pub expose_header: Value, diff --git a/src/api/common/xml/lifecycle.rs b/src/api/common/xml/lifecycle.rs index 2997d070..61f12a51 100644 --- a/src/api/common/xml/lifecycle.rs +++ b/src/api/common/xml/lifecycle.rs @@ -1,4 +1,5 @@ use serde::{Deserialize, Serialize}; +use utoipa::ToSchema; use garage_model::bucket_table::{ parse_lifecycle_date, LifecycleExpiration as GarageLifecycleExpiration, @@ -15,7 +16,8 @@ pub struct LifecycleConfiguration { pub lifecycle_rules: Vec, } -#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] +#[derive(Debug, ToSchema, Clone, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] +#[schema(as = lifecycle::Rule)] pub struct LifecycleRule { #[serde(rename = "ID", skip_serializing_if = "Option::is_none")] pub id: Option, @@ -37,9 +39,13 @@ pub struct LifecycleRule { pub abort_incomplete_mpu: Option, } -#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord, Default)] +#[derive( + Debug, ToSchema, Clone, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord, Default, +)] +#[schema(as = lifecycle::Filter)] pub struct Filter { #[serde(rename = "And", skip_serializing_if = "Option::is_none")] + #[schema(no_recursion)] pub and: Option>, #[serde(rename = "Prefix", skip_serializing_if = "Option::is_none")] pub prefix: Option, @@ -52,7 +58,8 @@ pub struct Filter { pub size_lt: Option, } -#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] +#[derive(Debug, ToSchema, Clone, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] +#[schema(as = lifecycle::Expiration)] pub struct Expiration { #[serde(rename = "Days", skip_serializing_if = "Option::is_none")] pub days: Option, @@ -60,7 +67,8 @@ pub struct Expiration { pub at_date: Option, } -#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] +#[derive(Debug, ToSchema, Clone, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] +#[schema(as = lifecycle::AbortIncompleteMpu)] pub struct AbortIncompleteMpu { #[serde(rename = "DaysAfterInitiation")] pub days: IntValue, diff --git a/src/api/common/xml/mod.rs b/src/api/common/xml/mod.rs index 701e2834..be07949c 100644 --- a/src/api/common/xml/mod.rs +++ b/src/api/common/xml/mod.rs @@ -3,6 +3,7 @@ pub mod lifecycle; pub mod website; use serde::{Deserialize, Serialize, Serializer}; +use utoipa::ToSchema; pub fn to_xml_with_header(x: &T) -> Result { use quick_xml::se::{self, EmptyElementHandling, QuoteLevel}; @@ -32,7 +33,8 @@ pub fn xmlns_xsi_tag(_v: &(), s: S) -> Result { s.serialize_str("http://www.w3.org/2001/XMLSchema-instance") } -#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] +#[derive(Debug, ToSchema, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord, Clone)] +#[schema(as = xml::Value)] pub struct Value(#[serde(rename = "$value")] pub String); impl From<&str> for Value { @@ -41,5 +43,6 @@ impl From<&str> for Value { } } -#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] +#[derive(Debug, ToSchema, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord, Clone)] +#[schema(as = xml::IntValue)] pub struct IntValue(#[serde(rename = "$value")] pub i64); diff --git a/src/api/common/xml/website.rs b/src/api/common/xml/website.rs index fc9d0b27..7759aa2c 100644 --- a/src/api/common/xml/website.rs +++ b/src/api/common/xml/website.rs @@ -1,6 +1,7 @@ use serde::{Deserialize, Serialize}; +use utoipa::ToSchema; -use garage_model::bucket_table::{self, WebsiteConfig}; +use garage_model::bucket_table::{self, RoutingRule as GarageRoutingRule, WebsiteConfig}; use crate::common_error::CommonError as Error; use crate::xml::{xmlns_tag, IntValue, Value}; @@ -35,7 +36,8 @@ impl RoutingRules { } } -#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] +#[derive(Debug, ToSchema, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord, Clone)] +#[schema(as = website::RoutingRule)] pub struct RoutingRule { #[serde(rename = "Condition")] pub condition: Option, @@ -43,19 +45,22 @@ pub struct RoutingRule { pub redirect: Redirect, } -#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] +#[derive(Debug, ToSchema, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord, Clone)] +#[schema(as = website::Key)] pub struct Key { #[serde(rename = "Key")] pub key: Value, } -#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] +#[derive(Debug, ToSchema, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord, Clone)] +#[schema(as = website::Suffix)] pub struct Suffix { #[serde(rename = "Suffix")] pub suffix: Value, } -#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] +#[derive(Debug, ToSchema, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord, Clone)] +#[schema(as = website::Target)] pub struct Target { #[serde(rename = "HostName")] pub hostname: Value, @@ -63,7 +68,8 @@ pub struct Target { pub protocol: Option, } -#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] +#[derive(Debug, ToSchema, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord, Clone)] +#[schema(as = website::Condition)] pub struct Condition { #[serde( rename = "HttpErrorCodeReturnedEquals", @@ -74,7 +80,8 @@ pub struct Condition { pub prefix: Option, } -#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] +#[derive(Debug, ToSchema, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord, Clone)] +#[schema(as = website::Redirect)] pub struct Redirect { #[serde(rename = "HostName", skip_serializing_if = "Option::is_none")] pub hostname: Option, @@ -214,6 +221,22 @@ impl RoutingRule { } self.redirect.validate() } + + pub fn from_garage_routing_rule(rule: GarageRoutingRule) -> Self { + RoutingRule { + condition: rule.condition.map(|cond| Condition { + http_error_code: cond.http_error_code.map(|c| IntValue(c as i64)), + prefix: cond.prefix.map(Value), + }), + redirect: Redirect { + hostname: rule.redirect.hostname.map(Value), + http_redirect_code: Some(IntValue(rule.redirect.http_redirect_code as i64)), + protocol: rule.redirect.protocol.map(Value), + replace_full: rule.redirect.replace_key.map(Value), + replace_prefix: rule.redirect.replace_key_prefix.map(Value), + }, + } + } } impl Condition { diff --git a/src/api/s3/website.rs b/src/api/s3/website.rs index d1241805..0010a300 100644 --- a/src/api/s3/website.rs +++ b/src/api/s3/website.rs @@ -9,7 +9,7 @@ use garage_api_common::xml::website::*; use crate::api_server::{ReqBody, ResBody}; use crate::error::*; -use crate::xml::{to_xml_with_header, IntValue, Value}; +use crate::xml::{to_xml_with_header, Value}; pub const X_AMZ_WEBSITE_REDIRECT_LOCATION: HeaderName = HeaderName::from_static("x-amz-website-redirect-location"); @@ -31,21 +31,7 @@ pub async fn handle_get_website(ctx: ReqCtx) -> Result, Error> .routing_rules .clone() .into_iter() - .map(|rule| RoutingRule { - condition: rule.condition.map(|cond| Condition { - http_error_code: cond.http_error_code.map(|c| IntValue(c as i64)), - prefix: cond.prefix.map(Value), - }), - redirect: Redirect { - hostname: rule.redirect.hostname.map(Value), - http_redirect_code: Some(IntValue( - rule.redirect.http_redirect_code as i64, - )), - protocol: rule.redirect.protocol.map(Value), - replace_full: rule.redirect.replace_key.map(Value), - replace_prefix: rule.redirect.replace_key_prefix.map(Value), - }, - }) + .map(RoutingRule::from_garage_routing_rule) .collect(), }, }; From 19e5f83164af4e262f1cb5b68e948dfed85c4b29 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Fri, 6 Mar 2026 12:20:03 +0100 Subject: [PATCH 05/11] admin api: update cors and lifecycle rules in UpdateBucket --- doc/api/garage-admin-v2.json | 18 ++++++++++++++++++ src/api/admin/api.rs | 4 ++++ src/api/admin/bucket.rs | 32 ++++++++++++++++++++++++++++++++ src/garage/cli/remote/bucket.rs | 4 ++++ 4 files changed, 58 insertions(+) diff --git a/doc/api/garage-admin-v2.json b/doc/api/garage-admin-v2.json index 23a6971e..c94825c4 100644 --- a/doc/api/garage-admin-v2.json +++ b/doc/api/garage-admin-v2.json @@ -4285,6 +4285,24 @@ "UpdateBucketRequestBody": { "type": "object", "properties": { + "corsRules": { + "type": [ + "array", + "null" + ], + "items": { + "$ref": "#/components/schemas/cors.Rule" + } + }, + "lifecycleRules": { + "type": [ + "array", + "null" + ], + "items": { + "$ref": "#/components/schemas/lifecycle.Rule" + } + }, "quotas": { "oneOf": [ { diff --git a/src/api/admin/api.rs b/src/api/admin/api.rs index 5d751fbe..23f6d65b 100644 --- a/src/api/admin/api.rs +++ b/src/api/admin/api.rs @@ -949,6 +949,10 @@ pub struct UpdateBucketResponse(pub GetBucketInfoResponse); pub struct UpdateBucketRequestBody { pub website_access: Option, pub quotas: Option, + #[serde(default)] + pub cors_rules: Option>, + #[serde(default)] + pub lifecycle_rules: Option>, } #[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] diff --git a/src/api/admin/bucket.rs b/src/api/admin/bucket.rs index 2fc15f46..78146430 100644 --- a/src/api/admin/bucket.rs +++ b/src/api/admin/bucket.rs @@ -323,6 +323,38 @@ impl RequestHandler for UpdateBucketRequest { }); } + if let Some(cr) = self.body.cors_rules { + let cors_config = if cr.is_empty() { + None + } else { + let cc = xml::cors::CorsConfiguration { + xmlns: (), + cors_rules: cr, + }; + cc.validate()?; + Some(cc.into_garage_cors_config()?) + }; + + state.cors_config.update(cors_config); + } + + if let Some(lr) = self.body.lifecycle_rules { + let lifecycle_config = if lr.is_empty() { + None + } else { + let lc = xml::lifecycle::LifecycleConfiguration { + xmlns: (), + lifecycle_rules: lr, + }; + Some( + lc.validate_into_garage_lifecycle_config() + .ok_or_bad_request("Invalid lifecycle configuration")?, + ) + }; + + state.lifecycle_config.update(lifecycle_config); + } + garage.bucket_table.insert(&bucket).await?; Ok(UpdateBucketResponse( diff --git a/src/garage/cli/remote/bucket.rs b/src/garage/cli/remote/bucket.rs index aa5fa5ea..64272d11 100644 --- a/src/garage/cli/remote/bucket.rs +++ b/src/garage/cli/remote/bucket.rs @@ -324,6 +324,8 @@ impl Cli { body: UpdateBucketRequestBody { website_access: Some(wa), quotas: None, + cors_rules: None, + lifecycle_rules: None, }, }) .await?; @@ -374,6 +376,8 @@ impl Cli { body: UpdateBucketRequestBody { website_access: None, quotas: Some(new_quotas), + cors_rules: None, + lifecycle_rules: None, }, }) .await?; From af5f68a34d5ac51e6fae63d76f44dd2fa80def06 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Fri, 6 Mar 2026 12:26:17 +0100 Subject: [PATCH 06/11] admin api: allow updating website routing rules --- doc/api/garage-admin-v2.json | 9 ++++++ src/api/admin/api.rs | 2 ++ src/api/admin/bucket.rs | 24 ++++++++++++++-- src/api/common/xml/website.rs | 50 +++++++++++++++++---------------- src/garage/cli/remote/bucket.rs | 2 ++ 5 files changed, 60 insertions(+), 27 deletions(-) diff --git a/doc/api/garage-admin-v2.json b/doc/api/garage-admin-v2.json index c94825c4..3b5e66a3 100644 --- a/doc/api/garage-admin-v2.json +++ b/doc/api/garage-admin-v2.json @@ -4348,6 +4348,15 @@ "string", "null" ] + }, + "routingRules": { + "type": [ + "array", + "null" + ], + "items": { + "$ref": "#/components/schemas/website.RoutingRule" + } } } }, diff --git a/src/api/admin/api.rs b/src/api/admin/api.rs index 23f6d65b..8ffd6d0d 100644 --- a/src/api/admin/api.rs +++ b/src/api/admin/api.rs @@ -961,6 +961,8 @@ pub struct UpdateBucketWebsiteAccess { pub enabled: bool, pub index_document: Option, pub error_document: Option, + #[serde(default)] + pub routing_rules: Option>, } // ---- DeleteBucket ---- diff --git a/src/api/admin/bucket.rs b/src/api/admin/bucket.rs index 78146430..93766c7b 100644 --- a/src/api/admin/bucket.rs +++ b/src/api/admin/bucket.rs @@ -294,10 +294,28 @@ impl RequestHandler for UpdateBucketRequest { if let Some(wa) = self.body.website_access { if wa.enabled { - let (redirect_all, routing_rules) = match state.website_config.get() { - Some(wc) => (wc.redirect_all.clone(), wc.routing_rules.clone()), - None => (None, Vec::new()), + let redirect_all = state + .website_config + .get() + .as_ref() + .and_then(|wc| wc.redirect_all.clone()); + + let routing_rules = if let Some(rr) = wa.routing_rules { + for r in rr.iter() { + r.validate()?; + } + rr.into_iter() + .map(xml::website::RoutingRule::into_garage_routing_rule) + .collect::>() + } else { + state + .website_config + .get() + .as_ref() + .map(|wc| wc.routing_rules.clone()) + .unwrap_or_default() }; + state.website_config.update(Some(WebsiteConfig { index_document: wa.index_document.ok_or_bad_request( "Please specify indexDocument when enabling website access.", diff --git a/src/api/common/xml/website.rs b/src/api/common/xml/website.rs index 7759aa2c..a3889cda 100644 --- a/src/api/common/xml/website.rs +++ b/src/api/common/xml/website.rs @@ -149,30 +149,7 @@ impl WebsiteConfiguration { .routing_rules .rules .into_iter() - .map(|rule| { - bucket_table::RoutingRule { - condition: rule.condition.map(|condition| { - bucket_table::RedirectCondition { - http_error_code: condition.http_error_code.map(|c| c.0 as u16), - prefix: condition.prefix.map(|p| p.0), - } - }), - redirect: bucket_table::Redirect { - hostname: rule.redirect.hostname.map(|h| h.0), - protocol: rule.redirect.protocol.map(|p| p.0), - // aws default to 301, which i find punitive in case of - // misconfiguration (can be permanently cached on the - // user agent) - http_redirect_code: rule - .redirect - .http_redirect_code - .map(|c| c.0 as u16) - .unwrap_or(302), - replace_key_prefix: rule.redirect.replace_prefix.map(|k| k.0), - replace_key: rule.redirect.replace_full.map(|k| k.0), - }, - } - }) + .map(RoutingRule::into_garage_routing_rule) .collect(), }) } @@ -237,6 +214,31 @@ impl RoutingRule { }, } } + + pub fn into_garage_routing_rule(self) -> bucket_table::RoutingRule { + bucket_table::RoutingRule { + condition: self + .condition + .map(|condition| bucket_table::RedirectCondition { + http_error_code: condition.http_error_code.map(|c| c.0 as u16), + prefix: condition.prefix.map(|p| p.0), + }), + redirect: bucket_table::Redirect { + hostname: self.redirect.hostname.map(|h| h.0), + protocol: self.redirect.protocol.map(|p| p.0), + // aws default to 301, which i find punitive in case of + // misconfiguration (can be permanently cached on the + // user agent) + http_redirect_code: self + .redirect + .http_redirect_code + .map(|c| c.0 as u16) + .unwrap_or(302), + replace_key_prefix: self.redirect.replace_prefix.map(|k| k.0), + replace_key: self.redirect.replace_full.map(|k| k.0), + }, + } + } } impl Condition { diff --git a/src/garage/cli/remote/bucket.rs b/src/garage/cli/remote/bucket.rs index 64272d11..edfeaca5 100644 --- a/src/garage/cli/remote/bucket.rs +++ b/src/garage/cli/remote/bucket.rs @@ -309,12 +309,14 @@ impl Cli { error_document: opt .error_document .or_else(|| bucket_website_config.and_then(|x| x.error_document.clone())), + routing_rules: None, } } else { UpdateBucketWebsiteAccess { enabled: false, index_document: None, error_document: None, + routing_rules: None, } }; From 8abd0fee8616db4e0666b1ab186d815ee6b3b9c8 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Fri, 6 Mar 2026 12:50:11 +0100 Subject: [PATCH 07/11] admin api: add fixme comments for cleanup for v3 release --- src/api/admin/api.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/api/admin/api.rs b/src/api/admin/api.rs index 8ffd6d0d..c0630e21 100644 --- a/src/api/admin/api.rs +++ b/src/api/admin/api.rs @@ -284,9 +284,11 @@ pub struct GetClusterStatisticsRequest; #[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] #[serde(rename_all = "camelCase")] pub struct GetClusterStatisticsResponse { + // FIXME for v3: remove freeform field and move display logic to garage crate /// cluster statistics as a free-form string, kept for compatibility with nodes /// running older v2.x versions of garage pub freeform: String, + // FIXME for v3: remove serde(default) for all three fields below /// available storage space for object data in the entire cluster, in bytes #[serde(default)] pub data_avail: u64, @@ -889,6 +891,7 @@ pub struct GetBucketInfoResponse { pub struct GetBucketInfoWebsiteResponse { pub index_document: String, pub error_document: Option, + // FIXME for v3: remove serde(default) for field below #[serde(default)] pub routing_rules: Vec, } @@ -949,6 +952,7 @@ pub struct UpdateBucketResponse(pub GetBucketInfoResponse); pub struct UpdateBucketRequestBody { pub website_access: Option, pub quotas: Option, + // FIXME for v3: remove serde(default) for the two fields below #[serde(default)] pub cors_rules: Option>, #[serde(default)] @@ -961,6 +965,7 @@ pub struct UpdateBucketWebsiteAccess { pub enabled: bool, pub index_document: Option, pub error_document: Option, + // FIXME for v3: remove serde(default) for field below #[serde(default)] pub routing_rules: Option>, } @@ -1137,6 +1142,7 @@ pub struct LocalGetNodeInfoRequest; #[serde(rename_all = "camelCase")] pub struct LocalGetNodeInfoResponse { pub node_id: String, + // FIXME for v3: remove serde(default) for field below /// hostname of this node #[serde(default)] pub hostname: String, @@ -1158,9 +1164,11 @@ pub struct LocalGetNodeStatisticsRequest; #[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] #[serde(rename_all = "camelCase")] pub struct LocalGetNodeStatisticsResponse { + // FIXME for v3: remove freeform field and move display logic to garage crate /// node statistics as a free-form string, kept for compatibility with nodes /// running older v2.x versions of garage pub freeform: String, + // FIXME for v3: remove serde(default) for fields below /// metadata table statistics #[serde(default)] pub table_stats: Vec, From de10dc43d5da81503749ed9d65034a2297f9eca9 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Fri, 6 Mar 2026 15:27:12 +0100 Subject: [PATCH 08/11] admin api: return total buckets, objects and bytes in GetClusterStatistics --- doc/api/garage-admin-v2.json | 20 +++++++++++- src/api/admin/api.rs | 12 +++++++- src/api/admin/bucket.rs | 2 +- src/api/admin/cluster.rs | 60 +++++++++++++++++++++++++++++++++--- 4 files changed, 87 insertions(+), 7 deletions(-) diff --git a/doc/api/garage-admin-v2.json b/doc/api/garage-admin-v2.json index 3b5e66a3..c63feec4 100644 --- a/doc/api/garage-admin-v2.json +++ b/doc/api/garage-admin-v2.json @@ -2607,6 +2607,12 @@ "freeform" ], "properties": { + "bucketCount": { + "type": "integer", + "format": "int64", + "description": "number of buckets in the cluster", + "minimum": 0 + }, "dataAvail": { "type": "integer", "format": "int64", @@ -2617,7 +2623,7 @@ "type": "string", "description": "cluster statistics as a free-form string, kept for compatibility with nodes\nrunning older v2.x versions of garage" }, - "incompleteInfo": { + "incompleteAvailInfo": { "type": "boolean", "description": "true if the available storage space statistics are imprecise due to missing\ninformation of disconnected nodes. When this is the case, the actual\nspace available in the cluster might be lower than the reported values." }, @@ -2626,6 +2632,18 @@ "format": "int64", "description": "available storage space for object metadata in the entire cluster, in bytes", "minimum": 0 + }, + "totalObjectBytes": { + "type": "integer", + "format": "int64", + "description": "total size of objects stored in all buckets, before compression, deduplication and\nreplication (this is NOT equivalent to actual disk usage in the cluster)", + "minimum": 0 + }, + "totalObjectCount": { + "type": "integer", + "format": "int64", + "description": "total number of objects stored in all buckets", + "minimum": 0 } } }, diff --git a/src/api/admin/api.rs b/src/api/admin/api.rs index c0630e21..53bb2f0b 100644 --- a/src/api/admin/api.rs +++ b/src/api/admin/api.rs @@ -299,7 +299,17 @@ pub struct GetClusterStatisticsResponse { /// information of disconnected nodes. When this is the case, the actual /// space available in the cluster might be lower than the reported values. #[serde(default)] - pub incomplete_info: bool, + pub incomplete_avail_info: bool, + /// number of buckets in the cluster + #[serde(default)] + pub bucket_count: u64, + /// total number of objects stored in all buckets + #[serde(default)] + pub total_object_count: u64, + /// total size of objects stored in all buckets, before compression, deduplication and + /// replication (this is NOT equivalent to actual disk usage in the cluster) + #[serde(default)] + pub total_object_bytes: u64, } // ---- ConnectClusterNodes ---- diff --git a/src/api/admin/bucket.rs b/src/api/admin/bucket.rs index 93766c7b..e0b7b3ad 100644 --- a/src/api/admin/bucket.rs +++ b/src/api/admin/bucket.rs @@ -38,7 +38,7 @@ impl RequestHandler for ListBucketsRequest { &EmptyKey, None, Some(DeletedFilter::NotDeleted), - 10000, + 1_000_000, EnumerationOrder::Forward, ) .await?; diff --git a/src/api/admin/cluster.rs b/src/api/admin/cluster.rs index 96efa7f3..eee657e3 100644 --- a/src/api/admin/cluster.rs +++ b/src/api/admin/cluster.rs @@ -8,8 +8,10 @@ use garage_util::data::*; use garage_rpc::layout; use garage_rpc::layout::PARTITION_BITS; +use garage_table::*; use garage_model::garage::Garage; +use garage_model::s3::object_table; use crate::api::*; use crate::error::*; @@ -152,7 +154,6 @@ impl RequestHandler for GetClusterHealthRequest { impl RequestHandler for GetClusterStatisticsRequest { type Response = GetClusterStatisticsResponse; - // FIXME: return this as a JSON struct instead of text async fn handle( self, garage: &Arc, @@ -160,8 +161,45 @@ impl RequestHandler for GetClusterStatisticsRequest { ) -> Result { let mut ret = String::new(); - // Gather storage node and free space statistics for current nodes + // Gather info on number of buckets, objects and object size + let buckets = garage + .bucket_table + .get_range( + &EmptyKey, + None, + Some(DeletedFilter::NotDeleted), + 1_000_000, + EnumerationOrder::Forward, + ) + .await?; + + let bucket_stats = futures::future::try_join_all( + buckets + .iter() + .map(|b| garage.object_counter_table.table.get(&b.id, &EmptyKey)), + ) + .await?; + let layout = &garage.system.cluster_layout(); + + let bucket_stats = bucket_stats + .into_iter() + .filter_map(|cnt| cnt.map(|x| x.filtered_values(layout))) + .collect::>(); + + let bucket_count = buckets.len() as u64; + let total_object_count = bucket_stats + .iter() + .clone() + .map(|cnt| *cnt.get(object_table::OBJECTS).unwrap_or(&0) as u64) + .sum(); + let total_object_bytes = bucket_stats + .iter() + .clone() + .map(|cnt| *cnt.get(object_table::BYTES).unwrap_or(&0) as u64) + .sum(); + + // Gather storage node and free space statistics for current nodes let mut node_partition_count = HashMap::::new(); if let Ok(current_layout) = layout.current() { for short_id in current_layout.ring_assignment_data.iter() { @@ -242,9 +280,20 @@ impl RequestHandler for GetClusterStatisticsRequest { let incomplete_info = meta_part_avail.len() < node_partition_count.len() || data_part_avail.len() < node_partition_count.len(); + // Display bucket statistics + let bucket_stats = vec![ + format!("Number of buckets:\t{}", bucket_count), + format!("Total number of objects:\t{}", total_object_count), + format!( + "Total size of objects:\t{}", + bytesize::ByteSize(total_object_bytes) + ), + ]; + writeln!(&mut ret, "\n{}", format_table_to_string(bucket_stats)).unwrap(); + writeln!( &mut ret, - "\nEstimated available storage space cluster-wide (might be lower in practice):" + "Estimated available storage space cluster-wide (might be lower in practice):" ) .unwrap(); if incomplete_info { @@ -264,7 +313,10 @@ impl RequestHandler for GetClusterStatisticsRequest { freeform: ret, metadata_avail, data_avail, - incomplete_info, + incomplete_avail_info: incomplete_info, + bucket_count, + total_object_count, + total_object_bytes, }) } } From 456602036019440e6f1c16c64c49be6e956a78a2 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Fri, 6 Mar 2026 16:54:45 +0100 Subject: [PATCH 09/11] admin api: convert new fields to Option --- doc/api/garage-admin-v2.json | 77 +++++++++++++++++++++++++++++------- src/api/admin/api.rs | 49 ++++++++++++----------- src/api/admin/bucket.rs | 11 +++--- src/api/admin/cluster.rs | 12 +++--- src/api/admin/node.rs | 6 +-- 5 files changed, 102 insertions(+), 53 deletions(-) diff --git a/doc/api/garage-admin-v2.json b/doc/api/garage-admin-v2.json index c63feec4..df6620a0 100644 --- a/doc/api/garage-admin-v2.json +++ b/doc/api/garage-admin-v2.json @@ -2445,7 +2445,10 @@ "type": "string" }, "routingRules": { - "type": "array", + "type": [ + "array", + "null" + ], "items": { "$ref": "#/components/schemas/website.RoutingRule" } @@ -2608,13 +2611,19 @@ ], "properties": { "bucketCount": { - "type": "integer", + "type": [ + "integer", + "null" + ], "format": "int64", "description": "number of buckets in the cluster", "minimum": 0 }, "dataAvail": { - "type": "integer", + "type": [ + "integer", + "null" + ], "format": "int64", "description": "available storage space for object data in the entire cluster, in bytes", "minimum": 0 @@ -2624,23 +2633,35 @@ "description": "cluster statistics as a free-form string, kept for compatibility with nodes\nrunning older v2.x versions of garage" }, "incompleteAvailInfo": { - "type": "boolean", + "type": [ + "boolean", + "null" + ], "description": "true if the available storage space statistics are imprecise due to missing\ninformation of disconnected nodes. When this is the case, the actual\nspace available in the cluster might be lower than the reported values." }, "metadataAvail": { - "type": "integer", + "type": [ + "integer", + "null" + ], "format": "int64", "description": "available storage space for object metadata in the entire cluster, in bytes", "minimum": 0 }, "totalObjectBytes": { - "type": "integer", + "type": [ + "integer", + "null" + ], "format": "int64", "description": "total size of objects stored in all buckets, before compression, deduplication and\nreplication (this is NOT equivalent to actual disk usage in the cluster)", "minimum": 0 }, "totalObjectCount": { - "type": "integer", + "type": [ + "integer", + "null" + ], "format": "int64", "description": "total number of objects stored in all buckets", "minimum": 0 @@ -3134,7 +3155,10 @@ "description": "garage version running on this node" }, "hostname": { - "type": "string", + "type": [ + "string", + "null" + ], "description": "hostname of this node" }, "nodeId": { @@ -3153,15 +3177,25 @@ ], "properties": { "blockManagerStats": { - "$ref": "#/components/schemas/NodeBlockManagerStats", - "description": "block manager statistics" + "oneOf": [ + { + "type": "null" + }, + { + "$ref": "#/components/schemas/NodeBlockManagerStats", + "description": "block manager statistics" + } + ] }, "freeform": { "type": "string", "description": "node statistics as a free-form string, kept for compatibility with nodes\nrunning older v2.x versions of garage" }, "tableStats": { - "type": "array", + "type": [ + "array", + "null" + ], "items": { "$ref": "#/components/schemas/NodeTableStats" }, @@ -3484,7 +3518,10 @@ "description": "garage version running on this node" }, "hostname": { - "type": "string", + "type": [ + "string", + "null" + ], "description": "hostname of this node" }, "nodeId": { @@ -3529,15 +3566,25 @@ ], "properties": { "blockManagerStats": { - "$ref": "#/components/schemas/NodeBlockManagerStats", - "description": "block manager statistics" + "oneOf": [ + { + "type": "null" + }, + { + "$ref": "#/components/schemas/NodeBlockManagerStats", + "description": "block manager statistics" + } + ] }, "freeform": { "type": "string", "description": "node statistics as a free-form string, kept for compatibility with nodes\nrunning older v2.x versions of garage" }, "tableStats": { - "type": "array", + "type": [ + "array", + "null" + ], "items": { "$ref": "#/components/schemas/NodeTableStats" }, diff --git a/src/api/admin/api.rs b/src/api/admin/api.rs index 53bb2f0b..ed145ea7 100644 --- a/src/api/admin/api.rs +++ b/src/api/admin/api.rs @@ -288,28 +288,28 @@ pub struct GetClusterStatisticsResponse { /// cluster statistics as a free-form string, kept for compatibility with nodes /// running older v2.x versions of garage pub freeform: String, - // FIXME for v3: remove serde(default) for all three fields below + // FIXME for v3: remove Option<> and serde(default) for all fields below /// available storage space for object data in the entire cluster, in bytes - #[serde(default)] - pub data_avail: u64, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub data_avail: Option, /// available storage space for object metadata in the entire cluster, in bytes - #[serde(default)] - pub metadata_avail: u64, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub metadata_avail: Option, /// true if the available storage space statistics are imprecise due to missing /// information of disconnected nodes. When this is the case, the actual /// space available in the cluster might be lower than the reported values. - #[serde(default)] - pub incomplete_avail_info: bool, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub incomplete_avail_info: Option, /// number of buckets in the cluster - #[serde(default)] - pub bucket_count: u64, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub bucket_count: Option, /// total number of objects stored in all buckets - #[serde(default)] - pub total_object_count: u64, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub total_object_count: Option, /// total size of objects stored in all buckets, before compression, deduplication and /// replication (this is NOT equivalent to actual disk usage in the cluster) - #[serde(default)] - pub total_object_bytes: u64, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub total_object_bytes: Option, } // ---- ConnectClusterNodes ---- @@ -870,13 +870,14 @@ pub struct GetBucketInfoResponse { /// Whether website access is enabled for this bucket pub website_access: bool, /// Website configuration for this bucket + #[serde(default, skip_serializing_if = "Option::is_none")] pub website_config: Option, // FIXME for v3: remove serde(default) for the two fields below /// CORS rules for this bucket - #[serde(default)] + #[serde(default, skip_serializing_if = "Option::is_none")] pub cors_rules: Option>, /// Object lifecycle rules for this bucket - #[serde(default)] + #[serde(default, skip_serializing_if = "Option::is_none")] pub lifecycle_rules: Option>, /// List of access keys that have permissions granted on this bucket pub keys: Vec, @@ -902,8 +903,8 @@ pub struct GetBucketInfoWebsiteResponse { pub index_document: String, pub error_document: Option, // FIXME for v3: remove serde(default) for field below - #[serde(default)] - pub routing_rules: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub routing_rules: Option>, } #[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] @@ -1152,10 +1153,10 @@ pub struct LocalGetNodeInfoRequest; #[serde(rename_all = "camelCase")] pub struct LocalGetNodeInfoResponse { pub node_id: String, - // FIXME for v3: remove serde(default) for field below + // FIXME for v3: remove Option<> and serde(default) for field below /// hostname of this node - #[serde(default)] - pub hostname: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub hostname: Option, /// garage version running on this node pub garage_version: String, /// build-time features enabled for this garage release @@ -1180,11 +1181,11 @@ pub struct LocalGetNodeStatisticsResponse { pub freeform: String, // FIXME for v3: remove serde(default) for fields below /// metadata table statistics - #[serde(default)] - pub table_stats: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub table_stats: Option>, /// block manager statistics - #[serde(default)] - pub block_manager_stats: NodeBlockManagerStats, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub block_manager_stats: Option, } #[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] diff --git a/src/api/admin/bucket.rs b/src/api/admin/bucket.rs index e0b7b3ad..e723d3b4 100644 --- a/src/api/admin/bucket.rs +++ b/src/api/admin/bucket.rs @@ -744,11 +744,12 @@ async fn bucket_info_results( GetBucketInfoWebsiteResponse { index_document: wsc.index_document, error_document: wsc.error_document, - routing_rules: wsc - .routing_rules - .into_iter() - .map(xml::website::RoutingRule::from_garage_routing_rule) - .collect::>(), + routing_rules: Some( + wsc.routing_rules + .into_iter() + .map(xml::website::RoutingRule::from_garage_routing_rule) + .collect::>(), + ), } }), cors_rules: state.cors_config.get().as_ref().map(|rules| { diff --git a/src/api/admin/cluster.rs b/src/api/admin/cluster.rs index eee657e3..7284be5f 100644 --- a/src/api/admin/cluster.rs +++ b/src/api/admin/cluster.rs @@ -311,12 +311,12 @@ impl RequestHandler for GetClusterStatisticsRequest { Ok(GetClusterStatisticsResponse { freeform: ret, - metadata_avail, - data_avail, - incomplete_avail_info: incomplete_info, - bucket_count, - total_object_count, - total_object_bytes, + metadata_avail: Some(metadata_avail), + data_avail: Some(data_avail), + incomplete_avail_info: Some(incomplete_info), + bucket_count: Some(bucket_count), + total_object_count: Some(total_object_count), + total_object_bytes: Some(total_object_bytes), }) } } diff --git a/src/api/admin/node.rs b/src/api/admin/node.rs index 6e46a46d..12163f18 100644 --- a/src/api/admin/node.rs +++ b/src/api/admin/node.rs @@ -27,7 +27,7 @@ impl RequestHandler for LocalGetNodeInfoRequest { Ok(LocalGetNodeInfoResponse { node_id: hex::encode(garage.system.id), - hostname, + hostname: Some(hostname), garage_version: garage_util::version::garage_version().to_string(), garage_features: garage_util::version::garage_features() .map(|features| features.iter().map(ToString::to_string).collect()), @@ -146,8 +146,8 @@ impl RequestHandler for LocalGetNodeStatisticsRequest { Ok(LocalGetNodeStatisticsResponse { freeform: ret, - table_stats, - block_manager_stats, + table_stats: Some(table_stats), + block_manager_stats: Some(block_manager_stats), }) } } From 6131318c80b50c882d6569bce57a712409653dcf Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Tue, 10 Mar 2026 11:35:35 +0100 Subject: [PATCH 10/11] admin api: don't gather all bucket statistics if too many buckets --- src/api/admin/cluster.rs | 78 ++++++++++++++++++++++++---------------- 1 file changed, 48 insertions(+), 30 deletions(-) diff --git a/src/api/admin/cluster.rs b/src/api/admin/cluster.rs index 7284be5f..8477b116 100644 --- a/src/api/admin/cluster.rs +++ b/src/api/admin/cluster.rs @@ -173,31 +173,47 @@ impl RequestHandler for GetClusterStatisticsRequest { ) .await?; - let bucket_stats = futures::future::try_join_all( - buckets - .iter() - .map(|b| garage.object_counter_table.table.get(&b.id, &EmptyKey)), - ) - .await?; + let bucket_stats_opt = if buckets.len() < 1000 { + Some( + futures::future::try_join_all( + buckets + .iter() + .map(|b| garage.object_counter_table.table.get(&b.id, &EmptyKey)), + ) + .await?, + ) + } else { + None + }; let layout = &garage.system.cluster_layout(); - let bucket_stats = bucket_stats - .into_iter() - .filter_map(|cnt| cnt.map(|x| x.filtered_values(layout))) - .collect::>(); - let bucket_count = buckets.len() as u64; - let total_object_count = bucket_stats - .iter() - .clone() - .map(|cnt| *cnt.get(object_table::OBJECTS).unwrap_or(&0) as u64) - .sum(); - let total_object_bytes = bucket_stats - .iter() - .clone() - .map(|cnt| *cnt.get(object_table::BYTES).unwrap_or(&0) as u64) - .sum(); + let (total_object_count, total_object_bytes); + if let Some(bucket_stats) = bucket_stats_opt { + let bucket_stats = bucket_stats + .into_iter() + .filter_map(|cnt| cnt.map(|x| x.filtered_values(layout))) + .collect::>(); + + total_object_count = Some( + bucket_stats + .iter() + .clone() + .map(|cnt| *cnt.get(object_table::OBJECTS).unwrap_or(&0) as u64) + .sum(), + ); + total_object_bytes = Some( + bucket_stats + .iter() + .clone() + .map(|cnt| *cnt.get(object_table::BYTES).unwrap_or(&0) as u64) + .sum(), + ); + } else { + total_object_count = None; + total_object_bytes = None; + } // Gather storage node and free space statistics for current nodes let mut node_partition_count = HashMap::::new(); @@ -281,14 +297,16 @@ impl RequestHandler for GetClusterStatisticsRequest { || data_part_avail.len() < node_partition_count.len(); // Display bucket statistics - let bucket_stats = vec![ - format!("Number of buckets:\t{}", bucket_count), - format!("Total number of objects:\t{}", total_object_count), - format!( + let mut bucket_stats = vec![format!("Number of buckets:\t{}", bucket_count)]; + if let Some(toc) = total_object_count { + bucket_stats.push(format!("Total number of objects:\t{}", toc)); + } + if let Some(tob) = total_object_bytes { + bucket_stats.push(format!( "Total size of objects:\t{}", - bytesize::ByteSize(total_object_bytes) - ), - ]; + bytesize::ByteSize(tob) + )); + } writeln!(&mut ret, "\n{}", format_table_to_string(bucket_stats)).unwrap(); writeln!( @@ -315,8 +333,8 @@ impl RequestHandler for GetClusterStatisticsRequest { data_avail: Some(data_avail), incomplete_avail_info: Some(incomplete_info), bucket_count: Some(bucket_count), - total_object_count: Some(total_object_count), - total_object_bytes: Some(total_object_bytes), + total_object_count, + total_object_bytes, }) } } From b81eae3f6517a93235e7741f84d586f3f26ca252 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Tue, 17 Mar 2026 11:36:18 +0100 Subject: [PATCH 11/11] admin api: don't fail in getclusterstatistics when counting total objects/bytes --- src/api/admin/cluster.rs | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/src/api/admin/cluster.rs b/src/api/admin/cluster.rs index 8477b116..15fe370a 100644 --- a/src/api/admin/cluster.rs +++ b/src/api/admin/cluster.rs @@ -174,14 +174,13 @@ impl RequestHandler for GetClusterStatisticsRequest { .await?; let bucket_stats_opt = if buckets.len() < 1000 { - Some( - futures::future::try_join_all( - buckets - .iter() - .map(|b| garage.object_counter_table.table.get(&b.id, &EmptyKey)), - ) - .await?, + futures::future::try_join_all( + buckets + .iter() + .map(|b| garage.object_counter_table.table.get(&b.id, &EmptyKey)), ) + .await + .ok() } else { None };