// Copyright 2024 RustFS Team // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. //! Bulk Operations for Swift API //! //! This module implements bulk operations that allow batch processing of //! multiple objects in a single request, improving efficiency for operations //! that affect many files. //! //! # Operations //! //! ## Bulk Delete //! //! Delete multiple objects in a single request. //! //! **Endpoint**: `DELETE /?bulk-delete` //! //! **Request Body**: Newline-separated list of object paths //! ```text //! /container1/object1.txt //! /container2/folder/object2.txt //! /container1/object3.txt //! ``` //! //! **Response**: JSON with results for each object //! ```json //! { //! "Number Deleted": 2, //! "Number Not Found": 1, //! "Errors": [], //! "Response Status": "200 OK", //! "Response Body": "" //! } //! ``` //! //! ## Bulk Extract //! //! Extract files from an uploaded archive into a container. //! //! **Endpoint**: `PUT /{container}?extract-archive=tar` (or tar.gz, tar.bz2) //! //! **Request Body**: Archive file contents //! //! **Response**: JSON with results for each extracted file //! ```json //! { //! "Number Files Created": 10, //! "Errors": [], //! "Response Status": "201 Created", //! "Response Body": "" //! } //! ``` //! //! # Examples //! //! ```bash //! # Bulk delete //! echo -e "/container/file1.txt\n/container/file2.txt" | \ //! swift delete --bulk //! //! # Bulk extract //! tar czf archive.tar.gz files/ //! swift upload container --extract-archive archive.tar.gz //! ``` use super::{SwiftError, SwiftResult, container, object}; use axum::http::{Response, StatusCode}; use futures::StreamExt; use rustfs_credentials::Credentials; use s3s::Body; use serde::{Deserialize, Serialize}; use tracing::{debug, error}; const LOG_COMPONENT_PROTOCOLS: &str = "protocols"; const LOG_SUBSYSTEM_SWIFT_BULK: &str = "swift_bulk"; const EVENT_SWIFT_BULK_DELETE_STATE: &str = "swift_bulk_delete_state"; const EVENT_SWIFT_BULK_EXTRACT_STATE: &str = "swift_bulk_extract_state"; /// Result of a single delete operation #[derive(Debug, Clone, Serialize, Deserialize)] pub struct DeleteResult { /// Object path that was deleted pub path: String, /// HTTP status code for this operation pub status: u16, /// Error message if deletion failed #[serde(skip_serializing_if = "Option::is_none")] pub error: Option, } /// Bulk delete response #[derive(Debug, Clone, Serialize, Deserialize)] pub struct BulkDeleteResponse { /// Number of objects successfully deleted #[serde(rename = "Number Deleted")] pub number_deleted: usize, /// Number of objects not found #[serde(rename = "Number Not Found")] pub number_not_found: usize, /// List of errors encountered #[serde(rename = "Errors")] pub errors: Vec>, /// Overall response status #[serde(rename = "Response Status")] pub response_status: String, /// Response body (usually empty) #[serde(rename = "Response Body")] pub response_body: String, } impl Default for BulkDeleteResponse { fn default() -> Self { Self { number_deleted: 0, number_not_found: 0, errors: Vec::new(), response_status: "200 OK".to_string(), response_body: String::new(), } } } /// Result of a single file extraction #[derive(Debug, Clone, Serialize, Deserialize)] pub struct ExtractResult { /// File path that was extracted pub path: String, /// HTTP status code for this operation pub status: u16, /// Error message if extraction failed #[serde(skip_serializing_if = "Option::is_none")] pub error: Option, } /// Bulk extract response #[derive(Debug, Clone, Serialize, Deserialize)] pub struct BulkExtractResponse { /// Number of files successfully created #[serde(rename = "Number Files Created")] pub number_files_created: usize, /// List of errors encountered #[serde(rename = "Errors")] pub errors: Vec>, /// Overall response status #[serde(rename = "Response Status")] pub response_status: String, /// Response body (usually empty) #[serde(rename = "Response Body")] pub response_body: String, } impl Default for BulkExtractResponse { fn default() -> Self { Self { number_files_created: 0, errors: Vec::new(), response_status: "201 Created".to_string(), response_body: String::new(), } } } /// Parse object path from bulk delete request /// /// Paths should be in format: /container/object fn parse_object_path(path: &str) -> SwiftResult<(String, String)> { let path = path.trim(); if path.is_empty() { return Err(SwiftError::BadRequest("Empty path in bulk delete".to_string())); } // Remove leading slash let path = path.trim_start_matches('/'); // Split into container and object let parts: Vec<&str> = path.splitn(2, '/').collect(); if parts.len() != 2 { return Err(SwiftError::BadRequest(format!( "Invalid path format: {}. Expected /container/object", path ))); } if parts[0].is_empty() || parts[1].is_empty() { return Err(SwiftError::BadRequest(format!("Empty container or object name in path: {}", path))); } Ok((parts[0].to_string(), parts[1].to_string())) } /// Handle bulk delete request /// /// Deletes multiple objects specified in the request body pub async fn handle_bulk_delete(account: &str, body: String, credentials: &Credentials) -> SwiftResult> { debug!( event = EVENT_SWIFT_BULK_DELETE_STATE, component = LOG_COMPONENT_PROTOCOLS, subsystem = LOG_SUBSYSTEM_SWIFT_BULK, state = "started", account = %account, "swift bulk delete state changed" ); let mut response = BulkDeleteResponse::default(); let mut delete_results = Vec::new(); // Parse paths from body (newline-separated) let paths: Vec<&str> = body.lines().filter(|line| !line.trim().is_empty()).collect(); if paths.is_empty() { return Err(SwiftError::BadRequest("No paths provided for bulk delete".to_string())); } debug!( event = EVENT_SWIFT_BULK_DELETE_STATE, component = LOG_COMPONENT_PROTOCOLS, subsystem = LOG_SUBSYSTEM_SWIFT_BULK, state = "processing", account = %account, path_count = paths.len(), "swift bulk delete state changed" ); // Process each path for path in paths { let result = match parse_object_path(path) { Ok((container, object_key)) => { // Attempt to delete the object match object::delete_object(account, &container, &object_key, credentials).await { Ok(_) => { response.number_deleted += 1; DeleteResult { path: path.to_string(), status: 204, error: None, } } Err(SwiftError::NotFound(_)) => { response.number_not_found += 1; DeleteResult { path: path.to_string(), status: 404, error: Some("Not Found".to_string()), } } Err(e) => { error!( event = EVENT_SWIFT_BULK_DELETE_STATE, component = LOG_COMPONENT_PROTOCOLS, subsystem = LOG_SUBSYSTEM_SWIFT_BULK, result = "delete_failed", account = %account, path = %path, error = %e, "swift bulk delete state changed" ); response.errors.push(vec![path.to_string(), e.to_string()]); DeleteResult { path: path.to_string(), status: 500, error: Some(e.to_string()), } } } } Err(e) => { error!( event = EVENT_SWIFT_BULK_DELETE_STATE, component = LOG_COMPONENT_PROTOCOLS, subsystem = LOG_SUBSYSTEM_SWIFT_BULK, result = "invalid_path", account = %account, path = %path, error = %e, "swift bulk delete state changed" ); response.errors.push(vec![path.to_string(), e.to_string()]); DeleteResult { path: path.to_string(), status: 400, error: Some(e.to_string()), } } }; delete_results.push(result); } // Determine overall status if !response.errors.is_empty() { response.response_status = "400 Bad Request".to_string(); } // Serialize response let json = serde_json::to_string(&response) .map_err(|e| SwiftError::InternalServerError(format!("JSON serialization failed: {}", e)))?; let trans_id = super::handler::generate_trans_id(); Response::builder() .status(StatusCode::OK) .header("content-type", "application/json; charset=utf-8") .header("content-length", json.len().to_string()) .header("x-trans-id", trans_id.clone()) .header("x-openstack-request-id", trans_id) .body(Body::from(json)) .map_err(|e| SwiftError::InternalServerError(format!("Failed to build response: {}", e))) } /// Archive format supported for bulk extract #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum ArchiveFormat { /// Uncompressed tar Tar, /// Gzip-compressed tar TarGz, /// Bzip2-compressed tar TarBz2, } impl ArchiveFormat { /// Parse archive format from query parameter pub fn from_query(query: &str) -> SwiftResult { match query { "tar" => Ok(ArchiveFormat::Tar), "tar.gz" | "tgz" => Ok(ArchiveFormat::TarGz), "tar.bz2" | "tbz2" | "tbz" => Ok(ArchiveFormat::TarBz2), _ => Err(SwiftError::BadRequest(format!( "Unsupported archive format: {}. Supported: tar, tar.gz, tar.bz2", query ))), } } } /// Handle bulk extract request /// /// Extracts files from an uploaded archive into the specified container pub async fn handle_bulk_extract( account: &str, container: &str, format: ArchiveFormat, body: Vec, credentials: &Credentials, ) -> SwiftResult> { debug!( event = EVENT_SWIFT_BULK_EXTRACT_STATE, component = LOG_COMPONENT_PROTOCOLS, subsystem = LOG_SUBSYSTEM_SWIFT_BULK, state = "started", account = %account, container = %container, format = ?format, "swift bulk extract state changed" ); let mut response = BulkExtractResponse::default(); // Verify container exists if container::get_container_metadata(account, container, credentials) .await .is_err() { return Err(SwiftError::NotFound(format!("Container not found: {}", container))); } // Parse archive and collect all entries into memory (entire archive and file contents are buffered) let entries = extract_tar_entries(format, body).await?; // Now upload each entry (async operations) for (path_str, contents) in entries { // Upload file to container match object::put_object( account, container, &path_str, credentials, std::io::Cursor::new(contents), &http::HeaderMap::new(), ) .await { Ok(_) => { response.number_files_created += 1; debug!( event = EVENT_SWIFT_BULK_EXTRACT_STATE, component = LOG_COMPONENT_PROTOCOLS, subsystem = LOG_SUBSYSTEM_SWIFT_BULK, state = "file_created", container = %container, object = %path_str, "swift bulk extract state changed" ); } Err(e) => { error!( event = EVENT_SWIFT_BULK_EXTRACT_STATE, component = LOG_COMPONENT_PROTOCOLS, subsystem = LOG_SUBSYSTEM_SWIFT_BULK, result = "upload_failed", container = %container, object = %path_str, error = %e, "swift bulk extract state changed" ); response.errors.push(vec![path_str.clone(), e.to_string()]); } } } // Determine overall status if response.number_files_created == 0 { response.response_status = "400 Bad Request".to_string(); } else if !response.errors.is_empty() { response.response_status = "201 Created".to_string(); } // Serialize response let json = serde_json::to_string(&response) .map_err(|e| SwiftError::InternalServerError(format!("JSON serialization failed: {}", e)))?; let trans_id = super::handler::generate_trans_id(); let status = if response.number_files_created > 0 { StatusCode::CREATED } else { StatusCode::BAD_REQUEST }; Response::builder() .status(status) .header("content-type", "application/json; charset=utf-8") .header("content-length", json.len().to_string()) .header("x-trans-id", trans_id.clone()) .header("x-openstack-request-id", trans_id) .body(Body::from(json)) .map_err(|e| SwiftError::InternalServerError(format!("Failed to build response: {}", e))) } /// Extract tar entries using async I/O and return them as in-memory buffers async fn extract_tar_entries(format: ArchiveFormat, body: Vec) -> SwiftResult)>> { // Create appropriate reader based on format let reader: Box = match format { ArchiveFormat::Tar => Box::new(std::io::Cursor::new(body)), ArchiveFormat::TarGz => { let cursor = std::io::Cursor::new(body); Box::new(async_compression::tokio::bufread::GzipDecoder::new(tokio::io::BufReader::new(cursor))) } ArchiveFormat::TarBz2 => { let cursor = std::io::Cursor::new(body); Box::new(async_compression::tokio::bufread::BzDecoder::new(tokio::io::BufReader::new(cursor))) } }; // Parse tar archive let mut archive = tokio_tar::Archive::new(reader); let mut entries = Vec::new(); let mut entries_iter = archive .entries() .map_err(|e| SwiftError::BadRequest(format!("Failed to read tar archive: {}", e)))?; // Extract each entry while let Some(entry) = entries_iter.next().await { let mut entry = entry.map_err(|e| SwiftError::BadRequest(format!("Failed to read tar entry: {}", e)))?; // Get entry path let path = entry .path() .map_err(|e| SwiftError::BadRequest(format!("Invalid path in tar entry: {}", e)))?; let path_str = path.to_string_lossy().to_string(); // Skip directories if entry.header().entry_type().is_dir() { debug!( event = EVENT_SWIFT_BULK_EXTRACT_STATE, component = LOG_COMPONENT_PROTOCOLS, subsystem = LOG_SUBSYSTEM_SWIFT_BULK, state = "skipped_directory", path = %path_str, "swift bulk extract state changed" ); continue; } // Read file contents let mut contents = Vec::new(); if let Err(e) = tokio::io::AsyncReadExt::read_to_end(&mut entry, &mut contents).await { error!( event = EVENT_SWIFT_BULK_EXTRACT_STATE, component = LOG_COMPONENT_PROTOCOLS, subsystem = LOG_SUBSYSTEM_SWIFT_BULK, result = "tar_entry_read_failed", path = %path_str, error = %e, "swift bulk extract state changed" ); continue; } entries.push((path_str, contents)); } Ok(entries) } #[cfg(test)] mod tests { use super::*; #[test] fn test_parse_object_path() { // Valid paths assert_eq!( parse_object_path("/container/object.txt").unwrap(), ("container".to_string(), "object.txt".to_string()) ); assert_eq!( parse_object_path("container/folder/object.txt").unwrap(), ("container".to_string(), "folder/object.txt".to_string()) ); assert_eq!( parse_object_path("/my-container/path/to/file.txt").unwrap(), ("my-container".to_string(), "path/to/file.txt".to_string()) ); // With whitespace assert_eq!( parse_object_path(" /container/object.txt ").unwrap(), ("container".to_string(), "object.txt".to_string()) ); } #[test] fn test_parse_object_path_invalid() { // Empty path assert!(parse_object_path("").is_err()); assert!(parse_object_path(" ").is_err()); // Missing object assert!(parse_object_path("/container").is_err()); assert!(parse_object_path("/container/").is_err()); // Missing container assert!(parse_object_path("/").is_err()); assert!(parse_object_path("//object").is_err()); } #[test] fn test_archive_format_from_query() { assert_eq!(ArchiveFormat::from_query("tar").unwrap(), ArchiveFormat::Tar); assert_eq!(ArchiveFormat::from_query("tar.gz").unwrap(), ArchiveFormat::TarGz); assert_eq!(ArchiveFormat::from_query("tgz").unwrap(), ArchiveFormat::TarGz); assert_eq!(ArchiveFormat::from_query("tar.bz2").unwrap(), ArchiveFormat::TarBz2); assert_eq!(ArchiveFormat::from_query("tbz2").unwrap(), ArchiveFormat::TarBz2); assert_eq!(ArchiveFormat::from_query("tbz").unwrap(), ArchiveFormat::TarBz2); // Invalid formats assert!(ArchiveFormat::from_query("zip").is_err()); assert!(ArchiveFormat::from_query("rar").is_err()); assert!(ArchiveFormat::from_query("").is_err()); } #[test] fn test_bulk_delete_response_default() { let response = BulkDeleteResponse::default(); assert_eq!(response.number_deleted, 0); assert_eq!(response.number_not_found, 0); assert!(response.errors.is_empty()); assert_eq!(response.response_status, "200 OK"); assert!(response.response_body.is_empty()); } #[test] fn test_bulk_extract_response_default() { let response = BulkExtractResponse::default(); assert_eq!(response.number_files_created, 0); assert!(response.errors.is_empty()); assert_eq!(response.response_status, "201 Created"); assert!(response.response_body.is_empty()); } #[test] fn test_parse_multiple_paths() { let body = "/container1/file1.txt\n/container2/file2.txt\n/container1/folder/file3.txt"; let paths: Vec<&str> = body.lines().collect(); assert_eq!(paths.len(), 3); let (c1, o1) = parse_object_path(paths[0]).unwrap(); assert_eq!(c1, "container1"); assert_eq!(o1, "file1.txt"); let (c2, o2) = parse_object_path(paths[1]).unwrap(); assert_eq!(c2, "container2"); assert_eq!(o2, "file2.txt"); let (c3, o3) = parse_object_path(paths[2]).unwrap(); assert_eq!(c3, "container1"); assert_eq!(o3, "folder/file3.txt"); } #[test] fn test_parse_paths_with_empty_lines() { let body = "/container1/file1.txt\n\n/container2/file2.txt\n \n/container1/file3.txt"; let paths: Vec<&str> = body.lines().filter(|line| !line.trim().is_empty()).collect(); assert_eq!(paths.len(), 3); } /// Tests for the `extract_tar_entries` async function. /// /// Conditionally compiled with the `swift` feature, which gates the /// `tokio_tar` (astral-tokio-tar) and `async_compression` dependencies. #[cfg(feature = "swift")] mod tar_extraction { use super::*; use tokio::io::AsyncWriteExt; /// Builds an uncompressed tar archive in memory. /// /// * `files` – `(path, content)` pairs added as regular files. /// * `dirs` – paths added as directory entries. async fn make_tar(files: &[(&str, &[u8])], dirs: &[&str]) -> Vec { let buf = std::io::Cursor::new(Vec::new()); let mut builder = tokio_tar::Builder::new(buf); for &dir in dirs { let mut header = tokio_tar::Header::new_gnu(); header.set_entry_type(tokio_tar::EntryType::Directory); header.set_size(0); header.set_mode(0o755); header.set_cksum(); builder .append_data(&mut header, dir, std::io::Cursor::new(&[] as &[u8])) .await .unwrap(); } for &(name, data) in files { let mut header = tokio_tar::Header::new_gnu(); header.set_size(data.len() as u64); header.set_mode(0o644); header.set_cksum(); builder .append_data(&mut header, name, std::io::Cursor::new(data)) .await .unwrap(); } builder.into_inner().await.unwrap().into_inner() } #[tokio::test] async fn test_extract_plain_tar_paths_and_contents() { let tar_bytes = make_tar(&[("file1.txt", b"hello"), ("dir/file2.txt", b"world")], &[]).await; let entries = extract_tar_entries(ArchiveFormat::Tar, tar_bytes).await.unwrap(); assert_eq!(entries.len(), 2); assert_eq!(entries[0].0, "file1.txt"); assert_eq!(entries[0].1, b"hello"); assert_eq!(entries[1].0, "dir/file2.txt"); assert_eq!(entries[1].1, b"world"); } #[tokio::test] async fn test_extract_tar_skips_directories() { let tar_bytes = make_tar(&[("file.txt", b"content")], &["subdir/", "another/"]).await; let entries = extract_tar_entries(ArchiveFormat::Tar, tar_bytes).await.unwrap(); // Directory entries must be filtered out; only the regular file is returned. assert_eq!(entries.len(), 1); assert_eq!(entries[0].0, "file.txt"); } #[tokio::test] async fn test_extract_tar_gz() { let tar_bytes = make_tar(&[("file.txt", b"compressed content")], &[]).await; // Compress with gzip. let cursor = std::io::Cursor::new(Vec::new()); let mut encoder = async_compression::tokio::write::GzipEncoder::new(cursor); encoder.write_all(&tar_bytes).await.unwrap(); encoder.shutdown().await.unwrap(); let gz_bytes = encoder.into_inner().into_inner(); let entries = extract_tar_entries(ArchiveFormat::TarGz, gz_bytes).await.unwrap(); assert_eq!(entries.len(), 1); assert_eq!(entries[0].0, "file.txt"); assert_eq!(entries[0].1, b"compressed content"); } #[tokio::test] async fn test_extract_tar_bz2() { let tar_bytes = make_tar(&[("file.txt", b"bzip2 content")], &[]).await; // Compress with bzip2. let cursor = std::io::Cursor::new(Vec::new()); let mut encoder = async_compression::tokio::write::BzEncoder::new(cursor); encoder.write_all(&tar_bytes).await.unwrap(); encoder.shutdown().await.unwrap(); let bz2_bytes = encoder.into_inner().into_inner(); let entries = extract_tar_entries(ArchiveFormat::TarBz2, bz2_bytes).await.unwrap(); assert_eq!(entries.len(), 1); assert_eq!(entries[0].0, "file.txt"); assert_eq!(entries[0].1, b"bzip2 content"); } #[tokio::test] async fn test_extract_invalid_tar_returns_error() { // Fewer than 512 bytes → parser cannot read a complete tar header block. let bad_bytes = b"this is not a valid tar archive".to_vec(); let result = extract_tar_entries(ArchiveFormat::Tar, bad_bytes).await; assert!(result.is_err(), "expected error for invalid tar data"); } #[tokio::test] async fn test_extract_empty_tar() { let tar_bytes = make_tar(&[], &[]).await; let entries = extract_tar_entries(ArchiveFormat::Tar, tar_bytes).await.unwrap(); assert!(entries.is_empty()); } } }