rustfs/crates/protocols/src/swift/bulk.rs

// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

//! Bulk Operations for Swift API
//!
//! This module implements bulk operations that allow batch processing of
//! multiple objects in a single request, improving efficiency for operations
//! that affect many files.
//!
//! # Operations
//!
//! ## Bulk Delete
//!
//! Delete multiple objects in a single request.
//!
//! **Endpoint**: `DELETE /?bulk-delete`
//!
//! **Request Body**: Newline-separated list of object paths
//! ```text
//! /container1/object1.txt
//! /container2/folder/object2.txt
//! /container1/object3.txt
//! ```
//!
//! **Response**: JSON with results for each object
//! ```json
//! {
//!   "Number Deleted": 2,
//!   "Number Not Found": 1,
//!   "Errors": [],
//!   "Response Status": "200 OK",
//!   "Response Body": ""
//! }
//! ```
//!
//! ## Bulk Extract
//!
//! Extract files from an uploaded archive into a container.
//!
//! **Endpoint**: `PUT /{container}?extract-archive=tar` (or tar.gz, tar.bz2)
//!
//! **Request Body**: Archive file contents
//!
//! **Response**: JSON with results for each extracted file
//! ```json
//! {
//!   "Number Files Created": 10,
//!   "Errors": [],
//!   "Response Status": "201 Created",
//!   "Response Body": ""
//! }
//! ```
//!
//! # Examples
//!
//! ```bash
//! # Bulk delete
//! echo -e "/container/file1.txt\n/container/file2.txt" | \
//!   swift delete --bulk
//!
//! # Bulk extract
//! tar czf archive.tar.gz files/
//! swift upload container --extract-archive archive.tar.gz
//! ```

use super::{SwiftError, SwiftResult, container, object};
use axum::http::{Response, StatusCode};
use futures::StreamExt;
use rustfs_credentials::Credentials;
use s3s::Body;
use serde::{Deserialize, Serialize};
use tracing::{debug, error};

const LOG_COMPONENT_PROTOCOLS: &str = "protocols";
const LOG_SUBSYSTEM_SWIFT_BULK: &str = "swift_bulk";
const EVENT_SWIFT_BULK_DELETE_STATE: &str = "swift_bulk_delete_state";
const EVENT_SWIFT_BULK_EXTRACT_STATE: &str = "swift_bulk_extract_state";

/// Result of a single delete operation
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct DeleteResult {
    /// Object path that was deleted
    pub path: String,

    /// HTTP status code for this operation
    pub status: u16,

    /// Error message if deletion failed
    #[serde(skip_serializing_if = "Option::is_none")]
    pub error: Option<String>,
}

/// Bulk delete response
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct BulkDeleteResponse {
    /// Number of objects successfully deleted
    #[serde(rename = "Number Deleted")]
    pub number_deleted: usize,

    /// Number of objects not found
    #[serde(rename = "Number Not Found")]
    pub number_not_found: usize,

    /// List of errors encountered
    #[serde(rename = "Errors")]
    pub errors: Vec<Vec<String>>,

    /// Overall response status
    #[serde(rename = "Response Status")]
    pub response_status: String,

    /// Response body (usually empty)
    #[serde(rename = "Response Body")]
    pub response_body: String,
}

impl Default for BulkDeleteResponse {
    fn default() -> Self {
        Self {
            number_deleted: 0,
            number_not_found: 0,
            errors: Vec::new(),
            response_status: "200 OK".to_string(),
            response_body: String::new(),
        }
    }
}

/// Result of a single file extraction
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ExtractResult {
    /// File path that was extracted
    pub path: String,

    /// HTTP status code for this operation
    pub status: u16,

    /// Error message if extraction failed
    #[serde(skip_serializing_if = "Option::is_none")]
    pub error: Option<String>,
}

/// Bulk extract response
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct BulkExtractResponse {
    /// Number of files successfully created
    #[serde(rename = "Number Files Created")]
    pub number_files_created: usize,

    /// List of errors encountered
    #[serde(rename = "Errors")]
    pub errors: Vec<Vec<String>>,

    /// Overall response status
    #[serde(rename = "Response Status")]
    pub response_status: String,

    /// Response body (usually empty)
    #[serde(rename = "Response Body")]
    pub response_body: String,
}

impl Default for BulkExtractResponse {
    fn default() -> Self {
        Self {
            number_files_created: 0,
            errors: Vec::new(),
            response_status: "201 Created".to_string(),
            response_body: String::new(),
        }
    }
}

/// Parse object path from bulk delete request
///
/// Paths should be in format: /container/object
fn parse_object_path(path: &str) -> SwiftResult<(String, String)> {
    let path = path.trim();

    if path.is_empty() {
        return Err(SwiftError::BadRequest("Empty path in bulk delete".to_string()));
    }

    // Remove leading slash
    let path = path.trim_start_matches('/');

    // Split into container and object
    let parts: Vec<&str> = path.splitn(2, '/').collect();

    if parts.len() != 2 {
        return Err(SwiftError::BadRequest(format!(
            "Invalid path format: {}. Expected /container/object",
            path
        )));
    }

    if parts[0].is_empty() || parts[1].is_empty() {
        return Err(SwiftError::BadRequest(format!("Empty container or object name in path: {}", path)));
    }

    Ok((parts[0].to_string(), parts[1].to_string()))
}

/// Handle bulk delete request
///
/// Deletes multiple objects specified in the request body
pub async fn handle_bulk_delete(account: &str, body: String, credentials: &Credentials) -> SwiftResult<Response<Body>> {
    debug!(
        event = EVENT_SWIFT_BULK_DELETE_STATE,
        component = LOG_COMPONENT_PROTOCOLS,
        subsystem = LOG_SUBSYSTEM_SWIFT_BULK,
        state = "started",
        account = %account,
        "swift bulk delete state changed"
    );

    let mut response = BulkDeleteResponse::default();
    let mut delete_results = Vec::new();

    // Parse paths from body (newline-separated)
    let paths: Vec<&str> = body.lines().filter(|line| !line.trim().is_empty()).collect();

    if paths.is_empty() {
        return Err(SwiftError::BadRequest("No paths provided for bulk delete".to_string()));
    }

    debug!(
        event = EVENT_SWIFT_BULK_DELETE_STATE,
        component = LOG_COMPONENT_PROTOCOLS,
        subsystem = LOG_SUBSYSTEM_SWIFT_BULK,
        state = "processing",
        account = %account,
        path_count = paths.len(),
        "swift bulk delete state changed"
    );

    // Process each path
    for path in paths {
        let result = match parse_object_path(path) {
            Ok((container, object_key)) => {
                // Attempt to delete the object
                match object::delete_object(account, &container, &object_key, credentials).await {
                    Ok(_) => {
                        response.number_deleted += 1;
                        DeleteResult {
                            path: path.to_string(),
                            status: 204,
                            error: None,
                        }
                    }
                    Err(SwiftError::NotFound(_)) => {
                        response.number_not_found += 1;
                        DeleteResult {
                            path: path.to_string(),
                            status: 404,
                            error: Some("Not Found".to_string()),
                        }
                    }
                    Err(e) => {
                        error!(
                            event = EVENT_SWIFT_BULK_DELETE_STATE,
                            component = LOG_COMPONENT_PROTOCOLS,
                            subsystem = LOG_SUBSYSTEM_SWIFT_BULK,
                            result = "delete_failed",
                            account = %account,
                            path = %path,
                            error = %e,
                            "swift bulk delete state changed"
                        );
                        response.errors.push(vec![path.to_string(), e.to_string()]);
                        DeleteResult {
                            path: path.to_string(),
                            status: 500,
                            error: Some(e.to_string()),
                        }
                    }
                }
            }
            Err(e) => {
                error!(
                    event = EVENT_SWIFT_BULK_DELETE_STATE,
                    component = LOG_COMPONENT_PROTOCOLS,
                    subsystem = LOG_SUBSYSTEM_SWIFT_BULK,
                    result = "invalid_path",
                    account = %account,
                    path = %path,
                    error = %e,
                    "swift bulk delete state changed"
                );
                response.errors.push(vec![path.to_string(), e.to_string()]);
                DeleteResult {
                    path: path.to_string(),
                    status: 400,
                    error: Some(e.to_string()),
                }
            }
        };

        delete_results.push(result);
    }

    // Determine overall status
    if !response.errors.is_empty() {
        response.response_status = "400 Bad Request".to_string();
    }

    // Serialize response
    let json = serde_json::to_string(&response)
        .map_err(|e| SwiftError::InternalServerError(format!("JSON serialization failed: {}", e)))?;

    let trans_id = super::handler::generate_trans_id();
    Response::builder()
        .status(StatusCode::OK)
        .header("content-type", "application/json; charset=utf-8")
        .header("content-length", json.len().to_string())
        .header("x-trans-id", trans_id.clone())
        .header("x-openstack-request-id", trans_id)
        .body(Body::from(json))
        .map_err(|e| SwiftError::InternalServerError(format!("Failed to build response: {}", e)))
}

/// Archive format supported for bulk extract
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ArchiveFormat {
    /// Uncompressed tar
    Tar,
    /// Gzip-compressed tar
    TarGz,
    /// Bzip2-compressed tar
    TarBz2,
}

impl ArchiveFormat {
    /// Parse archive format from query parameter
    pub fn from_query(query: &str) -> SwiftResult<Self> {
        match query {
            "tar" => Ok(ArchiveFormat::Tar),
            "tar.gz" | "tgz" => Ok(ArchiveFormat::TarGz),
            "tar.bz2" | "tbz2" | "tbz" => Ok(ArchiveFormat::TarBz2),
            _ => Err(SwiftError::BadRequest(format!(
                "Unsupported archive format: {}. Supported: tar, tar.gz, tar.bz2",
                query
            ))),
        }
    }
}

/// Handle bulk extract request
///
/// Extracts files from an uploaded archive into the specified container
pub async fn handle_bulk_extract(
    account: &str,
    container: &str,
    format: ArchiveFormat,
    body: Vec<u8>,
    credentials: &Credentials,
) -> SwiftResult<Response<Body>> {
    debug!(
        event = EVENT_SWIFT_BULK_EXTRACT_STATE,
        component = LOG_COMPONENT_PROTOCOLS,
        subsystem = LOG_SUBSYSTEM_SWIFT_BULK,
        state = "started",
        account = %account,
        container = %container,
        format = ?format,
        "swift bulk extract state changed"
    );

    let mut response = BulkExtractResponse::default();

    // Verify container exists
    if container::get_container_metadata(account, container, credentials)
        .await
        .is_err()
    {
        return Err(SwiftError::NotFound(format!("Container not found: {}", container)));
    }

    // Parse archive and collect all entries into memory (entire archive and file contents are buffered)
    let entries = extract_tar_entries(format, body).await?;

    // Now upload each entry (async operations)
    for (path_str, contents) in entries {
        // Upload file to container
        match object::put_object(
            account,
            container,
            &path_str,
            credentials,
            std::io::Cursor::new(contents),
            &http::HeaderMap::new(),
        )
        .await
        {
            Ok(_) => {
                response.number_files_created += 1;
                debug!(
                    event = EVENT_SWIFT_BULK_EXTRACT_STATE,
                    component = LOG_COMPONENT_PROTOCOLS,
                    subsystem = LOG_SUBSYSTEM_SWIFT_BULK,
                    state = "file_created",
                    container = %container,
                    object = %path_str,
                    "swift bulk extract state changed"
                );
            }
            Err(e) => {
                error!(
                    event = EVENT_SWIFT_BULK_EXTRACT_STATE,
                    component = LOG_COMPONENT_PROTOCOLS,
                    subsystem = LOG_SUBSYSTEM_SWIFT_BULK,
                    result = "upload_failed",
                    container = %container,
                    object = %path_str,
                    error = %e,
                    "swift bulk extract state changed"
                );
                response.errors.push(vec![path_str.clone(), e.to_string()]);
            }
        }
    }

    // Determine overall status
    if response.number_files_created == 0 {
        response.response_status = "400 Bad Request".to_string();
    } else if !response.errors.is_empty() {
        response.response_status = "201 Created".to_string();
    }

    // Serialize response
    let json = serde_json::to_string(&response)
        .map_err(|e| SwiftError::InternalServerError(format!("JSON serialization failed: {}", e)))?;

    let trans_id = super::handler::generate_trans_id();
    let status = if response.number_files_created > 0 {
        StatusCode::CREATED
    } else {
        StatusCode::BAD_REQUEST
    };

    Response::builder()
        .status(status)
        .header("content-type", "application/json; charset=utf-8")
        .header("content-length", json.len().to_string())
        .header("x-trans-id", trans_id.clone())
        .header("x-openstack-request-id", trans_id)
        .body(Body::from(json))
        .map_err(|e| SwiftError::InternalServerError(format!("Failed to build response: {}", e)))
}

/// Extract tar entries using async I/O and return them as in-memory buffers
async fn extract_tar_entries(format: ArchiveFormat, body: Vec<u8>) -> SwiftResult<Vec<(String, Vec<u8>)>> {
    // Create appropriate reader based on format
    let reader: Box<dyn tokio::io::AsyncRead + Unpin + Send> = match format {
        ArchiveFormat::Tar => Box::new(std::io::Cursor::new(body)),
        ArchiveFormat::TarGz => {
            let cursor = std::io::Cursor::new(body);
            Box::new(async_compression::tokio::bufread::GzipDecoder::new(tokio::io::BufReader::new(cursor)))
        }
        ArchiveFormat::TarBz2 => {
            let cursor = std::io::Cursor::new(body);
            Box::new(async_compression::tokio::bufread::BzDecoder::new(tokio::io::BufReader::new(cursor)))
        }
    };

    // Parse tar archive
    let mut archive = tokio_tar::Archive::new(reader);
    let mut entries = Vec::new();
    let mut entries_iter = archive
        .entries()
        .map_err(|e| SwiftError::BadRequest(format!("Failed to read tar archive: {}", e)))?;

    // Extract each entry
    while let Some(entry) = entries_iter.next().await {
        let mut entry = entry.map_err(|e| SwiftError::BadRequest(format!("Failed to read tar entry: {}", e)))?;

        // Get entry path
        let path = entry
            .path()
            .map_err(|e| SwiftError::BadRequest(format!("Invalid path in tar entry: {}", e)))?;

        let path_str = path.to_string_lossy().to_string();

        // Skip directories
        if entry.header().entry_type().is_dir() {
            debug!(
                event = EVENT_SWIFT_BULK_EXTRACT_STATE,
                component = LOG_COMPONENT_PROTOCOLS,
                subsystem = LOG_SUBSYSTEM_SWIFT_BULK,
                state = "skipped_directory",
                path = %path_str,
                "swift bulk extract state changed"
            );
            continue;
        }

        // Read file contents
        let mut contents = Vec::new();
        if let Err(e) = tokio::io::AsyncReadExt::read_to_end(&mut entry, &mut contents).await {
            error!(
                event = EVENT_SWIFT_BULK_EXTRACT_STATE,
                component = LOG_COMPONENT_PROTOCOLS,
                subsystem = LOG_SUBSYSTEM_SWIFT_BULK,
                result = "tar_entry_read_failed",
                path = %path_str,
                error = %e,
                "swift bulk extract state changed"
            );
            continue;
        }

        entries.push((path_str, contents));
    }

    Ok(entries)
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn test_parse_object_path() {
        // Valid paths
        assert_eq!(
            parse_object_path("/container/object.txt").unwrap(),
            ("container".to_string(), "object.txt".to_string())
        );

        assert_eq!(
            parse_object_path("container/folder/object.txt").unwrap(),
            ("container".to_string(), "folder/object.txt".to_string())
        );

        assert_eq!(
            parse_object_path("/my-container/path/to/file.txt").unwrap(),
            ("my-container".to_string(), "path/to/file.txt".to_string())
        );

        // With whitespace
        assert_eq!(
            parse_object_path("  /container/object.txt  ").unwrap(),
            ("container".to_string(), "object.txt".to_string())
        );
    }

    #[test]
    fn test_parse_object_path_invalid() {
        // Empty path
        assert!(parse_object_path("").is_err());
        assert!(parse_object_path("   ").is_err());

        // Missing object
        assert!(parse_object_path("/container").is_err());
        assert!(parse_object_path("/container/").is_err());

        // Missing container
        assert!(parse_object_path("/").is_err());
        assert!(parse_object_path("//object").is_err());
    }

    #[test]
    fn test_archive_format_from_query() {
        assert_eq!(ArchiveFormat::from_query("tar").unwrap(), ArchiveFormat::Tar);
        assert_eq!(ArchiveFormat::from_query("tar.gz").unwrap(), ArchiveFormat::TarGz);
        assert_eq!(ArchiveFormat::from_query("tgz").unwrap(), ArchiveFormat::TarGz);
        assert_eq!(ArchiveFormat::from_query("tar.bz2").unwrap(), ArchiveFormat::TarBz2);
        assert_eq!(ArchiveFormat::from_query("tbz2").unwrap(), ArchiveFormat::TarBz2);
        assert_eq!(ArchiveFormat::from_query("tbz").unwrap(), ArchiveFormat::TarBz2);

        // Invalid formats
        assert!(ArchiveFormat::from_query("zip").is_err());
        assert!(ArchiveFormat::from_query("rar").is_err());
        assert!(ArchiveFormat::from_query("").is_err());
    }

    #[test]
    fn test_bulk_delete_response_default() {
        let response = BulkDeleteResponse::default();
        assert_eq!(response.number_deleted, 0);
        assert_eq!(response.number_not_found, 0);
        assert!(response.errors.is_empty());
        assert_eq!(response.response_status, "200 OK");
        assert!(response.response_body.is_empty());
    }

    #[test]
    fn test_bulk_extract_response_default() {
        let response = BulkExtractResponse::default();
        assert_eq!(response.number_files_created, 0);
        assert!(response.errors.is_empty());
        assert_eq!(response.response_status, "201 Created");
        assert!(response.response_body.is_empty());
    }

    #[test]
    fn test_parse_multiple_paths() {
        let body = "/container1/file1.txt\n/container2/file2.txt\n/container1/folder/file3.txt";
        let paths: Vec<&str> = body.lines().collect();

        assert_eq!(paths.len(), 3);

        let (c1, o1) = parse_object_path(paths[0]).unwrap();
        assert_eq!(c1, "container1");
        assert_eq!(o1, "file1.txt");

        let (c2, o2) = parse_object_path(paths[1]).unwrap();
        assert_eq!(c2, "container2");
        assert_eq!(o2, "file2.txt");

        let (c3, o3) = parse_object_path(paths[2]).unwrap();
        assert_eq!(c3, "container1");
        assert_eq!(o3, "folder/file3.txt");
    }

    #[test]
    fn test_parse_paths_with_empty_lines() {
        let body = "/container1/file1.txt\n\n/container2/file2.txt\n   \n/container1/file3.txt";
        let paths: Vec<&str> = body.lines().filter(|line| !line.trim().is_empty()).collect();

        assert_eq!(paths.len(), 3);
    }

    /// Tests for the `extract_tar_entries` async function.
    ///
    /// Conditionally compiled with the `swift` feature, which gates the
    /// `tokio_tar` (astral-tokio-tar) and `async_compression` dependencies.
    #[cfg(feature = "swift")]
    mod tar_extraction {
        use super::*;
        use tokio::io::AsyncWriteExt;

        /// Builds an uncompressed tar archive in memory.
        ///
        /// * `files` – `(path, content)` pairs added as regular files.
        /// * `dirs`  – paths added as directory entries.
        async fn make_tar(files: &[(&str, &[u8])], dirs: &[&str]) -> Vec<u8> {
            let buf = std::io::Cursor::new(Vec::new());
            let mut builder = tokio_tar::Builder::new(buf);

            for &dir in dirs {
                let mut header = tokio_tar::Header::new_gnu();
                header.set_entry_type(tokio_tar::EntryType::Directory);
                header.set_size(0);
                header.set_mode(0o755);
                header.set_cksum();
                builder
                    .append_data(&mut header, dir, std::io::Cursor::new(&[] as &[u8]))
                    .await
                    .unwrap();
            }

            for &(name, data) in files {
                let mut header = tokio_tar::Header::new_gnu();
                header.set_size(data.len() as u64);
                header.set_mode(0o644);
                header.set_cksum();
                builder
                    .append_data(&mut header, name, std::io::Cursor::new(data))
                    .await
                    .unwrap();
            }

            builder.into_inner().await.unwrap().into_inner()
        }

        #[tokio::test]
        async fn test_extract_plain_tar_paths_and_contents() {
            let tar_bytes = make_tar(&[("file1.txt", b"hello"), ("dir/file2.txt", b"world")], &[]).await;
            let entries = extract_tar_entries(ArchiveFormat::Tar, tar_bytes).await.unwrap();
            assert_eq!(entries.len(), 2);
            assert_eq!(entries[0].0, "file1.txt");
            assert_eq!(entries[0].1, b"hello");
            assert_eq!(entries[1].0, "dir/file2.txt");
            assert_eq!(entries[1].1, b"world");
        }

        #[tokio::test]
        async fn test_extract_tar_skips_directories() {
            let tar_bytes = make_tar(&[("file.txt", b"content")], &["subdir/", "another/"]).await;
            let entries = extract_tar_entries(ArchiveFormat::Tar, tar_bytes).await.unwrap();
            // Directory entries must be filtered out; only the regular file is returned.
            assert_eq!(entries.len(), 1);
            assert_eq!(entries[0].0, "file.txt");
        }

        #[tokio::test]
        async fn test_extract_tar_gz() {
            let tar_bytes = make_tar(&[("file.txt", b"compressed content")], &[]).await;

            // Compress with gzip.
            let cursor = std::io::Cursor::new(Vec::new());
            let mut encoder = async_compression::tokio::write::GzipEncoder::new(cursor);
            encoder.write_all(&tar_bytes).await.unwrap();
            encoder.shutdown().await.unwrap();
            let gz_bytes = encoder.into_inner().into_inner();

            let entries = extract_tar_entries(ArchiveFormat::TarGz, gz_bytes).await.unwrap();
            assert_eq!(entries.len(), 1);
            assert_eq!(entries[0].0, "file.txt");
            assert_eq!(entries[0].1, b"compressed content");
        }

        #[tokio::test]
        async fn test_extract_tar_bz2() {
            let tar_bytes = make_tar(&[("file.txt", b"bzip2 content")], &[]).await;

            // Compress with bzip2.
            let cursor = std::io::Cursor::new(Vec::new());
            let mut encoder = async_compression::tokio::write::BzEncoder::new(cursor);
            encoder.write_all(&tar_bytes).await.unwrap();
            encoder.shutdown().await.unwrap();
            let bz2_bytes = encoder.into_inner().into_inner();

            let entries = extract_tar_entries(ArchiveFormat::TarBz2, bz2_bytes).await.unwrap();
            assert_eq!(entries.len(), 1);
            assert_eq!(entries[0].0, "file.txt");
            assert_eq!(entries[0].1, b"bzip2 content");
        }

        #[tokio::test]
        async fn test_extract_invalid_tar_returns_error() {
            // Fewer than 512 bytes → parser cannot read a complete tar header block.
            let bad_bytes = b"this is not a valid tar archive".to_vec();
            let result = extract_tar_entries(ArchiveFormat::Tar, bad_bytes).await;
            assert!(result.is_err(), "expected error for invalid tar data");
        }

        #[tokio::test]
        async fn test_extract_empty_tar() {
            let tar_bytes = make_tar(&[], &[]).await;
            let entries = extract_tar_entries(ArchiveFormat::Tar, tar_bytes).await.unwrap();
            assert!(entries.is_empty());
        }
    }
}