feat(sftp): add SFTPv3 protocol support (#2875)

Co-authored-by: houseme <housemecn@gmail.com>

rustfs/Cargo.toml

@@ -48,10 +48,11 @@ metrics-gpu = ["rustfs-obs/gpu"]
 ftps = ["rustfs-protocols/ftps"]
 swift = ["rustfs-protocols/swift"]
 webdav = ["rustfs-protocols/webdav"]
+sftp = ["rustfs-protocols/sftp"]
 license = []
 io-scheduler-debug = []  # Enable debug information in I/O scheduler
 tracing-chunk-debug = []  # Enable per-chunk tracing in data plane (high noise, for debugging only)
-full = ["metrics-gpu", "ftps", "swift", "webdav"]
+full = ["metrics-gpu", "ftps", "swift", "webdav", "sftp"]
 manual-test-runners = []
 
 [lints]

rustfs/src/init.rs

@@ -704,3 +704,87 @@ pub async fn init_webdav_system() -> Result<Option<tokio::sync::broadcast::Sende
         Ok(Some(shutdown_tx))
     }
 }
+
+/// Start the SFTP server when RUSTFS_SFTP_ENABLE is set. Loads host
+/// keys from the configured directory, validates the SSH configuration,
+/// and spawns the listener task.
+#[cfg(feature = "sftp")]
+#[instrument(skip_all)]
+pub async fn init_sftp_system() -> Result<Option<tokio::sync::broadcast::Sender<()>>, Box<dyn std::error::Error + Send + Sync>> {
+    {
+        use crate::protocols::ProtocolStorageClient;
+        use rustfs_config::{
+            DEFAULT_SFTP_ADDRESS, DEFAULT_SFTP_BANNER, DEFAULT_SFTP_IDLE_TIMEOUT, DEFAULT_SFTP_PART_SIZE, DEFAULT_SFTP_READ_ONLY,
+            ENV_SFTP_ADDRESS, ENV_SFTP_BACKEND_OP_TIMEOUT_SECS, ENV_SFTP_BANNER, ENV_SFTP_ENABLE, ENV_SFTP_HANDLES_PER_SESSION,
+            ENV_SFTP_HOST_KEY_DIR, ENV_SFTP_IDLE_TIMEOUT, ENV_SFTP_PART_SIZE, ENV_SFTP_READ_CACHE_TOTAL_MEM_BYTES,
+            ENV_SFTP_READ_CACHE_WINDOW_BYTES, ENV_SFTP_READ_ONLY,
+        };
+        use rustfs_protocols::{SftpConfig, SftpServer};
+
+        let enabled = rustfs_utils::get_env_bool(ENV_SFTP_ENABLE, false);
+        if !enabled {
+            debug!("SFTP system is disabled");
+            return Ok(None);
+        }
+
+        let addr_str = rustfs_utils::get_env_str(ENV_SFTP_ADDRESS, DEFAULT_SFTP_ADDRESS);
+        let addr = rustfs_utils::net::parse_and_resolve_address(&addr_str)
+            .map_err(|e| format!("Invalid SFTP address '{}': {}", addr_str, e))?;
+
+        let host_key_dir = rustfs_utils::get_env_opt_str(ENV_SFTP_HOST_KEY_DIR)
+            .ok_or("RUSTFS_SFTP_HOST_KEY_DIR is required when SFTP is enabled")?;
+
+        let idle_timeout = rustfs_utils::get_env_u64(ENV_SFTP_IDLE_TIMEOUT, DEFAULT_SFTP_IDLE_TIMEOUT);
+        let part_size = rustfs_utils::get_env_u64(ENV_SFTP_PART_SIZE, DEFAULT_SFTP_PART_SIZE);
+        let handles_per_session =
+            SftpConfig::resolve_handles_per_session(rustfs_utils::get_env_opt_usize(ENV_SFTP_HANDLES_PER_SESSION));
+        let backend_op_timeout_secs =
+            SftpConfig::resolve_backend_op_timeout_secs(rustfs_utils::get_env_opt_u64(ENV_SFTP_BACKEND_OP_TIMEOUT_SECS));
+        let read_cache_window_bytes =
+            SftpConfig::resolve_read_cache_window_bytes(rustfs_utils::get_env_opt_u64(ENV_SFTP_READ_CACHE_WINDOW_BYTES));
+        let read_cache_total_mem_bytes =
+            SftpConfig::resolve_read_cache_total_mem_bytes(rustfs_utils::get_env_opt_u64(ENV_SFTP_READ_CACHE_TOTAL_MEM_BYTES));
+        let read_only = rustfs_utils::get_env_bool(ENV_SFTP_READ_ONLY, DEFAULT_SFTP_READ_ONLY);
+        let banner = rustfs_utils::get_env_str(ENV_SFTP_BANNER, DEFAULT_SFTP_BANNER);
+
+        let config = SftpConfig {
+            bind_addr: addr,
+            host_key_dir: std::path::PathBuf::from(&host_key_dir),
+            idle_timeout_secs: idle_timeout,
+            part_size,
+            handles_per_session,
+            backend_op_timeout_secs,
+            read_cache_window_bytes,
+            read_cache_total_mem_bytes,
+            read_only,
+            banner,
+        };
+
+        config.validate().await?;
+
+        // Load and validate host keys. Fails if zero found or any key
+        // file has insecure permissions.
+        let host_keys = SftpConfig::load_host_keys(&config.host_key_dir).await?;
+
+        let fs = crate::storage::ecfs::FS::new();
+        let storage_client = ProtocolStorageClient::new(fs);
+
+        let server = SftpServer::new(config.clone(), storage_client, host_keys)?;
+
+        info!("SFTP server configured on {}", config.bind_addr);
+
+        // Hook into shutdown support
+        let (shutdown_tx, shutdown_rx) = tokio::sync::broadcast::channel(1);
+
+        // Start SFTP server in background task
+        tokio::spawn(async move {
+            if let Err(e) = server.start(shutdown_rx).await {
+                error!("SFTP server error: {}", e);
+            }
+            info!("SFTP server shutdown completed");
+        });
+
+        info!("SFTP system initialized successfully");
+        Ok(Some(shutdown_tx))
+    }
+}

rustfs/src/lib.rs

@@ -64,7 +64,7 @@ pub mod init;
 pub mod license;
 pub mod memory_observability;
 pub mod profiling;
-#[cfg(any(feature = "ftps", feature = "webdav"))]
+#[cfg(any(feature = "ftps", feature = "webdav", feature = "sftp"))]
 pub mod protocols;
 pub mod server;
 pub mod storage;

rustfs/src/main.rs

@@ -24,6 +24,9 @@ use rustfs::init::{init_ftp_system, init_ftps_system};
 #[cfg(feature = "webdav")]
 use rustfs::init::init_webdav_system;
 
+#[cfg(feature = "sftp")]
+use rustfs::init::init_sftp_system;
+
 use rustfs::capacity::capacity_integration::init_capacity_management;
 use rustfs::license::{current_license, init_license, license_status};
 use rustfs::server::{
@@ -451,6 +454,26 @@ async fn run(config: rustfs::config::Config) -> Result<()> {
     #[cfg(not(feature = "webdav"))]
     let webdav_shutdown_tx: Option<tokio::sync::broadcast::Sender<()>> = None;
 
+    // Initialize SFTP system if enabled
+    #[cfg(feature = "sftp")]
+    let sftp_shutdown_tx = match init_sftp_system().await {
+        Ok(Some(tx)) => {
+            info!("SFTP system initialized successfully");
+            Some(tx)
+        }
+        Ok(None) => {
+            info!("SFTP system disabled");
+            None
+        }
+        Err(e) => {
+            error!("Failed to initialize SFTP system: {}", e);
+            return Err(Error::other(e));
+        }
+    };
+
+    #[cfg(not(feature = "sftp"))]
+    let sftp_shutdown_tx: Option<tokio::sync::broadcast::Sender<()>> = None;
+
     // Initialize buffer profiling system
     init_buffer_profile_system(&config);
 
@@ -595,9 +618,12 @@ async fn run(config: rustfs::config::Config) -> Result<()> {
                 &state_manager,
                 s3_shutdown_tx,
                 console_shutdown_tx,
-                ftp_shutdown_tx,
-                ftps_shutdown_tx,
-                webdav_shutdown_tx,
+                ProtocolShutdownSenders {
+                    ftp: ftp_shutdown_tx,
+                    ftps: ftps_shutdown_tx,
+                    webdav: webdav_shutdown_tx,
+                    sftp: sftp_shutdown_tx,
+                },
                 ctx.clone(),
             )
             .await;
@@ -608,9 +634,12 @@ async fn run(config: rustfs::config::Config) -> Result<()> {
                 &state_manager,
                 s3_shutdown_tx,
                 console_shutdown_tx,
-                ftp_shutdown_tx,
-                ftps_shutdown_tx,
-                webdav_shutdown_tx,
+                ProtocolShutdownSenders {
+                    ftp: ftp_shutdown_tx,
+                    ftps: ftps_shutdown_tx,
+                    webdav: webdav_shutdown_tx,
+                    sftp: sftp_shutdown_tx,
+                },
                 ctx.clone(),
             )
             .await;
@@ -621,16 +650,29 @@ async fn run(config: rustfs::config::Config) -> Result<()> {
     Ok(())
 }
 
+/// Shutdown channels for every protocol server. None means the protocol was
+/// disabled at startup.
+struct ProtocolShutdownSenders {
+    ftp: Option<tokio::sync::broadcast::Sender<()>>,
+    ftps: Option<tokio::sync::broadcast::Sender<()>>,
+    webdav: Option<tokio::sync::broadcast::Sender<()>>,
+    sftp: Option<tokio::sync::broadcast::Sender<()>>,
+}
+
 /// Handles the shutdown process of the server
 async fn handle_shutdown(
     state_manager: &ServiceStateManager,
     s3_shutdown_tx: Option<tokio::sync::broadcast::Sender<()>>,
     console_shutdown_tx: Option<tokio::sync::broadcast::Sender<()>>,
-    ftp_shutdown_tx: Option<tokio::sync::broadcast::Sender<()>>,
-    ftps_shutdown_tx: Option<tokio::sync::broadcast::Sender<()>>,
-    webdav_shutdown_tx: Option<tokio::sync::broadcast::Sender<()>>,
+    protocols: ProtocolShutdownSenders,
     ctx: CancellationToken,
 ) {
+    let ProtocolShutdownSenders {
+        ftp: ftp_shutdown_tx,
+        ftps: ftps_shutdown_tx,
+        webdav: webdav_shutdown_tx,
+        sftp: sftp_shutdown_tx,
+    } = protocols;
     ctx.cancel();
 
     info!(
@@ -694,6 +736,15 @@ async fn handle_shutdown(
         let _ = webdav_shutdown_tx.send(());
     }
 
+    // Shutdown SFTP server
+    if let Some(sftp_shutdown_tx) = sftp_shutdown_tx {
+        info!(
+            target: "rustfs::main::handle_shutdown",
+            "Shutting down SFTP server..."
+        );
+        let _ = sftp_shutdown_tx.send(());
+    }
+
     // Stop the notification system
     info!(
         target: "rustfs::main::handle_shutdown",

rustfs/src/protocols/client.rs

@@ -167,10 +167,11 @@ impl rustfs_protocols::common::client::s3::StorageBackend for ProtocolStorageCli
         let mut headers = HeaderMap::default();
         if let Some(ref body) = input.body {
             let (lower, upper) = body.size_hint();
-            if let Some(len) = upper {
-                headers.insert("content-length", len.to_string().parse().unwrap());
-            } else if lower > 0 {
-                headers.insert("content-length", lower.to_string().parse().unwrap());
+            let resolved_len = upper.or(if lower > 0 { Some(lower) } else { None });
+            if let Some(len) = resolved_len
+                && let Ok(header_value) = len.to_string().parse()
+            {
+                headers.insert("content-length", header_value);
             }
         }
 
@@ -433,6 +434,43 @@ impl rustfs_protocols::common::client::s3::StorageBackend for ProtocolStorageCli
         }
     }
 
+    async fn copy_object(
+        &self,
+        input: CopyObjectInput,
+        access_key: &str,
+        secret_key: &str,
+    ) -> Result<CopyObjectOutput, Self::Error> {
+        trace!("Protocol storage client CopyObject request: bucket={}, key={}", input.bucket, input.key);
+
+        let bucket = input.bucket.clone();
+        let key = input.key.clone();
+        let uri: http::Uri = format!("/{}{}", bucket, key).parse().map_err(|e| {
+            s3s::S3Error::with_message(
+                s3s::S3ErrorCode::InvalidRequest,
+                format!("invalid URI for bucket={} key={}: {}", bucket, key, e),
+            )
+        })?;
+
+        let req = self
+            .create_request(
+                input,
+                Method::PUT,
+                uri,
+                RequestParams {
+                    bucket: Some(bucket),
+                    object: Some(key),
+                    access_key,
+                    secret_key,
+                },
+            )
+            .await?;
+
+        match self.fs.copy_object(req).await {
+            Ok(response) => Ok(response.output),
+            Err(e) => Err(e),
+        }
+    }
+
     async fn delete_bucket(&self, bucket: &str, access_key: &str, secret_key: &str) -> Result<DeleteBucketOutput, Self::Error> {
         trace!("Protocol storage client DeleteBucket request: bucket={}", bucket);
 
@@ -460,4 +498,235 @@ impl rustfs_protocols::common::client::s3::StorageBackend for ProtocolStorageCli
             Err(e) => Err(e),
         }
     }
+
+    async fn create_multipart_upload(
+        &self,
+        input: CreateMultipartUploadInput,
+        access_key: &str,
+        secret_key: &str,
+    ) -> Result<CreateMultipartUploadOutput, Self::Error> {
+        trace!(
+            "Protocol storage client CreateMultipartUpload request: bucket={}, key={}",
+            input.bucket, input.key
+        );
+
+        let bucket = input.bucket.clone();
+        let key = input.key.clone();
+        let uri: http::Uri = format!("/{}{}?uploads", bucket, key).parse().map_err(|e| {
+            s3s::S3Error::with_message(
+                s3s::S3ErrorCode::InvalidRequest,
+                format!("invalid URI for bucket={} key={}: {}", bucket, key, e),
+            )
+        })?;
+
+        let req = self
+            .create_request(
+                input,
+                Method::POST,
+                uri,
+                RequestParams {
+                    bucket: Some(bucket),
+                    object: Some(key),
+                    access_key,
+                    secret_key,
+                },
+            )
+            .await?;
+
+        match self.fs.create_multipart_upload(req).await {
+            Ok(response) => Ok(response.output),
+            Err(e) => Err(e),
+        }
+    }
+
+    async fn upload_part(
+        &self,
+        input: UploadPartInput,
+        access_key: &str,
+        secret_key: &str,
+    ) -> Result<UploadPartOutput, Self::Error> {
+        trace!(
+            "Protocol storage client UploadPart request: bucket={}, key={}, part_number={}",
+            input.bucket, input.key, input.part_number
+        );
+
+        let bucket = input.bucket.clone();
+        let key = input.key.clone();
+        let part_number = input.part_number;
+        let upload_id = input.upload_id.clone();
+        let uri: http::Uri = format!("/{}{}?partNumber={}&uploadId={}", bucket, key, part_number, upload_id)
+            .parse()
+            .map_err(|e| {
+                s3s::S3Error::with_message(
+                    s3s::S3ErrorCode::InvalidRequest,
+                    format!("invalid URI for bucket={} key={} upload_id={}: {}", bucket, key, upload_id, e),
+                )
+            })?;
+
+        // Set content-length from the body size hint so ecfs can bound
+        // the read and validate the part size. Prefer the exact upper
+        // bound when the producer knows it (the common case for an
+        // owned-buffer body). Fall back to the lower bound for truly
+        // streaming bodies of unknown length. Omit the header when the
+        // size is wholly unknown. The request then goes chunked and
+        // ecfs reads until EOF. The parse step cannot fail for ASCII
+        // digit strings, but an if-let keeps the code panic-free if a
+        // future refactor changes the source of the length value.
+        let mut headers = HeaderMap::default();
+        if let Some(ref body) = input.body {
+            let (lower, upper) = body.size_hint();
+            let resolved_len = upper.or(if lower > 0 { Some(lower) } else { None });
+            if let Some(len) = resolved_len
+                && let Ok(header_value) = len.to_string().parse()
+            {
+                headers.insert("content-length", header_value);
+            }
+        }
+
+        let req = self
+            .create_request(
+                input,
+                Method::PUT,
+                uri,
+                RequestParams {
+                    bucket: Some(bucket),
+                    object: Some(key),
+                    access_key,
+                    secret_key,
+                },
+            )
+            .await?;
+        let req = S3Request { headers, ..req };
+
+        match self.fs.upload_part(req).await {
+            Ok(response) => Ok(response.output),
+            Err(e) => Err(e),
+        }
+    }
+
+    async fn complete_multipart_upload(
+        &self,
+        input: CompleteMultipartUploadInput,
+        access_key: &str,
+        secret_key: &str,
+    ) -> Result<CompleteMultipartUploadOutput, Self::Error> {
+        trace!(
+            "Protocol storage client CompleteMultipartUpload request: bucket={}, key={}",
+            input.bucket, input.key
+        );
+
+        let bucket = input.bucket.clone();
+        let key = input.key.clone();
+        let upload_id = input.upload_id.clone();
+        let uri: http::Uri = format!("/{}{}?uploadId={}", bucket, key, upload_id).parse().map_err(|e| {
+            s3s::S3Error::with_message(
+                s3s::S3ErrorCode::InvalidRequest,
+                format!("invalid URI for bucket={} key={} upload_id={}: {}", bucket, key, upload_id, e),
+            )
+        })?;
+
+        let req = self
+            .create_request(
+                input,
+                Method::POST,
+                uri,
+                RequestParams {
+                    bucket: Some(bucket),
+                    object: Some(key),
+                    access_key,
+                    secret_key,
+                },
+            )
+            .await?;
+
+        match self.fs.complete_multipart_upload(req).await {
+            Ok(response) => Ok(response.output),
+            Err(e) => Err(e),
+        }
+    }
+
+    async fn abort_multipart_upload(
+        &self,
+        input: AbortMultipartUploadInput,
+        access_key: &str,
+        secret_key: &str,
+    ) -> Result<AbortMultipartUploadOutput, Self::Error> {
+        trace!(
+            "Protocol storage client AbortMultipartUpload request: bucket={}, key={}, upload_id={}",
+            input.bucket, input.key, input.upload_id
+        );
+
+        let bucket = input.bucket.clone();
+        let key = input.key.clone();
+        let upload_id = input.upload_id.clone();
+        let uri: http::Uri = format!("/{}{}?uploadId={}", bucket, key, upload_id).parse().map_err(|e| {
+            s3s::S3Error::with_message(
+                s3s::S3ErrorCode::InvalidRequest,
+                format!("invalid URI for bucket={} key={} upload_id={}: {}", bucket, key, upload_id, e),
+            )
+        })?;
+
+        let req = self
+            .create_request(
+                input,
+                Method::DELETE,
+                uri,
+                RequestParams {
+                    bucket: Some(bucket),
+                    object: Some(key),
+                    access_key,
+                    secret_key,
+                },
+            )
+            .await?;
+
+        match self.fs.abort_multipart_upload(req).await {
+            Ok(response) => Ok(response.output),
+            Err(e) => Err(e),
+        }
+    }
+
+    async fn upload_part_copy(
+        &self,
+        input: UploadPartCopyInput,
+        access_key: &str,
+        secret_key: &str,
+    ) -> Result<UploadPartCopyOutput, Self::Error> {
+        trace!(
+            "Protocol storage client UploadPartCopy request: bucket={}, key={}, part_number={}",
+            input.bucket, input.key, input.part_number
+        );
+
+        let bucket = input.bucket.clone();
+        let key = input.key.clone();
+        let part_number = input.part_number;
+        let upload_id = input.upload_id.clone();
+        let uri: http::Uri = format!("/{}{}?partNumber={}&uploadId={}", bucket, key, part_number, upload_id)
+            .parse()
+            .map_err(|e| {
+                s3s::S3Error::with_message(
+                    s3s::S3ErrorCode::InvalidRequest,
+                    format!("invalid URI for bucket={} key={} upload_id={}: {}", bucket, key, upload_id, e),
+                )
+            })?;
+
+        let req = self
+            .create_request(
+                input,
+                Method::PUT,
+                uri,
+                RequestParams {
+                    bucket: Some(bucket),
+                    object: Some(key),
+                    access_key,
+                    secret_key,
+                },
+            )
+            .await?;
+
+        match self.fs.upload_part_copy(req).await {
+            Ok(response) => Ok(response.output),
+            Err(e) => Err(e),
+        }
+    }
 }