gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-12 16:16:55 +00:00
Code Issues Packages Projects Releases Wiki Activity

Feat/ftps&sftp (#1308)

Browse Source 
[feat] ftp / sftp
This commit is contained in:
yxrxy
2025-12-31 09:01:15 +08:00
committed by GitHub
parent 3c14947878
commit b8aa8214e2
38 changed files with 6162 additions and 57 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/e2e_test/Cargo.toml
+8
View File
@@ -26,6 +26,7 @@ workspace = true
[dependencies]
rustfs-ecstore.workspace = true
rustfs-common.workspace = true
rustfs-iam.workspace = true
flatbuffers.workspace = true
futures.workspace = true
rustfs-lock.workspace = true
@@ -51,3 +52,10 @@ base64 = { workspace = true }
rand = { workspace = true }
chrono = { workspace = true }
md5 = { workspace = true }
ssh2.workspace = true
suppaftp.workspace = true
rcgen.workspace = true
anyhow.workspace = true
native-tls.workspace = true
rustls.workspace = true
rustls-pemfile.workspace = true
crates/e2e_test/src/common.rs
+3 -3
View File
@@ -34,8 +34,8 @@ use tracing::{error, info, warn};
use uuid::Uuid;
// Common constants for all E2E tests
pub const DEFAULT_ACCESS_KEY: &str = "minioadmin";
pub const DEFAULT_SECRET_KEY: &str = "minioadmin";
pub const DEFAULT_ACCESS_KEY: &str = "rustfsadmin";
pub const DEFAULT_SECRET_KEY: &str = "rustfsadmin";
pub const TEST_BUCKET: &str = "e2e-test-bucket";
pub fn workspace_root() -> PathBuf {
let mut path = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
@@ -165,7 +165,7 @@ impl RustFSTestEnvironment {
}
/// Find an available port for the test
async fn find_available_port() -> Result<u16, Box<dyn std::error::Error + Send + Sync>> {
pub async fn find_available_port() -> Result<u16, Box<dyn std::error::Error + Send + Sync>> {
use std::net::TcpListener;
let listener = TcpListener::bind("127.0.0.1:0")?;
let port = listener.local_addr()?.port();
crates/e2e_test/src/lib.rs
+3
View File
@@ -40,3 +40,6 @@ mod content_encoding_test;
// Policy variables tests
#[cfg(test)]
mod policy;
#[cfg(test)]
mod protocols;
crates/e2e_test/src/protocols/README.md
+44
View File
@@ -0,0 +1,44 @@
# Protocol E2E Tests
FTPS and SFTP protocol end-to-end tests for RustFS.
## Prerequisites
### Required Tools
```bash
# Ubuntu/Debian
sudo apt-get install sshpass ssh-keygen
# RHEL/CentOS
sudo yum install sshpass openssh-clients
# macOS
brew install sshpass openssh
```
## Running Tests
Run all protocol tests:
```bash
cargo test --package e2e_test test_protocol_core_suite -- --test-threads=1 --nocapture
```
Run only FTPS tests:
```bash
cargo test --package e2e_test test_ftps_core_operations -- --test-threads=1 --nocapture
```
## Test Coverage
### FTPS Tests
- mkdir bucket
- cd to bucket
- put file
- ls list objects
- cd . (stay in current directory)
- cd / (return to root)
- cd nonexistent bucket (should fail)
- delete object
- cdup
- rmdir delete bucket
crates/e2e_test/src/protocols/ftps_core.rs
+211
View File
@@ -0,0 +1,211 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Core FTPS tests
use crate::common::rustfs_binary_path;
use crate::protocols::test_env::{DEFAULT_ACCESS_KEY, DEFAULT_SECRET_KEY, ProtocolTestEnvironment};
use anyhow::Result;
use native_tls::TlsConnector;
use rcgen::generate_simple_self_signed;
use std::io::Cursor;
use std::path::PathBuf;
use suppaftp::NativeTlsConnector;
use suppaftp::NativeTlsFtpStream;
use tokio::process::Command;
use tracing::info;
// Fixed FTPS port for testing
const FTPS_PORT: u16 = 9021;
const FTPS_ADDRESS: &str = "127.0.0.1:9021";
/// Test FTPS: put, ls, mkdir, rmdir, delete operations
pub async fn test_ftps_core_operations() -> Result<()> {
let env = ProtocolTestEnvironment::new().map_err(|e| anyhow::anyhow!("{}", e))?;
// Generate and write certificate
let cert = generate_simple_self_signed(vec!["localhost".to_string(), "127.0.0.1".to_string()])?;
let cert_path = PathBuf::from(&env.temp_dir).join("ftps.crt");
let key_path = PathBuf::from(&env.temp_dir).join("ftps.key");
let cert_pem = cert.cert.pem();
let key_pem = cert.signing_key.serialize_pem();
tokio::fs::write(&cert_path, &cert_pem).await?;
tokio::fs::write(&key_path, &key_pem).await?;
// Start server manually
info!("Starting FTPS server on {}", FTPS_ADDRESS);
let binary_path = rustfs_binary_path();
let mut server_process = Command::new(&binary_path)
.args([
"--ftps-enable",
"--ftps-address",
FTPS_ADDRESS,
"--ftps-certs-file",
cert_path.to_str().unwrap(),
"--ftps-key-file",
key_path.to_str().unwrap(),
&env.temp_dir,
])
.spawn()?;
// Ensure server is cleaned up even on failure
let result = async {
// Wait for server to be ready
ProtocolTestEnvironment::wait_for_port_ready(FTPS_PORT, 30)
.await
.map_err(|e| anyhow::anyhow!("{}", e))?;
// Create native TLS connector that accepts the certificate
let tls_connector = TlsConnector::builder().danger_accept_invalid_certs(true).build()?;
// Wrap in suppaftp's NativeTlsConnector
let tls_connector = NativeTlsConnector::from(tls_connector);
// Connect to FTPS server
let ftp_stream = NativeTlsFtpStream::connect(FTPS_ADDRESS).map_err(|e| anyhow::anyhow!("Failed to connect: {}", e))?;
// Upgrade to secure connection
let mut ftp_stream = ftp_stream
.into_secure(tls_connector, "127.0.0.1")
.map_err(|e| anyhow::anyhow!("Failed to upgrade to TLS: {}", e))?;
ftp_stream.login(DEFAULT_ACCESS_KEY, DEFAULT_SECRET_KEY)?;
info!("Testing FTPS: mkdir bucket");
let bucket_name = "testbucket";
ftp_stream.mkdir(bucket_name)?;
info!("PASS: mkdir bucket '{}' successful", bucket_name);
info!("Testing FTPS: cd to bucket");
ftp_stream.cwd(bucket_name)?;
info!("PASS: cd to bucket '{}' successful", bucket_name);
info!("Testing FTPS: put file");
let filename = "test.txt";
let content = "Hello, FTPS!";
ftp_stream.put_file(filename, &mut Cursor::new(content.as_bytes()))?;
info!("PASS: put file '{}' ({} bytes) successful", filename, content.len());
info!("Testing FTPS: ls list objects in bucket");
let list = ftp_stream.list(None)?;
assert!(list.iter().any(|line| line.contains(filename)), "File should appear in list");
info!("PASS: ls command successful, file '{}' found in bucket", filename);
info!("Testing FTPS: ls . (list current directory)");
let list_dot = ftp_stream.list(Some(".")).unwrap_or_else(|_| ftp_stream.list(None).unwrap());
assert!(list_dot.iter().any(|line| line.contains(filename)), "File should appear in ls .");
info!("PASS: ls . successful, file '{}' found", filename);
info!("Testing FTPS: ls / (list root directory)");
let list_root = ftp_stream.list(Some("/")).unwrap();
assert!(list_root.iter().any(|line| line.contains(bucket_name)), "Bucket should appear in ls /");
assert!(!list_root.iter().any(|line| line.contains(filename)), "File should not appear in ls /");
info!(
"PASS: ls / successful, bucket '{}' found, file '{}' not found in root",
bucket_name, filename
);
info!("Testing FTPS: ls /. (list root directory with /.)");
let list_root_dot = ftp_stream
.list(Some("/."))
.unwrap_or_else(|_| ftp_stream.list(Some("/")).unwrap());
assert!(
list_root_dot.iter().any(|line| line.contains(bucket_name)),
"Bucket should appear in ls /."
);
info!("PASS: ls /. successful, bucket '{}' found", bucket_name);
info!("Testing FTPS: ls /bucket (list bucket by absolute path)");
let list_bucket = ftp_stream.list(Some(&format!("/{}", bucket_name))).unwrap();
assert!(list_bucket.iter().any(|line| line.contains(filename)), "File should appear in ls /bucket");
info!("PASS: ls /{} successful, file '{}' found", bucket_name, filename);
info!("Testing FTPS: cd . (stay in current directory)");
ftp_stream.cwd(".")?;
info!("PASS: cd . successful (stays in current directory)");
info!("Testing FTPS: ls after cd . (should still see file)");
let list_after_dot = ftp_stream.list(None)?;
assert!(
list_after_dot.iter().any(|line| line.contains(filename)),
"File should still appear in list after cd ."
);
info!("PASS: ls after cd . successful, file '{}' still found in bucket", filename);
info!("Testing FTPS: cd / (go to root directory)");
ftp_stream.cwd("/")?;
info!("PASS: cd / successful (back to root directory)");
info!("Testing FTPS: ls after cd / (should see bucket only)");
let root_list_after = ftp_stream.list(None)?;
assert!(
!root_list_after.iter().any(|line| line.contains(filename)),
"File should not appear in root ls"
);
assert!(
root_list_after.iter().any(|line| line.contains(bucket_name)),
"Bucket should appear in root ls"
);
info!("PASS: ls after cd / successful, file not in root, bucket '{}' found in root", bucket_name);
info!("Testing FTPS: cd back to bucket");
ftp_stream.cwd(bucket_name)?;
info!("PASS: cd back to bucket '{}' successful", bucket_name);
info!("Testing FTPS: delete object");
ftp_stream.rm(filename)?;
info!("PASS: delete object '{}' successful", filename);
info!("Testing FTPS: ls verify object deleted");
let list_after = ftp_stream.list(None)?;
assert!(!list_after.iter().any(|line| line.contains(filename)), "File should be deleted");
info!("PASS: ls after delete successful, file '{}' is not found", filename);
info!("Testing FTPS: cd up to root directory");
ftp_stream.cdup()?;
info!("PASS: cd up to root directory successful");
info!("Testing FTPS: cd to nonexistent bucket (should fail)");
let nonexistent_bucket = "nonexistent-bucket";
let cd_result = ftp_stream.cwd(nonexistent_bucket);
assert!(cd_result.is_err(), "cd to nonexistent bucket should fail");
info!("PASS: cd to nonexistent bucket '{}' failed as expected", nonexistent_bucket);
info!("Testing FTPS: ls verify bucket exists in root");
let root_list = ftp_stream.list(None)?;
assert!(root_list.iter().any(|line| line.contains(bucket_name)), "Bucket should exist in root");
info!("PASS: ls root successful, bucket '{}' found in root", bucket_name);
info!("Testing FTPS: rmdir delete bucket");
ftp_stream.rmdir(bucket_name)?;
info!("PASS: rmdir bucket '{}' successful", bucket_name);
info!("Testing FTPS: ls verify bucket deleted");
let root_list_after = ftp_stream.list(None)?;
assert!(!root_list_after.iter().any(|line| line.contains(bucket_name)), "Bucket should be deleted");
info!("PASS: ls root after delete successful, bucket '{}' is not found", bucket_name);
ftp_stream.quit()?;
info!("FTPS core tests passed");
Ok(())
}
.await;
// Always cleanup server process
let _ = server_process.kill().await;
let _ = server_process.wait().await;
result
}
crates/e2e_test/src/protocols/mod.rs
+19
View File
@@ -0,0 +1,19 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Protocol tests for FTPS and SFTP
pub mod ftps_core;
pub mod test_env;
pub mod test_runner;
crates/e2e_test/src/protocols/test_env.rs
+72
View File
@@ -0,0 +1,72 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Protocol test environment for FTPS and SFTP
use std::net::TcpStream;
use std::time::Duration;
use tokio::time::sleep;
use tracing::{info, warn};
/// Default credentials
pub const DEFAULT_ACCESS_KEY: &str = "rustfsadmin";
pub const DEFAULT_SECRET_KEY: &str = "rustfsadmin";
/// Custom test environment that doesn't automatically stop servers
pub struct ProtocolTestEnvironment {
pub temp_dir: String,
}
impl ProtocolTestEnvironment {
/// Create a new test environment
/// This environment won't stop any server when dropped
pub fn new() -> Result<Self, Box<dyn std::error::Error + Send + Sync>> {
let temp_dir = format!("/tmp/rustfs_protocol_test_{}", uuid::Uuid::new_v4());
std::fs::create_dir_all(&temp_dir)?;
Ok(Self { temp_dir })
}
/// Wait for server to be ready
pub async fn wait_for_port_ready(port: u16, max_attempts: u32) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
let address = format!("127.0.0.1:{}", port);
info!("Waiting for server to be ready on {}", address);
for i in 0..max_attempts {
if TcpStream::connect(&address).is_ok() {
info!("Server is ready after {} s", i + 1);
return Ok(());
}
if i == max_attempts - 1 {
return Err(format!("Server did not become ready within {} s", max_attempts).into());
}
sleep(Duration::from_secs(1)).await;
}
Ok(())
}
}
// Implement Drop trait that doesn't stop servers
impl Drop for ProtocolTestEnvironment {
fn drop(&mut self) {
// Clean up temp directory only, don't stop any server
if let Err(e) = std::fs::remove_dir_all(&self.temp_dir) {
warn!("Failed to clean up temp directory {}: {}", self.temp_dir, e);
}
}
}
crates/e2e_test/src/protocols/test_runner.rs
+171
View File
@@ -0,0 +1,171 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Protocol test runner
use crate::common::init_logging;
use crate::protocols::ftps_core::test_ftps_core_operations;
use std::time::Instant;
use tokio::time::{Duration, sleep};
use tracing::{error, info};
/// Test result
#[derive(Debug, Clone)]
pub struct TestResult {
pub test_name: String,
pub success: bool,
pub error_message: Option<String>,
}
impl TestResult {
pub fn success(test_name: String) -> Self {
Self {
test_name,
success: true,
error_message: None,
}
}
pub fn failure(test_name: String, error: String) -> Self {
Self {
test_name,
success: false,
error_message: Some(error),
}
}
}
/// Protocol test suite
pub struct ProtocolTestSuite {
tests: Vec<TestDefinition>,
}
#[derive(Debug, Clone)]
struct TestDefinition {
name: String,
}
impl ProtocolTestSuite {
/// Create default test suite
pub fn new() -> Self {
let tests = vec![
TestDefinition {
name: "test_ftps_core_operations".to_string(),
},
// TestDefinition { name: "test_sftp_core_operations".to_string() },
];
Self { tests }
}
/// Run test suite
pub async fn run_test_suite(&self) -> Vec<TestResult> {
init_logging();
info!("Starting Protocol test suite");
let start_time = Instant::now();
let mut results = Vec::new();
info!("Scheduled {} tests", self.tests.len());
// Run tests
for (i, test_def) in self.tests.iter().enumerate() {
let test_description = match test_def.name.as_str() {
"test_ftps_core_operations" => {
info!("=== Starting FTPS Module Test ===");
"FTPS core operations (put, ls, mkdir, rmdir, delete)"
}
"test_sftp_core_operations" => {
info!("=== Starting SFTP Module Test ===");
"SFTP core operations (put, ls, mkdir, rmdir, delete)"
}
_ => "",
};
info!("Test {}/{} - {}", i + 1, self.tests.len(), test_description);
info!("Running: {}", test_def.name);
let test_start = Instant::now();
let result = self.run_single_test(test_def).await;
let test_duration = test_start.elapsed();
match result {
Ok(_) => {
info!("Test passed: {} ({:.2}s)", test_def.name, test_duration.as_secs_f64());
results.push(TestResult::success(test_def.name.clone()));
}
Err(e) => {
error!("Test failed: {} ({:.2}s): {}", test_def.name, test_duration.as_secs_f64(), e);
results.push(TestResult::failure(test_def.name.clone(), e.to_string()));
}
}
// Delay between tests to avoid resource conflicts
if i < self.tests.len() - 1 {
sleep(Duration::from_secs(2)).await;
}
}
// Print summary
self.print_summary(&results, start_time.elapsed());
results
}
/// Run a single test
async fn run_single_test(&self, test_def: &TestDefinition) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
match test_def.name.as_str() {
"test_ftps_core_operations" => test_ftps_core_operations().await.map_err(|e| e.into()),
// "test_sftp_core_operations" => test_sftp_core_operations().await.map_err(|e| e.into()),
_ => Err(format!("Test {} not implemented", test_def.name).into()),
}
}
/// Print test summary
fn print_summary(&self, results: &[TestResult], total_duration: Duration) {
info!("=== Test Suite Summary ===");
info!("Total duration: {:.2}s", total_duration.as_secs_f64());
info!("Total tests: {}", results.len());
let passed = results.iter().filter(|r| r.success).count();
let failed = results.len() - passed;
let success_rate = (passed as f64 / results.len() as f64) * 100.0;
info!("Passed: {} | Failed: {}", passed, failed);
info!("Success rate: {:.1}%", success_rate);
if failed > 0 {
error!("Failed tests:");
for result in results.iter().filter(|r| !r.success) {
error!(" - {}: {}", result.test_name, result.error_message.as_ref().unwrap());
}
}
}
}
/// Test suite
#[tokio::test]
async fn test_protocol_core_suite() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
let suite = ProtocolTestSuite::new();
let results = suite.run_test_suite().await;
let failed = results.iter().filter(|r| !r.success).count();
if failed > 0 {
return Err(format!("Protocol tests failed: {failed} failures").into());
}
info!("All protocol tests passed");
Ok(())
}
crates/iam/src/manager.rs
+22
View File
@@ -1258,6 +1258,28 @@ where
self.update_user_with_claims(access_key, u)
}
/// Add SSH public key for a user (for SFTP authentication)
pub async fn add_user_ssh_public_key(&self, access_key: &str, public_key: &str) -> Result<()> {
if access_key.is_empty() || public_key.is_empty() {
return Err(Error::InvalidArgument);
}
let users = self.cache.users.load();
let u = match users.get(access_key) {
Some(u) => u,
None => return Err(Error::NoSuchUser(access_key.to_string())),
};
let mut user_identity = u.clone();
user_identity.add_ssh_public_key(public_key);
self.api
.save_user_identity(access_key, UserType::Reg, user_identity.clone(), None)
.await?;
self.update_user_with_claims(access_key, user_identity)
}
pub async fn set_user_status(&self, access_key: &str, status: AccountStatus) -> Result<OffsetDateTime> {
if access_key.is_empty() {
return Err(Error::InvalidArgument);
crates/iam/src/sys.rs
+13
View File
@@ -637,6 +637,19 @@ impl<T: Store> IamSys<T> {
self.store.update_user_secret_key(access_key, secret_key).await
}
/// Add SSH public key for a user (for SFTP authentication)
pub async fn add_user_ssh_public_key(&self, access_key: &str, public_key: &str) -> Result<()> {
if !is_access_key_valid(access_key) {
return Err(IamError::InvalidAccessKeyLength);
}
if public_key.is_empty() {
return Err(IamError::InvalidArgument);
}
self.store.add_user_ssh_public_key(access_key, public_key).await
}
pub async fn check_key(&self, access_key: &str) -> Result<(Option<UserIdentity>, bool)> {
if let Some(sys_cred) = get_global_action_cred() {
if sys_cred.access_key == access_key {
crates/policy/src/auth/mod.rs
+21
View File
@@ -18,6 +18,8 @@ pub use credentials::*;
use rustfs_credentials::Credentials;
use serde::{Deserialize, Serialize};
use serde_json::json;
use std::collections::HashMap;
use time::OffsetDateTime;
#[derive(Serialize, Deserialize, Clone, Debug, Default)]
@@ -42,6 +44,25 @@ impl UserIdentity {
update_at: Some(OffsetDateTime::now_utc()),
}
}
/// Add an SSH public key to user identity for SFTP authentication
pub fn add_ssh_public_key(&mut self, public_key: &str) {
self.credentials
.claims
.get_or_insert_with(HashMap::new)
.insert("ssh_public_keys".to_string(), json!([public_key]));
}
/// Get all SSH public keys for user identity
pub fn get_ssh_public_keys(&self) -> Vec<String> {
self.credentials
.claims
.as_ref()
.and_then(|claims| claims.get("ssh_public_keys"))
.and_then(|keys| keys.as_array())
.map(|arr| arr.iter().filter_map(|v| v.as_str()).map(String::from).collect())
.unwrap_or_default()
}
}
impl From<Credentials> for UserIdentity {