diff --git a/.github/workflows/e2e-upgrade.yml b/.github/workflows/e2e-upgrade.yml new file mode 100644 index 000000000..27ce41f2d --- /dev/null +++ b/.github/workflows/e2e-upgrade.yml @@ -0,0 +1,105 @@ +# Copyright 2024 RustFS Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: Upgrade Compatibility + +on: + pull_request: + paths: + - ".github/workflows/e2e-upgrade.yml" + - "crates/e2e_test/src/common.rs" + - "crates/e2e_test/src/lib.rs" + - "crates/e2e_test/src/upgrade_compatibility_test.rs" + - "crates/ecstore/**" + - "crates/filemeta/**" + - "crates/kms/**" + - "crates/storage-api/**" + - "rustfs/**" + - "Cargo.lock" + push: + tags: + - "[0-9]*.[0-9]*.[0-9]*" + schedule: + - cron: "17 3 * * 1" + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +env: + CARGO_TERM_COLOR: always + RUST_BACKTRACE: 1 + UPGRADE_SOURCE_VERSION: 1.0.0-rc.2 + UPGRADE_SOURCE_ASSET: rustfs-linux-x86_64-gnu-v1.0.0-rc.2.zip + UPGRADE_SOURCE_SHA256: 7c789386bf85278f865b8e0d359bf4edb84d5aa408cc3fa54a18c25ca74cd6e7 + +jobs: + direct-upgrade: + name: Direct upgrade from rc.2 + runs-on: ubuntu-latest + timeout-minutes: 60 + env: + FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true" + steps: + - name: Checkout repository + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 + with: + persist-credentials: false + + - name: Setup Rust environment + uses: ./.github/actions/setup + with: + cache-shared-key: e2e-direct-upgrade + cache-save-if: ${{ github.ref == 'refs/heads/main' }} + install-build-packaging-tools: "false" + + - name: Download pinned previous release + env: + SOURCE_DIR: ${{ runner.temp }}/rustfs-upgrade-source + run: | + set -euo pipefail + mkdir -p "$SOURCE_DIR" + archive="$SOURCE_DIR/$UPGRADE_SOURCE_ASSET" + curl --fail --location --retry 3 --output "$archive" \ + "https://github.com/${GITHUB_REPOSITORY}/releases/download/${UPGRADE_SOURCE_VERSION}/${UPGRADE_SOURCE_ASSET}" + echo "$UPGRADE_SOURCE_SHA256 $archive" | sha256sum --check --strict + unzip -q "$archive" -d "$SOURCE_DIR" + chmod +x "$SOURCE_DIR/rustfs" + test -x "$SOURCE_DIR/rustfs" + echo "RUSTFS_UPGRADE_SOURCE_BINARY=$SOURCE_DIR/rustfs" >> "$GITHUB_ENV" + echo "RUSTFS_E2E_LOG_DIR=$RUNNER_TEMP/rustfs-upgrade-logs" >> "$GITHUB_ENV" + + - name: Build current RustFS binary + run: | + cargo build --locked -p rustfs --bin rustfs + : > target/debug/rustfs.features + + - name: Run direct-upgrade compatibility test + run: | + cargo test --locked -p e2e_test \ + upgrade_compatibility_test::direct_upgrade_from_rc2_preserves_object_contracts \ + -- --ignored --exact --nocapture + + - name: Upload server logs + if: always() + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 + with: + name: direct-upgrade-server-logs-${{ github.run_number }} + path: ${{ runner.temp }}/rustfs-upgrade-logs + if-no-files-found: warn + retention-days: 14 diff --git a/crates/e2e_test/README.md b/crates/e2e_test/README.md index 443cd9fa1..334936a68 100644 --- a/crates/e2e_test/README.md +++ b/crates/e2e_test/README.md @@ -27,6 +27,7 @@ Registered in [`src/lib.rs`](src/lib.rs). Grouped by concern: | **reliant** | [`src/reliant/`](src/reliant) | Tests that reuse an **externally started** server (SQL/select, conditional writes, lifecycle, deleted-object reads, node-interact). Run via [`scripts/run_e2e_tests.sh`](../../scripts/run_e2e_tests.sh); see [`src/reliant/README.md`](src/reliant/README.md) | | **cluster** | `cluster_concurrency_test`, `stale_multipart_cleanup_cluster_test`, `namespace_lock_quorum_test`, `admin_timeout_regression_test`, `object_lambda_test`, `replication_extension_test` | Multi-node scenarios via `RustFSTestClusterEnvironment` | | **chaos / reliability** | [`src/chaos.rs`](src/chaos.rs), `reliability_disk_fault_test`, `heal_erasure_disk_rebuild_test`, `server_startup_failfast_test` | Disk offline/replace/corrupt, EC rebuild, heal, fail-fast startup | +| **upgrade compatibility** | `upgrade_compatibility_test` | Pinned previous-release writes followed by current-build reads on the same data directory | ## How to run @@ -168,6 +169,7 @@ the same profile for membership and execution with one nightly worker. | `s3s-e2e` black-box | `e2e-tests` + `e2e-tests-rio-v2` jobs | **Active** (external conformance tool) | | ILM / lifecycle (ignored) | `test-ilm-integration-serial` lane, `-j1` | **Active** (backlog#1148 ilm-1) | | KMS suite | `e2e-full` job, merge queue + main | **Active** | +| Direct upgrade from pinned previous release | `e2e-upgrade.yml`, storage-sensitive PRs + release tags + weekly | **Active** | | Cluster faults (`e2e-nightly` profile) | consolidated nightly workflow | **Active** (backlog#1149 ci-7) | | Protocols (FTPS/WebDAV/SFTP) | consolidated nightly workflow, serial | **Active** (backlog#1149 ci-7) | | Replication (fast subset) | `e2e-smoke` profile, `e2e-tests` job, every PR | **Active** (backlog#1147 repl-1) | diff --git a/crates/e2e_test/src/common.rs b/crates/e2e_test/src/common.rs index a3a382956..93a533113 100644 --- a/crates/e2e_test/src/common.rs +++ b/crates/e2e_test/src/common.rs @@ -638,6 +638,18 @@ impl RustFSTestEnvironment { extra_args: Vec<&str>, extra_env: &[(&str, &str)], cleanup_existing: bool, + ) -> Result<(), Box> { + let binary_path = rustfs_binary_path(); + self.start_rustfs_server_inner_with_binary(&binary_path, extra_args, extra_env, cleanup_existing) + .await + } + + async fn start_rustfs_server_inner_with_binary( + &mut self, + binary_path: &Path, + extra_args: Vec<&str>, + extra_env: &[(&str, &str)], + cleanup_existing: bool, ) -> Result<(), Box> { if cleanup_existing { self.cleanup_existing_processes().await?; @@ -647,8 +659,7 @@ impl RustFSTestEnvironment { info!("Starting RustFS server with args: {:?}", args); - let binary_path = rustfs_binary_path(); - let mut command = Command::new(&binary_path); + let mut command = Command::new(binary_path); command.env("RUST_LOG", "rustfs=info,rustfs_notify=debug"); // The embedded console would bind the fixed default port :9001, which // collides with unrelated local services (e.g. Docker Desktop). Tests @@ -668,6 +679,19 @@ impl RustFSTestEnvironment { Ok(()) } + /// Start a specific RustFS binary against this environment's isolated + /// data directory. Upgrade tests use this to seed an old on-disk format + /// before restarting the same environment with the workspace binary. + pub async fn start_rustfs_server_from_binary( + &mut self, + binary_path: &Path, + extra_args: Vec<&str>, + extra_env: &[(&str, &str)], + ) -> Result<(), Box> { + self.start_rustfs_server_inner_with_binary(binary_path, extra_args, extra_env, true) + .await + } + /// Start RustFS server with basic configuration pub async fn start_rustfs_server(&mut self, extra_args: Vec<&str>) -> Result<(), Box> { self.start_rustfs_server_inner(extra_args, &[], true).await diff --git a/crates/e2e_test/src/lib.rs b/crates/e2e_test/src/lib.rs index 28c63b267..6ff615964 100644 --- a/crates/e2e_test/src/lib.rs +++ b/crates/e2e_test/src/lib.rs @@ -61,6 +61,10 @@ mod get_codec_streaming_compat_test; #[cfg(test)] mod version_id_regression_test; +// Pinned previous-release -> current-build on-disk compatibility. +#[cfg(test)] +mod upgrade_compatibility_test; + // Receiver-side replication LWW (rustfs/backlog#1953): stale inbound // replication metadata must not overwrite a newer local category state. #[cfg(test)] diff --git a/crates/e2e_test/src/upgrade_compatibility_test.rs b/crates/e2e_test/src/upgrade_compatibility_test.rs new file mode 100644 index 000000000..3485f9593 --- /dev/null +++ b/crates/e2e_test/src/upgrade_compatibility_test.rs @@ -0,0 +1,254 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::common::{RustFSTestEnvironment, init_logging}; +use aws_sdk_s3::Client; +use aws_sdk_s3::error::ProvideErrorMetadata; +use aws_sdk_s3::primitives::ByteStream; +use aws_sdk_s3::types::{ + BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, ServerSideEncryption, VersioningConfiguration, +}; +use std::path::PathBuf; + +type TestResult = Result<(), Box>; + +const SOURCE_BINARY_ENV: &str = "RUSTFS_UPGRADE_SOURCE_BINARY"; +const SSE_MASTER_KEY_ENV: &str = "RUSTFS_SSE_S3_MASTER_KEY"; +const SSE_MASTER_KEY: &str = "QkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkI="; +const PLAIN_BUCKET: &str = "upgrade-plain-data"; +const VERSIONED_BUCKET: &str = "upgrade-versioned-data"; + +fn source_binary() -> Result> { + let path = std::env::var_os(SOURCE_BINARY_ENV) + .map(PathBuf::from) + .ok_or("RUSTFS_UPGRADE_SOURCE_BINARY must point to the pinned previous release binary")?; + if !path.is_file() { + return Err(format!("upgrade source binary does not exist: {}", path.display()).into()); + } + Ok(path) +} + +async fn enable_versioning(client: &Client, bucket: &str) -> TestResult { + let configuration = VersioningConfiguration::builder() + .status(BucketVersioningStatus::Enabled) + .build(); + client + .put_bucket_versioning() + .bucket(bucket) + .versioning_configuration(configuration) + .send() + .await?; + Ok(()) +} + +async fn read_object( + client: &Client, + bucket: &str, + key: &str, + version_id: Option<&str>, +) -> Result<(Option, Vec), Box> { + let mut request = client.get_object().bucket(bucket).key(key); + if let Some(version_id) = version_id { + request = request.version_id(version_id); + } + let response = request.send().await?; + let encryption = response.server_side_encryption().cloned(); + let body = response.body.collect().await?.into_bytes().to_vec(); + Ok((encryption, body)) +} + +async fn write_multipart(client: &Client, bucket: &str, key: &str, parts: &[Vec]) -> TestResult { + let created = client.create_multipart_upload().bucket(bucket).key(key).send().await?; + let upload_id = created.upload_id().ok_or("CreateMultipartUpload omitted upload ID")?; + let mut completed_parts = Vec::with_capacity(parts.len()); + + for (index, part) in parts.iter().enumerate() { + let part_number = i32::try_from(index + 1)?; + let uploaded = client + .upload_part() + .bucket(bucket) + .key(key) + .upload_id(upload_id) + .part_number(part_number) + .body(ByteStream::from(part.clone())) + .send() + .await?; + completed_parts.push( + CompletedPart::builder() + .part_number(part_number) + .e_tag(uploaded.e_tag().ok_or("UploadPart omitted ETag")?) + .build(), + ); + } + + client + .complete_multipart_upload() + .bucket(bucket) + .key(key) + .upload_id(upload_id) + .multipart_upload(CompletedMultipartUpload::builder().set_parts(Some(completed_parts)).build()) + .send() + .await?; + Ok(()) +} + +#[tokio::test] +#[ignore = "requires a pinned previous RustFS release binary"] +async fn direct_upgrade_from_rc2_preserves_object_contracts() -> TestResult { + init_logging(); + let previous_binary = source_binary()?; + let mut env = RustFSTestEnvironment::new().await?; + let server_env = [(SSE_MASTER_KEY_ENV, SSE_MASTER_KEY)]; + env.start_rustfs_server_from_binary(&previous_binary, vec![], &server_env) + .await?; + + let old_client = env.create_s3_client(); + env.create_test_bucket(PLAIN_BUCKET).await?; + env.create_test_bucket(VERSIONED_BUCKET).await?; + enable_versioning(&old_client, VERSIONED_BUCKET).await?; + + let plain_key = "plain-object"; + let plain_bytes = b"written by the previous RustFS release"; + old_client + .put_object() + .bucket(PLAIN_BUCKET) + .key(plain_key) + .body(ByteStream::from_static(plain_bytes)) + .send() + .await?; + + let encrypted_key = "sse-s3-object"; + let encrypted_bytes = b"encrypted by the previous RustFS release"; + old_client + .put_object() + .bucket(PLAIN_BUCKET) + .key(encrypted_key) + .server_side_encryption(ServerSideEncryption::Aes256) + .body(ByteStream::from_static(encrypted_bytes)) + .send() + .await?; + + let multipart_key = "multipart-object"; + let multipart_parts = vec![vec![b'a'; 5 * 1024 * 1024], b"final multipart bytes".to_vec()]; + let multipart_bytes = multipart_parts.concat(); + write_multipart(&old_client, PLAIN_BUCKET, multipart_key, &multipart_parts).await?; + + let versioned_key = "versioned-object"; + let version1_bytes = b"version one from the previous release"; + let version1 = old_client + .put_object() + .bucket(VERSIONED_BUCKET) + .key(versioned_key) + .body(ByteStream::from_static(version1_bytes)) + .send() + .await? + .version_id() + .ok_or("first versioned PUT omitted version ID")? + .to_string(); + let version2_bytes = b"version two from the previous release"; + let version2 = old_client + .put_object() + .bucket(VERSIONED_BUCKET) + .key(versioned_key) + .body(ByteStream::from_static(version2_bytes)) + .send() + .await? + .version_id() + .ok_or("second versioned PUT omitted version ID")? + .to_string(); + let deleted = old_client + .delete_object() + .bucket(VERSIONED_BUCKET) + .key(versioned_key) + .send() + .await?; + assert_eq!(deleted.delete_marker(), Some(true)); + let delete_marker = deleted + .version_id() + .ok_or("versioned DELETE omitted delete marker version ID")? + .to_string(); + + env.restart_server_preserving_data(vec![], &server_env).await?; + let current_client = env.create_s3_client(); + + assert_eq!(read_object(¤t_client, PLAIN_BUCKET, plain_key, None).await?.1, plain_bytes); + + let (encryption, upgraded_encrypted_bytes) = read_object(¤t_client, PLAIN_BUCKET, encrypted_key, None).await?; + assert_eq!(encryption, Some(ServerSideEncryption::Aes256)); + assert_eq!(upgraded_encrypted_bytes, encrypted_bytes); + + assert_eq!(read_object(¤t_client, PLAIN_BUCKET, multipart_key, None).await?.1, multipart_bytes); + + assert_eq!( + read_object(¤t_client, VERSIONED_BUCKET, versioned_key, Some(&version1)) + .await? + .1, + version1_bytes + ); + assert_eq!( + read_object(¤t_client, VERSIONED_BUCKET, versioned_key, Some(&version2)) + .await? + .1, + version2_bytes + ); + + let current_read = current_client + .get_object() + .bucket(VERSIONED_BUCKET) + .key(versioned_key) + .send() + .await + .expect_err("the previous release's delete marker must remain current after upgrade"); + assert_eq!(current_read.raw_response().map(|response| response.status().as_u16()), Some(404)); + assert_eq!(current_read.as_service_error().and_then(ProvideErrorMetadata::code), Some("NoSuchKey")); + + let listed = current_client + .list_object_versions() + .bucket(VERSIONED_BUCKET) + .prefix(versioned_key) + .send() + .await?; + assert_eq!(listed.versions().len(), 2); + assert!( + listed + .versions() + .iter() + .any(|version| version.version_id() == Some(version1.as_str())) + ); + assert!( + listed + .versions() + .iter() + .any(|version| version.version_id() == Some(version2.as_str())) + ); + assert_eq!(listed.delete_markers().len(), 1); + assert_eq!(listed.delete_markers()[0].version_id(), Some(delete_marker.as_str())); + assert_eq!(listed.delete_markers()[0].is_latest(), Some(true)); + + let post_upgrade_key = "written-after-upgrade"; + let post_upgrade_bytes = b"written by the current RustFS build"; + current_client + .put_object() + .bucket(PLAIN_BUCKET) + .key(post_upgrade_key) + .body(ByteStream::from_static(post_upgrade_bytes)) + .send() + .await?; + assert_eq!( + read_object(¤t_client, PLAIN_BUCKET, post_upgrade_key, None).await?.1, + post_upgrade_bytes + ); + + Ok(()) +}