mirror of
https://github.com/rustfs/rustfs.git
synced 2026-11-20 06:53:44 +00:00
Compare commits
6 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 6e417a458e | |||
| 60ed7f1c40 | |||
| 7f2bd6b99b | |||
| a798df7906 | |||
| 793632c0da | |||
| 301133c1c8 |
@@ -1,2 +1,2 @@
|
||||
sha256-darwin=ab6f66c9808753082eabfb43c9e6f16508a8d065a44428284feeb5bef3fca9ba
|
||||
sha256-linux=dde1254dc9993c23d76d90b4689f1c1af9d525e16d793f4b6ab4df9c1f74f3d4
|
||||
sha256-darwin=9ccb03f4139e99ec11a63810523d504df39191b86e74b28a72dd9a075682293b
|
||||
sha256-linux=5ef052b03a03afbd00ebfcbbdcb754bbde414b7e6151aae3e2ee78024ce1d0b6
|
||||
|
||||
@@ -114,8 +114,14 @@
|
||||
}
|
||||
],
|
||||
"upgrade_sources": {
|
||||
"basis": "Exactly the two previous-release sources pinned in .github/workflows/e2e-upgrade.yml are exercised today (source.version / source.asset / source.sha256). No other N-1/N-2 compatibility is claimed, and these digests are not inferred from tags.",
|
||||
"basis": "The three previous-release sources pinned in .github/workflows/e2e-upgrade.yml are exercised today (source.version / source.asset / source.sha256). No other N-1/N-2 compatibility is claimed, and these digests are not inferred from tags.",
|
||||
"sources": [
|
||||
{
|
||||
"version": "1.0.5",
|
||||
"role": "gateway compatibility baseline",
|
||||
"artifact_name": "rustfs-linux-x86_64-gnu-v1.0.5.zip",
|
||||
"artifact_sha256": "5addae2f74756bf80104d0975e6347cda843bc2daffdfcd1f0cb80118b95cbd0"
|
||||
},
|
||||
{
|
||||
"version": "1.0.1",
|
||||
"role": "N-1",
|
||||
@@ -130,6 +136,6 @@
|
||||
}
|
||||
],
|
||||
"rollback_supported": false,
|
||||
"rollback_basis": "Only the bucket-config-rollback case (rollback_to_previous_release_reads_current_bucket_metadata) is exercised. Full data-plane rollback is not certified and is never a default claim."
|
||||
"rollback_basis": "The release upgrade lane checks bucket-config rollback. The hosted compatibility workflow also checks in-flight multipart rollback to 1.0.5. Full data-plane rollback is not certified and is never a default claim."
|
||||
}
|
||||
}
|
||||
|
||||
@@ -90,11 +90,16 @@ jobs:
|
||||
- version: 1.0.1
|
||||
asset: rustfs-linux-x86_64-gnu-v1.0.1.zip
|
||||
sha256: 9742e183cb0dfb0d48fa909797af572fe47b7f827569036baa4156e0d3958ee5
|
||||
- version: 1.0.5
|
||||
asset: rustfs-linux-x86_64-gnu-v1.0.5.zip
|
||||
sha256: 5addae2f74756bf80104d0975e6347cda843bc2daffdfcd1f0cb80118b95cbd0
|
||||
revision: 34e6449ed90e3d3a106f6655342e0b3127f33082
|
||||
env:
|
||||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
|
||||
UPGRADE_SOURCE_VERSION: ${{ matrix.source.version }}
|
||||
UPGRADE_SOURCE_ASSET: ${{ matrix.source.asset }}
|
||||
UPGRADE_SOURCE_SHA256: ${{ matrix.source.sha256 }}
|
||||
UPGRADE_SOURCE_REVISION: ${{ matrix.source.revision }}
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
@@ -132,6 +137,13 @@ jobs:
|
||||
unzip -q "$archive" -d "$SOURCE_DIR"
|
||||
chmod +x "$SOURCE_DIR/rustfs"
|
||||
test -x "$SOURCE_DIR/rustfs"
|
||||
if [[ -n "$UPGRADE_SOURCE_REVISION" ]]; then
|
||||
mkdir -p "$RUNNER_TEMP/rustfs-upgrade-logs"
|
||||
version_log="$RUNNER_TEMP/rustfs-upgrade-logs/source-version.txt"
|
||||
"$SOURCE_DIR/rustfs" --version | tee "$version_log"
|
||||
grep -Fx "rustfs $UPGRADE_SOURCE_VERSION" "$version_log"
|
||||
grep -Eq "^git commit[[:space:]]*:[[:space:]]*$UPGRADE_SOURCE_REVISION$" "$version_log"
|
||||
fi
|
||||
echo "RUSTFS_UPGRADE_SOURCE_BINARY=$SOURCE_DIR/rustfs" >> "$GITHUB_ENV"
|
||||
echo "RUSTFS_E2E_LOG_DIR=$RUNNER_TEMP/rustfs-upgrade-logs" >> "$GITHUB_ENV"
|
||||
|
||||
@@ -151,12 +163,14 @@ jobs:
|
||||
multipart-layout-upgrade:direct_upgrade_from_previous_release_preserves_multipart_layouts
|
||||
multipart-layout-baseline:previous_release_baseline_replicates_multipart_layouts
|
||||
)
|
||||
if [[ "$UPGRADE_SOURCE_VERSION" == "1.0.0-rc.5" ]]; then
|
||||
cases+=(odm-config-rollback:rc5_rollback_requires_restoring_odm_configuration)
|
||||
cases+=(checkpoint-upgrade:rc5_checkpoint_upgrade_rewinds_signed_progress_without_losing_objects)
|
||||
else
|
||||
cases+=(legacy-delete-reservation:stable_upgrade_deletes_foreign_reservations_without_losing_live_objects)
|
||||
fi
|
||||
case "$UPGRADE_SOURCE_VERSION" in
|
||||
1.0.0-rc.5)
|
||||
cases+=(odm-config-rollback:rc5_rollback_requires_restoring_odm_configuration)
|
||||
cases+=(checkpoint-upgrade:rc5_checkpoint_upgrade_rewinds_signed_progress_without_losing_objects) ;;
|
||||
1.0.1) cases+=(legacy-delete-reservation:stable_upgrade_deletes_foreign_reservations_without_losing_live_objects) ;;
|
||||
1.0.5) ;; # The bucket rollback case also completes an in-flight multipart upload.
|
||||
*) echo "::error::Unknown upgrade source $UPGRADE_SOURCE_VERSION"; exit 1 ;;
|
||||
esac
|
||||
status=0
|
||||
for entry in "${cases[@]}"; do
|
||||
artifact="${entry%%:*}"
|
||||
@@ -178,7 +192,7 @@ jobs:
|
||||
exit "$status"
|
||||
|
||||
- name: Run mixed-version MPU recovery
|
||||
if: ${{ !cancelled() && steps.verify_source.outcome == 'success' }}
|
||||
if: ${{ !cancelled() && steps.verify_source.outcome == 'success' && matrix.source.version != '1.0.5' }}
|
||||
timeout-minutes: 15
|
||||
env:
|
||||
NO_PROXY: 127.0.0.1,localhost,::1
|
||||
@@ -199,6 +213,24 @@ jobs:
|
||||
"$mpu_test" -- --exact --nocapture --test-threads=1 \
|
||||
2>&1 | tee "$RUSTFS_E2E_LOG_DIR/test.log"
|
||||
|
||||
- name: Verify gateway baseline G09 evidence
|
||||
if: ${{ success() && matrix.source.version == '1.0.5' }}
|
||||
shell: bash
|
||||
run: |
|
||||
for field in mixed_version_reader_evidence mixed_version_writer_evidence rollback_payload_evidence; do
|
||||
directory=mixed-version-upgrade
|
||||
if [[ "$field" == rollback_payload_evidence ]]; then directory=bucket-config-rollback; fi
|
||||
jq -e --arg field "$field" --arg current "$GITHUB_SHA" --arg previous "$UPGRADE_SOURCE_REVISION" \
|
||||
'.gate == "G09" and .field == $field and .current_revision == $current and .previous_revision == $previous
|
||||
and ($field != "rollback_payload_evidence" or .rollback_payload_replayed == true)' \
|
||||
"$RUNNER_TEMP/rustfs-upgrade-g09-evidence/$directory/G09-$field.json"
|
||||
done
|
||||
jq -e --arg current "$GITHUB_SHA" --arg previous "$UPGRADE_SOURCE_REVISION" \
|
||||
'.current_revision == $current and .previous_revision == $previous
|
||||
and .part_sizes == [5242880, 5242880, 4096] and .bytes == (.part_sizes | add)
|
||||
and (.rollback_read_sha256 | test("^[a-f0-9]{64}$")) and .rollback_read_sha256 == .roll_forward_read_sha256' \
|
||||
"$RUNNER_TEMP/rustfs-upgrade-g09-evidence/bucket-config-rollback/inflight-multipart-rollback.json"
|
||||
|
||||
- name: Upload scanner/heal G09 evidence
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
|
||||
+1
-1
@@ -13,7 +13,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
|
||||
### Deployment
|
||||
|
||||
- **The S3 protocol stack is now rustfs/gateway**, and it is the only S3 stack. The legacy s3s-based stack is removed, and so are the `RUSTFS_S3_STACK` environment variable and the `gateway-stack` cargo feature. There is no on-disk format change. `RUSTFS_S3_STACK` no longer selects anything: a server started with it set, to any value, refuses to start and prints `RUSTFS_S3_STACK is no longer supported; the gateway stack is the only S3 stack since <version>`, where `<version>` is the release's version. Remove the variable from service units, container environments, Helm values and orchestration files before upgrading. To roll back, run the previous release binary on the same data.
|
||||
- **The S3 protocol stack is now rustfs/gateway**, and it is the only S3 stack. The legacy s3s-based stack is removed, and so are the `RUSTFS_S3_STACK` environment variable and the `gateway-stack` cargo feature. There is no on-disk format change. `RUSTFS_S3_STACK` no longer selects anything: a server started with it set, to any value, refuses to start and prints `RUSTFS_S3_STACK is no longer supported; the gateway stack is the only S3 stack since <version>`, where `<version>` is the release's version. Remove the variable from service units, container environments, Helm values and orchestration files before upgrading. To roll back, follow the [S3 stack rollback procedure](docs/operations/s3-stack-rollback.md).
|
||||
|
||||
- Before upgrading, verify that the configured root access key is at least 3 bytes and the root secret key is at least 8 bytes. RustFS now rejects shorter values at startup and identifies the setting to update. Version 1.0.5 accepted these settings at startup and returned HTTP 500 when root requested STS credentials. Coordinate root credential changes across all nodes and clients. See [root credential rotation considerations](docs/operations/two-factor-auth.md#root-credentials-cannot-be-changed-at-runtime).
|
||||
|
||||
|
||||
@@ -31,7 +31,7 @@ use super::harness::{
|
||||
use crate::common::{
|
||||
AdminTransport, admin_add_canned_policy_via, admin_attach_user_policy_via, admin_create_user_via, init_logging,
|
||||
};
|
||||
use crate::upgrade_compatibility_test::wait_for_upgrade_write_readiness;
|
||||
use crate::upgrade_compatibility_test::{UpgradeSourceCapabilities, wait_for_upgrade_write_readiness};
|
||||
use aws_sdk_s3::Client;
|
||||
use aws_sdk_s3::error::ProvideErrorMetadata;
|
||||
use std::path::{Path, PathBuf};
|
||||
@@ -427,7 +427,12 @@ async fn assert_history_and_iam(dist: &DistCluster, seed: &UpgradeSeed, context:
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn compare_degraded_legacy_list_parts(dist: &mut DistCluster, seed: &UpgradeSeed, previous: &Path) -> TestResult {
|
||||
async fn compare_degraded_legacy_list_parts(
|
||||
dist: &mut DistCluster,
|
||||
seed: &UpgradeSeed,
|
||||
previous: &Path,
|
||||
source: &UpgradeSourceCapabilities,
|
||||
) -> TestResult {
|
||||
let decoded = String::from_utf8(base64_simd::URL_SAFE_NO_PAD.decode_to_vec(seed.multipart_upload.as_bytes())?)?;
|
||||
let (_, upload_dir) = decoded.split_once('.').ok_or("legacy upload ID lacks deployment delimiter")?;
|
||||
let hash = sha256_hex(format!("{}/legacy-multipart.bin", seed.history_bucket).as_bytes());
|
||||
@@ -489,16 +494,22 @@ async fn compare_degraded_legacy_list_parts(dist: &mut DistCluster, seed: &Upgra
|
||||
// Three online disks retain metadata quorum but have no part directory;
|
||||
// the fourth disk is offline. A legacy empty response is not a proof that
|
||||
// the unavailable participant has no part.
|
||||
let legacy = legacy.expect("previous coordinator should expose its legacy empty-list behavior");
|
||||
assert!(legacy.parts().is_empty());
|
||||
let current = current.expect_err("current coordinator must not infer empty from missing directories and offline disks");
|
||||
assert_eq!(current.raw_response().expect("current S3 response").status().as_u16(), 503);
|
||||
// Since #9149, SlowDownRead is reserved for GET/HEAD object reads; a
|
||||
// ListParts read-quorum failure is reported as ServiceUnavailable.
|
||||
assert_eq!(
|
||||
current.as_service_error().and_then(|error| error.meta().code()),
|
||||
Some("ServiceUnavailable")
|
||||
);
|
||||
let mut quorum_errors =
|
||||
vec![current.expect_err("current coordinator must not infer empty from missing directories and offline disks")];
|
||||
if source.requires_list_parts_read_quorum {
|
||||
quorum_errors.push(legacy.expect_err("previous coordinator must require evidence from unavailable disks"));
|
||||
} else {
|
||||
assert!(
|
||||
legacy
|
||||
.expect("previous coordinator should expose its legacy empty-list behavior")
|
||||
.parts()
|
||||
.is_empty()
|
||||
);
|
||||
}
|
||||
for error in quorum_errors {
|
||||
assert_eq!(error.raw_response().expect("S3 response").status().as_u16(), 503);
|
||||
assert_eq!(error.as_service_error().and_then(|error| error.meta().code()), Some("ServiceUnavailable"));
|
||||
}
|
||||
dist.cluster.start_node_from_binary(3, previous).await?;
|
||||
wait_for_upgrade_write_readiness(&clients, &seed.history_bucket, "mixed-list-restored", CREDENTIAL_TIMEOUT).await?;
|
||||
for (path, before) in original_files {
|
||||
@@ -520,7 +531,7 @@ async fn compare_degraded_legacy_list_parts(dist: &mut DistCluster, seed: &Upgra
|
||||
assert_eq!(listed.parts()[0].e_tag(), Some(seed.multipart_etag.as_str()));
|
||||
}
|
||||
eprintln!(
|
||||
"MPU mixed degraded: one offline disk, three missing part directories; previous returned empty, current returned 503 SlowDownRead; original part restored on both"
|
||||
"MPU mixed degraded: one offline disk, three missing part directories; checked each coordinator against its release contract; original part restored on both"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
@@ -592,6 +603,7 @@ async fn replace_node_and_wait_for_write_readiness(dist: &mut DistCluster, seed:
|
||||
async fn four_node_rolling_upgrade_preserves_history_and_iam_credentials() -> TestResult {
|
||||
init_logging();
|
||||
let previous = source_binary()?;
|
||||
let source = UpgradeSourceCapabilities::from_binary(&previous).await?;
|
||||
let mut dist = DistCluster::new_stopped(DistLayout::FourNodeFourDisk).await?;
|
||||
dist.cluster.set_env("RUSTFS_STORAGE_CLASS_RRS", "EC:0");
|
||||
capture_upgrade_logs(&mut dist, "rolling-upgrade")?;
|
||||
@@ -601,7 +613,7 @@ async fn four_node_rolling_upgrade_preserves_history_and_iam_credentials() -> Te
|
||||
|
||||
replace_node_and_wait_for_write_readiness(&mut dist, &seed, 0).await?;
|
||||
assert_history_and_iam(&dist, &seed, "one-current-node").await?;
|
||||
compare_degraded_legacy_list_parts(&mut dist, &seed, &previous).await?;
|
||||
compare_degraded_legacy_list_parts(&mut dist, &seed, &previous, &source).await?;
|
||||
|
||||
for node_idx in [1, 2] {
|
||||
replace_node_and_wait_for_write_readiness(&mut dist, &seed, node_idx).await?;
|
||||
|
||||
@@ -46,9 +46,12 @@ type BoxError = Box<dyn std::error::Error + Send + Sync>;
|
||||
|
||||
const SOURCE_BINARY_ENV: &str = "RUSTFS_UPGRADE_SOURCE_BINARY";
|
||||
const G09_EVIDENCE_DIR_ENV: &str = "RUSTFS_SCANNER_HEAL_G09_EVIDENCE_DIR";
|
||||
const RC2_COMMIT: &str = "e9f53180273b32a9c2b0aa8b5a384047bc7e875c";
|
||||
const RC5_COMMIT: &str = "40a2470feb567201165a5b809b7598bb4b1f68f5";
|
||||
const STABLE_1_0_1_COMMIT: &str = "6de965ae3c965a78ff819fbcd7acd4aa44177d92";
|
||||
const STABLE_1_0_4_COMMIT: &str = "46337942d55f3a8e6a97b147ca98d9ea8115555d";
|
||||
const STABLE_1_0_5_COMMIT: &str = "34e6449ed90e3d3a106f6655342e0b3127f33082";
|
||||
const STABLE_1_0_6_COMMIT: &str = "de9009d882644effc2628e777650025c3d8e7c65";
|
||||
const SSE_MASTER_KEY_ENV: &str = "RUSTFS_SSE_S3_MASTER_KEY";
|
||||
const SSE_MASTER_KEY: &str = "QkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkI=";
|
||||
const PLAIN_BUCKET: &str = "upgrade-plain-data";
|
||||
@@ -142,6 +145,41 @@ fn current_source_revision() -> Result<String, BoxError> {
|
||||
Ok(revision.to_string())
|
||||
}
|
||||
|
||||
pub(crate) struct UpgradeSourceCapabilities {
|
||||
supports_delete_recovery: bool,
|
||||
supports_multipart_publication: bool,
|
||||
pub(crate) requires_list_parts_read_quorum: bool,
|
||||
}
|
||||
|
||||
impl UpgradeSourceCapabilities {
|
||||
fn from_revision(revision: &str) -> Result<Self, BoxError> {
|
||||
// 1.0.4 supports the delete decision protocol (#8468). Version 1.0.5 also
|
||||
// shipped native MPU publication (#8919) and strict ListParts evidence (#9037, #9149).
|
||||
match revision {
|
||||
RC2_COMMIT | RC5_COMMIT | STABLE_1_0_1_COMMIT => Ok(Self {
|
||||
supports_delete_recovery: false,
|
||||
supports_multipart_publication: false,
|
||||
requires_list_parts_read_quorum: false,
|
||||
}),
|
||||
STABLE_1_0_4_COMMIT => Ok(Self {
|
||||
supports_delete_recovery: true,
|
||||
supports_multipart_publication: false,
|
||||
requires_list_parts_read_quorum: false,
|
||||
}),
|
||||
STABLE_1_0_5_COMMIT | STABLE_1_0_6_COMMIT => Ok(Self {
|
||||
supports_delete_recovery: true,
|
||||
supports_multipart_publication: true,
|
||||
requires_list_parts_read_quorum: true,
|
||||
}),
|
||||
_ => Err(format!("no pinned upgrade expectations for source revision {revision}").into()),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn from_binary(binary: &Path) -> Result<Self, BoxError> {
|
||||
Self::from_revision(&binary_source_revision(binary).await?)
|
||||
}
|
||||
}
|
||||
|
||||
async fn binary_source_revision(binary: &Path) -> Result<String, BoxError> {
|
||||
let output = tokio::process::Command::new(binary).arg("--version").output().await?;
|
||||
if !output.status.success() {
|
||||
@@ -225,6 +263,25 @@ mod g09_evidence_tests {
|
||||
use super::*;
|
||||
use tempfile::TempDir;
|
||||
|
||||
#[test]
|
||||
fn pinned_upgrade_sources_keep_distinct_delete_and_list_parts_contracts() -> TestResult {
|
||||
for (revision, delete_recovery, multipart_publication, list_parts_quorum) in [
|
||||
(RC2_COMMIT, false, false, false),
|
||||
(RC5_COMMIT, false, false, false),
|
||||
(STABLE_1_0_1_COMMIT, false, false, false),
|
||||
(STABLE_1_0_4_COMMIT, true, false, false),
|
||||
(STABLE_1_0_5_COMMIT, true, true, true),
|
||||
(STABLE_1_0_6_COMMIT, true, true, true),
|
||||
] {
|
||||
let source = UpgradeSourceCapabilities::from_revision(revision)?;
|
||||
assert_eq!(source.supports_delete_recovery, delete_recovery, "{revision}");
|
||||
assert_eq!(source.supports_multipart_publication, multipart_publication, "{revision}");
|
||||
assert_eq!(source.requires_list_parts_read_quorum, list_parts_quorum, "{revision}");
|
||||
}
|
||||
assert!(UpgradeSourceCapabilities::from_revision("ffffffffffffffffffffffffffffffffffffffff").is_err());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn source_revision_parser_requires_lowercase_forty_byte_sha() -> TestResult {
|
||||
let binary = Path::new("rustfs");
|
||||
@@ -1011,8 +1068,7 @@ async fn exercise_mixed_cluster(
|
||||
vec![u8::try_from(MULTIPART_WORKERS - 1)?; 64 * 1024]
|
||||
);
|
||||
|
||||
// 1.0.4 already implements the delete decision protocol. Use a separate
|
||||
// old-writer key so the original seed still spans the entire rollout.
|
||||
// Use a separate old-writer key so the original seed still spans the entire rollout.
|
||||
if previous_has_delete_recovery {
|
||||
let key = format!(".upgrade-delete/{phase}/previous-writer");
|
||||
previous_client
|
||||
@@ -2293,7 +2349,7 @@ async fn rolling_upgrade_from_previous_release_preserves_mixed_version_contracts
|
||||
let previous_binary = source_binary()?;
|
||||
// Bind the expected protocol to the verified release source, never to the
|
||||
// DELETE result itself. The older CI source pins retain their 503 contract.
|
||||
let previous_has_delete_recovery = binary_source_revision(&previous_binary).await? == STABLE_1_0_4_COMMIT;
|
||||
let source = UpgradeSourceCapabilities::from_binary(&previous_binary).await?;
|
||||
let evidence_context = g09_evidence_context(&previous_binary).await?;
|
||||
let current_binary = rustfs_binary_path();
|
||||
let mut cluster = RustFSTestClusterEnvironment::new(MIXED_NODE_COUNT).await?;
|
||||
@@ -2315,17 +2371,21 @@ async fn rolling_upgrade_from_previous_release_preserves_mixed_version_contracts
|
||||
cluster.stop_node(0)?;
|
||||
cluster.start_node_from_binary(0, ¤t_binary).await?;
|
||||
let mut mixed_deleted_keys = Vec::new();
|
||||
if let Some(key) = exercise_mixed_cluster(&cluster, "one-current-node", 0, 1, previous_has_delete_recovery).await? {
|
||||
if let Some(key) = exercise_mixed_cluster(&cluster, "one-current-node", 0, 1, source.supports_delete_recovery).await? {
|
||||
mixed_deleted_keys.push(key);
|
||||
}
|
||||
// One controlled out-of-policy attempt proves fail-closed behavior below
|
||||
// native write quorum; positive MPU writers remain paused while mixed.
|
||||
assert_mixed_pending_refusal(&cluster, &pending, &sources).await?;
|
||||
let mixed_completion = if source.supports_multipart_publication {
|
||||
let clients = cluster.create_all_clients()?;
|
||||
Some(finish_pending_multipart(&clients, &pending, pending_parts.concat()).await?)
|
||||
} else {
|
||||
assert_mixed_pending_refusal(&cluster, &pending, &sources).await?;
|
||||
None
|
||||
};
|
||||
|
||||
for (node_idx, phase) in [(1, "two-current-nodes"), (2, "one-previous-node")] {
|
||||
cluster.stop_node(node_idx)?;
|
||||
cluster.start_node_from_binary(node_idx, ¤t_binary).await?;
|
||||
if let Some(key) = exercise_mixed_cluster(&cluster, phase, 0, 3, previous_has_delete_recovery).await? {
|
||||
if let Some(key) = exercise_mixed_cluster(&cluster, phase, 0, 3, source.supports_delete_recovery).await? {
|
||||
mixed_deleted_keys.push(key);
|
||||
}
|
||||
}
|
||||
@@ -2335,6 +2395,13 @@ async fn rolling_upgrade_from_previous_release_preserves_mixed_version_contracts
|
||||
let clients = cluster.create_all_clients()?;
|
||||
wait_for_drive_readmission(&clients, MIXED_BUCKET, "homogeneous-current", DRIVE_READMISSION_TIMEOUT).await?;
|
||||
let completed_pending = finish_pending_multipart(&clients, &pending, pending_parts.concat()).await?;
|
||||
if let Some(mixed) = mixed_completion {
|
||||
assert_eq!(
|
||||
completed_pending.version_id, mixed.version_id,
|
||||
"full upgrade changed the completed version"
|
||||
);
|
||||
assert_eq!(completed_pending.etag, mixed.etag, "full upgrade changed the completed ETag");
|
||||
}
|
||||
exercise_homogeneous_multipart(&cluster, "homogeneous-current").await?;
|
||||
verify_load_bodies(&clients, "homogeneous-previous", "multipart").await?;
|
||||
assert_mixed_deletes_visible(&clients, &mixed_deleted_keys).await?;
|
||||
@@ -2451,7 +2518,7 @@ async fn rolling_upgrade_from_previous_release_preserves_mixed_version_contracts
|
||||
"phases": phases,
|
||||
"objects_per_phase": objects_per_phase,
|
||||
"mixed_load": "plain-put",
|
||||
"mixed_delete_protocol_supported": previous_has_delete_recovery,
|
||||
"mixed_delete_protocol_supported": source.supports_delete_recovery,
|
||||
"mixed_deleted_keys": mixed_deleted_keys,
|
||||
"homogeneous_multipart_phases": ["homogeneous-previous", "homogeneous-current", "homogeneous-rollback"],
|
||||
"multipart_objects_per_homogeneous_phase": MULTIPART_WORKERS * MULTIPART_UPLOADS_PER_WORKER,
|
||||
@@ -2461,7 +2528,7 @@ async fn rolling_upgrade_from_previous_release_preserves_mixed_version_contracts
|
||||
"all nodes list every mixed-version object after homogeneous-current convergence",
|
||||
"the previous-release seed survives the baseline restart and every upgrade phase",
|
||||
"previous readers read every current-native MPU body after quiescent rollback",
|
||||
"same legacy upload resumes with identical body, ETag and version after full upgrade"
|
||||
"the same previous-release upload remains readable after full upgrade and rollback"
|
||||
],
|
||||
}),
|
||||
)?;
|
||||
@@ -2483,9 +2550,13 @@ async fn rolling_upgrade_from_previous_release_preserves_mixed_version_contracts
|
||||
"previous-release writer publishes objects readable by current node",
|
||||
"mixed phases retain 16x16 plain PUT writers while positive multipart writers are paused",
|
||||
"16x16 multipart load runs in homogeneous previous, current and rollback phases",
|
||||
"one-current-node Complete returns typed 503 and preserves actual source authority and parts",
|
||||
"the same legacy upload completes and replays once fully upgraded without adding a version",
|
||||
if previous_has_delete_recovery {
|
||||
if source.supports_multipart_publication {
|
||||
"one-current-node Complete and replay return the same version and ETag, and all readers agree on the body"
|
||||
} else {
|
||||
"one-current-node Complete returns typed 503 and preserves actual source authority and parts"
|
||||
},
|
||||
"fully upgraded readers see the same body, ETag and sole version after replay",
|
||||
if source.supports_delete_recovery {
|
||||
"mixed-version DELETE succeeds and stays absent through every reader after upgrade and rollback"
|
||||
} else {
|
||||
"modern DELETE fails closed and preserves every byte while old peers remain"
|
||||
@@ -3386,6 +3457,8 @@ async fn direct_upgrade_from_previous_release_preserves_bucket_configuration() -
|
||||
#[tokio::test]
|
||||
#[ignore = "requires a pinned previous RustFS release binary"]
|
||||
async fn rollback_to_previous_release_reads_current_bucket_metadata() -> TestResult {
|
||||
use sha2::Digest;
|
||||
|
||||
init_logging();
|
||||
let previous_binary = source_binary()?;
|
||||
let evidence_context = g09_evidence_context(&previous_binary).await?;
|
||||
@@ -3423,6 +3496,16 @@ async fn rollback_to_previous_release_reads_current_bucket_metadata() -> TestRes
|
||||
let multipart_bytes = multipart_parts.concat();
|
||||
write_multipart(&new_client, ROLLBACK_BUCKET, multipart_key, &multipart_parts).await?;
|
||||
|
||||
// This release can finish an upload staged by the current gateway.
|
||||
let inflight = if binary_source_revision(&previous_binary).await? == STABLE_1_0_5_COMMIT {
|
||||
env.create_test_bucket(PENDING_MULTIPART_BUCKET).await?;
|
||||
let parts = [vec![1; LAYOUT_PART_SIZE], vec![2; LAYOUT_PART_SIZE], vec![3; 4096]];
|
||||
let pending = stage_multipart(&new_client, PENDING_MULTIPART_BUCKET, PENDING_MULTIPART_KEY, &parts).await?;
|
||||
Some((pending, parts.concat()))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
restart_from_binary(&mut env, &previous_binary, &server_env).await?;
|
||||
let old_client = env.create_s3_client();
|
||||
|
||||
@@ -3438,6 +3521,16 @@ async fn rollback_to_previous_release_reads_current_bucket_metadata() -> TestRes
|
||||
assert_eq!(encryption, Some(ServerSideEncryption::Aes256));
|
||||
assert_eq!(body, multipart_bytes, "the multipart object did not survive the rollback");
|
||||
|
||||
let rollback_multipart = if let Some((pending, expected)) = inflight {
|
||||
assert_pending_parts(&old_client, &pending).await?;
|
||||
complete_pending_multipart(&old_client, PENDING_MULTIPART_BUCKET, PENDING_MULTIPART_KEY, &pending).await?;
|
||||
let (_, body) = read_object(&old_client, PENDING_MULTIPART_BUCKET, PENDING_MULTIPART_KEY, None).await?;
|
||||
assert_eq!(body, expected);
|
||||
Some((pending.part_sizes, body))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
// A PUT on the rolled-back release must still honour the encryption
|
||||
// configuration it decoded out of the current build's metadata blob.
|
||||
let post_rollback_key = "rollback/written-after-rollback";
|
||||
@@ -3478,6 +3571,25 @@ async fn rollback_to_previous_release_reads_current_bucket_metadata() -> TestRes
|
||||
post_rollback_bytes
|
||||
);
|
||||
|
||||
if let Some((part_sizes, rollback_read)) = rollback_multipart {
|
||||
let (_, current_read) = read_object(¤t_again, PENDING_MULTIPART_BUCKET, PENDING_MULTIPART_KEY, None).await?;
|
||||
assert_eq!(current_read, rollback_read);
|
||||
if let Some(context) = &evidence_context {
|
||||
let evidence = serde_json::json!({
|
||||
"previous_revision": context.previous_revision,
|
||||
"current_revision": context.current_revision,
|
||||
"part_sizes": part_sizes,
|
||||
"bytes": current_read.len(),
|
||||
"rollback_read_sha256": format!("{:x}", sha2::Sha256::digest(&rollback_read)),
|
||||
"roll_forward_read_sha256": format!("{:x}", sha2::Sha256::digest(¤t_read)),
|
||||
});
|
||||
std::fs::write(
|
||||
context.directory.join("inflight-multipart-rollback.json"),
|
||||
serde_json::to_vec_pretty(&evidence)?,
|
||||
)?;
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(context) = evidence_context.as_ref() {
|
||||
write_g09_evidence(
|
||||
context,
|
||||
|
||||
@@ -0,0 +1,17 @@
|
||||
# S3 stack upgrade and rollback
|
||||
|
||||
The gateway release serves S3 with rustfs/gateway. Remove `RUSTFS_S3_STACK` from service units, container environments and Helm values before upgrading. Even an empty setting prevents startup. Keep the previous release binary or image digest with the deployment configuration.
|
||||
|
||||
## Roll back to the previous release
|
||||
|
||||
1. Check the target release against the [storage format version floors](rolling-restart.md#version-floors-that-break-mixed-version-fleets). If the cluster has written data or metadata that the target cannot read, stop here. The S3 stack switch itself requires no data migration.
|
||||
2. Stop one node, restore the previous RustFS binary or image and start it with the same volumes and configuration. Follow the [rolling restart procedure](rolling-restart.md#rolling-restart-procedure). Wait for the node to report ready and for its peers to report its drives online before replacing another node.
|
||||
3. Through the restored node, read an object written after the upgrade and compare its contents or checksum. Read a bucket configuration used by the application. If an upload was in progress, list its parts and compare their numbers, sizes and ETags with the client's record. Pause the rollout if any check fails.
|
||||
|
||||
## Client-visible error change
|
||||
|
||||
| Conformance case | What clients see |
|
||||
| --- | --- |
|
||||
| `c-sig-0598` (R-053) | For an access key the S3 parser cannot read, both versions return HTTP 403. The gateway returns the XML error code `AccessDenied`. The legacy binary returns `InvalidAccessKeyId`. Clients that classify these errors by code need to handle both during a rollout. |
|
||||
|
||||
The case is recorded in the [pinned gateway conformance rulings](https://github.com/rustfs/gateway/blob/60dcd2a55713dbcc9dbd6e165922e1a0df4473d7/conformance/rulings/rustfs-profile.toml#L298-L304).
|
||||
@@ -1169,18 +1169,7 @@ impl rustfs_protocols::common::client::s3::StorageBackend for ProtocolStorageCli
|
||||
|
||||
let bucket = input.bucket.as_str().to_owned();
|
||||
let key = input.key.as_str().to_owned();
|
||||
let upload_id = gateway_s3s_compat::leaf::upload_id_to_s3s(&input.upload_id);
|
||||
let input = multipart::upload_part_copy_input(input)?;
|
||||
let part_number = input.part_number;
|
||||
let part_number = part_number.to_string();
|
||||
let uri = build_object_uri(
|
||||
&bucket,
|
||||
&key,
|
||||
&[
|
||||
("partNumber", Some(part_number.as_str())),
|
||||
("uploadId", Some(upload_id.as_str())),
|
||||
],
|
||||
)?;
|
||||
let (uri, input) = multipart::prepare_upload_part_copy(input)?;
|
||||
|
||||
let envelope = Self::create_envelope(
|
||||
Method::PUT,
|
||||
@@ -1517,6 +1506,7 @@ mod tests {
|
||||
};
|
||||
use http_body_util::BodyExt;
|
||||
use rustfs_protocols::common::client::s3::StorageBackend;
|
||||
use rustfs_s3_types::S3ErrorCode;
|
||||
let root = rustfs_credentials::Credentials {
|
||||
access_key: "protocol-oracle-root".into(),
|
||||
secret_key: "protocol-oracle-secret".into(),
|
||||
@@ -1528,9 +1518,16 @@ mod tests {
|
||||
let slot = ServerContextSlot::new();
|
||||
assert!(slot.install(context));
|
||||
let backend = ProtocolStorageClient::new(slot);
|
||||
for (bucket, key, data, versioned) in [
|
||||
("protocol-version-oracle", "versioned-key", b"versioned data".as_slice(), true),
|
||||
("protocol-marker-oracle", "folder/", b"".as_slice(), false),
|
||||
for (bucket, key, data, versioned, deleted_get_code) in [
|
||||
(
|
||||
"protocol-version-oracle",
|
||||
"versioned-key",
|
||||
b"versioned data".as_slice(),
|
||||
true,
|
||||
S3ErrorCode::NoSuchKey,
|
||||
),
|
||||
// RustFS 1.0.5 returns NoSuchVersion for a missing directory marker.
|
||||
("protocol-marker-oracle", "folder/", b"".as_slice(), false, S3ErrorCode::NoSuchVersion),
|
||||
] {
|
||||
store
|
||||
.make_bucket(
|
||||
@@ -1593,7 +1590,7 @@ mod tests {
|
||||
.get_object(bucket, key, &root, None)
|
||||
.await
|
||||
.expect_err("protocol GET deleted object");
|
||||
assert_eq!(current.code(), &rustfs_s3_types::S3ErrorCode::NoSuchKey);
|
||||
assert_eq!(current.code(), &deleted_get_code, "GET {bucket}/{key} after DELETE");
|
||||
assert_eq!(current.status_code(), Some(http::StatusCode::NOT_FOUND));
|
||||
if versioned {
|
||||
let headers = current.headers().expect("delete marker error headers");
|
||||
|
||||
@@ -293,9 +293,9 @@ pub(super) fn upload_part_error(error: S3Error) -> S3Error {
|
||||
error
|
||||
}
|
||||
|
||||
pub(super) fn upload_part_copy_input(
|
||||
pub(super) fn prepare_upload_part_copy(
|
||||
input: dto::UploadPartCopyInput,
|
||||
) -> S3Result<crate::app::multipart_usecase::UploadPartCopyWriteInput> {
|
||||
) -> S3Result<(http::Uri, crate::app::multipart_usecase::UploadPartCopyWriteInput)> {
|
||||
super::owned::validate_copy_source(&input.copy_source, "UploadPartCopy copy_source")?;
|
||||
let copy_source_if_match =
|
||||
super::owned::copy_condition(input.copy_source_if_match.as_deref(), "copy_source_if_match", "UploadPartCopy input")?;
|
||||
@@ -314,11 +314,18 @@ pub(super) fn upload_part_copy_input(
|
||||
.copy_source_if_unmodified_since
|
||||
.map(|at| multipart_timestamp(at, "UploadPartCopy input", "copy_source_if_unmodified_since"))
|
||||
.transpose()?;
|
||||
Ok(crate::app::multipart_usecase::UploadPartCopyWriteInput {
|
||||
let upload_id = super::gateway_s3s_compat::leaf::upload_id_to_s3s(&input.upload_id);
|
||||
let part_number = input.part_number.to_string();
|
||||
let uri = super::build_object_uri(
|
||||
input.bucket.as_str(),
|
||||
input.key.as_str(),
|
||||
&[("partNumber", Some(&part_number)), ("uploadId", Some(&upload_id))],
|
||||
)?;
|
||||
let write = crate::app::multipart_usecase::UploadPartCopyWriteInput {
|
||||
target: crate::app::multipart_usecase::UploadPartTarget::Legacy {
|
||||
bucket: input.bucket.as_str().to_owned(),
|
||||
key: input.key.as_str().to_owned(),
|
||||
upload_id: super::gateway_s3s_compat::leaf::upload_id_to_s3s(&input.upload_id),
|
||||
upload_id,
|
||||
},
|
||||
part_number: input.part_number,
|
||||
copy_source: input.copy_source,
|
||||
@@ -333,7 +340,8 @@ pub(super) fn upload_part_copy_input(
|
||||
.copy_source_sse_customer_key
|
||||
.map(|value| value.expose_secret().to_owned()),
|
||||
copy_source_sse_customer_key_md5: nonempty(input.copy_source_sse_customer_key_md5),
|
||||
})
|
||||
};
|
||||
Ok((uri, write))
|
||||
}
|
||||
|
||||
pub(super) fn upload_part_copy_output(
|
||||
|
||||
@@ -422,9 +422,10 @@ fn protocol_upload_copy_keeps_destination_generation_fence() {
|
||||
)
|
||||
.await
|
||||
.expect("replacement bytes");
|
||||
let (_, input) = multipart::prepare_upload_part_copy(request).expect("owned input");
|
||||
let result = backend
|
||||
.owned_multipart_usecase()
|
||||
.execute_upload_part_copy_with_audit(envelope, multipart::upload_part_copy_input(request).expect("owned input"))
|
||||
.execute_upload_part_copy_with_audit(envelope, input)
|
||||
.await;
|
||||
let mut reader = store
|
||||
.get_object_reader(destination, "copy-target", None, HeaderMap::new(), &options)
|
||||
@@ -509,10 +510,12 @@ fn protocol_upload_copy_input_matches_original_codec() {
|
||||
gateway_s3s_compat::generated::ops::upload_part_copy::input_to_s3s(make(), source)
|
||||
.map_err(|error| conversion_error("UploadPartCopy input", error))
|
||||
});
|
||||
let actual = multipart::upload_part_copy_input(make());
|
||||
let actual = multipart::prepare_upload_part_copy(make());
|
||||
match (actual, expected) {
|
||||
(Err(actual), Err(expected)) => assert_eq!(error_facts(actual), error_facts(expected)),
|
||||
(Ok(actual), Ok(expected)) => {
|
||||
(Ok((uri, actual)), Ok(expected)) => {
|
||||
assert_eq!(uri.path(), "/bucket/copy-target");
|
||||
assert_eq!(uri.query(), Some("partNumber=1&uploadId=upload%20%3F%26"));
|
||||
let crate::app::multipart_usecase::UploadPartTarget::Legacy { bucket, key, upload_id } = actual.target else {
|
||||
panic!("protocol client must use the legacy upload target")
|
||||
};
|
||||
@@ -726,7 +729,8 @@ async fn protocol_upload_copy_capabilities_precede_part_validation() {
|
||||
.err()
|
||||
.map(S3Error::from)
|
||||
.expect("legacy refusal");
|
||||
let actual = owned_copy_future(&usecase, envelope, multipart::upload_part_copy_input(make()).expect("owned input"))
|
||||
let (_, input) = multipart::prepare_upload_part_copy(make()).expect("owned input");
|
||||
let actual = owned_copy_future(&usecase, envelope, input)
|
||||
.await
|
||||
.err()
|
||||
.expect("owned refusal");
|
||||
|
||||
@@ -766,13 +766,9 @@ fn the_gateway_stack_swaps_the_tagging_action_for_a_named_version() {
|
||||
with_stack(GATEWAY, the_tagging_action_follows_the_named_version);
|
||||
}
|
||||
|
||||
/// What a stack answers for a key outside the floor, independent of the operation and bucket.
|
||||
type UniformAnswers = (Answer, Answer);
|
||||
|
||||
/// Checks boundary keys across bucket configurations. The legacy stack answers per operation and
|
||||
/// bucket from the table below; the gateway stack refuses the key before the bucket is consulted,
|
||||
/// so it answers every row with `uniform`: `(NUL key, 1025-byte key)`.
|
||||
async fn keys_outside_the_floor_are_refused(server: rustfs::embedded::RustFSServer, uniform: Option<UniformAnswers>) {
|
||||
/// The gateway rejects boundary keys before consulting the bucket. RustFS serves no torrents,
|
||||
/// so a NUL key on GetObjectTorrent retains the release's NoSuchKey response.
|
||||
async fn keys_outside_the_floor_are_refused(server: rustfs::embedded::RustFSServer) {
|
||||
const BUCKET: &str = "object-meta-keys";
|
||||
let client = s3_client(&server.endpoint(), server.access_key(), server.secret_key());
|
||||
create_bucket_in_the_store(BUCKET).await;
|
||||
@@ -782,38 +778,25 @@ async fn keys_outside_the_floor_are_refused(server: rustfs::embedded::RustFSServ
|
||||
assert!(super::bucket_exists_in_the_store(reserved).await);
|
||||
put_object_in_the_store(reserved, KEY, HashMap::new()).await;
|
||||
assert_eq!(answer(client.get_object_acl().bucket(reserved).key(KEY).send().await), (None, None, None));
|
||||
let refused = |code: &str, status, message: &str| (Some(code.to_owned()), Some(status), Some(message.to_owned()));
|
||||
let invalid_nul = refused("InvalidArgument", 400, "the name contains a NUL byte");
|
||||
let missing_torrent = refused("NoSuchKey", 404, "The specified key does not exist.");
|
||||
let too_long = refused("KeyTooLongError", 400, "the name is longer than the maximum length in bytes");
|
||||
let long_key = "k".repeat(1025);
|
||||
for (operation, call) in KEY_CALLS {
|
||||
for (bucket, kind) in [(BUCKET, 0), (locked.as_str(), 1), ("object-meta-missing", 2), (reserved, 0)] {
|
||||
let (code, status, message) = match (*operation, kind) {
|
||||
("GetObjectTorrent", _) => ("NoSuchKey", 404, "The specified key does not exist."),
|
||||
("GetObjectLegalHold", 0) | ("GetObjectRetention", 0) => {
|
||||
("InvalidRequest", 400, "Bucket is missing ObjectLockConfiguration")
|
||||
}
|
||||
("GetObjectLegalHold", 1) | ("GetObjectRetention", 1) => {
|
||||
("InternalError", 500, "XML response contains a control character")
|
||||
}
|
||||
("GetObjectLegalHold", _) => ("NoSuchBucket", 404, "The specified bucket does not exist"),
|
||||
("GetObjectRetention", _) => ("InternalError", 500, "Failed to load Object Lock configuration"),
|
||||
_ => ("InvalidArgument", 400, "Invalid argument"),
|
||||
};
|
||||
let expected = match &uniform {
|
||||
Some((nul, _)) => nul.clone(),
|
||||
None => (Some(code.to_owned()), Some(status), Some(message.to_owned())),
|
||||
};
|
||||
assert_eq!(call(&client, bucket, "a\0b").await, expected, "{operation} bucket {bucket}");
|
||||
}
|
||||
let expected = match &uniform {
|
||||
Some((_, long)) => long.clone(),
|
||||
None => (Some("KeyTooLongError".to_owned()), Some(400), Some("Your key is too long.".to_owned())),
|
||||
let expected_nul = if *operation == "GetObjectTorrent" {
|
||||
&missing_torrent
|
||||
} else {
|
||||
&invalid_nul
|
||||
};
|
||||
assert_eq!(call(&client, BUCKET, &long_key).await, expected, "{operation} long key");
|
||||
for bucket in [BUCKET, locked.as_str(), "object-meta-missing", reserved] {
|
||||
assert_eq!(&call(&client, bucket, "a\0b").await, expected_nul, "{operation} bucket {bucket}");
|
||||
}
|
||||
assert_eq!(call(&client, BUCKET, &long_key).await, too_long, "{operation} long key");
|
||||
}
|
||||
server.shutdown().await;
|
||||
}
|
||||
|
||||
type Answer = (Option<String>, Option<u16>, Option<String>);
|
||||
|
||||
type KeyCall = for<'a> fn(&'a Client, &'a str, &'a str) -> BoxFuture<'a, (Option<String>, Option<u16>, Option<String>)>;
|
||||
|
||||
const KEY_CALLS: &[(&str, KeyCall)] = &[
|
||||
@@ -853,12 +836,7 @@ fn answer<T, E: ProvideErrorMetadata>(result: Result<T, SdkError<E>>) -> (Option
|
||||
|
||||
#[test]
|
||||
fn the_gateway_stack_refuses_keys_outside_the_floor() {
|
||||
let refused = |code: &str, message: &str| (Some(code.to_owned()), Some(400), Some(message.to_owned()));
|
||||
let uniform = (
|
||||
refused("InvalidArgument", "the name contains a NUL byte"),
|
||||
refused("KeyTooLongError", "the name is longer than the maximum length in bytes"),
|
||||
);
|
||||
with_stack(GATEWAY, move |server| keys_outside_the_floor_are_refused(server, Some(uniform)));
|
||||
with_stack(GATEWAY, keys_outside_the_floor_are_refused);
|
||||
}
|
||||
|
||||
fn legal_hold(status: &str) -> aws_sdk_s3::types::ObjectLockLegalHold {
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Exercise both pinned upgrade sources without building RustFS."""
|
||||
"""Exercise the pinned upgrade sources without building RustFS."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -10,6 +10,7 @@ import re
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import tomllib
|
||||
import unittest
|
||||
|
||||
from check_test_wiring import yaml_block
|
||||
@@ -17,6 +18,12 @@ from test_security_workflow import named_steps, shell_body
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
SOURCE_CASES = {
|
||||
"1.0.0-rc.5": ("rc5_rollback_requires_restoring_odm_configuration",
|
||||
"rc5_checkpoint_upgrade_rewinds_signed_progress_without_losing_objects"),
|
||||
"1.0.1": ("stable_upgrade_deletes_foreign_reservations_without_losing_live_objects",),
|
||||
"1.0.5": (),
|
||||
}
|
||||
|
||||
|
||||
class UpgradeWorkflowTests(unittest.TestCase):
|
||||
@@ -28,19 +35,16 @@ class UpgradeWorkflowTests(unittest.TestCase):
|
||||
# Explicit bash enables pipefail in Actions, so tee cannot hide a failure.
|
||||
self.assertIn(" shell: bash", self.step)
|
||||
self.assertIn(" fail-fast: false", self.job)
|
||||
self.assertEqual(re.findall(r"- version: ([^\s]+)", "\n".join(self.job)), ["1.0.0-rc.5", "1.0.1"])
|
||||
self.assertEqual(re.findall(r"- version: ([^\s]+)", "\n".join(self.job)), list(SOURCE_CASES))
|
||||
rust = (ROOT / "crates/e2e_test/src/upgrade_compatibility_test.rs").read_text()
|
||||
self.expected = set(re.findall(r'#\[ignore[^\n]*\]\s*async fn (\w+)\(', rust))
|
||||
self.assertTrue(self.expected)
|
||||
|
||||
def run_cases(self, version, failed_test="", fail_tee=False):
|
||||
excluded = ({"stable_upgrade_deletes_foreign_reservations_without_losing_live_objects"}
|
||||
if version == "1.0.0-rc.5" else {
|
||||
"rc5_rollback_requires_restoring_odm_configuration",
|
||||
"rc5_checkpoint_upgrade_rewinds_signed_progress_without_losing_objects",
|
||||
})
|
||||
expected = self.expected - excluded
|
||||
self.assertEqual(len(expected), 8 if version == "1.0.0-rc.5" else 7)
|
||||
shared = self.expected.difference(*SOURCE_CASES.values())
|
||||
self.assertEqual(len(shared), 6)
|
||||
expected = shared | set(SOURCE_CASES[version])
|
||||
self.assertTrue(expected <= self.expected)
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
root = Path(directory)
|
||||
commands = root / "commands.jsonl"
|
||||
@@ -82,14 +86,14 @@ sys.exit(17 if test == os.environ["FAILED_TEST"] else 0)
|
||||
return result
|
||||
|
||||
def test_all_ignored_upgrade_cases_run_once_with_separate_evidence(self):
|
||||
for version in ("1.0.0-rc.5", "1.0.1"):
|
||||
for version in SOURCE_CASES:
|
||||
with self.subTest(version=version):
|
||||
result = self.run_cases(version)
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
|
||||
def test_early_or_late_failure_does_not_hide_remaining_cases(self):
|
||||
for version, last in (("1.0.0-rc.5", "rc5_checkpoint_upgrade_rewinds_signed_progress_without_losing_objects"),
|
||||
("1.0.1", "stable_upgrade_deletes_foreign_reservations_without_losing_live_objects")):
|
||||
for version, specialized in SOURCE_CASES.items():
|
||||
last = specialized[-1] if specialized else "previous_release_baseline_replicates_multipart_layouts"
|
||||
for name in ("direct_upgrade_from_previous_release_preserves_object_contracts", last):
|
||||
with self.subTest(version=version, name=name):
|
||||
result = self.run_cases(version, failed_test=name)
|
||||
@@ -97,9 +101,50 @@ sys.exit(17 if test == os.environ["FAILED_TEST"] else 0)
|
||||
self.assertIn(f"::error::Upgrade compatibility failed: {name}", result.stdout)
|
||||
|
||||
def test_log_write_failure_fails_the_job(self):
|
||||
for version in ("1.0.0-rc.5", "1.0.1"):
|
||||
for version in SOURCE_CASES:
|
||||
self.assertNotEqual(self.run_cases(version, fail_tee=True).returncode, 0)
|
||||
|
||||
def test_official_reference_matches_the_workflow_and_release_plan(self):
|
||||
reference = tomllib.loads((ROOT / ".config/reference-binary.toml").read_text())
|
||||
platform = reference["platforms"]["linux-x86_64-gnu"]
|
||||
pin = ("1.0.5", platform["asset"], platform["sha256"])
|
||||
self.assertEqual(reference["version"], pin[0])
|
||||
matrix = re.findall(r"- version: (\S+)\s+asset: (\S+)\s+sha256: (\w+)", "\n".join(self.job))
|
||||
self.assertIn(pin, matrix)
|
||||
plan = json.loads((ROOT / ".config/release-validation-plan.json").read_text())
|
||||
self.assertEqual(set(matrix), {(s["version"], s["artifact_name"], s["artifact_sha256"])
|
||||
for s in plan["upgrade_sources"]["sources"]})
|
||||
legacy_mpu = "\n".join(self.steps["Run mixed-version MPU recovery"])
|
||||
self.assertIn("matrix.source.version != '1.0.5'", legacy_mpu)
|
||||
self.assertIn("mixed_version_failed_complete_requires_current_coordinator_for_abort", legacy_mpu)
|
||||
|
||||
def test_gateway_evidence_requires_both_roles_and_rollback_for_the_pinned_pair(self):
|
||||
step = self.steps["Verify gateway baseline G09 evidence"]
|
||||
self.assertIn("matrix.source.version == '1.0.5'", "\n".join(step))
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
root = Path(directory) / "rustfs-upgrade-g09-evidence"
|
||||
env = dict(os.environ, RUNNER_TEMP=directory, GITHUB_SHA="c" * 40, UPGRADE_SOURCE_REVISION="a" * 40)
|
||||
for field in ("mixed_version_reader_evidence", "mixed_version_writer_evidence", "rollback_payload_evidence"):
|
||||
folder = root / ("bucket-config-rollback" if field == "rollback_payload_evidence" else "mixed-version-upgrade")
|
||||
folder.mkdir(exist_ok=True, parents=True)
|
||||
payload = dict(gate="G09", field=field, current_revision=env["GITHUB_SHA"],
|
||||
previous_revision=env["UPGRADE_SOURCE_REVISION"], rollback_payload_replayed=True)
|
||||
(folder / f"G09-{field}.json").write_text(json.dumps(payload))
|
||||
multipart = root / "bucket-config-rollback/inflight-multipart-rollback.json"
|
||||
payload = dict(current_revision=env["GITHUB_SHA"], previous_revision=env["UPGRADE_SOURCE_REVISION"],
|
||||
part_sizes=[5242880, 5242880, 4096], bytes=10489856,
|
||||
rollback_read_sha256="d" * 64, roll_forward_read_sha256="d" * 64)
|
||||
multipart.write_text(json.dumps(payload))
|
||||
command = ["bash", "-e", "-o", "pipefail", "-c", shell_body(step)]
|
||||
self.assertEqual(subprocess.run(command, env=env, capture_output=True).returncode, 0)
|
||||
wrong_pair = dict(env, UPGRADE_SOURCE_REVISION="b" * 40)
|
||||
self.assertNotEqual(subprocess.run(command, env=wrong_pair, capture_output=True).returncode, 0)
|
||||
multipart.write_text(json.dumps(dict(payload, roll_forward_read_sha256="e" * 64)))
|
||||
self.assertNotEqual(subprocess.run(command, env=env, capture_output=True).returncode, 0)
|
||||
multipart.write_text(json.dumps(payload))
|
||||
(root / "mixed-version-upgrade/G09-mixed_version_reader_evidence.json").unlink()
|
||||
self.assertNotEqual(subprocess.run(command, env=env, capture_output=True).returncode, 0)
|
||||
|
||||
def test_reports_are_uploaded_even_after_failure(self):
|
||||
for name, directory in (("Upload scanner/heal G09 evidence", "rustfs-upgrade-g09-evidence"),
|
||||
("Upload server logs", "rustfs-upgrade-logs")):
|
||||
|
||||
Reference in New Issue
Block a user