mirror of
https://github.com/rustfs/rustfs.git
synced 2026-11-20 15:07:34 +00:00
Compare commits
9 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 38aa318fb2 | |||
| e7c5d7c198 | |||
| e37aa27e40 | |||
| 6e417a458e | |||
| 60ed7f1c40 | |||
| 7f2bd6b99b | |||
| a798df7906 | |||
| 793632c0da | |||
| 301133c1c8 |
@@ -1,2 +1,2 @@
|
||||
sha256-darwin=ab6f66c9808753082eabfb43c9e6f16508a8d065a44428284feeb5bef3fca9ba
|
||||
sha256-linux=dde1254dc9993c23d76d90b4689f1c1af9d525e16d793f4b6ab4df9c1f74f3d4
|
||||
sha256-darwin=9ccb03f4139e99ec11a63810523d504df39191b86e74b28a72dd9a075682293b
|
||||
sha256-linux=5ef052b03a03afbd00ebfcbbdcb754bbde414b7e6151aae3e2ee78024ce1d0b6
|
||||
|
||||
@@ -114,8 +114,14 @@
|
||||
}
|
||||
],
|
||||
"upgrade_sources": {
|
||||
"basis": "Exactly the two previous-release sources pinned in .github/workflows/e2e-upgrade.yml are exercised today (source.version / source.asset / source.sha256). No other N-1/N-2 compatibility is claimed, and these digests are not inferred from tags.",
|
||||
"basis": "The three previous-release sources pinned in .github/workflows/e2e-upgrade.yml are exercised today (source.version / source.asset / source.sha256). No other N-1/N-2 compatibility is claimed, and these digests are not inferred from tags.",
|
||||
"sources": [
|
||||
{
|
||||
"version": "1.0.5",
|
||||
"role": "gateway compatibility baseline",
|
||||
"artifact_name": "rustfs-linux-x86_64-gnu-v1.0.5.zip",
|
||||
"artifact_sha256": "5addae2f74756bf80104d0975e6347cda843bc2daffdfcd1f0cb80118b95cbd0"
|
||||
},
|
||||
{
|
||||
"version": "1.0.1",
|
||||
"role": "N-1",
|
||||
@@ -130,6 +136,6 @@
|
||||
}
|
||||
],
|
||||
"rollback_supported": false,
|
||||
"rollback_basis": "Only the bucket-config-rollback case (rollback_to_previous_release_reads_current_bucket_metadata) is exercised. Full data-plane rollback is not certified and is never a default claim."
|
||||
"rollback_basis": "The release upgrade lane checks bucket-config rollback. The hosted compatibility workflow also checks in-flight multipart rollback to 1.0.5. Full data-plane rollback is not certified and is never a default claim."
|
||||
}
|
||||
}
|
||||
|
||||
@@ -90,11 +90,16 @@ jobs:
|
||||
- version: 1.0.1
|
||||
asset: rustfs-linux-x86_64-gnu-v1.0.1.zip
|
||||
sha256: 9742e183cb0dfb0d48fa909797af572fe47b7f827569036baa4156e0d3958ee5
|
||||
- version: 1.0.5
|
||||
asset: rustfs-linux-x86_64-gnu-v1.0.5.zip
|
||||
sha256: 5addae2f74756bf80104d0975e6347cda843bc2daffdfcd1f0cb80118b95cbd0
|
||||
revision: 34e6449ed90e3d3a106f6655342e0b3127f33082
|
||||
env:
|
||||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
|
||||
UPGRADE_SOURCE_VERSION: ${{ matrix.source.version }}
|
||||
UPGRADE_SOURCE_ASSET: ${{ matrix.source.asset }}
|
||||
UPGRADE_SOURCE_SHA256: ${{ matrix.source.sha256 }}
|
||||
UPGRADE_SOURCE_REVISION: ${{ matrix.source.revision }}
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
@@ -132,6 +137,13 @@ jobs:
|
||||
unzip -q "$archive" -d "$SOURCE_DIR"
|
||||
chmod +x "$SOURCE_DIR/rustfs"
|
||||
test -x "$SOURCE_DIR/rustfs"
|
||||
if [[ -n "$UPGRADE_SOURCE_REVISION" ]]; then
|
||||
mkdir -p "$RUNNER_TEMP/rustfs-upgrade-logs"
|
||||
version_log="$RUNNER_TEMP/rustfs-upgrade-logs/source-version.txt"
|
||||
"$SOURCE_DIR/rustfs" --version | tee "$version_log"
|
||||
grep -Fx "rustfs $UPGRADE_SOURCE_VERSION" "$version_log"
|
||||
grep -Eq "^git commit[[:space:]]*:[[:space:]]*$UPGRADE_SOURCE_REVISION$" "$version_log"
|
||||
fi
|
||||
echo "RUSTFS_UPGRADE_SOURCE_BINARY=$SOURCE_DIR/rustfs" >> "$GITHUB_ENV"
|
||||
echo "RUSTFS_E2E_LOG_DIR=$RUNNER_TEMP/rustfs-upgrade-logs" >> "$GITHUB_ENV"
|
||||
|
||||
@@ -151,12 +163,14 @@ jobs:
|
||||
multipart-layout-upgrade:direct_upgrade_from_previous_release_preserves_multipart_layouts
|
||||
multipart-layout-baseline:previous_release_baseline_replicates_multipart_layouts
|
||||
)
|
||||
if [[ "$UPGRADE_SOURCE_VERSION" == "1.0.0-rc.5" ]]; then
|
||||
cases+=(odm-config-rollback:rc5_rollback_requires_restoring_odm_configuration)
|
||||
cases+=(checkpoint-upgrade:rc5_checkpoint_upgrade_rewinds_signed_progress_without_losing_objects)
|
||||
else
|
||||
cases+=(legacy-delete-reservation:stable_upgrade_deletes_foreign_reservations_without_losing_live_objects)
|
||||
fi
|
||||
case "$UPGRADE_SOURCE_VERSION" in
|
||||
1.0.0-rc.5)
|
||||
cases+=(odm-config-rollback:rc5_rollback_requires_restoring_odm_configuration)
|
||||
cases+=(checkpoint-upgrade:rc5_checkpoint_upgrade_rewinds_signed_progress_without_losing_objects) ;;
|
||||
1.0.1) cases+=(legacy-delete-reservation:stable_upgrade_deletes_foreign_reservations_without_losing_live_objects) ;;
|
||||
1.0.5) ;; # The bucket rollback case also completes an in-flight multipart upload.
|
||||
*) echo "::error::Unknown upgrade source $UPGRADE_SOURCE_VERSION"; exit 1 ;;
|
||||
esac
|
||||
status=0
|
||||
for entry in "${cases[@]}"; do
|
||||
artifact="${entry%%:*}"
|
||||
@@ -178,7 +192,7 @@ jobs:
|
||||
exit "$status"
|
||||
|
||||
- name: Run mixed-version MPU recovery
|
||||
if: ${{ !cancelled() && steps.verify_source.outcome == 'success' }}
|
||||
if: ${{ !cancelled() && steps.verify_source.outcome == 'success' && matrix.source.version != '1.0.5' }}
|
||||
timeout-minutes: 15
|
||||
env:
|
||||
NO_PROXY: 127.0.0.1,localhost,::1
|
||||
@@ -199,6 +213,24 @@ jobs:
|
||||
"$mpu_test" -- --exact --nocapture --test-threads=1 \
|
||||
2>&1 | tee "$RUSTFS_E2E_LOG_DIR/test.log"
|
||||
|
||||
- name: Verify gateway baseline G09 evidence
|
||||
if: ${{ success() && matrix.source.version == '1.0.5' }}
|
||||
shell: bash
|
||||
run: |
|
||||
for field in mixed_version_reader_evidence mixed_version_writer_evidence rollback_payload_evidence; do
|
||||
directory=mixed-version-upgrade
|
||||
if [[ "$field" == rollback_payload_evidence ]]; then directory=bucket-config-rollback; fi
|
||||
jq -e --arg field "$field" --arg current "$GITHUB_SHA" --arg previous "$UPGRADE_SOURCE_REVISION" \
|
||||
'.gate == "G09" and .field == $field and .current_revision == $current and .previous_revision == $previous
|
||||
and ($field != "rollback_payload_evidence" or .rollback_payload_replayed == true)' \
|
||||
"$RUNNER_TEMP/rustfs-upgrade-g09-evidence/$directory/G09-$field.json"
|
||||
done
|
||||
jq -e --arg current "$GITHUB_SHA" --arg previous "$UPGRADE_SOURCE_REVISION" \
|
||||
'.current_revision == $current and .previous_revision == $previous
|
||||
and .part_sizes == [5242880, 5242880, 4096] and .bytes == (.part_sizes | add)
|
||||
and (.rollback_read_sha256 | test("^[a-f0-9]{64}$")) and .rollback_read_sha256 == .roll_forward_read_sha256' \
|
||||
"$RUNNER_TEMP/rustfs-upgrade-g09-evidence/bucket-config-rollback/inflight-multipart-rollback.json"
|
||||
|
||||
- name: Upload scanner/heal G09 evidence
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
|
||||
+1
-1
@@ -13,7 +13,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
|
||||
### Deployment
|
||||
|
||||
- **The S3 protocol stack is now rustfs/gateway**, and it is the only S3 stack. The legacy s3s-based stack is removed, and so are the `RUSTFS_S3_STACK` environment variable and the `gateway-stack` cargo feature. There is no on-disk format change. `RUSTFS_S3_STACK` no longer selects anything: a server started with it set, to any value, refuses to start and prints `RUSTFS_S3_STACK is no longer supported; the gateway stack is the only S3 stack since <version>`, where `<version>` is the release's version. Remove the variable from service units, container environments, Helm values and orchestration files before upgrading. To roll back, run the previous release binary on the same data.
|
||||
- **The S3 protocol stack is now rustfs/gateway**, and it is the only S3 stack. The legacy s3s-based stack is removed, and so are the `RUSTFS_S3_STACK` environment variable and the `gateway-stack` cargo feature. There is no on-disk format change. `RUSTFS_S3_STACK` no longer selects anything: a server started with it set, to any value, refuses to start and prints `RUSTFS_S3_STACK is no longer supported; the gateway stack is the only S3 stack since <version>`, where `<version>` is the release's version. Remove the variable from service units, container environments, Helm values and orchestration files before upgrading. To roll back, follow the [S3 stack rollback procedure](docs/operations/s3-stack-rollback.md).
|
||||
|
||||
- Before upgrading, verify that the configured root access key is at least 3 bytes and the root secret key is at least 8 bytes. RustFS now rejects shorter values at startup and identifies the setting to update. Version 1.0.5 accepted these settings at startup and returned HTTP 500 when root requested STS credentials. Coordinate root credential changes across all nodes and clients. See [root credential rotation considerations](docs/operations/two-factor-auth.md#root-credentials-cannot-be-changed-at-runtime).
|
||||
|
||||
|
||||
@@ -34,7 +34,10 @@
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use crate::common::{RustFSTestEnvironment, init_logging, local_http_client};
|
||||
use crate::common::{
|
||||
AdminTransport, RustFSTestEnvironment, admin_attach_user_policy_via, admin_create_user, admin_request_with_session_token,
|
||||
init_logging, local_http_client, parse_assume_role_credentials,
|
||||
};
|
||||
use hmac::{Hmac, KeyInit as _, Mac};
|
||||
use http::header::HOST;
|
||||
use rustfs_s3_types::Body;
|
||||
@@ -396,6 +399,54 @@ mod tests {
|
||||
let (status, _) = signed_request(&env.url, http::Method::GET, ACCOUNT_INFO_PATH, None, user_ak, old_sk).await?;
|
||||
assert!(status.is_client_error(), "the old secret must stop working, status: {status}");
|
||||
|
||||
env.stop_server();
|
||||
drop(env);
|
||||
console_session_rotates_the_iam_password_and_revokes_the_old_session().await
|
||||
}
|
||||
|
||||
async fn console_session_rotates_the_iam_password_and_revokes_the_old_session() -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
init_logging();
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server(vec![]).await?;
|
||||
|
||||
let user = "console-password-user";
|
||||
let old_password = uuid::Uuid::new_v4().to_string();
|
||||
let new_password = uuid::Uuid::new_v4().to_string();
|
||||
admin_create_user(&env, user, &old_password).await?;
|
||||
admin_attach_user_policy_via(AdminTransport::Signed, &env.url, &env.access_key, &env.secret_key, "readonly", user)
|
||||
.await?;
|
||||
|
||||
let (status, response) = assume_role(&env.url, user, &old_password, None).await?;
|
||||
assert_eq!(status, reqwest::StatusCode::OK, "the current password must sign in");
|
||||
let (session_key, session_secret, token) = parse_assume_role_credentials(&response)?;
|
||||
let session_request = |method, path, body| {
|
||||
admin_request_with_session_token(&env.url, method, path, body, &session_key, &session_secret, Some(&token))
|
||||
};
|
||||
let password_request = |current: &str| {
|
||||
Some(serde_json::json!({ "current_secret_key": current, "new_secret_key": new_password }).to_string())
|
||||
};
|
||||
|
||||
let (status, body) =
|
||||
session_request(http::Method::POST, ACCOUNT_PASSWORD_PATH, password_request(&session_secret)).await?;
|
||||
assert_eq!(
|
||||
status,
|
||||
reqwest::StatusCode::BAD_REQUEST,
|
||||
"the session secret must not be accepted as the IAM password"
|
||||
);
|
||||
assert!(body.contains("InvalidRequest") && body.contains("the current secret key is incorrect"));
|
||||
|
||||
let (status, body) = session_request(http::Method::POST, ACCOUNT_PASSWORD_PATH, password_request(&old_password)).await?;
|
||||
assert_eq!(status, reqwest::StatusCode::OK, "the same session must accept the current IAM password");
|
||||
assert_eq!(serde_json::from_str::<serde_json::Value>(&body)?["sessions_revoked"], 1);
|
||||
|
||||
let (status, _) = session_request(http::Method::GET, ACCOUNT_INFO_PATH, None).await?;
|
||||
assert_eq!(status, reqwest::StatusCode::FORBIDDEN, "the pre-rotation session must be revoked");
|
||||
let (status, _) = assume_role(&env.url, user, &old_password, None).await?;
|
||||
assert_eq!(status, reqwest::StatusCode::FORBIDDEN, "the old password must no longer sign in");
|
||||
let (status, response) = assume_role(&env.url, user, &new_password, None).await?;
|
||||
assert_eq!(status, reqwest::StatusCode::OK, "the new password must sign in");
|
||||
parse_assume_role_credentials(&response)?;
|
||||
|
||||
env.stop_server();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -31,7 +31,7 @@ use super::harness::{
|
||||
use crate::common::{
|
||||
AdminTransport, admin_add_canned_policy_via, admin_attach_user_policy_via, admin_create_user_via, init_logging,
|
||||
};
|
||||
use crate::upgrade_compatibility_test::wait_for_upgrade_write_readiness;
|
||||
use crate::upgrade_compatibility_test::{UpgradeSourceCapabilities, wait_for_upgrade_write_readiness};
|
||||
use aws_sdk_s3::Client;
|
||||
use aws_sdk_s3::error::ProvideErrorMetadata;
|
||||
use std::path::{Path, PathBuf};
|
||||
@@ -427,7 +427,12 @@ async fn assert_history_and_iam(dist: &DistCluster, seed: &UpgradeSeed, context:
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn compare_degraded_legacy_list_parts(dist: &mut DistCluster, seed: &UpgradeSeed, previous: &Path) -> TestResult {
|
||||
async fn compare_degraded_legacy_list_parts(
|
||||
dist: &mut DistCluster,
|
||||
seed: &UpgradeSeed,
|
||||
previous: &Path,
|
||||
source: &UpgradeSourceCapabilities,
|
||||
) -> TestResult {
|
||||
let decoded = String::from_utf8(base64_simd::URL_SAFE_NO_PAD.decode_to_vec(seed.multipart_upload.as_bytes())?)?;
|
||||
let (_, upload_dir) = decoded.split_once('.').ok_or("legacy upload ID lacks deployment delimiter")?;
|
||||
let hash = sha256_hex(format!("{}/legacy-multipart.bin", seed.history_bucket).as_bytes());
|
||||
@@ -489,16 +494,22 @@ async fn compare_degraded_legacy_list_parts(dist: &mut DistCluster, seed: &Upgra
|
||||
// Three online disks retain metadata quorum but have no part directory;
|
||||
// the fourth disk is offline. A legacy empty response is not a proof that
|
||||
// the unavailable participant has no part.
|
||||
let legacy = legacy.expect("previous coordinator should expose its legacy empty-list behavior");
|
||||
assert!(legacy.parts().is_empty());
|
||||
let current = current.expect_err("current coordinator must not infer empty from missing directories and offline disks");
|
||||
assert_eq!(current.raw_response().expect("current S3 response").status().as_u16(), 503);
|
||||
// Since #9149, SlowDownRead is reserved for GET/HEAD object reads; a
|
||||
// ListParts read-quorum failure is reported as ServiceUnavailable.
|
||||
assert_eq!(
|
||||
current.as_service_error().and_then(|error| error.meta().code()),
|
||||
Some("ServiceUnavailable")
|
||||
);
|
||||
let mut quorum_errors =
|
||||
vec![current.expect_err("current coordinator must not infer empty from missing directories and offline disks")];
|
||||
if source.requires_list_parts_read_quorum {
|
||||
quorum_errors.push(legacy.expect_err("previous coordinator must require evidence from unavailable disks"));
|
||||
} else {
|
||||
assert!(
|
||||
legacy
|
||||
.expect("previous coordinator should expose its legacy empty-list behavior")
|
||||
.parts()
|
||||
.is_empty()
|
||||
);
|
||||
}
|
||||
for error in quorum_errors {
|
||||
assert_eq!(error.raw_response().expect("S3 response").status().as_u16(), 503);
|
||||
assert_eq!(error.as_service_error().and_then(|error| error.meta().code()), Some("ServiceUnavailable"));
|
||||
}
|
||||
dist.cluster.start_node_from_binary(3, previous).await?;
|
||||
wait_for_upgrade_write_readiness(&clients, &seed.history_bucket, "mixed-list-restored", CREDENTIAL_TIMEOUT).await?;
|
||||
for (path, before) in original_files {
|
||||
@@ -520,7 +531,7 @@ async fn compare_degraded_legacy_list_parts(dist: &mut DistCluster, seed: &Upgra
|
||||
assert_eq!(listed.parts()[0].e_tag(), Some(seed.multipart_etag.as_str()));
|
||||
}
|
||||
eprintln!(
|
||||
"MPU mixed degraded: one offline disk, three missing part directories; previous returned empty, current returned 503 SlowDownRead; original part restored on both"
|
||||
"MPU mixed degraded: one offline disk, three missing part directories; checked each coordinator against its release contract; original part restored on both"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
@@ -592,6 +603,7 @@ async fn replace_node_and_wait_for_write_readiness(dist: &mut DistCluster, seed:
|
||||
async fn four_node_rolling_upgrade_preserves_history_and_iam_credentials() -> TestResult {
|
||||
init_logging();
|
||||
let previous = source_binary()?;
|
||||
let source = UpgradeSourceCapabilities::from_binary(&previous).await?;
|
||||
let mut dist = DistCluster::new_stopped(DistLayout::FourNodeFourDisk).await?;
|
||||
dist.cluster.set_env("RUSTFS_STORAGE_CLASS_RRS", "EC:0");
|
||||
capture_upgrade_logs(&mut dist, "rolling-upgrade")?;
|
||||
@@ -601,7 +613,7 @@ async fn four_node_rolling_upgrade_preserves_history_and_iam_credentials() -> Te
|
||||
|
||||
replace_node_and_wait_for_write_readiness(&mut dist, &seed, 0).await?;
|
||||
assert_history_and_iam(&dist, &seed, "one-current-node").await?;
|
||||
compare_degraded_legacy_list_parts(&mut dist, &seed, &previous).await?;
|
||||
compare_degraded_legacy_list_parts(&mut dist, &seed, &previous, &source).await?;
|
||||
|
||||
for node_idx in [1, 2] {
|
||||
replace_node_and_wait_for_write_readiness(&mut dist, &seed, node_idx).await?;
|
||||
|
||||
@@ -46,9 +46,12 @@ type BoxError = Box<dyn std::error::Error + Send + Sync>;
|
||||
|
||||
const SOURCE_BINARY_ENV: &str = "RUSTFS_UPGRADE_SOURCE_BINARY";
|
||||
const G09_EVIDENCE_DIR_ENV: &str = "RUSTFS_SCANNER_HEAL_G09_EVIDENCE_DIR";
|
||||
const RC2_COMMIT: &str = "e9f53180273b32a9c2b0aa8b5a384047bc7e875c";
|
||||
const RC5_COMMIT: &str = "40a2470feb567201165a5b809b7598bb4b1f68f5";
|
||||
const STABLE_1_0_1_COMMIT: &str = "6de965ae3c965a78ff819fbcd7acd4aa44177d92";
|
||||
const STABLE_1_0_4_COMMIT: &str = "46337942d55f3a8e6a97b147ca98d9ea8115555d";
|
||||
const STABLE_1_0_5_COMMIT: &str = "34e6449ed90e3d3a106f6655342e0b3127f33082";
|
||||
const STABLE_1_0_6_COMMIT: &str = "de9009d882644effc2628e777650025c3d8e7c65";
|
||||
const SSE_MASTER_KEY_ENV: &str = "RUSTFS_SSE_S3_MASTER_KEY";
|
||||
const SSE_MASTER_KEY: &str = "QkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkI=";
|
||||
const PLAIN_BUCKET: &str = "upgrade-plain-data";
|
||||
@@ -142,6 +145,41 @@ fn current_source_revision() -> Result<String, BoxError> {
|
||||
Ok(revision.to_string())
|
||||
}
|
||||
|
||||
pub(crate) struct UpgradeSourceCapabilities {
|
||||
supports_delete_recovery: bool,
|
||||
supports_multipart_publication: bool,
|
||||
pub(crate) requires_list_parts_read_quorum: bool,
|
||||
}
|
||||
|
||||
impl UpgradeSourceCapabilities {
|
||||
fn from_revision(revision: &str) -> Result<Self, BoxError> {
|
||||
// 1.0.4 supports the delete decision protocol (#8468). Version 1.0.5 also
|
||||
// shipped native MPU publication (#8919) and strict ListParts evidence (#9037, #9149).
|
||||
match revision {
|
||||
RC2_COMMIT | RC5_COMMIT | STABLE_1_0_1_COMMIT => Ok(Self {
|
||||
supports_delete_recovery: false,
|
||||
supports_multipart_publication: false,
|
||||
requires_list_parts_read_quorum: false,
|
||||
}),
|
||||
STABLE_1_0_4_COMMIT => Ok(Self {
|
||||
supports_delete_recovery: true,
|
||||
supports_multipart_publication: false,
|
||||
requires_list_parts_read_quorum: false,
|
||||
}),
|
||||
STABLE_1_0_5_COMMIT | STABLE_1_0_6_COMMIT => Ok(Self {
|
||||
supports_delete_recovery: true,
|
||||
supports_multipart_publication: true,
|
||||
requires_list_parts_read_quorum: true,
|
||||
}),
|
||||
_ => Err(format!("no pinned upgrade expectations for source revision {revision}").into()),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn from_binary(binary: &Path) -> Result<Self, BoxError> {
|
||||
Self::from_revision(&binary_source_revision(binary).await?)
|
||||
}
|
||||
}
|
||||
|
||||
async fn binary_source_revision(binary: &Path) -> Result<String, BoxError> {
|
||||
let output = tokio::process::Command::new(binary).arg("--version").output().await?;
|
||||
if !output.status.success() {
|
||||
@@ -225,6 +263,25 @@ mod g09_evidence_tests {
|
||||
use super::*;
|
||||
use tempfile::TempDir;
|
||||
|
||||
#[test]
|
||||
fn pinned_upgrade_sources_keep_distinct_delete_and_list_parts_contracts() -> TestResult {
|
||||
for (revision, delete_recovery, multipart_publication, list_parts_quorum) in [
|
||||
(RC2_COMMIT, false, false, false),
|
||||
(RC5_COMMIT, false, false, false),
|
||||
(STABLE_1_0_1_COMMIT, false, false, false),
|
||||
(STABLE_1_0_4_COMMIT, true, false, false),
|
||||
(STABLE_1_0_5_COMMIT, true, true, true),
|
||||
(STABLE_1_0_6_COMMIT, true, true, true),
|
||||
] {
|
||||
let source = UpgradeSourceCapabilities::from_revision(revision)?;
|
||||
assert_eq!(source.supports_delete_recovery, delete_recovery, "{revision}");
|
||||
assert_eq!(source.supports_multipart_publication, multipart_publication, "{revision}");
|
||||
assert_eq!(source.requires_list_parts_read_quorum, list_parts_quorum, "{revision}");
|
||||
}
|
||||
assert!(UpgradeSourceCapabilities::from_revision("ffffffffffffffffffffffffffffffffffffffff").is_err());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn source_revision_parser_requires_lowercase_forty_byte_sha() -> TestResult {
|
||||
let binary = Path::new("rustfs");
|
||||
@@ -1011,8 +1068,7 @@ async fn exercise_mixed_cluster(
|
||||
vec![u8::try_from(MULTIPART_WORKERS - 1)?; 64 * 1024]
|
||||
);
|
||||
|
||||
// 1.0.4 already implements the delete decision protocol. Use a separate
|
||||
// old-writer key so the original seed still spans the entire rollout.
|
||||
// Use a separate old-writer key so the original seed still spans the entire rollout.
|
||||
if previous_has_delete_recovery {
|
||||
let key = format!(".upgrade-delete/{phase}/previous-writer");
|
||||
previous_client
|
||||
@@ -2293,7 +2349,7 @@ async fn rolling_upgrade_from_previous_release_preserves_mixed_version_contracts
|
||||
let previous_binary = source_binary()?;
|
||||
// Bind the expected protocol to the verified release source, never to the
|
||||
// DELETE result itself. The older CI source pins retain their 503 contract.
|
||||
let previous_has_delete_recovery = binary_source_revision(&previous_binary).await? == STABLE_1_0_4_COMMIT;
|
||||
let source = UpgradeSourceCapabilities::from_binary(&previous_binary).await?;
|
||||
let evidence_context = g09_evidence_context(&previous_binary).await?;
|
||||
let current_binary = rustfs_binary_path();
|
||||
let mut cluster = RustFSTestClusterEnvironment::new(MIXED_NODE_COUNT).await?;
|
||||
@@ -2315,17 +2371,21 @@ async fn rolling_upgrade_from_previous_release_preserves_mixed_version_contracts
|
||||
cluster.stop_node(0)?;
|
||||
cluster.start_node_from_binary(0, ¤t_binary).await?;
|
||||
let mut mixed_deleted_keys = Vec::new();
|
||||
if let Some(key) = exercise_mixed_cluster(&cluster, "one-current-node", 0, 1, previous_has_delete_recovery).await? {
|
||||
if let Some(key) = exercise_mixed_cluster(&cluster, "one-current-node", 0, 1, source.supports_delete_recovery).await? {
|
||||
mixed_deleted_keys.push(key);
|
||||
}
|
||||
// One controlled out-of-policy attempt proves fail-closed behavior below
|
||||
// native write quorum; positive MPU writers remain paused while mixed.
|
||||
assert_mixed_pending_refusal(&cluster, &pending, &sources).await?;
|
||||
let mixed_completion = if source.supports_multipart_publication {
|
||||
let clients = cluster.create_all_clients()?;
|
||||
Some(finish_pending_multipart(&clients, &pending, pending_parts.concat()).await?)
|
||||
} else {
|
||||
assert_mixed_pending_refusal(&cluster, &pending, &sources).await?;
|
||||
None
|
||||
};
|
||||
|
||||
for (node_idx, phase) in [(1, "two-current-nodes"), (2, "one-previous-node")] {
|
||||
cluster.stop_node(node_idx)?;
|
||||
cluster.start_node_from_binary(node_idx, ¤t_binary).await?;
|
||||
if let Some(key) = exercise_mixed_cluster(&cluster, phase, 0, 3, previous_has_delete_recovery).await? {
|
||||
if let Some(key) = exercise_mixed_cluster(&cluster, phase, 0, 3, source.supports_delete_recovery).await? {
|
||||
mixed_deleted_keys.push(key);
|
||||
}
|
||||
}
|
||||
@@ -2335,6 +2395,13 @@ async fn rolling_upgrade_from_previous_release_preserves_mixed_version_contracts
|
||||
let clients = cluster.create_all_clients()?;
|
||||
wait_for_drive_readmission(&clients, MIXED_BUCKET, "homogeneous-current", DRIVE_READMISSION_TIMEOUT).await?;
|
||||
let completed_pending = finish_pending_multipart(&clients, &pending, pending_parts.concat()).await?;
|
||||
if let Some(mixed) = mixed_completion {
|
||||
assert_eq!(
|
||||
completed_pending.version_id, mixed.version_id,
|
||||
"full upgrade changed the completed version"
|
||||
);
|
||||
assert_eq!(completed_pending.etag, mixed.etag, "full upgrade changed the completed ETag");
|
||||
}
|
||||
exercise_homogeneous_multipart(&cluster, "homogeneous-current").await?;
|
||||
verify_load_bodies(&clients, "homogeneous-previous", "multipart").await?;
|
||||
assert_mixed_deletes_visible(&clients, &mixed_deleted_keys).await?;
|
||||
@@ -2451,7 +2518,7 @@ async fn rolling_upgrade_from_previous_release_preserves_mixed_version_contracts
|
||||
"phases": phases,
|
||||
"objects_per_phase": objects_per_phase,
|
||||
"mixed_load": "plain-put",
|
||||
"mixed_delete_protocol_supported": previous_has_delete_recovery,
|
||||
"mixed_delete_protocol_supported": source.supports_delete_recovery,
|
||||
"mixed_deleted_keys": mixed_deleted_keys,
|
||||
"homogeneous_multipart_phases": ["homogeneous-previous", "homogeneous-current", "homogeneous-rollback"],
|
||||
"multipart_objects_per_homogeneous_phase": MULTIPART_WORKERS * MULTIPART_UPLOADS_PER_WORKER,
|
||||
@@ -2461,7 +2528,7 @@ async fn rolling_upgrade_from_previous_release_preserves_mixed_version_contracts
|
||||
"all nodes list every mixed-version object after homogeneous-current convergence",
|
||||
"the previous-release seed survives the baseline restart and every upgrade phase",
|
||||
"previous readers read every current-native MPU body after quiescent rollback",
|
||||
"same legacy upload resumes with identical body, ETag and version after full upgrade"
|
||||
"the same previous-release upload remains readable after full upgrade and rollback"
|
||||
],
|
||||
}),
|
||||
)?;
|
||||
@@ -2483,9 +2550,13 @@ async fn rolling_upgrade_from_previous_release_preserves_mixed_version_contracts
|
||||
"previous-release writer publishes objects readable by current node",
|
||||
"mixed phases retain 16x16 plain PUT writers while positive multipart writers are paused",
|
||||
"16x16 multipart load runs in homogeneous previous, current and rollback phases",
|
||||
"one-current-node Complete returns typed 503 and preserves actual source authority and parts",
|
||||
"the same legacy upload completes and replays once fully upgraded without adding a version",
|
||||
if previous_has_delete_recovery {
|
||||
if source.supports_multipart_publication {
|
||||
"one-current-node Complete and replay return the same version and ETag, and all readers agree on the body"
|
||||
} else {
|
||||
"one-current-node Complete returns typed 503 and preserves actual source authority and parts"
|
||||
},
|
||||
"fully upgraded readers see the same body, ETag and sole version after replay",
|
||||
if source.supports_delete_recovery {
|
||||
"mixed-version DELETE succeeds and stays absent through every reader after upgrade and rollback"
|
||||
} else {
|
||||
"modern DELETE fails closed and preserves every byte while old peers remain"
|
||||
@@ -3386,6 +3457,8 @@ async fn direct_upgrade_from_previous_release_preserves_bucket_configuration() -
|
||||
#[tokio::test]
|
||||
#[ignore = "requires a pinned previous RustFS release binary"]
|
||||
async fn rollback_to_previous_release_reads_current_bucket_metadata() -> TestResult {
|
||||
use sha2::Digest;
|
||||
|
||||
init_logging();
|
||||
let previous_binary = source_binary()?;
|
||||
let evidence_context = g09_evidence_context(&previous_binary).await?;
|
||||
@@ -3423,6 +3496,16 @@ async fn rollback_to_previous_release_reads_current_bucket_metadata() -> TestRes
|
||||
let multipart_bytes = multipart_parts.concat();
|
||||
write_multipart(&new_client, ROLLBACK_BUCKET, multipart_key, &multipart_parts).await?;
|
||||
|
||||
// This release can finish an upload staged by the current gateway.
|
||||
let inflight = if binary_source_revision(&previous_binary).await? == STABLE_1_0_5_COMMIT {
|
||||
env.create_test_bucket(PENDING_MULTIPART_BUCKET).await?;
|
||||
let parts = [vec![1; LAYOUT_PART_SIZE], vec![2; LAYOUT_PART_SIZE], vec![3; 4096]];
|
||||
let pending = stage_multipart(&new_client, PENDING_MULTIPART_BUCKET, PENDING_MULTIPART_KEY, &parts).await?;
|
||||
Some((pending, parts.concat()))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
restart_from_binary(&mut env, &previous_binary, &server_env).await?;
|
||||
let old_client = env.create_s3_client();
|
||||
|
||||
@@ -3438,6 +3521,16 @@ async fn rollback_to_previous_release_reads_current_bucket_metadata() -> TestRes
|
||||
assert_eq!(encryption, Some(ServerSideEncryption::Aes256));
|
||||
assert_eq!(body, multipart_bytes, "the multipart object did not survive the rollback");
|
||||
|
||||
let rollback_multipart = if let Some((pending, expected)) = inflight {
|
||||
assert_pending_parts(&old_client, &pending).await?;
|
||||
complete_pending_multipart(&old_client, PENDING_MULTIPART_BUCKET, PENDING_MULTIPART_KEY, &pending).await?;
|
||||
let (_, body) = read_object(&old_client, PENDING_MULTIPART_BUCKET, PENDING_MULTIPART_KEY, None).await?;
|
||||
assert_eq!(body, expected);
|
||||
Some((pending.part_sizes, body))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
// A PUT on the rolled-back release must still honour the encryption
|
||||
// configuration it decoded out of the current build's metadata blob.
|
||||
let post_rollback_key = "rollback/written-after-rollback";
|
||||
@@ -3478,6 +3571,25 @@ async fn rollback_to_previous_release_reads_current_bucket_metadata() -> TestRes
|
||||
post_rollback_bytes
|
||||
);
|
||||
|
||||
if let Some((part_sizes, rollback_read)) = rollback_multipart {
|
||||
let (_, current_read) = read_object(¤t_again, PENDING_MULTIPART_BUCKET, PENDING_MULTIPART_KEY, None).await?;
|
||||
assert_eq!(current_read, rollback_read);
|
||||
if let Some(context) = &evidence_context {
|
||||
let evidence = serde_json::json!({
|
||||
"previous_revision": context.previous_revision,
|
||||
"current_revision": context.current_revision,
|
||||
"part_sizes": part_sizes,
|
||||
"bytes": current_read.len(),
|
||||
"rollback_read_sha256": hex_simd::encode_to_string(sha2::Sha256::digest(&rollback_read), hex_simd::AsciiCase::Lower),
|
||||
"roll_forward_read_sha256": hex_simd::encode_to_string(sha2::Sha256::digest(¤t_read), hex_simd::AsciiCase::Lower),
|
||||
});
|
||||
std::fs::write(
|
||||
context.directory.join("inflight-multipart-rollback.json"),
|
||||
serde_json::to_vec_pretty(&evidence)?,
|
||||
)?;
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(context) = evidence_context.as_ref() {
|
||||
write_g09_evidence(
|
||||
context,
|
||||
|
||||
@@ -1786,7 +1786,7 @@ where
|
||||
}
|
||||
|
||||
pub async fn update_user_secret_key(&self, access_key: &str, secret_key: &str) -> Result<(OffsetDateTime, AccountStatus)> {
|
||||
self.update_user_secret_key_with_guard(access_key, secret_key, None)
|
||||
self.update_user_secret_key_with_guard(access_key, secret_key, None, None)
|
||||
.await
|
||||
.map(|(updated_at, status, _guard)| (updated_at, status))
|
||||
}
|
||||
@@ -1796,6 +1796,7 @@ where
|
||||
access_key: &str,
|
||||
secret_key: &str,
|
||||
expected_secret: Option<&str>,
|
||||
session: Option<&Credentials>,
|
||||
) -> Result<(
|
||||
OffsetDateTime,
|
||||
AccountStatus,
|
||||
@@ -1824,6 +1825,51 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
// Lock order is parent, then the presenting STS credential. Deletion and
|
||||
// session refresh use the same STS lock. Release it before the revocation sweep.
|
||||
let _session_guard = match session {
|
||||
Some(session) => {
|
||||
let guard = self.cache.sts_account_mutation_lock(&session.access_key).lock_owned().await;
|
||||
let current = self
|
||||
.api
|
||||
.load_user_identity(&session.access_key, UserType::Sts)
|
||||
.await
|
||||
.map_err(|error| {
|
||||
if is_err_no_such_user(&error) {
|
||||
Error::IAMActionNotAllowed
|
||||
} else {
|
||||
error
|
||||
}
|
||||
})?;
|
||||
let parent = if current.credentials.parent_user.is_empty() {
|
||||
current.credentials.claims_or_empty().get("parent").and_then(Value::as_str)
|
||||
} else {
|
||||
Some(current.credentials.parent_user.as_str())
|
||||
};
|
||||
use subtle::ConstantTimeEq as _;
|
||||
if !session.is_valid()
|
||||
|| !current.credentials.is_valid()
|
||||
|| !current.credentials.is_temp()
|
||||
|| current.credentials.is_service_account()
|
||||
|| current.credentials.access_key != session.access_key
|
||||
|| parent != Some(access_key)
|
||||
|| !bool::from(current.credentials.secret_key.as_bytes().ct_eq(session.secret_key.as_bytes()))
|
||||
|| !bool::from(
|
||||
current
|
||||
.credentials
|
||||
.session_token
|
||||
.as_bytes()
|
||||
.ct_eq(session.session_token.as_bytes()),
|
||||
)
|
||||
|| current.sts_generation != u.sts_generation
|
||||
{
|
||||
return Err(Error::IAMActionNotAllowed);
|
||||
}
|
||||
Some(guard)
|
||||
}
|
||||
None => None,
|
||||
};
|
||||
|
||||
let mut cred = u.credentials.clone();
|
||||
cred.secret_key = secret_key.to_string();
|
||||
|
||||
|
||||
+183
-6
@@ -1421,11 +1421,11 @@ impl<T: Store> IamSys<T> {
|
||||
access_key: &str,
|
||||
secret_key: &str,
|
||||
) -> Result<(OffsetDateTime, rustfs_madmin::AccountStatus, usize)> {
|
||||
self.rotate_user_secret_key_checked(access_key, secret_key, None).await
|
||||
self.rotate_user_secret_key_checked(access_key, secret_key, None, None).await
|
||||
}
|
||||
|
||||
/// Change an enabled regular caller's secret only if its signing secret is
|
||||
/// still current under the parent mutation lock. Status and policies are kept.
|
||||
/// Change an enabled regular user's secret only if the supplied current
|
||||
/// secret still matches under the parent mutation lock. Status and policies are kept.
|
||||
/// Persistence and revocation continue after caller cancellation.
|
||||
pub async fn change_own_secret_key(
|
||||
&self,
|
||||
@@ -1433,7 +1433,27 @@ impl<T: Store> IamSys<T> {
|
||||
expected_secret: &str,
|
||||
secret_key: &str,
|
||||
) -> Result<(OffsetDateTime, rustfs_madmin::AccountStatus, usize)> {
|
||||
self.rotate_user_secret_key_checked(access_key, secret_key, Some(expected_secret))
|
||||
self.rotate_user_secret_key_checked(access_key, secret_key, Some(expected_secret), None)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Change a local IAM password using an authenticated STS session and its parent password.
|
||||
///
|
||||
/// The parent and session mutation locks protect revalidation through the secret write.
|
||||
/// Persistence and session revocation continue after request cancellation.
|
||||
/// A revocation failure is returned after the new secret has been persisted.
|
||||
///
|
||||
/// # Errors
|
||||
/// Returns `IAMActionNotAllowed` for inactive sessions, parent mismatches, or an
|
||||
/// incorrect current password. Storage and revocation errors are propagated.
|
||||
pub async fn change_own_secret_key_from_session(
|
||||
&self,
|
||||
access_key: &str,
|
||||
expected_secret: &str,
|
||||
secret_key: &str,
|
||||
session: &Credentials,
|
||||
) -> Result<(OffsetDateTime, rustfs_madmin::AccountStatus, usize)> {
|
||||
self.rotate_user_secret_key_checked(access_key, secret_key, Some(expected_secret), Some(session))
|
||||
.await
|
||||
}
|
||||
|
||||
@@ -1442,16 +1462,18 @@ impl<T: Store> IamSys<T> {
|
||||
access_key: &str,
|
||||
secret_key: &str,
|
||||
expected_secret: Option<&str>,
|
||||
session: Option<&Credentials>,
|
||||
) -> Result<(OffsetDateTime, rustfs_madmin::AccountStatus, usize)> {
|
||||
let runtime = tokio::runtime::Handle::try_current().map_err(Error::other)?;
|
||||
let iam = self.clone();
|
||||
let access_key = access_key.to_string();
|
||||
let secret_key = secret_key.to_string();
|
||||
let expected_secret = expected_secret.map(str::to_string);
|
||||
let session = session.cloned();
|
||||
#[cfg(test)]
|
||||
let notification_probe = crate::LOAD_USER_NOTIFICATION_PROBE.try_with(Arc::clone).ok();
|
||||
let operation = async move {
|
||||
iam.rotate_user_secret_key_inner(&access_key, &secret_key, expected_secret.as_deref())
|
||||
iam.rotate_user_secret_key_inner(&access_key, &secret_key, expected_secret.as_deref(), session.as_ref())
|
||||
.await
|
||||
};
|
||||
#[cfg(test)]
|
||||
@@ -1471,6 +1493,7 @@ impl<T: Store> IamSys<T> {
|
||||
access_key: &str,
|
||||
secret_key: &str,
|
||||
expected_secret: Option<&str>,
|
||||
session: Option<&Credentials>,
|
||||
) -> Result<(OffsetDateTime, rustfs_madmin::AccountStatus, usize)> {
|
||||
if !is_access_key_valid(access_key) {
|
||||
return Err(IamError::InvalidAccessKeyLength);
|
||||
@@ -1482,7 +1505,7 @@ impl<T: Store> IamSys<T> {
|
||||
|
||||
let (updated_at, status, parent_guard) = self
|
||||
.store
|
||||
.update_user_secret_key_with_guard(access_key, secret_key, expected_secret)
|
||||
.update_user_secret_key_with_guard(access_key, secret_key, expected_secret, session)
|
||||
.await?;
|
||||
let revoked = self.revoke_sts_sessions_for_parent(access_key).await?;
|
||||
drop(parent_guard);
|
||||
@@ -4815,6 +4838,160 @@ mod tests {
|
||||
assert!(matches!(issuer.check_key(&legacy.access_key).await, Ok((None, false))));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn own_secret_change_revalidates_the_presenting_session_under_the_parent_lock() {
|
||||
ensure_test_global_credentials();
|
||||
for mutation in ["revoke", "expire", "disable", "parent", "secret", "token", "generation"] {
|
||||
let store = StsTestMockStore::new(false);
|
||||
let iam = Arc::new(IamSys::new(IamCache::new(store.clone()).await.expect("IAM")));
|
||||
let parent = "session-change-parent";
|
||||
let password = "currentParentSecret123";
|
||||
iam.create_user(
|
||||
parent,
|
||||
&AddOrUpdateUserReq {
|
||||
secret_key: password.to_string(),
|
||||
policy: None,
|
||||
status: rustfs_madmin::AccountStatus::Enabled,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("create parent");
|
||||
let session = rotation_test_session(parent, "password-session");
|
||||
let owner = iam.get_user(parent).await.expect("parent").credentials;
|
||||
iam.set_temp_user_from_parent(&session, &owner)
|
||||
.await
|
||||
.expect("publish session");
|
||||
|
||||
let guard = iam.store.cache.parent_mutation_lock(parent).lock_owned().await;
|
||||
let pending = {
|
||||
let iam = Arc::clone(&iam);
|
||||
let session = session.clone();
|
||||
tokio::spawn(async move {
|
||||
iam.change_own_secret_key_from_session(parent, password, "replacementParentSecret123", &session)
|
||||
.await
|
||||
})
|
||||
};
|
||||
if mutation == "revoke" {
|
||||
iam.delete_temp_account(&session.access_key, false)
|
||||
.await
|
||||
.expect("revoke presenting session");
|
||||
} else {
|
||||
let mut saved = store.saved_sts_users.lock().expect("saved identities");
|
||||
let current = saved.get_mut(&session.access_key).expect("durable session");
|
||||
match mutation {
|
||||
"expire" => current.credentials.expiration = Some(OffsetDateTime::UNIX_EPOCH),
|
||||
"disable" => current.credentials.status = "off".to_string(),
|
||||
"parent" => current.credentials.parent_user = "another-parent".to_string(),
|
||||
"secret" => current.credentials.secret_key = "replacementSessionSecret123".to_string(),
|
||||
"token" => current.credentials.session_token = "replacement-token".to_string(),
|
||||
"generation" => {
|
||||
assert!(current.sts_generation.take().is_some());
|
||||
}
|
||||
_ => unreachable!("known fixture mutation"),
|
||||
}
|
||||
}
|
||||
drop(guard);
|
||||
let error = tokio::time::timeout(std::time::Duration::from_secs(5), pending)
|
||||
.await
|
||||
.expect("queued rotation finishes")
|
||||
.expect("rotation task")
|
||||
.expect_err("stale session must fail");
|
||||
assert_eq!(error, Error::IAMActionNotAllowed, "{mutation}");
|
||||
let saved = store.saved_sts_users.lock().expect("saved identities");
|
||||
assert_eq!(saved.get(parent).expect("parent survives").credentials.secret_key, password, "{mutation}");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn session_secret_change_uses_the_durable_parent_password_with_a_stale_cache() {
|
||||
let iam = test_iam_sys().await;
|
||||
let parent = "stale-password-parent";
|
||||
iam.create_user(
|
||||
parent,
|
||||
&AddOrUpdateUserReq {
|
||||
secret_key: "cachedParentSecret123".to_string(),
|
||||
policy: None,
|
||||
status: rustfs_madmin::AccountStatus::Enabled,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("create parent");
|
||||
let session = rotation_test_session(parent, "fresh-password-session");
|
||||
let owner = iam.get_user(parent).await.expect("cached parent").credentials;
|
||||
iam.set_temp_user_from_parent(&session, &owner)
|
||||
.await
|
||||
.expect("publish session");
|
||||
iam.store
|
||||
.api
|
||||
.saved_sts_users
|
||||
.lock()
|
||||
.expect("saved identities")
|
||||
.get_mut(parent)
|
||||
.expect("durable parent")
|
||||
.credentials
|
||||
.secret_key = "durableParentSecret123".to_string();
|
||||
assert_eq!(
|
||||
iam.store
|
||||
.get_user(parent)
|
||||
.await
|
||||
.expect("stale cached parent")
|
||||
.credentials
|
||||
.secret_key,
|
||||
"cachedParentSecret123"
|
||||
);
|
||||
let probe = IamSys::<StsTestMockStore>::load_user_notification_probe(0, false, false);
|
||||
crate::LOAD_USER_NOTIFICATION_PROBE
|
||||
.scope(
|
||||
probe,
|
||||
iam.change_own_secret_key_from_session(parent, "durableParentSecret123", "replacementParentSecret123", &session),
|
||||
)
|
||||
.await
|
||||
.expect("durable current password authorizes the change");
|
||||
assert_eq!(
|
||||
iam.get_user(parent).await.expect("updated parent").credentials.secret_key,
|
||||
"replacementParentSecret123"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn current_session_can_change_its_parent_secret_and_revoke_itself() {
|
||||
let iam = test_iam_sys().await;
|
||||
let parent = "current-session-parent";
|
||||
let password = "currentParentSecret123";
|
||||
iam.create_user(
|
||||
parent,
|
||||
&AddOrUpdateUserReq {
|
||||
secret_key: password.to_string(),
|
||||
policy: None,
|
||||
status: rustfs_madmin::AccountStatus::Enabled,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("create parent");
|
||||
let session = rotation_test_session(parent, "current-password-session");
|
||||
let owner = iam.get_user(parent).await.expect("parent").credentials;
|
||||
iam.set_temp_user_from_parent(&session, &owner)
|
||||
.await
|
||||
.expect("publish session");
|
||||
let probe = IamSys::<StsTestMockStore>::load_user_notification_probe(0, false, false);
|
||||
let (_, _, revoked) = tokio::time::timeout(
|
||||
std::time::Duration::from_secs(5),
|
||||
crate::LOAD_USER_NOTIFICATION_PROBE.scope(
|
||||
probe,
|
||||
iam.change_own_secret_key_from_session(parent, password, "replacementParentSecret123", &session),
|
||||
),
|
||||
)
|
||||
.await
|
||||
.expect("rotation must release its session lock before revocation")
|
||||
.expect("rotate current session");
|
||||
assert_eq!(revoked, 1);
|
||||
assert!(iam.get_user(&session.access_key).await.is_none());
|
||||
assert_eq!(
|
||||
iam.get_user(parent).await.expect("rotated parent").credentials.secret_key,
|
||||
"replacementParentSecret123"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn self_secret_change_cannot_undo_concurrent_disable_rotation_or_deletion() {
|
||||
ensure_test_global_credentials();
|
||||
|
||||
@@ -1635,94 +1635,6 @@ async fn run_data_scanner_cycle_publishes_activity_for_owner_lifetime() {
|
||||
global_metrics().set_cycle(None).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn single_drive_cycle_persists_usage_and_advances_with_healing_marker() {
|
||||
use crate::scanner_io::{clear_dirty_usage_buckets_for_tests, tests::wait_for_namespace_commit_tails};
|
||||
use crate::storage_api::scan::{HEALING_MARKER_PATH, MakeBucketOptions};
|
||||
|
||||
clear_dirty_usage_buckets_for_tests();
|
||||
let temp_dir = tempfile::tempdir().expect("scanner cycle test directory should be created");
|
||||
let store = setup_scanner_cycle_store_at_path_with_layout(temp_dir.path(), true, 1, 1, 1).await;
|
||||
let bucket = "marked-drive-bucket";
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("fixture bucket should be created");
|
||||
let mut reader = PutObjReader::from_vec(vec![7; 16]);
|
||||
store
|
||||
.put_object(bucket, "object", &mut reader, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("fixture object should persist");
|
||||
wait_for_namespace_commit_tails(store.as_ref()).await;
|
||||
|
||||
let marker = temp_dir
|
||||
.path()
|
||||
.join("pool0/disk0")
|
||||
.join(RUSTFS_META_BUCKET)
|
||||
.join(HEALING_MARKER_PATH);
|
||||
let marker_contents = b"leftover marker";
|
||||
tokio::fs::write(&marker, marker_contents)
|
||||
.await
|
||||
.expect("fixture healing marker should persist");
|
||||
let set = &store.pools[0].disk_set[0];
|
||||
tokio::time::timeout(Duration::from_secs(10), async {
|
||||
while !set.get_online_disks_with_healing(true).await.1 {
|
||||
tokio::time::sleep(Duration::from_millis(100)).await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("disk info should observe the marker after its cache expires");
|
||||
|
||||
let ctx = CancellationToken::new();
|
||||
let mut cycle_info = CurrentCycle {
|
||||
next: 1,
|
||||
..Default::default()
|
||||
};
|
||||
let mut revision = DataUsageCacheRevision::Missing;
|
||||
let leader_epoch = 11;
|
||||
let outcome = tokio::time::timeout(
|
||||
Duration::from_secs(60),
|
||||
run_data_scanner_cycle(&ctx, &store, &mut cycle_info, &mut revision, leader_epoch),
|
||||
)
|
||||
.await
|
||||
.expect("scanner cycle should finish");
|
||||
assert!(
|
||||
matches!(
|
||||
outcome,
|
||||
ScannerCycleOutcome::Completed | ScannerCycleOutcome::CompletedWithPendingMaintenance
|
||||
),
|
||||
"unexpected cycle outcome: {outcome:?}",
|
||||
);
|
||||
|
||||
let mut set_snapshot = DataUsageCache::default();
|
||||
set_snapshot
|
||||
.load_with_revisions(Arc::clone(set), DATA_USAGE_CACHE_NAME)
|
||||
.await
|
||||
.expect("set usage snapshot should be readable");
|
||||
assert!(set_snapshot.info.snapshot_complete);
|
||||
assert_eq!(set_snapshot.info.next_cycle, 1);
|
||||
|
||||
let usage = read_config(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str())
|
||||
.await
|
||||
.expect("global usage snapshot should be durable");
|
||||
let usage: DataUsageInfo = serde_json::from_slice(&usage).expect("global usage snapshot should decode");
|
||||
assert!(usage.usage_snapshot_complete);
|
||||
assert!(usage.last_update.is_some());
|
||||
assert_eq!(usage.buckets_usage[bucket].objects_count, 1);
|
||||
|
||||
let persisted_cycle = read_config(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str())
|
||||
.await
|
||||
.expect("scanner cycle cursor should be durable");
|
||||
let (persisted_cycle, persisted_epoch) =
|
||||
decode_scanner_cycle_state(&persisted_cycle).expect("scanner cycle cursor should decode");
|
||||
assert_eq!(persisted_epoch, leader_epoch);
|
||||
assert_eq!(persisted_cycle.next, 2);
|
||||
assert_eq!(tokio::fs::read(marker).await.expect("healing marker should remain"), marker_contents);
|
||||
clear_dirty_usage_buckets_for_tests();
|
||||
global_metrics().set_cycle(None).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn coordinator_publishes_authority_during_pending_put() {
|
||||
|
||||
@@ -620,10 +620,7 @@ impl ScannerIOCache for SetDisks {
|
||||
.ok_or_else(|| StorageError::other("failed to persist empty scanner set scope"));
|
||||
}
|
||||
|
||||
let single_drive = self.set_drive_count == 1;
|
||||
// A single-drive set has no alternate scan source. A heal marker
|
||||
// still disables scanner heals, but must not exclude its only usage walk.
|
||||
let (disks, healing) = self.get_online_disks_with_healing(single_drive).await;
|
||||
let (disks, healing) = self.get_online_disks_with_healing(false).await;
|
||||
#[cfg(test)]
|
||||
let disks = super::tests::usage_remote_drain::walker_disks(&self, &buckets, disks);
|
||||
if disks.is_empty() {
|
||||
@@ -760,18 +757,7 @@ impl ScannerIOCache for SetDisks {
|
||||
let _ = updates.send(incomplete_scope).await;
|
||||
return Ok(ScannerSourceTurnReceipt::default());
|
||||
}
|
||||
// Coordinator workers need the full inventory only for cross-drive
|
||||
// healing. Remote workers own their disk inventory on the peer.
|
||||
let heal_disk_inventory = if !single_drive
|
||||
&& !healing
|
||||
&& workers
|
||||
.iter()
|
||||
.any(|(_, mode)| *mode == NamespaceScannerWorkerMode::Coordinator)
|
||||
{
|
||||
Some(Arc::new(scanner_set_disk_inventory(self.as_ref()).await))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let set_disk_inventory = Arc::new(scanner_set_disk_inventory(self.as_ref()).await);
|
||||
if unsupported_remote_disks > 0 {
|
||||
debug!(
|
||||
target: "rustfs::scanner::io",
|
||||
@@ -1024,6 +1010,7 @@ impl ScannerIOCache for SetDisks {
|
||||
let store_clone_clone = self.clone();
|
||||
let bucket_result_tx_clone = bucket_result_tx.clone();
|
||||
let bucket_observations_clone = bucket_observations.clone();
|
||||
let set_disk_inventory_clone = set_disk_inventory.clone();
|
||||
let disk_scan_semaphore_clone = disk_scan_semaphore.clone();
|
||||
let queued_disk_bucket_scans_clone = queued_disk_bucket_scans.clone();
|
||||
let active_disk_bucket_scans_clone = active_disk_bucket_scans.clone();
|
||||
@@ -1039,9 +1026,9 @@ impl ScannerIOCache for SetDisks {
|
||||
let worker_retired_buckets = retired_buckets.clone();
|
||||
#[cfg(feature = "e2e-test-hooks")]
|
||||
let worker_incarnations = current_bucket_incarnations.clone();
|
||||
let (remote_capabilities, heal_disk_inventory) = match worker_mode {
|
||||
NamespaceScannerWorkerMode::Coordinator => (None, heal_disk_inventory.clone()),
|
||||
NamespaceScannerWorkerMode::RemoteV4(capabilities) => (Some(capabilities), None),
|
||||
let remote_capabilities = match worker_mode {
|
||||
NamespaceScannerWorkerMode::RemoteV4(capabilities) => Some(capabilities),
|
||||
NamespaceScannerWorkerMode::Coordinator => None,
|
||||
};
|
||||
#[cfg(test)]
|
||||
let worker_index = futs.len();
|
||||
@@ -1684,7 +1671,7 @@ impl ScannerIOCache for SetDisks {
|
||||
let scan = crate::workload_admission::with_scanner_node_requests(node_tx, disk.clone().nsscanner_disk(
|
||||
scan_ctx.clone(),
|
||||
bucket_budget.clone(),
|
||||
heal_disk_inventory.as_deref().cloned().unwrap_or_default(),
|
||||
set_disk_inventory_clone.as_ref().clone(),
|
||||
cache,
|
||||
None,
|
||||
ScannerDiskScanOptions {
|
||||
|
||||
@@ -422,8 +422,6 @@ pub(crate) mod scan {
|
||||
#[cfg(test)]
|
||||
pub(crate) use rustfs_ecstore::api::bucket::metadata_sys::acquire_scanner_bucket_incarnation_fence as ecstore_scanner_ack_fence;
|
||||
#[cfg(test)]
|
||||
pub(crate) use rustfs_ecstore::api::disk::HEALING_MARKER_PATH;
|
||||
#[cfg(test)]
|
||||
pub(crate) use rustfs_ecstore::api::notification::new_global_notification_sys;
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,17 @@
|
||||
# S3 stack upgrade and rollback
|
||||
|
||||
The gateway release serves S3 with rustfs/gateway. Remove `RUSTFS_S3_STACK` from service units, container environments and Helm values before upgrading. Even an empty setting prevents startup. Keep the previous release binary or image digest with the deployment configuration.
|
||||
|
||||
## Roll back to the previous release
|
||||
|
||||
1. Check the target release against the [storage format version floors](rolling-restart.md#version-floors-that-break-mixed-version-fleets). If the cluster has written data or metadata that the target cannot read, stop here. The S3 stack switch itself requires no data migration.
|
||||
2. Stop one node, restore the previous RustFS binary or image and start it with the same volumes and configuration. Follow the [rolling restart procedure](rolling-restart.md#rolling-restart-procedure). Wait for the node to report ready and for its peers to report its drives online before replacing another node.
|
||||
3. Through the restored node, read an object written after the upgrade and compare its contents or checksum. Read a bucket configuration used by the application. If an upload was in progress, list its parts and compare their numbers, sizes and ETags with the client's record. Pause the rollout if any check fails.
|
||||
|
||||
## Client-visible error change
|
||||
|
||||
| Conformance case | What clients see |
|
||||
| --- | --- |
|
||||
| `c-sig-0598` (R-053) | For an access key the S3 parser cannot read, both versions return HTTP 403. The gateway returns the XML error code `AccessDenied`. The legacy binary returns `InvalidAccessKeyId`. Clients that classify these errors by code need to handle both during a rollout. |
|
||||
|
||||
The case is recorded in the [pinned gateway conformance rulings](https://github.com/rustfs/gateway/blob/60dcd2a55713dbcc9dbd6e165922e1a0df4473d7/conformance/rulings/rustfs-profile.toml#L298-L304).
|
||||
@@ -195,27 +195,14 @@ impl Operation for ChangeOwnPasswordHandler {
|
||||
let request: ChangePasswordRequest = serde_json::from_slice(&body)
|
||||
.map_err(|e| s3::error(S3ErrorCode::InvalidRequest, format!("invalid change-password request: {e}")))?;
|
||||
|
||||
let iam_store =
|
||||
current_ready_iam_handle().map_err(|_| s3::error(S3ErrorCode::InternalError, "iam is not initialized"))?;
|
||||
|
||||
let Some(stored) = iam_store.get_user(&caller.access_key).await else {
|
||||
// Reached only if the identity was deleted between authentication
|
||||
// and this lookup.
|
||||
emit_audit(
|
||||
&audit,
|
||||
AccountAuditRecord::failure(
|
||||
AccountAuditOperation::ChangeOwnPassword,
|
||||
&caller.access_key,
|
||||
caller.identity_type,
|
||||
AccountAuditFailure::Internal,
|
||||
),
|
||||
);
|
||||
return Err(s3::error(S3ErrorCode::InvalidRequest, "the calling identity no longer exists"));
|
||||
// An STS session signs with its own secret. A direct IAM request must
|
||||
// have been signed with the same password the caller is replacing.
|
||||
let expected_signing_secret = if caller.identity_type == IdentityType::Sts {
|
||||
&caller.credentials.secret_key
|
||||
} else {
|
||||
&request.current_secret_key
|
||||
};
|
||||
|
||||
if !constant_time_eq(&request.current_secret_key, &stored.credentials.secret_key)
|
||||
|| !constant_time_eq(&request.current_secret_key, &signing_secret)
|
||||
{
|
||||
if !constant_time_eq(expected_signing_secret, &signing_secret) {
|
||||
warn!(
|
||||
component = LOG_COMPONENT_ADMIN,
|
||||
subsystem = LOG_SUBSYSTEM_ACCOUNT,
|
||||
@@ -256,8 +243,10 @@ impl Operation for ChangeOwnPasswordHandler {
|
||||
}
|
||||
|
||||
let access_key = caller.access_key.clone();
|
||||
let new_secret_key = request.new_secret_key.clone();
|
||||
let current_secret_key = request.current_secret_key;
|
||||
let new_secret_key = request.new_secret_key;
|
||||
let identity_type = caller.identity_type;
|
||||
let session_credentials = (identity_type == IdentityType::Sts).then(|| caller.credentials.clone());
|
||||
let session_access_key = caller.session_access_key.clone();
|
||||
let audit_for_task = audit.clone();
|
||||
|
||||
@@ -268,16 +257,35 @@ impl Operation for ChangeOwnPasswordHandler {
|
||||
let iam_store =
|
||||
current_ready_iam_handle().map_err(|_| s3::error(S3ErrorCode::InternalError, "iam is not initialized"))?;
|
||||
|
||||
let (updated_at, status, revoked) = iam_store
|
||||
.change_own_secret_key(&access_key, &signing_secret, &new_secret_key)
|
||||
.await
|
||||
.map_err(|error| {
|
||||
if matches!(error, rustfs_iam::error::Error::IAMActionNotAllowed) {
|
||||
s3::error(S3ErrorCode::InvalidRequest, "the current secret key is incorrect")
|
||||
} else {
|
||||
iam_error_to_s3_error(error)
|
||||
}
|
||||
})?;
|
||||
let rotation = match session_credentials.as_ref() {
|
||||
Some(session) => {
|
||||
iam_store
|
||||
.change_own_secret_key_from_session(&access_key, ¤t_secret_key, &new_secret_key, session)
|
||||
.await
|
||||
}
|
||||
None => {
|
||||
iam_store
|
||||
.change_own_secret_key(&access_key, ¤t_secret_key, &new_secret_key)
|
||||
.await
|
||||
}
|
||||
};
|
||||
let (updated_at, status, revoked) = rotation.map_err(|error| {
|
||||
if matches!(error, rustfs_iam::error::Error::IAMActionNotAllowed) {
|
||||
emit_audit(
|
||||
&audit_for_task,
|
||||
AccountAuditRecord::failure(
|
||||
AccountAuditOperation::ChangeOwnPassword,
|
||||
&access_key,
|
||||
identity_type,
|
||||
AccountAuditFailure::InvalidCurrentSecret,
|
||||
)
|
||||
.with_session_access_key(session_access_key.as_deref()),
|
||||
);
|
||||
s3::error(S3ErrorCode::InvalidRequest, "the current secret key is incorrect")
|
||||
} else {
|
||||
iam_error_to_s3_error(error)
|
||||
}
|
||||
})?;
|
||||
|
||||
emit_audit(
|
||||
&audit_for_task,
|
||||
|
||||
@@ -87,6 +87,8 @@ pub(crate) enum CredentialMutationDenial {
|
||||
ServiceAccount,
|
||||
/// A derived credential whose parent could not be determined.
|
||||
UnresolvedParent,
|
||||
/// A storage delegation has no account-management authority.
|
||||
TableStorageSession,
|
||||
}
|
||||
|
||||
impl CredentialMutationDenial {
|
||||
@@ -97,6 +99,7 @@ impl CredentialMutationDenial {
|
||||
}
|
||||
Self::FederatedIdentity => "federated identities are managed by their identity provider",
|
||||
Self::ServiceAccount => "service account credentials cannot change the credentials of their parent identity",
|
||||
Self::TableStorageSession => "table-scoped storage credentials cannot manage account credentials",
|
||||
Self::UnresolvedParent => "the parent identity of this session could not be resolved",
|
||||
}
|
||||
}
|
||||
@@ -187,7 +190,7 @@ impl CallerIdentity {
|
||||
// root", and an unresolved parent must never fall through to a
|
||||
// permissive answer.
|
||||
//
|
||||
// These three denials apply to both capabilities: a machine credential
|
||||
// These denials apply to both capabilities: a machine credential
|
||||
// must not take over its parent, a federated identity is owned by its
|
||||
// IdP, and an unresolvable parent fails closed.
|
||||
let shared_denial = if unresolved_parent {
|
||||
@@ -196,6 +199,11 @@ impl CallerIdentity {
|
||||
Some(CredentialMutationDenial::ServiceAccount)
|
||||
} else if federated {
|
||||
Some(CredentialMutationDenial::FederatedIdentity)
|
||||
} else if ["rustfs:table-bucket", "rustfs:table-id", "rustfs:credential-scope-prefix"]
|
||||
.iter()
|
||||
.any(|claim| credentials.claims_or_empty().contains_key(*claim))
|
||||
{
|
||||
Some(CredentialMutationDenial::TableStorageSession)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
@@ -501,6 +509,20 @@ mod tests {
|
||||
assert_eq!(caller.mutation_denial, Some(CredentialMutationDenial::FederatedIdentity));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn table_storage_sessions_cannot_manage_parent_credentials() {
|
||||
for claim in ["rustfs:table-bucket", "rustfs:table-id", "rustfs:credential-scope-prefix"] {
|
||||
let mut credentials = sts_session("TABLEKEY", "table-owner");
|
||||
credentials.claims = Some(HashMap::from([(claim.to_string(), Value::String("scope".to_string()))]));
|
||||
let caller = CallerIdentity::from_credentials(credentials, false);
|
||||
|
||||
assert_eq!(caller.mutation_denial, Some(CredentialMutationDenial::TableStorageSession));
|
||||
assert!(!caller.mutability().password);
|
||||
assert!(caller.ensure_credential_mutation_allowed().is_err());
|
||||
assert!(caller.ensure_mfa_management_allowed().is_err());
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn derived_credential_without_a_parent_is_denied_rather_than_allowed() {
|
||||
// Fail closed: an unresolvable parent must not be treated as a
|
||||
|
||||
@@ -1169,18 +1169,7 @@ impl rustfs_protocols::common::client::s3::StorageBackend for ProtocolStorageCli
|
||||
|
||||
let bucket = input.bucket.as_str().to_owned();
|
||||
let key = input.key.as_str().to_owned();
|
||||
let upload_id = gateway_s3s_compat::leaf::upload_id_to_s3s(&input.upload_id);
|
||||
let input = multipart::upload_part_copy_input(input)?;
|
||||
let part_number = input.part_number;
|
||||
let part_number = part_number.to_string();
|
||||
let uri = build_object_uri(
|
||||
&bucket,
|
||||
&key,
|
||||
&[
|
||||
("partNumber", Some(part_number.as_str())),
|
||||
("uploadId", Some(upload_id.as_str())),
|
||||
],
|
||||
)?;
|
||||
let (uri, input) = multipart::prepare_upload_part_copy(input)?;
|
||||
|
||||
let envelope = Self::create_envelope(
|
||||
Method::PUT,
|
||||
@@ -1517,6 +1506,7 @@ mod tests {
|
||||
};
|
||||
use http_body_util::BodyExt;
|
||||
use rustfs_protocols::common::client::s3::StorageBackend;
|
||||
use rustfs_s3_types::S3ErrorCode;
|
||||
let root = rustfs_credentials::Credentials {
|
||||
access_key: "protocol-oracle-root".into(),
|
||||
secret_key: "protocol-oracle-secret".into(),
|
||||
@@ -1528,9 +1518,16 @@ mod tests {
|
||||
let slot = ServerContextSlot::new();
|
||||
assert!(slot.install(context));
|
||||
let backend = ProtocolStorageClient::new(slot);
|
||||
for (bucket, key, data, versioned) in [
|
||||
("protocol-version-oracle", "versioned-key", b"versioned data".as_slice(), true),
|
||||
("protocol-marker-oracle", "folder/", b"".as_slice(), false),
|
||||
for (bucket, key, data, versioned, deleted_get_code) in [
|
||||
(
|
||||
"protocol-version-oracle",
|
||||
"versioned-key",
|
||||
b"versioned data".as_slice(),
|
||||
true,
|
||||
S3ErrorCode::NoSuchKey,
|
||||
),
|
||||
// RustFS 1.0.5 returns NoSuchVersion for a missing directory marker.
|
||||
("protocol-marker-oracle", "folder/", b"".as_slice(), false, S3ErrorCode::NoSuchVersion),
|
||||
] {
|
||||
store
|
||||
.make_bucket(
|
||||
@@ -1593,7 +1590,7 @@ mod tests {
|
||||
.get_object(bucket, key, &root, None)
|
||||
.await
|
||||
.expect_err("protocol GET deleted object");
|
||||
assert_eq!(current.code(), &rustfs_s3_types::S3ErrorCode::NoSuchKey);
|
||||
assert_eq!(current.code(), &deleted_get_code, "GET {bucket}/{key} after DELETE");
|
||||
assert_eq!(current.status_code(), Some(http::StatusCode::NOT_FOUND));
|
||||
if versioned {
|
||||
let headers = current.headers().expect("delete marker error headers");
|
||||
|
||||
@@ -293,9 +293,9 @@ pub(super) fn upload_part_error(error: S3Error) -> S3Error {
|
||||
error
|
||||
}
|
||||
|
||||
pub(super) fn upload_part_copy_input(
|
||||
pub(super) fn prepare_upload_part_copy(
|
||||
input: dto::UploadPartCopyInput,
|
||||
) -> S3Result<crate::app::multipart_usecase::UploadPartCopyWriteInput> {
|
||||
) -> S3Result<(http::Uri, crate::app::multipart_usecase::UploadPartCopyWriteInput)> {
|
||||
super::owned::validate_copy_source(&input.copy_source, "UploadPartCopy copy_source")?;
|
||||
let copy_source_if_match =
|
||||
super::owned::copy_condition(input.copy_source_if_match.as_deref(), "copy_source_if_match", "UploadPartCopy input")?;
|
||||
@@ -314,11 +314,18 @@ pub(super) fn upload_part_copy_input(
|
||||
.copy_source_if_unmodified_since
|
||||
.map(|at| multipart_timestamp(at, "UploadPartCopy input", "copy_source_if_unmodified_since"))
|
||||
.transpose()?;
|
||||
Ok(crate::app::multipart_usecase::UploadPartCopyWriteInput {
|
||||
let upload_id = super::gateway_s3s_compat::leaf::upload_id_to_s3s(&input.upload_id);
|
||||
let part_number = input.part_number.to_string();
|
||||
let uri = super::build_object_uri(
|
||||
input.bucket.as_str(),
|
||||
input.key.as_str(),
|
||||
&[("partNumber", Some(&part_number)), ("uploadId", Some(&upload_id))],
|
||||
)?;
|
||||
let write = crate::app::multipart_usecase::UploadPartCopyWriteInput {
|
||||
target: crate::app::multipart_usecase::UploadPartTarget::Legacy {
|
||||
bucket: input.bucket.as_str().to_owned(),
|
||||
key: input.key.as_str().to_owned(),
|
||||
upload_id: super::gateway_s3s_compat::leaf::upload_id_to_s3s(&input.upload_id),
|
||||
upload_id,
|
||||
},
|
||||
part_number: input.part_number,
|
||||
copy_source: input.copy_source,
|
||||
@@ -333,7 +340,8 @@ pub(super) fn upload_part_copy_input(
|
||||
.copy_source_sse_customer_key
|
||||
.map(|value| value.expose_secret().to_owned()),
|
||||
copy_source_sse_customer_key_md5: nonempty(input.copy_source_sse_customer_key_md5),
|
||||
})
|
||||
};
|
||||
Ok((uri, write))
|
||||
}
|
||||
|
||||
pub(super) fn upload_part_copy_output(
|
||||
|
||||
@@ -422,9 +422,10 @@ fn protocol_upload_copy_keeps_destination_generation_fence() {
|
||||
)
|
||||
.await
|
||||
.expect("replacement bytes");
|
||||
let (_, input) = multipart::prepare_upload_part_copy(request).expect("owned input");
|
||||
let result = backend
|
||||
.owned_multipart_usecase()
|
||||
.execute_upload_part_copy_with_audit(envelope, multipart::upload_part_copy_input(request).expect("owned input"))
|
||||
.execute_upload_part_copy_with_audit(envelope, input)
|
||||
.await;
|
||||
let mut reader = store
|
||||
.get_object_reader(destination, "copy-target", None, HeaderMap::new(), &options)
|
||||
@@ -509,10 +510,12 @@ fn protocol_upload_copy_input_matches_original_codec() {
|
||||
gateway_s3s_compat::generated::ops::upload_part_copy::input_to_s3s(make(), source)
|
||||
.map_err(|error| conversion_error("UploadPartCopy input", error))
|
||||
});
|
||||
let actual = multipart::upload_part_copy_input(make());
|
||||
let actual = multipart::prepare_upload_part_copy(make());
|
||||
match (actual, expected) {
|
||||
(Err(actual), Err(expected)) => assert_eq!(error_facts(actual), error_facts(expected)),
|
||||
(Ok(actual), Ok(expected)) => {
|
||||
(Ok((uri, actual)), Ok(expected)) => {
|
||||
assert_eq!(uri.path(), "/bucket/copy-target");
|
||||
assert_eq!(uri.query(), Some("partNumber=1&uploadId=upload%20%3F%26"));
|
||||
let crate::app::multipart_usecase::UploadPartTarget::Legacy { bucket, key, upload_id } = actual.target else {
|
||||
panic!("protocol client must use the legacy upload target")
|
||||
};
|
||||
@@ -726,7 +729,8 @@ async fn protocol_upload_copy_capabilities_precede_part_validation() {
|
||||
.err()
|
||||
.map(S3Error::from)
|
||||
.expect("legacy refusal");
|
||||
let actual = owned_copy_future(&usecase, envelope, multipart::upload_part_copy_input(make()).expect("owned input"))
|
||||
let (_, input) = multipart::prepare_upload_part_copy(make()).expect("owned input");
|
||||
let actual = owned_copy_future(&usecase, envelope, input)
|
||||
.await
|
||||
.err()
|
||||
.expect("owned refusal");
|
||||
|
||||
@@ -144,59 +144,6 @@ async fn diagnostic_handlers_enforce_advertised_runtime_contract() {
|
||||
assert_eq!(value["measured"], true);
|
||||
assert_eq!(value["rx_bytes"], 7);
|
||||
|
||||
// Exercise the largest declared body allowed by the legacy claimed-route profile.
|
||||
let payload = Bytes::from(vec![b'x'; 1024 * 1024]);
|
||||
let size = payload.len();
|
||||
let response = signed_admin_request(
|
||||
&client,
|
||||
&endpoint,
|
||||
Method::POST,
|
||||
"/minio/admin/v3/speedtest/client/devnull",
|
||||
server.access_key(),
|
||||
server.secret_key(),
|
||||
&sha256_hex(&payload),
|
||||
)
|
||||
.body(payload)
|
||||
.send()
|
||||
.await
|
||||
.expect("signed streaming upload through the MinIO alias");
|
||||
let status = response.status();
|
||||
let body = response.text().await.expect("streaming response");
|
||||
assert!(status.is_success(), "unexpected upload status {status}: {body}");
|
||||
let value: serde_json::Value = serde_json::from_str(&body).expect("streaming upload JSON");
|
||||
assert_eq!(value["rx_bytes"], size);
|
||||
|
||||
let anonymous = client
|
||||
.post(format!("{endpoint}/rustfs/admin/v3/speedtest/client/devnull"))
|
||||
.header("content-length", 1)
|
||||
.body(reqwest::Body::wrap_stream(stream::pending::<Result<Bytes, std::io::Error>>()))
|
||||
.send()
|
||||
.await
|
||||
.expect("anonymous upload must be rejected before reading the body");
|
||||
assert_eq!(anonymous.status(), reqwest::StatusCode::FORBIDDEN);
|
||||
|
||||
let bad_digest = signed_admin_request(
|
||||
&client,
|
||||
&endpoint,
|
||||
Method::POST,
|
||||
"/rustfs/admin/v3/speedtest/client/devnull",
|
||||
server.access_key(),
|
||||
server.secret_key(),
|
||||
&sha256_hex(b"y"),
|
||||
)
|
||||
.body("x")
|
||||
.send()
|
||||
.await
|
||||
.expect("mismatched body must receive a response");
|
||||
assert_eq!(bad_digest.status(), reqwest::StatusCode::BAD_REQUEST);
|
||||
assert!(
|
||||
bad_digest
|
||||
.text()
|
||||
.await
|
||||
.expect("digest error body")
|
||||
.contains("XAmzContentSHA256Mismatch")
|
||||
);
|
||||
|
||||
let oversized = signed_admin_request(
|
||||
&client,
|
||||
&endpoint,
|
||||
@@ -227,11 +174,11 @@ async fn diagnostic_handlers_enforce_advertised_runtime_contract() {
|
||||
DEVNULL_PATH,
|
||||
server.access_key(),
|
||||
server.secret_key(),
|
||||
&sha256_hex(b"xy"),
|
||||
&sha256_hex(b"x"),
|
||||
)
|
||||
.build()
|
||||
.expect("signed stalled upload headers");
|
||||
let mut head = format!("POST {DEVNULL_PATH} HTTP/1.1\r\nContent-Length: 2\r\nExpect: 100-continue\r\n");
|
||||
let mut head = format!("POST {DEVNULL_PATH} HTTP/1.1\r\nContent-Length: 1\r\nExpect: 100-continue\r\n");
|
||||
for (name, value) in request.headers() {
|
||||
write!(head, "{name}: {}\r\n", value.to_str().expect("ASCII signing header")).expect("write request head");
|
||||
}
|
||||
@@ -248,7 +195,6 @@ async fn diagnostic_handlers_enforce_advertised_runtime_contract() {
|
||||
let mut status = String::new();
|
||||
socket.read_line(&mut status).await.expect("read stalled upload status");
|
||||
assert_eq!(status, "HTTP/1.1 100 Continue\r\n");
|
||||
socket.write_all(b"x").await.expect("send partial upload body");
|
||||
stalled.push(socket);
|
||||
}
|
||||
stalled
|
||||
|
||||
@@ -766,13 +766,9 @@ fn the_gateway_stack_swaps_the_tagging_action_for_a_named_version() {
|
||||
with_stack(GATEWAY, the_tagging_action_follows_the_named_version);
|
||||
}
|
||||
|
||||
/// What a stack answers for a key outside the floor, independent of the operation and bucket.
|
||||
type UniformAnswers = (Answer, Answer);
|
||||
|
||||
/// Checks boundary keys across bucket configurations. The legacy stack answers per operation and
|
||||
/// bucket from the table below; the gateway stack refuses the key before the bucket is consulted,
|
||||
/// so it answers every row with `uniform`: `(NUL key, 1025-byte key)`.
|
||||
async fn keys_outside_the_floor_are_refused(server: rustfs::embedded::RustFSServer, uniform: Option<UniformAnswers>) {
|
||||
/// The gateway rejects boundary keys before consulting the bucket. RustFS serves no torrents,
|
||||
/// so a NUL key on GetObjectTorrent retains the release's NoSuchKey response.
|
||||
async fn keys_outside_the_floor_are_refused(server: rustfs::embedded::RustFSServer) {
|
||||
const BUCKET: &str = "object-meta-keys";
|
||||
let client = s3_client(&server.endpoint(), server.access_key(), server.secret_key());
|
||||
create_bucket_in_the_store(BUCKET).await;
|
||||
@@ -782,38 +778,25 @@ async fn keys_outside_the_floor_are_refused(server: rustfs::embedded::RustFSServ
|
||||
assert!(super::bucket_exists_in_the_store(reserved).await);
|
||||
put_object_in_the_store(reserved, KEY, HashMap::new()).await;
|
||||
assert_eq!(answer(client.get_object_acl().bucket(reserved).key(KEY).send().await), (None, None, None));
|
||||
let refused = |code: &str, status, message: &str| (Some(code.to_owned()), Some(status), Some(message.to_owned()));
|
||||
let invalid_nul = refused("InvalidArgument", 400, "the name contains a NUL byte");
|
||||
let missing_torrent = refused("NoSuchKey", 404, "The specified key does not exist.");
|
||||
let too_long = refused("KeyTooLongError", 400, "the name is longer than the maximum length in bytes");
|
||||
let long_key = "k".repeat(1025);
|
||||
for (operation, call) in KEY_CALLS {
|
||||
for (bucket, kind) in [(BUCKET, 0), (locked.as_str(), 1), ("object-meta-missing", 2), (reserved, 0)] {
|
||||
let (code, status, message) = match (*operation, kind) {
|
||||
("GetObjectTorrent", _) => ("NoSuchKey", 404, "The specified key does not exist."),
|
||||
("GetObjectLegalHold", 0) | ("GetObjectRetention", 0) => {
|
||||
("InvalidRequest", 400, "Bucket is missing ObjectLockConfiguration")
|
||||
}
|
||||
("GetObjectLegalHold", 1) | ("GetObjectRetention", 1) => {
|
||||
("InternalError", 500, "XML response contains a control character")
|
||||
}
|
||||
("GetObjectLegalHold", _) => ("NoSuchBucket", 404, "The specified bucket does not exist"),
|
||||
("GetObjectRetention", _) => ("InternalError", 500, "Failed to load Object Lock configuration"),
|
||||
_ => ("InvalidArgument", 400, "Invalid argument"),
|
||||
};
|
||||
let expected = match &uniform {
|
||||
Some((nul, _)) => nul.clone(),
|
||||
None => (Some(code.to_owned()), Some(status), Some(message.to_owned())),
|
||||
};
|
||||
assert_eq!(call(&client, bucket, "a\0b").await, expected, "{operation} bucket {bucket}");
|
||||
}
|
||||
let expected = match &uniform {
|
||||
Some((_, long)) => long.clone(),
|
||||
None => (Some("KeyTooLongError".to_owned()), Some(400), Some("Your key is too long.".to_owned())),
|
||||
let expected_nul = if *operation == "GetObjectTorrent" {
|
||||
&missing_torrent
|
||||
} else {
|
||||
&invalid_nul
|
||||
};
|
||||
assert_eq!(call(&client, BUCKET, &long_key).await, expected, "{operation} long key");
|
||||
for bucket in [BUCKET, locked.as_str(), "object-meta-missing", reserved] {
|
||||
assert_eq!(&call(&client, bucket, "a\0b").await, expected_nul, "{operation} bucket {bucket}");
|
||||
}
|
||||
assert_eq!(call(&client, BUCKET, &long_key).await, too_long, "{operation} long key");
|
||||
}
|
||||
server.shutdown().await;
|
||||
}
|
||||
|
||||
type Answer = (Option<String>, Option<u16>, Option<String>);
|
||||
|
||||
type KeyCall = for<'a> fn(&'a Client, &'a str, &'a str) -> BoxFuture<'a, (Option<String>, Option<u16>, Option<String>)>;
|
||||
|
||||
const KEY_CALLS: &[(&str, KeyCall)] = &[
|
||||
@@ -853,12 +836,7 @@ fn answer<T, E: ProvideErrorMetadata>(result: Result<T, SdkError<E>>) -> (Option
|
||||
|
||||
#[test]
|
||||
fn the_gateway_stack_refuses_keys_outside_the_floor() {
|
||||
let refused = |code: &str, message: &str| (Some(code.to_owned()), Some(400), Some(message.to_owned()));
|
||||
let uniform = (
|
||||
refused("InvalidArgument", "the name contains a NUL byte"),
|
||||
refused("KeyTooLongError", "the name is longer than the maximum length in bytes"),
|
||||
);
|
||||
with_stack(GATEWAY, move |server| keys_outside_the_floor_are_refused(server, Some(uniform)));
|
||||
with_stack(GATEWAY, keys_outside_the_floor_are_refused);
|
||||
}
|
||||
|
||||
fn legal_hold(status: &str) -> aws_sdk_s3::types::ObjectLockLegalHold {
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Exercise both pinned upgrade sources without building RustFS."""
|
||||
"""Exercise the pinned upgrade sources without building RustFS."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -10,6 +10,7 @@ import re
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import tomllib
|
||||
import unittest
|
||||
|
||||
from check_test_wiring import yaml_block
|
||||
@@ -17,6 +18,12 @@ from test_security_workflow import named_steps, shell_body
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
SOURCE_CASES = {
|
||||
"1.0.0-rc.5": ("rc5_rollback_requires_restoring_odm_configuration",
|
||||
"rc5_checkpoint_upgrade_rewinds_signed_progress_without_losing_objects"),
|
||||
"1.0.1": ("stable_upgrade_deletes_foreign_reservations_without_losing_live_objects",),
|
||||
"1.0.5": (),
|
||||
}
|
||||
|
||||
|
||||
class UpgradeWorkflowTests(unittest.TestCase):
|
||||
@@ -28,19 +35,16 @@ class UpgradeWorkflowTests(unittest.TestCase):
|
||||
# Explicit bash enables pipefail in Actions, so tee cannot hide a failure.
|
||||
self.assertIn(" shell: bash", self.step)
|
||||
self.assertIn(" fail-fast: false", self.job)
|
||||
self.assertEqual(re.findall(r"- version: ([^\s]+)", "\n".join(self.job)), ["1.0.0-rc.5", "1.0.1"])
|
||||
self.assertEqual(re.findall(r"- version: ([^\s]+)", "\n".join(self.job)), list(SOURCE_CASES))
|
||||
rust = (ROOT / "crates/e2e_test/src/upgrade_compatibility_test.rs").read_text()
|
||||
self.expected = set(re.findall(r'#\[ignore[^\n]*\]\s*async fn (\w+)\(', rust))
|
||||
self.assertTrue(self.expected)
|
||||
|
||||
def run_cases(self, version, failed_test="", fail_tee=False):
|
||||
excluded = ({"stable_upgrade_deletes_foreign_reservations_without_losing_live_objects"}
|
||||
if version == "1.0.0-rc.5" else {
|
||||
"rc5_rollback_requires_restoring_odm_configuration",
|
||||
"rc5_checkpoint_upgrade_rewinds_signed_progress_without_losing_objects",
|
||||
})
|
||||
expected = self.expected - excluded
|
||||
self.assertEqual(len(expected), 8 if version == "1.0.0-rc.5" else 7)
|
||||
shared = self.expected.difference(*SOURCE_CASES.values())
|
||||
self.assertEqual(len(shared), 6)
|
||||
expected = shared | set(SOURCE_CASES[version])
|
||||
self.assertTrue(expected <= self.expected)
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
root = Path(directory)
|
||||
commands = root / "commands.jsonl"
|
||||
@@ -82,14 +86,14 @@ sys.exit(17 if test == os.environ["FAILED_TEST"] else 0)
|
||||
return result
|
||||
|
||||
def test_all_ignored_upgrade_cases_run_once_with_separate_evidence(self):
|
||||
for version in ("1.0.0-rc.5", "1.0.1"):
|
||||
for version in SOURCE_CASES:
|
||||
with self.subTest(version=version):
|
||||
result = self.run_cases(version)
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
|
||||
def test_early_or_late_failure_does_not_hide_remaining_cases(self):
|
||||
for version, last in (("1.0.0-rc.5", "rc5_checkpoint_upgrade_rewinds_signed_progress_without_losing_objects"),
|
||||
("1.0.1", "stable_upgrade_deletes_foreign_reservations_without_losing_live_objects")):
|
||||
for version, specialized in SOURCE_CASES.items():
|
||||
last = specialized[-1] if specialized else "previous_release_baseline_replicates_multipart_layouts"
|
||||
for name in ("direct_upgrade_from_previous_release_preserves_object_contracts", last):
|
||||
with self.subTest(version=version, name=name):
|
||||
result = self.run_cases(version, failed_test=name)
|
||||
@@ -97,9 +101,50 @@ sys.exit(17 if test == os.environ["FAILED_TEST"] else 0)
|
||||
self.assertIn(f"::error::Upgrade compatibility failed: {name}", result.stdout)
|
||||
|
||||
def test_log_write_failure_fails_the_job(self):
|
||||
for version in ("1.0.0-rc.5", "1.0.1"):
|
||||
for version in SOURCE_CASES:
|
||||
self.assertNotEqual(self.run_cases(version, fail_tee=True).returncode, 0)
|
||||
|
||||
def test_official_reference_matches_the_workflow_and_release_plan(self):
|
||||
reference = tomllib.loads((ROOT / ".config/reference-binary.toml").read_text())
|
||||
platform = reference["platforms"]["linux-x86_64-gnu"]
|
||||
pin = ("1.0.5", platform["asset"], platform["sha256"])
|
||||
self.assertEqual(reference["version"], pin[0])
|
||||
matrix = re.findall(r"- version: (\S+)\s+asset: (\S+)\s+sha256: (\w+)", "\n".join(self.job))
|
||||
self.assertIn(pin, matrix)
|
||||
plan = json.loads((ROOT / ".config/release-validation-plan.json").read_text())
|
||||
self.assertEqual(set(matrix), {(s["version"], s["artifact_name"], s["artifact_sha256"])
|
||||
for s in plan["upgrade_sources"]["sources"]})
|
||||
legacy_mpu = "\n".join(self.steps["Run mixed-version MPU recovery"])
|
||||
self.assertIn("matrix.source.version != '1.0.5'", legacy_mpu)
|
||||
self.assertIn("mixed_version_failed_complete_requires_current_coordinator_for_abort", legacy_mpu)
|
||||
|
||||
def test_gateway_evidence_requires_both_roles_and_rollback_for_the_pinned_pair(self):
|
||||
step = self.steps["Verify gateway baseline G09 evidence"]
|
||||
self.assertIn("matrix.source.version == '1.0.5'", "\n".join(step))
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
root = Path(directory) / "rustfs-upgrade-g09-evidence"
|
||||
env = dict(os.environ, RUNNER_TEMP=directory, GITHUB_SHA="c" * 40, UPGRADE_SOURCE_REVISION="a" * 40)
|
||||
for field in ("mixed_version_reader_evidence", "mixed_version_writer_evidence", "rollback_payload_evidence"):
|
||||
folder = root / ("bucket-config-rollback" if field == "rollback_payload_evidence" else "mixed-version-upgrade")
|
||||
folder.mkdir(exist_ok=True, parents=True)
|
||||
payload = dict(gate="G09", field=field, current_revision=env["GITHUB_SHA"],
|
||||
previous_revision=env["UPGRADE_SOURCE_REVISION"], rollback_payload_replayed=True)
|
||||
(folder / f"G09-{field}.json").write_text(json.dumps(payload))
|
||||
multipart = root / "bucket-config-rollback/inflight-multipart-rollback.json"
|
||||
payload = dict(current_revision=env["GITHUB_SHA"], previous_revision=env["UPGRADE_SOURCE_REVISION"],
|
||||
part_sizes=[5242880, 5242880, 4096], bytes=10489856,
|
||||
rollback_read_sha256="d" * 64, roll_forward_read_sha256="d" * 64)
|
||||
multipart.write_text(json.dumps(payload))
|
||||
command = ["bash", "-e", "-o", "pipefail", "-c", shell_body(step)]
|
||||
self.assertEqual(subprocess.run(command, env=env, capture_output=True).returncode, 0)
|
||||
wrong_pair = dict(env, UPGRADE_SOURCE_REVISION="b" * 40)
|
||||
self.assertNotEqual(subprocess.run(command, env=wrong_pair, capture_output=True).returncode, 0)
|
||||
multipart.write_text(json.dumps(dict(payload, roll_forward_read_sha256="e" * 64)))
|
||||
self.assertNotEqual(subprocess.run(command, env=env, capture_output=True).returncode, 0)
|
||||
multipart.write_text(json.dumps(payload))
|
||||
(root / "mixed-version-upgrade/G09-mixed_version_reader_evidence.json").unlink()
|
||||
self.assertNotEqual(subprocess.run(command, env=env, capture_output=True).returncode, 0)
|
||||
|
||||
def test_reports_are_uploaded_even_after_failure(self):
|
||||
for name, directory in (("Upload scanner/heal G09 evidence", "rustfs-upgrade-g09-evidence"),
|
||||
("Upload server logs", "rustfs-upgrade-logs")):
|
||||
|
||||
Reference in New Issue
Block a user