Compare commits

..

16 Commits

Author SHA1 Message Date
overtrue 6e417a458e test(upgrade): exercise the official 1.0.5 gateway baseline 2026-11-20 14:50:43 +08:00
overtrue 60ed7f1c40 test(upgrade): select mixed storage contracts by source revision 2026-11-20 14:32:11 +08:00
overtrue 7f2bd6b99b test(protocols): match the released error for a missing directory marker 2026-11-20 14:23:19 +08:00
overtrue a798df7906 refactor(protocols): prepare part copy routing and storage from one upload ID 2026-11-20 14:23:19 +08:00
overtrue 793632c0da docs(operations): document rollback after the S3 stack switch 2026-11-20 14:23:18 +08:00
overtrue 301133c1c8 test(gateway): check the release torrent error for boundary keys 2026-11-20 14:23:18 +08:00
overtrue a642ba2a89 chore(ci): tighten the s3s import limit to 33 files 2026-11-20 13:04:07 +08:00
overtrue d893d70b2e refactor(storage): return native errors from S3 storage helpers 2026-11-20 13:03:42 +08:00
overtrue cf15e4d1b6 refactor(http): verify control body checksums with the native hashers 2026-11-20 12:35:23 +08:00
overtrue 5a3e94bbbb chore(ci): ratchet the remaining s3s imports to 34 files 2026-11-20 12:21:40 +08:00
overtrue 29b389bf80 refactor(errors): keep API error codes native through storage handling 2026-11-20 12:21:15 +08:00
overtrue f94e8cae07 refactor(storage): use native listing and object lock inputs 2026-11-20 12:17:42 +08:00
overtrue 64675785fe refactor(bucket): return native S3 errors from admission and config writes 2026-11-20 12:01:48 +08:00
overtrue 5b703b61f9 test(ci): align storage guard fixtures with the surviving helper rule 2026-11-20 12:00:53 +08:00
overtrue 23420e357d refactor(multipart): use native S3 errors for upload validation and storage failures 2026-11-20 11:43:15 +08:00
overtrue 9da1edd8d6 test(e2e): compare complete persisted version maps 2026-11-20 11:39:25 +08:00
66 changed files with 923 additions and 886 deletions
+2 -2
View File
@@ -1,2 +1,2 @@
sha256-darwin=3cf5f1857837aabc5d00d261dc40a36ef0a87c4c8b40cc5b705c95477f8e4993
sha256-linux=6f56cfda9326f5321c2690bfb0232e737d5660e99813fa33460a0a800d6c94af
sha256-darwin=9ccb03f4139e99ec11a63810523d504df39191b86e74b28a72dd9a075682293b
sha256-linux=5ef052b03a03afbd00ebfcbbdcb754bbde414b7e6151aae3e2ee78024ce1d0b6
+8 -2
View File
@@ -114,8 +114,14 @@
}
],
"upgrade_sources": {
"basis": "Exactly the two previous-release sources pinned in .github/workflows/e2e-upgrade.yml are exercised today (source.version / source.asset / source.sha256). No other N-1/N-2 compatibility is claimed, and these digests are not inferred from tags.",
"basis": "The three previous-release sources pinned in .github/workflows/e2e-upgrade.yml are exercised today (source.version / source.asset / source.sha256). No other N-1/N-2 compatibility is claimed, and these digests are not inferred from tags.",
"sources": [
{
"version": "1.0.5",
"role": "gateway compatibility baseline",
"artifact_name": "rustfs-linux-x86_64-gnu-v1.0.5.zip",
"artifact_sha256": "5addae2f74756bf80104d0975e6347cda843bc2daffdfcd1f0cb80118b95cbd0"
},
{
"version": "1.0.1",
"role": "N-1",
@@ -130,6 +136,6 @@
}
],
"rollback_supported": false,
"rollback_basis": "Only the bucket-config-rollback case (rollback_to_previous_release_reads_current_bucket_metadata) is exercised. Full data-plane rollback is not certified and is never a default claim."
"rollback_basis": "The release upgrade lane checks bucket-config rollback. The hosted compatibility workflow also checks in-flight multipart rollback to 1.0.5. Full data-plane rollback is not certified and is never a default claim."
}
}
+2 -2
View File
@@ -13,6 +13,6 @@
# s3_error_lines lines holding s3_error!, same scope, minus those the script
# header's rule clears (rustfs_s3_types's macro); fails closed
# ecstore_files Rust files referencing s3s paths under crates/ecstore/src
files=39
s3_error_lines=90
files=33
s3_error_lines=89
ecstore_files=0
+39 -7
View File
@@ -90,11 +90,16 @@ jobs:
- version: 1.0.1
asset: rustfs-linux-x86_64-gnu-v1.0.1.zip
sha256: 9742e183cb0dfb0d48fa909797af572fe47b7f827569036baa4156e0d3958ee5
- version: 1.0.5
asset: rustfs-linux-x86_64-gnu-v1.0.5.zip
sha256: 5addae2f74756bf80104d0975e6347cda843bc2daffdfcd1f0cb80118b95cbd0
revision: 34e6449ed90e3d3a106f6655342e0b3127f33082
env:
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
UPGRADE_SOURCE_VERSION: ${{ matrix.source.version }}
UPGRADE_SOURCE_ASSET: ${{ matrix.source.asset }}
UPGRADE_SOURCE_SHA256: ${{ matrix.source.sha256 }}
UPGRADE_SOURCE_REVISION: ${{ matrix.source.revision }}
steps:
- name: Checkout repository
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
@@ -132,6 +137,13 @@ jobs:
unzip -q "$archive" -d "$SOURCE_DIR"
chmod +x "$SOURCE_DIR/rustfs"
test -x "$SOURCE_DIR/rustfs"
if [[ -n "$UPGRADE_SOURCE_REVISION" ]]; then
mkdir -p "$RUNNER_TEMP/rustfs-upgrade-logs"
version_log="$RUNNER_TEMP/rustfs-upgrade-logs/source-version.txt"
"$SOURCE_DIR/rustfs" --version | tee "$version_log"
grep -Fx "rustfs $UPGRADE_SOURCE_VERSION" "$version_log"
grep -Eq "^git commit[[:space:]]*:[[:space:]]*$UPGRADE_SOURCE_REVISION$" "$version_log"
fi
echo "RUSTFS_UPGRADE_SOURCE_BINARY=$SOURCE_DIR/rustfs" >> "$GITHUB_ENV"
echo "RUSTFS_E2E_LOG_DIR=$RUNNER_TEMP/rustfs-upgrade-logs" >> "$GITHUB_ENV"
@@ -151,12 +163,14 @@ jobs:
multipart-layout-upgrade:direct_upgrade_from_previous_release_preserves_multipart_layouts
multipart-layout-baseline:previous_release_baseline_replicates_multipart_layouts
)
if [[ "$UPGRADE_SOURCE_VERSION" == "1.0.0-rc.5" ]]; then
cases+=(odm-config-rollback:rc5_rollback_requires_restoring_odm_configuration)
cases+=(checkpoint-upgrade:rc5_checkpoint_upgrade_rewinds_signed_progress_without_losing_objects)
else
cases+=(legacy-delete-reservation:stable_upgrade_deletes_foreign_reservations_without_losing_live_objects)
fi
case "$UPGRADE_SOURCE_VERSION" in
1.0.0-rc.5)
cases+=(odm-config-rollback:rc5_rollback_requires_restoring_odm_configuration)
cases+=(checkpoint-upgrade:rc5_checkpoint_upgrade_rewinds_signed_progress_without_losing_objects) ;;
1.0.1) cases+=(legacy-delete-reservation:stable_upgrade_deletes_foreign_reservations_without_losing_live_objects) ;;
1.0.5) ;; # The bucket rollback case also completes an in-flight multipart upload.
*) echo "::error::Unknown upgrade source $UPGRADE_SOURCE_VERSION"; exit 1 ;;
esac
status=0
for entry in "${cases[@]}"; do
artifact="${entry%%:*}"
@@ -178,7 +192,7 @@ jobs:
exit "$status"
- name: Run mixed-version MPU recovery
if: ${{ !cancelled() && steps.verify_source.outcome == 'success' }}
if: ${{ !cancelled() && steps.verify_source.outcome == 'success' && matrix.source.version != '1.0.5' }}
timeout-minutes: 15
env:
NO_PROXY: 127.0.0.1,localhost,::1
@@ -199,6 +213,24 @@ jobs:
"$mpu_test" -- --exact --nocapture --test-threads=1 \
2>&1 | tee "$RUSTFS_E2E_LOG_DIR/test.log"
- name: Verify gateway baseline G09 evidence
if: ${{ success() && matrix.source.version == '1.0.5' }}
shell: bash
run: |
for field in mixed_version_reader_evidence mixed_version_writer_evidence rollback_payload_evidence; do
directory=mixed-version-upgrade
if [[ "$field" == rollback_payload_evidence ]]; then directory=bucket-config-rollback; fi
jq -e --arg field "$field" --arg current "$GITHUB_SHA" --arg previous "$UPGRADE_SOURCE_REVISION" \
'.gate == "G09" and .field == $field and .current_revision == $current and .previous_revision == $previous
and ($field != "rollback_payload_evidence" or .rollback_payload_replayed == true)' \
"$RUNNER_TEMP/rustfs-upgrade-g09-evidence/$directory/G09-$field.json"
done
jq -e --arg current "$GITHUB_SHA" --arg previous "$UPGRADE_SOURCE_REVISION" \
'.current_revision == $current and .previous_revision == $previous
and .part_sizes == [5242880, 5242880, 4096] and .bytes == (.part_sizes | add)
and (.rollback_read_sha256 | test("^[a-f0-9]{64}$")) and .rollback_read_sha256 == .roll_forward_read_sha256' \
"$RUNNER_TEMP/rustfs-upgrade-g09-evidence/bucket-config-rollback/inflight-multipart-rollback.json"
- name: Upload scanner/heal G09 evidence
if: always()
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+1 -1
View File
@@ -13,7 +13,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Deployment
- **The S3 protocol stack is now rustfs/gateway**, and it is the only S3 stack. The legacy s3s-based stack is removed, and so are the `RUSTFS_S3_STACK` environment variable and the `gateway-stack` cargo feature. There is no on-disk format change. `RUSTFS_S3_STACK` no longer selects anything: a server started with it set, to any value, refuses to start and prints `RUSTFS_S3_STACK is no longer supported; the gateway stack is the only S3 stack since <version>`, where `<version>` is the release's version. Remove the variable from service units, container environments, Helm values and orchestration files before upgrading. To roll back, run the previous release binary on the same data.
- **The S3 protocol stack is now rustfs/gateway**, and it is the only S3 stack. The legacy s3s-based stack is removed, and so are the `RUSTFS_S3_STACK` environment variable and the `gateway-stack` cargo feature. There is no on-disk format change. `RUSTFS_S3_STACK` no longer selects anything: a server started with it set, to any value, refuses to start and prints `RUSTFS_S3_STACK is no longer supported; the gateway stack is the only S3 stack since <version>`, where `<version>` is the release's version. Remove the variable from service units, container environments, Helm values and orchestration files before upgrading. To roll back, follow the [S3 stack rollback procedure](docs/operations/s3-stack-rollback.md).
- Before upgrading, verify that the configured root access key is at least 3 bytes and the root secret key is at least 8 bytes. RustFS now rejects shorter values at startup and identifies the setting to update. Version 1.0.5 accepted these settings at startup and returned HTTP 500 when root requested STS credentials. Coordinate root credential changes across all nodes and clients. See [root credential rotation considerations](docs/operations/two-factor-auth.md#root-credentials-cannot-be-changed-at-runtime).
Generated
+2
View File
@@ -4238,6 +4238,7 @@ dependencies = [
"s3s",
"same-file",
"serde",
"serde-value",
"serde_json",
"serde_urlencoded",
"sha1 0.11.0",
@@ -11093,6 +11094,7 @@ name = "rustfs-protos"
version = "1.0.5"
dependencies = [
"flatbuffers",
"hotpath",
"hyper",
"hyper-util",
"prost 0.14.6",
+1
View File
@@ -222,6 +222,7 @@ quick-xml = "0.42.0"
rmp = { version = "0.8.15" }
rmp-serde = { version = "1.3.1" }
serde = { version = "1.0.229" }
serde-value = "0.7.0"
serde_ignored = { version = "0.1" }
serde_with = { version = "3", default-features = false, features = ["macros", "std"] }
serde_json = { version = "1.0.154" }
-3
View File
@@ -32,9 +32,6 @@ pub const DEFAULT_HEAL_BITROT_CYCLE_SECS: u64 = 30 * 24 * 60 * 60;
/// - Note: Enabling auto-heal can improve system resilience by automatically addressing issues, but may increase resource usage; evaluate based on your operational requirements.
pub const ENV_HEAL_AUTO_HEAL_ENABLE: &str = "RUSTFS_HEAL_AUTO_HEAL_ENABLE";
/// Colon-separated Linux mount roots for automatic replacement of data directories below a mount point.
pub const ENV_HEAL_REPLACEMENT_MOUNT_ROOTS: &str = "RUSTFS_HEAL_REPLACEMENT_MOUNT_ROOTS";
/// Environment variable name that specifies the heal queue size.
///
/// - Purpose: Set the maximum number of heal requests that can be queued.
+1
View File
@@ -138,6 +138,7 @@ rustls = { workspace = true, default-features = false, features = ["aws-lc-rs",
clap = { workspace = true, features = ["derive", "env"] }
[dev-dependencies]
serde-value.workspace = true
s3s = { workspace = true, features = ["minio"] }
async-compression = { workspace = true, features = ["tokio", "bzip2", "lz4", "xz"] }
aws-config = { workspace = true }
+26 -14
View File
@@ -31,7 +31,7 @@ use super::harness::{
use crate::common::{
AdminTransport, admin_add_canned_policy_via, admin_attach_user_policy_via, admin_create_user_via, init_logging,
};
use crate::upgrade_compatibility_test::wait_for_upgrade_write_readiness;
use crate::upgrade_compatibility_test::{UpgradeSourceCapabilities, wait_for_upgrade_write_readiness};
use aws_sdk_s3::Client;
use aws_sdk_s3::error::ProvideErrorMetadata;
use std::path::{Path, PathBuf};
@@ -427,7 +427,12 @@ async fn assert_history_and_iam(dist: &DistCluster, seed: &UpgradeSeed, context:
Ok(())
}
async fn compare_degraded_legacy_list_parts(dist: &mut DistCluster, seed: &UpgradeSeed, previous: &Path) -> TestResult {
async fn compare_degraded_legacy_list_parts(
dist: &mut DistCluster,
seed: &UpgradeSeed,
previous: &Path,
source: &UpgradeSourceCapabilities,
) -> TestResult {
let decoded = String::from_utf8(base64_simd::URL_SAFE_NO_PAD.decode_to_vec(seed.multipart_upload.as_bytes())?)?;
let (_, upload_dir) = decoded.split_once('.').ok_or("legacy upload ID lacks deployment delimiter")?;
let hash = sha256_hex(format!("{}/legacy-multipart.bin", seed.history_bucket).as_bytes());
@@ -489,16 +494,22 @@ async fn compare_degraded_legacy_list_parts(dist: &mut DistCluster, seed: &Upgra
// Three online disks retain metadata quorum but have no part directory;
// the fourth disk is offline. A legacy empty response is not a proof that
// the unavailable participant has no part.
let legacy = legacy.expect("previous coordinator should expose its legacy empty-list behavior");
assert!(legacy.parts().is_empty());
let current = current.expect_err("current coordinator must not infer empty from missing directories and offline disks");
assert_eq!(current.raw_response().expect("current S3 response").status().as_u16(), 503);
// Since #9149, SlowDownRead is reserved for GET/HEAD object reads; a
// ListParts read-quorum failure is reported as ServiceUnavailable.
assert_eq!(
current.as_service_error().and_then(|error| error.meta().code()),
Some("ServiceUnavailable")
);
let mut quorum_errors =
vec![current.expect_err("current coordinator must not infer empty from missing directories and offline disks")];
if source.requires_list_parts_read_quorum {
quorum_errors.push(legacy.expect_err("previous coordinator must require evidence from unavailable disks"));
} else {
assert!(
legacy
.expect("previous coordinator should expose its legacy empty-list behavior")
.parts()
.is_empty()
);
}
for error in quorum_errors {
assert_eq!(error.raw_response().expect("S3 response").status().as_u16(), 503);
assert_eq!(error.as_service_error().and_then(|error| error.meta().code()), Some("ServiceUnavailable"));
}
dist.cluster.start_node_from_binary(3, previous).await?;
wait_for_upgrade_write_readiness(&clients, &seed.history_bucket, "mixed-list-restored", CREDENTIAL_TIMEOUT).await?;
for (path, before) in original_files {
@@ -520,7 +531,7 @@ async fn compare_degraded_legacy_list_parts(dist: &mut DistCluster, seed: &Upgra
assert_eq!(listed.parts()[0].e_tag(), Some(seed.multipart_etag.as_str()));
}
eprintln!(
"MPU mixed degraded: one offline disk, three missing part directories; previous returned empty, current returned 503 SlowDownRead; original part restored on both"
"MPU mixed degraded: one offline disk, three missing part directories; checked each coordinator against its release contract; original part restored on both"
);
Ok(())
}
@@ -592,6 +603,7 @@ async fn replace_node_and_wait_for_write_readiness(dist: &mut DistCluster, seed:
async fn four_node_rolling_upgrade_preserves_history_and_iam_credentials() -> TestResult {
init_logging();
let previous = source_binary()?;
let source = UpgradeSourceCapabilities::from_binary(&previous).await?;
let mut dist = DistCluster::new_stopped(DistLayout::FourNodeFourDisk).await?;
dist.cluster.set_env("RUSTFS_STORAGE_CLASS_RRS", "EC:0");
capture_upgrade_logs(&mut dist, "rolling-upgrade")?;
@@ -601,7 +613,7 @@ async fn four_node_rolling_upgrade_preserves_history_and_iam_credentials() -> Te
replace_node_and_wait_for_write_readiness(&mut dist, &seed, 0).await?;
assert_history_and_iam(&dist, &seed, "one-current-node").await?;
compare_degraded_legacy_list_parts(&mut dist, &seed, &previous).await?;
compare_degraded_legacy_list_parts(&mut dist, &seed, &previous, &source).await?;
for node_idx in [1, 2] {
replace_node_and_wait_for_write_readiness(&mut dist, &seed, node_idx).await?;
+45 -8
View File
@@ -27,6 +27,7 @@ use aws_sdk_s3::types::{BucketCannedAcl, Tag, Tagging};
use http::{HeaderMap, Method};
use md5::{Digest, Md5};
use rustfs_filemeta::{FileMeta, MetaObject};
use serde_value::Value;
use std::collections::BTreeMap;
use std::path::{Path, PathBuf};
@@ -71,7 +72,9 @@ fn bucket_metadata(volume: &Path, bucket: &str) -> Result<BucketMetadata, Error>
Ok(BucketMetadata::unmarshal(fields)?)
}
fn read_object(volume: &Path, key: &str) -> Result<MetaObject, Error> {
type VersionFields = BTreeMap<String, Value>;
fn read_object(volume: &Path, key: &str) -> Result<(MetaObject, VersionFields), Error> {
let meta = FileMeta::load(&std::fs::read(volume.join(OBJECTS).join(key).join("xl.meta"))?)?;
let [version] = meta.versions.as_slice() else {
return Err(format!("{key}: expected one version").into());
@@ -80,7 +83,23 @@ fn read_object(volume: &Path, key: &str) -> Result<MetaObject, Error> {
if object.mod_time.is_none() || object.data_dir.is_none_or(|id| id.is_nil()) {
return Err(format!("{key}: missing modification time or data directory").into());
}
Ok(object)
Ok((object, nonvolatile_fields(&version.meta, key)?))
}
fn nonvolatile_fields(raw: &[u8], key: &str) -> Result<VersionFields, Error> {
// The production reader skips unknown keys and supplies defaults for some missing keys.
// Keep the raw maps so both the version envelope and V2Obj preserve every field and value.
let mut fields: VersionFields = rmp_serde::from_slice(raw)?;
let Some(Value::Map(object_fields)) = fields.get_mut("V2Obj") else {
return Err(format!("{key}: missing raw V2Obj map").into());
};
for volatile in ["MTime", "DDir"] {
let value = object_fields
.get_mut(&Value::String(volatile.to_owned()))
.ok_or_else(|| format!("{key}: missing {volatile}"))?;
*value = Value::Unit;
}
Ok(fields)
}
async fn put_xml(env: &RustFSTestEnvironment, query: &str, xml: &str) -> Result<(), Error> {
@@ -104,7 +123,7 @@ async fn put_xml(env: &RustFSTestEnvironment, query: &str, xml: &str) -> Result<
struct Snapshot {
configs: BTreeMap<&'static str, Vec<u8>>,
objects: BTreeMap<&'static str, MetaObject>,
objects: BTreeMap<&'static str, VersionFields>,
multipart_files: BTreeMap<String, Vec<u8>>,
}
@@ -295,12 +314,10 @@ fn read_snapshot(volume: &Path) -> Result<Snapshot, Error> {
]);
let mut objects = BTreeMap::new();
for key in ["put", "copy", "tagged"] {
let mut object = read_object(volume, key)?;
object.mod_time = None;
object.data_dir = None;
objects.insert(key, object);
let (_, fields) = read_object(volume, key)?;
objects.insert(key, fields);
}
let multipart = read_object(volume, "multipart")?;
let (multipart, _) = read_object(volume, "multipart")?;
if multipart.part_numbers != [1, 2] || multipart.part_sizes != [FIRST_PART_SIZE, 100] {
return Err("xl.meta does not describe both uploaded parts".into());
}
@@ -410,3 +427,23 @@ async fn persistence_byte_parity() -> Result<(), Error> {
);
Ok(())
}
#[test]
fn version_comparison_preserves_unknown_fields_and_absent_values() -> Result<(), Error> {
let mut sample = serde_json::json!({"Type": 1, "v": 1, "V2Obj": {"DDir": vec![1; 16], "MTime": 1}});
let fields = |value: &serde_json::Value| nonvolatile_fields(&rmp_serde::to_vec(value)?, "fixture");
let original = fields(&sample)?;
sample["V2Obj"]["DDir"] = serde_json::json!(vec![2; 16]);
sample["V2Obj"]["MTime"] = serde_json::json!(2);
assert_eq!(fields(&sample)?, original);
sample["V2Obj"]["Future"] = serde_json::Value::Null;
let with_nil = fields(&sample)?;
assert_ne!(with_nil, original);
sample["V2Obj"]["Future"] = serde_json::json!(false);
let with_value = fields(&sample)?;
assert_ne!(with_value, with_nil);
sample["FutureEnvelope"] = serde_json::json!(1);
assert_ne!(fields(&sample)?, with_value);
assert!(fields(&serde_json::json!({"V2Obj": {"MTime": 1}})).is_err());
Ok(())
}
@@ -24,7 +24,7 @@
#[cfg(all(test, target_os = "linux"))]
mod tests {
use crate::chaos::{VersionShardCensus, census_object_version_on_disk};
use crate::common::{ClusterTopology, RustFSTestClusterEnvironment, admin_request, init_logging, local_http_client};
use crate::common::{ClusterTopology, RustFSTestClusterEnvironment, admin_request, init_logging};
use aws_sdk_s3::Client;
use aws_sdk_s3::primitives::ByteStream;
use aws_sdk_s3::types::{BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, VersioningConfiguration};
@@ -412,7 +412,7 @@ mod tests {
.arg(test_binary)
.arg("--exact")
.arg(test_name)
.arg("--include-ignored")
.arg("--ignored")
.arg("--nocapture")
.env(NAMESPACE_ENV, "1")
.status()?;
@@ -1524,110 +1524,6 @@ mod tests {
assert!(assert_no_replacement_status_records_in_status(&definitive, target).is_ok());
}
/// Linux mount namespaces are per-thread; keep mount setup and process
/// spawning on one OS thread so child RustFS nodes inherit the test mounts.
#[tokio::test(flavor = "current_thread")]
async fn test_privileged_4x1_subdirectory_replacement_waits_for_write_quorum() -> Result<(), Box<dyn Error + Send + Sync>> {
init_logging();
if !privileged_run_enabled()? {
return Ok(());
}
if std::env::var_os(NAMESPACE_ENV).is_none() {
return run_current_test_in_mount_namespace(
"replacement_privileged_e2e_test::tests::test_privileged_4x1_subdirectory_replacement_waits_for_write_quorum",
);
}
let mut mounts = MountNamespaceGuard::new()?;
let mut cluster = RustFSTestClusterEnvironment::new(4).await?;
let filesystem: serde_json::Value = serde_json::from_str(&run_command_stdout(
"findmnt",
&["--json", "--target", &cluster.temp_dir, "--output", "TARGET,FSTYPE"],
)?)?;
let xfs_root = filesystem["filesystems"][0]["target"]
.as_str()
.filter(|root| *root != "/" && filesystem["filesystems"][0]["fstype"] == "xfs");
for (index, node) in cluster.nodes.iter_mut().enumerate() {
let mount = PathBuf::from(&node.data_dir);
if xfs_root.is_none() {
mounts.mount_tmpfs(&mount, &format!("rustfs-replacement-4x1-{index}"))?;
}
let data = mount.join("data");
fs::create_dir(&data)?;
node.data_dir = path_to_string(&data, "replacement subdirectory")?;
node.data_dirs[0] = node.data_dir.clone();
}
let target = PathBuf::from(&cluster.nodes[TARGET_NODE].data_dir);
let mount = xfs_root.unwrap_or(target.parent().and_then(Path::to_str).ok_or("missing target mount root")?);
eprintln!(
"4x1 replacement fixture filesystem = {}",
if xfs_root.is_some() { "xfs" } else { "tmpfs" }
);
cluster.set_node_env(TARGET_NODE, "RUSTFS_HEAL_REPLACEMENT_MOUNT_ROOTS", mount)?;
cluster.set_env("RUSTFS_STORAGE_CLASS_STANDARD", "EC:2");
cluster.set_env("RUSTFS_HEAL_ENABLED", "true");
cluster.set_env("RUSTFS_HEAL_INTERVAL_SECS", "1");
cluster.set_env("RUSTFS_SCANNER_START_DELAY_SECS", "0");
cluster.set_env("RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE", "false");
cluster.start().await?;
let clients = cluster.create_all_clients()?;
let versions = seed_baseline(&clients[0], &target, 0).await?;
cluster.stop_node_gracefully(TARGET_NODE).await?;
cluster.stop_node_gracefully(0).await?;
fs::remove_dir_all(&target)?;
fs::create_dir(&target)?;
let metadata = ["format.json", "pool.bin/xl.meta", "pool.bin.identity/xl.meta"];
let system_dir = Path::new(".rustfs.sys");
let survivor_files = cluster.nodes[2..]
.iter()
.flat_map(|node| metadata.map(|name| Path::new(&node.data_dir).join(system_dir).join(name)))
.map(|file| fs::read(&file).map(|bytes| (file, bytes)))
.collect::<std::io::Result<Vec<_>>>()?;
cluster.start_node_process(TARGET_NODE).await?;
let write_health = format!("{}/minio/health/cluster", cluster.nodes[2].url);
let http = local_http_client();
timeout(Duration::from_secs(30), async {
loop {
let status = http.get(&write_health).send().await?.status();
if status.as_u16() == 503 {
break;
}
sleep(Duration::from_millis(200)).await;
}
Ok::<_, reqwest::Error>(())
})
.await??;
verify_bodies(&clients[2], &versions).await?;
let version = &versions[0];
let failed_delete = clients[2]
.delete_object()
.bucket(&version.bucket)
.key(&version.key)
.send()
.await
.expect_err("two of four drives must not delete an existing object");
assert_eq!(failed_delete.raw_response().map(|response| response.status().as_u16()), Some(503));
for name in metadata {
assert!(!target.join(system_dir).join(name).try_exists()?, "unadmitted target published {name}");
}
for (file, before) in survivor_files {
assert_eq!(fs::read(&file)?, before, "surviving pool evidence changed below write quorum");
}
cluster.start_node(0).await?;
wait_for_completed_replacement_with_census(&cluster, &target, &versions, 90).await?;
for name in metadata {
assert!(fs::metadata(target.join(system_dir).join(name))?.is_file());
}
cluster.stop_node_gracefully(TARGET_NODE).await?;
cluster.start_node(TARGET_NODE).await?;
assert!(incomplete_versions(&target, &versions)?.is_empty());
verify_bodies(&clients[TARGET_NODE], &versions).await
}
/// Linux mount namespaces are per-thread; keep mount setup and process
/// spawning on one OS thread so child RustFS nodes inherit the test mounts.
#[tokio::test(flavor = "current_thread")]
+125 -13
View File
@@ -46,9 +46,12 @@ type BoxError = Box<dyn std::error::Error + Send + Sync>;
const SOURCE_BINARY_ENV: &str = "RUSTFS_UPGRADE_SOURCE_BINARY";
const G09_EVIDENCE_DIR_ENV: &str = "RUSTFS_SCANNER_HEAL_G09_EVIDENCE_DIR";
const RC2_COMMIT: &str = "e9f53180273b32a9c2b0aa8b5a384047bc7e875c";
const RC5_COMMIT: &str = "40a2470feb567201165a5b809b7598bb4b1f68f5";
const STABLE_1_0_1_COMMIT: &str = "6de965ae3c965a78ff819fbcd7acd4aa44177d92";
const STABLE_1_0_4_COMMIT: &str = "46337942d55f3a8e6a97b147ca98d9ea8115555d";
const STABLE_1_0_5_COMMIT: &str = "34e6449ed90e3d3a106f6655342e0b3127f33082";
const STABLE_1_0_6_COMMIT: &str = "de9009d882644effc2628e777650025c3d8e7c65";
const SSE_MASTER_KEY_ENV: &str = "RUSTFS_SSE_S3_MASTER_KEY";
const SSE_MASTER_KEY: &str = "QkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkI=";
const PLAIN_BUCKET: &str = "upgrade-plain-data";
@@ -142,6 +145,41 @@ fn current_source_revision() -> Result<String, BoxError> {
Ok(revision.to_string())
}
pub(crate) struct UpgradeSourceCapabilities {
supports_delete_recovery: bool,
supports_multipart_publication: bool,
pub(crate) requires_list_parts_read_quorum: bool,
}
impl UpgradeSourceCapabilities {
fn from_revision(revision: &str) -> Result<Self, BoxError> {
// 1.0.4 supports the delete decision protocol (#8468). Version 1.0.5 also
// shipped native MPU publication (#8919) and strict ListParts evidence (#9037, #9149).
match revision {
RC2_COMMIT | RC5_COMMIT | STABLE_1_0_1_COMMIT => Ok(Self {
supports_delete_recovery: false,
supports_multipart_publication: false,
requires_list_parts_read_quorum: false,
}),
STABLE_1_0_4_COMMIT => Ok(Self {
supports_delete_recovery: true,
supports_multipart_publication: false,
requires_list_parts_read_quorum: false,
}),
STABLE_1_0_5_COMMIT | STABLE_1_0_6_COMMIT => Ok(Self {
supports_delete_recovery: true,
supports_multipart_publication: true,
requires_list_parts_read_quorum: true,
}),
_ => Err(format!("no pinned upgrade expectations for source revision {revision}").into()),
}
}
pub(crate) async fn from_binary(binary: &Path) -> Result<Self, BoxError> {
Self::from_revision(&binary_source_revision(binary).await?)
}
}
async fn binary_source_revision(binary: &Path) -> Result<String, BoxError> {
let output = tokio::process::Command::new(binary).arg("--version").output().await?;
if !output.status.success() {
@@ -225,6 +263,25 @@ mod g09_evidence_tests {
use super::*;
use tempfile::TempDir;
#[test]
fn pinned_upgrade_sources_keep_distinct_delete_and_list_parts_contracts() -> TestResult {
for (revision, delete_recovery, multipart_publication, list_parts_quorum) in [
(RC2_COMMIT, false, false, false),
(RC5_COMMIT, false, false, false),
(STABLE_1_0_1_COMMIT, false, false, false),
(STABLE_1_0_4_COMMIT, true, false, false),
(STABLE_1_0_5_COMMIT, true, true, true),
(STABLE_1_0_6_COMMIT, true, true, true),
] {
let source = UpgradeSourceCapabilities::from_revision(revision)?;
assert_eq!(source.supports_delete_recovery, delete_recovery, "{revision}");
assert_eq!(source.supports_multipart_publication, multipart_publication, "{revision}");
assert_eq!(source.requires_list_parts_read_quorum, list_parts_quorum, "{revision}");
}
assert!(UpgradeSourceCapabilities::from_revision("ffffffffffffffffffffffffffffffffffffffff").is_err());
Ok(())
}
#[test]
fn source_revision_parser_requires_lowercase_forty_byte_sha() -> TestResult {
let binary = Path::new("rustfs");
@@ -1011,8 +1068,7 @@ async fn exercise_mixed_cluster(
vec![u8::try_from(MULTIPART_WORKERS - 1)?; 64 * 1024]
);
// 1.0.4 already implements the delete decision protocol. Use a separate
// old-writer key so the original seed still spans the entire rollout.
// Use a separate old-writer key so the original seed still spans the entire rollout.
if previous_has_delete_recovery {
let key = format!(".upgrade-delete/{phase}/previous-writer");
previous_client
@@ -2293,7 +2349,7 @@ async fn rolling_upgrade_from_previous_release_preserves_mixed_version_contracts
let previous_binary = source_binary()?;
// Bind the expected protocol to the verified release source, never to the
// DELETE result itself. The older CI source pins retain their 503 contract.
let previous_has_delete_recovery = binary_source_revision(&previous_binary).await? == STABLE_1_0_4_COMMIT;
let source = UpgradeSourceCapabilities::from_binary(&previous_binary).await?;
let evidence_context = g09_evidence_context(&previous_binary).await?;
let current_binary = rustfs_binary_path();
let mut cluster = RustFSTestClusterEnvironment::new(MIXED_NODE_COUNT).await?;
@@ -2315,17 +2371,21 @@ async fn rolling_upgrade_from_previous_release_preserves_mixed_version_contracts
cluster.stop_node(0)?;
cluster.start_node_from_binary(0, &current_binary).await?;
let mut mixed_deleted_keys = Vec::new();
if let Some(key) = exercise_mixed_cluster(&cluster, "one-current-node", 0, 1, previous_has_delete_recovery).await? {
if let Some(key) = exercise_mixed_cluster(&cluster, "one-current-node", 0, 1, source.supports_delete_recovery).await? {
mixed_deleted_keys.push(key);
}
// One controlled out-of-policy attempt proves fail-closed behavior below
// native write quorum; positive MPU writers remain paused while mixed.
assert_mixed_pending_refusal(&cluster, &pending, &sources).await?;
let mixed_completion = if source.supports_multipart_publication {
let clients = cluster.create_all_clients()?;
Some(finish_pending_multipart(&clients, &pending, pending_parts.concat()).await?)
} else {
assert_mixed_pending_refusal(&cluster, &pending, &sources).await?;
None
};
for (node_idx, phase) in [(1, "two-current-nodes"), (2, "one-previous-node")] {
cluster.stop_node(node_idx)?;
cluster.start_node_from_binary(node_idx, &current_binary).await?;
if let Some(key) = exercise_mixed_cluster(&cluster, phase, 0, 3, previous_has_delete_recovery).await? {
if let Some(key) = exercise_mixed_cluster(&cluster, phase, 0, 3, source.supports_delete_recovery).await? {
mixed_deleted_keys.push(key);
}
}
@@ -2335,6 +2395,13 @@ async fn rolling_upgrade_from_previous_release_preserves_mixed_version_contracts
let clients = cluster.create_all_clients()?;
wait_for_drive_readmission(&clients, MIXED_BUCKET, "homogeneous-current", DRIVE_READMISSION_TIMEOUT).await?;
let completed_pending = finish_pending_multipart(&clients, &pending, pending_parts.concat()).await?;
if let Some(mixed) = mixed_completion {
assert_eq!(
completed_pending.version_id, mixed.version_id,
"full upgrade changed the completed version"
);
assert_eq!(completed_pending.etag, mixed.etag, "full upgrade changed the completed ETag");
}
exercise_homogeneous_multipart(&cluster, "homogeneous-current").await?;
verify_load_bodies(&clients, "homogeneous-previous", "multipart").await?;
assert_mixed_deletes_visible(&clients, &mixed_deleted_keys).await?;
@@ -2451,7 +2518,7 @@ async fn rolling_upgrade_from_previous_release_preserves_mixed_version_contracts
"phases": phases,
"objects_per_phase": objects_per_phase,
"mixed_load": "plain-put",
"mixed_delete_protocol_supported": previous_has_delete_recovery,
"mixed_delete_protocol_supported": source.supports_delete_recovery,
"mixed_deleted_keys": mixed_deleted_keys,
"homogeneous_multipart_phases": ["homogeneous-previous", "homogeneous-current", "homogeneous-rollback"],
"multipart_objects_per_homogeneous_phase": MULTIPART_WORKERS * MULTIPART_UPLOADS_PER_WORKER,
@@ -2461,7 +2528,7 @@ async fn rolling_upgrade_from_previous_release_preserves_mixed_version_contracts
"all nodes list every mixed-version object after homogeneous-current convergence",
"the previous-release seed survives the baseline restart and every upgrade phase",
"previous readers read every current-native MPU body after quiescent rollback",
"same legacy upload resumes with identical body, ETag and version after full upgrade"
"the same previous-release upload remains readable after full upgrade and rollback"
],
}),
)?;
@@ -2483,9 +2550,13 @@ async fn rolling_upgrade_from_previous_release_preserves_mixed_version_contracts
"previous-release writer publishes objects readable by current node",
"mixed phases retain 16x16 plain PUT writers while positive multipart writers are paused",
"16x16 multipart load runs in homogeneous previous, current and rollback phases",
"one-current-node Complete returns typed 503 and preserves actual source authority and parts",
"the same legacy upload completes and replays once fully upgraded without adding a version",
if previous_has_delete_recovery {
if source.supports_multipart_publication {
"one-current-node Complete and replay return the same version and ETag, and all readers agree on the body"
} else {
"one-current-node Complete returns typed 503 and preserves actual source authority and parts"
},
"fully upgraded readers see the same body, ETag and sole version after replay",
if source.supports_delete_recovery {
"mixed-version DELETE succeeds and stays absent through every reader after upgrade and rollback"
} else {
"modern DELETE fails closed and preserves every byte while old peers remain"
@@ -3386,6 +3457,8 @@ async fn direct_upgrade_from_previous_release_preserves_bucket_configuration() -
#[tokio::test]
#[ignore = "requires a pinned previous RustFS release binary"]
async fn rollback_to_previous_release_reads_current_bucket_metadata() -> TestResult {
use sha2::Digest;
init_logging();
let previous_binary = source_binary()?;
let evidence_context = g09_evidence_context(&previous_binary).await?;
@@ -3423,6 +3496,16 @@ async fn rollback_to_previous_release_reads_current_bucket_metadata() -> TestRes
let multipart_bytes = multipart_parts.concat();
write_multipart(&new_client, ROLLBACK_BUCKET, multipart_key, &multipart_parts).await?;
// This release can finish an upload staged by the current gateway.
let inflight = if binary_source_revision(&previous_binary).await? == STABLE_1_0_5_COMMIT {
env.create_test_bucket(PENDING_MULTIPART_BUCKET).await?;
let parts = [vec![1; LAYOUT_PART_SIZE], vec![2; LAYOUT_PART_SIZE], vec![3; 4096]];
let pending = stage_multipart(&new_client, PENDING_MULTIPART_BUCKET, PENDING_MULTIPART_KEY, &parts).await?;
Some((pending, parts.concat()))
} else {
None
};
restart_from_binary(&mut env, &previous_binary, &server_env).await?;
let old_client = env.create_s3_client();
@@ -3438,6 +3521,16 @@ async fn rollback_to_previous_release_reads_current_bucket_metadata() -> TestRes
assert_eq!(encryption, Some(ServerSideEncryption::Aes256));
assert_eq!(body, multipart_bytes, "the multipart object did not survive the rollback");
let rollback_multipart = if let Some((pending, expected)) = inflight {
assert_pending_parts(&old_client, &pending).await?;
complete_pending_multipart(&old_client, PENDING_MULTIPART_BUCKET, PENDING_MULTIPART_KEY, &pending).await?;
let (_, body) = read_object(&old_client, PENDING_MULTIPART_BUCKET, PENDING_MULTIPART_KEY, None).await?;
assert_eq!(body, expected);
Some((pending.part_sizes, body))
} else {
None
};
// A PUT on the rolled-back release must still honour the encryption
// configuration it decoded out of the current build's metadata blob.
let post_rollback_key = "rollback/written-after-rollback";
@@ -3478,6 +3571,25 @@ async fn rollback_to_previous_release_reads_current_bucket_metadata() -> TestRes
post_rollback_bytes
);
if let Some((part_sizes, rollback_read)) = rollback_multipart {
let (_, current_read) = read_object(&current_again, PENDING_MULTIPART_BUCKET, PENDING_MULTIPART_KEY, None).await?;
assert_eq!(current_read, rollback_read);
if let Some(context) = &evidence_context {
let evidence = serde_json::json!({
"previous_revision": context.previous_revision,
"current_revision": context.current_revision,
"part_sizes": part_sizes,
"bytes": current_read.len(),
"rollback_read_sha256": format!("{:x}", sha2::Sha256::digest(&rollback_read)),
"roll_forward_read_sha256": format!("{:x}", sha2::Sha256::digest(&current_read)),
});
std::fs::write(
context.directory.join("inflight-multipart-rollback.json"),
serde_json::to_vec_pretty(&evidence)?,
)?;
}
}
if let Some(context) = evidence_context.as_ref() {
write_g09_evidence(
context,
-1
View File
@@ -469,7 +469,6 @@ pub mod disk {
pub use crate::cluster::rpc::background_pipe::{BackgroundPipeReader, BackgroundPipeWriter, background_pipe};
pub use crate::cluster::rpc::background_reader_stream::BackgroundReaderStream;
pub use crate::disk::BOUNDED_DELETE_PREPARE_PROTOCOL_VERSION;
pub use crate::disk::ReplacementMount;
pub use crate::disk::background_memory::BackgroundEncodeBuffer;
pub use crate::disk::disk_store::get_object_disk_read_timeout;
#[cfg(feature = "test-util")]
@@ -6599,9 +6599,7 @@ mod tests {
);
assert_eq!(
remote
.pool_meta_native_recovery(PoolMetaNativeRecoveryRequest::Inspect {
object: crate::disk::PoolMetaNativeObject::PoolBin,
})
.pool_meta_native_recovery(PoolMetaNativeRecoveryRequest::Probe {})
.await,
Err(DiskError::UnsupportedDisk),
);
@@ -6612,7 +6610,7 @@ mod tests {
"/node_service.NodeService/UndoPoolMetaWrite",
"/node_service.NodeService/PoolMetaNativeRecovery"
],
"protected mutations and native inspection must never fall back to legacy RPCs",
"protected mutations and capability probes must never fall back to legacy RPCs",
);
peer.stop().await;
}
@@ -207,10 +207,10 @@ async fn pool_meta_native_physical_split_without_ordinary_commit_performs_no_mut
let before = snapshots(&fixture.store).await;
let mut state = clean_state(&fixture);
let fence = PoolMetaPersistenceFence::Owned(owner(&fixture.store).await);
let error = repair(&fixture.store, &mut state, &fence)
.await
.expect_err("reconstruction alone must never authorize a COMMIT");
assert!(error.to_string().contains("(no_ordinary_v3_authority)"), "{error}");
assert!(
repair(&fixture.store, &mut state, &fence).await.is_err(),
"reconstruction alone must never authorize a COMMIT"
);
assert_eq!(
snapshots(&fixture.store).await,
before,
@@ -222,24 +222,6 @@ async fn pool_meta_native_physical_split_without_ordinary_commit_performs_no_mut
.expect("missing ordinary authority alone must not install a permanent corruption latch");
}
#[tokio::test]
#[serial_test::serial(pool_meta_version_env)]
async fn pool_meta_native_inspection_rejects_an_absent_slot_without_changing_evidence() {
let fixture = fixture(1).await;
let before = snapshots(&fixture.store).await;
let fence = PoolMetaPersistenceFence::Owned(owner(&fixture.store).await);
let last_pool = fixture.store.pools.last().expect("fixture has a last pool");
let set = last_pool.get_disks_by_key(POOL_META_NAME);
let missing = set.disks.write().await[0].take().expect("fixture has a disk in slot zero");
let result = inspect(&fixture.store.pools, &fence).await;
set.disks.write().await[0] = Some(missing);
let error = result.err().expect("an absent slot cannot provide native evidence");
assert!(error.to_string().contains("(missing_pool_metadata_slot)"), "{error}");
assert_eq!(snapshots(&fixture.store).await, before);
}
async fn interrupted_first_commit_fixture() -> (PhysicalFixture, PoolMetaCommittedCandidate) {
let fixture = fixture(3).await;
let PoolMetaReplica::Valid {
@@ -19,7 +19,6 @@ use crate::disk::{
PoolMetaNativeRecoveryResponse as Response,
};
use bytes::Bytes;
use futures::TryStreamExt as _;
#[cfg(test)]
#[derive(Default)]
@@ -39,16 +38,8 @@ struct InspectedPool {
snapshots: Vec<PoolMetaNativeInspection>,
}
#[derive(Debug, thiserror::Error)]
#[error("native pool metadata recovery requires one proven V3 transaction and complete matching disk evidence ({0})")]
struct MissingEvidence(&'static str);
fn missing_evidence(reason: &'static str) -> Error {
Error::other(MissingEvidence(reason))
}
fn invalid() -> Error {
missing_evidence("inconsistent_evidence")
Error::other("native pool metadata recovery requires one proven V3 transaction and complete matching disk evidence")
}
fn known_validation(source: Error, phase: &'static str) -> Error {
@@ -80,19 +71,13 @@ fn latch_validation(state: &mut PoolMetaWriteState, error: Error) -> Error {
fn current(pool: &InspectedPool) -> Result<Vec<Bytes>> {
pool.snapshots
.iter()
.map(|snapshot| {
snapshot
.current
.clone()
.ok_or_else(|| missing_evidence("missing_pool_metadata_head"))
})
.map(|snapshot| snapshot.current.clone().ok_or_else(invalid))
.collect()
}
async fn inspect(pools: &[Arc<Sets>], fence: &PoolMetaPersistenceFence<'_>) -> Result<Vec<InspectedPool>> {
let mut inspected = Vec::with_capacity(pools.len());
// Check every slot before starting I/O. The authenticated, read-only
// Inspect RPC proves both protocol support and the disk's evidence.
// Pool -> disk order is stable; all capability probes precede every mutation.
for pool in pools {
fence.ensure_held()?;
let set = pool.get_disks_by_key(POOL_META_NAME);
@@ -102,8 +87,20 @@ async fn inspect(pools: &[Arc<Sets>], fence: &PoolMetaPersistenceFence<'_>) -> R
.await
.iter()
.cloned()
.map(|disk| disk.ok_or_else(|| missing_evidence("missing_pool_metadata_slot")))
.map(|disk| disk.ok_or_else(invalid))
.collect::<Result<Vec<_>>>()?;
let probes = futures::stream::iter(disks.iter().cloned().map(|disk| async move {
match disk.pool_meta_native_recovery(Request::Probe {}).await? {
Response::Probed => Ok(()),
_ => Err(invalid()),
}
}))
.buffered(4)
.collect::<Vec<Result<()>>>()
.await;
for result in probes {
result?;
}
inspected.push(InspectedPool {
disks,
snapshots: Vec::new(),
@@ -111,7 +108,7 @@ async fn inspect(pools: &[Arc<Sets>], fence: &PoolMetaPersistenceFence<'_>) -> R
}
for pool in &mut inspected {
fence.ensure_held()?;
pool.snapshots = futures::stream::iter(pool.disks.iter().cloned().map(|disk| async move {
let results = futures::stream::iter(pool.disks.iter().cloned().map(|disk| async move {
match disk
.pool_meta_native_recovery(Request::Inspect {
object: PoolMetaNativeObject::PoolBin,
@@ -119,12 +116,15 @@ async fn inspect(pools: &[Arc<Sets>], fence: &PoolMetaPersistenceFence<'_>) -> R
.await?
{
Response::Inspected(snapshot) if !snapshot.disk_id.is_nil() => Ok(snapshot),
_ => Err(missing_evidence("invalid_disk_identity_evidence")),
_ => Err(invalid()),
}
}))
.buffered(4)
.try_collect()
.await?;
.collect::<Vec<Result<PoolMetaNativeInspection>>>()
.await;
for result in results {
pool.snapshots.push(result?);
}
}
fence.ensure_held()?;
Ok(inspected)
@@ -142,7 +142,7 @@ async fn validate_identities(pools: &[Arc<Sets>], canonical: &[u8], fence: &Pool
.await
.iter()
.cloned()
.map(|disk| disk.ok_or_else(|| missing_evidence("missing_pool_identity_slot")))
.map(|disk| disk.ok_or_else(invalid))
.collect::<Result<Vec<_>>>()?;
let results = futures::stream::iter(disks.into_iter().enumerate().map(|(index, disk)| async move {
match disk.pool_meta_native_recovery(Request::InspectIdentity {}).await? {
@@ -151,7 +151,7 @@ async fn validate_identities(pools: &[Arc<Sets>], canonical: &[u8], fence: &Pool
.await
.map_err(|error| codec_validation(error, "native_identity_validation"))
}
_ => Err(missing_evidence("invalid_disk_identity_evidence")),
_ => Err(invalid()),
}
}))
.buffered(4)
@@ -479,7 +479,7 @@ pub(super) async fn repair(
})
.collect::<Vec<_>>();
if ordinary_commits.is_empty() && ordinary_prepares.is_empty() {
return Err(missing_evidence("no_ordinary_v3_authority"));
return Err(invalid());
}
let selection = select_pool_meta_replica_reads(reads)
.map_err(|error| latch_validation(state, known_validation(error, "native_metadata_validation")))?;
@@ -535,7 +535,7 @@ pub(super) async fn repair(
.iter()
.any(|read| !matches!(read.replica, PoolMetaIdentityReplica::Valid(_)))
{
return Err(missing_evidence("pool_identity_unavailable"));
return Err(invalid());
}
let identity = select_pool_meta_identity(identities, cluster_id)
.map_err(|error| latch_validation(state, known_validation(error, "native_identity_validation")))?;
@@ -847,10 +847,7 @@ mod tests {
async fn pool_meta_native_plan_rejects_missing_or_corrupt_evidence() {
let mut f = fixture().await;
let before = f.pools[1].snapshots[2].current.take();
let error = plan(&f.pools, &f.commit, f.revision, &f.identity, None)
.await
.expect_err("missing current shard must block the repair plan");
assert!(error.to_string().contains("(missing_pool_metadata_head)"), "{error}");
assert!(plan(&f.pools, &f.commit, f.revision, &f.identity, None).await.is_err());
f.pools[1].snapshots[2].current = before;
f.pools[1].snapshots[2].backups[0].data = Bytes::from_static(b"invalid metadata");
assert!(plan(&f.pools, &f.commit, f.revision, &f.identity, None).await.is_err());
+82 -98
View File
@@ -8268,9 +8268,8 @@ pub struct LocalDisk {
io_root: PathBuf,
#[cfg(target_os = "linux")]
mount_lease: Arc<std::fs::File>,
// The descriptor pins this identity even if the configured path is remounted.
#[cfg(target_os = "linux")]
mount_lease_identity: Option<ReplacementMountIdentity>,
mount_lease_mount_id: Option<u64>,
/// Public path for callers that need the configured disk layout. Internal
/// disk I/O uses `io_format_path`, which is rooted at `mount_lease`.
pub format_path: PathBuf,
@@ -8470,50 +8469,72 @@ struct ReplacementMountIdentity {
}
#[cfg(target_os = "linux")]
fn statx_replacement_mount_identity(
dir: impl std::os::fd::AsFd,
path: &Path,
flags: rustix::fs::AtFlags,
) -> Option<ReplacementMountIdentity> {
use rustix::fs::{FileType, StatxFlags, makedev, statx};
let fields = StatxFlags::TYPE | StatxFlags::INO | StatxFlags::MNT_ID;
let stat = statx(dir, path, flags, fields).ok()?;
(StatxFlags::from_bits_retain(stat.stx_mask).contains(fields) && FileType::from_raw_mode(stat.stx_mode.into()).is_dir())
.then_some(ReplacementMountIdentity {
device: makedev(stat.stx_dev_major, stat.stx_dev_minor),
inode: stat.stx_ino,
mount_id: stat.stx_mnt_id,
fn replacement_mount_identity(metadata: &std::fs::Metadata, mount_id: u64) -> ReplacementMountIdentity {
use std::os::unix::fs::MetadataExt as _;
ReplacementMountIdentity {
device: metadata.dev(),
inode: metadata.ino(),
mount_id,
}
}
#[cfg(target_os = "linux")]
fn mount_id_for_fd(fd: &std::fs::File) -> Option<u64> {
use rustix::fs::{AtFlags, StatxFlags};
use std::os::fd::AsRawFd as _;
rustix::fs::statx(fd, "", AtFlags::EMPTY_PATH, StatxFlags::MNT_ID)
.ok()
.filter(|statx| StatxFlags::from_bits_retain(statx.stx_mask).contains(StatxFlags::MNT_ID))
.map(|statx| statx.stx_mnt_id)
.or_else(|| mount_id_from_fdinfo(fd.as_raw_fd()))
}
#[cfg(target_os = "linux")]
fn mount_id_for_path(path: &Path) -> Option<u64> {
use rustix::fs::{AtFlags, CWD, StatxFlags};
rustix::fs::statx(CWD, path, AtFlags::empty(), StatxFlags::MNT_ID)
.ok()
.filter(|statx| StatxFlags::from_bits_retain(statx.stx_mask).contains(StatxFlags::MNT_ID))
.map(|statx| statx.stx_mnt_id)
.or_else(|| {
std::fs::canonicalize(path)
.ok()
.and_then(|path| mount_id_from_mountinfo(&path))
})
}
#[cfg(target_os = "linux")]
fn replacement_mount_identity_for_fd(directory: &std::fs::File) -> Option<ReplacementMountIdentity> {
statx_replacement_mount_identity(directory, Path::new(""), rustix::fs::AtFlags::EMPTY_PATH)
.or_else(|| replacement_mount_identity_from_fdinfo(directory))
}
#[cfg(target_os = "linux")]
fn replacement_mount_identity_for_path(path: &Path) -> Option<ReplacementMountIdentity> {
use rustix::fs::{AtFlags, CWD, Mode, OFlags, open};
statx_replacement_mount_identity(CWD, path, AtFlags::empty()).or_else(|| {
let directory = std::fs::File::from(open(path, OFlags::PATH | OFlags::DIRECTORY | OFlags::CLOEXEC, Mode::empty()).ok()?);
replacement_mount_identity_from_fdinfo(&directory)
})
}
#[cfg(target_os = "linux")]
fn replacement_mount_identity_from_fdinfo(directory: &std::fs::File) -> Option<ReplacementMountIdentity> {
use std::os::{fd::AsRawFd as _, unix::fs::MetadataExt as _};
let fd = directory.as_raw_fd();
let mount_id = std::fs::read_to_string(format!("/proc/self/fdinfo/{fd}"))
fn mount_id_from_fdinfo(fd: std::os::fd::RawFd) -> Option<u64> {
std::fs::read_to_string(format!("/proc/self/fdinfo/{fd}"))
.ok()?
.lines()
.find_map(|line| line.strip_prefix("mnt_id:")?.trim().parse().ok())?;
let metadata = directory.metadata().ok()?;
Some(ReplacementMountIdentity {
device: metadata.dev(),
inode: metadata.ino(),
mount_id,
.find_map(|line| line.strip_prefix("mnt_id:")?.trim().parse().ok())
}
#[cfg(target_os = "linux")]
fn mount_id_from_mountinfo(path: &Path) -> Option<u64> {
let mountinfo = std::fs::read_to_string("/proc/self/mountinfo").ok()?;
mount_id_from_mountinfo_contents(&mountinfo, path)
}
#[cfg(target_os = "linux")]
fn mount_id_from_mountinfo_contents(mountinfo: &str, path: &Path) -> Option<u64> {
let mountpoint = path
.to_string_lossy()
.replace('\\', "\\134")
.replace('\t', "\\011")
.replace('\n', "\\012")
.replace(' ', "\\040");
mountinfo.lines().find_map(|line| {
let mut fields = line.split_whitespace();
let mount_id = fields.next()?.parse().ok()?;
fields.next()?;
fields.next()?;
fields.next()?;
(fields.next()? == mountpoint).then_some(mount_id)
})
}
@@ -8570,54 +8591,25 @@ impl LocalDisk {
/// Auto-replacement is destructive, so it is admitted only when the
/// configured endpoint still names the directory held by `mount_lease`.
pub fn has_replacement_mount_lease(&self) -> bool {
#[cfg(target_os = "linux")]
return self
.current_replacement_mount_identity(Path::new(&self.endpoint.get_file_path()))
.is_some();
#[cfg(not(target_os = "linux"))]
false
}
#[cfg(target_os = "linux")]
fn current_replacement_mount_identity(&self, configured_path: &Path) -> Option<ReplacementMountIdentity> {
let pinned = self.mount_lease_identity?;
(replacement_mount_identity_for_path(configured_path)? == pinned).then_some(pinned)
}
pub(super) fn replacement_mount(&self, _mount_roots: &[PathBuf]) -> Option<super::ReplacementMount> {
#[cfg(target_os = "linux")]
{
let configured_path = self.endpoint.get_file_path();
let configured_path = Path::new(&configured_path);
let canonical_path = std::fs::canonicalize(&self.io_root).ok()?;
let expected = _mount_roots
.iter()
.filter(|root| root.is_absolute() && (configured_path.starts_with(root) || canonical_path.starts_with(root)))
.max_by_key(|root| root.components().count())
.unwrap_or(&canonical_path);
// The expected root must be its own mount boundary. If it is
// unmounted, the exposed directory and its parent share a mount ID.
if !canonical_path.starts_with(expected) {
return None;
}
// When the configured endpoint is the expected mount root, this
// live observation also supplies the mount ID for the boundary.
let identity = self.current_replacement_mount_identity(configured_path)?;
if (expected != configured_path && replacement_mount_identity_for_path(expected)?.mount_id != identity.mount_id)
|| replacement_mount_identity_for_path(expected.parent()?)?.mount_id == identity.mount_id
{
return None;
}
Some(super::ReplacementMount {
canonical_path,
physical_device_ids: rustfs_utils::os::get_physical_device_ids(self.io_root.to_str()?).ok()?,
filesystem_identity: format!("{}:{}:{}", identity.mount_id, identity.device, identity.inode),
})
let Ok(configured) = std::fs::metadata(self.endpoint.get_file_path()) else {
return false;
};
let Ok(pinned) = self.mount_lease.metadata() else {
return false;
};
let Some(configured_mount_id) = mount_id_for_path(Path::new(&self.endpoint.get_file_path())) else {
return false;
};
let Some(pinned_mount_id) = self.mount_lease_mount_id else {
return false;
};
replacement_mount_identity(&configured, configured_mount_id) == replacement_mount_identity(&pinned, pinned_mount_id)
}
#[cfg(not(target_os = "linux"))]
None
false
}
/// Return the descriptor-rooted path for the mount instance admitted for
@@ -8673,10 +8665,10 @@ impl LocalDisk {
PathBuf::from(format!(".rustfs-format-control/{format_device:x}/{}", Uuid::from_bytes(format_file_id)));
#[cfg(target_os = "linux")]
let (mount_lease, io_root, mount_lease_identity) = (
let (mount_lease, io_root, mount_lease_mount_id) = (
publication_root.directory().clone(),
publication_root.io_path().to_path_buf(),
replacement_mount_identity_for_fd(publication_root.directory()),
mount_id_for_fd(publication_root.directory()),
);
#[cfg(not(target_os = "linux"))]
let io_root = Self::open_mount_lease(&root)?;
@@ -8859,7 +8851,7 @@ impl LocalDisk {
#[cfg(target_os = "linux")]
mount_lease: mount_lease.clone(),
#[cfg(target_os = "linux")]
mount_lease_identity,
mount_lease_mount_id,
endpoint: ep.clone(),
format_path,
io_format_path,
@@ -41485,18 +41477,10 @@ mod test {
#[cfg(target_os = "linux")]
#[test]
fn mount_id_fdinfo_can_identify_a_directory_below_a_mount() {
let temp = tempfile::tempdir().expect("mount fixture directory");
let child_path = temp.path().join("data");
std::fs::create_dir(&child_path).expect("create data subdirectory");
let parent = std::fs::File::open(temp.path()).expect("open parent directory");
let child = std::fs::File::open(&child_path).expect("open data subdirectory");
let parent_identity = replacement_mount_identity_from_fdinfo(&parent).expect("read parent identity");
let child_identity = replacement_mount_identity_from_fdinfo(&child).expect("read child identity");
assert_eq!(child_identity.mount_id, parent_identity.mount_id);
assert_ne!(child_identity.inode, parent_identity.inode);
assert_eq!(replacement_mount_identity_for_fd(&child), Some(child_identity));
assert_eq!(replacement_mount_identity_for_path(&child_path), Some(child_identity));
fn mountinfo_fallback_uses_the_exact_escaped_mountpoint() {
let mountinfo = "101 1 0:30 / / rw - rootfs rootfs rw\n202 101 0:42 / /mnt/replacement\\040disk rw - tmpfs tmpfs rw\n";
assert_eq!(mount_id_from_mountinfo_contents(mountinfo, Path::new("/mnt/replacement disk")), Some(202));
assert_eq!(mount_id_from_mountinfo_contents(mountinfo, Path::new("/mnt/replacement")), None);
}
#[cfg(not(windows))]
-17
View File
@@ -239,14 +239,6 @@ pub enum PoolMetaNativeRecoveryResponse {
pub type DiskStore = Arc<Disk>;
/// Filesystem evidence for a replacement data directory on its expected mount.
#[derive(Debug)]
pub struct ReplacementMount {
pub canonical_path: PathBuf,
pub physical_device_ids: Vec<String>,
pub filesystem_identity: String,
}
pub trait DiskFileWriter: AsyncWrite + Send + Sync + Unpin + std::any::Any {}
impl<T: AsyncWrite + Send + Sync + Unpin + 'static> DiskFileWriter for T {}
@@ -1591,15 +1583,6 @@ impl Disk {
}
}
/// Observe the expected mount through the same local disk handle that will
/// execute recovery. Call from a blocking context.
pub fn replacement_mount(&self, mount_roots: &[PathBuf]) -> Option<ReplacementMount> {
match self {
Self::Local(disk) => disk.get_disk().replacement_mount(mount_roots),
Self::Remote(_) => None,
}
}
pub async fn acquire_replacement_execution_lease(&self) -> Result<std::sync::Arc<local::ReplacementExecutionLease>> {
match self {
Self::Local(disk) => disk.get_disk().acquire_replacement_execution_lease().await,
+22 -13
View File
@@ -14,9 +14,12 @@
use super::{
DiskError, DiskStore, HEALING_MARKER_PATH, RUSTFS_META_BUCKET,
replacement_readiness::{ReplacementTargets, local_replacement_disks, probe_replacement_mounts},
replacement_readiness::{auto_replacement_target_identities, replacement_target_disk},
resume::ReplacementTargetIdentity,
storage_api::{EcstoreConditionalFileUpdate, EcstoreDiskAPI, EcstoreDiskBytes, EcstoreReplacementExecutionLease},
storage_api::{
EcstoreConditionalFileUpdate, EcstoreDiskAPI, EcstoreDiskBytes, EcstoreReplacementExecutionLease,
ecstore_local_disk_map_read,
},
};
use crate::{Error, Result};
use std::{future::Future, sync::Arc};
@@ -32,27 +35,33 @@ pub struct ReplacementExecution {
impl ReplacementExecution {
pub(crate) async fn acquire(targets: &[String]) -> Result<Arc<Self>> {
let targets = ReplacementTargets::open(targets)
.await
.ok_or_else(|| Error::ReplacementTargetNotReady("replacement target is unavailable".to_string()))?;
let identities = targets
.identities()
let identities = auto_replacement_target_identities(targets)
.await
.ok_or_else(|| Error::ReplacementTargetNotReady("target mount admission failed".to_string()))?;
let local_disks = ecstore_local_disk_map_read()
.await
.values()
.flatten()
.filter(|disk| EcstoreDiskAPI::is_local(disk.as_ref()))
.cloned()
.collect::<Vec<_>>();
let mut disks = Vec::with_capacity(identities.len());
let mut leases = Vec::with_capacity(identities.len());
// Take leases in endpoint order. Revalidate those same handles against
// the current inventory so a reopened disk cannot supply their proof.
for disk in &targets.disks {
// Identities are sorted by endpoint, the same order on every opener.
for identity in &identities {
let disk = replacement_target_disk(&identity.endpoint, &local_disks)
.await
.ok_or_else(|| Error::ReplacementTargetNotReady("replacement target is unavailable".to_string()))?;
leases.push(disk.acquire_replacement_execution_lease().await?);
disks.push(disk);
}
let local_disks = local_replacement_disks().await;
if probe_replacement_mounts(&targets.disks, &local_disks).await.as_ref() != Some(&identities) {
if auto_replacement_target_identities(targets).await.as_ref() != Some(&identities) {
return Err(Error::ReplacementTargetNotReady(
"target changed while acquiring execution leases".to_string(),
));
}
Ok(Arc::new(Self {
disks: targets.disks,
disks,
identities,
_leases: leases,
}))
+126 -203
View File
@@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use std::collections::HashSet;
use std::{fs, path::Path};
use super::{
DiskOption, DiskStore, Endpoint, HealDiskExt as _, local_disk_map_read, new_disk, resume::ReplacementTargetIdentity,
@@ -24,15 +24,12 @@ use super::{
/// Directory-backed deployments already declare, through
/// `RUSTFS_UNSAFE_BYPASS_DISK_CHECK`, that their endpoints are plain
/// directories sharing a device with the host root. Those endpoints can never
/// pass the independent-mount check, so without this fallback a
/// satisfy [`auto_replacement_target_identity`], so without this fallback a
/// runtime-wiped or replaced directory disk would stay deferred forever. The
/// admission check itself is never bypassed; the fallback only routes the heal
/// through the ordinary format path that formats every unformatted disk in the
/// set, which is exactly what the pre-admission `heal_disk` path did.
pub(crate) fn directory_backed_replacement_fallback_enabled() -> bool {
if cfg!(target_os = "linux") && std::env::var_os(rustfs_config::ENV_HEAL_REPLACEMENT_MOUNT_ROOTS).is_some() {
return false;
}
rustfs_utils::get_env_bool_with_aliases(
rustfs_config::ENV_UNSAFE_BYPASS_DISK_CHECK,
&[rustfs_config::ENV_MINIO_CI],
@@ -41,66 +38,53 @@ pub(crate) fn directory_backed_replacement_fallback_enabled() -> bool {
}
pub(crate) async fn auto_replacement_target_ready(disk: &DiskStore, local_disks: &[DiskStore]) -> bool {
probe_replacement_mounts(std::slice::from_ref(disk), local_disks)
.await
.is_some()
auto_replacement_target_identity(disk, local_disks).await.is_some()
}
pub(super) async fn probe_replacement_mounts(
disks: &[DiskStore],
pub(crate) async fn auto_replacement_target_identity(
disk: &DiskStore,
local_disks: &[DiskStore],
) -> Option<Vec<ReplacementTargetIdentity>> {
if disks.is_empty() {
return Some(Vec::new());
}
let disks = disks.to_vec();
let local_disks = local_disks.to_vec();
) -> Option<ReplacementTargetIdentity> {
let lease_root = disk.replacement_mount_lease_root()?;
let endpoint = disk.endpoint().to_string();
let sibling_lease_roots = local_disks
.iter()
.filter(|sibling| sibling.endpoint().is_local && sibling.endpoint().to_string() != endpoint)
.map(|sibling| sibling.replacement_mount_lease_root())
.collect::<Option<Vec<_>>>()?;
tokio::task::spawn_blocking(move || {
let mount_roots = std::env::var_os(rustfs_config::ENV_HEAL_REPLACEMENT_MOUNT_ROOTS)
.as_ref()
.into_iter()
.flat_map(std::env::split_paths)
.collect::<Vec<_>>();
let mut identities = Vec::with_capacity(disks.len());
for disk in &disks {
let mount = disk.replacement_mount(&mount_roots)?;
if mount.physical_device_ids.is_empty() {
return None;
}
identities.push(ReplacementTargetIdentity {
endpoint: disk.endpoint().to_string(),
canonical_path: mount.canonical_path.into_os_string().into_string().ok()?,
physical_device_ids: mount.physical_device_ids,
filesystem_identity: mount.filesystem_identity,
});
}
let mut target_devices = HashSet::new();
for target in &identities {
if target
.physical_device_ids
.iter()
.any(|device| !target_devices.insert(device.as_str()))
{
return None;
}
}
let root_devices = rustfs_utils::os::get_physical_device_ids("/").ok()?;
if root_devices.is_empty() || root_devices.iter().any(|device| target_devices.contains(device.as_str())) {
let canonical_path = fs::canonicalize(&lease_root).ok()?;
let metadata = fs::metadata(&lease_root).ok()?;
let Ok(target_device_ids) = rustfs_utils::os::get_physical_device_ids(lease_root.to_string_lossy().as_ref()) else {
return None;
};
let Ok(root_device_ids) = rustfs_utils::os::get_physical_device_ids("/") else {
return None;
};
if target_device_ids.is_empty()
|| root_device_ids.is_empty()
|| target_device_ids.iter().any(|target| root_device_ids.contains(target))
|| !rustfs_utils::os::is_mount_point(&canonical_path).unwrap_or(false)
{
return None;
}
let targets: HashSet<_> = identities.iter().map(|identity| identity.endpoint.as_str()).collect();
for sibling in &local_disks {
let endpoint = sibling.endpoint();
if endpoint.is_local && !targets.contains(endpoint.to_string().as_str()) {
let root = sibling.replacement_mount_lease_root()?;
let devices = rustfs_utils::os::get_physical_device_ids(root.to_str()?).ok()?;
if devices.iter().any(|device| target_devices.contains(device.as_str())) {
return None;
}
}
if sibling_lease_roots.iter().any(|sibling_lease_root| {
rustfs_utils::os::get_physical_device_ids(sibling_lease_root.to_string_lossy().as_ref())
.map(|ids| ids.iter().any(|id| target_device_ids.contains(id)))
.unwrap_or(true)
}) {
return None;
}
Some(identities)
let filesystem_identity = filesystem_identity(&metadata, &canonical_path)?;
Some(ReplacementTargetIdentity {
endpoint,
canonical_path: canonical_path.to_string_lossy().into_owned(),
physical_device_ids: target_device_ids,
filesystem_identity,
})
})
.await
.ok()
@@ -122,76 +106,72 @@ fn local_replacement_endpoint(target: &str, local_grid_hosts: &[String]) -> Opti
Some(endpoint)
}
pub(super) async fn local_replacement_disks() -> Vec<DiskStore> {
local_disk_map_read()
.await
pub(super) async fn replacement_target_disk(target: &str, local_disks: &[DiskStore]) -> Option<DiskStore> {
if let Some(disk) = local_disks.iter().find(|disk| disk.endpoint().to_string() == target) {
return Some(disk.clone());
}
let local_grid_hosts = local_disks.iter().map(|disk| disk.endpoint().grid_host()).collect::<Vec<_>>();
let endpoint = local_replacement_endpoint(target, &local_grid_hosts)?;
new_disk(
&endpoint,
&DiskOption {
cleanup: false,
health_check: false,
},
)
.await
.ok()
}
pub(crate) async fn auto_replacement_target_identities(targets: &[String]) -> Option<Vec<ReplacementTargetIdentity>> {
let local_disk_map = local_disk_map_read().await;
let local_disks = local_disk_map
.values()
.flatten()
.filter(|disk| disk.endpoint().is_local)
.cloned()
.collect()
.collect::<Vec<_>>();
drop(local_disk_map);
let mut identities = Vec::with_capacity(targets.len());
for target in targets {
let disk = replacement_target_disk(target, &local_disks).await?;
identities.push(auto_replacement_target_identity(&disk, &local_disks).await?);
}
identities.sort_by(|left, right| left.endpoint.cmp(&right.endpoint));
identities.dedup_by(|left, right| left.endpoint == right.endpoint);
(identities.len() == targets.len()).then_some(identities)
}
pub(super) struct ReplacementTargets {
pub(super) disks: Vec<DiskStore>,
local_disks: Vec<DiskStore>,
#[cfg(target_os = "linux")]
fn filesystem_identity(metadata: &fs::Metadata, canonical_path: &Path) -> Option<String> {
use std::os::unix::fs::MetadataExt as _;
let escaped_path = canonical_path.to_string_lossy().replace(' ', "\\040");
let mountinfo = fs::read_to_string("/proc/self/mountinfo").ok()?;
let mount_id = mountinfo.lines().find_map(|line| {
let mut fields = line.split_whitespace();
let mount_id = fields.next()?;
fields.next()?;
fields.next()?;
fields.next()?;
(fields.next()? == escaped_path).then_some(mount_id)
})?;
Some(format!("{mount_id}:{}:{}", metadata.dev(), metadata.ino()))
}
impl ReplacementTargets {
pub(super) async fn open(targets: &[String]) -> Option<Self> {
let mut ordered: Vec<_> = targets.iter().map(String::as_str).collect();
ordered.sort_unstable();
if ordered.windows(2).any(|pair| pair[0] == pair[1]) {
return None;
}
#[cfg(all(unix, not(target_os = "linux")))]
fn filesystem_identity(metadata: &fs::Metadata, _canonical_path: &Path) -> Option<String> {
use std::os::unix::fs::MetadataExt as _;
let registry = local_disk_map_read().await;
let requested: Vec<_> = ordered
.into_iter()
.map(|target| {
let disk = registry
.get(target)
.and_then(Option::as_ref)
.filter(|disk| disk.endpoint().is_local)
.cloned();
(target, disk)
})
.collect();
let local_disks = registry
.values()
.flatten()
.filter(|disk| disk.endpoint().is_local)
.cloned()
.collect::<Vec<_>>();
drop(registry);
let mut disks = Vec::with_capacity(requested.len());
let mut hosts = None;
for (target, disk) in requested {
let disk = match disk {
Some(disk) => disk,
None => {
let local_hosts = hosts
.get_or_insert_with(|| local_disks.iter().map(|disk| disk.endpoint().grid_host()).collect::<Vec<_>>());
let endpoint = local_replacement_endpoint(target, local_hosts)?;
new_disk(
&endpoint,
&DiskOption {
cleanup: false,
health_check: false,
},
)
.await
.ok()?
}
};
disks.push(disk);
}
Some(Self { disks, local_disks })
}
Some(format!("{}:{}", metadata.dev(), metadata.ino()))
}
pub(super) async fn identities(&self) -> Option<Vec<ReplacementTargetIdentity>> {
probe_replacement_mounts(&self.disks, &self.local_disks).await
}
#[cfg(not(unix))]
fn filesystem_identity(_metadata: &fs::Metadata, _canonical_path: &Path) -> Option<String> {
None
}
#[cfg(test)]
@@ -289,8 +269,8 @@ mod tests {
mod linux_privileged_tests {
use super::*;
use std::error::Error;
use std::path::Path;
use std::process::Command;
use std::{fs, path::Path};
const ENABLE_ENV: &str = "RUSTFS_PRIVILEGED_MOUNT_READINESS_TESTS";
const NAMESPACE_ENV: &str = "RUSTFS_PRIVILEGED_MOUNT_READINESS_TESTS_IN_NAMESPACE";
@@ -416,86 +396,39 @@ mod tests {
#[ignore = "requires Linux root/CAP_SYS_ADMIN and RUSTFS_PRIVILEGED_MOUNT_READINESS_TESTS=1"]
fn auto_replacement_readiness_accepts_an_independent_mount() -> Result<(), Box<dyn Error + Send + Sync>> {
run_privileged_mount_test(|mut mounts| async move {
let temp = TempDir::new()?;
let parent = temp.path().join("parent");
let target = parent.join("replacement disk");
let data = target.join("data");
let temp = TempDir::new().expect("temporary replacement roots should be created");
let target = temp.path().join("target");
let sibling = temp.path().join("sibling");
fs::create_dir(&parent)?;
mounts.mount_tmpfs(&parent, "rustfs-readiness-parent")?;
// Leave a directory at the same path below the target mount.
fs::create_dir_all(&data)?;
std::fs::create_dir(&target).expect("target mountpoint should be created");
std::fs::create_dir(&sibling).expect("sibling mountpoint should be created");
mounts.mount_tmpfs(&target, "rustfs-readiness-target")?;
fs::create_dir(&data)?;
fs::create_dir(&sibling)?;
mounts.mount_tmpfs(&sibling, "rustfs-readiness-sibling")?;
let target_endpoint = Endpoint::try_from(target.to_string_lossy().as_ref())?;
let sibling_endpoint = Endpoint::try_from(sibling.to_string_lossy().as_ref())?;
let options = DiskOption {
cleanup: false,
health_check: false,
};
let target_disk = new_disk(&target_endpoint, &options).await?;
let sibling_disk = new_disk(&sibling_endpoint, &options).await?;
assert!(
auto_replacement_target_ready(&target_disk, &[target_disk.clone(), sibling_disk.clone()]).await,
"a separately mounted replacement target with no sibling device overlap must be admitted"
);
let disks = [target_disk, sibling_disk];
let mut identities = probe_replacement_mounts(&disks, &disks)
.await
.expect("independently mounted replacement targets should be admitted together");
assert_eq!(identities[0].endpoint, disks[0].endpoint().to_string());
assert_eq!(identities[1].endpoint, disks[1].endpoint().to_string());
identities.sort_unstable_by(|left, right| left.endpoint.cmp(&right.endpoint));
let endpoints = [identities[1].endpoint.clone(), identities[0].endpoint.clone()];
let execution = crate::heal::replacement_execution::ReplacementExecution::acquire(&endpoints).await?;
assert_eq!(execution.identities(), identities);
assert!(
crate::heal::replacement_execution::ReplacementExecution::acquire(&endpoints)
.await
.is_err(),
"a second executor must not take leases on the same targets"
);
drop(execution);
let endpoint = Endpoint::try_from(path_to_string(&data, "data directory")?.as_str())?;
let disk = new_disk(&endpoint, &options).await?;
let key = rustfs_config::ENV_HEAL_REPLACEMENT_MOUNT_ROOTS;
temp_env::async_with_vars([(key, None::<&str>)], async {
assert!(!auto_replacement_target_ready(&disk, &[]).await);
})
.await;
let roots = std::env::join_paths([&parent, &target])?;
temp_env::async_with_vars(
[
(key, Some(roots.to_str().ok_or("mount roots are not UTF-8")?)),
(rustfs_config::ENV_UNSAFE_BYPASS_DISK_CHECK, Some("true")),
],
async {
assert!(!directory_backed_replacement_fallback_enabled());
let identities = probe_replacement_mounts(std::slice::from_ref(&disk), &[])
.await
.expect("data directory on its configured independent mount should be admitted");
assert_eq!(Path::new(&identities[0].canonical_path), data.canonicalize()?);
let reopened = new_disk(&endpoint, &options).await?;
assert_eq!(probe_replacement_mounts(&[reopened], &[]).await, Some(identities));
run_command("umount", &["-l", &path_to_string(&target, "replacement mount")?])?;
assert!(!auto_replacement_target_ready(&disk, &[]).await);
let exposed_disk = new_disk(&endpoint, &options).await?;
assert!(
!auto_replacement_target_ready(&exposed_disk, &[]).await,
"a new handle on the exposed parent filesystem must also be rejected"
);
Ok(())
let target_disk = new_disk(
&target_endpoint,
&DiskOption {
cleanup: false,
health_check: false,
},
)
.await
.await?;
let sibling_disk = new_disk(
&sibling_endpoint,
&DiskOption {
cleanup: false,
health_check: false,
},
)
.await?;
let identity = auto_replacement_target_identity(&target_disk, &[target_disk.clone(), sibling_disk.clone()]).await;
assert!(
identity.is_some(),
"a separately mounted replacement target with no sibling device overlap must be admitted"
);
Ok(())
})
}
@@ -534,20 +467,10 @@ mod tests {
.await?;
assert!(
!auto_replacement_target_ready(&target_disk, &[target_disk.clone(), sibling_disk.clone()]).await,
"replacement readiness must reject a target sharing its physical device with a sibling endpoint"
);
assert!(
probe_replacement_mounts(&[target_disk.clone(), sibling_disk], &[])
auto_replacement_target_identity(&target_disk, &[target_disk.clone(), sibling_disk.clone()])
.await
.is_none(),
"a batch must reject overlapping targets even before they are registered"
);
assert!(auto_replacement_target_ready(&target_disk, &[]).await);
mounts.mount_bind(&source, &target)?;
assert!(
!auto_replacement_target_ready(&target_disk, &[]).await,
"rebinding the same inode must invalidate the held mount identity"
"replacement readiness must reject a target sharing its physical device with a sibling endpoint"
);
Ok(())
})
+2 -8
View File
@@ -519,14 +519,8 @@ pub(crate) fn replacement_target_identities_match(
expected: &[ReplacementTargetIdentity],
actual: &[ReplacementTargetIdentity],
) -> bool {
if expected == actual {
return true;
}
if expected.len() != actual.len() {
return false;
}
let mut expected: Vec<_> = expected.iter().collect();
let mut actual: Vec<_> = actual.iter().collect();
let mut expected = expected.to_vec();
let mut actual = actual.to_vec();
expected.sort_by(|left, right| left.endpoint.cmp(&right.endpoint));
actual.sort_by(|left, right| left.endpoint.cmp(&right.endpoint));
expected == actual
+18 -11
View File
@@ -38,7 +38,7 @@ use super::storage_api::storage::{
ObjectIO as _, ObjectOperations as _, StorageAdminApi,
};
use super::{
DiskError, DiskStore, ECStore, HealDiskExt as _, StorageError,
DiskError, DiskStore, ECStore, HealDiskExt as _, StorageError, local_disk_map_read,
resume::{ReplacementRecoveryCandidate, ReplacementTargetIdentity, merge_replacement_recovery_candidate},
};
pub use super::{HealObjectInfo, HealObjectOptions, HealPutObjReader};
@@ -2457,13 +2457,9 @@ impl HealStorageAPI for ECStoreHealStorage {
}
async fn replacement_target_identities(&self, targets: &[String]) -> Result<Vec<ReplacementTargetIdentity>> {
let error = || Error::ReplacementTargetNotReady("replacement target is not a stable mounted disk".to_string());
super::replacement_readiness::ReplacementTargets::open(targets)
super::replacement_readiness::auto_replacement_target_identities(targets)
.await
.ok_or_else(error)?
.identities()
.await
.ok_or_else(error)
.ok_or_else(|| Error::ReplacementTargetNotReady("replacement target is not a stable mounted disk".to_string()))
}
async fn replacement_targets_ready_for_retry(
@@ -2472,14 +2468,25 @@ impl HealStorageAPI for ECStoreHealStorage {
targets: &[String],
expected_identities: &[ReplacementTargetIdentity],
) -> Result<bool> {
let Some(targets) = super::replacement_readiness::ReplacementTargets::open(targets).await else {
return Ok(false);
let identities = match self.replacement_target_identities(targets).await {
Ok(identities) => identities,
Err(_) => return Ok(false),
};
if targets.identities().await.as_deref() != Some(expected_identities) {
if identities != expected_identities {
return Ok(false);
}
for disk in targets.disks {
let local_disks = local_disk_map_read()
.await
.values()
.flatten()
.filter(|disk| disk.endpoint().is_local)
.cloned()
.collect::<Vec<_>>();
for target in targets {
let Some(disk) = local_disks.iter().find(|disk| disk.endpoint().to_string() == *target) else {
return Ok(false);
};
let endpoint = disk.endpoint();
if crate::heal::utils::format_set_disk_id_from_i32(endpoint.pool_idx, endpoint.set_idx).as_deref()
!= Some(set_disk_id)
@@ -228,10 +228,10 @@ impl HealTask {
// first target format write. A task retry keeps this id; a newly
// admitted blank replacement gets a fresh id and cannot reuse cursor
// progress from an older disk at the same endpoint.
let replacement_resume = if let Some(execution) = &replacement_execution {
self.check_control_flags().await?;
// Bind durable intent to the mounts whose execution leases we hold.
let identities = execution.identities().to_vec();
let replacement_resume = if is_auto_replacement {
let identities = self
.await_with_control(self.storage.replacement_target_identities(&self.heal_endpoints))
.await?;
let disk = replacement_resume_disk.clone().ok_or_else(|| Error::TaskExecutionFailed {
message: format!("Replacement resume disk is missing for automatic heal {set_disk_id}"),
})?;
+5 -35
View File
@@ -1258,40 +1258,6 @@ async fn recovered_replacement_never_uses_a_fresh_resume_disk() {
assert!(!*storage.listed.lock().unwrap(), "an unavailable durable anchor must not list buckets");
}
#[tokio::test]
async fn automatic_replacement_does_not_format_a_mount_outside_its_execution_lease() {
let temp = TempDir::new().expect("resume disk directory");
let disk = make_resume_disk(&temp).await;
let leased = replacement_identity("replacement-a", "device-a", "mount-a");
let remounted = replacement_identity("replacement-a", "device-a", "mount-b");
let storage = Arc::new(MockStorage {
bucket_incarnation_id: Mutex::new(Some(Uuid::from_u128(1))),
replacement_target_identities_ready: Mutex::new(true),
replacement_target_identity_sequences: Mutex::new(VecDeque::from([vec![remounted.clone()], vec![remounted]])),
replacement_execution_fixture: Mutex::new(Some(crate::heal::storage::ReplacementExecution::for_test(
Vec::new(),
vec![leased.clone()],
))),
resume_disk: Mutex::new(Some(disk.clone())),
..Default::default()
});
let mut request = directory_backed_replacement_request();
request.heal_endpoints = vec!["replacement-a".to_string()];
let task = HealTask::from_request(request, storage.clone());
let error = task.execute().await.expect_err("a remount must be rejected before format");
assert!(error.to_string().contains("changed during format"), "{error}");
assert!(storage.replacement_format_calls.lock().unwrap().is_empty());
let state = ResumeManager::load_replacement_intent(disk, &task.id)
.await
.expect("durable intent should identify the leased target")
.get_state()
.await;
assert_eq!(state.replacement_phase, ReplacementPhase::Intent);
assert_eq!(state.replacement_target_identities, vec![leased]);
}
#[tokio::test]
async fn automatic_replacement_rejects_a_new_identity_after_format() {
let temp = TempDir::new().expect("temporary resume disk directory should be created");
@@ -1301,7 +1267,11 @@ async fn automatic_replacement_rejects_a_new_identity_after_format() {
let storage = Arc::new(MockStorage {
bucket_incarnation_id: Mutex::new(Some(Uuid::from_u128(1))),
replacement_target_identities_ready: Mutex::new(true),
replacement_target_identity_sequences: Mutex::new(VecDeque::from([vec![first_identity.clone()], vec![second_identity]])),
replacement_target_identity_sequences: Mutex::new(VecDeque::from([
vec![first_identity.clone()],
vec![first_identity.clone()],
vec![second_identity],
])),
resume_disk: Mutex::new(Some(disk.clone())),
..Default::default()
});
+5
View File
@@ -38,6 +38,8 @@ default = []
# event. Requires `RUSTFLAGS="--cfg hyper_unstable_tracing"`.
diagnostic-internode-http2-ping = ["dep:hyper", "rustfs-io-metrics/diagnostic-internode-http2-ping"]
hotpath = [
"hotpath/hotpath",
"hotpath/tokio",
"rustfs-common/hotpath",
"rustfs-config/hotpath",
"rustfs-io-metrics/hotpath",
@@ -46,6 +48,7 @@ hotpath = [
]
hotpath-alloc = [
"hotpath",
"hotpath/hotpath-alloc",
"rustfs-common/hotpath-alloc",
"rustfs-config/hotpath-alloc",
"rustfs-io-metrics/hotpath-alloc",
@@ -54,6 +57,7 @@ hotpath-alloc = [
]
hotpath-cpu = [
"hotpath",
"hotpath/hotpath-cpu",
"rustfs-common/hotpath-cpu",
"rustfs-config/hotpath-cpu",
"rustfs-io-metrics/hotpath-cpu",
@@ -62,6 +66,7 @@ hotpath-cpu = [
]
[dependencies]
hotpath.workspace = true
rustfs-common.workspace = true
rustfs-heal-contracts.workspace = true
rustfs-io-metrics.workspace = true
+1 -1
View File
@@ -59,7 +59,7 @@ hotpath-cpu = [
]
[dependencies]
hotpath = { workspace = true, optional = true }
hotpath.workspace = true
arc-swap.workspace = true
tokio = { workspace = true, features = ["io-util", "macros", "net", "rt-multi-thread", "sync", "time"] }
rand = { workspace = true, features = ["serde"] }
@@ -232,7 +232,7 @@ Heal selects the authoritative version from quorum metadata, refuses when more s
### 10.2 Drive replacement
1. Confirm the affected set still meets read quorum for its objects (impossible at `EC:0`, see §5) and record pool index, set index, slot, and the failed disk's UUID from the pool status.
2. Replace the drive at the **same** endpoint and mount path with an empty filesystem owned by RustFS. Do not copy the old drive's directories. A copied `format.json` claims an identity the set already tracks. On Linux, automatic replacement checks that the endpoint is an independent mount by default. When the endpoint is a data directory below the mount, set `RUSTFS_HEAL_REPLACEMENT_MOUNT_ROOTS` on that node to the colon-separated absolute mount paths, for example `/srv/disk1:/srv/disk2` for endpoints `/srv/disk1/rustfs` and `/srv/disk2/rustfs`. Use canonical mount paths without trailing slashes. The most specific matching root must be the active mount that contains the endpoint. The root must remain independent of the system and sibling drives. Setting this variable disables the unsafe directory-backed heal fallback on that node.
2. Replace the drive at the **same** endpoint and mount path with an empty filesystem owned by RustFS. Do not copy the old drive's directories; a copied `format.json` claims an identity the set already tracks.
3. Bring the node back. The empty drive is not a format-quorum member, so `retain_format_quorum_members` keeps its slot offline; the endpoint monitor (`monitor_and_connect_endpoints_task` in `crates/ecstore/src/core/sets.rs`) retries the connection every 15 seconds. The erasure-set heal task (`crates/heal/src/heal/task/heal_erasure_set.rs`) writes the format into the original slot, through `heal_replacement_format` when an automatic replacement intent exists (schema 7 records under `.rustfs.sys/buckets/ahm-replacement/`, created when startup or the disk scanner observes a changed mount identity) or through `heal_format` for an admin erasure-set heal (`POST /rustfs/admin/v3/heal/` with `pool` and `set`), and then rebuilds the shards. Legacy schema 5/6 intents follow [replacement-generation-recovery.md](replacement-generation-recovery.md). If the drive held the node's pool metadata, follow [pool-metadata-recovery.md](pool-metadata-recovery.md#disk-replacement-and-metadata-erasure) first.
4. Watch `GET /rustfs/admin/v4/heal/replacement-recovery`, background heal status, disk health, and bitrot counters. Completion criteria are in [scanner-runtime-controls.md](scanner-runtime-controls.md#replacement-recovery-completion): an `idle` queue or a readable object is not proof.
5. The replacement is done when recovery reports `completed` for that instance and sampled objects have `xl.meta` and parts on the new drive.
@@ -66,8 +66,6 @@ An existing deployment with committed, initialized identity and unchanged topolo
During startup, an unreadable identity or `pool.bin` replica returns typed `ReadUnavailable` before a partial metadata view is installed or the object store is published. Initialization retries from the complete identity and metadata read at the existing five-second retry interval, within the existing startup attempt budget. A later readable result must still pass the original identity, generation, topology, conditional-write and readback checks. Read-only unavailability does not create a permanent startup block; an existing validation or uncertain-write block is never cleared by a clean retry.
When native recovery reports `no_ordinary_v3_authority`, no ordinary V3 COMMIT or eligible PREPARE proves a generation to restore. `missing_pool_metadata_slot` and `missing_pool_identity_slot` identify absent members of the corresponding erasure sets. `missing_pool_metadata_head` identifies an inspected member without a current metadata shard. `pool_identity_unavailable` means an ordinary identity read could not establish a valid replica. Transport errors retain their original storage error. Investigate the failing stage and preserve the surviving drives before attempting another replacement.
The node checks for interrupted pool metadata transactions every five seconds. Failed recovery attempts back off to at most sixty seconds; each attempt has a thirty-second budget and stops on shutdown. Healthy nodes do not read metadata for this worker. Recovery:
1. Cancels old decommission workers and waits for their supervisors to drain them. It does not cancel a separate rebalance operation; an attached rebalance worker prevents recovery until quiescent.
+17
View File
@@ -0,0 +1,17 @@
# S3 stack upgrade and rollback
The gateway release serves S3 with rustfs/gateway. Remove `RUSTFS_S3_STACK` from service units, container environments and Helm values before upgrading. Even an empty setting prevents startup. Keep the previous release binary or image digest with the deployment configuration.
## Roll back to the previous release
1. Check the target release against the [storage format version floors](rolling-restart.md#version-floors-that-break-mixed-version-fleets). If the cluster has written data or metadata that the target cannot read, stop here. The S3 stack switch itself requires no data migration.
2. Stop one node, restore the previous RustFS binary or image and start it with the same volumes and configuration. Follow the [rolling restart procedure](rolling-restart.md#rolling-restart-procedure). Wait for the node to report ready and for its peers to report its drives online before replacing another node.
3. Through the restored node, read an object written after the upgrade and compare its contents or checksum. Read a bucket configuration used by the application. If an upload was in progress, list its parts and compare their numbers, sizes and ETags with the client's record. Pause the rollout if any check fails.
## Client-visible error change
| Conformance case | What clients see |
| --- | --- |
| `c-sig-0598` (R-053) | For an access key the S3 parser cannot read, both versions return HTTP 403. The gateway returns the XML error code `AccessDenied`. The legacy binary returns `InvalidAccessKeyId`. Clients that classify these errors by code need to handle both during a rollout. |
The case is recorded in the [pinned gateway conformance rulings](https://github.com/rustfs/gateway/blob/60dcd2a55713dbcc9dbd6e165922e1a0df4473d7/conformance/rulings/rustfs-profile.toml#L298-L304).
+9 -9
View File
@@ -332,7 +332,7 @@ fn is_lock_timeout(err: &crate::storage_api::error::StorageError) -> bool {
fn decommission_start_api_error(err: crate::storage_api::error::StorageError) -> ApiError {
if crate::storage_api::capacity::is_pool_activation_fleet_proof_error(&err) {
return ApiError {
code: S3ErrorCode::InternalError.into(),
code: S3ErrorCode::InternalError,
message: POOL_ACTIVATION_FLEET_PROOF_REQUIRED.to_string(),
source: Some(Box::new(err)),
};
@@ -341,7 +341,7 @@ fn decommission_start_api_error(err: crate::storage_api::error::StorageError) ->
// movement gate has a queued writer: the same transient contention.
if is_lock_timeout(&err) || matches!(err, crate::storage_api::error::StorageError::SlowDown) {
return ApiError {
code: S3ErrorCode::ServiceUnavailable.into(),
code: S3ErrorCode::ServiceUnavailable,
message: err.to_string(),
source: Some(Box::new(err)),
};
@@ -350,7 +350,7 @@ fn decommission_start_api_error(err: crate::storage_api::error::StorageError) ->
// should say so, not just "Invalid Request".
if matches!(err, crate::storage_api::error::StorageError::DecommissionAlreadyRunning) {
return ApiError {
code: S3ErrorCode::InvalidRequest.into(),
code: S3ErrorCode::InvalidRequest,
message: err.to_string(),
source: Some(Box::new(err)),
};
@@ -1261,14 +1261,14 @@ mod pools_handler_tests {
#[test]
fn test_contextualize_admin_pool_api_error_preserves_code_and_adds_pool_context() {
let err = crate::error::ApiError {
code: rustfs_s3_types::S3ErrorCode::InvalidRequest.into(),
code: rustfs_s3_types::S3ErrorCode::InvalidRequest,
message: "decommission already running".to_string(),
source: None,
};
let err = contextualize_admin_pool_api_error(err, "start decommission", "pools [1, 3]");
assert_eq!(err.code, rustfs_s3_types::S3ErrorCode::InvalidRequest.into());
assert_eq!(err.code, rustfs_s3_types::S3ErrorCode::InvalidRequest);
assert_eq!(
err.message,
"admin start decommission failed for pools [1, 3]: decommission already running"
@@ -1281,12 +1281,12 @@ mod pools_handler_tests {
let err = decommission_start_api_error(err);
assert_eq!(err.code, rustfs_s3_types::S3ErrorCode::InternalError.into());
assert_eq!(err.code, rustfs_s3_types::S3ErrorCode::InternalError);
assert_eq!(err.message, POOL_ACTIVATION_FLEET_PROOF_REQUIRED);
assert!(err.source.is_some());
let unrelated = decommission_start_api_error(crate::storage_api::error::StorageError::other("disk read failed"));
assert_eq!(unrelated.code, rustfs_s3_types::S3ErrorCode::InternalError.into());
assert_eq!(unrelated.code, rustfs_s3_types::S3ErrorCode::InternalError);
assert_eq!(unrelated.message, "We encountered an internal error, please try again.");
}
@@ -1299,7 +1299,7 @@ mod pools_handler_tests {
let err = decommission_start_api_error(err);
assert_eq!(err.code, S3ErrorCode::ServiceUnavailable.into());
assert_eq!(err.code, S3ErrorCode::ServiceUnavailable);
assert!(err.message.contains("data movement gate (decommission activation)"), "{}", err.message);
let err = rustfs_s3_types::S3Error::from(contextualize_admin_pool_api_error(err, "start decommission", "pools [0]"));
assert_eq!(
@@ -1313,7 +1313,7 @@ mod pools_handler_tests {
fn test_decommission_start_api_error_reports_a_refused_preflight_heal_as_retryable() {
let err = decommission_start_api_error(crate::storage_api::error::StorageError::SlowDown);
assert_eq!(err.code, S3ErrorCode::ServiceUnavailable.into());
assert_eq!(err.code, S3ErrorCode::ServiceUnavailable);
let err = rustfs_s3_types::S3Error::from(contextualize_admin_pool_api_error(err, "start decommission", "pools [0]"));
assert_eq!(err.status_code(), Some(http::StatusCode::SERVICE_UNAVAILABLE));
}
+1 -1
View File
@@ -37,7 +37,7 @@ use crate::server::collect_dependency_readiness_report as collect_runtime_depend
use crate::shared_types::DependencyReadiness;
use rustfs_data_usage::DataUsageInfo;
use rustfs_madmin::{InfoMessage, StorageInfo};
use s3s::S3ErrorCode;
use rustfs_s3_types::S3ErrorCode;
use std::collections::HashSet;
use std::sync::Arc;
use tracing::{debug, error, info, warn};
+4 -3
View File
@@ -98,14 +98,13 @@ use rustfs_policy::policy::{
{BucketPolicy, BucketPolicyArgs, Effect},
};
use rustfs_s3_ops::S3Operation;
use rustfs_s3_types::Region;
use rustfs_s3_types::{Region, S3Error, S3ErrorCode, S3Result};
use rustfs_targets::{
EventName,
arn::{ARN, TargetID, TargetIDError},
};
use rustfs_utils::http::get_header;
use rustfs_utils::obj::extract_user_defined_metadata;
use s3s::{S3Error, S3ErrorCode, S3Result};
use std::{
borrow::Cow,
collections::{HashMap, HashSet},
@@ -907,7 +906,9 @@ mod tests {
.expect_err("full queue rejects immediately");
assert_eq!(error.code(), &S3ErrorCode::SlowDown);
assert_eq!(error.headers().expect("retry headers")[http::header::RETRY_AFTER], "1");
let response = error.to_http_response().expect("serialize admission error");
let response = s3s::S3Error::from(error)
.to_http_response()
.expect("serialize admission error");
assert_eq!(response.status(), http::StatusCode::SERVICE_UNAVAILABLE);
assert_eq!(response.headers()[http::header::RETRY_AFTER], "1");
use http_body_util::BodyExt as _;
@@ -116,7 +116,7 @@ impl DefaultBucketUsecase {
async fn list_objects_page(&self, envelope: &mut RequestEnvelope, request: &PageRequest) -> S3Result<Page> {
let PageRequest { bucket, params, .. } = request;
validate_list_object_unordered_with_delimiter(params.delimiter.as_ref(), envelope.uri().query())?;
validate_list_object_unordered_with_delimiter(params.delimiter.as_deref(), envelope.uri().query())?;
let store = get_validated_store(bucket).await?;
let incl_deleted = get_header(envelope.headers(), rustfs_utils::http::SUFFIX_INCLUDE_DELETED).as_deref() == Some("true");
let table_list_access = envelope.table_list_access().cloned();
+1 -1
View File
@@ -626,7 +626,7 @@ impl DefaultBucketUsecase {
let params = parse_list_objects_v2_params(prefix, delimiter, max_keys, continuation_token, start_after)?;
validate_list_object_unordered_with_delimiter(params.delimiter.as_ref(), envelope.uri().query())?;
validate_list_object_unordered_with_delimiter(params.delimiter.as_deref(), envelope.uri().query())?;
let store = get_validated_store(&bucket).await?;
let incl_deleted = get_header(envelope.headers(), rustfs_utils::http::SUFFIX_INCLUDE_DELETED)
+15 -7
View File
@@ -103,6 +103,7 @@ use futures::StreamExt;
use http::{HeaderMap, Uri};
use metrics::counter;
use rustfs_s3_ops::S3Operation;
use rustfs_s3_types::{S3Error, S3ErrorCode, S3Result, s3_error};
use rustfs_targets::EventName;
use rustfs_utils::CompressionAlgorithm;
#[cfg(test)]
@@ -114,7 +115,6 @@ use rustfs_utils::http::{
headers::{AMZ_CHECKSUM_TYPE, AMZ_OBJECT_TAGGING, AMZ_STORAGE_CLASS},
insert_str,
};
use s3s::{S3Error, S3ErrorCode, S3Result, s3_error};
use std::collections::{HashMap, HashSet};
use std::sync::Arc;
use std::time::Duration;
@@ -192,12 +192,13 @@ fn multipart_max_total_object_size(metadata: &HashMap<String, String>) -> S3Resu
let value = get_consistent_str(metadata, SUFFIX_MAX_TOTAL_OBJECT_SIZE).ok_or_else(|| {
S3Error::with_message(
S3ErrorCode::InvalidRequest,
"multipart size capability metadata is missing or inconsistent".to_string(),
"multipart size capability metadata is missing or inconsistent",
)
})?;
value.parse::<u64>().map(Some).map_err(|_| {
S3Error::with_message(S3ErrorCode::InvalidRequest, "multipart size capability metadata is invalid".to_string())
})
value
.parse::<u64>()
.map(Some)
.map_err(|_| S3Error::with_message(S3ErrorCode::InvalidRequest, "multipart size capability metadata is invalid"))
}
async fn validate_table_catalog_object_mutation(bucket: &str, key: &str) -> S3Result<()> {
@@ -290,7 +291,11 @@ impl DefaultMultipartUsecase {
/// or completed".
fn multipart_upload_api_error(err: StorageError) -> S3Error {
if matches!(err, StorageError::MalformedUploadID(_)) {
return S3Error::new(S3ErrorCode::NoSuchUpload);
let mut error = S3Error::new(S3ErrorCode::NoSuchUpload);
if let Some(message) = error.code().default_message() {
error.set_message(message);
}
return error;
}
ApiError::from(err).into()
}
@@ -544,7 +549,7 @@ mod tests {
.execute_create_multipart_upload_from_s3s(build_request(create, Method::POST))
.await
.expect_err("reserved key must be rejected before accessing storage");
assert_eq!(error.code(), &S3ErrorCode::InvalidArgument);
assert_eq!(error.code(), &crate::app::storage_api::s3::S3ErrorCode::InvalidArgument);
let complete = CompleteMultipartUploadInput::builder()
.bucket("bucket".to_owned())
@@ -592,6 +597,9 @@ mod tests {
fn multipart_upload_api_error_reports_malformed_upload_id_as_no_such_upload() {
let malformed = multipart_upload_api_error(StorageError::MalformedUploadID("bucket/object-not-an-id".to_string()));
assert_eq!(malformed.code(), &S3ErrorCode::NoSuchUpload);
let legacy = crate::app::storage_api::s3::S3Error::new(crate::app::storage_api::s3::S3ErrorCode::NoSuchUpload);
assert_eq!(malformed.message(), legacy.message());
assert_eq!(malformed.status_code(), legacy.status_code());
let unknown = multipart_upload_api_error(StorageError::InvalidUploadID(
"bucket".to_string(),
+2 -2
View File
@@ -79,7 +79,7 @@ where
Ok::<_, StorageError>(page.recorded_owner.take().map(|owner| RecordedListParts { owner, page }))
})
.await
.map_err(|error| S3Error::from(multipart_upload_api_error(error)))?
.map_err(multipart_upload_api_error)?
.map_err(|refusal| S3Error::with_message(S3ErrorCode::NoSuchUpload, refusal.reason()))?;
Ok(record.page)
}
@@ -198,7 +198,7 @@ impl DefaultMultipartUsecase {
ListPartsTarget::Legacy { upload_id, .. } => store
.list_object_parts(bucket, key, upload_id, params.part_number_marker, params.max_parts, &opts)
.await
.map_err(|error| multipart_upload_api_error(error).into()),
.map_err(multipart_upload_api_error),
ListPartsTarget::Gateway {
bucket: bucket_name,
key: object_key,
@@ -165,7 +165,7 @@ fn list_parts_resolves_real_owners_and_retains_storage_errors() {
storage_error,
StorageError::InvalidUploadID(..) | StorageError::MalformedUploadID(..)
));
let expected = multipart_upload_api_error(storage_error);
let expected: LegacyS3Error = ApiError::from(storage_error).into();
for target in [
gateway_fields(&bucket, key, id, None, None),
ListPartsFields {
@@ -43,7 +43,7 @@ impl RecordedUpload for PartUpload {
async fn lookup_part_upload(store: &ECStore, bucket: &str, key: &str, id: &str, opts: &ObjectOptions) -> S3Result<MultipartInfo> {
let result = store.get_multipart_info(bucket, key, id, opts).await;
result.map_err(|error| multipart_upload_api_error(error).into())
result.map_err(multipart_upload_api_error)
}
pub(super) async fn lookup_recorded_part_upload(
@@ -56,7 +56,7 @@ pub(super) async fn lookup_recorded_part_upload(
let result = store.lookup_multipart_part_binding(bucket, key, id, opts).await;
#[cfg(test)]
let result = tests::observe_lookup(result);
match result.map_err(|error| S3Error::from(multipart_upload_api_error(error))) {
match result.map_err(multipart_upload_api_error) {
Ok((info, owner)) => Ok(owner.map(|owner| PartUpload { owner, info })),
Err(error) if error.code() == &S3ErrorCode::NoSuchUpload => Ok(None),
Err(error) => Err(error),
@@ -233,7 +233,8 @@ fn native_copy_part_rejects_unproved_owner_without_changing_parts() {
} else if matches!(mode, LookupResult::Outage) {
let Err(error) = result else { panic!("expected read quorum failure") };
let actual: crate::app::storage_api::s3::S3Error = error.into();
let expected = multipart_upload_api_error(StorageError::ErasureReadQuorum);
let expected: crate::app::storage_api::s3::S3Error =
ApiError::from(StorageError::ErasureReadQuorum).into();
assert_eq!(actual.code(), expected.code());
assert_eq!(actual.status_code(), expected.status_code());
assert_eq!(actual.message(), expected.message());
+2 -2
View File
@@ -1051,8 +1051,8 @@ mod tests {
.expect_err("short copy source must fail before destination write succeeds");
let api_error = ApiError::from(err);
assert_eq!(api_error.code, S3ErrorCode::ServiceUnavailable);
assert_ne!(api_error.code, S3ErrorCode::IncompleteBody);
assert_eq!(api_error.code, rustfs_s3_types::S3ErrorCode::ServiceUnavailable);
assert_ne!(api_error.code, rustfs_s3_types::S3ErrorCode::IncompleteBody);
}
#[tokio::test]
+4 -5
View File
@@ -503,7 +503,7 @@ fn project_delete_objects_pre_stat_error(
error: ApiError,
) -> S3Result<DeleteBatchError> {
// Bucket loss invalidates the shared request, not just one object.
if error.code == S3ErrorCode::NoSuchBucket {
if error.code == rustfs_s3_types::S3ErrorCode::NoSuchBucket {
return Err(error.into());
}
Ok(DeleteBatchError {
@@ -676,10 +676,9 @@ impl DefaultObjectUsecase {
let key = normalize_delete_object_key(&obj_id.key);
// A rejected wire key has no authorization resource. Its normalized text cannot become a target.
let key_error = if obj_id.key_too_long {
Some((
S3ErrorCode::InvalidArgument,
ApiError::error_code_to_message(&S3ErrorCode::InvalidArgument),
))
let code = rustfs_s3_types::S3ErrorCode::InvalidArgument;
let message = ApiError::error_code_to_message(&code);
Some((code, message))
} else {
validate_object_key(&key, "DeleteObjects")
.err()
+3 -3
View File
@@ -1250,7 +1250,7 @@ pub fn normalize_extract_entry_key(path: &str, prefix: Option<&str>, is_dir: boo
fn map_extract_archive_error(err: std::io::Error) -> S3Error {
let api_error = ApiError::from(err);
if api_error.code != S3ErrorCode::InternalError {
if api_error.code != rustfs_s3_types::S3ErrorCode::InternalError {
return api_error.into();
}
@@ -1323,7 +1323,7 @@ fn is_empty_extract_entry_path(path: &str) -> bool {
fn validate_extract_member_key(path: &str, limits: ArchiveLimits) -> Result<(), ExtractEntryError> {
validate_put_object_extract_entry_path(path, limits).map_err(ExtractEntryError::Recoverable)?;
validate_new_object_key(path, "PUT").map_err(ExtractEntryError::Recoverable)
validate_new_object_key(path, "PUT").map_err(|err| ExtractEntryError::Recoverable(err.into()))
}
fn record_extract_pax_metadata_bytes(
@@ -3666,7 +3666,7 @@ mod tests {
] {
let error = map_extract_archive_error(error);
assert_eq!(error.code(), &code);
assert_eq!(error.message(), Some(ApiError::error_code_to_message(&code).as_str()));
assert_eq!(error.message(), Some(ApiError::error_code_to_message(&code.into()).as_str()));
assert!(error.source().is_some());
}
let error = map_extract_archive_error(io::Error::new(io::ErrorKind::InvalidData, "invalid TAR header"));
+4 -3
View File
@@ -2602,8 +2602,9 @@ impl DefaultObjectUsecase {
};
return Err(with_delete_marker_read_headers(S3Error::new(code), info, input.version_id.as_deref()));
}
let result =
check_preconditions(envelope.headers(), info).and_then(|()| Self::validate_get_object_part_number(part_number, info));
let result = check_preconditions(envelope.headers(), info)
.map_err(S3Error::from)
.and_then(|()| Self::validate_get_object_part_number(part_number, info));
if result.is_err() {
let principal = SseKmsPrincipal::from_request(envelope);
crate::storage::sse::validate_conditional_read(
@@ -12313,7 +12314,7 @@ mod tests {
fn parse_get_object_part_number_rejects_above_s3_max() {
let err = parse_part_number_i32_to_usize(Some(10001), "GET").expect_err("partNumber above S3 max must fail");
assert_eq!(err.code(), &S3ErrorCode::InvalidArgument);
assert_eq!(err.code(), &rustfs_s3_types::S3ErrorCode::InvalidArgument);
assert_eq!(err.message(), Some("GET: partNumber must be between 1 and 10000"));
}
+5 -4
View File
@@ -22,7 +22,7 @@
//! multipart handlers' policy steps against the same storage contract.
use super::*;
use crate::app::storage_api::s3::{S3Error, S3ErrorCode};
use rustfs_s3_types::{S3Error, S3ErrorCode};
use crate::app::object_data_cache::invalidate_object_data_cache_after_complete_multipart_success;
use crate::app::runtime_sources::current_app_context;
@@ -80,7 +80,8 @@ pub(crate) struct InternalPutContext {
const INTERNAL_PUT_METHOD_NAME: &str = "PUT";
fn api_error_from_s3(err: S3Error) -> ApiError {
fn api_error_from_s3(err: impl Into<S3Error>) -> ApiError {
let err = err.into();
ApiError {
code: err.code().clone(),
message: err.message().unwrap_or_default().to_string(),
@@ -139,7 +140,7 @@ async fn validate_internal_write_target(key: &str, bucket: &str, headers: &Heade
headers.get("content-type").and_then(|value| value.to_str().ok()),
headers.get("content-encoding").and_then(|value| value.to_str().ok()),
)
.map_err(|err| api_error_from_s3(err.into()))
.map_err(api_error_from_s3)
}
fn internal_events_wanted() -> bool {
@@ -333,7 +334,7 @@ impl DefaultObjectUsecase {
.put_object_core(write, super::native_body::from_io_stream(body), None, start_time)
.await
.map_err(|error| ApiError {
code: error.code().clone().into(),
code: error.code().clone(),
message: error.message().unwrap_or_default().to_owned(),
source: None,
})?;
@@ -32,7 +32,7 @@
//! `odm-source-etag`.
use super::*;
use crate::app::storage_api::s3::S3ErrorCode;
use rustfs_s3_types::S3ErrorCode;
use crate::app::storage_api::multipart_usecase::contract::multipart::CompletePart;
use crate::on_demand_migration::{
+2 -2
View File
@@ -150,7 +150,7 @@ pub(crate) fn reject_oversize_single_upload(size: i64) -> S3Result<()> {
if u64::try_from(size).is_ok_and(|size| size > rustfs_config::MAX_SINGLE_PUT_OBJECT_SIZE) {
return Err(S3Error::with_message(
S3ErrorCode::EntityTooLarge,
ApiError::error_code_to_message(&S3ErrorCode::EntityTooLarge),
ApiError::error_code_to_message(&rustfs_s3_types::S3ErrorCode::EntityTooLarge),
));
}
Ok(())
@@ -5669,7 +5669,7 @@ mod oversize_single_upload_tests {
assert_eq!(*err.code(), S3ErrorCode::EntityTooLarge);
assert_eq!(
err.message(),
Some(ApiError::error_code_to_message(&S3ErrorCode::EntityTooLarge).as_str())
Some(ApiError::error_code_to_message(&rustfs_s3_types::S3ErrorCode::EntityTooLarge).as_str())
);
}
}
+2 -1
View File
@@ -13,10 +13,11 @@
// limitations under the License.
use super::*;
use crate::app::storage_api::s3::{Body as S3Body, S3Error, S3ErrorCode, StreamingBlob};
use crate::app::storage_api::s3::{Body as S3Body, S3Error, StreamingBlob};
use crate::error::ApiError;
use futures::{StreamExt, poll};
use http_body_util::StreamBody;
use rustfs_s3_types::S3ErrorCode;
use std::io;
use tokio::io::AsyncReadExt;
use tokio::sync::mpsc;
+1 -1
View File
@@ -1136,7 +1136,7 @@ pub(crate) fn odm_check_source_preconditions(headers: &HeaderMap, head: &SourceH
user_defined: metadata.into(),
..Default::default()
};
check_preconditions(headers, &info)
Ok(check_preconditions(headers, &info)?)
}
/// S3 documents write-side `If-None-Match` only as `*` (create-only).
+1 -1
View File
@@ -69,7 +69,7 @@ impl ObjectMetaUsecase {
.get_bucket_info(bucket, &BucketOptions::default())
.await
.map_err(ApiError::from)?;
return Err(err.into());
return Err(err);
}
let opts: ObjectOptions = get_opts(bucket, key, version_id.clone(), None, envelope.headers())
+3 -3
View File
@@ -4,8 +4,8 @@ use super::storage_api::select_object::options::get_opts;
use super::storage_api::select_object::request_context::spawn_traced;
use super::storage_api::select_object::sse::{SseKmsPrincipal, authorize_sse_kms_object_read, project_sse_read_response_headers};
use super::storage_api::select_object::{
StoragePrepareSelectObjectSnapshotError, StorageSelectObjectSnapshot, get_validated_store_owned,
validate_sse_headers_for_read, validate_ssec_for_read,
StoragePrepareSelectObjectSnapshotError, StorageSelectObjectSnapshot, get_validated_store, validate_sse_headers_for_read,
validate_ssec_for_read,
};
use crate::app::runtime_sources::current_s3select_db;
use crate::app::{Answered, RequestEnvelope};
@@ -874,7 +874,7 @@ async fn prepare_select_object_snapshot(
let opts = get_opts(&input.bucket, &input.key, None, None, headers)
.await
.map_err(ApiError::from)?;
let store = get_validated_store_owned(&input.bucket).await?;
let store = get_validated_store(&input.bucket).await?;
let snapshot = store
.prepare_select_object_snapshot(&input.bucket, &input.key, headers, &opts)
.await
+2 -2
View File
@@ -1900,8 +1900,8 @@ pub(crate) mod select_object {
#[cfg(test)]
pub(crate) use crate::storage::storage_api::StorageError;
pub(crate) use crate::storage::storage_api::{
StoragePrepareSelectObjectSnapshotError, StorageSelectObjectSnapshot, get_validated_store_owned,
validate_sse_headers_for_read, validate_ssec_for_read,
StoragePrepareSelectObjectSnapshotError, StorageSelectObjectSnapshot, get_validated_store, validate_sse_headers_for_read,
validate_ssec_for_read,
};
}
+54 -30
View File
@@ -17,8 +17,9 @@ use crate::storage_api::error::{PoolMetadataError, QuotaError, StorageError, unr
use http::StatusCode;
use rustfs_kms::KmsUnavailableError;
use rustfs_lock::LockError;
use rustfs_s3_types::S3ErrorCode;
use s3s::S3Error as LegacyS3Error;
use s3s::stream::{aws_chunked_stream::AwsChunkedStreamError, upload_stream::UploadStreamError};
use s3s::{S3Error, S3ErrorCode};
const CONTENT_SHA256_MISMATCH_CODE: &str = "XAmzContentSHA256Mismatch";
const CONTENT_SHA256_MISMATCH_MESSAGE: &str = "The provided 'x-amz-content-sha256' header does not match what was computed.";
@@ -347,7 +348,7 @@ impl ApiErrorDiagnostic {
let mut diagnostic = ApiErrorDiagnostic::default();
// S3Error retains its payload through an inherent accessor; its
// std::error::Error implementation does not expose that source.
let mut current = if let Some(error) = error.downcast_ref::<S3Error>() {
let mut current = if let Some(error) = error.downcast_ref::<LegacyS3Error>() {
error.source().map(|source| source as &(dyn std::error::Error + 'static))
} else {
std::error::Error::source(error)
@@ -384,7 +385,7 @@ impl ApiErrorDiagnostic {
diagnostic.io_os_error = io.raw_os_error();
// io::Error::source can skip the wrapped error itself.
io.get_ref().map(|source| source as &(dyn std::error::Error + 'static))
} else if let Some(error) = error.downcast_ref::<S3Error>() {
} else if let Some(error) = error.downcast_ref::<LegacyS3Error>() {
error.source().map(|source| source as &(dyn std::error::Error + 'static))
} else {
error.source()
@@ -616,14 +617,14 @@ fn body_read_error_code(err: &(dyn std::error::Error + 'static)) -> Option<S3Err
if let Some(chunked) = error.downcast_ref::<AwsChunkedStreamError>()
&& !matches!(chunked, AwsChunkedStreamError::Underlying(_))
{
aws_chunked_error = Some(chunked.to_s3_error_code());
aws_chunked_error = Some(chunked.to_s3_error_code().into());
}
incomplete_body |= error.is::<rustfs_rio::IncompleteBody>()
|| matches!(upload_error, Some(UploadStreamError::LengthMismatch | UploadStreamError::Incomplete))
|| is_hyper_body_eof(error);
None
});
digest.or_else(|| {
digest.or({
if upload_limit_exceeded {
Some(S3ErrorCode::EntityTooLarge)
} else if server_source_failed {
@@ -711,7 +712,7 @@ impl From<ApiError> for rustfs_s3_types::S3Error {
} else {
None
};
let mut s3e = rustfs_s3_types::S3Error::with_message(err.code.into(), err.message);
let mut s3e = rustfs_s3_types::S3Error::with_message(err.code, err.message);
if let Some(cause) = cause {
let mut headers = http::HeaderMap::new();
// s3s replaces the generated headers when a custom map is supplied.
@@ -736,7 +737,7 @@ impl From<ApiError> for rustfs_s3_types::S3Error {
/// with `From`, which yields field for field what the s3s constructor builds
/// (code, message, explicit status, the cause itself), so both error types
/// come from one mapping.
impl From<ApiError> for S3Error {
impl From<ApiError> for LegacyS3Error {
fn from(err: ApiError) -> Self {
rustfs_s3_types::S3Error::from(err).into()
}
@@ -964,8 +965,9 @@ mod tests {
use metrics_util::debugging::{DebugValue, DebuggingRecorder};
use rustfs_io_metrics::s3_http_metrics::S3HttpRequestGuard;
use rustfs_s3_ops::S3Operation;
use rustfs_s3_types::S3ErrorCode;
use s3s::stream::{aws_chunked_stream::AwsChunkedStreamError, upload_stream::UploadStreamError};
use s3s::{S3Error, S3ErrorCode};
use s3s::{S3Error, S3ErrorCode as LegacyS3ErrorCode};
use std::io::{Error as IoError, ErrorKind};
fn error_chain_has_type<T>(err: &(dyn std::error::Error + 'static)) -> bool
@@ -982,7 +984,7 @@ mod tests {
let storage_error =
StorageError::Lock(LockError::timeout("private-bucket/private-key@latest", std::time::Duration::from_secs(5)));
let error = S3Error::from(ApiError::from(storage_error));
assert_eq!(error.code(), &S3ErrorCode::ServiceUnavailable);
assert_eq!(error.code(), &LegacyS3ErrorCode::ServiceUnavailable);
let source = error.source().expect("retain the typed storage cause");
assert!(source.downcast_ref::<StorageError>().is_some());
assert!(error_chain_has_type::<LockError>(source));
@@ -1008,7 +1010,7 @@ mod tests {
use http_body_util::BodyExt as _;
let error = S3Error::from(ApiError::from(StorageError::UnsupportedDisk));
assert_eq!(error.code(), &S3ErrorCode::ServiceUnavailable);
assert_eq!(error.code(), &LegacyS3ErrorCode::ServiceUnavailable);
assert_eq!(error.message(), Some(UNSUPPORTED_PARTICIPANT_MESSAGE));
assert!(matches!(
error.source().and_then(|source| source.downcast_ref::<StorageError>()),
@@ -1049,7 +1051,7 @@ mod tests {
] {
let expected_code = failure.code();
let error = S3Error::from(ApiError::from(failure));
assert_eq!(error.code(), &S3ErrorCode::ServiceUnavailable);
assert_eq!(error.code(), &LegacyS3ErrorCode::ServiceUnavailable);
assert_eq!(error.message(), Some("The service is unavailable. Please retry."));
let source = error
.source()
@@ -1082,7 +1084,7 @@ mod tests {
code: code.clone(),
source: Some(Box::new(source)),
});
assert_eq!(error.code(), &code);
assert_eq!(error.code(), &code.clone().into());
let source = error.source().expect("preserve wrapped lock timeout");
assert!(error_chain_has_type::<LockError>(source));
let response = error.to_http_response().expect("serialize wrapped timeout");
@@ -1126,7 +1128,7 @@ mod tests {
] {
let expected_code = failure.code();
let error = S3Error::from(ApiError::from(failure));
assert_eq!(error.code(), &S3ErrorCode::ServiceUnavailable);
assert_eq!(error.code(), &LegacyS3ErrorCode::ServiceUnavailable);
assert_eq!(error.message(), Some("The service is unavailable. Please retry."));
let source = error
.source()
@@ -1344,7 +1346,7 @@ mod tests {
assert_eq!(diagnostic.lock_reason, Some(expected));
assert!(!format!("{diagnostic:?}").contains("private"));
let s3: S3Error = api.into();
assert_eq!(s3.code(), &S3ErrorCode::ServiceUnavailable);
assert_eq!(s3.code(), &LegacyS3ErrorCode::ServiceUnavailable);
assert_eq!(super::ApiErrorDiagnostic::from_error(&s3).lock_reason, Some(expected));
}
@@ -1634,7 +1636,7 @@ mod tests {
AwsChunkedStreamError::TrailersTooLarge(16385, 16384),
AwsChunkedStreamError::TooManyTrailerHeaders(101, 100),
] {
let expected = error.to_s3_error_code();
let expected: S3ErrorCode = error.to_s3_error_code().into();
let expected_status = if expected == S3ErrorCode::SignatureDoesNotMatch {
StatusCode::FORBIDDEN
} else {
@@ -2366,6 +2368,28 @@ mod tests {
assert!(source.to_string().contains(detail), "the detail must survive on the source");
}
#[test]
fn custom_error_keeps_its_identity_and_cause_across_both_edges() {
let api = ApiError {
code: S3ErrorCode::Custom("InvalidRequest".into()),
message: "custom refusal".into(),
source: Some(Box::new(IoError::other("original cause"))),
};
let owned = rustfs_s3_types::S3Error::from(api);
assert!(matches!(owned.code(), S3ErrorCode::Custom(name) if name == "InvalidRequest"));
assert_eq!(owned.status_code(), None);
assert_eq!(owned.message(), Some("custom refusal"));
assert!(owned.headers().is_none());
assert!(owned.source().is_some_and(|source| source.is::<IoError>()));
let legacy = S3Error::from(owned);
assert!(matches!(legacy.code(), LegacyS3ErrorCode::Custom(name) if name == "InvalidRequest"));
assert_eq!(legacy.status_code(), None);
assert_eq!(legacy.message(), Some("custom refusal"));
assert!(legacy.headers().is_none());
assert!(legacy.source().is_some_and(|source| source.is::<IoError>()));
}
#[test]
fn test_generated_error_codes_keep_their_own_status() {
let s3_error = S3Error::from(ApiError::from(StorageError::other(rustfs_kms::KmsError::backend_error("down"))));
@@ -2397,7 +2421,7 @@ mod tests {
headers.insert("retry-after", http::HeaderValue::from_static("2"));
err.set_headers(headers);
assert_eq!(err.code(), &S3ErrorCode::ServiceUnavailable);
assert_eq!(err.code(), &LegacyS3ErrorCode::ServiceUnavailable);
assert_eq!(err.status_code(), Some(StatusCode::SERVICE_UNAVAILABLE));
assert_eq!(err.message(), Some("The service is unavailable. Please retry."));
assert_eq!(err.request_id(), Some("put-quorum-request"));
@@ -2428,16 +2452,16 @@ mod tests {
#[test]
fn put_read_quorum_error_preserves_other_error_contracts() {
for code in [
S3ErrorCode::SlowDown,
S3ErrorCode::AccessDenied,
S3ErrorCode::InvalidRequest,
S3ErrorCode::PreconditionFailed,
S3ErrorCode::BadDigest,
S3ErrorCode::IncompleteBody,
S3ErrorCode::NoSuchKey,
S3ErrorCode::NoSuchBucket,
S3ErrorCode::InternalError,
S3ErrorCode::ServiceUnavailable,
LegacyS3ErrorCode::SlowDown,
LegacyS3ErrorCode::AccessDenied,
LegacyS3ErrorCode::InvalidRequest,
LegacyS3ErrorCode::PreconditionFailed,
LegacyS3ErrorCode::BadDigest,
LegacyS3ErrorCode::IncompleteBody,
LegacyS3ErrorCode::NoSuchKey,
LegacyS3ErrorCode::NoSuchBucket,
LegacyS3ErrorCode::InternalError,
LegacyS3ErrorCode::ServiceUnavailable,
] {
let mut err = S3Error::with_message(code.clone(), "original message");
err.set_source(Box::new(StorageError::ErasureReadQuorum));
@@ -2693,7 +2717,7 @@ mod tests {
};
let s3_error: S3Error = api_error.into();
assert_eq!(*s3_error.code(), S3ErrorCode::NoSuchBucket);
assert_eq!(*s3_error.code(), LegacyS3ErrorCode::NoSuchBucket);
assert!(s3_error.message().unwrap_or("").contains("Bucket not found"));
assert!(s3_error.source().is_some());
}
@@ -2711,7 +2735,7 @@ mod tests {
let s3_error: S3Error = api_error.into();
assert_eq!(*s3_error.code(), S3ErrorCode::ServiceUnavailable);
assert_eq!(*s3_error.code(), LegacyS3ErrorCode::ServiceUnavailable);
assert_eq!(s3_error.status_code(), Some(http::StatusCode::SERVICE_UNAVAILABLE));
}
@@ -2748,7 +2772,7 @@ mod tests {
assert_eq!(api_error.message, MAX_VERSIONS_EXCEEDED_MESSAGE);
let s3_error: S3Error = api_error.into();
assert_eq!(s3_error.code(), &S3ErrorCode::Custom(MAX_VERSIONS_EXCEEDED_CODE.into()));
assert_eq!(s3_error.code(), &LegacyS3ErrorCode::Custom(MAX_VERSIONS_EXCEEDED_CODE.into()));
assert_eq!(s3_error.message(), Some(MAX_VERSIONS_EXCEEDED_MESSAGE));
assert_eq!(s3_error.status_code(), Some(StatusCode::BAD_REQUEST));
}
@@ -2762,7 +2786,7 @@ mod tests {
};
let s3_error: S3Error = api_error.into();
assert_eq!(*s3_error.code(), S3ErrorCode::InvalidArgument);
assert_eq!(*s3_error.code(), LegacyS3ErrorCode::InvalidArgument);
assert!(s3_error.message().unwrap_or("").contains("Invalid argument"));
}
+13 -16
View File
@@ -1169,18 +1169,7 @@ impl rustfs_protocols::common::client::s3::StorageBackend for ProtocolStorageCli
let bucket = input.bucket.as_str().to_owned();
let key = input.key.as_str().to_owned();
let upload_id = gateway_s3s_compat::leaf::upload_id_to_s3s(&input.upload_id);
let input = multipart::upload_part_copy_input(input)?;
let part_number = input.part_number;
let part_number = part_number.to_string();
let uri = build_object_uri(
&bucket,
&key,
&[
("partNumber", Some(part_number.as_str())),
("uploadId", Some(upload_id.as_str())),
],
)?;
let (uri, input) = multipart::prepare_upload_part_copy(input)?;
let envelope = Self::create_envelope(
Method::PUT,
@@ -1517,6 +1506,7 @@ mod tests {
};
use http_body_util::BodyExt;
use rustfs_protocols::common::client::s3::StorageBackend;
use rustfs_s3_types::S3ErrorCode;
let root = rustfs_credentials::Credentials {
access_key: "protocol-oracle-root".into(),
secret_key: "protocol-oracle-secret".into(),
@@ -1528,9 +1518,16 @@ mod tests {
let slot = ServerContextSlot::new();
assert!(slot.install(context));
let backend = ProtocolStorageClient::new(slot);
for (bucket, key, data, versioned) in [
("protocol-version-oracle", "versioned-key", b"versioned data".as_slice(), true),
("protocol-marker-oracle", "folder/", b"".as_slice(), false),
for (bucket, key, data, versioned, deleted_get_code) in [
(
"protocol-version-oracle",
"versioned-key",
b"versioned data".as_slice(),
true,
S3ErrorCode::NoSuchKey,
),
// RustFS 1.0.5 returns NoSuchVersion for a missing directory marker.
("protocol-marker-oracle", "folder/", b"".as_slice(), false, S3ErrorCode::NoSuchVersion),
] {
store
.make_bucket(
@@ -1593,7 +1590,7 @@ mod tests {
.get_object(bucket, key, &root, None)
.await
.expect_err("protocol GET deleted object");
assert_eq!(current.code(), &rustfs_s3_types::S3ErrorCode::NoSuchKey);
assert_eq!(current.code(), &deleted_get_code, "GET {bucket}/{key} after DELETE");
assert_eq!(current.status_code(), Some(http::StatusCode::NOT_FOUND));
if versioned {
let headers = current.headers().expect("delete marker error headers");
+13 -5
View File
@@ -293,9 +293,9 @@ pub(super) fn upload_part_error(error: S3Error) -> S3Error {
error
}
pub(super) fn upload_part_copy_input(
pub(super) fn prepare_upload_part_copy(
input: dto::UploadPartCopyInput,
) -> S3Result<crate::app::multipart_usecase::UploadPartCopyWriteInput> {
) -> S3Result<(http::Uri, crate::app::multipart_usecase::UploadPartCopyWriteInput)> {
super::owned::validate_copy_source(&input.copy_source, "UploadPartCopy copy_source")?;
let copy_source_if_match =
super::owned::copy_condition(input.copy_source_if_match.as_deref(), "copy_source_if_match", "UploadPartCopy input")?;
@@ -314,11 +314,18 @@ pub(super) fn upload_part_copy_input(
.copy_source_if_unmodified_since
.map(|at| multipart_timestamp(at, "UploadPartCopy input", "copy_source_if_unmodified_since"))
.transpose()?;
Ok(crate::app::multipart_usecase::UploadPartCopyWriteInput {
let upload_id = super::gateway_s3s_compat::leaf::upload_id_to_s3s(&input.upload_id);
let part_number = input.part_number.to_string();
let uri = super::build_object_uri(
input.bucket.as_str(),
input.key.as_str(),
&[("partNumber", Some(&part_number)), ("uploadId", Some(&upload_id))],
)?;
let write = crate::app::multipart_usecase::UploadPartCopyWriteInput {
target: crate::app::multipart_usecase::UploadPartTarget::Legacy {
bucket: input.bucket.as_str().to_owned(),
key: input.key.as_str().to_owned(),
upload_id: super::gateway_s3s_compat::leaf::upload_id_to_s3s(&input.upload_id),
upload_id,
},
part_number: input.part_number,
copy_source: input.copy_source,
@@ -333,7 +340,8 @@ pub(super) fn upload_part_copy_input(
.copy_source_sse_customer_key
.map(|value| value.expose_secret().to_owned()),
copy_source_sse_customer_key_md5: nonempty(input.copy_source_sse_customer_key_md5),
})
};
Ok((uri, write))
}
pub(super) fn upload_part_copy_output(
@@ -422,9 +422,10 @@ fn protocol_upload_copy_keeps_destination_generation_fence() {
)
.await
.expect("replacement bytes");
let (_, input) = multipart::prepare_upload_part_copy(request).expect("owned input");
let result = backend
.owned_multipart_usecase()
.execute_upload_part_copy_with_audit(envelope, multipart::upload_part_copy_input(request).expect("owned input"))
.execute_upload_part_copy_with_audit(envelope, input)
.await;
let mut reader = store
.get_object_reader(destination, "copy-target", None, HeaderMap::new(), &options)
@@ -509,10 +510,12 @@ fn protocol_upload_copy_input_matches_original_codec() {
gateway_s3s_compat::generated::ops::upload_part_copy::input_to_s3s(make(), source)
.map_err(|error| conversion_error("UploadPartCopy input", error))
});
let actual = multipart::upload_part_copy_input(make());
let actual = multipart::prepare_upload_part_copy(make());
match (actual, expected) {
(Err(actual), Err(expected)) => assert_eq!(error_facts(actual), error_facts(expected)),
(Ok(actual), Ok(expected)) => {
(Ok((uri, actual)), Ok(expected)) => {
assert_eq!(uri.path(), "/bucket/copy-target");
assert_eq!(uri.query(), Some("partNumber=1&uploadId=upload%20%3F%26"));
let crate::app::multipart_usecase::UploadPartTarget::Legacy { bucket, key, upload_id } = actual.target else {
panic!("protocol client must use the legacy upload target")
};
@@ -726,7 +729,8 @@ async fn protocol_upload_copy_capabilities_precede_part_validation() {
.err()
.map(S3Error::from)
.expect("legacy refusal");
let actual = owned_copy_future(&usecase, envelope, multipart::upload_part_copy_input(make()).expect("owned input"))
let (_, input) = multipart::prepare_upload_part_copy(make()).expect("owned input");
let actual = owned_copy_future(&usecase, envelope, input)
.await
.err()
.expect("owned refusal");
+49 -44
View File
@@ -1840,18 +1840,11 @@ fn control_body_digest_error<R: From<Bytes>>(code: &'static str) -> Response<R>
response
}
const CONTROL_BODY_CHECKSUM_HEADERS: [&str; 10] = [
"x-amz-checksum-crc32",
"x-amz-checksum-crc32c",
"x-amz-checksum-crc64nvme",
"x-amz-checksum-sha1",
"x-amz-checksum-sha256",
"x-amz-checksum-sha512",
"x-amz-checksum-xxhash64",
"x-amz-checksum-xxhash3",
"x-amz-checksum-xxhash128",
"x-amz-checksum-md5",
];
fn control_body_checksum_headers() -> impl Iterator<Item = (&'static str, rustfs_rio::ChecksumType)> {
rustfs_rio::BASE_CHECKSUM_TYPES
.iter()
.filter_map(|&kind| kind.key().map(|name| (name, kind)))
}
fn verify_control_body_digest(headers: &HeaderMap, bytes: &[u8]) -> std::result::Result<(), &'static str> {
use md5::Digest;
@@ -1869,37 +1862,16 @@ fn verify_control_body_digest(headers: &HeaderMap, bytes: &[u8]) -> std::result:
return Err("BadDigest");
}
}
let mut checksums = s3s::checksum::ChecksumHasher {
crc32: headers.contains_key("x-amz-checksum-crc32").then(Default::default),
crc32c: headers.contains_key("x-amz-checksum-crc32c").then(Default::default),
crc64nvme: headers.contains_key("x-amz-checksum-crc64nvme").then(Default::default),
sha1: headers.contains_key("x-amz-checksum-sha1").then(Default::default),
sha256: headers.contains_key("x-amz-checksum-sha256").then(Default::default),
sha512: headers.contains_key("x-amz-checksum-sha512").then(Default::default),
xxhash64: headers.contains_key("x-amz-checksum-xxhash64").then(Default::default),
xxhash3: headers.contains_key("x-amz-checksum-xxhash3").then(Default::default),
xxhash128: headers.contains_key("x-amz-checksum-xxhash128").then(Default::default),
md5: headers.contains_key("x-amz-checksum-md5").then(Default::default),
};
checksums.update(bytes);
let checksums = checksums.finalize();
let computed = [
checksums.checksum_crc32,
checksums.checksum_crc32c,
checksums.checksum_crc64nvme,
checksums.checksum_sha1,
checksums.checksum_sha256,
checksums.checksum_sha512,
checksums.checksum_xxhash64,
checksums.checksum_xxhash3,
checksums.checksum_xxhash128,
checksums.checksum_md5,
];
for (name, computed) in CONTROL_BODY_CHECKSUM_HEADERS.into_iter().zip(computed) {
if let Some(computed) = computed
&& (headers.get_all(name).iter().count() != 1
|| headers.get(name).map(HeaderValue::as_bytes) != Some(computed.as_bytes()))
{
for (name, kind) in control_body_checksum_headers() {
let mut values = headers.get_all(name).iter();
let Some(expected) = values.next() else {
continue;
};
if values.next().is_some() {
return Err("BadDigest");
}
let computed = rustfs_rio::Checksum::new_from_data(kind, bytes).ok_or("BadDigest")?;
if expected.as_bytes() != computed.encoded.as_bytes() {
return Err("BadDigest");
}
}
@@ -1948,7 +1920,7 @@ where
.is_some_and(|value| value.as_bytes().starts_with(b"AWS "))
|| url::form_urlencoded::parse(request.uri().query().unwrap_or_default().as_bytes())
.any(|(key, _)| key == "AWSAccessKeyId");
let has_checksum = CONTROL_BODY_CHECKSUM_HEADERS.iter().any(|name| headers.contains_key(*name));
let has_checksum = control_body_checksum_headers().any(|(name, _)| headers.contains_key(name));
let has_md5 = headers.contains_key("content-md5");
if required && !has_md5 && (sig_v2 || !has_checksum) {
return Ok(control_body_digest_error("MissingContentMD5"));
@@ -4212,6 +4184,39 @@ mod tests {
}
}
#[test]
fn control_body_checksums_match_wire_values_for_all_algorithms() {
let body = b"123456789";
let vectors = [
("x-amz-checksum-crc32", "y/Q5Jg=="),
("x-amz-checksum-crc32c", "4waSgw=="),
("x-amz-checksum-crc64nvme", "rosUhgp5mIg="),
("x-amz-checksum-sha1", "98O8HYCOBHMq32eZZczDTKeuNEE="),
("x-amz-checksum-sha256", "FeKw08M4keuw8e9gnsQZQgwg4yDOlMZfvIwzEkSOsiU="),
(
"x-amz-checksum-sha512",
"2eZ2LdHI6vbWGzxhkvxAjU1tXxF20MKRabwk5xw/J0rSf81YEbMT1oH35V7ALXPUmclUVba1u1A6z1dPuo/+hQ==",
),
("x-amz-checksum-xxhash64", "jLhB20DmroM="),
("x-amz-checksum-xxhash3", "ctyxi2ehff8="),
("x-amz-checksum-xxhash128", "MxGUd+3l3NXpcWQnaB1YYA=="),
("x-amz-checksum-md5", "JfnnlDI7RTiF9RgfG2JNCw=="),
];
let mut headers = HeaderMap::new();
for (name, value) in vectors {
headers.insert(name, HeaderValue::from_static(value));
}
assert_eq!(verify_control_body_digest(&headers, body), Ok(()));
for (name, value) in vectors {
headers.insert(name, HeaderValue::from_static("mismatch"));
assert_eq!(verify_control_body_digest(&headers, body), Err("BadDigest"), "{name}");
headers.insert(name, HeaderValue::from_static(value));
headers.append(name, HeaderValue::from_static(value));
assert_eq!(verify_control_body_digest(&headers, body), Err("BadDigest"), "duplicate {name}");
headers.insert(name, HeaderValue::from_static(value));
}
}
fn digest_v4_request(checksum: &str) -> HttpRequest<s3s::Body> {
use s3s_sigv4::{AmzDate, Payload, calculate_signature, create_canonical_request, create_string_to_sign};
let timestamp = chrono::Utc::now().format("%Y%m%dT%H%M%SZ").to_string();
@@ -28,13 +28,13 @@ use super::{SerializedS3Error, xml_escape};
use crate::error::ApiError;
use crate::server::layer::is_sts_query_request;
use crate::storage_api::server::layer::request_context::RequestContext;
use crate::storage_api::server::legacy_compat::S3ErrorCode;
use bytes::{Bytes, BytesMut};
use futures::future::Either;
use http::{HeaderValue, Request as HttpRequest, Response, StatusCode};
use http_body::Body;
use http_body_util::{BodyExt, Full};
use quick_xml::events::Event;
use rustfs_s3_types::S3ErrorCode;
use rustfs_utils::http::headers::AMZ_REQUEST_ID;
use std::borrow::Cow;
use std::future::Future;
+49 -10
View File
@@ -26,7 +26,7 @@ mod tests {
] {
let error = crate::storage::s3_support::validate_object_key(key, "PUT")
.expect_err("client keys must not alias internal directory markers");
assert_eq!(error.code(), &s3s::S3ErrorCode::InvalidArgument, "{key}");
assert_eq!(error.code(), &S3ErrorCode::InvalidArgument, "{key}");
}
for key in ["file", "folder/", "prefix/folder/", "__xldir__", "percent%20name"] {
crate::storage::s3_support::validate_object_key(key, "PUT")
@@ -37,18 +37,20 @@ mod tests {
use crate::config::WorkloadProfile;
use crate::server::cors;
use crate::storage::StorageError;
use crate::storage::request_markers::{RequestedObjectLegalHold, RequestedObjectLockRetention};
use crate::storage::s3_api::common::{rustfs_initiator, rustfs_owner};
use crate::storage::s3_support::parse_object_version_id;
use crate::storage::s3_support::{apply_bucket_default_lock_retention, map_bucket_object_lock_config_state};
use crate::storage::s3_support::{
parse_object_lock_legal_hold_native, parse_object_lock_retention_native, parse_object_version_id,
};
use crate::storage::storage_api::ecstore_bucket::metadata_sys::ObjectLockConfigState;
use crate::storage::storage_api::test_consumer::{
BucketMetadata, DEFAULT_READ_BUFFER_SIZE, StorageObjectInfo as ObjectInfo, apply_cors_headers,
apply_default_lock_retention_metadata, bucket_metadata_sys_initialized, check_preconditions, decode_tags_to_map,
get_adaptive_buffer_size_with_profile, get_buffer_size_opt_in, get_global_bucket_metadata_sys, matches_origin_pattern,
parse_etag, parse_object_lock_legal_hold, parse_object_lock_retention, process_lambda_configurations,
process_queue_configurations, process_topic_configurations, remove_object_lock_metadata_for_copy,
remove_object_lock_retention_metadata, set_bucket_metadata, validate_bucket_object_lock_enabled,
validate_list_object_unordered_with_delimiter,
parse_etag, process_lambda_configurations, process_queue_configurations, process_topic_configurations,
remove_object_lock_metadata_for_copy, remove_object_lock_retention_metadata, set_bucket_metadata,
validate_bucket_object_lock_enabled, validate_list_object_unordered_with_delimiter,
};
use http::{HeaderMap, HeaderValue, StatusCode};
use rustfs_config::MI_B;
@@ -56,6 +58,7 @@ mod tests {
PersistedFilterRule, PersistedLambdaFunctionConfiguration, PersistedNotificationConfigurationFilter,
PersistedQueueConfiguration, PersistedS3KeyFilter, PersistedTopicConfiguration,
};
use rustfs_s3_types::{S3ErrorCode, S3Result};
use rustfs_utils::http::{
AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER, AMZ_OBJECT_LOCK_MODE_LOWER, AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER,
MINIO_INTERNAL_PREFIX, RUSTFS_INTERNAL_PREFIX, SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP,
@@ -63,13 +66,23 @@ mod tests {
};
use rustfs_zip::CompressionFormat;
use s3s::dto::{
DefaultRetention, Delimiter, ObjectLockConfiguration, ObjectLockEnabled, ObjectLockLegalHold, ObjectLockLegalHoldStatus,
DefaultRetention, ObjectLockConfiguration, ObjectLockEnabled, ObjectLockLegalHold, ObjectLockLegalHoldStatus,
ObjectLockRetention, ObjectLockRetentionMode, ObjectLockRule,
};
use s3s::{S3Error, S3ErrorCode, s3_error};
use std::collections::HashMap;
use time::{OffsetDateTime, format_description::well_known::Rfc3339};
fn parse_object_lock_retention(retention: Option<ObjectLockRetention>) -> S3Result<HashMap<String, String>> {
parse_object_lock_retention_native(retention.map(|r| RequestedObjectLockRetention {
mode: r.mode.map(|m| m.as_str().to_owned()),
retain_until: r.retain_until_date.map(OffsetDateTime::from),
}))
}
fn parse_object_lock_legal_hold(legal_hold: Option<ObjectLockLegalHold>) -> S3Result<HashMap<String, String>> {
parse_object_lock_legal_hold_native(legal_hold.map(|h| RequestedObjectLegalHold(h.status.map(|s| s.as_str().to_owned()))))
}
#[test]
fn test_rustfs_owner_helpers_are_stable() {
let owner = rustfs_owner();
@@ -142,6 +155,8 @@ mod tests {
#[test]
fn test_s3_error_scenarios() {
use s3s::{S3Error, S3ErrorCode, s3_error};
// Test that we can create expected S3 errors for common validation cases
// Test incomplete body error
@@ -616,8 +631,7 @@ mod tests {
// [1] Normal case: No delimiter specified.
assert!(validate_list_object_unordered_with_delimiter(None, Some("allow-unordered=true")).is_ok());
let delim_str = "/".to_string();
let delimiter_some: Option<&Delimiter> = Some(&delim_str);
let delimiter_some = Some("/");
// [2] Normal case: Delimiter is present, but 'allow-unordered' is explicitly set to false.
assert!(validate_list_object_unordered_with_delimiter(delimiter_some, Some("allow-unordered=false")).is_ok());
@@ -1063,6 +1077,31 @@ mod tests {
assert!(check_preconditions(&headers18, &info18).is_ok());
}
#[test]
fn conditional_read_preserves_invalid_cache_header_cause_at_the_wire() {
let info = ObjectInfo {
user_defined: HashMap::from([("cache-control".to_owned(), "invalid\nvalue".to_owned())]).into(),
..Default::default()
};
let mut headers = HeaderMap::new();
headers.insert("if-none-match", HeaderValue::from_static("*"));
let error = check_preconditions(&headers, &info).expect_err("invalid cached metadata cannot form a 304 response");
assert_eq!(error.code(), &S3ErrorCode::InternalError);
assert_eq!(error.status_code(), Some(StatusCode::INTERNAL_SERVER_ERROR));
assert!(error.headers().is_none());
assert!(
error
.source()
.is_some_and(|cause| cause.is::<http::header::InvalidHeaderValue>())
);
let wire = s3s::S3Error::from(error);
assert_eq!(wire.status_code(), Some(StatusCode::INTERNAL_SERVER_ERROR));
assert!(
wire.source()
.is_some_and(|cause| cause.is::<http::header::InvalidHeaderValue>())
);
}
#[test]
fn test_check_preconditions_rfc304() {
let modified = OffsetDateTime::from_unix_timestamp(1700000000).expect("valid timestamp");
+7 -33
View File
@@ -18,6 +18,7 @@ use crate::config::{RustFSBufferConfig, WorkloadProfile, is_buffer_profile_enabl
use crate::error::ApiError;
use crate::server::cors;
use crate::storage::storage_api::ecstore_bucket::metadata_sys::{self, ObjectLockConfigState};
use crate::storage::storage_api::ecstore_bucket::object_lock::types::{LegalHoldStatus, RetentionMode};
use crate::storage::storage_api::s3_support_consumer::contract::bucket::{BucketOperations, BucketOptions};
use crate::storage::storage_api::s3_support_consumer::contract::multipart::MAX_MULTIPART_PART_NUMBER;
use http::header::{IF_MATCH, IF_MODIFIED_SINCE, IF_NONE_MATCH, IF_UNMODIFIED_SINCE};
@@ -28,16 +29,13 @@ use rustfs_gateway_types::persistence::{
PersistedLambdaFunctionConfiguration, PersistedNotificationConfigurationFilter, PersistedQueueConfiguration,
PersistedTopicConfiguration,
};
use rustfs_s3_types::{S3Error, S3ErrorCode, S3Result};
use rustfs_targets::EventName;
use rustfs_targets::arn::{TargetID, TargetIDError};
use rustfs_utils::http::{
AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER, AMZ_OBJECT_LOCK_MODE_LOWER, AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER,
SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP, contains_key_str, insert_str, remove_str,
};
use s3s::dto::{Delimiter, ObjectLockLegalHoldStatus, ObjectLockRetentionMode};
#[cfg(test)]
use s3s::dto::{ObjectLockLegalHold, ObjectLockRetention};
use s3s::{S3Error, S3ErrorCode, S3Result};
use serde_urlencoded::from_bytes;
use std::collections::HashMap;
use std::sync::{Arc, OnceLock, RwLock};
@@ -386,7 +384,7 @@ pub(crate) fn validate_new_object_key(key: &str, operation: &str) -> S3Result<()
/// 2. Rejects reserved directory markers and invalid path segments
/// 3. Logs debug information for keys containing spaces, plus signs, or percent signs
///
/// The s3s library handles URL decoding, so keys are already decoded when they reach this function.
/// Request routing decodes the key before this validation.
/// This validation ensures that invalid characters that could cause issues are rejected early.
pub(crate) fn validate_object_key(key: &str, operation: &str) -> S3Result<()> {
// Validate object key doesn't contain control characters
@@ -439,10 +437,7 @@ pub(crate) fn validate_object_key(key: &str, operation: &str) -> S3Result<()> {
/// According to S3 compatibility requirements, unordered listing cannot be combined with
/// hierarchical directory traversal (delimited listing). This validation ensures
/// conflicting parameters are caught before processing the request.
pub(crate) fn validate_list_object_unordered_with_delimiter(
delimiter: Option<&Delimiter>,
query_string: Option<&str>,
) -> S3Result<()> {
pub(crate) fn validate_list_object_unordered_with_delimiter(delimiter: Option<&str>, query_string: Option<&str>) -> S3Result<()> {
if delimiter.is_none() {
return Ok(());
}
@@ -463,14 +458,6 @@ pub(crate) fn validate_list_object_unordered_with_delimiter(
Ok(())
}
#[cfg(test)]
pub(crate) fn parse_object_lock_retention(retention: Option<ObjectLockRetention>) -> S3Result<HashMap<String, String>> {
parse_object_lock_retention_native(retention.map(|r| RequestedObjectLockRetention {
mode: r.mode.map(|m| m.as_str().to_owned()),
retain_until: r.retain_until_date.map(OffsetDateTime::from),
}))
}
pub(crate) fn parse_object_lock_retention_native(
retention: Option<RequestedObjectLockRetention>,
) -> S3Result<HashMap<String, String>> {
@@ -485,7 +472,7 @@ pub(crate) fn parse_object_lock_retention_native(
}
let mode = match v.mode {
Some(mode) => match mode.as_str() {
ObjectLockRetentionMode::COMPLIANCE | ObjectLockRetentionMode::GOVERNANCE => mode.as_str().to_string(),
RetentionMode::COMPLIANCE | RetentionMode::GOVERNANCE => mode.as_str().to_string(),
_ => {
return Err(S3Error::with_message(
S3ErrorCode::MalformedXML,
@@ -529,18 +516,13 @@ pub(crate) fn parse_object_lock_retention_native(
Ok(eval_metadata)
}
#[cfg(test)]
pub(crate) fn parse_object_lock_legal_hold(legal_hold: Option<ObjectLockLegalHold>) -> S3Result<HashMap<String, String>> {
parse_object_lock_legal_hold_native(legal_hold.map(|h| RequestedObjectLegalHold(h.status.map(|s| s.as_str().to_owned()))))
}
pub(crate) fn parse_object_lock_legal_hold_native(
legal_hold: Option<RequestedObjectLegalHold>,
) -> S3Result<HashMap<String, String>> {
let mut eval_metadata = HashMap::new();
if let Some(v) = legal_hold {
let status = match v.0.as_deref() {
Some(status @ (ObjectLockLegalHoldStatus::OFF | ObjectLockLegalHoldStatus::ON)) => status.to_owned(),
Some(status @ (LegalHoldStatus::OFF | LegalHoldStatus::ON)) => status.to_owned(),
_ => {
return Err(S3Error::with_message(
S3ErrorCode::MalformedXML,
@@ -929,15 +911,11 @@ pub fn invalidate_bucket_validation_cache(bucket: &str) {
/// Uses adaptive cache with 5s TTL to avoid repeated stat_volume() calls.
/// Returns store directly on cache hit without calling get_bucket_info().
pub(crate) async fn get_validated_store(bucket: &str) -> S3Result<Arc<super::ECStore>> {
Ok(get_validated_store_owned(bucket).await?)
}
pub(crate) async fn get_validated_store_owned(bucket: &str) -> rustfs_s3_types::S3Result<Arc<super::ECStore>> {
let Some(store) = runtime_sources::current_object_store_handle() else {
return Err(rustfs_s3_types::s3_error!(InternalError, "Not init"));
};
validate_bucket_exists_owned(&store, bucket).await?;
validate_bucket_exists(&store, bucket).await?;
Ok(store)
}
@@ -947,10 +925,6 @@ pub(crate) async fn get_validated_store_owned(bucket: &str) -> rustfs_s3_types::
/// store through the request-bound server context (backlog#1052 S6) use this
/// to keep bucket validation on that store instead of the process-global one.
pub(crate) async fn validate_bucket_exists(store: &super::ECStore, bucket: &str) -> S3Result<()> {
Ok(validate_bucket_exists_owned(store, bucket).await?)
}
async fn validate_bucket_exists_owned(store: &super::ECStore, bucket: &str) -> rustfs_s3_types::S3Result<()> {
// Check cache — TTL is checked manually.
if let Some(inserted_at) = cache_get(bucket)
&& inserted_at.elapsed() < BUCKET_VALIDATION_TTL
+2 -2
View File
@@ -99,8 +99,8 @@ use rustfs_config::{DEFAULT_KMS_ENFORCE_SSE_KEY_POLICY, ENV_RUSTFS_KMS_ENFORCE_S
use rustfs_kms::{DataKey, KmsUnavailableError, is_data_key_envelope, types::ObjectEncryptionContext};
use rustfs_policy::policy::Args;
use rustfs_policy::policy::action::{Action, KmsAction};
use rustfs_s3_types::S3ErrorCode;
use rustfs_utils::{get_env_bool, get_env_opt_str};
use s3s::S3ErrorCode;
use serde::{Deserialize, Serialize};
use serde_json::Value;
#[cfg(feature = "rio-v2")]
@@ -4975,9 +4975,9 @@ pub(crate) mod tests {
use http::{HeaderMap, HeaderValue};
use rustfs_kms::types::ObjectEncryptionContext;
use rustfs_rio::{DecryptReader, EncryptReader};
use rustfs_s3_types::S3ErrorCode;
use rustfs_utils::http::headers::AMZ_SERVER_SIDE_ENCRYPTION_KMS_CONTEXT;
use rustfs_utils::http::headers::{AMZ_ENCRYPTION_AES, AMZ_ENCRYPTION_KMS};
use s3s::S3ErrorCode;
use std::collections::HashMap;
use std::sync::{Arc, OnceLock};
use temp_env::async_with_vars;
+8 -9
View File
@@ -137,12 +137,11 @@ pub(crate) type StorageObjectToDelete = contract::object::ObjectToDelete;
pub(crate) type StoragePutObjReader = super::PutObjReader;
pub(crate) use super::s3_support::{
RFC1123, apply_bucket_default_lock_retention, apply_cors_headers, check_preconditions, evaluate_read_preconditions,
get_buffer_size_opt_in, get_validated_store, get_validated_store_owned, has_read_preconditions,
load_bucket_object_lock_config_state, parse_object_lock_legal_hold_native, parse_object_lock_retention_native,
parse_object_version_id, parse_part_number_i32_to_usize, process_lambda_configurations, process_queue_configurations,
process_topic_configurations, remove_object_lock_metadata_for_copy, validate_bucket_exists,
validate_bucket_object_lock_enabled_state, validate_list_object_unordered_with_delimiter, validate_new_object_key,
validate_object_key,
get_buffer_size_opt_in, get_validated_store, has_read_preconditions, load_bucket_object_lock_config_state,
parse_object_lock_legal_hold_native, parse_object_lock_retention_native, parse_object_version_id,
parse_part_number_i32_to_usize, process_lambda_configurations, process_queue_configurations, process_topic_configurations,
remove_object_lock_metadata_for_copy, validate_bucket_exists, validate_bucket_object_lock_enabled_state,
validate_list_object_unordered_with_delimiter, validate_new_object_key, validate_object_key,
};
pub(crate) use super::sse::{
DecryptionRequest, EncryptionRequest, ObjectDekRewrapOutcome, PrepareEncryptionRequest, SseKmsPrincipal,
@@ -486,9 +485,9 @@ pub(crate) mod test_consumer {
pub(crate) use super::super::{
apply_cors_headers, apply_default_lock_retention_metadata, check_preconditions, decode_tags_to_map,
get_adaptive_buffer_size_with_profile, get_buffer_size_opt_in, matches_origin_pattern, parse_etag,
parse_object_lock_legal_hold, parse_object_lock_retention, process_lambda_configurations, process_queue_configurations,
process_topic_configurations, remove_object_lock_metadata_for_copy, remove_object_lock_retention_metadata,
validate_bucket_object_lock_enabled, validate_list_object_unordered_with_delimiter,
process_lambda_configurations, process_queue_configurations, process_topic_configurations,
remove_object_lock_metadata_for_copy, remove_object_lock_retention_metadata, validate_bucket_object_lock_enabled,
validate_list_object_unordered_with_delimiter,
};
pub(crate) use super::{
BucketMetadata, DEFAULT_READ_BUFFER_SIZE, bucket_metadata_sys_initialized, get_global_bucket_metadata_sys,
-8
View File
@@ -274,14 +274,6 @@ pub(crate) mod server {
}
}
/// The s3s error types the s3s-compat tower layers in
/// `server::legacy_compat` render and match on, funneled here so those
/// files stay off the direct s3s surface (s3s footprint ratchet). T2.12
/// (rustfs/backlog#2771) removes the layers and with them this module.
pub(crate) mod legacy_compat {
pub(crate) use s3s::S3ErrorCode;
}
pub(crate) mod module_switch {
pub(crate) use crate::storage::storage_api::{
Error, read_config, read_config_no_lock, save_config_no_lock, with_config_object_read_lock,
@@ -766,13 +766,9 @@ fn the_gateway_stack_swaps_the_tagging_action_for_a_named_version() {
with_stack(GATEWAY, the_tagging_action_follows_the_named_version);
}
/// What a stack answers for a key outside the floor, independent of the operation and bucket.
type UniformAnswers = (Answer, Answer);
/// Checks boundary keys across bucket configurations. The legacy stack answers per operation and
/// bucket from the table below; the gateway stack refuses the key before the bucket is consulted,
/// so it answers every row with `uniform`: `(NUL key, 1025-byte key)`.
async fn keys_outside_the_floor_are_refused(server: rustfs::embedded::RustFSServer, uniform: Option<UniformAnswers>) {
/// The gateway rejects boundary keys before consulting the bucket. RustFS serves no torrents,
/// so a NUL key on GetObjectTorrent retains the release's NoSuchKey response.
async fn keys_outside_the_floor_are_refused(server: rustfs::embedded::RustFSServer) {
const BUCKET: &str = "object-meta-keys";
let client = s3_client(&server.endpoint(), server.access_key(), server.secret_key());
create_bucket_in_the_store(BUCKET).await;
@@ -782,38 +778,25 @@ async fn keys_outside_the_floor_are_refused(server: rustfs::embedded::RustFSServ
assert!(super::bucket_exists_in_the_store(reserved).await);
put_object_in_the_store(reserved, KEY, HashMap::new()).await;
assert_eq!(answer(client.get_object_acl().bucket(reserved).key(KEY).send().await), (None, None, None));
let refused = |code: &str, status, message: &str| (Some(code.to_owned()), Some(status), Some(message.to_owned()));
let invalid_nul = refused("InvalidArgument", 400, "the name contains a NUL byte");
let missing_torrent = refused("NoSuchKey", 404, "The specified key does not exist.");
let too_long = refused("KeyTooLongError", 400, "the name is longer than the maximum length in bytes");
let long_key = "k".repeat(1025);
for (operation, call) in KEY_CALLS {
for (bucket, kind) in [(BUCKET, 0), (locked.as_str(), 1), ("object-meta-missing", 2), (reserved, 0)] {
let (code, status, message) = match (*operation, kind) {
("GetObjectTorrent", _) => ("NoSuchKey", 404, "The specified key does not exist."),
("GetObjectLegalHold", 0) | ("GetObjectRetention", 0) => {
("InvalidRequest", 400, "Bucket is missing ObjectLockConfiguration")
}
("GetObjectLegalHold", 1) | ("GetObjectRetention", 1) => {
("InternalError", 500, "XML response contains a control character")
}
("GetObjectLegalHold", _) => ("NoSuchBucket", 404, "The specified bucket does not exist"),
("GetObjectRetention", _) => ("InternalError", 500, "Failed to load Object Lock configuration"),
_ => ("InvalidArgument", 400, "Invalid argument"),
};
let expected = match &uniform {
Some((nul, _)) => nul.clone(),
None => (Some(code.to_owned()), Some(status), Some(message.to_owned())),
};
assert_eq!(call(&client, bucket, "a\0b").await, expected, "{operation} bucket {bucket}");
}
let expected = match &uniform {
Some((_, long)) => long.clone(),
None => (Some("KeyTooLongError".to_owned()), Some(400), Some("Your key is too long.".to_owned())),
let expected_nul = if *operation == "GetObjectTorrent" {
&missing_torrent
} else {
&invalid_nul
};
assert_eq!(call(&client, BUCKET, &long_key).await, expected, "{operation} long key");
for bucket in [BUCKET, locked.as_str(), "object-meta-missing", reserved] {
assert_eq!(&call(&client, bucket, "a\0b").await, expected_nul, "{operation} bucket {bucket}");
}
assert_eq!(call(&client, BUCKET, &long_key).await, too_long, "{operation} long key");
}
server.shutdown().await;
}
type Answer = (Option<String>, Option<u16>, Option<String>);
type KeyCall = for<'a> fn(&'a Client, &'a str, &'a str) -> BoxFuture<'a, (Option<String>, Option<u16>, Option<String>)>;
const KEY_CALLS: &[(&str, KeyCall)] = &[
@@ -853,12 +836,7 @@ fn answer<T, E: ProvideErrorMetadata>(result: Result<T, SdkError<E>>) -> (Option
#[test]
fn the_gateway_stack_refuses_keys_outside_the_floor() {
let refused = |code: &str, message: &str| (Some(code.to_owned()), Some(400), Some(message.to_owned()));
let uniform = (
refused("InvalidArgument", "the name contains a NUL byte"),
refused("KeyTooLongError", "the name is longer than the maximum length in bytes"),
);
with_stack(GATEWAY, move |server| keys_outside_the_floor_are_refused(server, Some(uniform)));
with_stack(GATEWAY, keys_outside_the_floor_are_refused);
}
fn legal_hold(status: &str) -> aws_sdk_s3::types::ObjectLockLegalHold {
+6 -8
View File
@@ -162,13 +162,12 @@ done < <(awk '
pending { sub(/^[[:space:]]*'"'"'/, ""); sub(/'"'"'[[:space:]]*\\$/, ""); print; pending = 0; next }
/module_relative_rg rustfs\/src\/(app|storage) \\$/ { pending = 1 }
' "$ARCH_RULES")
[[ ${#TEMPLATES[@]} == 6 ]] || fail "expected 6 depth-aware rules in $ARCH_RULES, found ${#TEMPLATES[@]}"
[[ ${#TEMPLATES[@]} == 5 ]] || fail "expected 5 depth-aware rules in $ARCH_RULES, found ${#TEMPLATES[@]}"
STORAGE_HELPER_RULE="${TEMPLATES[0]}"
STORAGE_OWNER_RULE="${TEMPLATES[1]}"
RUNTIME_IMPORT_RULE="${TEMPLATES[2]}"
RUNTIME_PATH_RULE="${TEMPLATES[3]}"
ECFS_TEST_RULE="${TEMPLATES[4]}"
ECFS_TEST_SPLIT_RULE="${TEMPLATES[5]}"
STORAGE_HELPER_TEST_RULE="${TEMPLATES[4]}"
# relative_hits <module-dir> <template> <relative-file> <line>: rg output for one file.
relative_hits() {
@@ -208,9 +207,8 @@ flagged rustfs/src/app "$STORAGE_OWNER_RULE" rustfs/src/app/bucket_usecase.rs 's
flagged rustfs/src/app "$STORAGE_OWNER_RULE" rustfs/src/app/bucket_usecase/cors.rs 'super::super::super::metadata_sys::get();'
flagged rustfs/src/app "$RUNTIME_IMPORT_RULE" rustfs/src/app/bucket_usecase/cors.rs 'use super::super::ECStore;'
flagged rustfs/src/app "$RUNTIME_PATH_RULE" rustfs/src/app/lifecycle_transition_api_test/put.rs 'super::super::get_server_info();'
flagged rustfs/src/storage "$ECFS_TEST_RULE" rustfs/src/storage/ecfs_test.rs ' use super::super::{BucketMetadata};'
flagged rustfs/src/storage "$ECFS_TEST_SPLIT_RULE" rustfs/src/storage/legacy_edge/tests/object_meta.rs 'use super::super::super::{BucketMetadata};'
flagged rustfs/src/storage "$ECFS_TEST_SPLIT_RULE" rustfs/src/storage/legacy_edge/tests/object_meta.rs ' use super::super::super::super::{check_preconditions};'
flagged rustfs/src/storage "$STORAGE_HELPER_TEST_RULE" rustfs/src/storage/helpers_test.rs ' use super::super::{BucketMetadata};'
flagged rustfs/src/storage "$STORAGE_HELPER_TEST_RULE" rustfs/src/storage/helpers_test.rs 'use crate::storage::{check_preconditions};'
# Ordinary forms: a split file importing from its own parent module is not flagged.
new_case
@@ -219,12 +217,12 @@ not_flagged rustfs/src/app "$STORAGE_OWNER_RULE" rustfs/src/app/bucket_usecase/c
not_flagged rustfs/src/app "$STORAGE_OWNER_RULE" rustfs/src/app/bucket_usecase/cors.rs 'super::tagging::helper();'
not_flagged rustfs/src/app "$RUNTIME_IMPORT_RULE" rustfs/src/app/bucket_usecase/cors.rs 'use super::{ECStore, DefaultBucketUsecase};'
not_flagged rustfs/src/app "$RUNTIME_PATH_RULE" rustfs/src/app/bucket_usecase/cors.rs 'super::get_server_info();'
not_flagged rustfs/src/storage "$ECFS_TEST_SPLIT_RULE" rustfs/src/storage/legacy_edge/tests/object_meta.rs ' use super::{BucketMetadata};'
not_flagged rustfs/src/storage "$STORAGE_HELPER_TEST_RULE" rustfs/src/storage/helpers_test.rs ' use super::{BucketMetadata};'
# A file outside the module directory, or a missing -- separator, is an error, not a silent pass.
new_case
status=0
module_relative_rg rustfs/src/app 'x' -n -- rustfs/src/storage/ecfs.rs 2>/dev/null || status=$?
module_relative_rg rustfs/src/app 'x' -n -- rustfs/src/storage/helpers_test.rs 2>/dev/null || status=$?
[[ "$status" == 2 ]] || fail "file outside the module dir: status $status"
status=0
module_relative_rg rustfs/src/app 'x' -n rustfs/src/app/a.rs 2>/dev/null || status=$?
+58 -13
View File
@@ -1,5 +1,5 @@
#!/usr/bin/env python3
"""Exercise both pinned upgrade sources without building RustFS."""
"""Exercise the pinned upgrade sources without building RustFS."""
from __future__ import annotations
@@ -10,6 +10,7 @@ import re
import subprocess
import sys
import tempfile
import tomllib
import unittest
from check_test_wiring import yaml_block
@@ -17,6 +18,12 @@ from test_security_workflow import named_steps, shell_body
ROOT = Path(__file__).resolve().parents[1]
SOURCE_CASES = {
"1.0.0-rc.5": ("rc5_rollback_requires_restoring_odm_configuration",
"rc5_checkpoint_upgrade_rewinds_signed_progress_without_losing_objects"),
"1.0.1": ("stable_upgrade_deletes_foreign_reservations_without_losing_live_objects",),
"1.0.5": (),
}
class UpgradeWorkflowTests(unittest.TestCase):
@@ -28,19 +35,16 @@ class UpgradeWorkflowTests(unittest.TestCase):
# Explicit bash enables pipefail in Actions, so tee cannot hide a failure.
self.assertIn(" shell: bash", self.step)
self.assertIn(" fail-fast: false", self.job)
self.assertEqual(re.findall(r"- version: ([^\s]+)", "\n".join(self.job)), ["1.0.0-rc.5", "1.0.1"])
self.assertEqual(re.findall(r"- version: ([^\s]+)", "\n".join(self.job)), list(SOURCE_CASES))
rust = (ROOT / "crates/e2e_test/src/upgrade_compatibility_test.rs").read_text()
self.expected = set(re.findall(r'#\[ignore[^\n]*\]\s*async fn (\w+)\(', rust))
self.assertTrue(self.expected)
def run_cases(self, version, failed_test="", fail_tee=False):
excluded = ({"stable_upgrade_deletes_foreign_reservations_without_losing_live_objects"}
if version == "1.0.0-rc.5" else {
"rc5_rollback_requires_restoring_odm_configuration",
"rc5_checkpoint_upgrade_rewinds_signed_progress_without_losing_objects",
})
expected = self.expected - excluded
self.assertEqual(len(expected), 8 if version == "1.0.0-rc.5" else 7)
shared = self.expected.difference(*SOURCE_CASES.values())
self.assertEqual(len(shared), 6)
expected = shared | set(SOURCE_CASES[version])
self.assertTrue(expected <= self.expected)
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
commands = root / "commands.jsonl"
@@ -82,14 +86,14 @@ sys.exit(17 if test == os.environ["FAILED_TEST"] else 0)
return result
def test_all_ignored_upgrade_cases_run_once_with_separate_evidence(self):
for version in ("1.0.0-rc.5", "1.0.1"):
for version in SOURCE_CASES:
with self.subTest(version=version):
result = self.run_cases(version)
self.assertEqual(result.returncode, 0, result.stderr)
def test_early_or_late_failure_does_not_hide_remaining_cases(self):
for version, last in (("1.0.0-rc.5", "rc5_checkpoint_upgrade_rewinds_signed_progress_without_losing_objects"),
("1.0.1", "stable_upgrade_deletes_foreign_reservations_without_losing_live_objects")):
for version, specialized in SOURCE_CASES.items():
last = specialized[-1] if specialized else "previous_release_baseline_replicates_multipart_layouts"
for name in ("direct_upgrade_from_previous_release_preserves_object_contracts", last):
with self.subTest(version=version, name=name):
result = self.run_cases(version, failed_test=name)
@@ -97,9 +101,50 @@ sys.exit(17 if test == os.environ["FAILED_TEST"] else 0)
self.assertIn(f"::error::Upgrade compatibility failed: {name}", result.stdout)
def test_log_write_failure_fails_the_job(self):
for version in ("1.0.0-rc.5", "1.0.1"):
for version in SOURCE_CASES:
self.assertNotEqual(self.run_cases(version, fail_tee=True).returncode, 0)
def test_official_reference_matches_the_workflow_and_release_plan(self):
reference = tomllib.loads((ROOT / ".config/reference-binary.toml").read_text())
platform = reference["platforms"]["linux-x86_64-gnu"]
pin = ("1.0.5", platform["asset"], platform["sha256"])
self.assertEqual(reference["version"], pin[0])
matrix = re.findall(r"- version: (\S+)\s+asset: (\S+)\s+sha256: (\w+)", "\n".join(self.job))
self.assertIn(pin, matrix)
plan = json.loads((ROOT / ".config/release-validation-plan.json").read_text())
self.assertEqual(set(matrix), {(s["version"], s["artifact_name"], s["artifact_sha256"])
for s in plan["upgrade_sources"]["sources"]})
legacy_mpu = "\n".join(self.steps["Run mixed-version MPU recovery"])
self.assertIn("matrix.source.version != '1.0.5'", legacy_mpu)
self.assertIn("mixed_version_failed_complete_requires_current_coordinator_for_abort", legacy_mpu)
def test_gateway_evidence_requires_both_roles_and_rollback_for_the_pinned_pair(self):
step = self.steps["Verify gateway baseline G09 evidence"]
self.assertIn("matrix.source.version == '1.0.5'", "\n".join(step))
with tempfile.TemporaryDirectory() as directory:
root = Path(directory) / "rustfs-upgrade-g09-evidence"
env = dict(os.environ, RUNNER_TEMP=directory, GITHUB_SHA="c" * 40, UPGRADE_SOURCE_REVISION="a" * 40)
for field in ("mixed_version_reader_evidence", "mixed_version_writer_evidence", "rollback_payload_evidence"):
folder = root / ("bucket-config-rollback" if field == "rollback_payload_evidence" else "mixed-version-upgrade")
folder.mkdir(exist_ok=True, parents=True)
payload = dict(gate="G09", field=field, current_revision=env["GITHUB_SHA"],
previous_revision=env["UPGRADE_SOURCE_REVISION"], rollback_payload_replayed=True)
(folder / f"G09-{field}.json").write_text(json.dumps(payload))
multipart = root / "bucket-config-rollback/inflight-multipart-rollback.json"
payload = dict(current_revision=env["GITHUB_SHA"], previous_revision=env["UPGRADE_SOURCE_REVISION"],
part_sizes=[5242880, 5242880, 4096], bytes=10489856,
rollback_read_sha256="d" * 64, roll_forward_read_sha256="d" * 64)
multipart.write_text(json.dumps(payload))
command = ["bash", "-e", "-o", "pipefail", "-c", shell_body(step)]
self.assertEqual(subprocess.run(command, env=env, capture_output=True).returncode, 0)
wrong_pair = dict(env, UPGRADE_SOURCE_REVISION="b" * 40)
self.assertNotEqual(subprocess.run(command, env=wrong_pair, capture_output=True).returncode, 0)
multipart.write_text(json.dumps(dict(payload, roll_forward_read_sha256="e" * 64)))
self.assertNotEqual(subprocess.run(command, env=env, capture_output=True).returncode, 0)
multipart.write_text(json.dumps(payload))
(root / "mixed-version-upgrade/G09-mixed_version_reader_evidence.json").unlink()
self.assertNotEqual(subprocess.run(command, env=env, capture_output=True).returncode, 0)
def test_reports_are_uploaded_even_after_failure(self):
for name, directory in (("Upload scanner/heal G09 evidence", "rustfs-upgrade-g09-evidence"),
("Upload server logs", "rustfs-upgrade-logs")):