mirror of
https://github.com/rustfs/rustfs.git
synced 2026-11-20 06:53:44 +00:00
Compare commits
13 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 4c8f4c5969 | |||
| 0998de5281 | |||
| 9e6f786b35 | |||
| 0abacd305a | |||
| c780d86555 | |||
| 95d1a4193d | |||
| c1044cd873 | |||
| 1680db7ed8 | |||
| 161d0c7ae8 | |||
| 19a45dcb7b | |||
| 674e55fa6d | |||
| 7cf2c173ea | |||
| 6b40ead7c1 |
@@ -1,2 +1,2 @@
|
||||
sha256-darwin=9ccb03f4139e99ec11a63810523d504df39191b86e74b28a72dd9a075682293b
|
||||
sha256-linux=5ef052b03a03afbd00ebfcbbdcb754bbde414b7e6151aae3e2ee78024ce1d0b6
|
||||
sha256-darwin=3cf5f1857837aabc5d00d261dc40a36ef0a87c4c8b40cc5b705c95477f8e4993
|
||||
sha256-linux=6f56cfda9326f5321c2690bfb0232e737d5660e99813fa33460a0a800d6c94af
|
||||
|
||||
@@ -114,14 +114,8 @@
|
||||
}
|
||||
],
|
||||
"upgrade_sources": {
|
||||
"basis": "The three previous-release sources pinned in .github/workflows/e2e-upgrade.yml are exercised today (source.version / source.asset / source.sha256). No other N-1/N-2 compatibility is claimed, and these digests are not inferred from tags.",
|
||||
"basis": "Exactly the two previous-release sources pinned in .github/workflows/e2e-upgrade.yml are exercised today (source.version / source.asset / source.sha256). No other N-1/N-2 compatibility is claimed, and these digests are not inferred from tags.",
|
||||
"sources": [
|
||||
{
|
||||
"version": "1.0.5",
|
||||
"role": "gateway compatibility baseline",
|
||||
"artifact_name": "rustfs-linux-x86_64-gnu-v1.0.5.zip",
|
||||
"artifact_sha256": "5addae2f74756bf80104d0975e6347cda843bc2daffdfcd1f0cb80118b95cbd0"
|
||||
},
|
||||
{
|
||||
"version": "1.0.1",
|
||||
"role": "N-1",
|
||||
@@ -136,6 +130,6 @@
|
||||
}
|
||||
],
|
||||
"rollback_supported": false,
|
||||
"rollback_basis": "The release upgrade lane checks bucket-config rollback. The hosted compatibility workflow also checks in-flight multipart rollback to 1.0.5. Full data-plane rollback is not certified and is never a default claim."
|
||||
"rollback_basis": "Only the bucket-config-rollback case (rollback_to_previous_release_reads_current_bucket_metadata) is exercised. Full data-plane rollback is not certified and is never a default claim."
|
||||
}
|
||||
}
|
||||
|
||||
@@ -13,6 +13,6 @@
|
||||
# s3_error_lines lines holding s3_error!, same scope, minus those the script
|
||||
# header's rule clears (rustfs_s3_types's macro); fails closed
|
||||
# ecstore_files Rust files referencing s3s paths under crates/ecstore/src
|
||||
files=33
|
||||
s3_error_lines=89
|
||||
files=39
|
||||
s3_error_lines=90
|
||||
ecstore_files=0
|
||||
|
||||
@@ -90,16 +90,11 @@ jobs:
|
||||
- version: 1.0.1
|
||||
asset: rustfs-linux-x86_64-gnu-v1.0.1.zip
|
||||
sha256: 9742e183cb0dfb0d48fa909797af572fe47b7f827569036baa4156e0d3958ee5
|
||||
- version: 1.0.5
|
||||
asset: rustfs-linux-x86_64-gnu-v1.0.5.zip
|
||||
sha256: 5addae2f74756bf80104d0975e6347cda843bc2daffdfcd1f0cb80118b95cbd0
|
||||
revision: 34e6449ed90e3d3a106f6655342e0b3127f33082
|
||||
env:
|
||||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
|
||||
UPGRADE_SOURCE_VERSION: ${{ matrix.source.version }}
|
||||
UPGRADE_SOURCE_ASSET: ${{ matrix.source.asset }}
|
||||
UPGRADE_SOURCE_SHA256: ${{ matrix.source.sha256 }}
|
||||
UPGRADE_SOURCE_REVISION: ${{ matrix.source.revision }}
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
@@ -137,13 +132,6 @@ jobs:
|
||||
unzip -q "$archive" -d "$SOURCE_DIR"
|
||||
chmod +x "$SOURCE_DIR/rustfs"
|
||||
test -x "$SOURCE_DIR/rustfs"
|
||||
if [[ -n "$UPGRADE_SOURCE_REVISION" ]]; then
|
||||
mkdir -p "$RUNNER_TEMP/rustfs-upgrade-logs"
|
||||
version_log="$RUNNER_TEMP/rustfs-upgrade-logs/source-version.txt"
|
||||
"$SOURCE_DIR/rustfs" --version | tee "$version_log"
|
||||
grep -Fx "rustfs $UPGRADE_SOURCE_VERSION" "$version_log"
|
||||
grep -Eq "^git commit[[:space:]]*:[[:space:]]*$UPGRADE_SOURCE_REVISION$" "$version_log"
|
||||
fi
|
||||
echo "RUSTFS_UPGRADE_SOURCE_BINARY=$SOURCE_DIR/rustfs" >> "$GITHUB_ENV"
|
||||
echo "RUSTFS_E2E_LOG_DIR=$RUNNER_TEMP/rustfs-upgrade-logs" >> "$GITHUB_ENV"
|
||||
|
||||
@@ -163,14 +151,12 @@ jobs:
|
||||
multipart-layout-upgrade:direct_upgrade_from_previous_release_preserves_multipart_layouts
|
||||
multipart-layout-baseline:previous_release_baseline_replicates_multipart_layouts
|
||||
)
|
||||
case "$UPGRADE_SOURCE_VERSION" in
|
||||
1.0.0-rc.5)
|
||||
cases+=(odm-config-rollback:rc5_rollback_requires_restoring_odm_configuration)
|
||||
cases+=(checkpoint-upgrade:rc5_checkpoint_upgrade_rewinds_signed_progress_without_losing_objects) ;;
|
||||
1.0.1) cases+=(legacy-delete-reservation:stable_upgrade_deletes_foreign_reservations_without_losing_live_objects) ;;
|
||||
1.0.5) ;; # The bucket rollback case also completes an in-flight multipart upload.
|
||||
*) echo "::error::Unknown upgrade source $UPGRADE_SOURCE_VERSION"; exit 1 ;;
|
||||
esac
|
||||
if [[ "$UPGRADE_SOURCE_VERSION" == "1.0.0-rc.5" ]]; then
|
||||
cases+=(odm-config-rollback:rc5_rollback_requires_restoring_odm_configuration)
|
||||
cases+=(checkpoint-upgrade:rc5_checkpoint_upgrade_rewinds_signed_progress_without_losing_objects)
|
||||
else
|
||||
cases+=(legacy-delete-reservation:stable_upgrade_deletes_foreign_reservations_without_losing_live_objects)
|
||||
fi
|
||||
status=0
|
||||
for entry in "${cases[@]}"; do
|
||||
artifact="${entry%%:*}"
|
||||
@@ -192,7 +178,7 @@ jobs:
|
||||
exit "$status"
|
||||
|
||||
- name: Run mixed-version MPU recovery
|
||||
if: ${{ !cancelled() && steps.verify_source.outcome == 'success' && matrix.source.version != '1.0.5' }}
|
||||
if: ${{ !cancelled() && steps.verify_source.outcome == 'success' }}
|
||||
timeout-minutes: 15
|
||||
env:
|
||||
NO_PROXY: 127.0.0.1,localhost,::1
|
||||
@@ -213,24 +199,6 @@ jobs:
|
||||
"$mpu_test" -- --exact --nocapture --test-threads=1 \
|
||||
2>&1 | tee "$RUSTFS_E2E_LOG_DIR/test.log"
|
||||
|
||||
- name: Verify gateway baseline G09 evidence
|
||||
if: ${{ success() && matrix.source.version == '1.0.5' }}
|
||||
shell: bash
|
||||
run: |
|
||||
for field in mixed_version_reader_evidence mixed_version_writer_evidence rollback_payload_evidence; do
|
||||
directory=mixed-version-upgrade
|
||||
if [[ "$field" == rollback_payload_evidence ]]; then directory=bucket-config-rollback; fi
|
||||
jq -e --arg field "$field" --arg current "$GITHUB_SHA" --arg previous "$UPGRADE_SOURCE_REVISION" \
|
||||
'.gate == "G09" and .field == $field and .current_revision == $current and .previous_revision == $previous
|
||||
and ($field != "rollback_payload_evidence" or .rollback_payload_replayed == true)' \
|
||||
"$RUNNER_TEMP/rustfs-upgrade-g09-evidence/$directory/G09-$field.json"
|
||||
done
|
||||
jq -e --arg current "$GITHUB_SHA" --arg previous "$UPGRADE_SOURCE_REVISION" \
|
||||
'.current_revision == $current and .previous_revision == $previous
|
||||
and .part_sizes == [5242880, 5242880, 4096] and .bytes == (.part_sizes | add)
|
||||
and (.rollback_read_sha256 | test("^[a-f0-9]{64}$")) and .rollback_read_sha256 == .roll_forward_read_sha256' \
|
||||
"$RUNNER_TEMP/rustfs-upgrade-g09-evidence/bucket-config-rollback/inflight-multipart-rollback.json"
|
||||
|
||||
- name: Upload scanner/heal G09 evidence
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
|
||||
+1
-1
@@ -13,7 +13,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
|
||||
### Deployment
|
||||
|
||||
- **The S3 protocol stack is now rustfs/gateway**, and it is the only S3 stack. The legacy s3s-based stack is removed, and so are the `RUSTFS_S3_STACK` environment variable and the `gateway-stack` cargo feature. There is no on-disk format change. `RUSTFS_S3_STACK` no longer selects anything: a server started with it set, to any value, refuses to start and prints `RUSTFS_S3_STACK is no longer supported; the gateway stack is the only S3 stack since <version>`, where `<version>` is the release's version. Remove the variable from service units, container environments, Helm values and orchestration files before upgrading. To roll back, follow the [S3 stack rollback procedure](docs/operations/s3-stack-rollback.md).
|
||||
- **The S3 protocol stack is now rustfs/gateway**, and it is the only S3 stack. The legacy s3s-based stack is removed, and so are the `RUSTFS_S3_STACK` environment variable and the `gateway-stack` cargo feature. There is no on-disk format change. `RUSTFS_S3_STACK` no longer selects anything: a server started with it set, to any value, refuses to start and prints `RUSTFS_S3_STACK is no longer supported; the gateway stack is the only S3 stack since <version>`, where `<version>` is the release's version. Remove the variable from service units, container environments, Helm values and orchestration files before upgrading. To roll back, run the previous release binary on the same data.
|
||||
|
||||
- Before upgrading, verify that the configured root access key is at least 3 bytes and the root secret key is at least 8 bytes. RustFS now rejects shorter values at startup and identifies the setting to update. Version 1.0.5 accepted these settings at startup and returned HTTP 500 when root requested STS credentials. Coordinate root credential changes across all nodes and clients. See [root credential rotation considerations](docs/operations/two-factor-auth.md#root-credentials-cannot-be-changed-at-runtime).
|
||||
|
||||
|
||||
Generated
-2
@@ -4238,7 +4238,6 @@ dependencies = [
|
||||
"s3s",
|
||||
"same-file",
|
||||
"serde",
|
||||
"serde-value",
|
||||
"serde_json",
|
||||
"serde_urlencoded",
|
||||
"sha1 0.11.0",
|
||||
@@ -11094,7 +11093,6 @@ name = "rustfs-protos"
|
||||
version = "1.0.5"
|
||||
dependencies = [
|
||||
"flatbuffers",
|
||||
"hotpath",
|
||||
"hyper",
|
||||
"hyper-util",
|
||||
"prost 0.14.6",
|
||||
|
||||
@@ -222,7 +222,6 @@ quick-xml = "0.42.0"
|
||||
rmp = { version = "0.8.15" }
|
||||
rmp-serde = { version = "1.3.1" }
|
||||
serde = { version = "1.0.229" }
|
||||
serde-value = "0.7.0"
|
||||
serde_ignored = { version = "0.1" }
|
||||
serde_with = { version = "3", default-features = false, features = ["macros", "std"] }
|
||||
serde_json = { version = "1.0.154" }
|
||||
|
||||
@@ -32,6 +32,9 @@ pub const DEFAULT_HEAL_BITROT_CYCLE_SECS: u64 = 30 * 24 * 60 * 60;
|
||||
/// - Note: Enabling auto-heal can improve system resilience by automatically addressing issues, but may increase resource usage; evaluate based on your operational requirements.
|
||||
pub const ENV_HEAL_AUTO_HEAL_ENABLE: &str = "RUSTFS_HEAL_AUTO_HEAL_ENABLE";
|
||||
|
||||
/// Colon-separated Linux mount roots for automatic replacement of data directories below a mount point.
|
||||
pub const ENV_HEAL_REPLACEMENT_MOUNT_ROOTS: &str = "RUSTFS_HEAL_REPLACEMENT_MOUNT_ROOTS";
|
||||
|
||||
/// Environment variable name that specifies the heal queue size.
|
||||
///
|
||||
/// - Purpose: Set the maximum number of heal requests that can be queued.
|
||||
|
||||
@@ -138,7 +138,6 @@ rustls = { workspace = true, default-features = false, features = ["aws-lc-rs",
|
||||
clap = { workspace = true, features = ["derive", "env"] }
|
||||
|
||||
[dev-dependencies]
|
||||
serde-value.workspace = true
|
||||
s3s = { workspace = true, features = ["minio"] }
|
||||
async-compression = { workspace = true, features = ["tokio", "bzip2", "lz4", "xz"] }
|
||||
aws-config = { workspace = true }
|
||||
|
||||
@@ -31,7 +31,7 @@ use super::harness::{
|
||||
use crate::common::{
|
||||
AdminTransport, admin_add_canned_policy_via, admin_attach_user_policy_via, admin_create_user_via, init_logging,
|
||||
};
|
||||
use crate::upgrade_compatibility_test::{UpgradeSourceCapabilities, wait_for_upgrade_write_readiness};
|
||||
use crate::upgrade_compatibility_test::wait_for_upgrade_write_readiness;
|
||||
use aws_sdk_s3::Client;
|
||||
use aws_sdk_s3::error::ProvideErrorMetadata;
|
||||
use std::path::{Path, PathBuf};
|
||||
@@ -427,12 +427,7 @@ async fn assert_history_and_iam(dist: &DistCluster, seed: &UpgradeSeed, context:
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn compare_degraded_legacy_list_parts(
|
||||
dist: &mut DistCluster,
|
||||
seed: &UpgradeSeed,
|
||||
previous: &Path,
|
||||
source: &UpgradeSourceCapabilities,
|
||||
) -> TestResult {
|
||||
async fn compare_degraded_legacy_list_parts(dist: &mut DistCluster, seed: &UpgradeSeed, previous: &Path) -> TestResult {
|
||||
let decoded = String::from_utf8(base64_simd::URL_SAFE_NO_PAD.decode_to_vec(seed.multipart_upload.as_bytes())?)?;
|
||||
let (_, upload_dir) = decoded.split_once('.').ok_or("legacy upload ID lacks deployment delimiter")?;
|
||||
let hash = sha256_hex(format!("{}/legacy-multipart.bin", seed.history_bucket).as_bytes());
|
||||
@@ -494,22 +489,16 @@ async fn compare_degraded_legacy_list_parts(
|
||||
// Three online disks retain metadata quorum but have no part directory;
|
||||
// the fourth disk is offline. A legacy empty response is not a proof that
|
||||
// the unavailable participant has no part.
|
||||
let mut quorum_errors =
|
||||
vec![current.expect_err("current coordinator must not infer empty from missing directories and offline disks")];
|
||||
if source.requires_list_parts_read_quorum {
|
||||
quorum_errors.push(legacy.expect_err("previous coordinator must require evidence from unavailable disks"));
|
||||
} else {
|
||||
assert!(
|
||||
legacy
|
||||
.expect("previous coordinator should expose its legacy empty-list behavior")
|
||||
.parts()
|
||||
.is_empty()
|
||||
);
|
||||
}
|
||||
for error in quorum_errors {
|
||||
assert_eq!(error.raw_response().expect("S3 response").status().as_u16(), 503);
|
||||
assert_eq!(error.as_service_error().and_then(|error| error.meta().code()), Some("ServiceUnavailable"));
|
||||
}
|
||||
let legacy = legacy.expect("previous coordinator should expose its legacy empty-list behavior");
|
||||
assert!(legacy.parts().is_empty());
|
||||
let current = current.expect_err("current coordinator must not infer empty from missing directories and offline disks");
|
||||
assert_eq!(current.raw_response().expect("current S3 response").status().as_u16(), 503);
|
||||
// Since #9149, SlowDownRead is reserved for GET/HEAD object reads; a
|
||||
// ListParts read-quorum failure is reported as ServiceUnavailable.
|
||||
assert_eq!(
|
||||
current.as_service_error().and_then(|error| error.meta().code()),
|
||||
Some("ServiceUnavailable")
|
||||
);
|
||||
dist.cluster.start_node_from_binary(3, previous).await?;
|
||||
wait_for_upgrade_write_readiness(&clients, &seed.history_bucket, "mixed-list-restored", CREDENTIAL_TIMEOUT).await?;
|
||||
for (path, before) in original_files {
|
||||
@@ -531,7 +520,7 @@ async fn compare_degraded_legacy_list_parts(
|
||||
assert_eq!(listed.parts()[0].e_tag(), Some(seed.multipart_etag.as_str()));
|
||||
}
|
||||
eprintln!(
|
||||
"MPU mixed degraded: one offline disk, three missing part directories; checked each coordinator against its release contract; original part restored on both"
|
||||
"MPU mixed degraded: one offline disk, three missing part directories; previous returned empty, current returned 503 SlowDownRead; original part restored on both"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
@@ -603,7 +592,6 @@ async fn replace_node_and_wait_for_write_readiness(dist: &mut DistCluster, seed:
|
||||
async fn four_node_rolling_upgrade_preserves_history_and_iam_credentials() -> TestResult {
|
||||
init_logging();
|
||||
let previous = source_binary()?;
|
||||
let source = UpgradeSourceCapabilities::from_binary(&previous).await?;
|
||||
let mut dist = DistCluster::new_stopped(DistLayout::FourNodeFourDisk).await?;
|
||||
dist.cluster.set_env("RUSTFS_STORAGE_CLASS_RRS", "EC:0");
|
||||
capture_upgrade_logs(&mut dist, "rolling-upgrade")?;
|
||||
@@ -613,7 +601,7 @@ async fn four_node_rolling_upgrade_preserves_history_and_iam_credentials() -> Te
|
||||
|
||||
replace_node_and_wait_for_write_readiness(&mut dist, &seed, 0).await?;
|
||||
assert_history_and_iam(&dist, &seed, "one-current-node").await?;
|
||||
compare_degraded_legacy_list_parts(&mut dist, &seed, &previous, &source).await?;
|
||||
compare_degraded_legacy_list_parts(&mut dist, &seed, &previous).await?;
|
||||
|
||||
for node_idx in [1, 2] {
|
||||
replace_node_and_wait_for_write_readiness(&mut dist, &seed, node_idx).await?;
|
||||
|
||||
@@ -27,7 +27,6 @@ use aws_sdk_s3::types::{BucketCannedAcl, Tag, Tagging};
|
||||
use http::{HeaderMap, Method};
|
||||
use md5::{Digest, Md5};
|
||||
use rustfs_filemeta::{FileMeta, MetaObject};
|
||||
use serde_value::Value;
|
||||
use std::collections::BTreeMap;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
@@ -72,9 +71,7 @@ fn bucket_metadata(volume: &Path, bucket: &str) -> Result<BucketMetadata, Error>
|
||||
Ok(BucketMetadata::unmarshal(fields)?)
|
||||
}
|
||||
|
||||
type VersionFields = BTreeMap<String, Value>;
|
||||
|
||||
fn read_object(volume: &Path, key: &str) -> Result<(MetaObject, VersionFields), Error> {
|
||||
fn read_object(volume: &Path, key: &str) -> Result<MetaObject, Error> {
|
||||
let meta = FileMeta::load(&std::fs::read(volume.join(OBJECTS).join(key).join("xl.meta"))?)?;
|
||||
let [version] = meta.versions.as_slice() else {
|
||||
return Err(format!("{key}: expected one version").into());
|
||||
@@ -83,23 +80,7 @@ fn read_object(volume: &Path, key: &str) -> Result<(MetaObject, VersionFields),
|
||||
if object.mod_time.is_none() || object.data_dir.is_none_or(|id| id.is_nil()) {
|
||||
return Err(format!("{key}: missing modification time or data directory").into());
|
||||
}
|
||||
Ok((object, nonvolatile_fields(&version.meta, key)?))
|
||||
}
|
||||
|
||||
fn nonvolatile_fields(raw: &[u8], key: &str) -> Result<VersionFields, Error> {
|
||||
// The production reader skips unknown keys and supplies defaults for some missing keys.
|
||||
// Keep the raw maps so both the version envelope and V2Obj preserve every field and value.
|
||||
let mut fields: VersionFields = rmp_serde::from_slice(raw)?;
|
||||
let Some(Value::Map(object_fields)) = fields.get_mut("V2Obj") else {
|
||||
return Err(format!("{key}: missing raw V2Obj map").into());
|
||||
};
|
||||
for volatile in ["MTime", "DDir"] {
|
||||
let value = object_fields
|
||||
.get_mut(&Value::String(volatile.to_owned()))
|
||||
.ok_or_else(|| format!("{key}: missing {volatile}"))?;
|
||||
*value = Value::Unit;
|
||||
}
|
||||
Ok(fields)
|
||||
Ok(object)
|
||||
}
|
||||
|
||||
async fn put_xml(env: &RustFSTestEnvironment, query: &str, xml: &str) -> Result<(), Error> {
|
||||
@@ -123,7 +104,7 @@ async fn put_xml(env: &RustFSTestEnvironment, query: &str, xml: &str) -> Result<
|
||||
|
||||
struct Snapshot {
|
||||
configs: BTreeMap<&'static str, Vec<u8>>,
|
||||
objects: BTreeMap<&'static str, VersionFields>,
|
||||
objects: BTreeMap<&'static str, MetaObject>,
|
||||
multipart_files: BTreeMap<String, Vec<u8>>,
|
||||
}
|
||||
|
||||
@@ -314,10 +295,12 @@ fn read_snapshot(volume: &Path) -> Result<Snapshot, Error> {
|
||||
]);
|
||||
let mut objects = BTreeMap::new();
|
||||
for key in ["put", "copy", "tagged"] {
|
||||
let (_, fields) = read_object(volume, key)?;
|
||||
objects.insert(key, fields);
|
||||
let mut object = read_object(volume, key)?;
|
||||
object.mod_time = None;
|
||||
object.data_dir = None;
|
||||
objects.insert(key, object);
|
||||
}
|
||||
let (multipart, _) = read_object(volume, "multipart")?;
|
||||
let multipart = read_object(volume, "multipart")?;
|
||||
if multipart.part_numbers != [1, 2] || multipart.part_sizes != [FIRST_PART_SIZE, 100] {
|
||||
return Err("xl.meta does not describe both uploaded parts".into());
|
||||
}
|
||||
@@ -427,23 +410,3 @@ async fn persistence_byte_parity() -> Result<(), Error> {
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn version_comparison_preserves_unknown_fields_and_absent_values() -> Result<(), Error> {
|
||||
let mut sample = serde_json::json!({"Type": 1, "v": 1, "V2Obj": {"DDir": vec![1; 16], "MTime": 1}});
|
||||
let fields = |value: &serde_json::Value| nonvolatile_fields(&rmp_serde::to_vec(value)?, "fixture");
|
||||
let original = fields(&sample)?;
|
||||
sample["V2Obj"]["DDir"] = serde_json::json!(vec![2; 16]);
|
||||
sample["V2Obj"]["MTime"] = serde_json::json!(2);
|
||||
assert_eq!(fields(&sample)?, original);
|
||||
sample["V2Obj"]["Future"] = serde_json::Value::Null;
|
||||
let with_nil = fields(&sample)?;
|
||||
assert_ne!(with_nil, original);
|
||||
sample["V2Obj"]["Future"] = serde_json::json!(false);
|
||||
let with_value = fields(&sample)?;
|
||||
assert_ne!(with_value, with_nil);
|
||||
sample["FutureEnvelope"] = serde_json::json!(1);
|
||||
assert_ne!(fields(&sample)?, with_value);
|
||||
assert!(fields(&serde_json::json!({"V2Obj": {"MTime": 1}})).is_err());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -24,7 +24,7 @@
|
||||
#[cfg(all(test, target_os = "linux"))]
|
||||
mod tests {
|
||||
use crate::chaos::{VersionShardCensus, census_object_version_on_disk};
|
||||
use crate::common::{ClusterTopology, RustFSTestClusterEnvironment, admin_request, init_logging};
|
||||
use crate::common::{ClusterTopology, RustFSTestClusterEnvironment, admin_request, init_logging, local_http_client};
|
||||
use aws_sdk_s3::Client;
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::{BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, VersioningConfiguration};
|
||||
@@ -412,7 +412,7 @@ mod tests {
|
||||
.arg(test_binary)
|
||||
.arg("--exact")
|
||||
.arg(test_name)
|
||||
.arg("--ignored")
|
||||
.arg("--include-ignored")
|
||||
.arg("--nocapture")
|
||||
.env(NAMESPACE_ENV, "1")
|
||||
.status()?;
|
||||
@@ -1524,6 +1524,110 @@ mod tests {
|
||||
assert!(assert_no_replacement_status_records_in_status(&definitive, target).is_ok());
|
||||
}
|
||||
|
||||
/// Linux mount namespaces are per-thread; keep mount setup and process
|
||||
/// spawning on one OS thread so child RustFS nodes inherit the test mounts.
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
async fn test_privileged_4x1_subdirectory_replacement_waits_for_write_quorum() -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
init_logging();
|
||||
if !privileged_run_enabled()? {
|
||||
return Ok(());
|
||||
}
|
||||
if std::env::var_os(NAMESPACE_ENV).is_none() {
|
||||
return run_current_test_in_mount_namespace(
|
||||
"replacement_privileged_e2e_test::tests::test_privileged_4x1_subdirectory_replacement_waits_for_write_quorum",
|
||||
);
|
||||
}
|
||||
|
||||
let mut mounts = MountNamespaceGuard::new()?;
|
||||
let mut cluster = RustFSTestClusterEnvironment::new(4).await?;
|
||||
let filesystem: serde_json::Value = serde_json::from_str(&run_command_stdout(
|
||||
"findmnt",
|
||||
&["--json", "--target", &cluster.temp_dir, "--output", "TARGET,FSTYPE"],
|
||||
)?)?;
|
||||
let xfs_root = filesystem["filesystems"][0]["target"]
|
||||
.as_str()
|
||||
.filter(|root| *root != "/" && filesystem["filesystems"][0]["fstype"] == "xfs");
|
||||
for (index, node) in cluster.nodes.iter_mut().enumerate() {
|
||||
let mount = PathBuf::from(&node.data_dir);
|
||||
if xfs_root.is_none() {
|
||||
mounts.mount_tmpfs(&mount, &format!("rustfs-replacement-4x1-{index}"))?;
|
||||
}
|
||||
let data = mount.join("data");
|
||||
fs::create_dir(&data)?;
|
||||
node.data_dir = path_to_string(&data, "replacement subdirectory")?;
|
||||
node.data_dirs[0] = node.data_dir.clone();
|
||||
}
|
||||
let target = PathBuf::from(&cluster.nodes[TARGET_NODE].data_dir);
|
||||
let mount = xfs_root.unwrap_or(target.parent().and_then(Path::to_str).ok_or("missing target mount root")?);
|
||||
eprintln!(
|
||||
"4x1 replacement fixture filesystem = {}",
|
||||
if xfs_root.is_some() { "xfs" } else { "tmpfs" }
|
||||
);
|
||||
cluster.set_node_env(TARGET_NODE, "RUSTFS_HEAL_REPLACEMENT_MOUNT_ROOTS", mount)?;
|
||||
cluster.set_env("RUSTFS_STORAGE_CLASS_STANDARD", "EC:2");
|
||||
cluster.set_env("RUSTFS_HEAL_ENABLED", "true");
|
||||
cluster.set_env("RUSTFS_HEAL_INTERVAL_SECS", "1");
|
||||
cluster.set_env("RUSTFS_SCANNER_START_DELAY_SECS", "0");
|
||||
cluster.set_env("RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE", "false");
|
||||
cluster.start().await?;
|
||||
let clients = cluster.create_all_clients()?;
|
||||
let versions = seed_baseline(&clients[0], &target, 0).await?;
|
||||
|
||||
cluster.stop_node_gracefully(TARGET_NODE).await?;
|
||||
cluster.stop_node_gracefully(0).await?;
|
||||
fs::remove_dir_all(&target)?;
|
||||
fs::create_dir(&target)?;
|
||||
|
||||
let metadata = ["format.json", "pool.bin/xl.meta", "pool.bin.identity/xl.meta"];
|
||||
let system_dir = Path::new(".rustfs.sys");
|
||||
let survivor_files = cluster.nodes[2..]
|
||||
.iter()
|
||||
.flat_map(|node| metadata.map(|name| Path::new(&node.data_dir).join(system_dir).join(name)))
|
||||
.map(|file| fs::read(&file).map(|bytes| (file, bytes)))
|
||||
.collect::<std::io::Result<Vec<_>>>()?;
|
||||
|
||||
cluster.start_node_process(TARGET_NODE).await?;
|
||||
let write_health = format!("{}/minio/health/cluster", cluster.nodes[2].url);
|
||||
let http = local_http_client();
|
||||
timeout(Duration::from_secs(30), async {
|
||||
loop {
|
||||
let status = http.get(&write_health).send().await?.status();
|
||||
if status.as_u16() == 503 {
|
||||
break;
|
||||
}
|
||||
sleep(Duration::from_millis(200)).await;
|
||||
}
|
||||
Ok::<_, reqwest::Error>(())
|
||||
})
|
||||
.await??;
|
||||
verify_bodies(&clients[2], &versions).await?;
|
||||
let version = &versions[0];
|
||||
let failed_delete = clients[2]
|
||||
.delete_object()
|
||||
.bucket(&version.bucket)
|
||||
.key(&version.key)
|
||||
.send()
|
||||
.await
|
||||
.expect_err("two of four drives must not delete an existing object");
|
||||
assert_eq!(failed_delete.raw_response().map(|response| response.status().as_u16()), Some(503));
|
||||
for name in metadata {
|
||||
assert!(!target.join(system_dir).join(name).try_exists()?, "unadmitted target published {name}");
|
||||
}
|
||||
for (file, before) in survivor_files {
|
||||
assert_eq!(fs::read(&file)?, before, "surviving pool evidence changed below write quorum");
|
||||
}
|
||||
|
||||
cluster.start_node(0).await?;
|
||||
wait_for_completed_replacement_with_census(&cluster, &target, &versions, 90).await?;
|
||||
for name in metadata {
|
||||
assert!(fs::metadata(target.join(system_dir).join(name))?.is_file());
|
||||
}
|
||||
cluster.stop_node_gracefully(TARGET_NODE).await?;
|
||||
cluster.start_node(TARGET_NODE).await?;
|
||||
assert!(incomplete_versions(&target, &versions)?.is_empty());
|
||||
verify_bodies(&clients[TARGET_NODE], &versions).await
|
||||
}
|
||||
|
||||
/// Linux mount namespaces are per-thread; keep mount setup and process
|
||||
/// spawning on one OS thread so child RustFS nodes inherit the test mounts.
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
|
||||
@@ -46,12 +46,9 @@ type BoxError = Box<dyn std::error::Error + Send + Sync>;
|
||||
|
||||
const SOURCE_BINARY_ENV: &str = "RUSTFS_UPGRADE_SOURCE_BINARY";
|
||||
const G09_EVIDENCE_DIR_ENV: &str = "RUSTFS_SCANNER_HEAL_G09_EVIDENCE_DIR";
|
||||
const RC2_COMMIT: &str = "e9f53180273b32a9c2b0aa8b5a384047bc7e875c";
|
||||
const RC5_COMMIT: &str = "40a2470feb567201165a5b809b7598bb4b1f68f5";
|
||||
const STABLE_1_0_1_COMMIT: &str = "6de965ae3c965a78ff819fbcd7acd4aa44177d92";
|
||||
const STABLE_1_0_4_COMMIT: &str = "46337942d55f3a8e6a97b147ca98d9ea8115555d";
|
||||
const STABLE_1_0_5_COMMIT: &str = "34e6449ed90e3d3a106f6655342e0b3127f33082";
|
||||
const STABLE_1_0_6_COMMIT: &str = "de9009d882644effc2628e777650025c3d8e7c65";
|
||||
const SSE_MASTER_KEY_ENV: &str = "RUSTFS_SSE_S3_MASTER_KEY";
|
||||
const SSE_MASTER_KEY: &str = "QkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkI=";
|
||||
const PLAIN_BUCKET: &str = "upgrade-plain-data";
|
||||
@@ -145,41 +142,6 @@ fn current_source_revision() -> Result<String, BoxError> {
|
||||
Ok(revision.to_string())
|
||||
}
|
||||
|
||||
pub(crate) struct UpgradeSourceCapabilities {
|
||||
supports_delete_recovery: bool,
|
||||
supports_multipart_publication: bool,
|
||||
pub(crate) requires_list_parts_read_quorum: bool,
|
||||
}
|
||||
|
||||
impl UpgradeSourceCapabilities {
|
||||
fn from_revision(revision: &str) -> Result<Self, BoxError> {
|
||||
// 1.0.4 supports the delete decision protocol (#8468). Version 1.0.5 also
|
||||
// shipped native MPU publication (#8919) and strict ListParts evidence (#9037, #9149).
|
||||
match revision {
|
||||
RC2_COMMIT | RC5_COMMIT | STABLE_1_0_1_COMMIT => Ok(Self {
|
||||
supports_delete_recovery: false,
|
||||
supports_multipart_publication: false,
|
||||
requires_list_parts_read_quorum: false,
|
||||
}),
|
||||
STABLE_1_0_4_COMMIT => Ok(Self {
|
||||
supports_delete_recovery: true,
|
||||
supports_multipart_publication: false,
|
||||
requires_list_parts_read_quorum: false,
|
||||
}),
|
||||
STABLE_1_0_5_COMMIT | STABLE_1_0_6_COMMIT => Ok(Self {
|
||||
supports_delete_recovery: true,
|
||||
supports_multipart_publication: true,
|
||||
requires_list_parts_read_quorum: true,
|
||||
}),
|
||||
_ => Err(format!("no pinned upgrade expectations for source revision {revision}").into()),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn from_binary(binary: &Path) -> Result<Self, BoxError> {
|
||||
Self::from_revision(&binary_source_revision(binary).await?)
|
||||
}
|
||||
}
|
||||
|
||||
async fn binary_source_revision(binary: &Path) -> Result<String, BoxError> {
|
||||
let output = tokio::process::Command::new(binary).arg("--version").output().await?;
|
||||
if !output.status.success() {
|
||||
@@ -263,25 +225,6 @@ mod g09_evidence_tests {
|
||||
use super::*;
|
||||
use tempfile::TempDir;
|
||||
|
||||
#[test]
|
||||
fn pinned_upgrade_sources_keep_distinct_delete_and_list_parts_contracts() -> TestResult {
|
||||
for (revision, delete_recovery, multipart_publication, list_parts_quorum) in [
|
||||
(RC2_COMMIT, false, false, false),
|
||||
(RC5_COMMIT, false, false, false),
|
||||
(STABLE_1_0_1_COMMIT, false, false, false),
|
||||
(STABLE_1_0_4_COMMIT, true, false, false),
|
||||
(STABLE_1_0_5_COMMIT, true, true, true),
|
||||
(STABLE_1_0_6_COMMIT, true, true, true),
|
||||
] {
|
||||
let source = UpgradeSourceCapabilities::from_revision(revision)?;
|
||||
assert_eq!(source.supports_delete_recovery, delete_recovery, "{revision}");
|
||||
assert_eq!(source.supports_multipart_publication, multipart_publication, "{revision}");
|
||||
assert_eq!(source.requires_list_parts_read_quorum, list_parts_quorum, "{revision}");
|
||||
}
|
||||
assert!(UpgradeSourceCapabilities::from_revision("ffffffffffffffffffffffffffffffffffffffff").is_err());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn source_revision_parser_requires_lowercase_forty_byte_sha() -> TestResult {
|
||||
let binary = Path::new("rustfs");
|
||||
@@ -1068,7 +1011,8 @@ async fn exercise_mixed_cluster(
|
||||
vec![u8::try_from(MULTIPART_WORKERS - 1)?; 64 * 1024]
|
||||
);
|
||||
|
||||
// Use a separate old-writer key so the original seed still spans the entire rollout.
|
||||
// 1.0.4 already implements the delete decision protocol. Use a separate
|
||||
// old-writer key so the original seed still spans the entire rollout.
|
||||
if previous_has_delete_recovery {
|
||||
let key = format!(".upgrade-delete/{phase}/previous-writer");
|
||||
previous_client
|
||||
@@ -2349,7 +2293,7 @@ async fn rolling_upgrade_from_previous_release_preserves_mixed_version_contracts
|
||||
let previous_binary = source_binary()?;
|
||||
// Bind the expected protocol to the verified release source, never to the
|
||||
// DELETE result itself. The older CI source pins retain their 503 contract.
|
||||
let source = UpgradeSourceCapabilities::from_binary(&previous_binary).await?;
|
||||
let previous_has_delete_recovery = binary_source_revision(&previous_binary).await? == STABLE_1_0_4_COMMIT;
|
||||
let evidence_context = g09_evidence_context(&previous_binary).await?;
|
||||
let current_binary = rustfs_binary_path();
|
||||
let mut cluster = RustFSTestClusterEnvironment::new(MIXED_NODE_COUNT).await?;
|
||||
@@ -2371,21 +2315,17 @@ async fn rolling_upgrade_from_previous_release_preserves_mixed_version_contracts
|
||||
cluster.stop_node(0)?;
|
||||
cluster.start_node_from_binary(0, ¤t_binary).await?;
|
||||
let mut mixed_deleted_keys = Vec::new();
|
||||
if let Some(key) = exercise_mixed_cluster(&cluster, "one-current-node", 0, 1, source.supports_delete_recovery).await? {
|
||||
if let Some(key) = exercise_mixed_cluster(&cluster, "one-current-node", 0, 1, previous_has_delete_recovery).await? {
|
||||
mixed_deleted_keys.push(key);
|
||||
}
|
||||
let mixed_completion = if source.supports_multipart_publication {
|
||||
let clients = cluster.create_all_clients()?;
|
||||
Some(finish_pending_multipart(&clients, &pending, pending_parts.concat()).await?)
|
||||
} else {
|
||||
assert_mixed_pending_refusal(&cluster, &pending, &sources).await?;
|
||||
None
|
||||
};
|
||||
// One controlled out-of-policy attempt proves fail-closed behavior below
|
||||
// native write quorum; positive MPU writers remain paused while mixed.
|
||||
assert_mixed_pending_refusal(&cluster, &pending, &sources).await?;
|
||||
|
||||
for (node_idx, phase) in [(1, "two-current-nodes"), (2, "one-previous-node")] {
|
||||
cluster.stop_node(node_idx)?;
|
||||
cluster.start_node_from_binary(node_idx, ¤t_binary).await?;
|
||||
if let Some(key) = exercise_mixed_cluster(&cluster, phase, 0, 3, source.supports_delete_recovery).await? {
|
||||
if let Some(key) = exercise_mixed_cluster(&cluster, phase, 0, 3, previous_has_delete_recovery).await? {
|
||||
mixed_deleted_keys.push(key);
|
||||
}
|
||||
}
|
||||
@@ -2395,13 +2335,6 @@ async fn rolling_upgrade_from_previous_release_preserves_mixed_version_contracts
|
||||
let clients = cluster.create_all_clients()?;
|
||||
wait_for_drive_readmission(&clients, MIXED_BUCKET, "homogeneous-current", DRIVE_READMISSION_TIMEOUT).await?;
|
||||
let completed_pending = finish_pending_multipart(&clients, &pending, pending_parts.concat()).await?;
|
||||
if let Some(mixed) = mixed_completion {
|
||||
assert_eq!(
|
||||
completed_pending.version_id, mixed.version_id,
|
||||
"full upgrade changed the completed version"
|
||||
);
|
||||
assert_eq!(completed_pending.etag, mixed.etag, "full upgrade changed the completed ETag");
|
||||
}
|
||||
exercise_homogeneous_multipart(&cluster, "homogeneous-current").await?;
|
||||
verify_load_bodies(&clients, "homogeneous-previous", "multipart").await?;
|
||||
assert_mixed_deletes_visible(&clients, &mixed_deleted_keys).await?;
|
||||
@@ -2518,7 +2451,7 @@ async fn rolling_upgrade_from_previous_release_preserves_mixed_version_contracts
|
||||
"phases": phases,
|
||||
"objects_per_phase": objects_per_phase,
|
||||
"mixed_load": "plain-put",
|
||||
"mixed_delete_protocol_supported": source.supports_delete_recovery,
|
||||
"mixed_delete_protocol_supported": previous_has_delete_recovery,
|
||||
"mixed_deleted_keys": mixed_deleted_keys,
|
||||
"homogeneous_multipart_phases": ["homogeneous-previous", "homogeneous-current", "homogeneous-rollback"],
|
||||
"multipart_objects_per_homogeneous_phase": MULTIPART_WORKERS * MULTIPART_UPLOADS_PER_WORKER,
|
||||
@@ -2528,7 +2461,7 @@ async fn rolling_upgrade_from_previous_release_preserves_mixed_version_contracts
|
||||
"all nodes list every mixed-version object after homogeneous-current convergence",
|
||||
"the previous-release seed survives the baseline restart and every upgrade phase",
|
||||
"previous readers read every current-native MPU body after quiescent rollback",
|
||||
"the same previous-release upload remains readable after full upgrade and rollback"
|
||||
"same legacy upload resumes with identical body, ETag and version after full upgrade"
|
||||
],
|
||||
}),
|
||||
)?;
|
||||
@@ -2550,13 +2483,9 @@ async fn rolling_upgrade_from_previous_release_preserves_mixed_version_contracts
|
||||
"previous-release writer publishes objects readable by current node",
|
||||
"mixed phases retain 16x16 plain PUT writers while positive multipart writers are paused",
|
||||
"16x16 multipart load runs in homogeneous previous, current and rollback phases",
|
||||
if source.supports_multipart_publication {
|
||||
"one-current-node Complete and replay return the same version and ETag, and all readers agree on the body"
|
||||
} else {
|
||||
"one-current-node Complete returns typed 503 and preserves actual source authority and parts"
|
||||
},
|
||||
"fully upgraded readers see the same body, ETag and sole version after replay",
|
||||
if source.supports_delete_recovery {
|
||||
"one-current-node Complete returns typed 503 and preserves actual source authority and parts",
|
||||
"the same legacy upload completes and replays once fully upgraded without adding a version",
|
||||
if previous_has_delete_recovery {
|
||||
"mixed-version DELETE succeeds and stays absent through every reader after upgrade and rollback"
|
||||
} else {
|
||||
"modern DELETE fails closed and preserves every byte while old peers remain"
|
||||
@@ -3457,8 +3386,6 @@ async fn direct_upgrade_from_previous_release_preserves_bucket_configuration() -
|
||||
#[tokio::test]
|
||||
#[ignore = "requires a pinned previous RustFS release binary"]
|
||||
async fn rollback_to_previous_release_reads_current_bucket_metadata() -> TestResult {
|
||||
use sha2::Digest;
|
||||
|
||||
init_logging();
|
||||
let previous_binary = source_binary()?;
|
||||
let evidence_context = g09_evidence_context(&previous_binary).await?;
|
||||
@@ -3496,16 +3423,6 @@ async fn rollback_to_previous_release_reads_current_bucket_metadata() -> TestRes
|
||||
let multipart_bytes = multipart_parts.concat();
|
||||
write_multipart(&new_client, ROLLBACK_BUCKET, multipart_key, &multipart_parts).await?;
|
||||
|
||||
// This release can finish an upload staged by the current gateway.
|
||||
let inflight = if binary_source_revision(&previous_binary).await? == STABLE_1_0_5_COMMIT {
|
||||
env.create_test_bucket(PENDING_MULTIPART_BUCKET).await?;
|
||||
let parts = [vec![1; LAYOUT_PART_SIZE], vec![2; LAYOUT_PART_SIZE], vec![3; 4096]];
|
||||
let pending = stage_multipart(&new_client, PENDING_MULTIPART_BUCKET, PENDING_MULTIPART_KEY, &parts).await?;
|
||||
Some((pending, parts.concat()))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
restart_from_binary(&mut env, &previous_binary, &server_env).await?;
|
||||
let old_client = env.create_s3_client();
|
||||
|
||||
@@ -3521,16 +3438,6 @@ async fn rollback_to_previous_release_reads_current_bucket_metadata() -> TestRes
|
||||
assert_eq!(encryption, Some(ServerSideEncryption::Aes256));
|
||||
assert_eq!(body, multipart_bytes, "the multipart object did not survive the rollback");
|
||||
|
||||
let rollback_multipart = if let Some((pending, expected)) = inflight {
|
||||
assert_pending_parts(&old_client, &pending).await?;
|
||||
complete_pending_multipart(&old_client, PENDING_MULTIPART_BUCKET, PENDING_MULTIPART_KEY, &pending).await?;
|
||||
let (_, body) = read_object(&old_client, PENDING_MULTIPART_BUCKET, PENDING_MULTIPART_KEY, None).await?;
|
||||
assert_eq!(body, expected);
|
||||
Some((pending.part_sizes, body))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
// A PUT on the rolled-back release must still honour the encryption
|
||||
// configuration it decoded out of the current build's metadata blob.
|
||||
let post_rollback_key = "rollback/written-after-rollback";
|
||||
@@ -3571,25 +3478,6 @@ async fn rollback_to_previous_release_reads_current_bucket_metadata() -> TestRes
|
||||
post_rollback_bytes
|
||||
);
|
||||
|
||||
if let Some((part_sizes, rollback_read)) = rollback_multipart {
|
||||
let (_, current_read) = read_object(¤t_again, PENDING_MULTIPART_BUCKET, PENDING_MULTIPART_KEY, None).await?;
|
||||
assert_eq!(current_read, rollback_read);
|
||||
if let Some(context) = &evidence_context {
|
||||
let evidence = serde_json::json!({
|
||||
"previous_revision": context.previous_revision,
|
||||
"current_revision": context.current_revision,
|
||||
"part_sizes": part_sizes,
|
||||
"bytes": current_read.len(),
|
||||
"rollback_read_sha256": format!("{:x}", sha2::Sha256::digest(&rollback_read)),
|
||||
"roll_forward_read_sha256": format!("{:x}", sha2::Sha256::digest(¤t_read)),
|
||||
});
|
||||
std::fs::write(
|
||||
context.directory.join("inflight-multipart-rollback.json"),
|
||||
serde_json::to_vec_pretty(&evidence)?,
|
||||
)?;
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(context) = evidence_context.as_ref() {
|
||||
write_g09_evidence(
|
||||
context,
|
||||
|
||||
@@ -469,6 +469,7 @@ pub mod disk {
|
||||
pub use crate::cluster::rpc::background_pipe::{BackgroundPipeReader, BackgroundPipeWriter, background_pipe};
|
||||
pub use crate::cluster::rpc::background_reader_stream::BackgroundReaderStream;
|
||||
pub use crate::disk::BOUNDED_DELETE_PREPARE_PROTOCOL_VERSION;
|
||||
pub use crate::disk::ReplacementMount;
|
||||
pub use crate::disk::background_memory::BackgroundEncodeBuffer;
|
||||
pub use crate::disk::disk_store::get_object_disk_read_timeout;
|
||||
#[cfg(feature = "test-util")]
|
||||
|
||||
@@ -6599,7 +6599,9 @@ mod tests {
|
||||
);
|
||||
assert_eq!(
|
||||
remote
|
||||
.pool_meta_native_recovery(PoolMetaNativeRecoveryRequest::Probe {})
|
||||
.pool_meta_native_recovery(PoolMetaNativeRecoveryRequest::Inspect {
|
||||
object: crate::disk::PoolMetaNativeObject::PoolBin,
|
||||
})
|
||||
.await,
|
||||
Err(DiskError::UnsupportedDisk),
|
||||
);
|
||||
@@ -6610,7 +6612,7 @@ mod tests {
|
||||
"/node_service.NodeService/UndoPoolMetaWrite",
|
||||
"/node_service.NodeService/PoolMetaNativeRecovery"
|
||||
],
|
||||
"protected mutations and capability probes must never fall back to legacy RPCs",
|
||||
"protected mutations and native inspection must never fall back to legacy RPCs",
|
||||
);
|
||||
peer.stop().await;
|
||||
}
|
||||
|
||||
@@ -207,10 +207,10 @@ async fn pool_meta_native_physical_split_without_ordinary_commit_performs_no_mut
|
||||
let before = snapshots(&fixture.store).await;
|
||||
let mut state = clean_state(&fixture);
|
||||
let fence = PoolMetaPersistenceFence::Owned(owner(&fixture.store).await);
|
||||
assert!(
|
||||
repair(&fixture.store, &mut state, &fence).await.is_err(),
|
||||
"reconstruction alone must never authorize a COMMIT"
|
||||
);
|
||||
let error = repair(&fixture.store, &mut state, &fence)
|
||||
.await
|
||||
.expect_err("reconstruction alone must never authorize a COMMIT");
|
||||
assert!(error.to_string().contains("(no_ordinary_v3_authority)"), "{error}");
|
||||
assert_eq!(
|
||||
snapshots(&fixture.store).await,
|
||||
before,
|
||||
@@ -222,6 +222,24 @@ async fn pool_meta_native_physical_split_without_ordinary_commit_performs_no_mut
|
||||
.expect("missing ordinary authority alone must not install a permanent corruption latch");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(pool_meta_version_env)]
|
||||
async fn pool_meta_native_inspection_rejects_an_absent_slot_without_changing_evidence() {
|
||||
let fixture = fixture(1).await;
|
||||
let before = snapshots(&fixture.store).await;
|
||||
let fence = PoolMetaPersistenceFence::Owned(owner(&fixture.store).await);
|
||||
let last_pool = fixture.store.pools.last().expect("fixture has a last pool");
|
||||
let set = last_pool.get_disks_by_key(POOL_META_NAME);
|
||||
let missing = set.disks.write().await[0].take().expect("fixture has a disk in slot zero");
|
||||
|
||||
let result = inspect(&fixture.store.pools, &fence).await;
|
||||
|
||||
set.disks.write().await[0] = Some(missing);
|
||||
let error = result.err().expect("an absent slot cannot provide native evidence");
|
||||
assert!(error.to_string().contains("(missing_pool_metadata_slot)"), "{error}");
|
||||
assert_eq!(snapshots(&fixture.store).await, before);
|
||||
}
|
||||
|
||||
async fn interrupted_first_commit_fixture() -> (PhysicalFixture, PoolMetaCommittedCandidate) {
|
||||
let fixture = fixture(3).await;
|
||||
let PoolMetaReplica::Valid {
|
||||
|
||||
@@ -19,6 +19,7 @@ use crate::disk::{
|
||||
PoolMetaNativeRecoveryResponse as Response,
|
||||
};
|
||||
use bytes::Bytes;
|
||||
use futures::TryStreamExt as _;
|
||||
|
||||
#[cfg(test)]
|
||||
#[derive(Default)]
|
||||
@@ -38,8 +39,16 @@ struct InspectedPool {
|
||||
snapshots: Vec<PoolMetaNativeInspection>,
|
||||
}
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
#[error("native pool metadata recovery requires one proven V3 transaction and complete matching disk evidence ({0})")]
|
||||
struct MissingEvidence(&'static str);
|
||||
|
||||
fn missing_evidence(reason: &'static str) -> Error {
|
||||
Error::other(MissingEvidence(reason))
|
||||
}
|
||||
|
||||
fn invalid() -> Error {
|
||||
Error::other("native pool metadata recovery requires one proven V3 transaction and complete matching disk evidence")
|
||||
missing_evidence("inconsistent_evidence")
|
||||
}
|
||||
|
||||
fn known_validation(source: Error, phase: &'static str) -> Error {
|
||||
@@ -71,13 +80,19 @@ fn latch_validation(state: &mut PoolMetaWriteState, error: Error) -> Error {
|
||||
fn current(pool: &InspectedPool) -> Result<Vec<Bytes>> {
|
||||
pool.snapshots
|
||||
.iter()
|
||||
.map(|snapshot| snapshot.current.clone().ok_or_else(invalid))
|
||||
.map(|snapshot| {
|
||||
snapshot
|
||||
.current
|
||||
.clone()
|
||||
.ok_or_else(|| missing_evidence("missing_pool_metadata_head"))
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
async fn inspect(pools: &[Arc<Sets>], fence: &PoolMetaPersistenceFence<'_>) -> Result<Vec<InspectedPool>> {
|
||||
let mut inspected = Vec::with_capacity(pools.len());
|
||||
// Pool -> disk order is stable; all capability probes precede every mutation.
|
||||
// Check every slot before starting I/O. The authenticated, read-only
|
||||
// Inspect RPC proves both protocol support and the disk's evidence.
|
||||
for pool in pools {
|
||||
fence.ensure_held()?;
|
||||
let set = pool.get_disks_by_key(POOL_META_NAME);
|
||||
@@ -87,20 +102,8 @@ async fn inspect(pools: &[Arc<Sets>], fence: &PoolMetaPersistenceFence<'_>) -> R
|
||||
.await
|
||||
.iter()
|
||||
.cloned()
|
||||
.map(|disk| disk.ok_or_else(invalid))
|
||||
.map(|disk| disk.ok_or_else(|| missing_evidence("missing_pool_metadata_slot")))
|
||||
.collect::<Result<Vec<_>>>()?;
|
||||
let probes = futures::stream::iter(disks.iter().cloned().map(|disk| async move {
|
||||
match disk.pool_meta_native_recovery(Request::Probe {}).await? {
|
||||
Response::Probed => Ok(()),
|
||||
_ => Err(invalid()),
|
||||
}
|
||||
}))
|
||||
.buffered(4)
|
||||
.collect::<Vec<Result<()>>>()
|
||||
.await;
|
||||
for result in probes {
|
||||
result?;
|
||||
}
|
||||
inspected.push(InspectedPool {
|
||||
disks,
|
||||
snapshots: Vec::new(),
|
||||
@@ -108,7 +111,7 @@ async fn inspect(pools: &[Arc<Sets>], fence: &PoolMetaPersistenceFence<'_>) -> R
|
||||
}
|
||||
for pool in &mut inspected {
|
||||
fence.ensure_held()?;
|
||||
let results = futures::stream::iter(pool.disks.iter().cloned().map(|disk| async move {
|
||||
pool.snapshots = futures::stream::iter(pool.disks.iter().cloned().map(|disk| async move {
|
||||
match disk
|
||||
.pool_meta_native_recovery(Request::Inspect {
|
||||
object: PoolMetaNativeObject::PoolBin,
|
||||
@@ -116,15 +119,12 @@ async fn inspect(pools: &[Arc<Sets>], fence: &PoolMetaPersistenceFence<'_>) -> R
|
||||
.await?
|
||||
{
|
||||
Response::Inspected(snapshot) if !snapshot.disk_id.is_nil() => Ok(snapshot),
|
||||
_ => Err(invalid()),
|
||||
_ => Err(missing_evidence("invalid_disk_identity_evidence")),
|
||||
}
|
||||
}))
|
||||
.buffered(4)
|
||||
.collect::<Vec<Result<PoolMetaNativeInspection>>>()
|
||||
.await;
|
||||
for result in results {
|
||||
pool.snapshots.push(result?);
|
||||
}
|
||||
.try_collect()
|
||||
.await?;
|
||||
}
|
||||
fence.ensure_held()?;
|
||||
Ok(inspected)
|
||||
@@ -142,7 +142,7 @@ async fn validate_identities(pools: &[Arc<Sets>], canonical: &[u8], fence: &Pool
|
||||
.await
|
||||
.iter()
|
||||
.cloned()
|
||||
.map(|disk| disk.ok_or_else(invalid))
|
||||
.map(|disk| disk.ok_or_else(|| missing_evidence("missing_pool_identity_slot")))
|
||||
.collect::<Result<Vec<_>>>()?;
|
||||
let results = futures::stream::iter(disks.into_iter().enumerate().map(|(index, disk)| async move {
|
||||
match disk.pool_meta_native_recovery(Request::InspectIdentity {}).await? {
|
||||
@@ -151,7 +151,7 @@ async fn validate_identities(pools: &[Arc<Sets>], canonical: &[u8], fence: &Pool
|
||||
.await
|
||||
.map_err(|error| codec_validation(error, "native_identity_validation"))
|
||||
}
|
||||
_ => Err(invalid()),
|
||||
_ => Err(missing_evidence("invalid_disk_identity_evidence")),
|
||||
}
|
||||
}))
|
||||
.buffered(4)
|
||||
@@ -479,7 +479,7 @@ pub(super) async fn repair(
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
if ordinary_commits.is_empty() && ordinary_prepares.is_empty() {
|
||||
return Err(invalid());
|
||||
return Err(missing_evidence("no_ordinary_v3_authority"));
|
||||
}
|
||||
let selection = select_pool_meta_replica_reads(reads)
|
||||
.map_err(|error| latch_validation(state, known_validation(error, "native_metadata_validation")))?;
|
||||
@@ -535,7 +535,7 @@ pub(super) async fn repair(
|
||||
.iter()
|
||||
.any(|read| !matches!(read.replica, PoolMetaIdentityReplica::Valid(_)))
|
||||
{
|
||||
return Err(invalid());
|
||||
return Err(missing_evidence("pool_identity_unavailable"));
|
||||
}
|
||||
let identity = select_pool_meta_identity(identities, cluster_id)
|
||||
.map_err(|error| latch_validation(state, known_validation(error, "native_identity_validation")))?;
|
||||
@@ -847,7 +847,10 @@ mod tests {
|
||||
async fn pool_meta_native_plan_rejects_missing_or_corrupt_evidence() {
|
||||
let mut f = fixture().await;
|
||||
let before = f.pools[1].snapshots[2].current.take();
|
||||
assert!(plan(&f.pools, &f.commit, f.revision, &f.identity, None).await.is_err());
|
||||
let error = plan(&f.pools, &f.commit, f.revision, &f.identity, None)
|
||||
.await
|
||||
.expect_err("missing current shard must block the repair plan");
|
||||
assert!(error.to_string().contains("(missing_pool_metadata_head)"), "{error}");
|
||||
f.pools[1].snapshots[2].current = before;
|
||||
f.pools[1].snapshots[2].backups[0].data = Bytes::from_static(b"invalid metadata");
|
||||
assert!(plan(&f.pools, &f.commit, f.revision, &f.identity, None).await.is_err());
|
||||
|
||||
@@ -8268,8 +8268,9 @@ pub struct LocalDisk {
|
||||
io_root: PathBuf,
|
||||
#[cfg(target_os = "linux")]
|
||||
mount_lease: Arc<std::fs::File>,
|
||||
// The descriptor pins this identity even if the configured path is remounted.
|
||||
#[cfg(target_os = "linux")]
|
||||
mount_lease_mount_id: Option<u64>,
|
||||
mount_lease_identity: Option<ReplacementMountIdentity>,
|
||||
/// Public path for callers that need the configured disk layout. Internal
|
||||
/// disk I/O uses `io_format_path`, which is rooted at `mount_lease`.
|
||||
pub format_path: PathBuf,
|
||||
@@ -8469,72 +8470,50 @@ struct ReplacementMountIdentity {
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
fn replacement_mount_identity(metadata: &std::fs::Metadata, mount_id: u64) -> ReplacementMountIdentity {
|
||||
use std::os::unix::fs::MetadataExt as _;
|
||||
|
||||
ReplacementMountIdentity {
|
||||
device: metadata.dev(),
|
||||
inode: metadata.ino(),
|
||||
mount_id,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
fn mount_id_for_fd(fd: &std::fs::File) -> Option<u64> {
|
||||
use rustix::fs::{AtFlags, StatxFlags};
|
||||
use std::os::fd::AsRawFd as _;
|
||||
|
||||
rustix::fs::statx(fd, "", AtFlags::EMPTY_PATH, StatxFlags::MNT_ID)
|
||||
.ok()
|
||||
.filter(|statx| StatxFlags::from_bits_retain(statx.stx_mask).contains(StatxFlags::MNT_ID))
|
||||
.map(|statx| statx.stx_mnt_id)
|
||||
.or_else(|| mount_id_from_fdinfo(fd.as_raw_fd()))
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
fn mount_id_for_path(path: &Path) -> Option<u64> {
|
||||
use rustix::fs::{AtFlags, CWD, StatxFlags};
|
||||
|
||||
rustix::fs::statx(CWD, path, AtFlags::empty(), StatxFlags::MNT_ID)
|
||||
.ok()
|
||||
.filter(|statx| StatxFlags::from_bits_retain(statx.stx_mask).contains(StatxFlags::MNT_ID))
|
||||
.map(|statx| statx.stx_mnt_id)
|
||||
.or_else(|| {
|
||||
std::fs::canonicalize(path)
|
||||
.ok()
|
||||
.and_then(|path| mount_id_from_mountinfo(&path))
|
||||
fn statx_replacement_mount_identity(
|
||||
dir: impl std::os::fd::AsFd,
|
||||
path: &Path,
|
||||
flags: rustix::fs::AtFlags,
|
||||
) -> Option<ReplacementMountIdentity> {
|
||||
use rustix::fs::{FileType, StatxFlags, makedev, statx};
|
||||
let fields = StatxFlags::TYPE | StatxFlags::INO | StatxFlags::MNT_ID;
|
||||
let stat = statx(dir, path, flags, fields).ok()?;
|
||||
(StatxFlags::from_bits_retain(stat.stx_mask).contains(fields) && FileType::from_raw_mode(stat.stx_mode.into()).is_dir())
|
||||
.then_some(ReplacementMountIdentity {
|
||||
device: makedev(stat.stx_dev_major, stat.stx_dev_minor),
|
||||
inode: stat.stx_ino,
|
||||
mount_id: stat.stx_mnt_id,
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
fn mount_id_from_fdinfo(fd: std::os::fd::RawFd) -> Option<u64> {
|
||||
std::fs::read_to_string(format!("/proc/self/fdinfo/{fd}"))
|
||||
fn replacement_mount_identity_for_fd(directory: &std::fs::File) -> Option<ReplacementMountIdentity> {
|
||||
statx_replacement_mount_identity(directory, Path::new(""), rustix::fs::AtFlags::EMPTY_PATH)
|
||||
.or_else(|| replacement_mount_identity_from_fdinfo(directory))
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
fn replacement_mount_identity_for_path(path: &Path) -> Option<ReplacementMountIdentity> {
|
||||
use rustix::fs::{AtFlags, CWD, Mode, OFlags, open};
|
||||
statx_replacement_mount_identity(CWD, path, AtFlags::empty()).or_else(|| {
|
||||
let directory = std::fs::File::from(open(path, OFlags::PATH | OFlags::DIRECTORY | OFlags::CLOEXEC, Mode::empty()).ok()?);
|
||||
replacement_mount_identity_from_fdinfo(&directory)
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
fn replacement_mount_identity_from_fdinfo(directory: &std::fs::File) -> Option<ReplacementMountIdentity> {
|
||||
use std::os::{fd::AsRawFd as _, unix::fs::MetadataExt as _};
|
||||
let fd = directory.as_raw_fd();
|
||||
let mount_id = std::fs::read_to_string(format!("/proc/self/fdinfo/{fd}"))
|
||||
.ok()?
|
||||
.lines()
|
||||
.find_map(|line| line.strip_prefix("mnt_id:")?.trim().parse().ok())
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
fn mount_id_from_mountinfo(path: &Path) -> Option<u64> {
|
||||
let mountinfo = std::fs::read_to_string("/proc/self/mountinfo").ok()?;
|
||||
mount_id_from_mountinfo_contents(&mountinfo, path)
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
fn mount_id_from_mountinfo_contents(mountinfo: &str, path: &Path) -> Option<u64> {
|
||||
let mountpoint = path
|
||||
.to_string_lossy()
|
||||
.replace('\\', "\\134")
|
||||
.replace('\t', "\\011")
|
||||
.replace('\n', "\\012")
|
||||
.replace(' ', "\\040");
|
||||
mountinfo.lines().find_map(|line| {
|
||||
let mut fields = line.split_whitespace();
|
||||
let mount_id = fields.next()?.parse().ok()?;
|
||||
fields.next()?;
|
||||
fields.next()?;
|
||||
fields.next()?;
|
||||
(fields.next()? == mountpoint).then_some(mount_id)
|
||||
.find_map(|line| line.strip_prefix("mnt_id:")?.trim().parse().ok())?;
|
||||
let metadata = directory.metadata().ok()?;
|
||||
Some(ReplacementMountIdentity {
|
||||
device: metadata.dev(),
|
||||
inode: metadata.ino(),
|
||||
mount_id,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -8592,26 +8571,55 @@ impl LocalDisk {
|
||||
/// configured endpoint still names the directory held by `mount_lease`.
|
||||
pub fn has_replacement_mount_lease(&self) -> bool {
|
||||
#[cfg(target_os = "linux")]
|
||||
{
|
||||
let Ok(configured) = std::fs::metadata(self.endpoint.get_file_path()) else {
|
||||
return false;
|
||||
};
|
||||
let Ok(pinned) = self.mount_lease.metadata() else {
|
||||
return false;
|
||||
};
|
||||
let Some(configured_mount_id) = mount_id_for_path(Path::new(&self.endpoint.get_file_path())) else {
|
||||
return false;
|
||||
};
|
||||
let Some(pinned_mount_id) = self.mount_lease_mount_id else {
|
||||
return false;
|
||||
};
|
||||
replacement_mount_identity(&configured, configured_mount_id) == replacement_mount_identity(&pinned, pinned_mount_id)
|
||||
}
|
||||
return self
|
||||
.current_replacement_mount_identity(Path::new(&self.endpoint.get_file_path()))
|
||||
.is_some();
|
||||
|
||||
#[cfg(not(target_os = "linux"))]
|
||||
false
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
fn current_replacement_mount_identity(&self, configured_path: &Path) -> Option<ReplacementMountIdentity> {
|
||||
let pinned = self.mount_lease_identity?;
|
||||
(replacement_mount_identity_for_path(configured_path)? == pinned).then_some(pinned)
|
||||
}
|
||||
|
||||
pub(super) fn replacement_mount(&self, _mount_roots: &[PathBuf]) -> Option<super::ReplacementMount> {
|
||||
#[cfg(target_os = "linux")]
|
||||
{
|
||||
let configured_path = self.endpoint.get_file_path();
|
||||
let configured_path = Path::new(&configured_path);
|
||||
let canonical_path = std::fs::canonicalize(&self.io_root).ok()?;
|
||||
let expected = _mount_roots
|
||||
.iter()
|
||||
.filter(|root| root.is_absolute() && (configured_path.starts_with(root) || canonical_path.starts_with(root)))
|
||||
.max_by_key(|root| root.components().count())
|
||||
.unwrap_or(&canonical_path);
|
||||
// The expected root must be its own mount boundary. If it is
|
||||
// unmounted, the exposed directory and its parent share a mount ID.
|
||||
if !canonical_path.starts_with(expected) {
|
||||
return None;
|
||||
}
|
||||
// When the configured endpoint is the expected mount root, this
|
||||
// live observation also supplies the mount ID for the boundary.
|
||||
let identity = self.current_replacement_mount_identity(configured_path)?;
|
||||
if (expected != configured_path && replacement_mount_identity_for_path(expected)?.mount_id != identity.mount_id)
|
||||
|| replacement_mount_identity_for_path(expected.parent()?)?.mount_id == identity.mount_id
|
||||
{
|
||||
return None;
|
||||
}
|
||||
Some(super::ReplacementMount {
|
||||
canonical_path,
|
||||
physical_device_ids: rustfs_utils::os::get_physical_device_ids(self.io_root.to_str()?).ok()?,
|
||||
filesystem_identity: format!("{}:{}:{}", identity.mount_id, identity.device, identity.inode),
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(not(target_os = "linux"))]
|
||||
None
|
||||
}
|
||||
|
||||
/// Return the descriptor-rooted path for the mount instance admitted for
|
||||
/// automatic replacement. Callers must not derive destructive identity
|
||||
/// from the mutable endpoint pathname.
|
||||
@@ -8665,10 +8673,10 @@ impl LocalDisk {
|
||||
PathBuf::from(format!(".rustfs-format-control/{format_device:x}/{}", Uuid::from_bytes(format_file_id)));
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
let (mount_lease, io_root, mount_lease_mount_id) = (
|
||||
let (mount_lease, io_root, mount_lease_identity) = (
|
||||
publication_root.directory().clone(),
|
||||
publication_root.io_path().to_path_buf(),
|
||||
mount_id_for_fd(publication_root.directory()),
|
||||
replacement_mount_identity_for_fd(publication_root.directory()),
|
||||
);
|
||||
#[cfg(not(target_os = "linux"))]
|
||||
let io_root = Self::open_mount_lease(&root)?;
|
||||
@@ -8851,7 +8859,7 @@ impl LocalDisk {
|
||||
#[cfg(target_os = "linux")]
|
||||
mount_lease: mount_lease.clone(),
|
||||
#[cfg(target_os = "linux")]
|
||||
mount_lease_mount_id,
|
||||
mount_lease_identity,
|
||||
endpoint: ep.clone(),
|
||||
format_path,
|
||||
io_format_path,
|
||||
@@ -41477,10 +41485,18 @@ mod test {
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
#[test]
|
||||
fn mountinfo_fallback_uses_the_exact_escaped_mountpoint() {
|
||||
let mountinfo = "101 1 0:30 / / rw - rootfs rootfs rw\n202 101 0:42 / /mnt/replacement\\040disk rw - tmpfs tmpfs rw\n";
|
||||
assert_eq!(mount_id_from_mountinfo_contents(mountinfo, Path::new("/mnt/replacement disk")), Some(202));
|
||||
assert_eq!(mount_id_from_mountinfo_contents(mountinfo, Path::new("/mnt/replacement")), None);
|
||||
fn mount_id_fdinfo_can_identify_a_directory_below_a_mount() {
|
||||
let temp = tempfile::tempdir().expect("mount fixture directory");
|
||||
let child_path = temp.path().join("data");
|
||||
std::fs::create_dir(&child_path).expect("create data subdirectory");
|
||||
let parent = std::fs::File::open(temp.path()).expect("open parent directory");
|
||||
let child = std::fs::File::open(&child_path).expect("open data subdirectory");
|
||||
let parent_identity = replacement_mount_identity_from_fdinfo(&parent).expect("read parent identity");
|
||||
let child_identity = replacement_mount_identity_from_fdinfo(&child).expect("read child identity");
|
||||
assert_eq!(child_identity.mount_id, parent_identity.mount_id);
|
||||
assert_ne!(child_identity.inode, parent_identity.inode);
|
||||
assert_eq!(replacement_mount_identity_for_fd(&child), Some(child_identity));
|
||||
assert_eq!(replacement_mount_identity_for_path(&child_path), Some(child_identity));
|
||||
}
|
||||
|
||||
#[cfg(not(windows))]
|
||||
|
||||
@@ -239,6 +239,14 @@ pub enum PoolMetaNativeRecoveryResponse {
|
||||
|
||||
pub type DiskStore = Arc<Disk>;
|
||||
|
||||
/// Filesystem evidence for a replacement data directory on its expected mount.
|
||||
#[derive(Debug)]
|
||||
pub struct ReplacementMount {
|
||||
pub canonical_path: PathBuf,
|
||||
pub physical_device_ids: Vec<String>,
|
||||
pub filesystem_identity: String,
|
||||
}
|
||||
|
||||
pub trait DiskFileWriter: AsyncWrite + Send + Sync + Unpin + std::any::Any {}
|
||||
impl<T: AsyncWrite + Send + Sync + Unpin + 'static> DiskFileWriter for T {}
|
||||
|
||||
@@ -1583,6 +1591,15 @@ impl Disk {
|
||||
}
|
||||
}
|
||||
|
||||
/// Observe the expected mount through the same local disk handle that will
|
||||
/// execute recovery. Call from a blocking context.
|
||||
pub fn replacement_mount(&self, mount_roots: &[PathBuf]) -> Option<ReplacementMount> {
|
||||
match self {
|
||||
Self::Local(disk) => disk.get_disk().replacement_mount(mount_roots),
|
||||
Self::Remote(_) => None,
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn acquire_replacement_execution_lease(&self) -> Result<std::sync::Arc<local::ReplacementExecutionLease>> {
|
||||
match self {
|
||||
Self::Local(disk) => disk.get_disk().acquire_replacement_execution_lease().await,
|
||||
|
||||
@@ -14,12 +14,9 @@
|
||||
|
||||
use super::{
|
||||
DiskError, DiskStore, HEALING_MARKER_PATH, RUSTFS_META_BUCKET,
|
||||
replacement_readiness::{auto_replacement_target_identities, replacement_target_disk},
|
||||
replacement_readiness::{ReplacementTargets, local_replacement_disks, probe_replacement_mounts},
|
||||
resume::ReplacementTargetIdentity,
|
||||
storage_api::{
|
||||
EcstoreConditionalFileUpdate, EcstoreDiskAPI, EcstoreDiskBytes, EcstoreReplacementExecutionLease,
|
||||
ecstore_local_disk_map_read,
|
||||
},
|
||||
storage_api::{EcstoreConditionalFileUpdate, EcstoreDiskAPI, EcstoreDiskBytes, EcstoreReplacementExecutionLease},
|
||||
};
|
||||
use crate::{Error, Result};
|
||||
use std::{future::Future, sync::Arc};
|
||||
@@ -35,33 +32,27 @@ pub struct ReplacementExecution {
|
||||
|
||||
impl ReplacementExecution {
|
||||
pub(crate) async fn acquire(targets: &[String]) -> Result<Arc<Self>> {
|
||||
let identities = auto_replacement_target_identities(targets)
|
||||
let targets = ReplacementTargets::open(targets)
|
||||
.await
|
||||
.ok_or_else(|| Error::ReplacementTargetNotReady("replacement target is unavailable".to_string()))?;
|
||||
let identities = targets
|
||||
.identities()
|
||||
.await
|
||||
.ok_or_else(|| Error::ReplacementTargetNotReady("target mount admission failed".to_string()))?;
|
||||
let local_disks = ecstore_local_disk_map_read()
|
||||
.await
|
||||
.values()
|
||||
.flatten()
|
||||
.filter(|disk| EcstoreDiskAPI::is_local(disk.as_ref()))
|
||||
.cloned()
|
||||
.collect::<Vec<_>>();
|
||||
let mut disks = Vec::with_capacity(identities.len());
|
||||
let mut leases = Vec::with_capacity(identities.len());
|
||||
// Identities are sorted by endpoint, the same order on every opener.
|
||||
for identity in &identities {
|
||||
let disk = replacement_target_disk(&identity.endpoint, &local_disks)
|
||||
.await
|
||||
.ok_or_else(|| Error::ReplacementTargetNotReady("replacement target is unavailable".to_string()))?;
|
||||
// Take leases in endpoint order. Revalidate those same handles against
|
||||
// the current inventory so a reopened disk cannot supply their proof.
|
||||
for disk in &targets.disks {
|
||||
leases.push(disk.acquire_replacement_execution_lease().await?);
|
||||
disks.push(disk);
|
||||
}
|
||||
if auto_replacement_target_identities(targets).await.as_ref() != Some(&identities) {
|
||||
let local_disks = local_replacement_disks().await;
|
||||
if probe_replacement_mounts(&targets.disks, &local_disks).await.as_ref() != Some(&identities) {
|
||||
return Err(Error::ReplacementTargetNotReady(
|
||||
"target changed while acquiring execution leases".to_string(),
|
||||
));
|
||||
}
|
||||
Ok(Arc::new(Self {
|
||||
disks,
|
||||
disks: targets.disks,
|
||||
identities,
|
||||
_leases: leases,
|
||||
}))
|
||||
|
||||
@@ -12,7 +12,7 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use std::{fs, path::Path};
|
||||
use std::collections::HashSet;
|
||||
|
||||
use super::{
|
||||
DiskOption, DiskStore, Endpoint, HealDiskExt as _, local_disk_map_read, new_disk, resume::ReplacementTargetIdentity,
|
||||
@@ -24,12 +24,15 @@ use super::{
|
||||
/// Directory-backed deployments already declare, through
|
||||
/// `RUSTFS_UNSAFE_BYPASS_DISK_CHECK`, that their endpoints are plain
|
||||
/// directories sharing a device with the host root. Those endpoints can never
|
||||
/// satisfy [`auto_replacement_target_identity`], so without this fallback a
|
||||
/// pass the independent-mount check, so without this fallback a
|
||||
/// runtime-wiped or replaced directory disk would stay deferred forever. The
|
||||
/// admission check itself is never bypassed; the fallback only routes the heal
|
||||
/// through the ordinary format path that formats every unformatted disk in the
|
||||
/// set, which is exactly what the pre-admission `heal_disk` path did.
|
||||
pub(crate) fn directory_backed_replacement_fallback_enabled() -> bool {
|
||||
if cfg!(target_os = "linux") && std::env::var_os(rustfs_config::ENV_HEAL_REPLACEMENT_MOUNT_ROOTS).is_some() {
|
||||
return false;
|
||||
}
|
||||
rustfs_utils::get_env_bool_with_aliases(
|
||||
rustfs_config::ENV_UNSAFE_BYPASS_DISK_CHECK,
|
||||
&[rustfs_config::ENV_MINIO_CI],
|
||||
@@ -38,53 +41,66 @@ pub(crate) fn directory_backed_replacement_fallback_enabled() -> bool {
|
||||
}
|
||||
|
||||
pub(crate) async fn auto_replacement_target_ready(disk: &DiskStore, local_disks: &[DiskStore]) -> bool {
|
||||
auto_replacement_target_identity(disk, local_disks).await.is_some()
|
||||
probe_replacement_mounts(std::slice::from_ref(disk), local_disks)
|
||||
.await
|
||||
.is_some()
|
||||
}
|
||||
|
||||
pub(crate) async fn auto_replacement_target_identity(
|
||||
disk: &DiskStore,
|
||||
pub(super) async fn probe_replacement_mounts(
|
||||
disks: &[DiskStore],
|
||||
local_disks: &[DiskStore],
|
||||
) -> Option<ReplacementTargetIdentity> {
|
||||
let lease_root = disk.replacement_mount_lease_root()?;
|
||||
let endpoint = disk.endpoint().to_string();
|
||||
let sibling_lease_roots = local_disks
|
||||
.iter()
|
||||
.filter(|sibling| sibling.endpoint().is_local && sibling.endpoint().to_string() != endpoint)
|
||||
.map(|sibling| sibling.replacement_mount_lease_root())
|
||||
.collect::<Option<Vec<_>>>()?;
|
||||
) -> Option<Vec<ReplacementTargetIdentity>> {
|
||||
if disks.is_empty() {
|
||||
return Some(Vec::new());
|
||||
}
|
||||
let disks = disks.to_vec();
|
||||
let local_disks = local_disks.to_vec();
|
||||
tokio::task::spawn_blocking(move || {
|
||||
let canonical_path = fs::canonicalize(&lease_root).ok()?;
|
||||
let metadata = fs::metadata(&lease_root).ok()?;
|
||||
let Ok(target_device_ids) = rustfs_utils::os::get_physical_device_ids(lease_root.to_string_lossy().as_ref()) else {
|
||||
return None;
|
||||
};
|
||||
let Ok(root_device_ids) = rustfs_utils::os::get_physical_device_ids("/") else {
|
||||
return None;
|
||||
};
|
||||
if target_device_ids.is_empty()
|
||||
|| root_device_ids.is_empty()
|
||||
|| target_device_ids.iter().any(|target| root_device_ids.contains(target))
|
||||
|| !rustfs_utils::os::is_mount_point(&canonical_path).unwrap_or(false)
|
||||
{
|
||||
return None;
|
||||
let mount_roots = std::env::var_os(rustfs_config::ENV_HEAL_REPLACEMENT_MOUNT_ROOTS)
|
||||
.as_ref()
|
||||
.into_iter()
|
||||
.flat_map(std::env::split_paths)
|
||||
.collect::<Vec<_>>();
|
||||
let mut identities = Vec::with_capacity(disks.len());
|
||||
for disk in &disks {
|
||||
let mount = disk.replacement_mount(&mount_roots)?;
|
||||
if mount.physical_device_ids.is_empty() {
|
||||
return None;
|
||||
}
|
||||
identities.push(ReplacementTargetIdentity {
|
||||
endpoint: disk.endpoint().to_string(),
|
||||
canonical_path: mount.canonical_path.into_os_string().into_string().ok()?,
|
||||
physical_device_ids: mount.physical_device_ids,
|
||||
filesystem_identity: mount.filesystem_identity,
|
||||
});
|
||||
}
|
||||
|
||||
if sibling_lease_roots.iter().any(|sibling_lease_root| {
|
||||
rustfs_utils::os::get_physical_device_ids(sibling_lease_root.to_string_lossy().as_ref())
|
||||
.map(|ids| ids.iter().any(|id| target_device_ids.contains(id)))
|
||||
.unwrap_or(true)
|
||||
}) {
|
||||
let mut target_devices = HashSet::new();
|
||||
for target in &identities {
|
||||
if target
|
||||
.physical_device_ids
|
||||
.iter()
|
||||
.any(|device| !target_devices.insert(device.as_str()))
|
||||
{
|
||||
return None;
|
||||
}
|
||||
}
|
||||
let root_devices = rustfs_utils::os::get_physical_device_ids("/").ok()?;
|
||||
if root_devices.is_empty() || root_devices.iter().any(|device| target_devices.contains(device.as_str())) {
|
||||
return None;
|
||||
}
|
||||
|
||||
let filesystem_identity = filesystem_identity(&metadata, &canonical_path)?;
|
||||
|
||||
Some(ReplacementTargetIdentity {
|
||||
endpoint,
|
||||
canonical_path: canonical_path.to_string_lossy().into_owned(),
|
||||
physical_device_ids: target_device_ids,
|
||||
filesystem_identity,
|
||||
})
|
||||
let targets: HashSet<_> = identities.iter().map(|identity| identity.endpoint.as_str()).collect();
|
||||
for sibling in &local_disks {
|
||||
let endpoint = sibling.endpoint();
|
||||
if endpoint.is_local && !targets.contains(endpoint.to_string().as_str()) {
|
||||
let root = sibling.replacement_mount_lease_root()?;
|
||||
let devices = rustfs_utils::os::get_physical_device_ids(root.to_str()?).ok()?;
|
||||
if devices.iter().any(|device| target_devices.contains(device.as_str())) {
|
||||
return None;
|
||||
}
|
||||
}
|
||||
}
|
||||
Some(identities)
|
||||
})
|
||||
.await
|
||||
.ok()
|
||||
@@ -106,72 +122,76 @@ fn local_replacement_endpoint(target: &str, local_grid_hosts: &[String]) -> Opti
|
||||
Some(endpoint)
|
||||
}
|
||||
|
||||
pub(super) async fn replacement_target_disk(target: &str, local_disks: &[DiskStore]) -> Option<DiskStore> {
|
||||
if let Some(disk) = local_disks.iter().find(|disk| disk.endpoint().to_string() == target) {
|
||||
return Some(disk.clone());
|
||||
}
|
||||
|
||||
let local_grid_hosts = local_disks.iter().map(|disk| disk.endpoint().grid_host()).collect::<Vec<_>>();
|
||||
let endpoint = local_replacement_endpoint(target, &local_grid_hosts)?;
|
||||
|
||||
new_disk(
|
||||
&endpoint,
|
||||
&DiskOption {
|
||||
cleanup: false,
|
||||
health_check: false,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.ok()
|
||||
}
|
||||
|
||||
pub(crate) async fn auto_replacement_target_identities(targets: &[String]) -> Option<Vec<ReplacementTargetIdentity>> {
|
||||
let local_disk_map = local_disk_map_read().await;
|
||||
let local_disks = local_disk_map
|
||||
pub(super) async fn local_replacement_disks() -> Vec<DiskStore> {
|
||||
local_disk_map_read()
|
||||
.await
|
||||
.values()
|
||||
.flatten()
|
||||
.filter(|disk| disk.endpoint().is_local)
|
||||
.cloned()
|
||||
.collect::<Vec<_>>();
|
||||
drop(local_disk_map);
|
||||
.collect()
|
||||
}
|
||||
|
||||
let mut identities = Vec::with_capacity(targets.len());
|
||||
for target in targets {
|
||||
let disk = replacement_target_disk(target, &local_disks).await?;
|
||||
identities.push(auto_replacement_target_identity(&disk, &local_disks).await?);
|
||||
pub(super) struct ReplacementTargets {
|
||||
pub(super) disks: Vec<DiskStore>,
|
||||
local_disks: Vec<DiskStore>,
|
||||
}
|
||||
|
||||
impl ReplacementTargets {
|
||||
pub(super) async fn open(targets: &[String]) -> Option<Self> {
|
||||
let mut ordered: Vec<_> = targets.iter().map(String::as_str).collect();
|
||||
ordered.sort_unstable();
|
||||
if ordered.windows(2).any(|pair| pair[0] == pair[1]) {
|
||||
return None;
|
||||
}
|
||||
|
||||
let registry = local_disk_map_read().await;
|
||||
let requested: Vec<_> = ordered
|
||||
.into_iter()
|
||||
.map(|target| {
|
||||
let disk = registry
|
||||
.get(target)
|
||||
.and_then(Option::as_ref)
|
||||
.filter(|disk| disk.endpoint().is_local)
|
||||
.cloned();
|
||||
(target, disk)
|
||||
})
|
||||
.collect();
|
||||
let local_disks = registry
|
||||
.values()
|
||||
.flatten()
|
||||
.filter(|disk| disk.endpoint().is_local)
|
||||
.cloned()
|
||||
.collect::<Vec<_>>();
|
||||
drop(registry);
|
||||
let mut disks = Vec::with_capacity(requested.len());
|
||||
let mut hosts = None;
|
||||
for (target, disk) in requested {
|
||||
let disk = match disk {
|
||||
Some(disk) => disk,
|
||||
None => {
|
||||
let local_hosts = hosts
|
||||
.get_or_insert_with(|| local_disks.iter().map(|disk| disk.endpoint().grid_host()).collect::<Vec<_>>());
|
||||
let endpoint = local_replacement_endpoint(target, local_hosts)?;
|
||||
new_disk(
|
||||
&endpoint,
|
||||
&DiskOption {
|
||||
cleanup: false,
|
||||
health_check: false,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.ok()?
|
||||
}
|
||||
};
|
||||
disks.push(disk);
|
||||
}
|
||||
Some(Self { disks, local_disks })
|
||||
}
|
||||
identities.sort_by(|left, right| left.endpoint.cmp(&right.endpoint));
|
||||
identities.dedup_by(|left, right| left.endpoint == right.endpoint);
|
||||
(identities.len() == targets.len()).then_some(identities)
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
fn filesystem_identity(metadata: &fs::Metadata, canonical_path: &Path) -> Option<String> {
|
||||
use std::os::unix::fs::MetadataExt as _;
|
||||
|
||||
let escaped_path = canonical_path.to_string_lossy().replace(' ', "\\040");
|
||||
let mountinfo = fs::read_to_string("/proc/self/mountinfo").ok()?;
|
||||
let mount_id = mountinfo.lines().find_map(|line| {
|
||||
let mut fields = line.split_whitespace();
|
||||
let mount_id = fields.next()?;
|
||||
fields.next()?;
|
||||
fields.next()?;
|
||||
fields.next()?;
|
||||
(fields.next()? == escaped_path).then_some(mount_id)
|
||||
})?;
|
||||
Some(format!("{mount_id}:{}:{}", metadata.dev(), metadata.ino()))
|
||||
}
|
||||
|
||||
#[cfg(all(unix, not(target_os = "linux")))]
|
||||
fn filesystem_identity(metadata: &fs::Metadata, _canonical_path: &Path) -> Option<String> {
|
||||
use std::os::unix::fs::MetadataExt as _;
|
||||
|
||||
Some(format!("{}:{}", metadata.dev(), metadata.ino()))
|
||||
}
|
||||
|
||||
#[cfg(not(unix))]
|
||||
fn filesystem_identity(_metadata: &fs::Metadata, _canonical_path: &Path) -> Option<String> {
|
||||
None
|
||||
pub(super) async fn identities(&self) -> Option<Vec<ReplacementTargetIdentity>> {
|
||||
probe_replacement_mounts(&self.disks, &self.local_disks).await
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -269,8 +289,8 @@ mod tests {
|
||||
mod linux_privileged_tests {
|
||||
use super::*;
|
||||
use std::error::Error;
|
||||
use std::path::Path;
|
||||
use std::process::Command;
|
||||
use std::{fs, path::Path};
|
||||
|
||||
const ENABLE_ENV: &str = "RUSTFS_PRIVILEGED_MOUNT_READINESS_TESTS";
|
||||
const NAMESPACE_ENV: &str = "RUSTFS_PRIVILEGED_MOUNT_READINESS_TESTS_IN_NAMESPACE";
|
||||
@@ -396,39 +416,86 @@ mod tests {
|
||||
#[ignore = "requires Linux root/CAP_SYS_ADMIN and RUSTFS_PRIVILEGED_MOUNT_READINESS_TESTS=1"]
|
||||
fn auto_replacement_readiness_accepts_an_independent_mount() -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
run_privileged_mount_test(|mut mounts| async move {
|
||||
let temp = TempDir::new().expect("temporary replacement roots should be created");
|
||||
let target = temp.path().join("target");
|
||||
let temp = TempDir::new()?;
|
||||
let parent = temp.path().join("parent");
|
||||
let target = parent.join("replacement disk");
|
||||
let data = target.join("data");
|
||||
let sibling = temp.path().join("sibling");
|
||||
std::fs::create_dir(&target).expect("target mountpoint should be created");
|
||||
std::fs::create_dir(&sibling).expect("sibling mountpoint should be created");
|
||||
fs::create_dir(&parent)?;
|
||||
mounts.mount_tmpfs(&parent, "rustfs-readiness-parent")?;
|
||||
// Leave a directory at the same path below the target mount.
|
||||
fs::create_dir_all(&data)?;
|
||||
mounts.mount_tmpfs(&target, "rustfs-readiness-target")?;
|
||||
fs::create_dir(&data)?;
|
||||
fs::create_dir(&sibling)?;
|
||||
mounts.mount_tmpfs(&sibling, "rustfs-readiness-sibling")?;
|
||||
|
||||
let target_endpoint = Endpoint::try_from(target.to_string_lossy().as_ref())?;
|
||||
let sibling_endpoint = Endpoint::try_from(sibling.to_string_lossy().as_ref())?;
|
||||
let target_disk = new_disk(
|
||||
&target_endpoint,
|
||||
&DiskOption {
|
||||
cleanup: false,
|
||||
health_check: false,
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
let sibling_disk = new_disk(
|
||||
&sibling_endpoint,
|
||||
&DiskOption {
|
||||
cleanup: false,
|
||||
health_check: false,
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
let options = DiskOption {
|
||||
cleanup: false,
|
||||
health_check: false,
|
||||
};
|
||||
let target_disk = new_disk(&target_endpoint, &options).await?;
|
||||
let sibling_disk = new_disk(&sibling_endpoint, &options).await?;
|
||||
|
||||
let identity = auto_replacement_target_identity(&target_disk, &[target_disk.clone(), sibling_disk.clone()]).await;
|
||||
assert!(
|
||||
identity.is_some(),
|
||||
auto_replacement_target_ready(&target_disk, &[target_disk.clone(), sibling_disk.clone()]).await,
|
||||
"a separately mounted replacement target with no sibling device overlap must be admitted"
|
||||
);
|
||||
Ok(())
|
||||
let disks = [target_disk, sibling_disk];
|
||||
let mut identities = probe_replacement_mounts(&disks, &disks)
|
||||
.await
|
||||
.expect("independently mounted replacement targets should be admitted together");
|
||||
assert_eq!(identities[0].endpoint, disks[0].endpoint().to_string());
|
||||
assert_eq!(identities[1].endpoint, disks[1].endpoint().to_string());
|
||||
identities.sort_unstable_by(|left, right| left.endpoint.cmp(&right.endpoint));
|
||||
let endpoints = [identities[1].endpoint.clone(), identities[0].endpoint.clone()];
|
||||
let execution = crate::heal::replacement_execution::ReplacementExecution::acquire(&endpoints).await?;
|
||||
assert_eq!(execution.identities(), identities);
|
||||
assert!(
|
||||
crate::heal::replacement_execution::ReplacementExecution::acquire(&endpoints)
|
||||
.await
|
||||
.is_err(),
|
||||
"a second executor must not take leases on the same targets"
|
||||
);
|
||||
drop(execution);
|
||||
|
||||
let endpoint = Endpoint::try_from(path_to_string(&data, "data directory")?.as_str())?;
|
||||
let disk = new_disk(&endpoint, &options).await?;
|
||||
let key = rustfs_config::ENV_HEAL_REPLACEMENT_MOUNT_ROOTS;
|
||||
temp_env::async_with_vars([(key, None::<&str>)], async {
|
||||
assert!(!auto_replacement_target_ready(&disk, &[]).await);
|
||||
})
|
||||
.await;
|
||||
|
||||
let roots = std::env::join_paths([&parent, &target])?;
|
||||
temp_env::async_with_vars(
|
||||
[
|
||||
(key, Some(roots.to_str().ok_or("mount roots are not UTF-8")?)),
|
||||
(rustfs_config::ENV_UNSAFE_BYPASS_DISK_CHECK, Some("true")),
|
||||
],
|
||||
async {
|
||||
assert!(!directory_backed_replacement_fallback_enabled());
|
||||
let identities = probe_replacement_mounts(std::slice::from_ref(&disk), &[])
|
||||
.await
|
||||
.expect("data directory on its configured independent mount should be admitted");
|
||||
assert_eq!(Path::new(&identities[0].canonical_path), data.canonicalize()?);
|
||||
let reopened = new_disk(&endpoint, &options).await?;
|
||||
assert_eq!(probe_replacement_mounts(&[reopened], &[]).await, Some(identities));
|
||||
|
||||
run_command("umount", &["-l", &path_to_string(&target, "replacement mount")?])?;
|
||||
assert!(!auto_replacement_target_ready(&disk, &[]).await);
|
||||
|
||||
let exposed_disk = new_disk(&endpoint, &options).await?;
|
||||
assert!(
|
||||
!auto_replacement_target_ready(&exposed_disk, &[]).await,
|
||||
"a new handle on the exposed parent filesystem must also be rejected"
|
||||
);
|
||||
Ok(())
|
||||
},
|
||||
)
|
||||
.await
|
||||
})
|
||||
}
|
||||
|
||||
@@ -467,10 +534,20 @@ mod tests {
|
||||
.await?;
|
||||
|
||||
assert!(
|
||||
auto_replacement_target_identity(&target_disk, &[target_disk.clone(), sibling_disk.clone()])
|
||||
!auto_replacement_target_ready(&target_disk, &[target_disk.clone(), sibling_disk.clone()]).await,
|
||||
"replacement readiness must reject a target sharing its physical device with a sibling endpoint"
|
||||
);
|
||||
assert!(
|
||||
probe_replacement_mounts(&[target_disk.clone(), sibling_disk], &[])
|
||||
.await
|
||||
.is_none(),
|
||||
"replacement readiness must reject a target sharing its physical device with a sibling endpoint"
|
||||
"a batch must reject overlapping targets even before they are registered"
|
||||
);
|
||||
assert!(auto_replacement_target_ready(&target_disk, &[]).await);
|
||||
mounts.mount_bind(&source, &target)?;
|
||||
assert!(
|
||||
!auto_replacement_target_ready(&target_disk, &[]).await,
|
||||
"rebinding the same inode must invalidate the held mount identity"
|
||||
);
|
||||
Ok(())
|
||||
})
|
||||
|
||||
@@ -519,8 +519,14 @@ pub(crate) fn replacement_target_identities_match(
|
||||
expected: &[ReplacementTargetIdentity],
|
||||
actual: &[ReplacementTargetIdentity],
|
||||
) -> bool {
|
||||
let mut expected = expected.to_vec();
|
||||
let mut actual = actual.to_vec();
|
||||
if expected == actual {
|
||||
return true;
|
||||
}
|
||||
if expected.len() != actual.len() {
|
||||
return false;
|
||||
}
|
||||
let mut expected: Vec<_> = expected.iter().collect();
|
||||
let mut actual: Vec<_> = actual.iter().collect();
|
||||
expected.sort_by(|left, right| left.endpoint.cmp(&right.endpoint));
|
||||
actual.sort_by(|left, right| left.endpoint.cmp(&right.endpoint));
|
||||
expected == actual
|
||||
|
||||
@@ -38,7 +38,7 @@ use super::storage_api::storage::{
|
||||
ObjectIO as _, ObjectOperations as _, StorageAdminApi,
|
||||
};
|
||||
use super::{
|
||||
DiskError, DiskStore, ECStore, HealDiskExt as _, StorageError, local_disk_map_read,
|
||||
DiskError, DiskStore, ECStore, HealDiskExt as _, StorageError,
|
||||
resume::{ReplacementRecoveryCandidate, ReplacementTargetIdentity, merge_replacement_recovery_candidate},
|
||||
};
|
||||
pub use super::{HealObjectInfo, HealObjectOptions, HealPutObjReader};
|
||||
@@ -2457,9 +2457,13 @@ impl HealStorageAPI for ECStoreHealStorage {
|
||||
}
|
||||
|
||||
async fn replacement_target_identities(&self, targets: &[String]) -> Result<Vec<ReplacementTargetIdentity>> {
|
||||
super::replacement_readiness::auto_replacement_target_identities(targets)
|
||||
let error = || Error::ReplacementTargetNotReady("replacement target is not a stable mounted disk".to_string());
|
||||
super::replacement_readiness::ReplacementTargets::open(targets)
|
||||
.await
|
||||
.ok_or_else(|| Error::ReplacementTargetNotReady("replacement target is not a stable mounted disk".to_string()))
|
||||
.ok_or_else(error)?
|
||||
.identities()
|
||||
.await
|
||||
.ok_or_else(error)
|
||||
}
|
||||
|
||||
async fn replacement_targets_ready_for_retry(
|
||||
@@ -2468,25 +2472,14 @@ impl HealStorageAPI for ECStoreHealStorage {
|
||||
targets: &[String],
|
||||
expected_identities: &[ReplacementTargetIdentity],
|
||||
) -> Result<bool> {
|
||||
let identities = match self.replacement_target_identities(targets).await {
|
||||
Ok(identities) => identities,
|
||||
Err(_) => return Ok(false),
|
||||
let Some(targets) = super::replacement_readiness::ReplacementTargets::open(targets).await else {
|
||||
return Ok(false);
|
||||
};
|
||||
if identities != expected_identities {
|
||||
if targets.identities().await.as_deref() != Some(expected_identities) {
|
||||
return Ok(false);
|
||||
}
|
||||
|
||||
let local_disks = local_disk_map_read()
|
||||
.await
|
||||
.values()
|
||||
.flatten()
|
||||
.filter(|disk| disk.endpoint().is_local)
|
||||
.cloned()
|
||||
.collect::<Vec<_>>();
|
||||
for target in targets {
|
||||
let Some(disk) = local_disks.iter().find(|disk| disk.endpoint().to_string() == *target) else {
|
||||
return Ok(false);
|
||||
};
|
||||
for disk in targets.disks {
|
||||
let endpoint = disk.endpoint();
|
||||
if crate::heal::utils::format_set_disk_id_from_i32(endpoint.pool_idx, endpoint.set_idx).as_deref()
|
||||
!= Some(set_disk_id)
|
||||
|
||||
@@ -228,10 +228,10 @@ impl HealTask {
|
||||
// first target format write. A task retry keeps this id; a newly
|
||||
// admitted blank replacement gets a fresh id and cannot reuse cursor
|
||||
// progress from an older disk at the same endpoint.
|
||||
let replacement_resume = if is_auto_replacement {
|
||||
let identities = self
|
||||
.await_with_control(self.storage.replacement_target_identities(&self.heal_endpoints))
|
||||
.await?;
|
||||
let replacement_resume = if let Some(execution) = &replacement_execution {
|
||||
self.check_control_flags().await?;
|
||||
// Bind durable intent to the mounts whose execution leases we hold.
|
||||
let identities = execution.identities().to_vec();
|
||||
let disk = replacement_resume_disk.clone().ok_or_else(|| Error::TaskExecutionFailed {
|
||||
message: format!("Replacement resume disk is missing for automatic heal {set_disk_id}"),
|
||||
})?;
|
||||
|
||||
@@ -1258,6 +1258,40 @@ async fn recovered_replacement_never_uses_a_fresh_resume_disk() {
|
||||
assert!(!*storage.listed.lock().unwrap(), "an unavailable durable anchor must not list buckets");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn automatic_replacement_does_not_format_a_mount_outside_its_execution_lease() {
|
||||
let temp = TempDir::new().expect("resume disk directory");
|
||||
let disk = make_resume_disk(&temp).await;
|
||||
let leased = replacement_identity("replacement-a", "device-a", "mount-a");
|
||||
let remounted = replacement_identity("replacement-a", "device-a", "mount-b");
|
||||
let storage = Arc::new(MockStorage {
|
||||
bucket_incarnation_id: Mutex::new(Some(Uuid::from_u128(1))),
|
||||
replacement_target_identities_ready: Mutex::new(true),
|
||||
replacement_target_identity_sequences: Mutex::new(VecDeque::from([vec![remounted.clone()], vec![remounted]])),
|
||||
replacement_execution_fixture: Mutex::new(Some(crate::heal::storage::ReplacementExecution::for_test(
|
||||
Vec::new(),
|
||||
vec![leased.clone()],
|
||||
))),
|
||||
resume_disk: Mutex::new(Some(disk.clone())),
|
||||
..Default::default()
|
||||
});
|
||||
let mut request = directory_backed_replacement_request();
|
||||
request.heal_endpoints = vec!["replacement-a".to_string()];
|
||||
let task = HealTask::from_request(request, storage.clone());
|
||||
|
||||
let error = task.execute().await.expect_err("a remount must be rejected before format");
|
||||
assert!(error.to_string().contains("changed during format"), "{error}");
|
||||
assert!(storage.replacement_format_calls.lock().unwrap().is_empty());
|
||||
|
||||
let state = ResumeManager::load_replacement_intent(disk, &task.id)
|
||||
.await
|
||||
.expect("durable intent should identify the leased target")
|
||||
.get_state()
|
||||
.await;
|
||||
assert_eq!(state.replacement_phase, ReplacementPhase::Intent);
|
||||
assert_eq!(state.replacement_target_identities, vec![leased]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn automatic_replacement_rejects_a_new_identity_after_format() {
|
||||
let temp = TempDir::new().expect("temporary resume disk directory should be created");
|
||||
@@ -1267,11 +1301,7 @@ async fn automatic_replacement_rejects_a_new_identity_after_format() {
|
||||
let storage = Arc::new(MockStorage {
|
||||
bucket_incarnation_id: Mutex::new(Some(Uuid::from_u128(1))),
|
||||
replacement_target_identities_ready: Mutex::new(true),
|
||||
replacement_target_identity_sequences: Mutex::new(VecDeque::from([
|
||||
vec![first_identity.clone()],
|
||||
vec![first_identity.clone()],
|
||||
vec![second_identity],
|
||||
])),
|
||||
replacement_target_identity_sequences: Mutex::new(VecDeque::from([vec![first_identity.clone()], vec![second_identity]])),
|
||||
resume_disk: Mutex::new(Some(disk.clone())),
|
||||
..Default::default()
|
||||
});
|
||||
|
||||
@@ -38,8 +38,6 @@ default = []
|
||||
# event. Requires `RUSTFLAGS="--cfg hyper_unstable_tracing"`.
|
||||
diagnostic-internode-http2-ping = ["dep:hyper", "rustfs-io-metrics/diagnostic-internode-http2-ping"]
|
||||
hotpath = [
|
||||
"hotpath/hotpath",
|
||||
"hotpath/tokio",
|
||||
"rustfs-common/hotpath",
|
||||
"rustfs-config/hotpath",
|
||||
"rustfs-io-metrics/hotpath",
|
||||
@@ -48,7 +46,6 @@ hotpath = [
|
||||
]
|
||||
hotpath-alloc = [
|
||||
"hotpath",
|
||||
"hotpath/hotpath-alloc",
|
||||
"rustfs-common/hotpath-alloc",
|
||||
"rustfs-config/hotpath-alloc",
|
||||
"rustfs-io-metrics/hotpath-alloc",
|
||||
@@ -57,7 +54,6 @@ hotpath-alloc = [
|
||||
]
|
||||
hotpath-cpu = [
|
||||
"hotpath",
|
||||
"hotpath/hotpath-cpu",
|
||||
"rustfs-common/hotpath-cpu",
|
||||
"rustfs-config/hotpath-cpu",
|
||||
"rustfs-io-metrics/hotpath-cpu",
|
||||
@@ -66,7 +62,6 @@ hotpath-cpu = [
|
||||
]
|
||||
|
||||
[dependencies]
|
||||
hotpath.workspace = true
|
||||
rustfs-common.workspace = true
|
||||
rustfs-heal-contracts.workspace = true
|
||||
rustfs-io-metrics.workspace = true
|
||||
|
||||
@@ -59,7 +59,7 @@ hotpath-cpu = [
|
||||
]
|
||||
|
||||
[dependencies]
|
||||
hotpath.workspace = true
|
||||
hotpath = { workspace = true, optional = true }
|
||||
arc-swap.workspace = true
|
||||
tokio = { workspace = true, features = ["io-util", "macros", "net", "rt-multi-thread", "sync", "time"] }
|
||||
rand = { workspace = true, features = ["serde"] }
|
||||
|
||||
@@ -232,7 +232,7 @@ Heal selects the authoritative version from quorum metadata, refuses when more s
|
||||
### 10.2 Drive replacement
|
||||
|
||||
1. Confirm the affected set still meets read quorum for its objects (impossible at `EC:0`, see §5) and record pool index, set index, slot, and the failed disk's UUID from the pool status.
|
||||
2. Replace the drive at the **same** endpoint and mount path with an empty filesystem owned by RustFS. Do not copy the old drive's directories; a copied `format.json` claims an identity the set already tracks.
|
||||
2. Replace the drive at the **same** endpoint and mount path with an empty filesystem owned by RustFS. Do not copy the old drive's directories. A copied `format.json` claims an identity the set already tracks. On Linux, automatic replacement checks that the endpoint is an independent mount by default. When the endpoint is a data directory below the mount, set `RUSTFS_HEAL_REPLACEMENT_MOUNT_ROOTS` on that node to the colon-separated absolute mount paths, for example `/srv/disk1:/srv/disk2` for endpoints `/srv/disk1/rustfs` and `/srv/disk2/rustfs`. Use canonical mount paths without trailing slashes. The most specific matching root must be the active mount that contains the endpoint. The root must remain independent of the system and sibling drives. Setting this variable disables the unsafe directory-backed heal fallback on that node.
|
||||
3. Bring the node back. The empty drive is not a format-quorum member, so `retain_format_quorum_members` keeps its slot offline; the endpoint monitor (`monitor_and_connect_endpoints_task` in `crates/ecstore/src/core/sets.rs`) retries the connection every 15 seconds. The erasure-set heal task (`crates/heal/src/heal/task/heal_erasure_set.rs`) writes the format into the original slot, through `heal_replacement_format` when an automatic replacement intent exists (schema 7 records under `.rustfs.sys/buckets/ahm-replacement/`, created when startup or the disk scanner observes a changed mount identity) or through `heal_format` for an admin erasure-set heal (`POST /rustfs/admin/v3/heal/` with `pool` and `set`), and then rebuilds the shards. Legacy schema 5/6 intents follow [replacement-generation-recovery.md](replacement-generation-recovery.md). If the drive held the node's pool metadata, follow [pool-metadata-recovery.md](pool-metadata-recovery.md#disk-replacement-and-metadata-erasure) first.
|
||||
4. Watch `GET /rustfs/admin/v4/heal/replacement-recovery`, background heal status, disk health, and bitrot counters. Completion criteria are in [scanner-runtime-controls.md](scanner-runtime-controls.md#replacement-recovery-completion): an `idle` queue or a readable object is not proof.
|
||||
5. The replacement is done when recovery reports `completed` for that instance and sampled objects have `xl.meta` and parts on the new drive.
|
||||
|
||||
@@ -66,6 +66,8 @@ An existing deployment with committed, initialized identity and unchanged topolo
|
||||
|
||||
During startup, an unreadable identity or `pool.bin` replica returns typed `ReadUnavailable` before a partial metadata view is installed or the object store is published. Initialization retries from the complete identity and metadata read at the existing five-second retry interval, within the existing startup attempt budget. A later readable result must still pass the original identity, generation, topology, conditional-write and readback checks. Read-only unavailability does not create a permanent startup block; an existing validation or uncertain-write block is never cleared by a clean retry.
|
||||
|
||||
When native recovery reports `no_ordinary_v3_authority`, no ordinary V3 COMMIT or eligible PREPARE proves a generation to restore. `missing_pool_metadata_slot` and `missing_pool_identity_slot` identify absent members of the corresponding erasure sets. `missing_pool_metadata_head` identifies an inspected member without a current metadata shard. `pool_identity_unavailable` means an ordinary identity read could not establish a valid replica. Transport errors retain their original storage error. Investigate the failing stage and preserve the surviving drives before attempting another replacement.
|
||||
|
||||
The node checks for interrupted pool metadata transactions every five seconds. Failed recovery attempts back off to at most sixty seconds; each attempt has a thirty-second budget and stops on shutdown. Healthy nodes do not read metadata for this worker. Recovery:
|
||||
|
||||
1. Cancels old decommission workers and waits for their supervisors to drain them. It does not cancel a separate rebalance operation; an attached rebalance worker prevents recovery until quiescent.
|
||||
|
||||
@@ -1,17 +0,0 @@
|
||||
# S3 stack upgrade and rollback
|
||||
|
||||
The gateway release serves S3 with rustfs/gateway. Remove `RUSTFS_S3_STACK` from service units, container environments and Helm values before upgrading. Even an empty setting prevents startup. Keep the previous release binary or image digest with the deployment configuration.
|
||||
|
||||
## Roll back to the previous release
|
||||
|
||||
1. Check the target release against the [storage format version floors](rolling-restart.md#version-floors-that-break-mixed-version-fleets). If the cluster has written data or metadata that the target cannot read, stop here. The S3 stack switch itself requires no data migration.
|
||||
2. Stop one node, restore the previous RustFS binary or image and start it with the same volumes and configuration. Follow the [rolling restart procedure](rolling-restart.md#rolling-restart-procedure). Wait for the node to report ready and for its peers to report its drives online before replacing another node.
|
||||
3. Through the restored node, read an object written after the upgrade and compare its contents or checksum. Read a bucket configuration used by the application. If an upload was in progress, list its parts and compare their numbers, sizes and ETags with the client's record. Pause the rollout if any check fails.
|
||||
|
||||
## Client-visible error change
|
||||
|
||||
| Conformance case | What clients see |
|
||||
| --- | --- |
|
||||
| `c-sig-0598` (R-053) | For an access key the S3 parser cannot read, both versions return HTTP 403. The gateway returns the XML error code `AccessDenied`. The legacy binary returns `InvalidAccessKeyId`. Clients that classify these errors by code need to handle both during a rollout. |
|
||||
|
||||
The case is recorded in the [pinned gateway conformance rulings](https://github.com/rustfs/gateway/blob/60dcd2a55713dbcc9dbd6e165922e1a0df4473d7/conformance/rulings/rustfs-profile.toml#L298-L304).
|
||||
@@ -332,7 +332,7 @@ fn is_lock_timeout(err: &crate::storage_api::error::StorageError) -> bool {
|
||||
fn decommission_start_api_error(err: crate::storage_api::error::StorageError) -> ApiError {
|
||||
if crate::storage_api::capacity::is_pool_activation_fleet_proof_error(&err) {
|
||||
return ApiError {
|
||||
code: S3ErrorCode::InternalError,
|
||||
code: S3ErrorCode::InternalError.into(),
|
||||
message: POOL_ACTIVATION_FLEET_PROOF_REQUIRED.to_string(),
|
||||
source: Some(Box::new(err)),
|
||||
};
|
||||
@@ -341,7 +341,7 @@ fn decommission_start_api_error(err: crate::storage_api::error::StorageError) ->
|
||||
// movement gate has a queued writer: the same transient contention.
|
||||
if is_lock_timeout(&err) || matches!(err, crate::storage_api::error::StorageError::SlowDown) {
|
||||
return ApiError {
|
||||
code: S3ErrorCode::ServiceUnavailable,
|
||||
code: S3ErrorCode::ServiceUnavailable.into(),
|
||||
message: err.to_string(),
|
||||
source: Some(Box::new(err)),
|
||||
};
|
||||
@@ -350,7 +350,7 @@ fn decommission_start_api_error(err: crate::storage_api::error::StorageError) ->
|
||||
// should say so, not just "Invalid Request".
|
||||
if matches!(err, crate::storage_api::error::StorageError::DecommissionAlreadyRunning) {
|
||||
return ApiError {
|
||||
code: S3ErrorCode::InvalidRequest,
|
||||
code: S3ErrorCode::InvalidRequest.into(),
|
||||
message: err.to_string(),
|
||||
source: Some(Box::new(err)),
|
||||
};
|
||||
@@ -1261,14 +1261,14 @@ mod pools_handler_tests {
|
||||
#[test]
|
||||
fn test_contextualize_admin_pool_api_error_preserves_code_and_adds_pool_context() {
|
||||
let err = crate::error::ApiError {
|
||||
code: rustfs_s3_types::S3ErrorCode::InvalidRequest,
|
||||
code: rustfs_s3_types::S3ErrorCode::InvalidRequest.into(),
|
||||
message: "decommission already running".to_string(),
|
||||
source: None,
|
||||
};
|
||||
|
||||
let err = contextualize_admin_pool_api_error(err, "start decommission", "pools [1, 3]");
|
||||
|
||||
assert_eq!(err.code, rustfs_s3_types::S3ErrorCode::InvalidRequest);
|
||||
assert_eq!(err.code, rustfs_s3_types::S3ErrorCode::InvalidRequest.into());
|
||||
assert_eq!(
|
||||
err.message,
|
||||
"admin start decommission failed for pools [1, 3]: decommission already running"
|
||||
@@ -1281,12 +1281,12 @@ mod pools_handler_tests {
|
||||
|
||||
let err = decommission_start_api_error(err);
|
||||
|
||||
assert_eq!(err.code, rustfs_s3_types::S3ErrorCode::InternalError);
|
||||
assert_eq!(err.code, rustfs_s3_types::S3ErrorCode::InternalError.into());
|
||||
assert_eq!(err.message, POOL_ACTIVATION_FLEET_PROOF_REQUIRED);
|
||||
assert!(err.source.is_some());
|
||||
|
||||
let unrelated = decommission_start_api_error(crate::storage_api::error::StorageError::other("disk read failed"));
|
||||
assert_eq!(unrelated.code, rustfs_s3_types::S3ErrorCode::InternalError);
|
||||
assert_eq!(unrelated.code, rustfs_s3_types::S3ErrorCode::InternalError.into());
|
||||
assert_eq!(unrelated.message, "We encountered an internal error, please try again.");
|
||||
}
|
||||
|
||||
@@ -1299,7 +1299,7 @@ mod pools_handler_tests {
|
||||
|
||||
let err = decommission_start_api_error(err);
|
||||
|
||||
assert_eq!(err.code, S3ErrorCode::ServiceUnavailable);
|
||||
assert_eq!(err.code, S3ErrorCode::ServiceUnavailable.into());
|
||||
assert!(err.message.contains("data movement gate (decommission activation)"), "{}", err.message);
|
||||
let err = rustfs_s3_types::S3Error::from(contextualize_admin_pool_api_error(err, "start decommission", "pools [0]"));
|
||||
assert_eq!(
|
||||
@@ -1313,7 +1313,7 @@ mod pools_handler_tests {
|
||||
fn test_decommission_start_api_error_reports_a_refused_preflight_heal_as_retryable() {
|
||||
let err = decommission_start_api_error(crate::storage_api::error::StorageError::SlowDown);
|
||||
|
||||
assert_eq!(err.code, S3ErrorCode::ServiceUnavailable);
|
||||
assert_eq!(err.code, S3ErrorCode::ServiceUnavailable.into());
|
||||
let err = rustfs_s3_types::S3Error::from(contextualize_admin_pool_api_error(err, "start decommission", "pools [0]"));
|
||||
assert_eq!(err.status_code(), Some(http::StatusCode::SERVICE_UNAVAILABLE));
|
||||
}
|
||||
|
||||
@@ -37,7 +37,7 @@ use crate::server::collect_dependency_readiness_report as collect_runtime_depend
|
||||
use crate::shared_types::DependencyReadiness;
|
||||
use rustfs_data_usage::DataUsageInfo;
|
||||
use rustfs_madmin::{InfoMessage, StorageInfo};
|
||||
use rustfs_s3_types::S3ErrorCode;
|
||||
use s3s::S3ErrorCode;
|
||||
use std::collections::HashSet;
|
||||
use std::sync::Arc;
|
||||
use tracing::{debug, error, info, warn};
|
||||
|
||||
@@ -98,13 +98,14 @@ use rustfs_policy::policy::{
|
||||
{BucketPolicy, BucketPolicyArgs, Effect},
|
||||
};
|
||||
use rustfs_s3_ops::S3Operation;
|
||||
use rustfs_s3_types::{Region, S3Error, S3ErrorCode, S3Result};
|
||||
use rustfs_s3_types::Region;
|
||||
use rustfs_targets::{
|
||||
EventName,
|
||||
arn::{ARN, TargetID, TargetIDError},
|
||||
};
|
||||
use rustfs_utils::http::get_header;
|
||||
use rustfs_utils::obj::extract_user_defined_metadata;
|
||||
use s3s::{S3Error, S3ErrorCode, S3Result};
|
||||
use std::{
|
||||
borrow::Cow,
|
||||
collections::{HashMap, HashSet},
|
||||
@@ -906,9 +907,7 @@ mod tests {
|
||||
.expect_err("full queue rejects immediately");
|
||||
assert_eq!(error.code(), &S3ErrorCode::SlowDown);
|
||||
assert_eq!(error.headers().expect("retry headers")[http::header::RETRY_AFTER], "1");
|
||||
let response = s3s::S3Error::from(error)
|
||||
.to_http_response()
|
||||
.expect("serialize admission error");
|
||||
let response = error.to_http_response().expect("serialize admission error");
|
||||
assert_eq!(response.status(), http::StatusCode::SERVICE_UNAVAILABLE);
|
||||
assert_eq!(response.headers()[http::header::RETRY_AFTER], "1");
|
||||
use http_body_util::BodyExt as _;
|
||||
|
||||
@@ -116,7 +116,7 @@ impl DefaultBucketUsecase {
|
||||
|
||||
async fn list_objects_page(&self, envelope: &mut RequestEnvelope, request: &PageRequest) -> S3Result<Page> {
|
||||
let PageRequest { bucket, params, .. } = request;
|
||||
validate_list_object_unordered_with_delimiter(params.delimiter.as_deref(), envelope.uri().query())?;
|
||||
validate_list_object_unordered_with_delimiter(params.delimiter.as_ref(), envelope.uri().query())?;
|
||||
let store = get_validated_store(bucket).await?;
|
||||
let incl_deleted = get_header(envelope.headers(), rustfs_utils::http::SUFFIX_INCLUDE_DELETED).as_deref() == Some("true");
|
||||
let table_list_access = envelope.table_list_access().cloned();
|
||||
|
||||
@@ -626,7 +626,7 @@ impl DefaultBucketUsecase {
|
||||
|
||||
let params = parse_list_objects_v2_params(prefix, delimiter, max_keys, continuation_token, start_after)?;
|
||||
|
||||
validate_list_object_unordered_with_delimiter(params.delimiter.as_deref(), envelope.uri().query())?;
|
||||
validate_list_object_unordered_with_delimiter(params.delimiter.as_ref(), envelope.uri().query())?;
|
||||
|
||||
let store = get_validated_store(&bucket).await?;
|
||||
let incl_deleted = get_header(envelope.headers(), rustfs_utils::http::SUFFIX_INCLUDE_DELETED)
|
||||
|
||||
@@ -103,7 +103,6 @@ use futures::StreamExt;
|
||||
use http::{HeaderMap, Uri};
|
||||
use metrics::counter;
|
||||
use rustfs_s3_ops::S3Operation;
|
||||
use rustfs_s3_types::{S3Error, S3ErrorCode, S3Result, s3_error};
|
||||
use rustfs_targets::EventName;
|
||||
use rustfs_utils::CompressionAlgorithm;
|
||||
#[cfg(test)]
|
||||
@@ -115,6 +114,7 @@ use rustfs_utils::http::{
|
||||
headers::{AMZ_CHECKSUM_TYPE, AMZ_OBJECT_TAGGING, AMZ_STORAGE_CLASS},
|
||||
insert_str,
|
||||
};
|
||||
use s3s::{S3Error, S3ErrorCode, S3Result, s3_error};
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
@@ -192,13 +192,12 @@ fn multipart_max_total_object_size(metadata: &HashMap<String, String>) -> S3Resu
|
||||
let value = get_consistent_str(metadata, SUFFIX_MAX_TOTAL_OBJECT_SIZE).ok_or_else(|| {
|
||||
S3Error::with_message(
|
||||
S3ErrorCode::InvalidRequest,
|
||||
"multipart size capability metadata is missing or inconsistent",
|
||||
"multipart size capability metadata is missing or inconsistent".to_string(),
|
||||
)
|
||||
})?;
|
||||
value
|
||||
.parse::<u64>()
|
||||
.map(Some)
|
||||
.map_err(|_| S3Error::with_message(S3ErrorCode::InvalidRequest, "multipart size capability metadata is invalid"))
|
||||
value.parse::<u64>().map(Some).map_err(|_| {
|
||||
S3Error::with_message(S3ErrorCode::InvalidRequest, "multipart size capability metadata is invalid".to_string())
|
||||
})
|
||||
}
|
||||
|
||||
async fn validate_table_catalog_object_mutation(bucket: &str, key: &str) -> S3Result<()> {
|
||||
@@ -291,11 +290,7 @@ impl DefaultMultipartUsecase {
|
||||
/// or completed".
|
||||
fn multipart_upload_api_error(err: StorageError) -> S3Error {
|
||||
if matches!(err, StorageError::MalformedUploadID(_)) {
|
||||
let mut error = S3Error::new(S3ErrorCode::NoSuchUpload);
|
||||
if let Some(message) = error.code().default_message() {
|
||||
error.set_message(message);
|
||||
}
|
||||
return error;
|
||||
return S3Error::new(S3ErrorCode::NoSuchUpload);
|
||||
}
|
||||
ApiError::from(err).into()
|
||||
}
|
||||
@@ -549,7 +544,7 @@ mod tests {
|
||||
.execute_create_multipart_upload_from_s3s(build_request(create, Method::POST))
|
||||
.await
|
||||
.expect_err("reserved key must be rejected before accessing storage");
|
||||
assert_eq!(error.code(), &crate::app::storage_api::s3::S3ErrorCode::InvalidArgument);
|
||||
assert_eq!(error.code(), &S3ErrorCode::InvalidArgument);
|
||||
|
||||
let complete = CompleteMultipartUploadInput::builder()
|
||||
.bucket("bucket".to_owned())
|
||||
@@ -597,9 +592,6 @@ mod tests {
|
||||
fn multipart_upload_api_error_reports_malformed_upload_id_as_no_such_upload() {
|
||||
let malformed = multipart_upload_api_error(StorageError::MalformedUploadID("bucket/object-not-an-id".to_string()));
|
||||
assert_eq!(malformed.code(), &S3ErrorCode::NoSuchUpload);
|
||||
let legacy = crate::app::storage_api::s3::S3Error::new(crate::app::storage_api::s3::S3ErrorCode::NoSuchUpload);
|
||||
assert_eq!(malformed.message(), legacy.message());
|
||||
assert_eq!(malformed.status_code(), legacy.status_code());
|
||||
|
||||
let unknown = multipart_upload_api_error(StorageError::InvalidUploadID(
|
||||
"bucket".to_string(),
|
||||
|
||||
@@ -79,7 +79,7 @@ where
|
||||
Ok::<_, StorageError>(page.recorded_owner.take().map(|owner| RecordedListParts { owner, page }))
|
||||
})
|
||||
.await
|
||||
.map_err(multipart_upload_api_error)?
|
||||
.map_err(|error| S3Error::from(multipart_upload_api_error(error)))?
|
||||
.map_err(|refusal| S3Error::with_message(S3ErrorCode::NoSuchUpload, refusal.reason()))?;
|
||||
Ok(record.page)
|
||||
}
|
||||
@@ -198,7 +198,7 @@ impl DefaultMultipartUsecase {
|
||||
ListPartsTarget::Legacy { upload_id, .. } => store
|
||||
.list_object_parts(bucket, key, upload_id, params.part_number_marker, params.max_parts, &opts)
|
||||
.await
|
||||
.map_err(multipart_upload_api_error),
|
||||
.map_err(|error| multipart_upload_api_error(error).into()),
|
||||
ListPartsTarget::Gateway {
|
||||
bucket: bucket_name,
|
||||
key: object_key,
|
||||
|
||||
@@ -165,7 +165,7 @@ fn list_parts_resolves_real_owners_and_retains_storage_errors() {
|
||||
storage_error,
|
||||
StorageError::InvalidUploadID(..) | StorageError::MalformedUploadID(..)
|
||||
));
|
||||
let expected: LegacyS3Error = ApiError::from(storage_error).into();
|
||||
let expected = multipart_upload_api_error(storage_error);
|
||||
for target in [
|
||||
gateway_fields(&bucket, key, id, None, None),
|
||||
ListPartsFields {
|
||||
|
||||
@@ -43,7 +43,7 @@ impl RecordedUpload for PartUpload {
|
||||
|
||||
async fn lookup_part_upload(store: &ECStore, bucket: &str, key: &str, id: &str, opts: &ObjectOptions) -> S3Result<MultipartInfo> {
|
||||
let result = store.get_multipart_info(bucket, key, id, opts).await;
|
||||
result.map_err(multipart_upload_api_error)
|
||||
result.map_err(|error| multipart_upload_api_error(error).into())
|
||||
}
|
||||
|
||||
pub(super) async fn lookup_recorded_part_upload(
|
||||
@@ -56,7 +56,7 @@ pub(super) async fn lookup_recorded_part_upload(
|
||||
let result = store.lookup_multipart_part_binding(bucket, key, id, opts).await;
|
||||
#[cfg(test)]
|
||||
let result = tests::observe_lookup(result);
|
||||
match result.map_err(multipart_upload_api_error) {
|
||||
match result.map_err(|error| S3Error::from(multipart_upload_api_error(error))) {
|
||||
Ok((info, owner)) => Ok(owner.map(|owner| PartUpload { owner, info })),
|
||||
Err(error) if error.code() == &S3ErrorCode::NoSuchUpload => Ok(None),
|
||||
Err(error) => Err(error),
|
||||
|
||||
@@ -233,8 +233,7 @@ fn native_copy_part_rejects_unproved_owner_without_changing_parts() {
|
||||
} else if matches!(mode, LookupResult::Outage) {
|
||||
let Err(error) = result else { panic!("expected read quorum failure") };
|
||||
let actual: crate::app::storage_api::s3::S3Error = error.into();
|
||||
let expected: crate::app::storage_api::s3::S3Error =
|
||||
ApiError::from(StorageError::ErasureReadQuorum).into();
|
||||
let expected = multipart_upload_api_error(StorageError::ErasureReadQuorum);
|
||||
assert_eq!(actual.code(), expected.code());
|
||||
assert_eq!(actual.status_code(), expected.status_code());
|
||||
assert_eq!(actual.message(), expected.message());
|
||||
|
||||
@@ -1051,8 +1051,8 @@ mod tests {
|
||||
.expect_err("short copy source must fail before destination write succeeds");
|
||||
let api_error = ApiError::from(err);
|
||||
|
||||
assert_eq!(api_error.code, rustfs_s3_types::S3ErrorCode::ServiceUnavailable);
|
||||
assert_ne!(api_error.code, rustfs_s3_types::S3ErrorCode::IncompleteBody);
|
||||
assert_eq!(api_error.code, S3ErrorCode::ServiceUnavailable);
|
||||
assert_ne!(api_error.code, S3ErrorCode::IncompleteBody);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -503,7 +503,7 @@ fn project_delete_objects_pre_stat_error(
|
||||
error: ApiError,
|
||||
) -> S3Result<DeleteBatchError> {
|
||||
// Bucket loss invalidates the shared request, not just one object.
|
||||
if error.code == rustfs_s3_types::S3ErrorCode::NoSuchBucket {
|
||||
if error.code == S3ErrorCode::NoSuchBucket {
|
||||
return Err(error.into());
|
||||
}
|
||||
Ok(DeleteBatchError {
|
||||
@@ -676,9 +676,10 @@ impl DefaultObjectUsecase {
|
||||
let key = normalize_delete_object_key(&obj_id.key);
|
||||
// A rejected wire key has no authorization resource. Its normalized text cannot become a target.
|
||||
let key_error = if obj_id.key_too_long {
|
||||
let code = rustfs_s3_types::S3ErrorCode::InvalidArgument;
|
||||
let message = ApiError::error_code_to_message(&code);
|
||||
Some((code, message))
|
||||
Some((
|
||||
S3ErrorCode::InvalidArgument,
|
||||
ApiError::error_code_to_message(&S3ErrorCode::InvalidArgument),
|
||||
))
|
||||
} else {
|
||||
validate_object_key(&key, "DeleteObjects")
|
||||
.err()
|
||||
|
||||
@@ -1250,7 +1250,7 @@ pub fn normalize_extract_entry_key(path: &str, prefix: Option<&str>, is_dir: boo
|
||||
|
||||
fn map_extract_archive_error(err: std::io::Error) -> S3Error {
|
||||
let api_error = ApiError::from(err);
|
||||
if api_error.code != rustfs_s3_types::S3ErrorCode::InternalError {
|
||||
if api_error.code != S3ErrorCode::InternalError {
|
||||
return api_error.into();
|
||||
}
|
||||
|
||||
@@ -1323,7 +1323,7 @@ fn is_empty_extract_entry_path(path: &str) -> bool {
|
||||
|
||||
fn validate_extract_member_key(path: &str, limits: ArchiveLimits) -> Result<(), ExtractEntryError> {
|
||||
validate_put_object_extract_entry_path(path, limits).map_err(ExtractEntryError::Recoverable)?;
|
||||
validate_new_object_key(path, "PUT").map_err(|err| ExtractEntryError::Recoverable(err.into()))
|
||||
validate_new_object_key(path, "PUT").map_err(ExtractEntryError::Recoverable)
|
||||
}
|
||||
|
||||
fn record_extract_pax_metadata_bytes(
|
||||
@@ -3666,7 +3666,7 @@ mod tests {
|
||||
] {
|
||||
let error = map_extract_archive_error(error);
|
||||
assert_eq!(error.code(), &code);
|
||||
assert_eq!(error.message(), Some(ApiError::error_code_to_message(&code.into()).as_str()));
|
||||
assert_eq!(error.message(), Some(ApiError::error_code_to_message(&code).as_str()));
|
||||
assert!(error.source().is_some());
|
||||
}
|
||||
let error = map_extract_archive_error(io::Error::new(io::ErrorKind::InvalidData, "invalid TAR header"));
|
||||
|
||||
@@ -2602,9 +2602,8 @@ impl DefaultObjectUsecase {
|
||||
};
|
||||
return Err(with_delete_marker_read_headers(S3Error::new(code), info, input.version_id.as_deref()));
|
||||
}
|
||||
let result = check_preconditions(envelope.headers(), info)
|
||||
.map_err(S3Error::from)
|
||||
.and_then(|()| Self::validate_get_object_part_number(part_number, info));
|
||||
let result =
|
||||
check_preconditions(envelope.headers(), info).and_then(|()| Self::validate_get_object_part_number(part_number, info));
|
||||
if result.is_err() {
|
||||
let principal = SseKmsPrincipal::from_request(envelope);
|
||||
crate::storage::sse::validate_conditional_read(
|
||||
@@ -12314,7 +12313,7 @@ mod tests {
|
||||
fn parse_get_object_part_number_rejects_above_s3_max() {
|
||||
let err = parse_part_number_i32_to_usize(Some(10001), "GET").expect_err("partNumber above S3 max must fail");
|
||||
|
||||
assert_eq!(err.code(), &rustfs_s3_types::S3ErrorCode::InvalidArgument);
|
||||
assert_eq!(err.code(), &S3ErrorCode::InvalidArgument);
|
||||
assert_eq!(err.message(), Some("GET: partNumber must be between 1 and 10000"));
|
||||
}
|
||||
|
||||
|
||||
@@ -22,7 +22,7 @@
|
||||
//! multipart handlers' policy steps against the same storage contract.
|
||||
|
||||
use super::*;
|
||||
use rustfs_s3_types::{S3Error, S3ErrorCode};
|
||||
use crate::app::storage_api::s3::{S3Error, S3ErrorCode};
|
||||
|
||||
use crate::app::object_data_cache::invalidate_object_data_cache_after_complete_multipart_success;
|
||||
use crate::app::runtime_sources::current_app_context;
|
||||
@@ -80,8 +80,7 @@ pub(crate) struct InternalPutContext {
|
||||
|
||||
const INTERNAL_PUT_METHOD_NAME: &str = "PUT";
|
||||
|
||||
fn api_error_from_s3(err: impl Into<S3Error>) -> ApiError {
|
||||
let err = err.into();
|
||||
fn api_error_from_s3(err: S3Error) -> ApiError {
|
||||
ApiError {
|
||||
code: err.code().clone(),
|
||||
message: err.message().unwrap_or_default().to_string(),
|
||||
@@ -140,7 +139,7 @@ async fn validate_internal_write_target(key: &str, bucket: &str, headers: &Heade
|
||||
headers.get("content-type").and_then(|value| value.to_str().ok()),
|
||||
headers.get("content-encoding").and_then(|value| value.to_str().ok()),
|
||||
)
|
||||
.map_err(api_error_from_s3)
|
||||
.map_err(|err| api_error_from_s3(err.into()))
|
||||
}
|
||||
|
||||
fn internal_events_wanted() -> bool {
|
||||
@@ -334,7 +333,7 @@ impl DefaultObjectUsecase {
|
||||
.put_object_core(write, super::native_body::from_io_stream(body), None, start_time)
|
||||
.await
|
||||
.map_err(|error| ApiError {
|
||||
code: error.code().clone(),
|
||||
code: error.code().clone().into(),
|
||||
message: error.message().unwrap_or_default().to_owned(),
|
||||
source: None,
|
||||
})?;
|
||||
|
||||
@@ -32,7 +32,7 @@
|
||||
//! `odm-source-etag`.
|
||||
|
||||
use super::*;
|
||||
use rustfs_s3_types::S3ErrorCode;
|
||||
use crate::app::storage_api::s3::S3ErrorCode;
|
||||
|
||||
use crate::app::storage_api::multipart_usecase::contract::multipart::CompletePart;
|
||||
use crate::on_demand_migration::{
|
||||
|
||||
@@ -150,7 +150,7 @@ pub(crate) fn reject_oversize_single_upload(size: i64) -> S3Result<()> {
|
||||
if u64::try_from(size).is_ok_and(|size| size > rustfs_config::MAX_SINGLE_PUT_OBJECT_SIZE) {
|
||||
return Err(S3Error::with_message(
|
||||
S3ErrorCode::EntityTooLarge,
|
||||
ApiError::error_code_to_message(&rustfs_s3_types::S3ErrorCode::EntityTooLarge),
|
||||
ApiError::error_code_to_message(&S3ErrorCode::EntityTooLarge),
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
@@ -5669,7 +5669,7 @@ mod oversize_single_upload_tests {
|
||||
assert_eq!(*err.code(), S3ErrorCode::EntityTooLarge);
|
||||
assert_eq!(
|
||||
err.message(),
|
||||
Some(ApiError::error_code_to_message(&rustfs_s3_types::S3ErrorCode::EntityTooLarge).as_str())
|
||||
Some(ApiError::error_code_to_message(&S3ErrorCode::EntityTooLarge).as_str())
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -13,11 +13,10 @@
|
||||
// limitations under the License.
|
||||
|
||||
use super::*;
|
||||
use crate::app::storage_api::s3::{Body as S3Body, S3Error, StreamingBlob};
|
||||
use crate::app::storage_api::s3::{Body as S3Body, S3Error, S3ErrorCode, StreamingBlob};
|
||||
use crate::error::ApiError;
|
||||
use futures::{StreamExt, poll};
|
||||
use http_body_util::StreamBody;
|
||||
use rustfs_s3_types::S3ErrorCode;
|
||||
use std::io;
|
||||
use tokio::io::AsyncReadExt;
|
||||
use tokio::sync::mpsc;
|
||||
|
||||
@@ -1136,7 +1136,7 @@ pub(crate) fn odm_check_source_preconditions(headers: &HeaderMap, head: &SourceH
|
||||
user_defined: metadata.into(),
|
||||
..Default::default()
|
||||
};
|
||||
Ok(check_preconditions(headers, &info)?)
|
||||
check_preconditions(headers, &info)
|
||||
}
|
||||
|
||||
/// S3 documents write-side `If-None-Match` only as `*` (create-only).
|
||||
|
||||
@@ -69,7 +69,7 @@ impl ObjectMetaUsecase {
|
||||
.get_bucket_info(bucket, &BucketOptions::default())
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
return Err(err);
|
||||
return Err(err.into());
|
||||
}
|
||||
|
||||
let opts: ObjectOptions = get_opts(bucket, key, version_id.clone(), None, envelope.headers())
|
||||
|
||||
@@ -4,8 +4,8 @@ use super::storage_api::select_object::options::get_opts;
|
||||
use super::storage_api::select_object::request_context::spawn_traced;
|
||||
use super::storage_api::select_object::sse::{SseKmsPrincipal, authorize_sse_kms_object_read, project_sse_read_response_headers};
|
||||
use super::storage_api::select_object::{
|
||||
StoragePrepareSelectObjectSnapshotError, StorageSelectObjectSnapshot, get_validated_store, validate_sse_headers_for_read,
|
||||
validate_ssec_for_read,
|
||||
StoragePrepareSelectObjectSnapshotError, StorageSelectObjectSnapshot, get_validated_store_owned,
|
||||
validate_sse_headers_for_read, validate_ssec_for_read,
|
||||
};
|
||||
use crate::app::runtime_sources::current_s3select_db;
|
||||
use crate::app::{Answered, RequestEnvelope};
|
||||
@@ -874,7 +874,7 @@ async fn prepare_select_object_snapshot(
|
||||
let opts = get_opts(&input.bucket, &input.key, None, None, headers)
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
let store = get_validated_store(&input.bucket).await?;
|
||||
let store = get_validated_store_owned(&input.bucket).await?;
|
||||
let snapshot = store
|
||||
.prepare_select_object_snapshot(&input.bucket, &input.key, headers, &opts)
|
||||
.await
|
||||
|
||||
@@ -1900,8 +1900,8 @@ pub(crate) mod select_object {
|
||||
#[cfg(test)]
|
||||
pub(crate) use crate::storage::storage_api::StorageError;
|
||||
pub(crate) use crate::storage::storage_api::{
|
||||
StoragePrepareSelectObjectSnapshotError, StorageSelectObjectSnapshot, get_validated_store, validate_sse_headers_for_read,
|
||||
validate_ssec_for_read,
|
||||
StoragePrepareSelectObjectSnapshotError, StorageSelectObjectSnapshot, get_validated_store_owned,
|
||||
validate_sse_headers_for_read, validate_ssec_for_read,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
+30
-54
@@ -17,9 +17,8 @@ use crate::storage_api::error::{PoolMetadataError, QuotaError, StorageError, unr
|
||||
use http::StatusCode;
|
||||
use rustfs_kms::KmsUnavailableError;
|
||||
use rustfs_lock::LockError;
|
||||
use rustfs_s3_types::S3ErrorCode;
|
||||
use s3s::S3Error as LegacyS3Error;
|
||||
use s3s::stream::{aws_chunked_stream::AwsChunkedStreamError, upload_stream::UploadStreamError};
|
||||
use s3s::{S3Error, S3ErrorCode};
|
||||
|
||||
const CONTENT_SHA256_MISMATCH_CODE: &str = "XAmzContentSHA256Mismatch";
|
||||
const CONTENT_SHA256_MISMATCH_MESSAGE: &str = "The provided 'x-amz-content-sha256' header does not match what was computed.";
|
||||
@@ -348,7 +347,7 @@ impl ApiErrorDiagnostic {
|
||||
let mut diagnostic = ApiErrorDiagnostic::default();
|
||||
// S3Error retains its payload through an inherent accessor; its
|
||||
// std::error::Error implementation does not expose that source.
|
||||
let mut current = if let Some(error) = error.downcast_ref::<LegacyS3Error>() {
|
||||
let mut current = if let Some(error) = error.downcast_ref::<S3Error>() {
|
||||
error.source().map(|source| source as &(dyn std::error::Error + 'static))
|
||||
} else {
|
||||
std::error::Error::source(error)
|
||||
@@ -385,7 +384,7 @@ impl ApiErrorDiagnostic {
|
||||
diagnostic.io_os_error = io.raw_os_error();
|
||||
// io::Error::source can skip the wrapped error itself.
|
||||
io.get_ref().map(|source| source as &(dyn std::error::Error + 'static))
|
||||
} else if let Some(error) = error.downcast_ref::<LegacyS3Error>() {
|
||||
} else if let Some(error) = error.downcast_ref::<S3Error>() {
|
||||
error.source().map(|source| source as &(dyn std::error::Error + 'static))
|
||||
} else {
|
||||
error.source()
|
||||
@@ -617,14 +616,14 @@ fn body_read_error_code(err: &(dyn std::error::Error + 'static)) -> Option<S3Err
|
||||
if let Some(chunked) = error.downcast_ref::<AwsChunkedStreamError>()
|
||||
&& !matches!(chunked, AwsChunkedStreamError::Underlying(_))
|
||||
{
|
||||
aws_chunked_error = Some(chunked.to_s3_error_code().into());
|
||||
aws_chunked_error = Some(chunked.to_s3_error_code());
|
||||
}
|
||||
incomplete_body |= error.is::<rustfs_rio::IncompleteBody>()
|
||||
|| matches!(upload_error, Some(UploadStreamError::LengthMismatch | UploadStreamError::Incomplete))
|
||||
|| is_hyper_body_eof(error);
|
||||
None
|
||||
});
|
||||
digest.or({
|
||||
digest.or_else(|| {
|
||||
if upload_limit_exceeded {
|
||||
Some(S3ErrorCode::EntityTooLarge)
|
||||
} else if server_source_failed {
|
||||
@@ -712,7 +711,7 @@ impl From<ApiError> for rustfs_s3_types::S3Error {
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let mut s3e = rustfs_s3_types::S3Error::with_message(err.code, err.message);
|
||||
let mut s3e = rustfs_s3_types::S3Error::with_message(err.code.into(), err.message);
|
||||
if let Some(cause) = cause {
|
||||
let mut headers = http::HeaderMap::new();
|
||||
// s3s replaces the generated headers when a custom map is supplied.
|
||||
@@ -737,7 +736,7 @@ impl From<ApiError> for rustfs_s3_types::S3Error {
|
||||
/// with `From`, which yields field for field what the s3s constructor builds
|
||||
/// (code, message, explicit status, the cause itself), so both error types
|
||||
/// come from one mapping.
|
||||
impl From<ApiError> for LegacyS3Error {
|
||||
impl From<ApiError> for S3Error {
|
||||
fn from(err: ApiError) -> Self {
|
||||
rustfs_s3_types::S3Error::from(err).into()
|
||||
}
|
||||
@@ -965,9 +964,8 @@ mod tests {
|
||||
use metrics_util::debugging::{DebugValue, DebuggingRecorder};
|
||||
use rustfs_io_metrics::s3_http_metrics::S3HttpRequestGuard;
|
||||
use rustfs_s3_ops::S3Operation;
|
||||
use rustfs_s3_types::S3ErrorCode;
|
||||
use s3s::stream::{aws_chunked_stream::AwsChunkedStreamError, upload_stream::UploadStreamError};
|
||||
use s3s::{S3Error, S3ErrorCode as LegacyS3ErrorCode};
|
||||
use s3s::{S3Error, S3ErrorCode};
|
||||
use std::io::{Error as IoError, ErrorKind};
|
||||
|
||||
fn error_chain_has_type<T>(err: &(dyn std::error::Error + 'static)) -> bool
|
||||
@@ -984,7 +982,7 @@ mod tests {
|
||||
let storage_error =
|
||||
StorageError::Lock(LockError::timeout("private-bucket/private-key@latest", std::time::Duration::from_secs(5)));
|
||||
let error = S3Error::from(ApiError::from(storage_error));
|
||||
assert_eq!(error.code(), &LegacyS3ErrorCode::ServiceUnavailable);
|
||||
assert_eq!(error.code(), &S3ErrorCode::ServiceUnavailable);
|
||||
let source = error.source().expect("retain the typed storage cause");
|
||||
assert!(source.downcast_ref::<StorageError>().is_some());
|
||||
assert!(error_chain_has_type::<LockError>(source));
|
||||
@@ -1010,7 +1008,7 @@ mod tests {
|
||||
use http_body_util::BodyExt as _;
|
||||
|
||||
let error = S3Error::from(ApiError::from(StorageError::UnsupportedDisk));
|
||||
assert_eq!(error.code(), &LegacyS3ErrorCode::ServiceUnavailable);
|
||||
assert_eq!(error.code(), &S3ErrorCode::ServiceUnavailable);
|
||||
assert_eq!(error.message(), Some(UNSUPPORTED_PARTICIPANT_MESSAGE));
|
||||
assert!(matches!(
|
||||
error.source().and_then(|source| source.downcast_ref::<StorageError>()),
|
||||
@@ -1051,7 +1049,7 @@ mod tests {
|
||||
] {
|
||||
let expected_code = failure.code();
|
||||
let error = S3Error::from(ApiError::from(failure));
|
||||
assert_eq!(error.code(), &LegacyS3ErrorCode::ServiceUnavailable);
|
||||
assert_eq!(error.code(), &S3ErrorCode::ServiceUnavailable);
|
||||
assert_eq!(error.message(), Some("The service is unavailable. Please retry."));
|
||||
let source = error
|
||||
.source()
|
||||
@@ -1084,7 +1082,7 @@ mod tests {
|
||||
code: code.clone(),
|
||||
source: Some(Box::new(source)),
|
||||
});
|
||||
assert_eq!(error.code(), &code.clone().into());
|
||||
assert_eq!(error.code(), &code);
|
||||
let source = error.source().expect("preserve wrapped lock timeout");
|
||||
assert!(error_chain_has_type::<LockError>(source));
|
||||
let response = error.to_http_response().expect("serialize wrapped timeout");
|
||||
@@ -1128,7 +1126,7 @@ mod tests {
|
||||
] {
|
||||
let expected_code = failure.code();
|
||||
let error = S3Error::from(ApiError::from(failure));
|
||||
assert_eq!(error.code(), &LegacyS3ErrorCode::ServiceUnavailable);
|
||||
assert_eq!(error.code(), &S3ErrorCode::ServiceUnavailable);
|
||||
assert_eq!(error.message(), Some("The service is unavailable. Please retry."));
|
||||
let source = error
|
||||
.source()
|
||||
@@ -1346,7 +1344,7 @@ mod tests {
|
||||
assert_eq!(diagnostic.lock_reason, Some(expected));
|
||||
assert!(!format!("{diagnostic:?}").contains("private"));
|
||||
let s3: S3Error = api.into();
|
||||
assert_eq!(s3.code(), &LegacyS3ErrorCode::ServiceUnavailable);
|
||||
assert_eq!(s3.code(), &S3ErrorCode::ServiceUnavailable);
|
||||
assert_eq!(super::ApiErrorDiagnostic::from_error(&s3).lock_reason, Some(expected));
|
||||
}
|
||||
|
||||
@@ -1636,7 +1634,7 @@ mod tests {
|
||||
AwsChunkedStreamError::TrailersTooLarge(16385, 16384),
|
||||
AwsChunkedStreamError::TooManyTrailerHeaders(101, 100),
|
||||
] {
|
||||
let expected: S3ErrorCode = error.to_s3_error_code().into();
|
||||
let expected = error.to_s3_error_code();
|
||||
let expected_status = if expected == S3ErrorCode::SignatureDoesNotMatch {
|
||||
StatusCode::FORBIDDEN
|
||||
} else {
|
||||
@@ -2368,28 +2366,6 @@ mod tests {
|
||||
assert!(source.to_string().contains(detail), "the detail must survive on the source");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn custom_error_keeps_its_identity_and_cause_across_both_edges() {
|
||||
let api = ApiError {
|
||||
code: S3ErrorCode::Custom("InvalidRequest".into()),
|
||||
message: "custom refusal".into(),
|
||||
source: Some(Box::new(IoError::other("original cause"))),
|
||||
};
|
||||
let owned = rustfs_s3_types::S3Error::from(api);
|
||||
assert!(matches!(owned.code(), S3ErrorCode::Custom(name) if name == "InvalidRequest"));
|
||||
assert_eq!(owned.status_code(), None);
|
||||
assert_eq!(owned.message(), Some("custom refusal"));
|
||||
assert!(owned.headers().is_none());
|
||||
assert!(owned.source().is_some_and(|source| source.is::<IoError>()));
|
||||
|
||||
let legacy = S3Error::from(owned);
|
||||
assert!(matches!(legacy.code(), LegacyS3ErrorCode::Custom(name) if name == "InvalidRequest"));
|
||||
assert_eq!(legacy.status_code(), None);
|
||||
assert_eq!(legacy.message(), Some("custom refusal"));
|
||||
assert!(legacy.headers().is_none());
|
||||
assert!(legacy.source().is_some_and(|source| source.is::<IoError>()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_generated_error_codes_keep_their_own_status() {
|
||||
let s3_error = S3Error::from(ApiError::from(StorageError::other(rustfs_kms::KmsError::backend_error("down"))));
|
||||
@@ -2421,7 +2397,7 @@ mod tests {
|
||||
headers.insert("retry-after", http::HeaderValue::from_static("2"));
|
||||
err.set_headers(headers);
|
||||
|
||||
assert_eq!(err.code(), &LegacyS3ErrorCode::ServiceUnavailable);
|
||||
assert_eq!(err.code(), &S3ErrorCode::ServiceUnavailable);
|
||||
assert_eq!(err.status_code(), Some(StatusCode::SERVICE_UNAVAILABLE));
|
||||
assert_eq!(err.message(), Some("The service is unavailable. Please retry."));
|
||||
assert_eq!(err.request_id(), Some("put-quorum-request"));
|
||||
@@ -2452,16 +2428,16 @@ mod tests {
|
||||
#[test]
|
||||
fn put_read_quorum_error_preserves_other_error_contracts() {
|
||||
for code in [
|
||||
LegacyS3ErrorCode::SlowDown,
|
||||
LegacyS3ErrorCode::AccessDenied,
|
||||
LegacyS3ErrorCode::InvalidRequest,
|
||||
LegacyS3ErrorCode::PreconditionFailed,
|
||||
LegacyS3ErrorCode::BadDigest,
|
||||
LegacyS3ErrorCode::IncompleteBody,
|
||||
LegacyS3ErrorCode::NoSuchKey,
|
||||
LegacyS3ErrorCode::NoSuchBucket,
|
||||
LegacyS3ErrorCode::InternalError,
|
||||
LegacyS3ErrorCode::ServiceUnavailable,
|
||||
S3ErrorCode::SlowDown,
|
||||
S3ErrorCode::AccessDenied,
|
||||
S3ErrorCode::InvalidRequest,
|
||||
S3ErrorCode::PreconditionFailed,
|
||||
S3ErrorCode::BadDigest,
|
||||
S3ErrorCode::IncompleteBody,
|
||||
S3ErrorCode::NoSuchKey,
|
||||
S3ErrorCode::NoSuchBucket,
|
||||
S3ErrorCode::InternalError,
|
||||
S3ErrorCode::ServiceUnavailable,
|
||||
] {
|
||||
let mut err = S3Error::with_message(code.clone(), "original message");
|
||||
err.set_source(Box::new(StorageError::ErasureReadQuorum));
|
||||
@@ -2717,7 +2693,7 @@ mod tests {
|
||||
};
|
||||
|
||||
let s3_error: S3Error = api_error.into();
|
||||
assert_eq!(*s3_error.code(), LegacyS3ErrorCode::NoSuchBucket);
|
||||
assert_eq!(*s3_error.code(), S3ErrorCode::NoSuchBucket);
|
||||
assert!(s3_error.message().unwrap_or("").contains("Bucket not found"));
|
||||
assert!(s3_error.source().is_some());
|
||||
}
|
||||
@@ -2735,7 +2711,7 @@ mod tests {
|
||||
|
||||
let s3_error: S3Error = api_error.into();
|
||||
|
||||
assert_eq!(*s3_error.code(), LegacyS3ErrorCode::ServiceUnavailable);
|
||||
assert_eq!(*s3_error.code(), S3ErrorCode::ServiceUnavailable);
|
||||
assert_eq!(s3_error.status_code(), Some(http::StatusCode::SERVICE_UNAVAILABLE));
|
||||
}
|
||||
|
||||
@@ -2772,7 +2748,7 @@ mod tests {
|
||||
assert_eq!(api_error.message, MAX_VERSIONS_EXCEEDED_MESSAGE);
|
||||
|
||||
let s3_error: S3Error = api_error.into();
|
||||
assert_eq!(s3_error.code(), &LegacyS3ErrorCode::Custom(MAX_VERSIONS_EXCEEDED_CODE.into()));
|
||||
assert_eq!(s3_error.code(), &S3ErrorCode::Custom(MAX_VERSIONS_EXCEEDED_CODE.into()));
|
||||
assert_eq!(s3_error.message(), Some(MAX_VERSIONS_EXCEEDED_MESSAGE));
|
||||
assert_eq!(s3_error.status_code(), Some(StatusCode::BAD_REQUEST));
|
||||
}
|
||||
@@ -2786,7 +2762,7 @@ mod tests {
|
||||
};
|
||||
|
||||
let s3_error: S3Error = api_error.into();
|
||||
assert_eq!(*s3_error.code(), LegacyS3ErrorCode::InvalidArgument);
|
||||
assert_eq!(*s3_error.code(), S3ErrorCode::InvalidArgument);
|
||||
assert!(s3_error.message().unwrap_or("").contains("Invalid argument"));
|
||||
}
|
||||
|
||||
|
||||
@@ -1169,7 +1169,18 @@ impl rustfs_protocols::common::client::s3::StorageBackend for ProtocolStorageCli
|
||||
|
||||
let bucket = input.bucket.as_str().to_owned();
|
||||
let key = input.key.as_str().to_owned();
|
||||
let (uri, input) = multipart::prepare_upload_part_copy(input)?;
|
||||
let upload_id = gateway_s3s_compat::leaf::upload_id_to_s3s(&input.upload_id);
|
||||
let input = multipart::upload_part_copy_input(input)?;
|
||||
let part_number = input.part_number;
|
||||
let part_number = part_number.to_string();
|
||||
let uri = build_object_uri(
|
||||
&bucket,
|
||||
&key,
|
||||
&[
|
||||
("partNumber", Some(part_number.as_str())),
|
||||
("uploadId", Some(upload_id.as_str())),
|
||||
],
|
||||
)?;
|
||||
|
||||
let envelope = Self::create_envelope(
|
||||
Method::PUT,
|
||||
@@ -1506,7 +1517,6 @@ mod tests {
|
||||
};
|
||||
use http_body_util::BodyExt;
|
||||
use rustfs_protocols::common::client::s3::StorageBackend;
|
||||
use rustfs_s3_types::S3ErrorCode;
|
||||
let root = rustfs_credentials::Credentials {
|
||||
access_key: "protocol-oracle-root".into(),
|
||||
secret_key: "protocol-oracle-secret".into(),
|
||||
@@ -1518,16 +1528,9 @@ mod tests {
|
||||
let slot = ServerContextSlot::new();
|
||||
assert!(slot.install(context));
|
||||
let backend = ProtocolStorageClient::new(slot);
|
||||
for (bucket, key, data, versioned, deleted_get_code) in [
|
||||
(
|
||||
"protocol-version-oracle",
|
||||
"versioned-key",
|
||||
b"versioned data".as_slice(),
|
||||
true,
|
||||
S3ErrorCode::NoSuchKey,
|
||||
),
|
||||
// RustFS 1.0.5 returns NoSuchVersion for a missing directory marker.
|
||||
("protocol-marker-oracle", "folder/", b"".as_slice(), false, S3ErrorCode::NoSuchVersion),
|
||||
for (bucket, key, data, versioned) in [
|
||||
("protocol-version-oracle", "versioned-key", b"versioned data".as_slice(), true),
|
||||
("protocol-marker-oracle", "folder/", b"".as_slice(), false),
|
||||
] {
|
||||
store
|
||||
.make_bucket(
|
||||
@@ -1590,7 +1593,7 @@ mod tests {
|
||||
.get_object(bucket, key, &root, None)
|
||||
.await
|
||||
.expect_err("protocol GET deleted object");
|
||||
assert_eq!(current.code(), &deleted_get_code, "GET {bucket}/{key} after DELETE");
|
||||
assert_eq!(current.code(), &rustfs_s3_types::S3ErrorCode::NoSuchKey);
|
||||
assert_eq!(current.status_code(), Some(http::StatusCode::NOT_FOUND));
|
||||
if versioned {
|
||||
let headers = current.headers().expect("delete marker error headers");
|
||||
|
||||
@@ -293,9 +293,9 @@ pub(super) fn upload_part_error(error: S3Error) -> S3Error {
|
||||
error
|
||||
}
|
||||
|
||||
pub(super) fn prepare_upload_part_copy(
|
||||
pub(super) fn upload_part_copy_input(
|
||||
input: dto::UploadPartCopyInput,
|
||||
) -> S3Result<(http::Uri, crate::app::multipart_usecase::UploadPartCopyWriteInput)> {
|
||||
) -> S3Result<crate::app::multipart_usecase::UploadPartCopyWriteInput> {
|
||||
super::owned::validate_copy_source(&input.copy_source, "UploadPartCopy copy_source")?;
|
||||
let copy_source_if_match =
|
||||
super::owned::copy_condition(input.copy_source_if_match.as_deref(), "copy_source_if_match", "UploadPartCopy input")?;
|
||||
@@ -314,18 +314,11 @@ pub(super) fn prepare_upload_part_copy(
|
||||
.copy_source_if_unmodified_since
|
||||
.map(|at| multipart_timestamp(at, "UploadPartCopy input", "copy_source_if_unmodified_since"))
|
||||
.transpose()?;
|
||||
let upload_id = super::gateway_s3s_compat::leaf::upload_id_to_s3s(&input.upload_id);
|
||||
let part_number = input.part_number.to_string();
|
||||
let uri = super::build_object_uri(
|
||||
input.bucket.as_str(),
|
||||
input.key.as_str(),
|
||||
&[("partNumber", Some(&part_number)), ("uploadId", Some(&upload_id))],
|
||||
)?;
|
||||
let write = crate::app::multipart_usecase::UploadPartCopyWriteInput {
|
||||
Ok(crate::app::multipart_usecase::UploadPartCopyWriteInput {
|
||||
target: crate::app::multipart_usecase::UploadPartTarget::Legacy {
|
||||
bucket: input.bucket.as_str().to_owned(),
|
||||
key: input.key.as_str().to_owned(),
|
||||
upload_id,
|
||||
upload_id: super::gateway_s3s_compat::leaf::upload_id_to_s3s(&input.upload_id),
|
||||
},
|
||||
part_number: input.part_number,
|
||||
copy_source: input.copy_source,
|
||||
@@ -340,8 +333,7 @@ pub(super) fn prepare_upload_part_copy(
|
||||
.copy_source_sse_customer_key
|
||||
.map(|value| value.expose_secret().to_owned()),
|
||||
copy_source_sse_customer_key_md5: nonempty(input.copy_source_sse_customer_key_md5),
|
||||
};
|
||||
Ok((uri, write))
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) fn upload_part_copy_output(
|
||||
|
||||
@@ -422,10 +422,9 @@ fn protocol_upload_copy_keeps_destination_generation_fence() {
|
||||
)
|
||||
.await
|
||||
.expect("replacement bytes");
|
||||
let (_, input) = multipart::prepare_upload_part_copy(request).expect("owned input");
|
||||
let result = backend
|
||||
.owned_multipart_usecase()
|
||||
.execute_upload_part_copy_with_audit(envelope, input)
|
||||
.execute_upload_part_copy_with_audit(envelope, multipart::upload_part_copy_input(request).expect("owned input"))
|
||||
.await;
|
||||
let mut reader = store
|
||||
.get_object_reader(destination, "copy-target", None, HeaderMap::new(), &options)
|
||||
@@ -510,12 +509,10 @@ fn protocol_upload_copy_input_matches_original_codec() {
|
||||
gateway_s3s_compat::generated::ops::upload_part_copy::input_to_s3s(make(), source)
|
||||
.map_err(|error| conversion_error("UploadPartCopy input", error))
|
||||
});
|
||||
let actual = multipart::prepare_upload_part_copy(make());
|
||||
let actual = multipart::upload_part_copy_input(make());
|
||||
match (actual, expected) {
|
||||
(Err(actual), Err(expected)) => assert_eq!(error_facts(actual), error_facts(expected)),
|
||||
(Ok((uri, actual)), Ok(expected)) => {
|
||||
assert_eq!(uri.path(), "/bucket/copy-target");
|
||||
assert_eq!(uri.query(), Some("partNumber=1&uploadId=upload%20%3F%26"));
|
||||
(Ok(actual), Ok(expected)) => {
|
||||
let crate::app::multipart_usecase::UploadPartTarget::Legacy { bucket, key, upload_id } = actual.target else {
|
||||
panic!("protocol client must use the legacy upload target")
|
||||
};
|
||||
@@ -729,8 +726,7 @@ async fn protocol_upload_copy_capabilities_precede_part_validation() {
|
||||
.err()
|
||||
.map(S3Error::from)
|
||||
.expect("legacy refusal");
|
||||
let (_, input) = multipart::prepare_upload_part_copy(make()).expect("owned input");
|
||||
let actual = owned_copy_future(&usecase, envelope, input)
|
||||
let actual = owned_copy_future(&usecase, envelope, multipart::upload_part_copy_input(make()).expect("owned input"))
|
||||
.await
|
||||
.err()
|
||||
.expect("owned refusal");
|
||||
|
||||
+44
-49
@@ -1840,11 +1840,18 @@ fn control_body_digest_error<R: From<Bytes>>(code: &'static str) -> Response<R>
|
||||
response
|
||||
}
|
||||
|
||||
fn control_body_checksum_headers() -> impl Iterator<Item = (&'static str, rustfs_rio::ChecksumType)> {
|
||||
rustfs_rio::BASE_CHECKSUM_TYPES
|
||||
.iter()
|
||||
.filter_map(|&kind| kind.key().map(|name| (name, kind)))
|
||||
}
|
||||
const CONTROL_BODY_CHECKSUM_HEADERS: [&str; 10] = [
|
||||
"x-amz-checksum-crc32",
|
||||
"x-amz-checksum-crc32c",
|
||||
"x-amz-checksum-crc64nvme",
|
||||
"x-amz-checksum-sha1",
|
||||
"x-amz-checksum-sha256",
|
||||
"x-amz-checksum-sha512",
|
||||
"x-amz-checksum-xxhash64",
|
||||
"x-amz-checksum-xxhash3",
|
||||
"x-amz-checksum-xxhash128",
|
||||
"x-amz-checksum-md5",
|
||||
];
|
||||
|
||||
fn verify_control_body_digest(headers: &HeaderMap, bytes: &[u8]) -> std::result::Result<(), &'static str> {
|
||||
use md5::Digest;
|
||||
@@ -1862,16 +1869,37 @@ fn verify_control_body_digest(headers: &HeaderMap, bytes: &[u8]) -> std::result:
|
||||
return Err("BadDigest");
|
||||
}
|
||||
}
|
||||
for (name, kind) in control_body_checksum_headers() {
|
||||
let mut values = headers.get_all(name).iter();
|
||||
let Some(expected) = values.next() else {
|
||||
continue;
|
||||
};
|
||||
if values.next().is_some() {
|
||||
return Err("BadDigest");
|
||||
}
|
||||
let computed = rustfs_rio::Checksum::new_from_data(kind, bytes).ok_or("BadDigest")?;
|
||||
if expected.as_bytes() != computed.encoded.as_bytes() {
|
||||
let mut checksums = s3s::checksum::ChecksumHasher {
|
||||
crc32: headers.contains_key("x-amz-checksum-crc32").then(Default::default),
|
||||
crc32c: headers.contains_key("x-amz-checksum-crc32c").then(Default::default),
|
||||
crc64nvme: headers.contains_key("x-amz-checksum-crc64nvme").then(Default::default),
|
||||
sha1: headers.contains_key("x-amz-checksum-sha1").then(Default::default),
|
||||
sha256: headers.contains_key("x-amz-checksum-sha256").then(Default::default),
|
||||
sha512: headers.contains_key("x-amz-checksum-sha512").then(Default::default),
|
||||
xxhash64: headers.contains_key("x-amz-checksum-xxhash64").then(Default::default),
|
||||
xxhash3: headers.contains_key("x-amz-checksum-xxhash3").then(Default::default),
|
||||
xxhash128: headers.contains_key("x-amz-checksum-xxhash128").then(Default::default),
|
||||
md5: headers.contains_key("x-amz-checksum-md5").then(Default::default),
|
||||
};
|
||||
checksums.update(bytes);
|
||||
let checksums = checksums.finalize();
|
||||
let computed = [
|
||||
checksums.checksum_crc32,
|
||||
checksums.checksum_crc32c,
|
||||
checksums.checksum_crc64nvme,
|
||||
checksums.checksum_sha1,
|
||||
checksums.checksum_sha256,
|
||||
checksums.checksum_sha512,
|
||||
checksums.checksum_xxhash64,
|
||||
checksums.checksum_xxhash3,
|
||||
checksums.checksum_xxhash128,
|
||||
checksums.checksum_md5,
|
||||
];
|
||||
for (name, computed) in CONTROL_BODY_CHECKSUM_HEADERS.into_iter().zip(computed) {
|
||||
if let Some(computed) = computed
|
||||
&& (headers.get_all(name).iter().count() != 1
|
||||
|| headers.get(name).map(HeaderValue::as_bytes) != Some(computed.as_bytes()))
|
||||
{
|
||||
return Err("BadDigest");
|
||||
}
|
||||
}
|
||||
@@ -1920,7 +1948,7 @@ where
|
||||
.is_some_and(|value| value.as_bytes().starts_with(b"AWS "))
|
||||
|| url::form_urlencoded::parse(request.uri().query().unwrap_or_default().as_bytes())
|
||||
.any(|(key, _)| key == "AWSAccessKeyId");
|
||||
let has_checksum = control_body_checksum_headers().any(|(name, _)| headers.contains_key(name));
|
||||
let has_checksum = CONTROL_BODY_CHECKSUM_HEADERS.iter().any(|name| headers.contains_key(*name));
|
||||
let has_md5 = headers.contains_key("content-md5");
|
||||
if required && !has_md5 && (sig_v2 || !has_checksum) {
|
||||
return Ok(control_body_digest_error("MissingContentMD5"));
|
||||
@@ -4184,39 +4212,6 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn control_body_checksums_match_wire_values_for_all_algorithms() {
|
||||
let body = b"123456789";
|
||||
let vectors = [
|
||||
("x-amz-checksum-crc32", "y/Q5Jg=="),
|
||||
("x-amz-checksum-crc32c", "4waSgw=="),
|
||||
("x-amz-checksum-crc64nvme", "rosUhgp5mIg="),
|
||||
("x-amz-checksum-sha1", "98O8HYCOBHMq32eZZczDTKeuNEE="),
|
||||
("x-amz-checksum-sha256", "FeKw08M4keuw8e9gnsQZQgwg4yDOlMZfvIwzEkSOsiU="),
|
||||
(
|
||||
"x-amz-checksum-sha512",
|
||||
"2eZ2LdHI6vbWGzxhkvxAjU1tXxF20MKRabwk5xw/J0rSf81YEbMT1oH35V7ALXPUmclUVba1u1A6z1dPuo/+hQ==",
|
||||
),
|
||||
("x-amz-checksum-xxhash64", "jLhB20DmroM="),
|
||||
("x-amz-checksum-xxhash3", "ctyxi2ehff8="),
|
||||
("x-amz-checksum-xxhash128", "MxGUd+3l3NXpcWQnaB1YYA=="),
|
||||
("x-amz-checksum-md5", "JfnnlDI7RTiF9RgfG2JNCw=="),
|
||||
];
|
||||
let mut headers = HeaderMap::new();
|
||||
for (name, value) in vectors {
|
||||
headers.insert(name, HeaderValue::from_static(value));
|
||||
}
|
||||
assert_eq!(verify_control_body_digest(&headers, body), Ok(()));
|
||||
for (name, value) in vectors {
|
||||
headers.insert(name, HeaderValue::from_static("mismatch"));
|
||||
assert_eq!(verify_control_body_digest(&headers, body), Err("BadDigest"), "{name}");
|
||||
headers.insert(name, HeaderValue::from_static(value));
|
||||
headers.append(name, HeaderValue::from_static(value));
|
||||
assert_eq!(verify_control_body_digest(&headers, body), Err("BadDigest"), "duplicate {name}");
|
||||
headers.insert(name, HeaderValue::from_static(value));
|
||||
}
|
||||
}
|
||||
|
||||
fn digest_v4_request(checksum: &str) -> HttpRequest<s3s::Body> {
|
||||
use s3s_sigv4::{AmzDate, Payload, calculate_signature, create_canonical_request, create_string_to_sign};
|
||||
let timestamp = chrono::Utc::now().format("%Y%m%dT%H%M%SZ").to_string();
|
||||
|
||||
@@ -28,13 +28,13 @@ use super::{SerializedS3Error, xml_escape};
|
||||
use crate::error::ApiError;
|
||||
use crate::server::layer::is_sts_query_request;
|
||||
use crate::storage_api::server::layer::request_context::RequestContext;
|
||||
use crate::storage_api::server::legacy_compat::S3ErrorCode;
|
||||
use bytes::{Bytes, BytesMut};
|
||||
use futures::future::Either;
|
||||
use http::{HeaderValue, Request as HttpRequest, Response, StatusCode};
|
||||
use http_body::Body;
|
||||
use http_body_util::{BodyExt, Full};
|
||||
use quick_xml::events::Event;
|
||||
use rustfs_s3_types::S3ErrorCode;
|
||||
use rustfs_utils::http::headers::AMZ_REQUEST_ID;
|
||||
use std::borrow::Cow;
|
||||
use std::future::Future;
|
||||
|
||||
@@ -26,7 +26,7 @@ mod tests {
|
||||
] {
|
||||
let error = crate::storage::s3_support::validate_object_key(key, "PUT")
|
||||
.expect_err("client keys must not alias internal directory markers");
|
||||
assert_eq!(error.code(), &S3ErrorCode::InvalidArgument, "{key}");
|
||||
assert_eq!(error.code(), &s3s::S3ErrorCode::InvalidArgument, "{key}");
|
||||
}
|
||||
for key in ["file", "folder/", "prefix/folder/", "__xldir__", "percent%20name"] {
|
||||
crate::storage::s3_support::validate_object_key(key, "PUT")
|
||||
@@ -37,20 +37,18 @@ mod tests {
|
||||
use crate::config::WorkloadProfile;
|
||||
use crate::server::cors;
|
||||
use crate::storage::StorageError;
|
||||
use crate::storage::request_markers::{RequestedObjectLegalHold, RequestedObjectLockRetention};
|
||||
use crate::storage::s3_api::common::{rustfs_initiator, rustfs_owner};
|
||||
use crate::storage::s3_support::parse_object_version_id;
|
||||
use crate::storage::s3_support::{apply_bucket_default_lock_retention, map_bucket_object_lock_config_state};
|
||||
use crate::storage::s3_support::{
|
||||
parse_object_lock_legal_hold_native, parse_object_lock_retention_native, parse_object_version_id,
|
||||
};
|
||||
use crate::storage::storage_api::ecstore_bucket::metadata_sys::ObjectLockConfigState;
|
||||
use crate::storage::storage_api::test_consumer::{
|
||||
BucketMetadata, DEFAULT_READ_BUFFER_SIZE, StorageObjectInfo as ObjectInfo, apply_cors_headers,
|
||||
apply_default_lock_retention_metadata, bucket_metadata_sys_initialized, check_preconditions, decode_tags_to_map,
|
||||
get_adaptive_buffer_size_with_profile, get_buffer_size_opt_in, get_global_bucket_metadata_sys, matches_origin_pattern,
|
||||
parse_etag, process_lambda_configurations, process_queue_configurations, process_topic_configurations,
|
||||
remove_object_lock_metadata_for_copy, remove_object_lock_retention_metadata, set_bucket_metadata,
|
||||
validate_bucket_object_lock_enabled, validate_list_object_unordered_with_delimiter,
|
||||
parse_etag, parse_object_lock_legal_hold, parse_object_lock_retention, process_lambda_configurations,
|
||||
process_queue_configurations, process_topic_configurations, remove_object_lock_metadata_for_copy,
|
||||
remove_object_lock_retention_metadata, set_bucket_metadata, validate_bucket_object_lock_enabled,
|
||||
validate_list_object_unordered_with_delimiter,
|
||||
};
|
||||
use http::{HeaderMap, HeaderValue, StatusCode};
|
||||
use rustfs_config::MI_B;
|
||||
@@ -58,7 +56,6 @@ mod tests {
|
||||
PersistedFilterRule, PersistedLambdaFunctionConfiguration, PersistedNotificationConfigurationFilter,
|
||||
PersistedQueueConfiguration, PersistedS3KeyFilter, PersistedTopicConfiguration,
|
||||
};
|
||||
use rustfs_s3_types::{S3ErrorCode, S3Result};
|
||||
use rustfs_utils::http::{
|
||||
AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER, AMZ_OBJECT_LOCK_MODE_LOWER, AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER,
|
||||
MINIO_INTERNAL_PREFIX, RUSTFS_INTERNAL_PREFIX, SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP,
|
||||
@@ -66,23 +63,13 @@ mod tests {
|
||||
};
|
||||
use rustfs_zip::CompressionFormat;
|
||||
use s3s::dto::{
|
||||
DefaultRetention, ObjectLockConfiguration, ObjectLockEnabled, ObjectLockLegalHold, ObjectLockLegalHoldStatus,
|
||||
DefaultRetention, Delimiter, ObjectLockConfiguration, ObjectLockEnabled, ObjectLockLegalHold, ObjectLockLegalHoldStatus,
|
||||
ObjectLockRetention, ObjectLockRetentionMode, ObjectLockRule,
|
||||
};
|
||||
use s3s::{S3Error, S3ErrorCode, s3_error};
|
||||
use std::collections::HashMap;
|
||||
use time::{OffsetDateTime, format_description::well_known::Rfc3339};
|
||||
|
||||
fn parse_object_lock_retention(retention: Option<ObjectLockRetention>) -> S3Result<HashMap<String, String>> {
|
||||
parse_object_lock_retention_native(retention.map(|r| RequestedObjectLockRetention {
|
||||
mode: r.mode.map(|m| m.as_str().to_owned()),
|
||||
retain_until: r.retain_until_date.map(OffsetDateTime::from),
|
||||
}))
|
||||
}
|
||||
|
||||
fn parse_object_lock_legal_hold(legal_hold: Option<ObjectLockLegalHold>) -> S3Result<HashMap<String, String>> {
|
||||
parse_object_lock_legal_hold_native(legal_hold.map(|h| RequestedObjectLegalHold(h.status.map(|s| s.as_str().to_owned()))))
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_rustfs_owner_helpers_are_stable() {
|
||||
let owner = rustfs_owner();
|
||||
@@ -155,8 +142,6 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn test_s3_error_scenarios() {
|
||||
use s3s::{S3Error, S3ErrorCode, s3_error};
|
||||
|
||||
// Test that we can create expected S3 errors for common validation cases
|
||||
|
||||
// Test incomplete body error
|
||||
@@ -631,7 +616,8 @@ mod tests {
|
||||
// [1] Normal case: No delimiter specified.
|
||||
assert!(validate_list_object_unordered_with_delimiter(None, Some("allow-unordered=true")).is_ok());
|
||||
|
||||
let delimiter_some = Some("/");
|
||||
let delim_str = "/".to_string();
|
||||
let delimiter_some: Option<&Delimiter> = Some(&delim_str);
|
||||
// [2] Normal case: Delimiter is present, but 'allow-unordered' is explicitly set to false.
|
||||
assert!(validate_list_object_unordered_with_delimiter(delimiter_some, Some("allow-unordered=false")).is_ok());
|
||||
|
||||
@@ -1077,31 +1063,6 @@ mod tests {
|
||||
assert!(check_preconditions(&headers18, &info18).is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn conditional_read_preserves_invalid_cache_header_cause_at_the_wire() {
|
||||
let info = ObjectInfo {
|
||||
user_defined: HashMap::from([("cache-control".to_owned(), "invalid\nvalue".to_owned())]).into(),
|
||||
..Default::default()
|
||||
};
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert("if-none-match", HeaderValue::from_static("*"));
|
||||
let error = check_preconditions(&headers, &info).expect_err("invalid cached metadata cannot form a 304 response");
|
||||
assert_eq!(error.code(), &S3ErrorCode::InternalError);
|
||||
assert_eq!(error.status_code(), Some(StatusCode::INTERNAL_SERVER_ERROR));
|
||||
assert!(error.headers().is_none());
|
||||
assert!(
|
||||
error
|
||||
.source()
|
||||
.is_some_and(|cause| cause.is::<http::header::InvalidHeaderValue>())
|
||||
);
|
||||
let wire = s3s::S3Error::from(error);
|
||||
assert_eq!(wire.status_code(), Some(StatusCode::INTERNAL_SERVER_ERROR));
|
||||
assert!(
|
||||
wire.source()
|
||||
.is_some_and(|cause| cause.is::<http::header::InvalidHeaderValue>())
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_check_preconditions_rfc304() {
|
||||
let modified = OffsetDateTime::from_unix_timestamp(1700000000).expect("valid timestamp");
|
||||
|
||||
@@ -18,7 +18,6 @@ use crate::config::{RustFSBufferConfig, WorkloadProfile, is_buffer_profile_enabl
|
||||
use crate::error::ApiError;
|
||||
use crate::server::cors;
|
||||
use crate::storage::storage_api::ecstore_bucket::metadata_sys::{self, ObjectLockConfigState};
|
||||
use crate::storage::storage_api::ecstore_bucket::object_lock::types::{LegalHoldStatus, RetentionMode};
|
||||
use crate::storage::storage_api::s3_support_consumer::contract::bucket::{BucketOperations, BucketOptions};
|
||||
use crate::storage::storage_api::s3_support_consumer::contract::multipart::MAX_MULTIPART_PART_NUMBER;
|
||||
use http::header::{IF_MATCH, IF_MODIFIED_SINCE, IF_NONE_MATCH, IF_UNMODIFIED_SINCE};
|
||||
@@ -29,13 +28,16 @@ use rustfs_gateway_types::persistence::{
|
||||
PersistedLambdaFunctionConfiguration, PersistedNotificationConfigurationFilter, PersistedQueueConfiguration,
|
||||
PersistedTopicConfiguration,
|
||||
};
|
||||
use rustfs_s3_types::{S3Error, S3ErrorCode, S3Result};
|
||||
use rustfs_targets::EventName;
|
||||
use rustfs_targets::arn::{TargetID, TargetIDError};
|
||||
use rustfs_utils::http::{
|
||||
AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER, AMZ_OBJECT_LOCK_MODE_LOWER, AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER,
|
||||
SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP, contains_key_str, insert_str, remove_str,
|
||||
};
|
||||
use s3s::dto::{Delimiter, ObjectLockLegalHoldStatus, ObjectLockRetentionMode};
|
||||
#[cfg(test)]
|
||||
use s3s::dto::{ObjectLockLegalHold, ObjectLockRetention};
|
||||
use s3s::{S3Error, S3ErrorCode, S3Result};
|
||||
use serde_urlencoded::from_bytes;
|
||||
use std::collections::HashMap;
|
||||
use std::sync::{Arc, OnceLock, RwLock};
|
||||
@@ -384,7 +386,7 @@ pub(crate) fn validate_new_object_key(key: &str, operation: &str) -> S3Result<()
|
||||
/// 2. Rejects reserved directory markers and invalid path segments
|
||||
/// 3. Logs debug information for keys containing spaces, plus signs, or percent signs
|
||||
///
|
||||
/// Request routing decodes the key before this validation.
|
||||
/// The s3s library handles URL decoding, so keys are already decoded when they reach this function.
|
||||
/// This validation ensures that invalid characters that could cause issues are rejected early.
|
||||
pub(crate) fn validate_object_key(key: &str, operation: &str) -> S3Result<()> {
|
||||
// Validate object key doesn't contain control characters
|
||||
@@ -437,7 +439,10 @@ pub(crate) fn validate_object_key(key: &str, operation: &str) -> S3Result<()> {
|
||||
/// According to S3 compatibility requirements, unordered listing cannot be combined with
|
||||
/// hierarchical directory traversal (delimited listing). This validation ensures
|
||||
/// conflicting parameters are caught before processing the request.
|
||||
pub(crate) fn validate_list_object_unordered_with_delimiter(delimiter: Option<&str>, query_string: Option<&str>) -> S3Result<()> {
|
||||
pub(crate) fn validate_list_object_unordered_with_delimiter(
|
||||
delimiter: Option<&Delimiter>,
|
||||
query_string: Option<&str>,
|
||||
) -> S3Result<()> {
|
||||
if delimiter.is_none() {
|
||||
return Ok(());
|
||||
}
|
||||
@@ -458,6 +463,14 @@ pub(crate) fn validate_list_object_unordered_with_delimiter(delimiter: Option<&s
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn parse_object_lock_retention(retention: Option<ObjectLockRetention>) -> S3Result<HashMap<String, String>> {
|
||||
parse_object_lock_retention_native(retention.map(|r| RequestedObjectLockRetention {
|
||||
mode: r.mode.map(|m| m.as_str().to_owned()),
|
||||
retain_until: r.retain_until_date.map(OffsetDateTime::from),
|
||||
}))
|
||||
}
|
||||
|
||||
pub(crate) fn parse_object_lock_retention_native(
|
||||
retention: Option<RequestedObjectLockRetention>,
|
||||
) -> S3Result<HashMap<String, String>> {
|
||||
@@ -472,7 +485,7 @@ pub(crate) fn parse_object_lock_retention_native(
|
||||
}
|
||||
let mode = match v.mode {
|
||||
Some(mode) => match mode.as_str() {
|
||||
RetentionMode::COMPLIANCE | RetentionMode::GOVERNANCE => mode.as_str().to_string(),
|
||||
ObjectLockRetentionMode::COMPLIANCE | ObjectLockRetentionMode::GOVERNANCE => mode.as_str().to_string(),
|
||||
_ => {
|
||||
return Err(S3Error::with_message(
|
||||
S3ErrorCode::MalformedXML,
|
||||
@@ -516,13 +529,18 @@ pub(crate) fn parse_object_lock_retention_native(
|
||||
Ok(eval_metadata)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn parse_object_lock_legal_hold(legal_hold: Option<ObjectLockLegalHold>) -> S3Result<HashMap<String, String>> {
|
||||
parse_object_lock_legal_hold_native(legal_hold.map(|h| RequestedObjectLegalHold(h.status.map(|s| s.as_str().to_owned()))))
|
||||
}
|
||||
|
||||
pub(crate) fn parse_object_lock_legal_hold_native(
|
||||
legal_hold: Option<RequestedObjectLegalHold>,
|
||||
) -> S3Result<HashMap<String, String>> {
|
||||
let mut eval_metadata = HashMap::new();
|
||||
if let Some(v) = legal_hold {
|
||||
let status = match v.0.as_deref() {
|
||||
Some(status @ (LegalHoldStatus::OFF | LegalHoldStatus::ON)) => status.to_owned(),
|
||||
Some(status @ (ObjectLockLegalHoldStatus::OFF | ObjectLockLegalHoldStatus::ON)) => status.to_owned(),
|
||||
_ => {
|
||||
return Err(S3Error::with_message(
|
||||
S3ErrorCode::MalformedXML,
|
||||
@@ -911,11 +929,15 @@ pub fn invalidate_bucket_validation_cache(bucket: &str) {
|
||||
/// Uses adaptive cache with 5s TTL to avoid repeated stat_volume() calls.
|
||||
/// Returns store directly on cache hit without calling get_bucket_info().
|
||||
pub(crate) async fn get_validated_store(bucket: &str) -> S3Result<Arc<super::ECStore>> {
|
||||
Ok(get_validated_store_owned(bucket).await?)
|
||||
}
|
||||
|
||||
pub(crate) async fn get_validated_store_owned(bucket: &str) -> rustfs_s3_types::S3Result<Arc<super::ECStore>> {
|
||||
let Some(store) = runtime_sources::current_object_store_handle() else {
|
||||
return Err(rustfs_s3_types::s3_error!(InternalError, "Not init"));
|
||||
};
|
||||
|
||||
validate_bucket_exists(&store, bucket).await?;
|
||||
validate_bucket_exists_owned(&store, bucket).await?;
|
||||
|
||||
Ok(store)
|
||||
}
|
||||
@@ -925,6 +947,10 @@ pub(crate) async fn get_validated_store(bucket: &str) -> S3Result<Arc<super::ECS
|
||||
/// store through the request-bound server context (backlog#1052 S6) use this
|
||||
/// to keep bucket validation on that store instead of the process-global one.
|
||||
pub(crate) async fn validate_bucket_exists(store: &super::ECStore, bucket: &str) -> S3Result<()> {
|
||||
Ok(validate_bucket_exists_owned(store, bucket).await?)
|
||||
}
|
||||
|
||||
async fn validate_bucket_exists_owned(store: &super::ECStore, bucket: &str) -> rustfs_s3_types::S3Result<()> {
|
||||
// Check cache — TTL is checked manually.
|
||||
if let Some(inserted_at) = cache_get(bucket)
|
||||
&& inserted_at.elapsed() < BUCKET_VALIDATION_TTL
|
||||
|
||||
@@ -99,8 +99,8 @@ use rustfs_config::{DEFAULT_KMS_ENFORCE_SSE_KEY_POLICY, ENV_RUSTFS_KMS_ENFORCE_S
|
||||
use rustfs_kms::{DataKey, KmsUnavailableError, is_data_key_envelope, types::ObjectEncryptionContext};
|
||||
use rustfs_policy::policy::Args;
|
||||
use rustfs_policy::policy::action::{Action, KmsAction};
|
||||
use rustfs_s3_types::S3ErrorCode;
|
||||
use rustfs_utils::{get_env_bool, get_env_opt_str};
|
||||
use s3s::S3ErrorCode;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::Value;
|
||||
#[cfg(feature = "rio-v2")]
|
||||
@@ -4975,9 +4975,9 @@ pub(crate) mod tests {
|
||||
use http::{HeaderMap, HeaderValue};
|
||||
use rustfs_kms::types::ObjectEncryptionContext;
|
||||
use rustfs_rio::{DecryptReader, EncryptReader};
|
||||
use rustfs_s3_types::S3ErrorCode;
|
||||
use rustfs_utils::http::headers::AMZ_SERVER_SIDE_ENCRYPTION_KMS_CONTEXT;
|
||||
use rustfs_utils::http::headers::{AMZ_ENCRYPTION_AES, AMZ_ENCRYPTION_KMS};
|
||||
use s3s::S3ErrorCode;
|
||||
use std::collections::HashMap;
|
||||
use std::sync::{Arc, OnceLock};
|
||||
use temp_env::async_with_vars;
|
||||
|
||||
@@ -137,11 +137,12 @@ pub(crate) type StorageObjectToDelete = contract::object::ObjectToDelete;
|
||||
pub(crate) type StoragePutObjReader = super::PutObjReader;
|
||||
pub(crate) use super::s3_support::{
|
||||
RFC1123, apply_bucket_default_lock_retention, apply_cors_headers, check_preconditions, evaluate_read_preconditions,
|
||||
get_buffer_size_opt_in, get_validated_store, has_read_preconditions, load_bucket_object_lock_config_state,
|
||||
parse_object_lock_legal_hold_native, parse_object_lock_retention_native, parse_object_version_id,
|
||||
parse_part_number_i32_to_usize, process_lambda_configurations, process_queue_configurations, process_topic_configurations,
|
||||
remove_object_lock_metadata_for_copy, validate_bucket_exists, validate_bucket_object_lock_enabled_state,
|
||||
validate_list_object_unordered_with_delimiter, validate_new_object_key, validate_object_key,
|
||||
get_buffer_size_opt_in, get_validated_store, get_validated_store_owned, has_read_preconditions,
|
||||
load_bucket_object_lock_config_state, parse_object_lock_legal_hold_native, parse_object_lock_retention_native,
|
||||
parse_object_version_id, parse_part_number_i32_to_usize, process_lambda_configurations, process_queue_configurations,
|
||||
process_topic_configurations, remove_object_lock_metadata_for_copy, validate_bucket_exists,
|
||||
validate_bucket_object_lock_enabled_state, validate_list_object_unordered_with_delimiter, validate_new_object_key,
|
||||
validate_object_key,
|
||||
};
|
||||
pub(crate) use super::sse::{
|
||||
DecryptionRequest, EncryptionRequest, ObjectDekRewrapOutcome, PrepareEncryptionRequest, SseKmsPrincipal,
|
||||
@@ -485,9 +486,9 @@ pub(crate) mod test_consumer {
|
||||
pub(crate) use super::super::{
|
||||
apply_cors_headers, apply_default_lock_retention_metadata, check_preconditions, decode_tags_to_map,
|
||||
get_adaptive_buffer_size_with_profile, get_buffer_size_opt_in, matches_origin_pattern, parse_etag,
|
||||
process_lambda_configurations, process_queue_configurations, process_topic_configurations,
|
||||
remove_object_lock_metadata_for_copy, remove_object_lock_retention_metadata, validate_bucket_object_lock_enabled,
|
||||
validate_list_object_unordered_with_delimiter,
|
||||
parse_object_lock_legal_hold, parse_object_lock_retention, process_lambda_configurations, process_queue_configurations,
|
||||
process_topic_configurations, remove_object_lock_metadata_for_copy, remove_object_lock_retention_metadata,
|
||||
validate_bucket_object_lock_enabled, validate_list_object_unordered_with_delimiter,
|
||||
};
|
||||
pub(crate) use super::{
|
||||
BucketMetadata, DEFAULT_READ_BUFFER_SIZE, bucket_metadata_sys_initialized, get_global_bucket_metadata_sys,
|
||||
|
||||
@@ -274,6 +274,14 @@ pub(crate) mod server {
|
||||
}
|
||||
}
|
||||
|
||||
/// The s3s error types the s3s-compat tower layers in
|
||||
/// `server::legacy_compat` render and match on, funneled here so those
|
||||
/// files stay off the direct s3s surface (s3s footprint ratchet). T2.12
|
||||
/// (rustfs/backlog#2771) removes the layers and with them this module.
|
||||
pub(crate) mod legacy_compat {
|
||||
pub(crate) use s3s::S3ErrorCode;
|
||||
}
|
||||
|
||||
pub(crate) mod module_switch {
|
||||
pub(crate) use crate::storage::storage_api::{
|
||||
Error, read_config, read_config_no_lock, save_config_no_lock, with_config_object_read_lock,
|
||||
|
||||
@@ -766,9 +766,13 @@ fn the_gateway_stack_swaps_the_tagging_action_for_a_named_version() {
|
||||
with_stack(GATEWAY, the_tagging_action_follows_the_named_version);
|
||||
}
|
||||
|
||||
/// The gateway rejects boundary keys before consulting the bucket. RustFS serves no torrents,
|
||||
/// so a NUL key on GetObjectTorrent retains the release's NoSuchKey response.
|
||||
async fn keys_outside_the_floor_are_refused(server: rustfs::embedded::RustFSServer) {
|
||||
/// What a stack answers for a key outside the floor, independent of the operation and bucket.
|
||||
type UniformAnswers = (Answer, Answer);
|
||||
|
||||
/// Checks boundary keys across bucket configurations. The legacy stack answers per operation and
|
||||
/// bucket from the table below; the gateway stack refuses the key before the bucket is consulted,
|
||||
/// so it answers every row with `uniform`: `(NUL key, 1025-byte key)`.
|
||||
async fn keys_outside_the_floor_are_refused(server: rustfs::embedded::RustFSServer, uniform: Option<UniformAnswers>) {
|
||||
const BUCKET: &str = "object-meta-keys";
|
||||
let client = s3_client(&server.endpoint(), server.access_key(), server.secret_key());
|
||||
create_bucket_in_the_store(BUCKET).await;
|
||||
@@ -778,25 +782,38 @@ async fn keys_outside_the_floor_are_refused(server: rustfs::embedded::RustFSServ
|
||||
assert!(super::bucket_exists_in_the_store(reserved).await);
|
||||
put_object_in_the_store(reserved, KEY, HashMap::new()).await;
|
||||
assert_eq!(answer(client.get_object_acl().bucket(reserved).key(KEY).send().await), (None, None, None));
|
||||
let refused = |code: &str, status, message: &str| (Some(code.to_owned()), Some(status), Some(message.to_owned()));
|
||||
let invalid_nul = refused("InvalidArgument", 400, "the name contains a NUL byte");
|
||||
let missing_torrent = refused("NoSuchKey", 404, "The specified key does not exist.");
|
||||
let too_long = refused("KeyTooLongError", 400, "the name is longer than the maximum length in bytes");
|
||||
let long_key = "k".repeat(1025);
|
||||
for (operation, call) in KEY_CALLS {
|
||||
let expected_nul = if *operation == "GetObjectTorrent" {
|
||||
&missing_torrent
|
||||
} else {
|
||||
&invalid_nul
|
||||
};
|
||||
for bucket in [BUCKET, locked.as_str(), "object-meta-missing", reserved] {
|
||||
assert_eq!(&call(&client, bucket, "a\0b").await, expected_nul, "{operation} bucket {bucket}");
|
||||
for (bucket, kind) in [(BUCKET, 0), (locked.as_str(), 1), ("object-meta-missing", 2), (reserved, 0)] {
|
||||
let (code, status, message) = match (*operation, kind) {
|
||||
("GetObjectTorrent", _) => ("NoSuchKey", 404, "The specified key does not exist."),
|
||||
("GetObjectLegalHold", 0) | ("GetObjectRetention", 0) => {
|
||||
("InvalidRequest", 400, "Bucket is missing ObjectLockConfiguration")
|
||||
}
|
||||
("GetObjectLegalHold", 1) | ("GetObjectRetention", 1) => {
|
||||
("InternalError", 500, "XML response contains a control character")
|
||||
}
|
||||
("GetObjectLegalHold", _) => ("NoSuchBucket", 404, "The specified bucket does not exist"),
|
||||
("GetObjectRetention", _) => ("InternalError", 500, "Failed to load Object Lock configuration"),
|
||||
_ => ("InvalidArgument", 400, "Invalid argument"),
|
||||
};
|
||||
let expected = match &uniform {
|
||||
Some((nul, _)) => nul.clone(),
|
||||
None => (Some(code.to_owned()), Some(status), Some(message.to_owned())),
|
||||
};
|
||||
assert_eq!(call(&client, bucket, "a\0b").await, expected, "{operation} bucket {bucket}");
|
||||
}
|
||||
assert_eq!(call(&client, BUCKET, &long_key).await, too_long, "{operation} long key");
|
||||
let expected = match &uniform {
|
||||
Some((_, long)) => long.clone(),
|
||||
None => (Some("KeyTooLongError".to_owned()), Some(400), Some("Your key is too long.".to_owned())),
|
||||
};
|
||||
assert_eq!(call(&client, BUCKET, &long_key).await, expected, "{operation} long key");
|
||||
}
|
||||
server.shutdown().await;
|
||||
}
|
||||
|
||||
type Answer = (Option<String>, Option<u16>, Option<String>);
|
||||
|
||||
type KeyCall = for<'a> fn(&'a Client, &'a str, &'a str) -> BoxFuture<'a, (Option<String>, Option<u16>, Option<String>)>;
|
||||
|
||||
const KEY_CALLS: &[(&str, KeyCall)] = &[
|
||||
@@ -836,7 +853,12 @@ fn answer<T, E: ProvideErrorMetadata>(result: Result<T, SdkError<E>>) -> (Option
|
||||
|
||||
#[test]
|
||||
fn the_gateway_stack_refuses_keys_outside_the_floor() {
|
||||
with_stack(GATEWAY, keys_outside_the_floor_are_refused);
|
||||
let refused = |code: &str, message: &str| (Some(code.to_owned()), Some(400), Some(message.to_owned()));
|
||||
let uniform = (
|
||||
refused("InvalidArgument", "the name contains a NUL byte"),
|
||||
refused("KeyTooLongError", "the name is longer than the maximum length in bytes"),
|
||||
);
|
||||
with_stack(GATEWAY, move |server| keys_outside_the_floor_are_refused(server, Some(uniform)));
|
||||
}
|
||||
|
||||
fn legal_hold(status: &str) -> aws_sdk_s3::types::ObjectLockLegalHold {
|
||||
|
||||
@@ -162,12 +162,13 @@ done < <(awk '
|
||||
pending { sub(/^[[:space:]]*'"'"'/, ""); sub(/'"'"'[[:space:]]*\\$/, ""); print; pending = 0; next }
|
||||
/module_relative_rg rustfs\/src\/(app|storage) \\$/ { pending = 1 }
|
||||
' "$ARCH_RULES")
|
||||
[[ ${#TEMPLATES[@]} == 5 ]] || fail "expected 5 depth-aware rules in $ARCH_RULES, found ${#TEMPLATES[@]}"
|
||||
[[ ${#TEMPLATES[@]} == 6 ]] || fail "expected 6 depth-aware rules in $ARCH_RULES, found ${#TEMPLATES[@]}"
|
||||
STORAGE_HELPER_RULE="${TEMPLATES[0]}"
|
||||
STORAGE_OWNER_RULE="${TEMPLATES[1]}"
|
||||
RUNTIME_IMPORT_RULE="${TEMPLATES[2]}"
|
||||
RUNTIME_PATH_RULE="${TEMPLATES[3]}"
|
||||
STORAGE_HELPER_TEST_RULE="${TEMPLATES[4]}"
|
||||
ECFS_TEST_RULE="${TEMPLATES[4]}"
|
||||
ECFS_TEST_SPLIT_RULE="${TEMPLATES[5]}"
|
||||
|
||||
# relative_hits <module-dir> <template> <relative-file> <line>: rg output for one file.
|
||||
relative_hits() {
|
||||
@@ -207,8 +208,9 @@ flagged rustfs/src/app "$STORAGE_OWNER_RULE" rustfs/src/app/bucket_usecase.rs 's
|
||||
flagged rustfs/src/app "$STORAGE_OWNER_RULE" rustfs/src/app/bucket_usecase/cors.rs 'super::super::super::metadata_sys::get();'
|
||||
flagged rustfs/src/app "$RUNTIME_IMPORT_RULE" rustfs/src/app/bucket_usecase/cors.rs 'use super::super::ECStore;'
|
||||
flagged rustfs/src/app "$RUNTIME_PATH_RULE" rustfs/src/app/lifecycle_transition_api_test/put.rs 'super::super::get_server_info();'
|
||||
flagged rustfs/src/storage "$STORAGE_HELPER_TEST_RULE" rustfs/src/storage/helpers_test.rs ' use super::super::{BucketMetadata};'
|
||||
flagged rustfs/src/storage "$STORAGE_HELPER_TEST_RULE" rustfs/src/storage/helpers_test.rs 'use crate::storage::{check_preconditions};'
|
||||
flagged rustfs/src/storage "$ECFS_TEST_RULE" rustfs/src/storage/ecfs_test.rs ' use super::super::{BucketMetadata};'
|
||||
flagged rustfs/src/storage "$ECFS_TEST_SPLIT_RULE" rustfs/src/storage/legacy_edge/tests/object_meta.rs 'use super::super::super::{BucketMetadata};'
|
||||
flagged rustfs/src/storage "$ECFS_TEST_SPLIT_RULE" rustfs/src/storage/legacy_edge/tests/object_meta.rs ' use super::super::super::super::{check_preconditions};'
|
||||
|
||||
# Ordinary forms: a split file importing from its own parent module is not flagged.
|
||||
new_case
|
||||
@@ -217,12 +219,12 @@ not_flagged rustfs/src/app "$STORAGE_OWNER_RULE" rustfs/src/app/bucket_usecase/c
|
||||
not_flagged rustfs/src/app "$STORAGE_OWNER_RULE" rustfs/src/app/bucket_usecase/cors.rs 'super::tagging::helper();'
|
||||
not_flagged rustfs/src/app "$RUNTIME_IMPORT_RULE" rustfs/src/app/bucket_usecase/cors.rs 'use super::{ECStore, DefaultBucketUsecase};'
|
||||
not_flagged rustfs/src/app "$RUNTIME_PATH_RULE" rustfs/src/app/bucket_usecase/cors.rs 'super::get_server_info();'
|
||||
not_flagged rustfs/src/storage "$STORAGE_HELPER_TEST_RULE" rustfs/src/storage/helpers_test.rs ' use super::{BucketMetadata};'
|
||||
not_flagged rustfs/src/storage "$ECFS_TEST_SPLIT_RULE" rustfs/src/storage/legacy_edge/tests/object_meta.rs ' use super::{BucketMetadata};'
|
||||
|
||||
# A file outside the module directory, or a missing -- separator, is an error, not a silent pass.
|
||||
new_case
|
||||
status=0
|
||||
module_relative_rg rustfs/src/app 'x' -n -- rustfs/src/storage/helpers_test.rs 2>/dev/null || status=$?
|
||||
module_relative_rg rustfs/src/app 'x' -n -- rustfs/src/storage/ecfs.rs 2>/dev/null || status=$?
|
||||
[[ "$status" == 2 ]] || fail "file outside the module dir: status $status"
|
||||
status=0
|
||||
module_relative_rg rustfs/src/app 'x' -n rustfs/src/app/a.rs 2>/dev/null || status=$?
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Exercise the pinned upgrade sources without building RustFS."""
|
||||
"""Exercise both pinned upgrade sources without building RustFS."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -10,7 +10,6 @@ import re
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import tomllib
|
||||
import unittest
|
||||
|
||||
from check_test_wiring import yaml_block
|
||||
@@ -18,12 +17,6 @@ from test_security_workflow import named_steps, shell_body
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
SOURCE_CASES = {
|
||||
"1.0.0-rc.5": ("rc5_rollback_requires_restoring_odm_configuration",
|
||||
"rc5_checkpoint_upgrade_rewinds_signed_progress_without_losing_objects"),
|
||||
"1.0.1": ("stable_upgrade_deletes_foreign_reservations_without_losing_live_objects",),
|
||||
"1.0.5": (),
|
||||
}
|
||||
|
||||
|
||||
class UpgradeWorkflowTests(unittest.TestCase):
|
||||
@@ -35,16 +28,19 @@ class UpgradeWorkflowTests(unittest.TestCase):
|
||||
# Explicit bash enables pipefail in Actions, so tee cannot hide a failure.
|
||||
self.assertIn(" shell: bash", self.step)
|
||||
self.assertIn(" fail-fast: false", self.job)
|
||||
self.assertEqual(re.findall(r"- version: ([^\s]+)", "\n".join(self.job)), list(SOURCE_CASES))
|
||||
self.assertEqual(re.findall(r"- version: ([^\s]+)", "\n".join(self.job)), ["1.0.0-rc.5", "1.0.1"])
|
||||
rust = (ROOT / "crates/e2e_test/src/upgrade_compatibility_test.rs").read_text()
|
||||
self.expected = set(re.findall(r'#\[ignore[^\n]*\]\s*async fn (\w+)\(', rust))
|
||||
self.assertTrue(self.expected)
|
||||
|
||||
def run_cases(self, version, failed_test="", fail_tee=False):
|
||||
shared = self.expected.difference(*SOURCE_CASES.values())
|
||||
self.assertEqual(len(shared), 6)
|
||||
expected = shared | set(SOURCE_CASES[version])
|
||||
self.assertTrue(expected <= self.expected)
|
||||
excluded = ({"stable_upgrade_deletes_foreign_reservations_without_losing_live_objects"}
|
||||
if version == "1.0.0-rc.5" else {
|
||||
"rc5_rollback_requires_restoring_odm_configuration",
|
||||
"rc5_checkpoint_upgrade_rewinds_signed_progress_without_losing_objects",
|
||||
})
|
||||
expected = self.expected - excluded
|
||||
self.assertEqual(len(expected), 8 if version == "1.0.0-rc.5" else 7)
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
root = Path(directory)
|
||||
commands = root / "commands.jsonl"
|
||||
@@ -86,14 +82,14 @@ sys.exit(17 if test == os.environ["FAILED_TEST"] else 0)
|
||||
return result
|
||||
|
||||
def test_all_ignored_upgrade_cases_run_once_with_separate_evidence(self):
|
||||
for version in SOURCE_CASES:
|
||||
for version in ("1.0.0-rc.5", "1.0.1"):
|
||||
with self.subTest(version=version):
|
||||
result = self.run_cases(version)
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
|
||||
def test_early_or_late_failure_does_not_hide_remaining_cases(self):
|
||||
for version, specialized in SOURCE_CASES.items():
|
||||
last = specialized[-1] if specialized else "previous_release_baseline_replicates_multipart_layouts"
|
||||
for version, last in (("1.0.0-rc.5", "rc5_checkpoint_upgrade_rewinds_signed_progress_without_losing_objects"),
|
||||
("1.0.1", "stable_upgrade_deletes_foreign_reservations_without_losing_live_objects")):
|
||||
for name in ("direct_upgrade_from_previous_release_preserves_object_contracts", last):
|
||||
with self.subTest(version=version, name=name):
|
||||
result = self.run_cases(version, failed_test=name)
|
||||
@@ -101,50 +97,9 @@ sys.exit(17 if test == os.environ["FAILED_TEST"] else 0)
|
||||
self.assertIn(f"::error::Upgrade compatibility failed: {name}", result.stdout)
|
||||
|
||||
def test_log_write_failure_fails_the_job(self):
|
||||
for version in SOURCE_CASES:
|
||||
for version in ("1.0.0-rc.5", "1.0.1"):
|
||||
self.assertNotEqual(self.run_cases(version, fail_tee=True).returncode, 0)
|
||||
|
||||
def test_official_reference_matches_the_workflow_and_release_plan(self):
|
||||
reference = tomllib.loads((ROOT / ".config/reference-binary.toml").read_text())
|
||||
platform = reference["platforms"]["linux-x86_64-gnu"]
|
||||
pin = ("1.0.5", platform["asset"], platform["sha256"])
|
||||
self.assertEqual(reference["version"], pin[0])
|
||||
matrix = re.findall(r"- version: (\S+)\s+asset: (\S+)\s+sha256: (\w+)", "\n".join(self.job))
|
||||
self.assertIn(pin, matrix)
|
||||
plan = json.loads((ROOT / ".config/release-validation-plan.json").read_text())
|
||||
self.assertEqual(set(matrix), {(s["version"], s["artifact_name"], s["artifact_sha256"])
|
||||
for s in plan["upgrade_sources"]["sources"]})
|
||||
legacy_mpu = "\n".join(self.steps["Run mixed-version MPU recovery"])
|
||||
self.assertIn("matrix.source.version != '1.0.5'", legacy_mpu)
|
||||
self.assertIn("mixed_version_failed_complete_requires_current_coordinator_for_abort", legacy_mpu)
|
||||
|
||||
def test_gateway_evidence_requires_both_roles_and_rollback_for_the_pinned_pair(self):
|
||||
step = self.steps["Verify gateway baseline G09 evidence"]
|
||||
self.assertIn("matrix.source.version == '1.0.5'", "\n".join(step))
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
root = Path(directory) / "rustfs-upgrade-g09-evidence"
|
||||
env = dict(os.environ, RUNNER_TEMP=directory, GITHUB_SHA="c" * 40, UPGRADE_SOURCE_REVISION="a" * 40)
|
||||
for field in ("mixed_version_reader_evidence", "mixed_version_writer_evidence", "rollback_payload_evidence"):
|
||||
folder = root / ("bucket-config-rollback" if field == "rollback_payload_evidence" else "mixed-version-upgrade")
|
||||
folder.mkdir(exist_ok=True, parents=True)
|
||||
payload = dict(gate="G09", field=field, current_revision=env["GITHUB_SHA"],
|
||||
previous_revision=env["UPGRADE_SOURCE_REVISION"], rollback_payload_replayed=True)
|
||||
(folder / f"G09-{field}.json").write_text(json.dumps(payload))
|
||||
multipart = root / "bucket-config-rollback/inflight-multipart-rollback.json"
|
||||
payload = dict(current_revision=env["GITHUB_SHA"], previous_revision=env["UPGRADE_SOURCE_REVISION"],
|
||||
part_sizes=[5242880, 5242880, 4096], bytes=10489856,
|
||||
rollback_read_sha256="d" * 64, roll_forward_read_sha256="d" * 64)
|
||||
multipart.write_text(json.dumps(payload))
|
||||
command = ["bash", "-e", "-o", "pipefail", "-c", shell_body(step)]
|
||||
self.assertEqual(subprocess.run(command, env=env, capture_output=True).returncode, 0)
|
||||
wrong_pair = dict(env, UPGRADE_SOURCE_REVISION="b" * 40)
|
||||
self.assertNotEqual(subprocess.run(command, env=wrong_pair, capture_output=True).returncode, 0)
|
||||
multipart.write_text(json.dumps(dict(payload, roll_forward_read_sha256="e" * 64)))
|
||||
self.assertNotEqual(subprocess.run(command, env=env, capture_output=True).returncode, 0)
|
||||
multipart.write_text(json.dumps(payload))
|
||||
(root / "mixed-version-upgrade/G09-mixed_version_reader_evidence.json").unlink()
|
||||
self.assertNotEqual(subprocess.run(command, env=env, capture_output=True).returncode, 0)
|
||||
|
||||
def test_reports_are_uploaded_even_after_failure(self):
|
||||
for name, directory in (("Upload scanner/heal G09 evidence", "rustfs-upgrade-g09-evidence"),
|
||||
("Upload server logs", "rustfs-upgrade-logs")):
|
||||
|
||||
Reference in New Issue
Block a user