gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-09-06 03:59:14 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(admin): add isolated bucket metadata diagnostics (#7225)

Browse Source
This commit is contained in:
Zhengchao An
2026-09-06 01:30:18 +08:00
committed by GitHub
parent c9acc33720
commit 14cef91423
2 changed files with 516 additions and 235 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/operations/bucket-metadata-recovery.md
+22
View File
@@ -0,0 +1,22 @@
# Bucket metadata diagnostics and recovery
`GET /rustfs/admin/v3/export-bucket-metadata` keeps its strict behavior: an unreadable configuration fails the export. The optional `bucket` query selects one bucket; omitting it selects all buckets.
To inspect readable configurations while identifying failures, use the same authenticated endpoint with `?diagnostic=true`. This requires the existing `ExportBucketMetadataAction` permission. A successful response has:
- Filename `bucket-meta-diagnostic.zip` and header `x-rustfs-bucket-metadata-export: diagnostic`.
- Readable entries under `_diagnostic/<bucket>/<config>`; target credentials remain redacted.
- `_diagnostic-manifest.json`, containing `version: 1`, `mode: "diagnostic"`, `complete`, and an `errors` array. Each error identifies `bucket`, `config`, and the fixed code `configuration_unavailable`. The archive excludes unreadable payloads and parser error details.
`complete` reports whether all supported configuration reads succeeded. A diagnostic archive is never a restorable backup, including when `complete` is true. Import rejects the manifest or reserved directory before any bucket creation or configuration write. The reserved directory is not a valid bucket name, so older importers cannot restore diagnostic entries as ordinary bucket configurations.
## Recover unreadable replication targets
MinIO target configuration may be an array or KMS-encrypted data that RustFS cannot decode. Diagnosis preserves the failure instead of interpreting it as an empty target set.
1. Inspect the diagnostic manifest to identify affected buckets. Preserve a separate backup of the original source configuration and any credentials needed for recovery.
2. Prepare a ZIP containing `<bucket>/bucket-targets.json` with a valid RustFS replacement, whose top-level shape is `{"targets": [...]}`. Supply the intended target settings and credentials; exported credentials are redacted. Use `{"targets": []}` only when intentionally clearing all targets, and reconcile any replication rules that reference removed targets.
3. Submit the ZIP to the existing authenticated `PUT /rustfs/admin/v3/import-bucket-metadata` endpoint with `ImportBucketMetadataAction` permission. Import validates the replacement and persists it against the bucket incarnation; it does not need to parse the old target payload successfully.
4. Verify target listing and the intended replication configuration. Retry the ordinary strict metadata export to confirm the unreadable configuration no longer blocks it.
Do not submit the diagnostic archive itself to the import endpoint. Copy only reviewed replacement entries into an ordinary import archive.
rustfs/src/admin/handlers/bucket_meta.rs
+494 -235
View File
@@ -64,6 +64,9 @@ use time::OffsetDateTime;
use tracing::warn;
use zip::{ZipArchive, ZipWriter, write::SimpleFileOptions};
const DIAGNOSTIC_EXPORT_PREFIX: &str = "_diagnostic";
const DIAGNOSTIC_EXPORT_MANIFEST: &str = "_diagnostic-manifest.json";
const LOG_COMPONENT_ADMIN: &str = "admin";
const LOG_SUBSYSTEM_BUCKET_META: &str = "bucket_meta";
const EVENT_ADMIN_BUCKET_META_STATE: &str = "admin_bucket_meta_state";
@@ -97,9 +100,198 @@ fn checked_versioning_xml(validated: &VersioningConfiguration, raw: Vec<u8>) ->
checked_raw_xml(validated, raw, deserialize::<VersioningConfiguration>)
}
async fn exported_bucket_config(bucket: &str, conf: &str) -> S3Result<Option<Vec<u8>>> {
match conf {
BUCKET_POLICY_CONFIG => {
let config: BucketPolicy = match metadata_sys::get_bucket_policy(bucket).await {
Ok((res, _)) => res,
Err(e) => {
if e == StorageError::ConfigNotFound {
return Ok(None);
}
return Err(s3_error!(InternalError, "failed to load bucket metadata: {e}"));
}
};
let config_json =
serde_json::to_vec(&config).map_err(|e| s3_error!(InternalError, "failed to serialize config: {e}"))?;
Ok(Some(config_json))
}
BUCKET_NOTIFICATION_CONFIG => {
let config: s3s::dto::NotificationConfiguration = match metadata_sys::get_notification_config(bucket).await {
Ok(Some(res)) => res,
Err(e) => {
if e == StorageError::ConfigNotFound {
return Ok(None);
}
return Err(s3_error!(InternalError, "get bucket metadata failed: {e}"));
}
Ok(None) => return Ok(None),
};
let raw_config = metadata_sys::get(bucket)
.await
.map_err(|e| export_internal_error(format!("get bucket metadata failed: {e}")))?
.notification_config_xml
.clone();
let config_xml = checked_raw_xml(&config, raw_config, deserialize::<s3s::dto::NotificationConfiguration>)?;
Ok(Some(config_xml))
}
BUCKET_LIFECYCLE_CONFIG => {
let config: BucketLifecycleConfiguration = match metadata_sys::get_lifecycle_config(bucket).await {
Ok((res, _)) => res,
Err(e) => {
if e == StorageError::ConfigNotFound {
return Ok(None);
}
return Err(s3_error!(InternalError, "failed to load bucket metadata: {e}"));
}
};
let raw_config = metadata_sys::get(bucket)
.await
.map_err(|e| export_internal_error(format!("failed to load bucket metadata: {e}")))?
.lifecycle_config_xml
.clone();
let config_xml = checked_raw_xml(&config, raw_config, deserialize::<BucketLifecycleConfiguration>)?;
Ok(Some(config_xml))
}
BUCKET_TAGGING_CONFIG => {
let config: Tagging = match metadata_sys::get_tagging_config(bucket).await {
Ok((res, _)) => res,
Err(e) => {
if e == StorageError::ConfigNotFound {
return Ok(None);
}
return Err(s3_error!(InternalError, "failed to load bucket metadata: {e}"));
}
};
let raw_config = metadata_sys::get(bucket)
.await
.map_err(|e| export_internal_error(format!("failed to load bucket metadata: {e}")))?
.tagging_config_xml
.clone();
let config_xml = checked_raw_xml(&config, raw_config, deserialize::<Tagging>)?;
Ok(Some(config_xml))
}
BUCKET_QUOTA_CONFIG_FILE => {
let config: BucketQuota = match metadata_sys::get_quota_config(bucket).await {
Ok((res, _)) => res,
Err(e) => {
if e == StorageError::ConfigNotFound {
return Ok(None);
}
return Err(s3_error!(InternalError, "get bucket metadata failed: {e}"));
}
};
let config_json =
serde_json::to_vec(&config).map_err(|e| s3_error!(InternalError, "serialize config failed: {e}"))?;
Ok(Some(config_json))
}
OBJECT_LOCK_CONFIG => {
let config = match metadata_sys::get_object_lock_config(bucket).await {
Ok((res, _)) => res,
Err(e) => {
if e == StorageError::ConfigNotFound {
return Ok(None);
}
return Err(s3_error!(InternalError, "get bucket metadata failed: {e}"));
}
};
let raw_config = metadata_sys::get(bucket)
.await
.map_err(|e| export_internal_error(format!("get bucket metadata failed: {e}")))?
.object_lock_config_xml
.clone();
let config_xml = checked_raw_xml(&config, raw_config, deserialize::<ObjectLockConfiguration>)?;
Ok(Some(config_xml))
}
BUCKET_SSECONFIG => {
let config = match metadata_sys::get_sse_config(bucket).await {
Ok((res, _)) => res,
Err(e) => {
if e == StorageError::ConfigNotFound {
return Ok(None);
}
return Err(s3_error!(InternalError, "get bucket metadata failed: {e}"));
}
};
let raw_config = metadata_sys::get(bucket)
.await
.map_err(|e| export_internal_error(format!("get bucket metadata failed: {e}")))?
.encryption_config_xml
.clone();
let config_xml = checked_raw_xml(&config, raw_config, deserialize::<ServerSideEncryptionConfiguration>)?;
Ok(Some(config_xml))
}
BUCKET_VERSIONING_CONFIG => {
let config = match metadata_sys::get_versioning_config(bucket).await {
Ok((res, _)) => res,
Err(e) => {
if e == StorageError::ConfigNotFound {
return Ok(None);
}
return Err(s3_error!(InternalError, "get bucket metadata failed: {e}"));
}
};
let raw_config = metadata_sys::get(bucket)
.await
.map_err(|e| export_internal_error(format!("get bucket metadata failed: {e}")))?
.versioning_config_xml
.clone();
let config_xml = checked_versioning_xml(&config, raw_config)?;
Ok(Some(config_xml))
}
BUCKET_REPLICATION_CONFIG => {
let config = match metadata_sys::get_replication_config(bucket).await {
Ok((res, _)) => res,
Err(e) => {
if e == StorageError::ConfigNotFound {
return Ok(None);
}
return Err(s3_error!(InternalError, "get bucket metadata failed: {e}"));
}
};
let raw_config = metadata_sys::get(bucket)
.await
.map_err(|e| export_internal_error(format!("get bucket metadata failed: {e}")))?
.replication_config_xml
.clone();
let config_xml = checked_raw_xml(&config, raw_config, deserialize::<ReplicationConfiguration>)?;
Ok(Some(config_xml))
}
BUCKET_TARGETS_FILE => {
let config: BucketTargets = match metadata_sys::get_bucket_targets_config(bucket).await {
Ok(res) => res,
Err(e) => {
if e == StorageError::ConfigNotFound {
return Ok(None);
}
return Err(s3_error!(InternalError, "get bucket metadata failed: {e}"));
}
};
let config_json = serde_json::to_vec(&config.redacted_credentials())
.map_err(|e| s3_error!(InternalError, "serialize config failed: {e}"))?;
Ok(Some(config_json))
}
_ => Ok(None),
}
}
#[derive(Debug, Default, serde::Deserialize)]
pub struct ExportBucketMetadataQuery {
#[serde(default)]
pub bucket: String,
#[serde(default)]
pub diagnostic: bool,
}
pub struct ExportBucketMetadata {}
@@ -169,6 +361,7 @@ impl Operation for ExportBucketMetadata {
};
let mut zip_writer = ZipWriter::new(Cursor::new(Vec::new()));
let mut errors = Vec::new();
let confs = [
BUCKET_POLICY_CONFIG,
@@ -186,244 +379,49 @@ impl Operation for ExportBucketMetadata {
for bucket in buckets {
for &conf in confs.iter() {
let conf_path = path_join_buf(&[bucket.name.as_str(), conf]);
match conf {
BUCKET_POLICY_CONFIG => {
let config: BucketPolicy = match metadata_sys::get_bucket_policy(&bucket.name).await {
Ok((res, _)) => res,
Err(e) => {
if e == StorageError::ConfigNotFound {
continue;
}
return Err(s3_error!(InternalError, "failed to load bucket metadata: {e}"));
}
};
let config_json = serde_json::to_vec(&config)
.map_err(|e| s3_error!(InternalError, "failed to serialize config: {e}"))?;
zip_writer
.start_file(conf_path, SimpleFileOptions::default())
.map_err(|e| s3_error!(InternalError, "failed to start archive entry: {e}"))?;
zip_writer
.write_all(&config_json)
.map_err(|e| s3_error!(InternalError, "failed to write archive entry: {e}"))?;
let config = match exported_bucket_config(&bucket.name, conf).await {
Ok(Some(config)) => config,
Ok(None) => continue,
Err(error) if !query.diagnostic => return Err(error),
Err(_) => {
errors.push(serde_json::json!({
"bucket": bucket.name,
"config": conf,
"code": "configuration_unavailable",
}));
continue;
}
BUCKET_NOTIFICATION_CONFIG => {
let config: s3s::dto::NotificationConfiguration =
match metadata_sys::get_notification_config(&bucket.name).await {
Ok(Some(res)) => res,
Err(e) => {
if e == StorageError::ConfigNotFound {
continue;
}
return Err(s3_error!(InternalError, "get bucket metadata failed: {e}"));
}
Ok(None) => continue,
};
let raw_config = metadata_sys::get(&bucket.name)
.await
.map_err(|e| export_internal_error(format!("get bucket metadata failed: {e}")))?
.notification_config_xml
.clone();
let config_xml =
checked_raw_xml(&config, raw_config, deserialize::<s3s::dto::NotificationConfiguration>)?;
zip_writer
.start_file(conf_path, SimpleFileOptions::default())
.map_err(|e| s3_error!(InternalError, "start file failed: {e}"))?;
zip_writer
.write_all(&config_xml)
.map_err(|e| s3_error!(InternalError, "write file failed: {e}"))?;
}
BUCKET_LIFECYCLE_CONFIG => {
let config: BucketLifecycleConfiguration = match metadata_sys::get_lifecycle_config(&bucket.name).await {
Ok((res, _)) => res,
Err(e) => {
if e == StorageError::ConfigNotFound {
continue;
}
return Err(s3_error!(InternalError, "failed to load bucket metadata: {e}"));
}
};
let raw_config = metadata_sys::get(&bucket.name)
.await
.map_err(|e| export_internal_error(format!("failed to load bucket metadata: {e}")))?
.lifecycle_config_xml
.clone();
let config_xml = checked_raw_xml(&config, raw_config, deserialize::<BucketLifecycleConfiguration>)?;
zip_writer
.start_file(conf_path, SimpleFileOptions::default())
.map_err(|e| s3_error!(InternalError, "failed to start archive entry: {e}"))?;
zip_writer
.write_all(&config_xml)
.map_err(|e| s3_error!(InternalError, "failed to write archive entry: {e}"))?;
}
BUCKET_TAGGING_CONFIG => {
let config: Tagging = match metadata_sys::get_tagging_config(&bucket.name).await {
Ok((res, _)) => res,
Err(e) => {
if e == StorageError::ConfigNotFound {
continue;
}
return Err(s3_error!(InternalError, "failed to load bucket metadata: {e}"));
}
};
let raw_config = metadata_sys::get(&bucket.name)
.await
.map_err(|e| export_internal_error(format!("failed to load bucket metadata: {e}")))?
.tagging_config_xml
.clone();
let config_xml = checked_raw_xml(&config, raw_config, deserialize::<Tagging>)?;
zip_writer
.start_file(conf_path, SimpleFileOptions::default())
.map_err(|e| s3_error!(InternalError, "failed to start archive entry: {e}"))?;
zip_writer
.write_all(&config_xml)
.map_err(|e| s3_error!(InternalError, "failed to write archive entry: {e}"))?;
}
BUCKET_QUOTA_CONFIG_FILE => {
let config: BucketQuota = match metadata_sys::get_quota_config(&bucket.name).await {
Ok((res, _)) => res,
Err(e) => {
if e == StorageError::ConfigNotFound {
continue;
}
return Err(s3_error!(InternalError, "get bucket metadata failed: {e}"));
}
};
let config_json =
serde_json::to_vec(&config).map_err(|e| s3_error!(InternalError, "serialize config failed: {e}"))?;
zip_writer
.start_file(conf_path, SimpleFileOptions::default())
.map_err(|e| s3_error!(InternalError, "start file failed: {e}"))?;
zip_writer
.write_all(&config_json)
.map_err(|e| s3_error!(InternalError, "write file failed: {e}"))?;
}
OBJECT_LOCK_CONFIG => {
let config = match metadata_sys::get_object_lock_config(&bucket.name).await {
Ok((res, _)) => res,
Err(e) => {
if e == StorageError::ConfigNotFound {
continue;
}
return Err(s3_error!(InternalError, "get bucket metadata failed: {e}"));
}
};
let raw_config = metadata_sys::get(&bucket.name)
.await
.map_err(|e| export_internal_error(format!("get bucket metadata failed: {e}")))?
.object_lock_config_xml
.clone();
let config_xml = checked_raw_xml(&config, raw_config, deserialize::<ObjectLockConfiguration>)?;
zip_writer
.start_file(conf_path, SimpleFileOptions::default())
.map_err(|e| s3_error!(InternalError, "start file failed: {e}"))?;
zip_writer
.write_all(&config_xml)
.map_err(|e| s3_error!(InternalError, "write file failed: {e}"))?;
}
BUCKET_SSECONFIG => {
let config = match metadata_sys::get_sse_config(&bucket.name).await {
Ok((res, _)) => res,
Err(e) => {
if e == StorageError::ConfigNotFound {
continue;
}
return Err(s3_error!(InternalError, "get bucket metadata failed: {e}"));
}
};
let raw_config = metadata_sys::get(&bucket.name)
.await
.map_err(|e| export_internal_error(format!("get bucket metadata failed: {e}")))?
.encryption_config_xml
.clone();
let config_xml = checked_raw_xml(&config, raw_config, deserialize::<ServerSideEncryptionConfiguration>)?;
zip_writer
.start_file(conf_path, SimpleFileOptions::default())
.map_err(|e| s3_error!(InternalError, "start file failed: {e}"))?;
zip_writer
.write_all(&config_xml)
.map_err(|e| s3_error!(InternalError, "write file failed: {e}"))?;
}
BUCKET_VERSIONING_CONFIG => {
let config = match metadata_sys::get_versioning_config(&bucket.name).await {
Ok((res, _)) => res,
Err(e) => {
if e == StorageError::ConfigNotFound {
continue;
}
return Err(s3_error!(InternalError, "get bucket metadata failed: {e}"));
}
};
let raw_config = metadata_sys::get(&bucket.name)
.await
.map_err(|e| export_internal_error(format!("get bucket metadata failed: {e}")))?
.versioning_config_xml
.clone();
let config_xml = checked_versioning_xml(&config, raw_config)?;
zip_writer
.start_file(conf_path, SimpleFileOptions::default())
.map_err(|e| s3_error!(InternalError, "start file failed: {e}"))?;
zip_writer
.write_all(&config_xml)
.map_err(|e| s3_error!(InternalError, "write file failed: {e}"))?;
}
BUCKET_REPLICATION_CONFIG => {
let config = match metadata_sys::get_replication_config(&bucket.name).await {
Ok((res, _)) => res,
Err(e) => {
if e == StorageError::ConfigNotFound {
continue;
}
return Err(s3_error!(InternalError, "get bucket metadata failed: {e}"));
}
};
let raw_config = metadata_sys::get(&bucket.name)
.await
.map_err(|e| export_internal_error(format!("get bucket metadata failed: {e}")))?
.replication_config_xml
.clone();
let config_xml = checked_raw_xml(&config, raw_config, deserialize::<ReplicationConfiguration>)?;
zip_writer
.start_file(conf_path, SimpleFileOptions::default())
.map_err(|e| s3_error!(InternalError, "start file failed: {e}"))?;
zip_writer
.write_all(&config_xml)
.map_err(|e| s3_error!(InternalError, "write file failed: {e}"))?;
}
BUCKET_TARGETS_FILE => {
let config: BucketTargets = match metadata_sys::get_bucket_targets_config(&bucket.name).await {
Ok(res) => res,
Err(e) => {
if e == StorageError::ConfigNotFound {
continue;
}
return Err(s3_error!(InternalError, "get bucket metadata failed: {e}"));
}
};
let config_json = serde_json::to_vec(&config.redacted_credentials())
.map_err(|e| s3_error!(InternalError, "serialize config failed: {e}"))?;
zip_writer
.start_file(conf_path, SimpleFileOptions::default())
.map_err(|e| s3_error!(InternalError, "start file failed: {e}"))?;
zip_writer
.write_all(&config_json)
.map_err(|e| s3_error!(InternalError, "write file failed: {e}"))?;
}
_ => {}
}
};
let conf_path = if query.diagnostic {
path_join_buf(&[DIAGNOSTIC_EXPORT_PREFIX, &conf_path])
} else {
conf_path
};
zip_writer
.start_file(conf_path, SimpleFileOptions::default())
.map_err(|e| s3_error!(InternalError, "failed to start archive entry: {e}"))?;
zip_writer
.write_all(&config)
.map_err(|e| s3_error!(InternalError, "failed to write archive entry: {e}"))?;
}
}
if query.diagnostic {
let manifest = serde_json::to_vec(&serde_json::json!({
"version": 1,
"mode": "diagnostic",
"complete": errors.is_empty(),
"errors": errors,
}))
.map_err(|e| s3_error!(InternalError, "failed to serialize diagnostic manifest: {e}"))?;
zip_writer
.start_file(DIAGNOSTIC_EXPORT_MANIFEST, SimpleFileOptions::default())
.map_err(|e| s3_error!(InternalError, "failed to start diagnostic manifest: {e}"))?;
zip_writer
.write_all(&manifest)
.map_err(|e| s3_error!(InternalError, "failed to write diagnostic manifest: {e}"))?;
}
let zip_bytes = zip_writer
.finish()
.map_err(|e| s3_error!(InternalError, "failed to finalize export archive: {e}"))?;
@@ -431,8 +429,17 @@ impl Operation for ExportBucketMetadata {
header.insert(CONTENT_TYPE, "application/zip".parse().expect("valid header value"));
header.insert(
CONTENT_DISPOSITION,
"attachment; filename=bucket-meta.zip".parse().expect("valid header value"),
if query.diagnostic {
"attachment; filename=bucket-meta-diagnostic.zip"
} else {
"attachment; filename=bucket-meta.zip"
}
.parse()
.expect("valid header value"),
);
if query.diagnostic {
header.insert("x-rustfs-bucket-metadata-export", "diagnostic".parse().expect("valid header value"));
}
header.insert(CONTENT_LENGTH, zip_bytes.get_ref().len().to_string().parse().expect("valid header value"));
Ok(S3Response::with_headers((StatusCode::OK, Body::from(zip_bytes.into_inner())), header))
}
@@ -499,6 +506,18 @@ impl Operation for ImportBucketMetadata {
file_contents.push((file_path, content));
}
// Reject the whole archive before creating buckets or writing configs,
// even when the marker is malformed or follows ordinary config entries.
if file_contents.iter().any(|(path, _)| {
path == DIAGNOSTIC_EXPORT_MANIFEST
|| path == DIAGNOSTIC_EXPORT_PREFIX
|| path
.strip_prefix(DIAGNOSTIC_EXPORT_PREFIX)
.is_some_and(|suffix| suffix.starts_with('/'))
}) {
return Err(s3_error!(InvalidRequest, "diagnostic bucket metadata archives cannot be imported"));
}
let durable_quota_import = imported_quota_requires_fleet_proof(&file_contents)?;
let quota_fleet_proof =
if durable_quota_import {
@@ -1419,6 +1438,246 @@ mod backup_zip_compatibility_tests {
assert_eq!(response.output.0, StatusCode::OK);
}
#[tokio::test]
#[serial_test::serial]
async fn diagnostic_export_isolated_errors_and_import_recovers_unreadable_targets() {
const HEALTHY: &str = "diagnostic-healthy";
const UNREADABLE: &str = "diagnostic-unreadable";
const REPLACEMENT_TARGETS: &[u8] = br#"{"targets":[]}"#;
const SECRET: &str = "diagnostic-must-not-expose-this-secret";
let _ = rustfs_credentials::init_global_action_credentials(
Some(ROOT_ACCESS_KEY.to_string()),
Some(ROOT_SECRET_KEY.to_string()),
);
let temp = tempfile::tempdir().expect("create diagnostic export test root");
let env = rustfs_test_utils::TestECStoreEnv::builder()
.base_dir(temp.path())
.disk_count(1)
.build()
.await;
env.make_bucket(HEALTHY, false).await;
env.make_bucket(UNREADABLE, false).await;
rustfs_iam::store::object::ObjectStore::new(Arc::clone(&env.ecstore))
.save_iam_config(serde_json::json!({"version": 1}), format!("{}/format.json", *IAM_CONFIG_PREFIX))
.await
.expect("seed IAM format");
let iam = rustfs_iam::build_iam_sys(Arc::clone(&env.ecstore))
.await
.expect("build test IAM");
publish_test_app_context(Arc::new(AppContext::with_default_interfaces(
Arc::clone(&env.ecstore),
iam,
Arc::new(rustfs_kms::KmsServiceManager::new()),
)));
metadata_sys::update(HEALTHY, BUCKET_VERSIONING_CONFIG, VERSIONING_XML.to_vec())
.await
.expect("seed healthy bucket config");
let minio_blob = hex_simd::decode_to_vec(
include_str!("../../../../crates/ecstore/tests/fixtures/minio/bucket_metadata.blob.hex").trim(),
)
.expect("decode real MinIO metadata fixture");
let minio_metadata = BucketMetadata::unmarshal(&minio_blob[4..]).expect("read MinIO metadata fixture");
let targets_array = format!(r#"[{{"credentials":{{"secretKey":"{SECRET}"}}}}]"#).into_bytes();
for unreadable_targets in [targets_array, minio_metadata.bucket_targets_config_json] {
let mut metadata = metadata_sys::get_config_from_disk(UNREADABLE)
.await
.expect("load bucket before simulating MinIO targets");
let incarnation = metadata.bucket_incarnation_id;
metadata.bucket_targets_config_json = unreadable_targets.clone();
metadata
.save_with_store(Arc::clone(&env.ecstore))
.await
.expect("persist unreadable targets fixture");
crate::storage::storage_api::set_bucket_metadata(UNREADABLE.to_string(), metadata)
.await
.expect("publish unreadable targets fixture");
assert!(metadata_sys::get_bucket_targets_config(UNREADABLE).await.is_err());
let strict_error = ExportBucketMetadata {}
.call(
admin_request(Method::GET, Uri::from_static("/rustfs/admin/v3/export-bucket-metadata"), Vec::new()),
Params::new(),
)
.await
.expect_err("a complete export must fail closed on unreadable targets");
assert_eq!(*strict_error.code(), s3s::S3ErrorCode::InternalError);
let response = ExportBucketMetadata {}
.call(
admin_request(
Method::GET,
Uri::from_static("/rustfs/admin/v3/export-bucket-metadata?diagnostic=true"),
Vec::new(),
),
Params::new(),
)
.await
.expect("one unreadable bucket must not abort diagnostic export");
assert_eq!(response.output.0, StatusCode::OK);
assert_eq!(response.headers["x-rustfs-bucket-metadata-export"], "diagnostic");
assert_eq!(response.headers[CONTENT_DISPOSITION], "attachment; filename=bucket-meta-diagnostic.zip");
let bytes = response.output.1.collect().await.expect("read diagnostic archive").to_bytes();
let mut archive = ZipArchive::new(Cursor::new(&bytes)).expect("open diagnostic archive");
let mut files = HashMap::new();
for index in 0..archive.len() {
let mut file = archive.by_index(index).expect("read diagnostic entry");
let mut content = Vec::new();
file.read_to_end(&mut content).expect("read diagnostic config");
assert!(!content.windows(SECRET.len()).any(|window| window == SECRET.as_bytes()));
assert!(file.name() == DIAGNOSTIC_EXPORT_MANIFEST || file.name().starts_with("_diagnostic/"));
files.insert(file.name().to_string(), content);
}
assert_eq!(files[&format!("_diagnostic/{HEALTHY}/{BUCKET_VERSIONING_CONFIG}")], VERSIONING_XML);
assert!(files.contains_key(&format!("_diagnostic/{UNREADABLE}/{BUCKET_VERSIONING_CONFIG}")));
assert!(!files.contains_key(&format!("_diagnostic/{UNREADABLE}/{BUCKET_TARGETS_FILE}")));
let manifest: serde_json::Value =
serde_json::from_slice(&files[DIAGNOSTIC_EXPORT_MANIFEST]).expect("decode diagnostic manifest");
assert_eq!(
manifest,
serde_json::json!({
"version": 1,
"mode": "diagnostic",
"complete": false,
"errors": [{ "bucket": UNREADABLE, "config": BUCKET_TARGETS_FILE, "code": "configuration_unavailable" }],
})
);
let error = ImportBucketMetadata {}
.call(
admin_request(Method::PUT, Uri::from_static("/rustfs/admin/v3/import-bucket-metadata"), bytes.to_vec()),
Params::new(),
)
.await
.expect_err("diagnostic exports are never backups");
assert_eq!(*error.code(), s3s::S3ErrorCode::InvalidRequest);
assert_eq!(
metadata_sys::get_config_from_disk(UNREADABLE)
.await
.expect("load rejected import state")
.bucket_targets_config_json,
unreadable_targets
);
import_archive(zip_with_entries(UNREADABLE, &[(BUCKET_TARGETS_FILE, REPLACEMENT_TARGETS)])).await;
let recovered = metadata_sys::get_config_from_disk(UNREADABLE)
.await
.expect("read recovered targets from disk");
assert_eq!(recovered.bucket_incarnation_id, incarnation);
assert_eq!(recovered.bucket_targets_config_json, REPLACEMENT_TARGETS);
assert!(
metadata_sys::get_bucket_targets_config(UNREADABLE)
.await
.expect("existing import API must recover targets readers")
.is_empty()
);
}
// The marker may be malformed, come last, or be removed while the
// reserved directory remains. None may allow an earlier config write.
for marker in [
DIAGNOSTIC_EXPORT_MANIFEST.to_string(),
DIAGNOSTIC_EXPORT_PREFIX.to_string(),
format!("{DIAGNOSTIC_EXPORT_PREFIX}/bucket/config"),
] {
let mut writer = ZipWriter::new(Cursor::new(Vec::new()));
writer
.start_file(format!("{HEALTHY}/{BUCKET_VERSIONING_CONFIG}"), SimpleFileOptions::default())
.expect("start ordinary config before diagnostic marker");
writer
.write_all(b"<VersioningConfiguration><Status>Suspended</Status></VersioningConfiguration>")
.expect("write ordinary config before diagnostic marker");
writer
.start_file(
format!("diagnostic-never-created/{BUCKET_VERSIONING_CONFIG}"),
SimpleFileOptions::default(),
)
.expect("start a nonexistent bucket config before diagnostic marker");
writer.write_all(VERSIONING_XML).expect("write nonexistent bucket config");
writer
.start_file(marker, SimpleFileOptions::default())
.expect("start diagnostic marker");
writer.write_all(b"not json").expect("write malformed diagnostic marker");
let error = ImportBucketMetadata {}
.call(
admin_request(
Method::PUT,
Uri::from_static("/rustfs/admin/v3/import-bucket-metadata"),
writer.finish().expect("finish marked archive").into_inner(),
),
Params::new(),
)
.await
.expect_err("diagnostic preflight must reject before any config write");
assert_eq!(*error.code(), s3s::S3ErrorCode::InvalidRequest);
assert_eq!(
metadata_sys::get_config_from_disk(HEALTHY)
.await
.expect("read healthy config after rejected import")
.versioning_config_xml,
VERSIONING_XML
);
assert!(
env.ecstore
.get_bucket_info("diagnostic-never-created", &BucketOptions::default())
.await
.is_err(),
"diagnostic preflight must reject before bucket creation"
);
}
let response = ExportBucketMetadata {}
.call(
admin_request(
Method::GET,
Uri::from_static("/rustfs/admin/v3/export-bucket-metadata?diagnostic=true"),
Vec::new(),
),
Params::new(),
)
.await
.expect("diagnostic export after recovery");
let bytes = response
.output
.1
.collect()
.await
.expect("read complete diagnostic archive")
.to_bytes();
let mut archive = ZipArchive::new(Cursor::new(&bytes)).expect("open complete diagnostic archive");
let manifest: serde_json::Value = serde_json::from_reader(
archive
.by_name(DIAGNOSTIC_EXPORT_MANIFEST)
.expect("complete diagnostic manifest"),
)
.expect("parse complete diagnostic manifest");
assert_eq!(manifest["complete"], true);
let error = ImportBucketMetadata {}
.call(
admin_request(Method::PUT, Uri::from_static("/rustfs/admin/v3/import-bucket-metadata"), bytes.to_vec()),
Params::new(),
)
.await
.expect_err("complete diagnostics must still reject import");
assert_eq!(*error.code(), s3s::S3ErrorCode::InvalidRequest);
let response = ExportBucketMetadata {}
.call(
admin_request(Method::GET, Uri::from_static("/rustfs/admin/v3/export-bucket-metadata"), Vec::new()),
Params::new(),
)
.await
.expect("ordinary cluster export must work after API recovery");
assert!(!response.headers.contains_key("x-rustfs-bucket-metadata-export"));
assert_eq!(response.headers[CONTENT_DISPOSITION], "attachment; filename=bucket-meta.zip");
let bytes = response.output.1.collect().await.expect("read ordinary archive").to_bytes();
let mut archive = ZipArchive::new(Cursor::new(bytes)).expect("open ordinary archive");
assert!(archive.by_name(DIAGNOSTIC_EXPORT_MANIFEST).is_err());
assert!(archive.by_name(&format!("{HEALTHY}/{BUCKET_VERSIONING_CONFIG}")).is_ok());
assert!(archive.by_name(&format!("{UNREADABLE}/{BUCKET_TARGETS_FILE}")).is_ok());
}
#[tokio::test]
#[serial_test::serial]
async fn g_zip_001_002_003_use_real_admin_archive_and_persistence_paths() {