From e279a4f48a6bc74f23cfffe8431249bb5504c4bc Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Fri, 17 Jul 2026 19:00:31 +0800 Subject: [PATCH] fix(extract): treat tar mtime=0 as unset to fix unreadable entries (#4948) An extracted entry whose tar header mtime is 0 was stored with mod_time = UNIX_EPOCH, which xl.meta encodes as 0 nanos (= no mod_time). On read-back the version failed valid() and parsing fell into the legacy rmp_serde fallback, so every read of the object returned 500 (invalid type: integer 0, expected an OffsetDateTime). Treat mtime 0 as unset (tar convention) and fall back to the upload time. Also fix two test-side issues uncovered behind the 500: the pax fixture must use a ustar header for its XHeader entry, and the SSE-S3 extract tests must provision RUSTFS_SSE_S3_MASTER_KEY. Re-admit the 19 quarantined tests to the e2e-full merge gate. Fixes #4842 --- .config/nextest.toml | 4 ---- crates/e2e_test/src/multipart_auth_test.rs | 14 +++++++++++--- rustfs/src/app/object_usecase.rs | 4 ++++ 3 files changed, 15 insertions(+), 7 deletions(-) diff --git a/.config/nextest.toml b/.config/nextest.toml index 42fa9a90c..8f51a1fdb 100644 --- a/.config/nextest.toml +++ b/.config/nextest.toml @@ -286,8 +286,6 @@ path = "junit.xml" # ci-profile quarantine (docs/testing/README.md): every entry MUST cite one # OPEN issue, and the fixing PR MUST delete the exclusion. The passing # negative-path siblings of each family stay in as regression guards. -# * rustfs#4842 — extract/snowball expand pipeline 500s (mtime=0 -# OffsetDateTime deserialization + same-path failures). # * rustfs#4843 — over-limit archive entry paths hard-reject the whole # archive even under ignore-errors semantics. # * rustfs#4844 — anonymous POST-object with SSE-S3 / bucket-default SSE @@ -303,8 +301,6 @@ default-filter = """ & !test(/^protocols::/) & !test(/^(admin_timeout_regression_test|cluster_concurrency_test|cluster_multidrive_pool_test|heal_erasure_disk_rebuild_test|namespace_lock_quorum_test|object_lambda_test|stale_multipart_cleanup_cluster_test)::/) & !test(/^replication_extension_test::/) - & !test(/^multipart_auth_test::test_signed_put_object_extract_(accepts_compat_header|expands_tar_entries_with_prefix_headers|expands_tar_gz_archive|expands_tbz2_archive|expands_tgz_archive|expands_txz_archive|expands_tzst_archive|normalizes_prefix_header_value|preserves_directory_markers_by_default|preserves_object_lock_legal_hold|preserves_object_lock_retention|preserves_pax_metadata_and_version_id|preserves_request_metadata_on_extracted_objects|preserves_sse_c|preserves_sse_s3_and_redirect|preserves_storage_class|uses_bucket_default_sse_s3)$/) - & !test(/^snowball_auto_extract_test::tests::snowball_auto_extract_(prefers_exact_minio_prefix_over_suffix_fallback|supports_minio_prefix_and_directory_markers)$/) & !test(/^multipart_auth_test::test_signed_put_object_extract_skips_invalid_entry_when_ignore_errors_enabled$/) & !test(/^snowball_auto_extract_test::tests::snowball_auto_extract_(ignores_invalid_entries_when_requested|supports_standard_headers_with_combined_extract_options)$/) & !test(/^multipart_auth_test::test_anonymous_post_object_(accepts_sse_s3|rejects_sse_s3_missing_from_policy_conditions|uses_bucket_default_sse_kms|uses_bucket_default_sse_s3)$/) diff --git a/crates/e2e_test/src/multipart_auth_test.rs b/crates/e2e_test/src/multipart_auth_test.rs index 4117ca43c..0cc4eb326 100644 --- a/crates/e2e_test/src/multipart_auth_test.rs +++ b/crates/e2e_test/src/multipart_auth_test.rs @@ -105,7 +105,11 @@ async fn make_tar_with_pax_entry(path: &str, data: &[u8], mtime: Option, pa pax_payload.extend(build_pax_record(key, value)); } - let mut pax_header = tokio_tar::Header::new_gnu(); + // Pax extension entries must carry a POSIX ustar header — this is what real + // tar writers emit, and the server-side reader rejects an XHeader typeflag on + // GNU-format headers ("extension typeflag is not permitted on an unrecognized + // header"). + let mut pax_header = tokio_tar::Header::new_ustar(); pax_header.set_entry_type(tokio_tar::EntryType::XHeader); pax_header.set_size(pax_payload.len() as u64); pax_header.set_mode(0o644); @@ -5041,7 +5045,9 @@ async fn test_signed_put_object_extract_preserves_sse_s3_and_redirect() -> Resul init_logging(); let mut env = RustFSTestEnvironment::new().await?; - env.start_rustfs_server(vec![]).await?; + let sse_master_key = base64::engine::general_purpose::STANDARD.encode([0x42u8; 32]); + env.start_rustfs_server_with_env(vec![], &[("RUSTFS_SSE_S3_MASTER_KEY", sse_master_key.as_str())]) + .await?; let bucket = "signed-extract-sse-s3-redirect"; let archive_key = "encrypted-metadata.tar"; @@ -5318,7 +5324,9 @@ async fn test_signed_put_object_extract_uses_bucket_default_sse_s3() -> Result<( init_logging(); let mut env = RustFSTestEnvironment::new().await?; - env.start_rustfs_server(vec![]).await?; + let sse_master_key = base64::engine::general_purpose::STANDARD.encode([0x42u8; 32]); + env.start_rustfs_server_with_env(vec![], &[("RUSTFS_SSE_S3_MASTER_KEY", sse_master_key.as_str())]) + .await?; let bucket = "signed-extract-default-sse-s3"; let archive_key = "default-encryption.tar"; diff --git a/rustfs/src/app/object_usecase.rs b/rustfs/src/app/object_usecase.rs index a783c68ce..199120ef0 100644 --- a/rustfs/src/app/object_usecase.rs +++ b/rustfs/src/app/object_usecase.rs @@ -6872,10 +6872,14 @@ impl DefaultObjectUsecase { } let mut size = i64::try_from(entry_size).map_err(|_| s3_error!(InvalidArgument, "Archive entry size does not fit into i64"))?; + // mtime 0 means "unset" in tar headers, and xl.meta cannot represent an + // epoch mod_time anyway (0 nanos decodes as no-mod_time, making the version + // unreadable — rustfs#4842), so fall back to the upload time instead. let archive_entry_mod_time = f .header() .mtime() .ok() + .filter(|&modified_at_secs| modified_at_secs > 0) .and_then(|modified_at_secs| OffsetDateTime::from_unix_timestamp(modified_at_secs as i64).ok()); let mut metadata = HashMap::new(); let has_explicit_object_lock_retention = object_lock_mode.is_some() || object_lock_retain_until_date.is_some();