Compare commits

..

6 Commits

Author SHA1 Message Date
Hauser Zhi 212cb2a47e refactor(scanner): scope heal inventory to coordinator workers
Build the inventory only for coordinator-owned cross-drive healing and expose the regression marker directly through the scan storage domain.

Co-Authored-By: heihutu <heihutu@gmail.com>
Co-Authored-By: zhi22915 <qiuzgang@gmail.com>
2026-11-20 01:24:53 +08:00
houseme 3c0a0cf7f0 fix(scanner): use scan domain for test marker import
Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com>
2026-11-19 16:30:48 +00:00
Hauser Zhi 7f3ea0c73c perf(scanner): skip unused heal disk inventory
Avoid building and cloning the cross-drive inventory for single-drive scans and scans with healing suppressed. Exercise durable usage publication and cursor advancement through the runtime cycle.

Co-Authored-By: heihutu <heihutu@gmail.com>
Co-Authored-By: zhi22915 <qiuzgang@gmail.com>
2026-11-20 00:24:56 +08:00
houseme f9cafcb8a2 fix(scanner): route healing marker through storage boundary
Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com>
2026-11-19 15:58:58 +00:00
houseme 156defd1f1 fix(scanner): scan online single drives marked as healing
Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com>
2026-11-19 14:45:52 +00:00
houseme 4398882cc9 Initial plan 2026-11-19 14:37:06 +00:00
53 changed files with 5418 additions and 5553 deletions
+2 -2
View File
@@ -143,7 +143,7 @@ setup = 'ecstore-base-stack'
# The multipart refusal fixture composes real-store setup, abort, quota and legal-hold futures.
[[profile.default.scripts]]
filter = 'package(rustfs-ecstore) | package(rustfs-s3select-api) | package(rustfs-scanner) | (package(rustfs) & test(/^(app::multipart_usecase::complete::tests::completion_absent_binding_refusal_order_baseline|app::multipart_usecase::complete::tests::concurrent_completions_share_durable_bucket_quota_reservations|app::object::delete::tests::compressed_delete_requests_update_observed_usage_without_releasing_quota_floor|app::object::internal_put::tests::internal_multipart_roundtrip_completes_and_abort_leaves_nothing|app::object::restore::tests::execute_restore_object_maps_failures_to_typed_s3_errors|storage::access_owned::tests::generation_guards::(delete_object_access_captures_authorized_bucket_incarnation|copy_operations_reject_recreated_source_bucket_after_authorization|request_slot_keeps_bucket_policy_bound_to_its_store))$/))'
filter = 'package(rustfs-ecstore) | package(rustfs-s3select-api) | package(rustfs-scanner) | (package(rustfs) & test(/^(app::multipart_usecase::complete::tests::completion_absent_binding_refusal_order_baseline|app::multipart_usecase::complete::tests::concurrent_completions_share_durable_bucket_quota_reservations|app::object::delete::tests::compressed_delete_requests_update_observed_usage_without_releasing_quota_floor|app::object::internal_put::tests::internal_multipart_roundtrip_completes_and_abort_leaves_nothing|app::object::restore::tests::execute_restore_object_maps_failures_to_typed_s3_errors|storage::access::tests::(delete_object_access_captures_authorized_bucket_incarnation|copy_operations_reject_recreated_source_bucket_after_authorization|request_slot_keeps_bucket_policy_bound_to_its_store))$/))'
setup = 'ecstore-base-stack'
# These handler/store fixtures and the relocated-pool GET compose deep storage
@@ -466,7 +466,7 @@ filter = '(package(rustfs) & test(/^(admin::handlers::replication::target_repair
setup = 'ecstore-base-stack'
[[profile.ci.scripts]]
filter = 'package(rustfs-ecstore) | package(rustfs-s3select-api) | package(rustfs-scanner) | (package(rustfs) & test(/^(app::multipart_usecase::complete::tests::completion_absent_binding_refusal_order_baseline|app::multipart_usecase::complete::tests::concurrent_completions_share_durable_bucket_quota_reservations|app::object::delete::tests::compressed_delete_requests_update_observed_usage_without_releasing_quota_floor|app::object::internal_put::tests::internal_multipart_roundtrip_completes_and_abort_leaves_nothing|app::object::restore::tests::execute_restore_object_maps_failures_to_typed_s3_errors|storage::access_owned::tests::generation_guards::(delete_object_access_captures_authorized_bucket_incarnation|copy_operations_reject_recreated_source_bucket_after_authorization|request_slot_keeps_bucket_policy_bound_to_its_store))$/))'
filter = 'package(rustfs-ecstore) | package(rustfs-s3select-api) | package(rustfs-scanner) | (package(rustfs) & test(/^(app::multipart_usecase::complete::tests::completion_absent_binding_refusal_order_baseline|app::multipart_usecase::complete::tests::concurrent_completions_share_durable_bucket_quota_reservations|app::object::delete::tests::compressed_delete_requests_update_observed_usage_without_releasing_quota_floor|app::object::internal_put::tests::internal_multipart_roundtrip_completes_and_abort_leaves_nothing|app::object::restore::tests::execute_restore_object_maps_failures_to_typed_s3_errors|storage::access::tests::(delete_object_access_captures_authorized_bucket_incarnation|copy_operations_reject_recreated_source_bucket_after_authorization|request_slot_keeps_bucket_policy_bound_to_its_store))$/))'
setup = 'ecstore-base-stack'
[[profile.ci.scripts]]
+2 -2
View File
@@ -13,6 +13,6 @@
# s3_error_lines lines holding s3_error!, same scope, minus those the script
# header's rule clears (rustfs_s3_types's macro); fails closed
# ecstore_files Rust files referencing s3s paths under crates/ecstore/src
files=49
s3_error_lines=95
files=50
s3_error_lines=103
ecstore_files=0
+88
View File
@@ -1635,6 +1635,94 @@ async fn run_data_scanner_cycle_publishes_activity_for_owner_lifetime() {
global_metrics().set_cycle(None).await;
}
#[tokio::test]
#[serial]
async fn single_drive_cycle_persists_usage_and_advances_with_healing_marker() {
use crate::scanner_io::{clear_dirty_usage_buckets_for_tests, tests::wait_for_namespace_commit_tails};
use crate::storage_api::scan::{HEALING_MARKER_PATH, MakeBucketOptions};
clear_dirty_usage_buckets_for_tests();
let temp_dir = tempfile::tempdir().expect("scanner cycle test directory should be created");
let store = setup_scanner_cycle_store_at_path_with_layout(temp_dir.path(), true, 1, 1, 1).await;
let bucket = "marked-drive-bucket";
store
.make_bucket(bucket, &MakeBucketOptions::default())
.await
.expect("fixture bucket should be created");
let mut reader = PutObjReader::from_vec(vec![7; 16]);
store
.put_object(bucket, "object", &mut reader, &ObjectOptions::default())
.await
.expect("fixture object should persist");
wait_for_namespace_commit_tails(store.as_ref()).await;
let marker = temp_dir
.path()
.join("pool0/disk0")
.join(RUSTFS_META_BUCKET)
.join(HEALING_MARKER_PATH);
let marker_contents = b"leftover marker";
tokio::fs::write(&marker, marker_contents)
.await
.expect("fixture healing marker should persist");
let set = &store.pools[0].disk_set[0];
tokio::time::timeout(Duration::from_secs(10), async {
while !set.get_online_disks_with_healing(true).await.1 {
tokio::time::sleep(Duration::from_millis(100)).await;
}
})
.await
.expect("disk info should observe the marker after its cache expires");
let ctx = CancellationToken::new();
let mut cycle_info = CurrentCycle {
next: 1,
..Default::default()
};
let mut revision = DataUsageCacheRevision::Missing;
let leader_epoch = 11;
let outcome = tokio::time::timeout(
Duration::from_secs(60),
run_data_scanner_cycle(&ctx, &store, &mut cycle_info, &mut revision, leader_epoch),
)
.await
.expect("scanner cycle should finish");
assert!(
matches!(
outcome,
ScannerCycleOutcome::Completed | ScannerCycleOutcome::CompletedWithPendingMaintenance
),
"unexpected cycle outcome: {outcome:?}",
);
let mut set_snapshot = DataUsageCache::default();
set_snapshot
.load_with_revisions(Arc::clone(set), DATA_USAGE_CACHE_NAME)
.await
.expect("set usage snapshot should be readable");
assert!(set_snapshot.info.snapshot_complete);
assert_eq!(set_snapshot.info.next_cycle, 1);
let usage = read_config(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str())
.await
.expect("global usage snapshot should be durable");
let usage: DataUsageInfo = serde_json::from_slice(&usage).expect("global usage snapshot should decode");
assert!(usage.usage_snapshot_complete);
assert!(usage.last_update.is_some());
assert_eq!(usage.buckets_usage[bucket].objects_count, 1);
let persisted_cycle = read_config(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str())
.await
.expect("scanner cycle cursor should be durable");
let (persisted_cycle, persisted_epoch) =
decode_scanner_cycle_state(&persisted_cycle).expect("scanner cycle cursor should decode");
assert_eq!(persisted_epoch, leader_epoch);
assert_eq!(persisted_cycle.next, 2);
assert_eq!(tokio::fs::read(marker).await.expect("healing marker should remain"), marker_contents);
clear_dirty_usage_buckets_for_tests();
global_metrics().set_cycle(None).await;
}
#[tokio::test]
#[serial]
async fn coordinator_publishes_authority_during_pending_put() {
+20 -7
View File
@@ -620,7 +620,10 @@ impl ScannerIOCache for SetDisks {
.ok_or_else(|| StorageError::other("failed to persist empty scanner set scope"));
}
let (disks, healing) = self.get_online_disks_with_healing(false).await;
let single_drive = self.set_drive_count == 1;
// A single-drive set has no alternate scan source. A heal marker
// still disables scanner heals, but must not exclude its only usage walk.
let (disks, healing) = self.get_online_disks_with_healing(single_drive).await;
#[cfg(test)]
let disks = super::tests::usage_remote_drain::walker_disks(&self, &buckets, disks);
if disks.is_empty() {
@@ -757,7 +760,18 @@ impl ScannerIOCache for SetDisks {
let _ = updates.send(incomplete_scope).await;
return Ok(ScannerSourceTurnReceipt::default());
}
let set_disk_inventory = Arc::new(scanner_set_disk_inventory(self.as_ref()).await);
// Coordinator workers need the full inventory only for cross-drive
// healing. Remote workers own their disk inventory on the peer.
let heal_disk_inventory = if !single_drive
&& !healing
&& workers
.iter()
.any(|(_, mode)| *mode == NamespaceScannerWorkerMode::Coordinator)
{
Some(Arc::new(scanner_set_disk_inventory(self.as_ref()).await))
} else {
None
};
if unsupported_remote_disks > 0 {
debug!(
target: "rustfs::scanner::io",
@@ -1010,7 +1024,6 @@ impl ScannerIOCache for SetDisks {
let store_clone_clone = self.clone();
let bucket_result_tx_clone = bucket_result_tx.clone();
let bucket_observations_clone = bucket_observations.clone();
let set_disk_inventory_clone = set_disk_inventory.clone();
let disk_scan_semaphore_clone = disk_scan_semaphore.clone();
let queued_disk_bucket_scans_clone = queued_disk_bucket_scans.clone();
let active_disk_bucket_scans_clone = active_disk_bucket_scans.clone();
@@ -1026,9 +1039,9 @@ impl ScannerIOCache for SetDisks {
let worker_retired_buckets = retired_buckets.clone();
#[cfg(feature = "e2e-test-hooks")]
let worker_incarnations = current_bucket_incarnations.clone();
let remote_capabilities = match worker_mode {
NamespaceScannerWorkerMode::RemoteV4(capabilities) => Some(capabilities),
NamespaceScannerWorkerMode::Coordinator => None,
let (remote_capabilities, heal_disk_inventory) = match worker_mode {
NamespaceScannerWorkerMode::Coordinator => (None, heal_disk_inventory.clone()),
NamespaceScannerWorkerMode::RemoteV4(capabilities) => (Some(capabilities), None),
};
#[cfg(test)]
let worker_index = futs.len();
@@ -1671,7 +1684,7 @@ impl ScannerIOCache for SetDisks {
let scan = crate::workload_admission::with_scanner_node_requests(node_tx, disk.clone().nsscanner_disk(
scan_ctx.clone(),
bucket_budget.clone(),
set_disk_inventory_clone.as_ref().clone(),
heal_disk_inventory.as_deref().cloned().unwrap_or_default(),
cache,
None,
ScannerDiskScanOptions {
+2
View File
@@ -422,6 +422,8 @@ pub(crate) mod scan {
#[cfg(test)]
pub(crate) use rustfs_ecstore::api::bucket::metadata_sys::acquire_scanner_bucket_incarnation_fence as ecstore_scanner_ack_fence;
#[cfg(test)]
pub(crate) use rustfs_ecstore::api::disk::HEALING_MARKER_PATH;
#[cfg(test)]
pub(crate) use rustfs_ecstore::api::notification::new_global_notification_sys;
}
@@ -79,12 +79,3 @@ mod object_lock;
mod put;
mod restore;
mod transition;
/// PutObject through its test seam, from a test's s3s fixture request.
async fn put_through_seam(
usecase: &DefaultObjectUsecase,
req: S3Request<PutObjectInput>,
) -> rustfs_s3_types::S3Result<crate::app::object_usecase::PutObjectWriteOutput> {
let (envelope, input) = crate::app::object_usecase::PutObjectWriteInput::from_test_request(req);
crate::app::test_seams::put_object(usecase, envelope, input).await
}
@@ -16,6 +16,7 @@
use super::common::*;
use super::*;
use crate::app::test_seams;
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
#[serial]
@@ -69,7 +70,7 @@ async fn copy_object_if_none_match_existing_destination_returns_precondition_fai
let mut req = build_request(copy_input, Method::PUT);
req.headers.insert(IF_NONE_MATCH, header.clone());
let err = Box::pin(copy_owned(&usecase, req))
let err = Box::pin(usecase.execute_copy_object(req))
.await
.expect_err("conditional COPY must reject the replacement");
assert_eq!(err.code(), &expected_code, "header={header:?}, has_dto={has_dto}");
@@ -92,6 +93,7 @@ async fn copy_object_if_none_match_existing_destination_returns_precondition_fai
#[ignore = "global-state object-version integration test: runs serialized in the CI ILM Integration (serial) lane"]
async fn undo_copy_requires_the_observed_current_version() {
let (_disk_paths, ecstore) = setup_test_env().await;
let fs = FS::new();
let usecase = DefaultObjectUsecase::from_global();
let bucket = format!("test-undo-copy-{}", &Uuid::new_v4().simple().to_string()[..8]);
let object = "test/object.txt";
@@ -106,14 +108,16 @@ async fn undo_copy_requires_the_observed_current_version() {
.build()
.unwrap()
};
let original = Box::pin(put_through_seam(&usecase, build_request(put(b"original"), Method::PUT)))
let original = Box::pin(test_seams::put_object(&usecase, &fs, build_request(put(b"original"), Method::PUT)))
.await
.expect("initial version should be written")
.output
.version_id
.expect("versioned PUT should return a version ID");
let observed_current = Box::pin(put_through_seam(&usecase, build_request(put(b"replacement"), Method::PUT)))
let observed_current = Box::pin(test_seams::put_object(&usecase, &fs, build_request(put(b"replacement"), Method::PUT)))
.await
.expect("replacement version should be written")
.output
.version_id
.expect("versioned PUT should return a version ID");
@@ -135,7 +139,7 @@ async fn undo_copy_requires_the_observed_current_version() {
HeaderValue::from_str(&Uuid::new_v4().to_string()).unwrap(),
);
let versions_before = live_object_version_count(&ecstore, bucket.as_str(), object).await;
let err = Box::pin(copy_owned(&usecase, stale_req))
let err = Box::pin(usecase.execute_copy_object(stale_req))
.await
.expect_err("stale destination must fail");
assert_eq!(err.code(), &S3ErrorCode::PreconditionFailed);
@@ -146,7 +150,7 @@ async fn undo_copy_requires_the_observed_current_version() {
matching_req
.headers
.insert(RUSTFS_EXPECTED_CURRENT_VERSION_ID, HeaderValue::from_str(&observed_current).unwrap());
Box::pin(copy_owned(&usecase, matching_req))
Box::pin(usecase.execute_copy_object(matching_req))
.await
.expect("matching destination version should permit undo copy");
assert_eq!(read_object_bytes(&ecstore, bucket.as_str(), object).await, b"original");
@@ -171,15 +175,18 @@ async fn undo_copy_serializes_a_concurrent_put_until_after_commit() {
.build()
.unwrap()
};
let fs = FS::new();
let usecase = DefaultObjectUsecase::from_global();
let original = Box::pin(put_through_seam(&usecase, build_request(put(b"original"), Method::PUT)))
let original = Box::pin(test_seams::put_object(&usecase, &fs, build_request(put(b"original"), Method::PUT)))
.await
.expect("original should be written")
.output
.version_id
.expect("original should be versioned");
let observed = Box::pin(put_through_seam(&usecase, build_request(put(b"observed"), Method::PUT)))
let observed = Box::pin(test_seams::put_object(&usecase, &fs, build_request(put(b"observed"), Method::PUT)))
.await
.expect("observed should be written")
.output
.version_id
.expect("observed should be versioned");
let encoded_object = encode_dir_object(object);
@@ -211,7 +218,7 @@ async fn undo_copy_serializes_a_concurrent_put_until_after_commit() {
let undo = tokio::spawn(async move {
let usecase = DefaultObjectUsecase::from_global();
Box::pin(copy_owned(&usecase, undo_req)).await
Box::pin(usecase.execute_copy_object(undo_req)).await
});
tokio::task::yield_now().await;
let put = PutObjectInput::builder()
@@ -222,8 +229,9 @@ async fn undo_copy_serializes_a_concurrent_put_until_after_commit() {
.build()
.unwrap();
let concurrent_put = tokio::spawn(async move {
let fs = FS::new();
let usecase = DefaultObjectUsecase::from_global();
Box::pin(put_through_seam(&usecase, build_request(put, Method::PUT))).await
Box::pin(test_seams::put_object(&usecase, &fs, build_request(put, Method::PUT))).await
});
drop(blocker);
@@ -241,6 +249,7 @@ async fn undo_copy_serializes_a_concurrent_put_until_after_commit() {
#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-1)"]
async fn copy_object_allows_new_version_for_locked_destination_but_blocks_explicit_overwrite() {
let (_disk_paths, ecstore) = setup_test_env().await;
let fs = FS::new();
let usecase = DefaultObjectUsecase::from_global();
let bucket = format!("test-copy-object-lock-{}", &Uuid::new_v4().simple().to_string()[..8]);
@@ -266,7 +275,7 @@ async fn copy_object_allows_new_version_for_locked_destination_but_blocks_explic
.build()
.unwrap();
Box::pin(put_through_seam(&usecase, build_request(locked_input, Method::PUT)))
Box::pin(test_seams::put_object(&usecase, &fs, build_request(locked_input, Method::PUT)))
.await
.expect("Failed to upload locked destination object");
@@ -290,10 +299,11 @@ async fn copy_object_allows_new_version_for_locked_destination_but_blocks_explic
.build()
.unwrap();
let copy_output = Box::pin(copy_owned(&usecase, build_request(copy_input, Method::PUT)))
let copy_output = Box::pin(usecase.execute_copy_object(build_request(copy_input, Method::PUT)))
.await
.expect("CopyObject should create a new version over a locked current version");
let copied_version_id = copy_output
.output
.version_id
.expect("versioned CopyObject should return the created version ID");
@@ -313,7 +323,7 @@ async fn copy_object_allows_new_version_for_locked_destination_but_blocks_explic
.build()
.unwrap();
let err = Box::pin(copy_owned(&usecase, build_request(explicit_overwrite_input, Method::PUT)))
let err = Box::pin(usecase.execute_copy_object(build_request(explicit_overwrite_input, Method::PUT)))
.await
.expect_err("explicit CopyObject overwrite of a locked version should be blocked");
@@ -369,13 +379,13 @@ async fn concurrent_reverse_copy_object_does_not_deadlock_with_reader_locks() {
let a_to_b = tokio::spawn(async move {
start_a.wait().await;
let usecase = DefaultObjectUsecase::from_global();
Box::pin(copy_owned(&usecase, build_request(a_to_b_input, Method::PUT))).await
Box::pin(usecase.execute_copy_object(build_request(a_to_b_input, Method::PUT))).await
});
let start_b = start.clone();
let b_to_a = tokio::spawn(async move {
start_b.wait().await;
let usecase = DefaultObjectUsecase::from_global();
Box::pin(copy_owned(&usecase, build_request(b_to_a_input, Method::PUT))).await
Box::pin(usecase.execute_copy_object(build_request(b_to_a_input, Method::PUT))).await
});
start.wait().await;
@@ -412,6 +422,7 @@ async fn concurrent_reverse_copy_object_does_not_deadlock_with_reader_locks() {
#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-1)"]
async fn put_and_copy_object_transition_immediately_via_usecases() {
let (_disk_paths, ecstore) = setup_test_env().await;
let fs = FS::new();
let usecase = DefaultObjectUsecase::from_global();
let tier_name = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase();
@@ -434,7 +445,7 @@ async fn put_and_copy_object_transition_immediately_via_usecases() {
.build()
.unwrap();
Box::pin(put_through_seam(&usecase, build_request(put_input, Method::PUT)))
Box::pin(test_seams::put_object(&usecase, &fs, build_request(put_input, Method::PUT)))
.await
.expect("Failed to put object through usecase");
@@ -470,7 +481,7 @@ async fn put_and_copy_object_transition_immediately_via_usecases() {
.build()
.unwrap();
Box::pin(copy_owned(&usecase, build_request(copy_input, Method::PUT)))
Box::pin(usecase.execute_copy_object(build_request(copy_input, Method::PUT)))
.await
.expect("Failed to copy object through usecase");
@@ -518,7 +529,7 @@ async fn compensation_driven_copy_still_completes_transition() {
.unwrap();
with_forced_immediate_enqueue_timeout(&backend, || async {
Box::pin(copy_owned(&usecase, build_request(copy_input, Method::PUT)))
Box::pin(usecase.execute_copy_object(build_request(copy_input, Method::PUT)))
.await
.expect("Failed to copy object through usecase");
})
@@ -605,7 +616,7 @@ async fn copy_object_computes_replication_decision_and_strips_stale_status() {
.key(dst_object.to_string())
.build()
.unwrap();
Box::pin(copy_owned(&usecase, build_request(copy_input, Method::PUT)))
Box::pin(usecase.execute_copy_object(build_request(copy_input, Method::PUT)))
.await
.expect("Failed to copy object through usecase");
@@ -641,13 +652,3 @@ async fn copy_object_computes_replication_decision_and_strips_stale_status() {
"destination must not inherit the surfaced x-amz-replication-status header key"
);
}
/// CopyObject through the owned entry point the gateway handler calls, its error in the s3s
/// form these tests compare.
async fn copy_owned(
usecase: &DefaultObjectUsecase,
req: S3Request<CopyObjectInput>,
) -> crate::app::storage_api::s3::S3Result<crate::app::object_usecase::CopyObjectWriteOutput> {
let (envelope, input) = crate::app::object_usecase::CopyObjectWriteInput::from_test_request(req);
usecase.execute_copy_object_owned(envelope, input).await.map_err(Into::into)
}
@@ -43,6 +43,7 @@ async fn wait_for_object_absence(ecstore: &Arc<ECStore>, bucket: &str, object: &
#[ignore = "global-state object-version integration test: runs serialized in the CI ILM Integration (serial) lane"]
async fn undo_delete_only_removes_the_current_matching_delete_marker() {
let (_disk_paths, ecstore) = setup_test_env().await;
let fs = FS::new();
let usecase = DefaultObjectUsecase::from_global();
let bucket = format!("test-undo-delete-{}", &Uuid::new_v4().simple().to_string()[..8]);
let object = "test/object.txt";
@@ -55,7 +56,7 @@ async fn undo_delete_only_removes_the_current_matching_delete_marker() {
.content_length(Some(13))
.build()
.unwrap();
Box::pin(put_through_seam(&usecase, build_request(put, Method::PUT)))
Box::pin(test_seams::put_object(&usecase, &fs, build_request(put, Method::PUT)))
.await
.expect("object version should be written");
let marker = Box::pin(
@@ -22,6 +22,7 @@ use super::*;
#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-1)"]
async fn put_object_if_none_match_existing_object_returns_precondition_failed() {
let (_disk_paths, ecstore) = setup_test_env().await;
let fs = FS::new();
let usecase = DefaultObjectUsecase::from_global();
let bucket = format!("test-put-if-none-match-{}", &Uuid::new_v4().simple().to_string()[..8]);
@@ -38,7 +39,7 @@ async fn put_object_if_none_match_existing_object_returns_precondition_failed()
.content_length(Some(initial_payload.len() as i64))
.build()
.unwrap();
Box::pin(put_owned(&usecase, build_request(initial_input, Method::PUT)))
Box::pin(usecase.execute_put_object(&fs, build_request(initial_input, Method::PUT)))
.await
.expect("Failed to upload initial object through usecase");
@@ -72,7 +73,7 @@ async fn put_object_if_none_match_existing_object_returns_precondition_failed()
let mut req = build_request(replacement_input, Method::PUT);
req.headers.insert(IF_NONE_MATCH, header.clone());
let err = Box::pin(put_owned(&usecase, req))
let err = Box::pin(usecase.execute_put_object(&fs, req))
.await
.expect_err("conditional PUT must reject the replacement");
assert_eq!(err.code(), &expected_code, "header={header:?}, has_dto={has_dto}");
@@ -107,6 +108,7 @@ async fn put_object_computes_replication_decision_exactly_once() {
use std::sync::atomic::Ordering;
let (_disk_paths, ecstore) = setup_test_env().await;
let fs = FS::new();
let usecase = DefaultObjectUsecase::from_global();
let bucket = format!("test-repl-decision-once-{}", &Uuid::new_v4().simple().to_string()[..8]);
@@ -127,7 +129,7 @@ async fn put_object_computes_replication_decision_exactly_once() {
.content_length(Some(payload.len() as i64))
.build()
.unwrap();
Box::pin(put_owned(&usecase, build_request(input, Method::PUT)))
Box::pin(usecase.execute_put_object(&fs, build_request(input, Method::PUT)))
.await
.expect("Failed to upload object through usecase");
@@ -205,7 +207,7 @@ async fn put_object_extract_computes_replication_decision_per_entry() {
.unwrap();
let mut req = build_request(input, Method::PUT);
req.extensions.insert(ReqInfo::default());
Box::pin(put_extract_owned(&usecase, req))
Box::pin(usecase.execute_put_object_extract(req))
.await
.expect("Failed to extract archive through usecase");
@@ -217,26 +219,3 @@ async fn put_object_extract_computes_replication_decision_per_entry() {
PutObjectExtract parity)"
);
}
/// PutObject through the owned entry point the gateway handler calls, its error in the s3s form
/// these tests compare.
async fn put_owned(
usecase: &DefaultObjectUsecase,
req: S3Request<PutObjectInput>,
) -> s3s::S3Result<crate::app::object_usecase::PutObjectWriteOutput> {
let (envelope, input) = crate::app::object_usecase::PutObjectWriteInput::from_test_request(req);
Box::pin(usecase.execute_put_object_owned(envelope, input))
.await
.map_err(Into::into)
}
/// As [`put_owned`], through the archive-extraction entry point.
async fn put_extract_owned(
usecase: &DefaultObjectUsecase,
req: S3Request<PutObjectInput>,
) -> s3s::S3Result<crate::app::object_usecase::PutObjectWriteOutput> {
let (envelope, input) = crate::app::object_usecase::PutObjectWriteInput::from_test_request(req);
Box::pin(usecase.execute_put_object_extract_owned(envelope, input))
.await
.map_err(Into::into)
}
+53 -70
View File
@@ -931,16 +931,6 @@ impl DefaultObjectUsecase {
#[cfg(test)]
mod tests {
use super::*;
/// CopyObject through the owned entry point the gateway handler calls, its error in the s3s
/// form these tests compare.
async fn copy_owned(
usecase: &DefaultObjectUsecase,
req: S3Request<CopyObjectInput>,
) -> crate::app::storage_api::s3::S3Result<crate::app::object_usecase::CopyObjectWriteOutput> {
let (envelope, input) = crate::app::object_usecase::CopyObjectWriteInput::from_test_request(req);
usecase.execute_copy_object_owned(envelope, input).await.map_err(Into::into)
}
use crate::app::storage_api::s3::S3ErrorCode;
use crate::app::storage_api::s3::dto::ServerSideEncryption;
use http::{HeaderValue, Method};
@@ -1071,7 +1061,7 @@ mod tests {
let req = build_request(input, Method::PUT);
let usecase = DefaultObjectUsecase::without_context();
let err = Box::pin(copy_owned(&usecase, req)).await.unwrap_err();
let err = Box::pin(usecase.execute_copy_object(req)).await.unwrap_err();
assert_eq!(err.code(), &S3ErrorCode::InvalidRequest);
}
@@ -1087,7 +1077,8 @@ mod tests {
.key(".rustfs-table/metadata/current.json".to_string())
.build()
.unwrap();
let error = copy_owned(&DefaultObjectUsecase::without_context(), build_request(input, Method::PUT))
let error = DefaultObjectUsecase::without_context()
.execute_copy_object(build_request(input, Method::PUT))
.await
.expect_err("reserved catalog destination must fail closed without bucket metadata");
assert_eq!(error.code(), &S3ErrorCode::InvalidRequest);
@@ -1111,7 +1102,7 @@ mod tests {
let req = build_request(input, Method::PUT);
let usecase = DefaultObjectUsecase::without_context();
let err = Box::pin(copy_owned(&usecase, req)).await.unwrap_err();
let err = Box::pin(usecase.execute_copy_object(req)).await.unwrap_err();
assert_eq!(err.code(), &S3ErrorCode::InvalidStorageClass);
}
@@ -1132,7 +1123,7 @@ mod tests {
let req = build_request(input, Method::PUT);
let usecase = DefaultObjectUsecase::without_context();
let err = Box::pin(copy_owned(&usecase, req)).await.unwrap_err();
let err = Box::pin(usecase.execute_copy_object(req)).await.unwrap_err();
// Self-copy with explicit storage class change must pass the self-copy guard.
assert_ne!(err.code(), &S3ErrorCode::InvalidRequest);
}
@@ -1141,9 +1132,7 @@ mod tests {
#[serial_test::serial]
async fn execute_self_copy_when_object_name_equals_bucket_observes_lock_order() {
use crate::app::storage_api::test::contract::bucket::{BucketOperations as _, DeleteBucketOptions, MakeBucketOptions};
use crate::storage_api::server::http::access::access_owned::object_commit::{
CopyObjectAuthorization, CopySourceObject, authorize_copy_object,
};
use s3s::access::S3Access as _;
let store = crate::app::gating_test_env::shared_gating_ecstore().await;
if current_app_context().is_none() {
@@ -1153,6 +1142,7 @@ mod tests {
let context = Arc::new(AppContext::new(Arc::clone(&store), ambient.iam(), ambient.kms()));
let server_ctx = crate::app::runtime_sources::ServerContextSlot::new();
assert!(server_ctx.install(Arc::clone(&context)));
let fs = FS::with_server_ctx(server_ctx);
let bucket = format!("self-copy-lock-order-{}", Uuid::new_v4());
store
@@ -1197,33 +1187,18 @@ mod tests {
.expect("self-copy input should build");
let mut req = build_request(input, Method::PUT);
req.extensions.insert(crate::storage::req_info::ReqInfo::default());
let (mut envelope, input) = CopyObjectWriteInput::from_test_request(req);
let facts = CopyObjectAuthorization {
bucket: &bucket,
key: &bucket,
source: Some(CopySourceObject {
bucket: &bucket,
key: &bucket,
version_id: None,
}),
query_version: None,
tagging: None,
tagging_directive: None,
retain_until: None,
legal_hold: false,
retention: false,
};
authorize_copy_object(&mut envelope, &server_ctx, facts)
fs.copy_object(&mut req)
.await
.expect("authorize self-copy whose object key equals its bucket");
let usecase = DefaultObjectUsecase::with_context(Some(context));
let response =
tokio::time::timeout(std::time::Duration::from_secs(30), usecase.execute_copy_object_owned(envelope, input))
.await
.expect("lifecycle, authority, and exact object locks must not deadlock")
.expect("self-copy whose object key equals its bucket should succeed");
assert!(response.e_tag.is_some());
let response = tokio::time::timeout(
std::time::Duration::from_secs(30),
DefaultObjectUsecase::with_context(Some(context)).execute_copy_object(req),
)
.await
.expect("lifecycle, authority, and exact object locks must not deadlock")
.expect("self-copy whose object key equals its bucket should succeed");
assert!(response.output.copy_object_result.is_some());
let info = store
.get_object_info(&bucket, &bucket, &ObjectOptions::default())
.await
@@ -1260,7 +1235,7 @@ mod tests {
let req = build_request(input, Method::PUT);
let usecase = DefaultObjectUsecase::without_context();
let err = Box::pin(copy_owned(&usecase, req)).await.unwrap_err();
let err = Box::pin(usecase.execute_copy_object(req)).await.unwrap_err();
// Tiered self-copy with STANDARD storage class must pass all validation checks.
// The call fails at store init (no store in unit tests), not at validation.
assert_ne!(err.code(), &S3ErrorCode::InvalidRequest);
@@ -1285,7 +1260,7 @@ mod tests {
let req = build_request(input, Method::PUT);
let usecase = DefaultObjectUsecase::without_context();
let err = Box::pin(copy_owned(&usecase, req)).await.unwrap_err();
let err = Box::pin(usecase.execute_copy_object(req)).await.unwrap_err();
// Must not be rejected by the self-copy guard; it fails later at store init instead.
assert_ne!(err.code(), &S3ErrorCode::InvalidRequest);
}
@@ -1366,16 +1341,17 @@ mod tests {
[(rustfs_config::ENV_OBJECT_LOCK_OPTIMIZATION_ENABLE, Some("false"))],
tokio::time::timeout(
Duration::from_secs(10),
copy_owned(&DefaultObjectUsecase::with_context(Some(context)), req),
DefaultObjectUsecase::with_context(Some(context)).execute_copy_object(req),
),
)
.await
.expect("nonbuffered source reader must not deadlock behind the self-copy write lock")
.expect("expected-current historical self-copy should succeed");
assert!(response.e_tag.is_some());
assert!(response.output.copy_object_result.is_some());
let source_version_id_text = source_version_id.to_string();
assert_eq!(response.copy_source_version_id.as_deref(), Some(source_version_id_text.as_str()));
assert_eq!(response.output.copy_source_version_id.as_deref(), Some(source_version_id_text.as_str()));
let destination_version_id = response
.output
.version_id
.clone()
.expect("versioned self-copy should return its destination version");
@@ -1461,12 +1437,13 @@ mod tests {
.key(destination.to_string())
.build()
.expect("directory copy input should build");
let response = copy_owned(&DefaultObjectUsecase::with_context(Some(context)), build_request(input, Method::PUT))
let response = DefaultObjectUsecase::with_context(Some(context))
.execute_copy_object(build_request(input, Method::PUT))
.await
.expect("copying to a directory marker should succeed");
assert_eq!(
response.version_id.as_deref(),
response.output.version_id.as_deref(),
Some(NULL_VERSION_ID),
"CopyObject must expose the null destination version without leaking the internal nil UUID"
);
@@ -1500,7 +1477,7 @@ mod tests {
let req = build_request(input, Method::PUT);
let usecase = DefaultObjectUsecase::without_context();
let err = Box::pin(copy_owned(&usecase, req)).await.unwrap_err();
let err = Box::pin(usecase.execute_copy_object(req)).await.unwrap_err();
assert_ne!(err.code(), &S3ErrorCode::InvalidRequest);
}
@@ -1521,7 +1498,7 @@ mod tests {
let req = build_request(input, Method::PUT);
let usecase = DefaultObjectUsecase::without_context();
let err = Box::pin(copy_owned(&usecase, req)).await.unwrap_err();
let err = Box::pin(usecase.execute_copy_object(req)).await.unwrap_err();
assert_eq!(err.code(), &S3ErrorCode::InvalidArgument);
}
@@ -1543,7 +1520,7 @@ mod tests {
HeaderValue::from_str(&Uuid::new_v4().to_string()).unwrap(),
);
let err = Box::pin(copy_owned(&DefaultObjectUsecase::without_context(), req))
let err = Box::pin(DefaultObjectUsecase::without_context().execute_copy_object(req))
.await
.unwrap_err();
assert_eq!(err.code(), &S3ErrorCode::InvalidRequest);
@@ -1552,8 +1529,9 @@ mod tests {
#[tokio::test]
#[serial_test::serial]
async fn metadata_only_copy_escapes_legacy_etags_in_xml() {
use crate::app::storage_api::test::ReqInfo;
use crate::app::storage_api::test::contract::bucket::{BucketOperations as _, MakeBucketOptions};
use http_body_util::BodyExt as _;
use s3s::xml::{SerializeContent, Serializer};
let (store, context) = real_store_test_context().await;
let bucket = format!("copy-legacy-etag-{}", Uuid::new_v4());
let key = "owned.txt";
@@ -1580,25 +1558,30 @@ mod tests {
.await
.expect("legacy object");
assert_eq!(before.etag.as_deref(), Some(etag));
// The use case hands the stored ETag on verbatim; the gateway's encoder escapes it.
let slot = crate::app::runtime_sources::ServerContextSlot::new();
assert!(slot.install(Arc::clone(&context)));
let gateway = crate::server::TestGateway::over(&context, slot);
let response = gateway
.signed(
Method::PUT,
&format!("/{bucket}/{key}"),
vec![
("x-amz-copy-source", format!("/{bucket}/{key}")),
("x-amz-metadata-directive", "REPLACE".to_owned()),
],
Bytes::new(),
)
let input = CopyObjectInput::builder()
.bucket(bucket.clone())
.key(key.to_string())
.copy_source(CopySource::Bucket {
bucket: bucket.clone().into(),
key: key.into(),
version_id: None,
})
.metadata_directive(Some(s3s::dto::MetadataDirective::from_static("REPLACE")))
.build()
.expect("self-copy request");
let mut request = build_request(input, Method::PUT);
request.extensions.insert(ReqInfo::default());
let result = Box::pin(DefaultObjectUsecase::with_context(Some(context)).execute_copy_object(request))
.await
.expect("metadata-only self-copy");
assert_eq!(response.status(), http::StatusCode::OK);
let body = response.into_body().collect().await.expect("copy result body").to_bytes();
let xml = String::from_utf8(body.to_vec()).expect("XML UTF-8");
let mut bytes = Vec::new();
result
.output
.copy_object_result
.expect("copy result")
.serialize_content(&mut Serializer::new(&mut bytes))
.expect("serialize copy result");
let xml = String::from_utf8(bytes).expect("XML UTF-8");
assert!(!xml.contains("<Injected>"), "{xml}");
assert!(xml.contains("&lt;Injected&gt;victim&lt;/Injected&gt;"), "{xml}");
let stored = store
@@ -1640,7 +1623,7 @@ mod tests {
.expect("copy input must build");
let usecase = DefaultObjectUsecase::with_context(Some(Arc::clone(&context)));
let err = Box::pin(copy_owned(&usecase, build_request(input, Method::PUT)))
let err = Box::pin(usecase.execute_copy_object(build_request(input, Method::PUT)))
.await
.expect_err("an unreadable bucket encryption configuration must refuse the copy");
-12
View File
@@ -70,7 +70,6 @@ pub(crate) struct CopyObjectWriteInput {
pub(crate) checksum_algorithm: Option<String>,
}
#[derive(Debug)]
pub(crate) struct CopyObjectWriteOutput {
pub(crate) e_tag: Option<String>,
pub(crate) last_modified: Option<OffsetDateTime>,
@@ -92,14 +91,3 @@ pub(crate) struct CopyObjectWriteOutput {
pub(crate) checksum_xxhash128: Option<String>,
pub(crate) checksum_type: Option<String>,
}
#[cfg(test)]
impl CopyObjectWriteInput {
/// A test's s3s fixture request as the envelope and input a gateway handler hands the use case.
pub(crate) fn from_test_request(
req: crate::app::storage_api::s3::S3Request<crate::app::storage_api::s3::dto::CopyObjectInput>,
) -> (crate::app::RequestEnvelope, Self) {
let envelope = crate::app::RequestEnvelope::from_s3s(&req);
(envelope, crate::storage::ecfs::copy_input_from_legacy(req.input))
}
}
+31 -25
View File
@@ -2082,6 +2082,7 @@ mod tests {
#[serial_test::serial]
fn repeated_versioned_delete_returns_204_with_legacy_residue() {
crate::app::gating_test_env::run_large_stack_test("repeated-versioned-delete-http", || async {
use crate::app::storage_api::s3::{Body, S3ServiceBuilder};
use crate::app::storage_api::test::contract::bucket::{BucketOperations as _, MakeBucketOptions};
use http_body_util::BodyExt;
@@ -2089,8 +2090,8 @@ mod tests {
let ambient = crate::app::gating_test_env::shared_gating_ambient().await;
let context = Arc::new(AppContext::new(Arc::clone(&store), ambient.iam(), ambient.kms()));
let slot = crate::app::runtime_sources::ServerContextSlot::new();
assert!(slot.install(Arc::clone(&context)));
let gateway = crate::server::TestGateway::over(&context, slot);
assert!(slot.install(context));
let service = S3ServiceBuilder::new(FS::with_server_ctx(slot)).build();
let bucket = format!("repeated-delete-http-{}", Uuid::new_v4());
let key = "history";
let payload = b"historical version behind repeated delete markers";
@@ -2116,14 +2117,26 @@ mod tests {
)
.await
.expect("write historical version");
// HTTP decoding and serialization through the gateway stack, signed by its root.
let gateway = &gateway;
let request =
|method: Method, uri: String| async move { gateway.signed(method, &uri, Vec::new(), Bytes::new()).await };
// Exercise HTTP decoding and serialization with the production FS
// dispatcher. Authentication is supplied by the owner fixture.
let request = |method: Method, uri: String| {
http::Request::builder()
.method(method)
.uri(uri)
.header("host", "localhost")
.extension(crate::storage::req_info::ReqInfo {
cred: Some(rustfs_credentials::Credentials::default()),
is_owner: true,
..Default::default()
})
.body(Body::empty())
.expect("HTTP request")
};
let mut marker_ids = Vec::new();
let mut backup = None;
for attempt in 0..2 {
let response = request(Method::DELETE, format!("/{bucket}/{key}"))
let response = service
.call(request(Method::DELETE, format!("/{bucket}/{key}")))
.await
.expect("DELETE HTTP response");
assert_eq!(response.status(), StatusCode::NO_CONTENT);
@@ -2159,12 +2172,13 @@ mod tests {
}
let (path, bytes) = backup.expect("legacy backup fixture");
assert_eq!(tokio::fs::read(path).await.expect("legacy backup remains"), bytes);
let response = request(
Method::GET,
format!("/{bucket}/{key}?versionId={}", original.version_id.expect("historical version ID")),
)
.await
.expect("historical GET HTTP response");
let response = service
.call(request(
Method::GET,
format!("/{bucket}/{key}?versionId={}", original.version_id.expect("historical version ID")),
))
.await
.expect("historical GET HTTP response");
assert_eq!(response.status(), StatusCode::OK);
assert_eq!(
BodyExt::collect(response.into_body())
@@ -3607,8 +3621,9 @@ mod tests {
}
#[test]
fn invalid_delete_key_still_records_an_audit() {
fn invalid_legacy_delete_key_still_records_an_audit() {
crate::app::gating_test_env::run_large_stack_test("invalid-delete-audit", || async {
use s3s::S3 as _;
let bucket = format!("invalid-delete-audit-{}", Uuid::new_v4());
let probe = crate::app::storage_api::object_usecase::helper::MutationCompletionProbe::install(&bucket);
let input = DeleteObjectInput::builder()
@@ -3618,17 +3633,8 @@ mod tests {
.expect("delete input");
let mut request = build_request(input, Method::DELETE);
request.uri = format!("/{bucket}/key").parse().expect("URI");
let envelope = RequestEnvelope::from_s3s(&request);
let input = request.input;
let error = Box::pin(DefaultObjectUsecase::from_global().execute_delete_object_fields(
envelope,
input.bucket,
input.key,
input.version_id,
))
.await
.expect_err("invalid key");
assert_eq!(error.code(), &rustfs_s3_types::S3ErrorCode::InvalidArgument);
let error = FS::new().delete_object(request).await.expect_err("invalid key");
assert_eq!(error.code(), &S3ErrorCode::InvalidArgument);
let audits = probe.audits.lock().expect("audits");
assert_eq!(audits.len(), 1);
assert_eq!(audits[0].api.status.as_deref(), Some("failure"));
+7 -15
View File
@@ -2832,15 +2832,6 @@ pub(super) mod post_archive_preflight;
#[cfg(test)]
mod tests {
use super::*;
/// Archive extraction through the owned entry point the gateway handler calls, its error in
/// the s3s form these tests compare.
async fn extract_owned(usecase: &DefaultObjectUsecase, req: S3Request<PutObjectInput>) -> S3Result<PutObjectWriteOutput> {
let (envelope, input) = PutObjectWriteInput::from_test_request(req);
Box::pin(usecase.execute_put_object_extract_owned(envelope, input))
.await
.map_err(S3Error::from)
}
use crate::app::storage_api::s3::StdError;
use crate::app::storage_api::s3::dto::{ObjectLockRetentionMode, StreamingBlob};
use http::{HeaderMap, HeaderName, HeaderValue};
@@ -3721,7 +3712,8 @@ mod tests {
request
.headers
.insert("x-amz-meta-snowball-auto-extract", HeaderValue::from_static("true"));
let error = extract_owned(&DefaultObjectUsecase::with_context(Some(context)), request)
let error = DefaultObjectUsecase::with_context(Some(context))
.execute_put_object_extract(request)
.await
.expect_err("a bad payload digest must reject the archive request");
assert_eq!(error.code().as_str(), "XAmzContentSHA256Mismatch");
@@ -3804,7 +3796,7 @@ mod tests {
.expect("extract request");
let mut request = build_request(input, Method::PUT);
request.extensions.insert(ReqInfo::default());
let result = Box::pin(extract_owned(&usecase, request)).await;
let result = Box::pin(usecase.execute_put_object_extract(request)).await;
if explicit_version && !replicate {
assert_eq!(
result
@@ -3908,7 +3900,7 @@ mod tests {
.expect("allowed member exists");
assert_eq!(info.size, 1);
} else {
assert_eq!(result.expect_err("IAM must deny the member").code().as_str(), "AccessDenied");
assert_eq!(result.err().expect("IAM must deny the member").code().as_str(), "AccessDenied");
if member == "denied-existing" {
let after = store
.get_object_info(&bucket, member, &options)
@@ -4018,7 +4010,7 @@ mod tests {
result.expect("unchanged public member is allowed");
assert_eq!(after.size, 1);
} else {
assert_eq!(result.expect_err("private member must be denied").code().as_str(), "AccessDenied");
assert_eq!(result.err().expect("private member must be denied").code().as_str(), "AccessDenied");
assert_eq!(after.etag, before.etag, "denied member must not overwrite");
assert_eq!(after.user_tags.as_str(), "classification=private");
}
@@ -4097,7 +4089,7 @@ mod tests {
result.expect("private member is authorized");
assert_eq!(stored.expect("private member stored").user_tags.as_str(), "classification=private");
} else {
assert_eq!(result.expect_err("public member must be denied").code().as_str(), "AccessDenied");
assert_eq!(result.err().expect("public member must be denied").code().as_str(), "AccessDenied");
assert!(is_err_object_not_found(&stored.expect_err("denied member must not be stored")));
}
}
@@ -4198,7 +4190,7 @@ mod tests {
}
} else {
assert_eq!(
result.expect_err("member privilege required").code().as_str(),
result.err().expect("member privilege required").code().as_str(),
"AccessDenied",
"{action}: {permission}"
);
+3 -7
View File
@@ -482,13 +482,9 @@ async fn ssec_wire_contract(usecase: &DefaultObjectUsecase, server: &crate::embe
.sse_customer_key_md5(Some(digest.clone()))
.build()
.unwrap();
{
let (envelope, input) =
crate::app::object_usecase::PutObjectWriteInput::from_test_request(build_request(input, Method::PUT));
crate::app::test_seams::put_object(usecase, envelope, input)
}
.await
.unwrap();
crate::app::test_seams::put_object(usecase, &FS::new(), build_request(input, Method::PUT))
.await
.unwrap();
for (case, valid, wrong_key, condition, range) in [
("ordinary", true, false, false, false),
("missing", false, false, false, false),
+7 -8
View File
@@ -1246,13 +1246,15 @@ mod tests {
crate::app::gating_test_env::run_large_stack_test("head-metadata-permissions", || async {
use crate::app::runtime_sources::{AppContext, ServerContextSlot};
use crate::app::storage_api::test::ReqInfo;
use crate::app::storage_api::test::authorize_head_object;
use crate::app::storage_api::test::S3Access as _;
use crate::app::storage_api::test::contract::bucket::{BucketOperations as _, MakeBucketOptions};
use crate::app::storage_api::test::ecfs::FS;
let ambient = crate::app::gating_test_env::shared_gating_ambient().await;
let store = crate::app::gating_test_env::shared_gating_ecstore().await;
let context = Arc::new(AppContext::new(store.clone(), ambient.iam(), ambient.kms()));
let slot = ServerContextSlot::new();
assert!(slot.install(context.clone()));
let fs = FS::with_server_ctx(slot.clone());
let usecase = DefaultObjectUsecase::with_context(Some(context));
let bucket = format!("head-metadata-{}", Uuid::new_v4().simple());
store
@@ -1308,7 +1310,7 @@ mod tests {
.unwrap();
let version_id = object.version_id.map(|version| version.to_string());
for (version, tagging_directive) in [(None, false), (None, true), (version_id, true)] {
let mut req = head_input(&bucket, "protected", version.clone());
let mut req = head_input(&bucket, "protected", version);
if tagging_directive {
req.headers
.insert(rustfs_utils::http::AMZ_TAGGING_DIRECTIVE_LOWER, HeaderValue::from_static("ACCESS"));
@@ -1319,14 +1321,11 @@ mod tests {
req.headers
.insert("x-rustfs-source-replication-check", HeaderValue::from_static("true"));
}
let mut envelope = RequestEnvelope::from_s3s(&req);
authorize_head_object(&slot, &mut envelope, &bucket, "protected", version.as_deref())
.await
.expect("ordinary HEAD read is permitted");
let response = Box::pin(usecase.execute_head_object_fields(envelope, HeadObjectFields::from_s3s(req.input)))
fs.head_object(&mut req).await.expect("ordinary HEAD read is permitted");
let response = Box::pin(crate::app::test_seams::head_object(&usecase, req))
.await
.expect("HEAD succeeds");
assert_eq!(response.output.dto.content_length, Some(4));
assert_eq!(response.output.content_length, Some(4));
for header in [
AMZ_TAG_COUNT,
AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER,
+76 -70
View File
@@ -2605,23 +2605,6 @@ mod tests {
use http::{HeaderMap, HeaderName, HeaderValue, Method};
use rustfs_s3_types::S3ErrorCode as OwnedS3ErrorCode;
/// PutObject through the owned entry point the gateway handler calls, its error in the s3s
/// form these tests compare.
async fn put_owned(usecase: &DefaultObjectUsecase, req: S3Request<PutObjectInput>) -> S3Result<PutObjectWriteOutput> {
let (envelope, input) = PutObjectWriteInput::from_test_request(req);
Box::pin(usecase.execute_put_object_owned(envelope, input))
.await
.map_err(S3Error::from)
}
/// As [`put_owned`], through the archive-extraction entry point.
async fn put_extract_owned(usecase: &DefaultObjectUsecase, req: S3Request<PutObjectInput>) -> S3Result<PutObjectWriteOutput> {
let (envelope, input) = PutObjectWriteInput::from_test_request(req);
Box::pin(usecase.execute_put_object_extract_owned(envelope, input))
.await
.map_err(S3Error::from)
}
#[tokio::test]
#[serial_test::serial]
async fn s3_put_uses_tail_drained_completion_before_acknowledgement() {
@@ -2643,9 +2626,10 @@ mod tests {
.build()
.expect("PUT completion request");
let usecase = DefaultObjectUsecase::with_context(Some(context));
let fs = FS::new();
let (response, observed) = PUT_STORAGE_TEST_COMPLETION
.scope(std::cell::Cell::new(None), async {
let response = Box::pin(put_owned(&usecase, build_request(input, Method::PUT))).await;
let response = Box::pin(usecase.execute_put_object(&fs, build_request(input, Method::PUT))).await;
(response, PUT_STORAGE_TEST_COMPLETION.with(std::cell::Cell::get))
})
.await;
@@ -2660,9 +2644,7 @@ mod tests {
#[test]
#[serial_test::serial]
/// The PutObject use cases count nothing: the gateway's dispatch counts each request once before
/// the use case runs (`GatewayBackend::serve`), so a count here would count it twice.
fn owned_put_entry_points_never_count_the_operation() {
fn put_edge_counts_polled_requests_once_before_usecase_refusal() {
use crate::app::storage_api::test::contract::bucket::{BucketOperations as _, MakeBucketOptions};
use metrics_util::debugging::{DebugValue, DebuggingRecorder, Snapshotter};
@@ -2708,6 +2690,7 @@ mod tests {
.block_on(Box::pin(store.make_bucket(&bucket, &MakeBucketOptions::default())))
.expect("count test bucket");
let usecase = DefaultObjectUsecase::with_context(Some(context));
let fs = FS::new();
let recorder = DebuggingRecorder::new();
let snapshotter = recorder.snapshotter();
let request = |invalid_class: bool| {
@@ -2724,35 +2707,39 @@ mod tests {
let baseline = official_count();
let mut observed = 0;
metrics::with_local_recorder(&recorder, || {
drop(put_owned(&usecase, request(false)));
drop(usecase.execute_put_object(&fs, request(false)));
assert_eq!(put_count(&snapshotter, &mut observed), 0, "constructing a future does not count a PUT");
assert_eq!(official_count() - baseline, 0, "official operation total");
let error = runtime
.block_on(Box::pin(put_owned(&usecase, request(true))))
.block_on(Box::pin(usecase.execute_put_object(&fs, request(true))))
.expect_err("invalid class");
assert_eq!(error.code(), &S3ErrorCode::InvalidStorageClass);
assert_eq!(put_count(&snapshotter, &mut observed), 0, "the use case does not count a refused PUT");
assert_eq!(official_count() - baseline, 0, "official operation total");
assert_eq!(put_count(&snapshotter, &mut observed), 1, "the owned edge counts before usecase refusal");
assert_eq!(official_count() - baseline, 1, "official operation total");
runtime
.block_on(Box::pin(put_owned(&usecase, request(false))))
.block_on(Box::pin(usecase.execute_put_object(&fs, request(false))))
.expect("successful PUT");
assert_eq!(put_count(&snapshotter, &mut observed), 0, "the use case does not count an accepted PUT");
assert_eq!(official_count() - baseline, 0, "official operation total");
assert_eq!(
put_count(&snapshotter, &mut observed),
2,
"the usecase must not count the accepted PUT again"
);
assert_eq!(official_count() - baseline, 2, "official operation total");
let input = PutObjectInput {
bucket: bucket.clone(),
key: "archive.tar".to_owned(),
..Default::default()
};
let error = runtime
.block_on(Box::pin(put_extract_owned(&usecase, build_request(input, Method::PUT))))
.block_on(Box::pin(usecase.execute_put_object_extract(build_request(input, Method::PUT))))
.expect_err("missing archive body");
assert_eq!(error.code(), &S3ErrorCode::IncompleteBody);
assert_eq!(
put_count(&snapshotter, &mut observed),
0,
"the archive use case does not count a refused PUT"
3,
"the direct archive edge counts once before validation"
);
assert_eq!(official_count() - baseline, 0, "official operation total");
assert_eq!(official_count() - baseline, 3, "official operation total");
});
}
@@ -2785,16 +2772,13 @@ mod tests {
let snapshotter = recorder.snapshotter();
let mut request = S3HttpRequestGuard::new("PUT");
let error = metrics::with_local_recorder(&recorder, || {
let fs = FS::new();
let mut future = Box::pin(PUT_STORAGE_TEST_FAILURE.scope(
(
PutStorageTestStage::Commit,
StorageError::Lock(LockError::timeout("private-bucket/private-key", Duration::from_secs(5))),
),
// The gateway's dispatch names the operation before the use case runs.
Box::pin(async {
rustfs_io_metrics::record_s3_op(S3Operation::PutObject);
put_owned(&usecase, build_request(input, Method::PUT)).await
}),
Box::pin(usecase.execute_put_object(&fs, build_request(input, Method::PUT))),
));
runtime.block_on(std::future::poll_fn(|cx| {
request.in_scope(|| std::future::Future::poll(future.as_mut(), cx))
@@ -2875,7 +2859,7 @@ mod tests {
let err = PUT_STORAGE_TEST_FAILURE
.scope(
(stage, StorageError::InsufficientReadQuorum("bucket".into(), "object".into())),
put_owned(&usecase, req),
usecase.execute_put_object(&FS::new(), req),
)
.await
.expect_err("read quorum failure must refuse the upload");
@@ -2935,7 +2919,7 @@ mod tests {
.expect("capacity PUT input must build");
let request = build_request(input, Method::PUT);
let error = PUT_STORAGE_TEST_FAILURE
.scope((PutStorageTestStage::Commit, failure), put_owned(&usecase, request))
.scope((PutStorageTestStage::Commit, failure), usecase.execute_put_object(&FS::new(), request))
.await
.expect_err("capacity refusal must fail PUT");
assert_eq!(error.code(), &S3ErrorCode::ServiceUnavailable);
@@ -3289,7 +3273,11 @@ mod tests {
.build()
.expect("PUT input must build");
let put_usecase = DefaultObjectUsecase::with_context(Some(context));
let put = tokio::spawn(async move { put_owned(&put_usecase, build_request(put_input, Method::PUT)).await });
let put = tokio::spawn(async move {
put_usecase
.execute_put_object(&FS::new(), build_request(put_input, Method::PUT))
.await
});
tokio::time::timeout(Duration::from_secs(2), async {
while !object_traffic_health.snapshot().write_stalled {
tokio::task::yield_now().await;
@@ -3349,7 +3337,7 @@ mod tests {
let put_usecase = DefaultObjectUsecase::with_context(Some(Arc::clone(&context)));
let mut request = build_request(put_input, Method::PUT);
request.uri = format!("/{bucket}/{object}").parse().expect("PUT URI");
let put = tokio::spawn(async move { put_owned(&put_usecase, request).await });
let put = tokio::spawn(async move { put_usecase.execute_put_object(&FS::new(), request).await });
tokio::time::timeout(Duration::from_secs(10), post_store_entered.wait())
.await
@@ -3392,7 +3380,8 @@ mod tests {
.expect("normal PUT");
let mut normal_request = build_request(normal_input, Method::PUT);
normal_request.uri = format!("/{bucket}/normal.bin").parse().expect("URI");
put_owned(&DefaultObjectUsecase::with_context(Some(context)), normal_request)
DefaultObjectUsecase::with_context(Some(context))
.execute_put_object(&FS::new(), normal_request)
.await
.expect("normal response completion");
let audits = probe.audits.lock().expect("audits");
@@ -3429,7 +3418,8 @@ mod tests {
let extra_body_usecase = DefaultObjectUsecase::with_context(Some(Arc::clone(&context)));
let mut extra_body_request = build_request(extra_body_input, Method::PUT);
extra_body_request.headers = streaming_headers(Some("0"));
let extra_body_err = put_owned(&extra_body_usecase, extra_body_request)
let extra_body_err = extra_body_usecase
.execute_put_object(&FS::new(), extra_body_request)
.await
.expect_err("decoded zero-byte PUT with body data must fail");
assert_eq!(extra_body_err.code(), &S3ErrorCode::UnexpectedContent);
@@ -3465,7 +3455,7 @@ mod tests {
let zero_usecase = DefaultObjectUsecase::with_context(Some(Arc::clone(&context)));
let mut zero_request = build_request(zero_input, Method::PUT);
zero_request.headers = streaming_headers(Some("0"));
let zero_put = tokio::spawn(async move { put_owned(&zero_usecase, zero_request).await });
let zero_put = tokio::spawn(async move { zero_usecase.execute_put_object(&FS::new(), zero_request).await });
tokio::time::timeout(Duration::from_secs(30), body_polled_rx)
.await
@@ -3517,8 +3507,11 @@ mod tests {
.build()
.expect("zero-copy PUT input must build");
let zero_copy_usecase = DefaultObjectUsecase::with_context(Some(context));
let zero_copy_put =
tokio::spawn(async move { put_owned(&zero_copy_usecase, build_request(zero_copy_input, Method::PUT)).await });
let zero_copy_put = tokio::spawn(async move {
zero_copy_usecase
.execute_put_object(&FS::new(), build_request(zero_copy_input, Method::PUT))
.await
});
tokio::time::timeout(Duration::from_secs(30), async {
while !object_traffic_health.snapshot().write_stalled {
@@ -3980,7 +3973,7 @@ mod tests {
let mut request = build_request(input, Method::PUT);
request.extensions.insert(control.clone());
let usecase = DefaultObjectUsecase::with_context(Some(Arc::clone(&context)));
let owner = tokio::spawn(async move { put_owned(&usecase, request).await });
let owner = tokio::spawn(async move { usecase.execute_put_object(&FS::new(), request).await });
control.wait_until_pending_for_test().await;
control.advance_wait_for_test(Duration::from_secs(60));
sender.send(Ok(Frame::data(Bytes::new()))).unwrap();
@@ -4365,7 +4358,7 @@ mod tests {
async fn post_object_metadata_cannot_dispatch_legacy_archive_extraction() {
let mut req = build_request(PutObjectInput::default(), Method::POST);
req.extensions.insert(PostObjectRequestMarker);
let err = Box::pin(put_extract_owned(&DefaultObjectUsecase::without_context(), req))
let err = Box::pin(DefaultObjectUsecase::without_context().execute_put_object_extract(req))
.await
.expect_err("a legacy form cannot authorize archive replay");
assert_eq!(err.code(), &S3ErrorCode::NotImplemented);
@@ -4384,7 +4377,7 @@ mod tests {
.insert(http::header::IF_NONE_MATCH, HeaderValue::from_static("\"different-etag\""));
let usecase = DefaultObjectUsecase::without_context();
let err = Box::pin(put_owned(&usecase, req))
let err = Box::pin(usecase.execute_put_object(&FS::new(), req))
.await
.expect_err("POST header-only ETag precondition must fail before storage access");
assert_eq!(err.code(), &S3ErrorCode::NotImplemented);
@@ -4414,7 +4407,7 @@ mod tests {
} else {
Box::pin(usecase.execute_put_object_owned(envelope, input)).await
};
let error = result.expect_err("unbound gateway archive must be refused");
let error = result.err().expect("unbound gateway archive must be refused");
assert_eq!(error.code().as_str(), "NotImplemented", "direct={direct_extract}");
assert_eq!(
polls.load(std::sync::atomic::Ordering::SeqCst),
@@ -4437,8 +4430,9 @@ mod tests {
req.extensions.insert(PostObjectRequestMarker);
let usecase = DefaultObjectUsecase::without_context();
let fs = FS::new();
let err = Box::pin(put_owned(&usecase, req)).await.unwrap_err();
let err = Box::pin(usecase.execute_put_object(&fs, req)).await.unwrap_err();
assert_eq!(err.code(), &S3ErrorCode::NotImplemented);
}
@@ -4455,8 +4449,9 @@ mod tests {
req.headers.insert(AMZ_SNOWBALL_EXTRACT, HeaderValue::from_static("true"));
let usecase = DefaultObjectUsecase::without_context();
let fs = FS::new();
let err = Box::pin(put_owned(&usecase, req)).await.unwrap_err();
let err = Box::pin(usecase.execute_put_object(&fs, req)).await.unwrap_err();
assert_eq!(err.code(), &S3ErrorCode::NotImplemented);
}
@@ -4473,8 +4468,9 @@ mod tests {
req.headers.insert(AMZ_SNOWBALL_EXTRACT, HeaderValue::from_static("true"));
let usecase = DefaultObjectUsecase::without_context();
let fs = FS::new();
let err = Box::pin(put_owned(&usecase, req)).await.unwrap_err();
let err = Box::pin(usecase.execute_put_object(&fs, req)).await.unwrap_err();
assert_eq!(err.code(), &S3ErrorCode::InvalidStorageClass);
}
@@ -4495,8 +4491,9 @@ mod tests {
let req = build_request(input, Method::PUT);
let usecase = DefaultObjectUsecase::without_context();
let fs = FS::new();
let err = Box::pin(put_owned(&usecase, req)).await.unwrap_err();
let err = Box::pin(usecase.execute_put_object(&fs, req)).await.unwrap_err();
assert_eq!(err.code(), &S3ErrorCode::EntityTooLarge);
assert_eq!(polls.load(std::sync::atomic::Ordering::SeqCst), 0, "body must not be polled");
}
@@ -4532,8 +4529,9 @@ mod tests {
HeaderValue::from_str(&decoded.to_string()).unwrap(),
);
let usecase = DefaultObjectUsecase::without_context();
let fs = FS::new();
let err = Box::pin(put_owned(&usecase, req)).await.unwrap_err();
let err = Box::pin(usecase.execute_put_object(&fs, req)).await.unwrap_err();
if expect_too_large {
assert_eq!(err.code(), &S3ErrorCode::EntityTooLarge, "decoded {decoded}");
assert_eq!(polls.load(std::sync::atomic::Ordering::SeqCst), 0, "body must not be polled");
@@ -4561,8 +4559,9 @@ mod tests {
.insert(AMZ_SERVER_SIDE_ENCRYPTION, HeaderValue::from_static("aws:kms"));
let usecase = DefaultObjectUsecase::without_context();
let fs = FS::new();
let err = Box::pin(put_owned(&usecase, req)).await.unwrap_err();
let err = Box::pin(usecase.execute_put_object(&fs, req)).await.unwrap_err();
assert_eq!(err.code(), &S3ErrorCode::NotImplemented);
}
@@ -4580,8 +4579,9 @@ mod tests {
.insert(AMZ_SERVER_SIDE_ENCRYPTION_KMS_ID, HeaderValue::from_static("test-kms-key-id"));
let usecase = DefaultObjectUsecase::without_context();
let fs = FS::new();
let err = Box::pin(put_owned(&usecase, req)).await.unwrap_err();
let err = Box::pin(usecase.execute_put_object(&fs, req)).await.unwrap_err();
assert_eq!(err.code(), &S3ErrorCode::NotImplemented);
}
@@ -4596,8 +4596,9 @@ mod tests {
let req = build_request(input, Method::PUT);
let usecase = DefaultObjectUsecase::without_context();
let fs = FS::new();
let err = Box::pin(put_owned(&usecase, req)).await.unwrap_err();
let err = Box::pin(usecase.execute_put_object(&fs, req)).await.unwrap_err();
assert_eq!(err.code(), &S3ErrorCode::InvalidStorageClass);
}
@@ -4819,12 +4820,13 @@ mod tests {
.expect("SSE-C PUT input must build");
let usecase = DefaultObjectUsecase::from_global();
let response = put_owned(&usecase, build_request(input, Method::PUT))
let response = usecase
.execute_put_object(&FS::new(), build_request(input, Method::PUT))
.await
.unwrap_or_else(|err| panic!("a {algorithm} default bucket must accept an SSE-C PUT: {err:?}"));
assert_eq!(response.server_side_encryption, None);
assert_eq!(response.sse_customer_algorithm.as_deref(), Some("AES256"));
assert_eq!(response.sse_customer_key_md5.as_deref(), Some(key_md5.as_str()));
assert_eq!(response.output.server_side_encryption, None);
assert_eq!(response.output.sse_customer_algorithm.as_deref(), Some("AES256"));
assert_eq!(response.output.sse_customer_key_md5.as_deref(), Some(key_md5.as_str()));
let encoded_key = base64_simd::STANDARD.encode_to_string(customer_key);
let mut read_headers = HeaderMap::new();
@@ -4938,7 +4940,8 @@ mod tests {
..Default::default()
});
let err = put_owned(&DefaultObjectUsecase::from_global(), request)
let err = DefaultObjectUsecase::from_global()
.execute_put_object(&FS::new(), request)
.await
.expect_err("quota-enabled ciphertext replication should fail at ingress");
assert_eq!(err.code(), &S3ErrorCode::InvalidRequest);
@@ -4988,7 +4991,8 @@ mod tests {
.build()
.expect("legacy quota PUT input should build");
let err = put_owned(&DefaultObjectUsecase::from_global(), build_request(input, Method::PUT))
let err = DefaultObjectUsecase::from_global()
.execute_put_object(&FS::new(), build_request(input, Method::PUT))
.await
.expect_err("4 GiB used plus a 2 GiB PUT must exceed a 5 GiB legacy quota");
assert_eq!(err.code(), &S3ErrorCode::InvalidRequest);
@@ -5034,7 +5038,8 @@ mod tests {
.content_length(Some(i64::try_from(payload.len()).expect("test payload length must fit i64")))
.build()
.expect("PUT input should build");
put_owned(&DefaultObjectUsecase::from_global(), build_request(input, Method::PUT))
DefaultObjectUsecase::from_global()
.execute_put_object(&FS::new(), build_request(input, Method::PUT))
.await
.expect("a ledger-admitted bucket must accept a write without scanner usage");
@@ -5046,7 +5051,8 @@ mod tests {
.content_length(Some(i64::try_from(too_large.len()).expect("test payload length must fit i64")))
.build()
.expect("PUT input should build");
let err = put_owned(&DefaultObjectUsecase::from_global(), build_request(input, Method::PUT))
let err = DefaultObjectUsecase::from_global()
.execute_put_object(&FS::new(), build_request(input, Method::PUT))
.await
.expect_err("the ledger must still reject a write above the quota limit");
assert_eq!(err.code(), &S3ErrorCode::InvalidRequest);
@@ -5449,7 +5455,7 @@ mod tests {
.expect("PUT input must build");
let usecase = DefaultObjectUsecase::with_context(Some(Arc::clone(&context)));
let err = Box::pin(put_owned(&usecase, build_request(input, Method::PUT)))
let err = Box::pin(usecase.execute_put_object(&FS::new(), build_request(input, Method::PUT)))
.await
.expect_err("an unreadable bucket encryption configuration must refuse the write");
@@ -5494,7 +5500,7 @@ mod tests {
metrics::with_local_recorder(&recorder, || {
request.in_scope(|| {
runtime.block_on(async {
Box::pin(put_owned(&usecase, build_request(input, Method::PUT)))
Box::pin(usecase.execute_put_object(&FS::new(), build_request(input, Method::PUT)))
.await
.expect("S3 PUT must commit");
assert_eq!(current_s3_http_operation_name(), Some("s3:PutObject"));
@@ -5561,7 +5567,7 @@ mod tests {
.expect("PUT input must build");
let usecase = DefaultObjectUsecase::with_context(Some(Arc::clone(&context)));
Box::pin(put_owned(&usecase, build_request(input, Method::PUT)))
Box::pin(usecase.execute_put_object(&FS::new(), build_request(input, Method::PUT)))
.await
.expect("a bucket without default encryption must still accept a plaintext write");
@@ -5607,7 +5613,7 @@ mod tests {
req.headers.insert(AMZ_SNOWBALL_EXTRACT, HeaderValue::from_static("true"));
let usecase = DefaultObjectUsecase::with_context(Some(Arc::clone(&context)));
let err = Box::pin(put_owned(&usecase, req))
let err = Box::pin(usecase.execute_put_object(&FS::new(), req))
.await
.expect_err("an unreadable bucket encryption configuration must refuse the extract upload");
+1 -12
View File
@@ -62,7 +62,7 @@ pub(crate) struct PutObjectWriteInput {
pub(crate) checksum_crc64nvme: Option<String>,
}
#[derive(Debug, Default)]
#[derive(Default)]
pub(crate) struct PutObjectWriteOutput {
pub(crate) e_tag: Option<String>,
pub(crate) version_id: Option<String>,
@@ -88,14 +88,3 @@ impl PutObjectWriteInput {
self.trailing_headers = Some(trailers);
}
}
#[cfg(test)]
impl PutObjectWriteInput {
/// A test's s3s fixture request as the envelope and input a gateway handler hands the use case.
pub(crate) fn from_test_request(
req: crate::app::storage_api::s3::S3Request<crate::app::storage_api::s3::PutObjectInput>,
) -> (crate::app::RequestEnvelope, Self) {
let envelope = crate::app::RequestEnvelope::from_s3s(&req);
(envelope, Self::from_legacy(req.input, req.trailing_headers))
}
}
+2 -2
View File
@@ -73,7 +73,7 @@ pub(crate) mod s3 {
#[cfg(test)]
pub(crate) use s3s::dto::{GetObjectInput, HeadObjectInput, ListObjectsV2Input, ListObjectsV2Output};
#[cfg(test)]
pub(crate) use s3s::dto::{GetObjectOutput, HeadObjectOutput, PutObjectInput};
pub(crate) use s3s::dto::{GetObjectOutput, HeadObjectOutput, PutObjectInput, PutObjectOutput};
#[cfg(test)]
pub(crate) use s3s::dto::{ListObjectsInput, StreamingBlob, UploadPartInput, UploadPartOutput};
pub(crate) use s3s::header;
@@ -1951,6 +1951,7 @@ pub(crate) mod context {
#[cfg(test)]
pub(crate) mod test {
pub(crate) use super::EndpointServerPools;
pub(crate) use s3s::access::S3Access;
pub(crate) mod contract {
pub(crate) mod bucket {
pub(crate) use super::super::super::storage_contracts::{
@@ -1981,7 +1982,6 @@ pub(crate) mod test {
pub(crate) use super::access::ReqInfo;
pub(crate) use super::options::VERSIONING_CONFIG_LOOKUPS;
pub(crate) use crate::storage::storage_api::test_consumer::authorize_head_object;
pub(crate) mod sse {
pub(crate) use crate::storage::storage_api::test_consumer::sse::{
RecordingKmsKeyAuthorizer, TestAuthorizationHooks, sse_kms_object_metadata,
+3 -10
View File
@@ -27,18 +27,11 @@ pub(crate) async fn get_object(
use crate::app::{Answered, RequestEnvelope};
rustfs_io_metrics::record_s3_op(rustfs_s3_ops::S3Operation::GetObject);
let envelope = RequestEnvelope::from_s3s(&req);
let Answered { output, status, headers } = get_object_fields(usecase, envelope, GetObjectFields::from_s3s(req.input)).await?;
let Answered { output, status, headers } = usecase
.execute_get_object_fields(envelope, GetObjectFields::from_s3s(req.input))
.await?;
let mut response = S3Response::new(output.into_s3s().map_err(crate::app::dto_values::seam_error)?);
response.status = status;
response.headers = headers;
Ok(response)
}
/// Calls `DefaultObjectUsecase::execute_get_object_fields` with owned inputs.
pub(crate) async fn get_object_fields(
usecase: &DefaultObjectUsecase,
envelope: crate::app::RequestEnvelope,
input: crate::app::object_usecase::GetObjectFields,
) -> rustfs_s3_types::S3Result<crate::app::Answered<crate::app::object_usecase::GetObjectOutputFacts>> {
usecase.execute_get_object_fields(envelope, input).await
}
+3 -11
View File
@@ -27,19 +27,11 @@ pub(crate) async fn head_object(
use crate::app::{Answered, RequestEnvelope};
rustfs_io_metrics::record_s3_op(rustfs_s3_ops::S3Operation::HeadObject);
let envelope = RequestEnvelope::from_s3s(&req);
let Answered { output, status, headers } =
head_object_fields(usecase, envelope, HeadObjectFields::from_s3s(req.input)).await?;
let Answered { output, status, headers } = usecase
.execute_head_object_fields(envelope, HeadObjectFields::from_s3s(req.input))
.await?;
let mut response = S3Response::new(output.into_s3s().map_err(crate::app::dto_values::seam_error)?);
response.status = status;
response.headers = headers;
Ok(response)
}
/// Calls `DefaultObjectUsecase::execute_head_object_fields` with owned inputs.
pub(crate) async fn head_object_fields(
usecase: &DefaultObjectUsecase,
envelope: crate::app::RequestEnvelope,
input: crate::app::object_usecase::HeadObjectFields,
) -> rustfs_s3_types::S3Result<crate::app::Answered<crate::app::object_usecase::HeadObjectOutputFacts>> {
usecase.execute_head_object_fields(envelope, input).await
}
+1 -10
View File
@@ -24,16 +24,7 @@ pub(crate) async fn list_objects_v2(
req: S3Request<ListObjectsV2Input>,
) -> S3Result<S3Response<ListObjectsV2Output>> {
crate::storage::ecfs::list_objects_v2_through_legacy_edge(req, |envelope, input| {
list_objects_v2_owned(usecase, envelope, input)
usecase.execute_list_objects_v2(envelope, input)
})
.await
}
/// Calls `DefaultBucketUsecase::execute_list_objects_v2` with owned inputs.
pub(crate) async fn list_objects_v2_owned(
usecase: &DefaultBucketUsecase,
envelope: crate::app::RequestEnvelope,
input: rustfs_gateway::dto::ListObjectsV2Input,
) -> rustfs_s3_types::S3Result<rustfs_gateway::dto::ListObjectsV2Output> {
usecase.execute_list_objects_v2(envelope, input).await
}
+3 -3
View File
@@ -23,7 +23,7 @@ mod head_object;
mod list_objects_v2;
mod put_object;
pub(crate) use get_object::{get_object, get_object_fields};
pub(crate) use head_object::{head_object, head_object_fields};
pub(crate) use list_objects_v2::{list_objects_v2, list_objects_v2_owned};
pub(crate) use get_object::get_object;
pub(crate) use head_object::head_object;
pub(crate) use list_objects_v2::list_objects_v2;
pub(crate) use put_object::put_object;
+10 -9
View File
@@ -12,18 +12,19 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::app::RequestEnvelope;
use crate::app::object_usecase::{DefaultObjectUsecase, PutObjectWriteInput, PutObjectWriteOutput};
use crate::app::object_usecase::DefaultObjectUsecase;
use crate::app::storage_api::s3::{PutObjectInput, PutObjectOutput, S3Request, S3Response, S3Result};
use crate::app::storage_api::test::ecfs::FS;
use std::future::Future;
/// Calls `DefaultObjectUsecase::execute_put_object_owned`, the entry point the gateway handler calls.
/// Calls `DefaultObjectUsecase::execute_put_object` with today's arguments.
///
/// Owned by the PutObject batch (D). Tests outside that batch's files call this seam instead of the use-case method;
/// when batch D changes the use-case signature it changes only this body.
pub(crate) fn put_object(
usecase: &DefaultObjectUsecase,
envelope: RequestEnvelope,
input: PutObjectWriteInput,
) -> impl Future<Output = rustfs_s3_types::S3Result<PutObjectWriteOutput>> + '_ {
Box::pin(usecase.execute_put_object_owned(envelope, input))
pub(crate) fn put_object<'a>(
usecase: &'a DefaultObjectUsecase,
fs: &'a FS,
req: S3Request<PutObjectInput>,
) -> impl Future<Output = S3Result<S3Response<PutObjectOutput>>> + 'a {
usecase.execute_put_object(fs, req)
}
-80
View File
@@ -347,86 +347,6 @@ impl GatewayStack {
}
}
/// The gateway stack over a server context, with requests signed by a root it publishes on that
/// context: for tests that drive an operation through the wire codec, authentication and
/// authorization instead of calling its use case.
#[cfg(test)]
pub(crate) struct TestGateway(GatewayStack);
#[cfg(test)]
impl TestGateway {
pub(crate) const ACCESS_KEY: &str = "gatewaytestroot";
pub(crate) const SECRET_KEY: &str = "gateway-test-root-secret";
/// Publishes the root on `context` and assembles the stack over `slot`, which holds `context`.
pub(crate) fn over(context: &crate::runtime_sources::AppContext, slot: Arc<ServerContextSlot>) -> Self {
assert!(
context.publish_action_credentials(rustfs_credentials::Credentials {
access_key: Self::ACCESS_KEY.to_owned(),
secret_key: Self::SECRET_KEY.to_owned(),
status: "on".to_owned(),
..Default::default()
}),
"the test context has no root yet"
);
let stack = GatewayStack::assemble(GatewayAssembly {
server_ctx: slot,
root_access_key: Self::ACCESS_KEY.to_owned(),
root_secret_key: Self::SECRET_KEY.to_owned(),
region: None,
server_domains: Vec::new(),
body_read_timeout: Duration::from_secs(300),
keystone_auth: None,
})
.expect("the gateway stack assembles");
Self(stack)
}
/// `method target` with `headers` and `body`, header-signed by the root over an unsigned
/// payload. A `Result`, so a call site reads as a service call does.
pub(crate) async fn signed(
&self,
method: http::Method,
target: &str,
headers: Vec<(&str, String)>,
body: bytes::Bytes,
) -> Result<Response<rustfs_gateway::Body>, std::convert::Infallible> {
use crate::server::gateway_handlers::test_support::{Signed, signed_request_with_headers};
let (path, query) = target.split_once('?').unwrap_or((target, ""));
// The signer takes the canonical spelling of a subresource, `uploads=` for `uploads`.
let query = query
.split('&')
.filter(|pair| !pair.is_empty())
.map(|pair| {
if pair.contains('=') {
pair.to_owned()
} else {
format!("{pair}=")
}
})
.collect::<Vec<_>>()
.join("&");
let mut signed_headers = http::HeaderMap::new();
// The length is the body's own, set below; the signer does not sign it.
for (name, value) in headers
.into_iter()
.filter(|(name, _)| !name.eq_ignore_ascii_case("content-length"))
{
signed_headers.append(
http::HeaderName::from_bytes(name.as_bytes()).expect("a header name"),
value.parse().expect("a header value"),
);
}
let request =
signed_request_with_headers(Signed::Header, method, path, &query, Self::ACCESS_KEY, Self::SECRET_KEY, signed_headers);
let mut request = request.map(|_| http_body_util::Full::new(body.clone()));
request
.headers_mut()
.insert(http::header::CONTENT_LENGTH, http::HeaderValue::from(body.len()));
Ok(self.0.call(request).await)
}
}
/// The early gate over `table` and the two metadata listings: an operation it serves is allowed on
/// to its handler, which decides; every other operation is denied.
fn early_gate(table: OperationTable) -> impl Authorizer {
-2
View File
@@ -66,8 +66,6 @@ pub(crate) use event::{
reconcile_event_notifier_from_store, start_persisted_event_notifier_reconciler, with_notify_runtime_reconcile_lock,
};
#[cfg(test)]
pub(crate) use gateway_stack::TestGateway;
#[cfg(test)]
pub(crate) use health::{
HealthPayloadContext, HealthReadinessSource, build_component_details, build_health_payload, health_check_state,
readiness_source_for_probe,
File diff suppressed because it is too large Load Diff
@@ -13,16 +13,18 @@
// limitations under the License.
//! Multipart initiation policy decisions take precedence over generation-read failures.
//!
//! Tests: `authorize_create_multipart_upload` on the owned request, with every write action the
//! initiation can need excluded in turn. Not tested: the listing hooks or the request decoder.
//! Upstream: `storage::authorization::tests::fixture`. Downstream: none.
use crate::storage::access_owned::listing_multipart::{CreateMultipartAuthorization, authorize_create_multipart_upload};
use crate::storage::authorization::tests::fixture::{request, server_context};
use crate::storage::authorization::{AuthorizationRequest as _, BucketGenerationGuard};
use super::access_owned::listing_multipart::{CreateMultipartAuthorization, authorize_create_multipart_upload};
use crate::app::RequestEnvelope;
use crate::runtime_sources::{AppContext, ServerContextSlot};
use crate::storage::access::tests::{TestKms, UnreadyIam, build_request, ensure_req_info};
use crate::storage::access::{
AuthorizationRequest, BucketGenerationGuard, CreateMultipartUploadInput, FS, ObjectLockLegalHoldStatus, ObjectLockMode,
S3Access as _,
};
use crate::storage::storage_api::contract::bucket::{BucketOperations as _, MakeBucketOptions};
use http::Method;
use http::{HeaderValue, Method};
use std::sync::Arc;
use time::OffsetDateTime;
#[test]
@@ -36,7 +38,9 @@ fn create_multipart_authorization_surfaces_generation_errors_after_policy_checks
"s3:PutObjectRetention",
];
let store = crate::app::gating_test_env::shared_gating_ecstore().await;
let slot = server_context(store.clone());
let slot = ServerContextSlot::new();
let fs = FS::with_server_ctx(slot.clone());
assert!(slot.install(Arc::new(AppContext::new(store.clone(), Arc::new(UnreadyIam), Arc::new(TestKms)))));
let bucket = format!("create-auth-order-{}", uuid::Uuid::new_v4().simple());
store
.make_bucket(&bucket, &MakeBucketOptions::default())
@@ -57,15 +61,31 @@ fn create_multipart_authorization_surfaces_generation_errors_after_policy_checks
.await
.expect("publish policy");
let retain_until = OffsetDateTime::now_utc() + time::Duration::days(1);
let mut owned = request(Method::POST)
.header("x-rustfs-source-replication-request", "true")
.server_context(&slot)
.req_info()
.build();
owned.install_bucket_generation(BucketGenerationGuard {
let mut legacy = build_request(
CreateMultipartUploadInput {
bucket: bucket.clone(),
key: "object".to_owned(),
object_lock_legal_hold_status: Some(ObjectLockLegalHoldStatus::from_static("OFF")),
object_lock_mode: Some(ObjectLockMode::from_static("GOVERNANCE")),
object_lock_retain_until_date: Some(retain_until.into()),
..Default::default()
},
Method::POST,
);
legacy
.headers
.insert("x-rustfs-source-replication-request", HeaderValue::from_static("true"));
legacy.extensions.insert(slot.clone());
ensure_req_info(&mut legacy);
legacy.install_bucket_generation(BucketGenerationGuard {
bucket: "other-bucket".to_owned(),
incarnation_id: uuid::Uuid::new_v4(),
});
let mut owned = RequestEnvelope::from_s3s(&legacy);
let legacy = fs
.create_multipart_upload(&mut legacy)
.await
.expect_err("policy or generation refusal");
let owned = authorize_create_multipart_upload(
&slot,
&mut owned,
@@ -81,6 +101,7 @@ fn create_multipart_authorization_surfaces_generation_errors_after_policy_checks
.await
.expect_err("policy or generation refusal");
let expected = if excluded.is_some() { "AccessDenied" } else { "InternalError" };
assert_eq!(legacy.code().as_str(), expected, "legacy {excluded:?}");
assert_eq!(owned.code().as_str(), expected, "owned {excluded:?}");
}
});
-3
View File
@@ -25,6 +25,3 @@ pub(crate) mod object_meta;
pub(crate) mod object_misc;
pub(crate) mod object_read;
pub(crate) mod object_write;
#[cfg(test)]
mod tests;
@@ -12,23 +12,19 @@
// See the License for the specific language governing permissions and
// limitations under the License.
//! The copy-source, copy and completion authorization on the owned request envelope: the
//! decision and the state handed on, allowed and refused.
//!
//! Tests: `authorize_copy_source`, `authorize_copy_object`, `authorize_upload_part_copy` and
//! `authorize_complete_multipart_upload`. Not tested: decoding `x-amz-copy-source` (the facts are
//! written out as the gateway handler derives them) or the copy body.
//! Upstream: `storage::authorization::tests::fixture`. Downstream: none.
//! Parity of the copy-source authorization across both request edges: the same decision and the
//! same state handed on, for an `S3Request` and for a `RequestEnvelope`, in both directions.
use crate::storage::access_owned::object_commit::{
CopyObjectAuthorization, CopySourceObject, authorize_complete_multipart_upload, authorize_copy_object, authorize_copy_source,
authorize_upload_part_copy,
use super::access_owned::object_commit::{CopySourceObject, authorize_copy_source};
use crate::app::{RequestEnvelope, RequestHead};
use crate::storage::access::tests::{TestKms, UnreadyIam, build_request, ensure_req_info};
use crate::storage::access::{
AuthorizationRequest, CopyObjectInput, CopySource, CopySourceBucketGenerationGuard, ECStore, FS, ReqInfo, S3Access as _,
UploadPartCopyInput,
};
use crate::storage::authorization::tests::fixture::{request, server_context};
use crate::storage::authorization::{AuthorizationRequest, CopySourceBucketGenerationGuard};
use crate::storage::storage_api::contract::bucket::{BucketOperations as _, MakeBucketOptions};
use crate::storage::storage_api::runtime_sources_consumer::{ECStore, ServerContextSlot};
use http::Method;
use crate::storage::storage_api::runtime_sources_consumer::{AppContext, ServerContextSlot};
use http::{Extensions, HeaderMap, Method, Uri};
use rustfs_s3_types::{S3ErrorCode, S3Result};
use serial_test::serial;
use std::sync::Arc;
@@ -59,22 +55,42 @@ fn observe<R: AuthorizationRequest>(
}
}
/// Runs the copy-source authorization once with the given facts: `served` is the request's
/// server context, `capture` the one the source incarnation is read from.
async fn on_owned_edge(
/// Runs the copy-source authorization once on each edge with the same facts: `served` is the
/// request's server context, `capture` the one the source incarnation is read from.
async fn on_both_edges(
served: &Arc<ServerContextSlot>,
capture: &ServerContextSlot,
with_req_info: bool,
source: Option<(&str, &str, Option<&str>)>,
) -> Observed {
let object = source.map(|(bucket, key, version_id)| CopySourceObject { bucket, key, version_id });
let mut builder = request(Method::PUT);
if with_req_info {
builder = builder.req_info();
}
let mut envelope = builder.server_context(served).build();
let result = authorize_copy_source(&mut envelope, capture, object).await;
observe(&envelope, result, envelope.copy_source_bucket_generation())
) -> (Observed, Observed) {
let extensions = || {
let mut extensions = Extensions::new();
if with_req_info {
extensions.insert(ReqInfo::default());
}
extensions.insert(Arc::clone(served));
extensions
};
let object = || source.map(|(bucket, key, version_id)| CopySourceObject { bucket, key, version_id });
let mut legacy = build_request((), Method::PUT);
legacy.extensions = extensions();
let result = authorize_copy_source(&mut legacy, capture, object()).await;
let generation = legacy.extensions.get::<CopySourceBucketGenerationGuard>();
let legacy = observe(&legacy, result, generation);
let head = RequestHead {
method: Method::PUT,
uri: Uri::from_static("/"),
headers: HeaderMap::new(),
credentials: None,
region: None,
service: None,
};
let mut envelope = RequestEnvelope::from_parts(head, extensions());
let result = authorize_copy_source(&mut envelope, capture, object()).await;
let owned = observe(&envelope, result, envelope.copy_source_bucket_generation());
(legacy, owned)
}
async fn source_bucket(store: &ECStore, actions: &[&str]) -> String {
@@ -109,8 +125,9 @@ fn copy_source_authorization_decides_and_hands_on_the_same_on_both_edges() {
async fn copy_source_parity_inner() {
let store = crate::app::gating_test_env::shared_gating_ecstore().await;
let server = server_context(Arc::clone(&store));
let unready = ServerContextSlot::new();
let server = Arc::new(ServerContextSlot::new());
assert!(server.install(Arc::new(AppContext::new(Arc::clone(&store), Arc::new(UnreadyIam), Arc::new(TestKms)))));
let unready = Arc::new(ServerContextSlot::new());
let readable = source_bucket(&store, &["s3:GetObject"]).await;
let version_readable = source_bucket(&store, &["s3:GetObjectVersion"]).await;
@@ -221,9 +238,9 @@ async fn copy_source_parity_inner() {
// Every case is checked before failing, so one run names every case a defect breaks.
let mut mismatches = Vec::new();
for (case, slot, with_req_info, source, expected) in cases {
let owned = on_owned_edge(&server, &slot, with_req_info, source).await;
if owned != expected {
mismatches.push(format!("{case}: expected {expected:?}, owned {owned:?}"));
let (legacy, owned) = on_both_edges(&server, &slot, with_req_info, source).await;
if legacy != expected || owned != expected {
mismatches.push(format!("{case}: expected {expected:?}, legacy {legacy:?}, owned {owned:?}"));
}
}
assert!(mismatches.is_empty(), "{}", mismatches.join("\n"));
@@ -239,62 +256,57 @@ fn copy_hooks_decide_the_source_before_the_destination() {
/// `InvalidRequest`; that refusal must never be reached for a source the caller may not read.
async fn copy_hooks_source_first_inner() {
let store = crate::app::gating_test_env::shared_gating_ecstore().await;
let server = server_context(Arc::clone(&store));
let server = ServerContextSlot::new();
assert!(server.install(Arc::new(AppContext::new(Arc::clone(&store), Arc::new(UnreadyIam), Arc::new(TestKms)))));
let fs = FS::with_server_ctx(server);
let closed = source_bucket(&store, &[]).await;
let readable = source_bucket(&store, &["s3:GetObject"]).await;
let destination = source_bucket(&store, &["s3:PutObject"]).await;
let arn = "arn:aws:s3:us-east-1:123456789012:accesspoint/my-access-point/object/key";
// The copy source bucket as the request decoder hands it on (key `key`, no version); `None`
// is an access-point ARN such as
// `arn:aws:s3:us-east-1:123456789012:accesspoint/my-access-point/object/key`.
for (case, source_bucket_name, copy_expected, part_expected) in [
for (case, spelling, copy_expected, part_expected) in [
(
"a denied source",
Some(closed.as_str()),
format!("{closed}/key"),
S3ErrorCode::AccessDenied,
S3ErrorCode::AccessDenied,
),
("an ARN source", None, S3ErrorCode::NotImplemented, S3ErrorCode::NotImplemented),
("an ARN source", arn.to_owned(), S3ErrorCode::NotImplemented, S3ErrorCode::NotImplemented),
// Control: with the source readable, the destination half is reached and refuses.
(
"a readable source",
Some(readable.as_str()),
format!("{readable}/key"),
S3ErrorCode::InvalidRequest,
S3ErrorCode::AccessDenied,
),
] {
let source = || {
source_bucket_name.map(|bucket| CopySourceObject {
bucket,
key: "key",
version_id: None,
})
};
let mut copy = request(Method::PUT).req_info().server_context(&server).build();
let copy_error = authorize_copy_object(
&mut copy,
&server,
CopyObjectAuthorization {
bucket: &destination,
key: "copy",
source: source(),
query_version: None,
tagging: Some("a=b"),
tagging_directive: None,
retain_until: None,
legal_hold: false,
retention: false,
},
)
.await
.expect_err(case);
assert_eq!(copy_error.code(), &copy_expected, "CopyObject: {case}");
let source = || CopySource::parse(&spelling).expect("the request parser accepts the source");
let copy = CopyObjectInput::builder()
.bucket(destination.clone())
.key("copy".to_string())
.copy_source(source())
.tagging(Some("a=b".to_string()))
.build()
.expect("copy input");
let mut copy = build_request(copy, Method::PUT);
ensure_req_info(&mut copy);
copy.extensions.insert(fs.server_ctx().clone());
let copy_error = fs.copy_object(&mut copy).await.expect_err(case);
assert_eq!(S3ErrorCode::from(copy_error.code().clone()), copy_expected, "CopyObject: {case}");
let mut part = request(Method::PUT).req_info().server_context(&server).build();
let part_error = authorize_upload_part_copy(&mut part, &server, &closed, "copy", source())
.await
.expect_err(case);
assert_eq!(part_error.code(), &part_expected, "UploadPartCopy: {case}");
let part = UploadPartCopyInput::builder()
.bucket(closed.clone())
.key("copy".to_string())
.copy_source(source())
.upload_id("unused".to_string())
.part_number(1)
.build()
.expect("part copy input");
let mut part = build_request(part, Method::PUT);
ensure_req_info(&mut part);
part.extensions.insert(fs.server_ctx().clone());
let part_error = fs.upload_part_copy(&mut part).await.expect_err(case);
assert_eq!(S3ErrorCode::from(part_error.code().clone()), part_expected, "UploadPartCopy: {case}");
}
}
@@ -302,7 +314,8 @@ async fn copy_hooks_source_first_inner() {
#[serial]
async fn completion_authorization_requires_write_read_and_replication_permissions() {
let store = crate::app::gating_test_env::shared_gating_ecstore().await;
let server = server_context(Arc::clone(&store));
let server = ServerContextSlot::new();
assert!(server.install(Arc::new(AppContext::new(Arc::clone(&store), Arc::new(UnreadyIam), Arc::new(TestKms)))));
for (actions, specific_etag, replica, allowed) in [
(vec!["s3:PutObject"], false, false, true),
(vec![], false, false, false),
@@ -312,16 +325,40 @@ async fn completion_authorization_requires_write_read_and_replication_permission
(vec!["s3:GetObject"], false, false, false),
] {
let bucket = source_bucket(&store, &actions).await;
let mut builder = request(Method::POST).server_context(&server).req_info();
let mut request = build_request((), Method::POST);
request.extensions.insert(Arc::clone(&server));
request.extensions.insert(ReqInfo::default());
if replica {
builder = builder.header("x-amz-replication-status", "REPLICA");
request
.headers
.insert("x-amz-replication-status", http::HeaderValue::from_static("REPLICA"));
}
let mut envelope = builder.build();
let result = authorize_complete_multipart_upload(&mut envelope, &server, &bucket, "object", specific_etag).await;
if allowed {
result.expect("permitted completion");
} else {
assert_eq!(result.expect_err("completion refused").code(), &S3ErrorCode::AccessDenied);
let mut envelope = RequestEnvelope::from_s3s(&request);
for owned in [false, true] {
let result = if owned {
super::access_owned::object_commit::authorize_complete_multipart_upload(
&mut envelope,
&server,
&bucket,
"object",
specific_etag,
)
.await
} else {
super::access_owned::object_commit::authorize_complete_multipart_upload(
&mut request,
&server,
&bucket,
"object",
specific_etag,
)
.await
};
if allowed {
result.expect("permitted completion");
} else {
assert_eq!(result.expect_err("completion refused").code(), &S3ErrorCode::AccessDenied);
}
}
assert_eq!(envelope.bucket_generation().is_some(), allowed);
assert_eq!(envelope.req_info().unwrap().bucket.as_deref(), Some(bucket.as_str()));
@@ -12,22 +12,20 @@
// See the License for the specific language governing permissions and
// limitations under the License.
//! The DeleteObject decision on the owned request envelope: the answer and the state handed on
//! Both request forms through the one decision: the same answer and the same state handed on
//! (the caller's `ReqInfo` and the guard), allowed and refused.
//!
//! Tests: `decide_delete_object` for anonymous, policy-less, IAM and owner callers, the headers
//! that add actions, and replica deletes. Not tested: the delete handler or the request decoder.
//! Upstream: `storage::authorization::tests::fixture`. Downstream: none.
use crate::storage::access_owned::object_misc::decide_delete_object;
use crate::storage::authorization::tests::fixture::{TestKms, UnreadyIam, request};
use crate::storage::authorization::{AuthorizationRequest, BucketGenerationGuard, PendingDeleteBucketGenerationGuard};
use crate::storage::req_info::ReqInfo;
use super::access_owned::object_misc::decide_delete_object;
use crate::app::RequestEnvelope;
use crate::storage::access::{
AuthorizationRequest, BucketGenerationGuard, PendingDeleteBucketGenerationGuard, ReqInfo, S3Request,
};
use crate::storage::storage_api::contract::bucket::{BucketOperations as _, MakeBucketOptions};
use crate::storage::storage_api::runtime_sources_consumer::{AppContext, IamInterface, ServerContextSlot};
use http::{HeaderMap, Method};
use crate::storage::storage_api::runtime_sources_consumer::{AppContext, IamInterface, KmsInterface, ServerContextSlot};
use http::{Extensions, HeaderMap, HeaderValue, Method, Uri};
use rustfs_iam::store::{Store as _, UserType};
use rustfs_iam::{store::object::ObjectStore, sys::IamSys};
use rustfs_kms::KmsServiceManager;
use rustfs_madmin::{AccountStatus, AddOrUpdateUserReq};
use rustfs_policy::policy::Policy;
use rustfs_s3_types::{S3Error, S3ErrorCode, S3Result};
@@ -169,20 +167,28 @@ fn cases() -> Vec<Case> {
]
}
fn case_request(case: &Case, server_ctx: &Arc<ServerContextSlot>) -> crate::app::RequestEnvelope {
fn request(case: &Case, server_ctx: &Arc<ServerContextSlot>) -> S3Request<()> {
let mut headers = HeaderMap::new();
insert_header(&mut headers, SUFFIX_SOURCE_VERSION_ID, SOURCE_VERSION);
let mut builder = request(Method::DELETE)
.uri("/e2-delete-parity/key")
.headers(headers)
.server_context(server_ctx);
for (name, value) in case.headers {
builder = builder.header(name, value);
headers.insert(*name, HeaderValue::from_static(value));
}
let mut extensions = Extensions::new();
if let Some(caller) = &case.caller {
builder = builder.extension(caller.clone());
extensions.insert(caller.clone());
}
extensions.insert(Arc::clone(server_ctx));
S3Request {
input: (),
method: Method::DELETE,
uri: Uri::from_static("/e2-delete-parity/key"),
headers,
extensions,
credentials: None,
region: None,
service: None,
trailing_headers: None,
}
builder.build()
}
async fn decide<R: AuthorizationRequest + ?Sized>(req: &mut R, bucket: &str, case: &Case) -> Result<Handed, S3ErrorCode> {
@@ -197,14 +203,19 @@ async fn decide<R: AuthorizationRequest + ?Sized>(req: &mut R, bucket: &str, cas
})
}
/// Runs every case; one run names every case a defect breaks.
/// Runs every case in both request forms; one run names every case a defect breaks.
async fn assert_cases(cases: Vec<Case>, bucket: &str, server_ctx: &Arc<ServerContextSlot>) {
let mut wrong = Vec::new();
for case in cases {
let mut envelope = case_request(&case, server_ctx);
let answer = decide(&mut envelope, bucket, &case).await;
if answer != case.expected {
wrong.push(format!("{}: {answer:?}", case.name));
let mut legacy = request(&case, server_ctx);
let mut envelope = RequestEnvelope::from_s3s(&legacy);
for (form, answer) in [
("legacy request", decide(&mut legacy, bucket, &case).await),
("envelope", decide(&mut envelope, bucket, &case).await),
] {
if answer != case.expected {
wrong.push(format!("{} ({form}): {answer:?}", case.name));
}
}
}
assert!(wrong.is_empty(), "{wrong:#?}");
@@ -364,3 +375,23 @@ impl IamInterface for ReadyIam {
true
}
}
struct UnreadyIam;
impl IamInterface for UnreadyIam {
fn handle(&self) -> Arc<IamSys<ObjectStore>> {
panic!("a bucket-policy decision does not resolve IAM")
}
fn is_ready(&self) -> bool {
false
}
}
struct TestKms;
impl KmsInterface for TestKms {
fn handle(&self) -> Arc<KmsServiceManager> {
Arc::new(KmsServiceManager::new())
}
}
@@ -12,18 +12,13 @@
// See the License for the specific language governing permissions and
// limitations under the License.
//! Read authorization tests on the owned request envelope.
//!
//! Tests: `authorize_head_object`, `authorize_get_object` and `authorize_get_object_attributes`
//! against bucket policies, and the resource each leaves in `ReqInfo`. Not tested: the read
//! handlers or the request decoder.
//! Upstream: `storage::authorization::tests::fixture`. Downstream: none.
//! Read authorization tests for the legacy request and owned envelope.
use crate::storage::access_owned::object_read::{authorize_get_object, authorize_get_object_attributes, authorize_head_object};
use crate::storage::authorization::tests::fixture::request;
use super::access_owned::object_read::authorize_head_object;
use crate::app::RequestEnvelope;
use crate::storage::access::{ReqInfo, S3Request, ServerContextSlot};
use crate::storage::storage_api::contract::bucket::{BucketOperations as _, MakeBucketOptions};
use crate::storage::storage_api::runtime_sources_consumer::ServerContextSlot;
use http::Method;
use http::{Extensions, HeaderMap, HeaderValue, Method, Uri};
use rustfs_s3_types::S3ErrorCode;
use std::sync::Arc;
@@ -67,11 +62,23 @@ fn head_replication_marker_requires_its_own_permission() {
]
};
for header in headers {
let mut builder = request(Method::HEAD).uri("/bucket/key").req_info().server_context(&slot);
let mut request = S3Request {
input: (),
method: Method::HEAD,
uri: Uri::from_static("/bucket/key"),
headers: HeaderMap::new(),
extensions: Extensions::new(),
credentials: None,
region: None,
service: None,
trailing_headers: None,
};
request.extensions.insert(ReqInfo::default());
request.extensions.insert(Arc::clone(&slot));
if let Some(header) = header {
builder = builder.header(header, "true");
request.headers.insert(header, HeaderValue::from_static("true"));
}
let mut envelope = builder.build();
let mut envelope = RequestEnvelope::from_s3s(&request);
let result = authorize_head_object(&slot, &mut envelope, &bucket, "key", None).await;
let permitted = header.is_none() || (allow && !deny);
if permitted {
@@ -121,23 +128,41 @@ fn get_authorization_binds_version_and_explicit_denial() {
.into_iter()
.flat_map(|version| ["key", "other"].map(|key| (version, key)))
{
let mut envelope = request(Method::GET)
.uri("/bucket/key")
.req_info()
.server_context(&slot)
.build();
let result = authorize_get_object(&slot, &mut envelope, &bucket, key, version).await;
let mut request = S3Request {
input: (),
method: Method::GET,
uri: Uri::from_static("/bucket/key"),
headers: HeaderMap::new(),
extensions: Extensions::new(),
credentials: None,
region: None,
service: None,
trailing_headers: None,
};
request.extensions.insert(ReqInfo::default());
request.extensions.insert(Arc::clone(&slot));
let mut envelope = RequestEnvelope::from_s3s(&request);
let legacy =
super::access_owned::object_read::authorize_get_object(&slot, &mut request, &bucket, key, version).await;
let native =
super::access_owned::object_read::authorize_get_object(&slot, &mut envelope, &bucket, key, version).await;
let allowed = key == "key" && if version.is_some() { version_allowed } else { current_allowed };
let label = format!("actions={actions:?} deny={explicit_deny} version={version:?} key={key}");
let expected = if allowed { None } else { Some(&S3ErrorCode::AccessDenied) };
if result.as_ref().err().map(|error| error.code()) != expected {
failures.push(format!("{label}: expected {expected:?}, got {result:?}"));
for (edge, result) in [("legacy", legacy), ("native", native)] {
let label = format!("actions={actions:?} deny={explicit_deny} version={version:?} key={key} edge={edge}");
let expected = if allowed { None } else { Some(&S3ErrorCode::AccessDenied) };
if result.as_ref().err().map(|error| error.code()) != expected {
failures.push(format!("{label}: expected {expected:?}, got {result:?}"));
}
}
let info = envelope.req_info().unwrap();
let actual = (info.bucket.as_deref(), info.object.as_deref(), info.version_id.as_deref());
let expected = (Some(bucket.as_str()), Some(key), version);
if actual != expected {
failures.push(format!("{label}: expected resource {expected:?}, got {actual:?}"));
for (edge, info) in [
("legacy", request.extensions.get::<ReqInfo>().unwrap()),
("native", envelope.req_info().unwrap()),
] {
let actual = (info.bucket.as_deref(), info.object.as_deref(), info.version_id.as_deref());
let expected = (Some(bucket.as_str()), Some(key), version);
if actual != expected {
failures.push(format!("{edge}: expected resource {expected:?}, got {actual:?}"));
}
}
}
}
@@ -148,10 +173,12 @@ fn get_authorization_binds_version_and_explicit_denial() {
#[test]
#[serial_test::serial]
fn attributes_authorization_requires_each_action_on_the_original_version() {
use super::super::access_owned::object_read::authorize_get_object_attributes;
crate::app::gating_test_env::run_large_stack_test("attributes-authorization", || async {
let context = crate::app::gating_test_env::shared_gating_ambient().await;
let store = context.object_store();
let slot = ServerContextSlot::new();
let slot = std::sync::Arc::new(ServerContextSlot::new());
assert!(slot.install(Arc::clone(&context)));
for (version, required) in [
(None, ["s3:GetObjectAttributes", "s3:GetObject"]),
@@ -195,23 +222,37 @@ fn attributes_authorization_requires_each_action_on_the_original_version() {
)
.await
.unwrap();
let mut envelope = request(Method::GET)
.uri("/bucket/key?attributes")
.req_info()
.server_context(&slot)
.build();
let result = authorize_get_object_attributes(&mut envelope, &bucket, key, version).await;
let mut request = S3Request {
input: (),
method: Method::GET,
uri: Uri::from_static("/bucket/key?attributes"),
headers: HeaderMap::new(),
extensions: Extensions::new(),
credentials: None,
region: None,
service: None,
trailing_headers: None,
};
request.extensions.insert(ReqInfo::default());
request.extensions.insert(Arc::clone(&slot));
let mut envelope = RequestEnvelope::from_s3s(&request);
let legacy = authorize_get_object_attributes(&mut request, &bucket, key, version).await;
let native = authorize_get_object_attributes(&mut envelope, &bucket, key, version).await;
let expected = (!permitted).then_some(&S3ErrorCode::AccessDenied);
let info = envelope.req_info().unwrap();
assert_eq!(
result.as_ref().err().map(|error| error.code()),
expected,
"allow={allowed:?}, deny={denied:?}, key={key}, version={version:?}",
);
assert_eq!(
(info.bucket.as_deref(), info.object.as_deref(), info.version_id.as_deref()),
(Some(bucket.as_str()), Some(key), version)
);
for (edge, result, info) in [
("legacy", legacy, request.extensions.get::<ReqInfo>().unwrap()),
("native", native, envelope.req_info().unwrap()),
] {
assert_eq!(
result.as_ref().err().map(|error| error.code()),
expected,
"edge={edge}, allow={allowed:?}, deny={denied:?}, key={key}, version={version:?}",
);
assert_eq!(
(info.bucket.as_deref(), info.object.as_deref(), info.version_id.as_deref()),
(Some(bucket.as_str()), Some(key), version)
);
}
}
}
});
@@ -1,655 +0,0 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Tag- and condition-dependent decisions of the owned object authorization functions.
//!
//! Responsible for: the tag evidence a local miss leaves for the replication proxy, the read
//! permission a specific `If-Match` write requires, the version permission an include-deleted
//! listing requires, and the source and destination tag decisions of CopyObject, including the
//! fresh decision the copy use case makes against the source it actually reads.
//! Not responsible for: bucket-generation snapshots (see `generation_guards.rs`), or the policy
//! evaluation itself.
//! Upstream: the owned request fixture in `storage::authorization::tests::fixture`.
//! Downstream: `storage::access_owned::{object_read, object_meta, object_write, object_commit,
//! listing_multipart}` and the CopyObject use case.
use crate::app::RequestEnvelope;
use crate::app::object_usecase::{CopyObjectWriteInput, DefaultObjectUsecase};
use crate::storage::access_owned::listing_multipart::authorize_bucket_listing;
use crate::storage::access_owned::object_commit::{
CopyObjectAuthorization, CopySourceObject, authorize_complete_multipart_upload, authorize_copy_object,
};
use crate::storage::access_owned::object_meta::{authorize_object_tagging_request, authorize_put_object_tagging};
use crate::storage::access_owned::object_read::{authorize_get_object, authorize_head_object};
use crate::storage::access_owned::object_write::{PutObjectAuthorization, authorize_put_object};
use crate::storage::authorization::tests::fixture::{TestKms, UnreadyIam, request, server_context};
use crate::storage::req_info::ReqInfo;
use crate::storage::request_markers::RequestedObjectTags;
use crate::storage::storage_api::contract::bucket::{BucketOperations as _, MakeBucketOptions};
use crate::storage::storage_api::contract::object::{ObjectIO as _, ObjectOperations as _};
use crate::storage::storage_api::runtime_sources_consumer::{AppContext, IamInterface, ServerContextSlot};
use http::Method;
use rustfs_gateway_types::CopySourceValue;
use rustfs_iam::{store::object::ObjectStore, sys::IamSys};
use rustfs_policy::policy::action::S3Action;
use rustfs_s3_types::S3ErrorCode;
use serial_test::serial;
use std::{collections::HashMap, sync::Arc};
/// The copy use case input the legacy edge built from a CopyObject request that carries only
/// these fields.
fn copy_write_input(
bucket: &str,
key: &str,
copy_source: CopySourceValue,
metadata_directive: Option<&str>,
tagging: Option<&str>,
tagging_directive: Option<&str>,
) -> CopyObjectWriteInput {
CopyObjectWriteInput {
bucket: bucket.to_owned(),
key: key.to_owned(),
copy_source: Some(copy_source),
version_id: None,
specific_if_none_match: false,
server_side_encryption: None,
ssekms_key_id: None,
sse_customer_algorithm: None,
sse_customer_key: None,
sse_customer_key_md5: None,
copy_source_sse_customer_algorithm: None,
copy_source_sse_customer_key: None,
copy_source_sse_customer_key_md5: None,
metadata_directive: metadata_directive.map(str::to_owned),
metadata: None,
tagging: tagging.map(str::to_owned),
tagging_directive: tagging_directive.map(str::to_owned),
copy_source_if_match: None,
copy_source_if_none_match: None,
cache_control: None,
content_disposition: None,
content_encoding: None,
content_language: None,
content_type: None,
expires: None,
website_redirect_location: None,
object_lock_legal_hold_status: None,
object_lock_mode: None,
object_lock_retain_until_date: None,
storage_class: None,
checksum_algorithm: None,
}
}
/// CopyObject's authorization facts for a request naming only a bucket-path source, tags and
/// a tagging directive.
fn copy_facts<'a>(
bucket: &'a str,
key: &'a str,
source: CopySourceObject<'a>,
tagging: Option<&'a str>,
tagging_directive: Option<&'a str>,
) -> CopyObjectAuthorization<'a> {
CopyObjectAuthorization {
bucket,
key,
source: Some(source),
query_version: None,
tagging,
tagging_directive,
retain_until: None,
legal_hold: false,
retention: false,
}
}
#[tokio::test]
#[serial]
async fn tag_dependent_access_hooks_preserve_local_miss_for_replication_proxy() {
let store = crate::app::gating_test_env::shared_gating_ecstore().await;
let slot = server_context(Arc::clone(&store));
let bucket = format!("tag-proxy-boundary-{}", uuid::Uuid::new_v4());
store
.make_bucket(&bucket, &MakeBucketOptions::default())
.await
.expect("create bucket");
let policy_json = format!(
r#"{{"Version":"2012-10-17","Statement":[
{{"Effect":"Allow","Principal":{{"AWS":"*"}},"Action":["s3:GetObject","s3:GetObjectTagging","s3:PutObjectTagging","s3:DeleteObjectTagging"],"Resource":["arn:aws:s3:::{bucket}/*"]}},
{{"Effect":"Deny","Principal":{{"AWS":"*"}},"Action":["s3:GetObject","s3:GetObjectTagging","s3:PutObjectTagging","s3:DeleteObjectTagging"],"Resource":["arn:aws:s3:::{bucket}/*"],"Condition":{{"StringEquals":{{"s3:ExistingObjectTag/protected":"yes"}}}}}}
]}}"#
);
let policy_bytes = policy_json.into_bytes();
store
.update_bucket_metadata_config(
&bucket,
crate::storage::storage_api::ecstore_bucket::metadata::BUCKET_POLICY_CONFIG,
policy_bytes,
)
.await
.expect("publish policy");
let check = |envelope: &RequestEnvelope| {
let evidence = envelope
.object_tag_conditions()
.expect("authorization must retain tag dependency");
assert!(evidence.values.is_empty(), "missing local object has no tags");
assert!(
!crate::app::object_meta::tagging::proxy_allowed(envelope),
"local absence cannot authorize a tagged replica"
);
};
let mut envelope = request(Method::GET).req_info().server_context(&slot).build();
authorize_get_object(&slot, &mut envelope, &bucket, "missing", None)
.await
.expect("local miss does not match the tag deny");
check(&envelope);
let mut envelope = request(Method::HEAD).req_info().server_context(&slot).build();
authorize_head_object(&slot, &mut envelope, &bucket, "missing", None)
.await
.expect("local miss does not match the tag deny");
check(&envelope);
let mut envelope = request(Method::GET).req_info().server_context(&slot).build();
authorize_object_tagging_request(&mut envelope, &bucket, "missing", None, S3Action::GetObjectTaggingAction)
.await
.expect("local miss does not match the tag deny");
check(&envelope);
let mut envelope = request(Method::DELETE).req_info().server_context(&slot).build();
authorize_object_tagging_request(&mut envelope, &bucket, "missing", None, S3Action::DeleteObjectTaggingAction)
.await
.expect("local miss does not match the tag deny");
check(&envelope);
let mut envelope = request(Method::PUT).req_info().server_context(&slot).build();
authorize_put_object_tagging(&mut envelope, &bucket, "missing", None, RequestedObjectTags(Vec::new()))
.await
.expect("local miss does not match the tag deny");
check(&envelope);
}
#[tokio::test]
#[serial]
async fn conditional_writes_require_read_permission_only_for_specific_etags() {
let store = crate::app::gating_test_env::shared_gating_ecstore().await;
let slot = server_context(Arc::clone(&store));
for readable in [false, true] {
let bucket = format!("etag-oracle-{}", uuid::Uuid::new_v4());
store
.make_bucket(&bucket, &MakeBucketOptions::default())
.await
.expect("create bucket");
let read_effect = if readable { "Allow" } else { "Deny" };
let policy_json = format!(
r#"{{"Version":"2012-10-17","Statement":[
{{"Effect":"Allow","Principal":{{"AWS":"*"}},"Action":["s3:PutObject"],"Resource":["arn:aws:s3:::{bucket}/*"]}},
{{"Effect":"{read_effect}","Principal":{{"AWS":"*"}},"Action":["s3:GetObject","s3:GetObjectVersion"],"Resource":["arn:aws:s3:::{bucket}/*"]}}
]}}"#
);
let policy_bytes = policy_json.into_bytes();
store
.update_bucket_metadata_config(
&bucket,
crate::storage::storage_api::ecstore_bucket::metadata::BUCKET_POLICY_CONFIG,
policy_bytes,
)
.await
.expect("publish policy");
for condition in ["absent", "wildcard", "dto", "header"] {
let specific = matches!(condition, "dto" | "header");
// The decoded `If-Match` value is specific only for a strong ETag; the wildcard and
// the header-only case leave the decoded value non-specific.
let specific_dto_etag = condition == "dto";
let mut put = request(Method::PUT).req_info().server_context(&slot);
let mut complete = request(Method::POST).req_info().server_context(&slot);
if condition == "header" {
put = put.header("if-match", "\"guessed-etag\"");
complete = complete.header("if-match", "\"guessed-etag\"");
}
let mut put = put.build();
let mut complete = complete.build();
let put_facts = PutObjectAuthorization {
bucket: &bucket,
key: "object",
version_id: None,
retain_until: None,
specific_etag: specific_dto_etag,
legal_hold: false,
retention: false,
post: false,
};
for result in [
authorize_put_object(&mut put, &slot, put_facts, |_| Ok(())).await,
authorize_complete_multipart_upload(&mut complete, &slot, &bucket, "object", specific_dto_etag).await,
] {
if specific && !readable {
assert_eq!(
result
.expect_err("a write-only caller must not compare content fingerprints")
.code(),
&S3ErrorCode::AccessDenied
);
} else {
result.expect("readable or wildcard/ordinary writes remain authorized");
}
}
}
}
}
#[tokio::test]
#[serial]
async fn include_deleted_listing_enforces_version_permissions() {
use rustfs_iam::store::Store as _;
use rustfs_madmin::{AccountStatus, AddOrUpdateUserReq};
struct ReadyIam(Arc<IamSys<ObjectStore>>);
impl IamInterface for ReadyIam {
fn handle(&self) -> Arc<IamSys<ObjectStore>> {
self.0.clone()
}
fn is_ready(&self) -> bool {
true
}
}
let (_directory, _paths, store) = crate::app::gating_test_env::isolated_multi_pool_ecstore().await;
ObjectStore::new(store.clone())
.save_iam_config(
serde_json::json!({"version":1}),
format!("{}/format.json", *rustfs_iam::store::object::IAM_CONFIG_PREFIX),
)
.await
.expect("seed IAM format");
let iam = rustfs_iam::build_iam_sys(store.clone()).await.expect("initialize IAM");
iam.create_user(
"list-reader",
&AddOrUpdateUserReq {
secret_key: "list-reader-secret".to_string(),
policy: None,
status: AccountStatus::Enabled,
},
)
.await
.expect("create reader");
let user = iam.get_user("list-reader").await.expect("reader identity").credentials;
iam.set_policy(
"list-only",
rustfs_policy::policy::Policy::parse_config(
br#"{"Version":"2012-10-17","Statement":[
{"Effect":"Allow","Action":"s3:ListBucket","Resource":"arn:aws:s3:::*"}]}"#,
)
.expect("list policy"),
)
.await
.expect("store identity policy");
iam.policy_db_set("list-reader", rustfs_iam::store::UserType::Reg, false, "list-only")
.await
.expect("attach list policy");
let bucket = format!("deleted-list-auth-{}", uuid::Uuid::new_v4());
store
.make_bucket(&bucket, &MakeBucketOptions::default())
.await
.expect("create bucket");
let slot = ServerContextSlot::new();
assert!(slot.install(Arc::new(AppContext::new(Arc::clone(&store), Arc::new(ReadyIam(iam)), Arc::new(TestKms)))));
for (grant_versions, deny_versions) in [(false, false), (true, false), (true, true)] {
let mut statements = vec![serde_json::json!({
"Effect": "Allow", "Principal": {"AWS": "*"}, "Action": ["s3:ListBucket"], "Resource": format!("arn:aws:s3:::{bucket}")
})];
if grant_versions {
statements.push(serde_json::json!({
"Effect": "Allow", "Principal": {"AWS": "*"}, "Action": ["s3:ListBucketVersions"], "Resource": format!("arn:aws:s3:::{bucket}")
}));
}
if deny_versions {
statements.push(serde_json::json!({
"Effect": "Deny", "Principal": {"AWS": "*"}, "Action": ["s3:ListBucketVersions"], "Resource": format!("arn:aws:s3:::{bucket}")
}));
}
let policy = serde_json::json!({"Version": "2012-10-17", "Statement": statements});
store
.update_bucket_metadata_config(
&bucket,
crate::storage::storage_api::ecstore_bucket::metadata::BUCKET_POLICY_CONFIG,
serde_json::to_vec(&policy).unwrap(),
)
.await
.expect("publish listing policy");
for header in ["x-rustfs-include-deleted", "x-minio-include-deleted"] {
for (value, authenticated) in [("true", true), ("false", true), ("true", false), ("false", false)] {
let mut envelope = request(Method::GET)
.server_context(&slot)
.extension(ReqInfo {
bucket: Some(bucket.clone()),
cred: authenticated.then(|| user.clone()),
..Default::default()
})
.header(header, value)
.build();
let result = authorize_bucket_listing(&mut envelope, &bucket, S3Action::ListBucketAction).await;
if value == "true" && (deny_versions || (authenticated && !grant_versions)) {
assert_eq!(
result.expect_err("version visibility requires permission").code(),
&S3ErrorCode::AccessDenied,
"{header}={value}, authenticated={authenticated}"
);
} else {
result.expect("authorized listings remain available");
}
}
}
}
}
#[test]
fn copy_tags_authorize_source_and_effective_destination_tags() {
crate::app::gating_test_env::run_large_stack_test("copy-tag-authorization", || async {
let store = crate::app::gating_test_env::shared_gating_ecstore().await;
let slot = ServerContextSlot::new();
let context = Arc::new(AppContext::new(store.clone(), Arc::new(UnreadyIam), Arc::new(TestKms)));
assert!(slot.install(context.clone()));
let bucket = format!("copy-tags-{}", uuid::Uuid::new_v4());
store
.make_bucket(&bucket, &MakeBucketOptions::default())
.await
.expect("bucket");
let mut reader = crate::storage::PutObjReader::from_vec(b"confidential source".to_vec());
let source = store
.put_object(
&bucket,
"source",
&mut reader,
&crate::storage::ObjectOptions {
versioned: true,
user_defined: HashMap::from([(
rustfs_utils::http::AMZ_OBJECT_TAGGING.to_string(),
"visibility=private".to_string(),
)]),
..Default::default()
},
)
.await
.expect("tagged historical source");
let historical = source.version_id.expect("versioned source").to_string();
let mut reader = crate::storage::PutObjReader::from_vec(b"current source".to_vec());
store
.put_object(
&bucket,
"source",
&mut reader,
&crate::storage::ObjectOptions {
versioned: true,
user_defined: HashMap::from([(
rustfs_utils::http::AMZ_OBJECT_TAGGING.to_string(),
"visibility=private".to_string(),
)]),
..Default::default()
},
)
.await
.expect("current tagged source");
for (key, tags) in [("source-null", "visibility=private"), ("source-empty", "")] {
let mut reader = crate::storage::PutObjReader::from_vec(b"unversioned source".to_vec());
store
.put_object(
&bucket,
key,
&mut reader,
&crate::storage::ObjectOptions {
user_defined: HashMap::from([(rustfs_utils::http::AMZ_OBJECT_TAGGING.to_string(), tags.to_string())]),
..Default::default()
},
)
.await
.expect("unversioned source");
}
for (index, (directive, replacement, explicit_version, tag_permission, deny_public, expected)) in [
(None, None, None, "missing", false, false),
(Some("COPY"), None, None, "deny", false, false),
(None, None, None, "current", false, true),
(Some("COPY"), None, Some("historical"), "current", false, false),
(Some("COPY"), None, Some("historical"), "version", false, true),
(Some("COPY"), None, Some("null"), "current", false, false),
(Some("COPY"), None, Some("null"), "version", false, true),
(None, None, None, "empty", false, true),
(Some("REPLACE"), Some("visibility=private"), None, "deny", false, true),
(Some("REPLACE"), Some(""), None, "deny", false, true),
(Some("REPLACE"), Some("visibility=public"), None, "deny", true, false),
]
.into_iter()
.enumerate()
{
let mut statements = vec![
serde_json::json!({"Effect":"Allow","Principal":{"AWS":"*"},"Action":["s3:GetObject","s3:GetObjectVersion"],"Resource":format!("arn:aws:s3:::{bucket}/source*")}),
serde_json::json!({"Effect":"Allow","Principal":{"AWS":"*"},"Action":"s3:PutObject","Resource":format!("arn:aws:s3:::{bucket}/copied-*")}),
];
if !matches!(tag_permission, "missing" | "empty") {
let action = if tag_permission == "version" {
"s3:GetObjectVersionTagging"
} else {
"s3:GetObjectTagging"
};
statements.push(serde_json::json!({"Effect":if tag_permission == "deny" { "Deny" } else { "Allow" },"Principal":{"AWS":"*"},"Action":action,"Resource":format!("arn:aws:s3:::{bucket}/source*")}));
}
if deny_public {
statements.push(serde_json::json!({"Effect":"Deny","Principal":{"AWS":"*"},"Action":"s3:PutObject","Resource":format!("arn:aws:s3:::{bucket}/*"),"Condition":{"StringEquals":{"s3:RequestObjectTag/visibility":"public"}}}));
}
let policy = serde_json::to_vec(&serde_json::json!({"Version":"2012-10-17","Statement":statements})).expect("policy");
store
.update_bucket_metadata_config(
&bucket,
crate::storage::storage_api::ecstore_bucket::metadata::BUCKET_POLICY_CONFIG,
policy,
)
.await
.expect("policy publication");
let destination = format!("copied-{index}");
let source_key = if tag_permission == "empty" {
"source-empty"
} else if explicit_version == Some("null") {
"source-null"
} else {
"source"
};
let source_version = explicit_version.map(|version| {
if version == "historical" {
historical.clone()
} else {
"null".to_owned()
}
});
let mut envelope = request(Method::PUT).req_info().server_context(&slot).build();
let facts = copy_facts(
&bucket,
&destination,
CopySourceObject {
bucket: &bucket,
key: source_key,
version_id: source_version.as_deref(),
},
replacement,
directive,
);
let access = authorize_copy_object(&mut envelope, &slot, facts).await;
if !expected {
assert_eq!(
access.expect_err("copy must enforce tagging and destination policy").code(),
&S3ErrorCode::AccessDenied,
"case {index}"
);
assert!(
store
.get_object_info(&bucket, &destination, &crate::storage::ObjectOptions::default())
.await
.is_err()
);
continue;
}
access.expect("authorized copy");
let input = copy_write_input(
&bucket,
&destination,
CopySourceValue::Bucket {
bucket: bucket.clone(),
key: source_key.to_owned(),
version_id: source_version,
},
None,
replacement,
directive,
);
DefaultObjectUsecase::with_context(Some(context.clone()))
.execute_copy_object_owned(envelope, input)
.await
.expect("copy preserves allowed tags");
assert_eq!(
store
.get_object_tags(&bucket, &destination, &crate::storage::ObjectOptions::default())
.await
.expect("destination tags"),
replacement.unwrap_or(if tag_permission == "empty" { "" } else { "visibility=private" })
);
}
});
}
#[test]
fn copy_tags_rechecks_actual_source_after_access_snapshot() {
crate::app::gating_test_env::run_large_stack_test("copy-tag-snapshot", || async {
let store = crate::app::gating_test_env::shared_gating_ecstore().await;
let slot = ServerContextSlot::new();
let context = Arc::new(AppContext::new(store.clone(), Arc::new(UnreadyIam), Arc::new(TestKms)));
assert!(slot.install(context.clone()));
for mode in ["tag-change", "new-tags", "allowed-tags", "source-tag-change", "self-copy"] {
let bucket = format!("copy-snapshot-{}", uuid::Uuid::new_v4());
store
.make_bucket(&bucket, &MakeBucketOptions::default())
.await
.expect("bucket");
let destination = if mode == "self-copy" { "source" } else { "destination" };
let initial = if mode == "new-tags" { "" } else { "visibility=private" };
let mut reader = crate::storage::PutObjReader::from_vec(b"source".to_vec());
store
.put_object(
&bucket,
"source",
&mut reader,
&crate::storage::ObjectOptions {
user_defined: HashMap::from([(rustfs_utils::http::AMZ_OBJECT_TAGGING.to_string(), initial.to_string())]),
..Default::default()
},
)
.await
.expect("source");
let mut statements = vec![
serde_json::json!({"Effect":"Allow","Principal":{"AWS":"*"},"Action":"s3:GetObject","Resource":format!("arn:aws:s3:::{bucket}/source")}),
serde_json::json!({"Effect":"Allow","Principal":{"AWS":"*"},"Action":"s3:PutObject","Resource":format!("arn:aws:s3:::{bucket}/{destination}")}),
serde_json::json!({"Effect":"Deny","Principal":{"AWS":"*"},"Action":"s3:PutObject","Resource":format!("arn:aws:s3:::{bucket}/*"),"Condition":{"StringEquals":{"s3:RequestObjectTag/visibility":"public"}}}),
];
if mode != "new-tags" {
statements.push(serde_json::json!({"Effect":"Allow","Principal":{"AWS":"*"},"Action":"s3:GetObjectTagging","Resource":format!("arn:aws:s3:::{bucket}/source")}));
// A conditional Allow must work at the preliminary hook as well.
statements[1]["Condition"] = serde_json::json!({"StringEquals":{"s3:RequestObjectTag/visibility":"private"}});
if mode == "source-tag-change" {
statements[1].as_object_mut().expect("statement").remove("Condition");
statements.remove(2);
statements.last_mut().expect("tag read statement")["Condition"] =
serde_json::json!({"StringEquals":{"s3:ExistingObjectTag/visibility":"private"}});
} else if mode == "self-copy" {
statements[1]["Condition"]["StringEquals"]["s3:ExistingObjectTag/visibility"] = serde_json::json!("private");
statements.last_mut().expect("tag read statement")["Condition"] =
serde_json::json!({"StringEquals":{"s3:ExistingObjectTag/visibility":"private"}});
}
}
store
.update_bucket_metadata_config(
&bucket,
crate::storage::storage_api::ecstore_bucket::metadata::BUCKET_POLICY_CONFIG,
serde_json::to_vec(&serde_json::json!({"Version":"2012-10-17","Statement":statements})).expect("policy"),
)
.await
.expect("publish policy");
let mut envelope = request(Method::PUT).req_info().server_context(&slot).build();
let facts = copy_facts(
&bucket,
destination,
CopySourceObject {
bucket: &bucket,
key: "source",
version_id: None,
},
None,
None,
);
authorize_copy_object(&mut envelope, &slot, facts)
.await
.expect("preliminary snapshot permits copy");
if !matches!(mode, "allowed-tags" | "self-copy") {
store
.put_object_tags(&bucket, "source", "visibility=public", &crate::storage::ObjectOptions::default())
.await
.expect("change source after access hook");
}
let input = copy_write_input(
&bucket,
destination,
CopySourceValue::Bucket {
bucket: bucket.clone(),
key: "source".to_owned(),
version_id: None,
},
(mode == "self-copy").then_some("REPLACE"),
None,
None,
);
let probe = crate::storage::helper::MutationCompletionProbe::install(&bucket);
let result = tokio::time::timeout(
std::time::Duration::from_secs(10),
DefaultObjectUsecase::with_context(Some(context.clone())).execute_copy_object_owned(envelope, input),
)
.await
.expect("copy authorization must not reacquire its own object lock");
if matches!(mode, "allowed-tags" | "self-copy") {
result.expect("unchanged allowed tags complete");
{
let events = probe.events.lock().unwrap();
assert_eq!(events.len(), 1, "one committed copy event");
assert_eq!(events[0].event_name, rustfs_s3_types::EventName::ObjectCreatedCopy);
assert_eq!(events[0].object.name, destination);
}
assert_eq!(
store
.get_object_tags(&bucket, destination, &crate::storage::ObjectOptions::default())
.await
.expect("copied tags"),
"visibility=private"
);
} else {
assert_eq!(
result.expect_err("actual source tags require fresh authorization").code(),
&S3ErrorCode::AccessDenied
);
assert!(
store
.get_object_info(&bucket, "destination", &crate::storage::ObjectOptions::default())
.await
.is_err(),
"denial must precede destination publication"
);
assert!(probe.events.lock().unwrap().is_empty(), "refused copy cannot emit a success event");
}
}
});
}
@@ -1,757 +0,0 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Bucket-generation snapshots and early refusals of the owned object authorization functions.
//!
//! Responsible for: the destination and copy-source bucket generations an authorization captures
//! and keeps across a recreation of the same bucket name, the use case refusing to act on the
//! recreated bucket, the fail-closed refusal of a request without its generation guard or whose
//! policy metadata is unavailable, the write-offset refusal ahead of authorization, and the
//! request facts DeleteObjects leaves for its use case.
//! Not responsible for: tag- or condition-dependent decisions (see `copy_tags.rs`), or the policy
//! evaluation itself.
//! Upstream: the owned request fixture in `storage::authorization::tests::fixture`.
//! Downstream: `storage::access_owned::{object_write, object_misc, object_commit, listing_multipart}`,
//! the generation guards in `storage::authorization`, and the Delete, Restore, Copy and
//! UploadPartCopy use cases.
use crate::app::multipart_usecase::{DefaultMultipartUsecase, UploadPartCopyWriteInput, UploadPartTarget};
use crate::app::object_usecase::{CopyObjectWriteInput, DefaultObjectUsecase};
use crate::storage::access_owned::listing_multipart::authorize_abort_multipart_upload;
use crate::storage::access_owned::object_commit::{
CopyObjectAuthorization, CopySourceObject, authorize_complete_multipart_upload, authorize_copy_object,
authorize_upload_part_copy,
};
use crate::storage::access_owned::object_misc::{authorize_delete_object, authorize_delete_objects, authorize_restore_object};
use crate::storage::access_owned::object_write::{PutObjectAuthorization, authorize_put_object};
use crate::storage::authorization::tests::fixture::{TestKms, UnreadyIam, request, server_context};
use crate::storage::authorization::{
apply_bucket_generation_guard, apply_copy_source_bucket_generation_guard, existing_object_tag_conditions,
has_write_offset_bytes_header, install_object_generation_test_hook,
};
use crate::storage::req_info::ReqInfo;
use crate::storage::storage_api::contract::bucket::{BucketOperations as _, DeleteBucketOptions, MakeBucketOptions};
use crate::storage::storage_api::contract::multipart::MultipartOperations as _;
use crate::storage::storage_api::contract::object::{ObjectIO as _, ObjectOperations as _};
use crate::storage::storage_api::runtime_sources_consumer::{AppContext, ServerContextSlot};
use http::{HeaderMap, Method};
use rustfs_gateway_types::CopySourceValue;
use rustfs_s3_types::S3ErrorCode;
use serial_test::serial;
use std::sync::Arc;
/// The copy use case input the legacy edge built from a CopyObject request that names only a
/// destination and a bucket-path source.
fn copy_write_input(bucket: &str, key: &str, copy_source: CopySourceValue) -> CopyObjectWriteInput {
CopyObjectWriteInput {
bucket: bucket.to_owned(),
key: key.to_owned(),
copy_source: Some(copy_source),
version_id: None,
specific_if_none_match: false,
server_side_encryption: None,
ssekms_key_id: None,
sse_customer_algorithm: None,
sse_customer_key: None,
sse_customer_key_md5: None,
copy_source_sse_customer_algorithm: None,
copy_source_sse_customer_key: None,
copy_source_sse_customer_key_md5: None,
metadata_directive: None,
metadata: None,
tagging: None,
tagging_directive: None,
copy_source_if_match: None,
copy_source_if_none_match: None,
cache_control: None,
content_disposition: None,
content_encoding: None,
content_language: None,
content_type: None,
expires: None,
website_redirect_location: None,
object_lock_legal_hold_status: None,
object_lock_mode: None,
object_lock_retain_until_date: None,
storage_class: None,
checksum_algorithm: None,
}
}
/// CopyObject's authorization facts for a request naming only a destination and an unversioned
/// bucket-path source.
fn copy_facts<'a>(bucket: &'a str, key: &'a str, source_bucket: &'a str, source_key: &'a str) -> CopyObjectAuthorization<'a> {
CopyObjectAuthorization {
bucket,
key,
source: Some(CopySourceObject {
bucket: source_bucket,
key: source_key,
version_id: None,
}),
query_version: None,
tagging: None,
tagging_directive: None,
retain_until: None,
legal_hold: false,
retention: false,
}
}
/// PutObject's authorization facts for a request naming only a bucket and key.
fn put_facts<'a>(bucket: &'a str, key: &'a str) -> PutObjectAuthorization<'a> {
PutObjectAuthorization {
bucket,
key,
version_id: None,
retain_until: None,
specific_etag: false,
legal_hold: false,
retention: false,
post: false,
}
}
#[tokio::test]
#[serial]
async fn put_object_access_captures_authorized_bucket_incarnation() {
let store = crate::app::gating_test_env::shared_gating_ecstore().await;
let slot = server_context(Arc::clone(&store));
let bucket = "generation-guard-collision";
store
.make_bucket(bucket, &MakeBucketOptions::default())
.await
.expect("test bucket should be created");
let policy_json = format!(
r#"{{"Version":"2012-10-17","Statement":[{{"Effect":"Allow","Principal":{{"AWS":"*"}},"Action":["s3:PutObject"],"Resource":["arn:aws:s3:::{bucket}/*"]}}]}}"#
);
let mut metadata = (*crate::storage::get_bucket_metadata(bucket)
.await
.expect("new bucket metadata should be cached"))
.clone();
metadata.policy_config = Some(serde_json::from_str(&policy_json).expect("test bucket policy should parse"));
metadata.policy_config_json = policy_json.into_bytes();
crate::storage::storage_api::set_bucket_metadata(bucket.to_string(), metadata)
.await
.expect("test bucket policy should be published");
let mut envelope = request(Method::PUT).req_info().server_context(&slot).build();
authorize_put_object(&mut envelope, &slot, put_facts(bucket, bucket), |_| Ok(()))
.await
.expect("anonymous PutObject should be authorized by the test policy");
let mut opts = crate::storage::ObjectOptions::default();
apply_bucket_generation_guard(&envelope, bucket, &mut opts).expect("request snapshot should apply to PutObject options");
assert_eq!(
opts.expected_bucket_incarnation_id,
Some(
store
.bucket_incarnation_id(bucket)
.await
.expect("bucket incarnation should remain readable")
)
);
}
#[test]
#[serial]
fn delete_object_access_keeps_snapshot_across_recreation_before_admission() {
crate::app::gating_test_env::run_large_stack_test(
"delete-object-generation-guard",
delete_object_access_keeps_snapshot_across_recreation_before_admission_inner,
);
}
async fn delete_object_access_keeps_snapshot_across_recreation_before_admission_inner() {
let store = crate::app::gating_test_env::shared_gating_ecstore().await;
let slot = ServerContextSlot::new();
let app_context = Arc::new(AppContext::new(Arc::clone(&store), Arc::new(UnreadyIam), Arc::new(TestKms)));
assert!(slot.install(Arc::clone(&app_context)));
let bucket = format!("delete-generation-guard-{}", uuid::Uuid::new_v4());
store
.make_bucket(&bucket, &MakeBucketOptions::default())
.await
.expect("test bucket should be created");
assert!(
existing_object_tag_conditions(store.as_ref(), &bucket, "missing-object", None)
.await
.expect("a missing object must be treated as having no policy tags")
.is_empty()
);
assert!(
existing_object_tag_conditions(store.as_ref(), &format!("missing-{bucket}"), "missing-object", None)
.await
.expect("a missing bucket must be indistinguishable during pre-authorization tag lookup")
.is_empty()
);
let policy_json = format!(
r#"{{"Version":"2012-10-17","Statement":[{{"Effect":"Allow","Principal":{{"AWS":"*"}},"Action":["s3:DeleteObject"],"Resource":["arn:aws:s3:::{bucket}/*"]}}]}}"#
);
let mut metadata = (*crate::storage::get_bucket_metadata(&bucket)
.await
.expect("new bucket metadata should be cached"))
.clone();
metadata.policy_config = Some(serde_json::from_str(&policy_json).expect("test bucket policy should parse"));
metadata.policy_config_json = policy_json.clone().into_bytes();
crate::storage::storage_api::set_bucket_metadata(bucket.clone(), metadata)
.await
.expect("test bucket policy should be published");
let mut envelope = request(Method::DELETE).req_info().server_context(&slot).build();
let captured_incarnation_id = store
.bucket_incarnation_id(&bucket)
.await
.expect("read the bucket generation before admission");
let (loaded_tx, loaded_rx) = tokio::sync::oneshot::channel();
let (resume_tx, resume_rx) = tokio::sync::oneshot::channel();
install_object_generation_test_hook(bucket.clone(), loaded_tx, resume_rx);
let access = {
let slot = slot.clone();
let bucket = bucket.clone();
tokio::spawn(async move {
let result = authorize_delete_object(&slot, &mut envelope, &bucket, "object", None).await;
(result, envelope)
})
};
tokio::time::timeout(std::time::Duration::from_secs(10), loaded_rx)
.await
.expect("DeleteObject should reach the generation snapshot hook")
.expect("DeleteObject access should remain paused before admission");
store
.delete_bucket(&bucket, &DeleteBucketOptions::default())
.await
.expect("delete the authorized bucket generation");
store
.make_bucket(&bucket, &MakeBucketOptions::default())
.await
.expect("recreate the same bucket name");
let mut metadata = (*crate::storage::get_bucket_metadata(&bucket)
.await
.expect("recreated bucket metadata should be cached"))
.clone();
metadata.policy_config = Some(serde_json::from_str(&policy_json).expect("test policy should parse"));
metadata.policy_config_json = policy_json.into_bytes();
crate::storage::storage_api::set_bucket_metadata(bucket.clone(), metadata)
.await
.expect("republish the same policy for admission");
let mut reader = crate::storage::PutObjReader::from_vec(b"new generation".to_vec());
store
.put_object(&bucket, "object", &mut reader, &crate::storage::ObjectOptions::default())
.await
.expect("put the new-generation object");
assert_ne!(
captured_incarnation_id,
store
.bucket_incarnation_id(&bucket)
.await
.expect("read the recreated bucket generation")
);
resume_tx
.send(())
.expect("DeleteObject should still be paused before admission");
let (result, envelope) = access.await.expect("DeleteObject access task should join");
result.expect("the equivalent policy should authorize DeleteObject");
let mut opts = crate::storage::ObjectOptions::default();
apply_bucket_generation_guard(&envelope, &bucket, &mut opts).expect("request snapshot should apply to DeleteObject options");
assert_eq!(opts.expected_bucket_incarnation_id, Some(captured_incarnation_id));
let err = Box::pin(DefaultObjectUsecase::with_context(Some(app_context)).execute_delete_object_fields(
envelope,
bucket.clone(),
"object".to_string(),
None,
))
.await
.expect_err("the old authorization must not delete from the recreated bucket");
assert_eq!(err.code(), &S3ErrorCode::NoSuchBucket);
store
.get_object_info(&bucket, "object", &crate::storage::ObjectOptions::default())
.await
.expect("the new-generation object must survive the stale request");
}
#[tokio::test]
#[serial]
async fn copy_operations_reject_recreated_source_bucket_after_authorization() {
let store = crate::app::gating_test_env::shared_gating_ecstore().await;
let slot = ServerContextSlot::new();
let app_context = Arc::new(AppContext::new(Arc::clone(&store), Arc::new(UnreadyIam), Arc::new(TestKms)));
assert!(slot.install(Arc::clone(&app_context)));
let suffix = uuid::Uuid::new_v4();
let src_bucket = format!("copy-src-generation-{suffix}");
let dst_bucket = format!("copy-dst-generation-{suffix}");
for bucket in [&src_bucket, &dst_bucket] {
store
.make_bucket(bucket, &MakeBucketOptions::default())
.await
.expect("create copy test bucket");
}
let src_policy_json = format!(
r#"{{"Version":"2012-10-17","Statement":[{{"Effect":"Allow","Principal":{{"AWS":"*"}},"Action":["s3:GetObject"],"Resource":["arn:aws:s3:::{src_bucket}/*"]}}]}}"#
);
let mut src_metadata = (*crate::storage::get_bucket_metadata(&src_bucket)
.await
.expect("source bucket metadata should be cached"))
.clone();
src_metadata.policy_config = Some(serde_json::from_str(&src_policy_json).expect("source policy should parse"));
src_metadata.policy_config_json = src_policy_json.into_bytes();
crate::storage::storage_api::set_bucket_metadata(src_bucket.clone(), src_metadata)
.await
.expect("publish source policy");
let dst_policy_json = format!(
r#"{{"Version":"2012-10-17","Statement":[{{"Effect":"Allow","Principal":{{"AWS":"*"}},"Action":["s3:PutObject"],"Resource":["arn:aws:s3:::{dst_bucket}/*"]}}]}}"#
);
let mut dst_metadata = (*crate::storage::get_bucket_metadata(&dst_bucket)
.await
.expect("destination bucket metadata should be cached"))
.clone();
dst_metadata.policy_config = Some(serde_json::from_str(&dst_policy_json).expect("destination policy should parse"));
dst_metadata.policy_config_json = dst_policy_json.into_bytes();
crate::storage::storage_api::set_bucket_metadata(dst_bucket.clone(), dst_metadata)
.await
.expect("publish destination policy");
let mut envelope = request(Method::PUT).req_info().server_context(&slot).build();
authorize_copy_object(&mut envelope, &slot, copy_facts(&dst_bucket, "copied", &src_bucket, "secret"))
.await
.expect("test policies should authorize the copy request");
let mut source_opts = crate::storage::ObjectOptions::default();
apply_copy_source_bucket_generation_guard(&envelope, &src_bucket, &mut source_opts)
.expect("authorized source generation should be attached");
let authorized_source_incarnation_id = source_opts
.expected_bucket_incarnation_id
.expect("source incarnation should be captured");
let upload = store
.new_multipart_upload(&dst_bucket, "multipart-copy", &crate::storage::ObjectOptions::default())
.await
.expect("create destination multipart upload");
let upload_id = upload.upload_id;
let mut upload_envelope = request(Method::PUT).req_info().server_context(&slot).build();
authorize_upload_part_copy(
&mut upload_envelope,
&slot,
&dst_bucket,
"multipart-copy",
Some(CopySourceObject {
bucket: &src_bucket,
key: "secret",
version_id: None,
}),
)
.await
.expect("test policies should authorize the part copy request");
store
.delete_bucket(&src_bucket, &DeleteBucketOptions::default())
.await
.expect("delete the authorized source generation");
store
.make_bucket(&src_bucket, &MakeBucketOptions::default())
.await
.expect("recreate the source bucket name");
assert_ne!(
authorized_source_incarnation_id,
store
.bucket_incarnation_id(&src_bucket)
.await
.expect("read recreated source incarnation")
);
let mut reader = crate::storage::PutObjReader::from_vec(b"new generation secret".to_vec());
store
.put_object(&src_bucket, "secret", &mut reader, &crate::storage::ObjectOptions::default())
.await
.expect("write the new-generation source object");
let source = || CopySourceValue::Bucket {
bucket: src_bucket.clone(),
key: "secret".to_owned(),
version_id: None,
};
let err = DefaultObjectUsecase::with_context(Some(Arc::clone(&app_context)))
.execute_copy_object_owned(envelope, copy_write_input(&dst_bucket, "copied", source()))
.await
.expect_err("stale authorization must not read from the recreated source bucket");
assert_eq!(err.code(), &S3ErrorCode::NoSuchBucket);
let upload_input = UploadPartCopyWriteInput {
target: UploadPartTarget::Legacy {
bucket: dst_bucket.clone(),
key: "multipart-copy".to_owned(),
upload_id,
},
part_number: 1,
copy_source: source(),
copy_source_range: None,
copy_source_if_match: None,
copy_source_if_none_match: None,
sse_customer_algorithm: None,
sse_customer_key: None,
sse_customer_key_md5: None,
copy_source_sse_customer_algorithm: None,
copy_source_sse_customer_key: None,
copy_source_sse_customer_key_md5: None,
};
let err = DefaultMultipartUsecase::with_context(Some(app_context))
.execute_upload_part_copy_owned(upload_envelope, upload_input)
.await
.err()
.expect("stale authorization must not read the recreated source into a part");
assert_eq!(err.code(), &S3ErrorCode::NoSuchBucket);
store
.get_object_info(&dst_bucket, "copied", &crate::storage::ObjectOptions::default())
.await
.expect_err("the new-generation source object must not be copied");
}
#[tokio::test]
#[serial]
async fn copy_object_rejects_recreated_destination_bucket_after_authorization() {
let store = crate::app::gating_test_env::shared_gating_ecstore().await;
let slot = ServerContextSlot::new();
let context = Arc::new(AppContext::new(store.clone(), Arc::new(UnreadyIam), Arc::new(TestKms)));
assert!(slot.install(context.clone()));
let src = format!("copy-src-{}", uuid::Uuid::new_v4());
let dst = format!("copy-dst-{}", uuid::Uuid::new_v4());
let policy = |bucket: &str, action: &str| {
serde_json::to_vec(&serde_json::json!({"Version":"2012-10-17","Statement":[{
"Effect":"Allow","Principal":{"AWS":"*"},"Action":action,"Resource":format!("arn:aws:s3:::{bucket}/*")
}]}))
.unwrap()
};
for (bucket, action) in [(&src, "s3:GetObject"), (&dst, "s3:PutObject")] {
store.make_bucket(bucket, &MakeBucketOptions::default()).await.unwrap();
store
.update_bucket_metadata_config(
bucket,
crate::storage::storage_api::ecstore_bucket::metadata::BUCKET_POLICY_CONFIG,
policy(bucket, action),
)
.await
.unwrap();
}
let opts = crate::storage::ObjectOptions::default();
let mut body = crate::storage::PutObjReader::from_vec(b"source contents".to_vec());
store.put_object(&src, "source", &mut body, &opts).await.unwrap();
let mut envelope = request(Method::PUT).req_info().server_context(&slot).build();
let captured_incarnation = store.bucket_incarnation_id(&dst).await.unwrap();
let (loaded_tx, loaded_rx) = tokio::sync::oneshot::channel();
let (resume_tx, resume_rx) = tokio::sync::oneshot::channel();
install_object_generation_test_hook(dst.clone(), loaded_tx, resume_rx);
let access = {
let (slot, src, dst) = (slot.clone(), src.clone(), dst.clone());
tokio::spawn(async move {
let result = authorize_copy_object(&mut envelope, &slot, copy_facts(&dst, "copied", &src, "source")).await;
(result, envelope)
})
};
tokio::time::timeout(std::time::Duration::from_secs(10), loaded_rx)
.await
.expect("destination generation capture must be reached")
.expect("copy access ended before capture");
store.delete_bucket(&dst, &DeleteBucketOptions::default()).await.unwrap();
store.make_bucket(&dst, &MakeBucketOptions::default()).await.unwrap();
store
.update_bucket_metadata_config(
&dst,
crate::storage::storage_api::ecstore_bucket::metadata::BUCKET_POLICY_CONFIG,
policy(&dst, "s3:PutObject"),
)
.await
.unwrap();
assert_ne!(captured_incarnation, store.bucket_incarnation_id(&dst).await.unwrap());
resume_tx.send(()).expect("resume copy before admission");
let (result, envelope) = access.await.unwrap();
result.expect("the equivalent policy should admit the copy");
let mut captured = crate::storage::ObjectOptions::default();
apply_bucket_generation_guard(&envelope, &dst, &mut captured).expect("copy carries the destination snapshot");
assert_eq!(captured.expected_bucket_incarnation_id, Some(captured_incarnation));
let input = copy_write_input(
&dst,
"copied",
CopySourceValue::Bucket {
bucket: src.clone(),
key: "source".to_owned(),
version_id: None,
},
);
let error = DefaultObjectUsecase::with_context(Some(context))
.execute_copy_object_owned(envelope, input)
.await
.expect_err("must refuse before publishing to the recreated bucket");
assert_eq!(error.code(), &S3ErrorCode::NoSuchBucket);
let missing = store
.get_object_info(&dst, "copied", &opts)
.await
.expect_err("stale copy wrote an object");
assert!(crate::storage::is_err_object_not_found(&missing), "{missing}");
}
#[test]
#[serial]
fn restore_object_access_keeps_snapshot_across_recreation_before_admission() {
crate::app::gating_test_env::run_large_stack_test(
"restore-object-generation-guard",
restore_object_access_keeps_snapshot_across_recreation_before_admission_inner,
);
}
async fn restore_object_access_keeps_snapshot_across_recreation_before_admission_inner() {
let store = crate::app::gating_test_env::shared_gating_ecstore().await;
let slot = ServerContextSlot::new();
let app_context = Arc::new(AppContext::new(Arc::clone(&store), Arc::new(UnreadyIam), Arc::new(TestKms)));
assert!(slot.install(Arc::clone(&app_context)));
let bucket = format!("restore-generation-guard-{}", uuid::Uuid::new_v4());
store
.make_bucket(&bucket, &MakeBucketOptions::default())
.await
.expect("create the bucket generation being authorized");
let authorized_incarnation_id = store
.bucket_incarnation_id(&bucket)
.await
.expect("read the authorized bucket incarnation");
let policy_json = format!(
r#"{{"Version":"2012-10-17","Statement":[{{"Effect":"Allow","Principal":{{"AWS":"*"}},"Action":["s3:RestoreObject"],"Resource":["arn:aws:s3:::{bucket}/*"]}}]}}"#
);
let mut metadata = (*crate::storage::get_bucket_metadata(&bucket)
.await
.expect("authorized bucket metadata should be cached"))
.clone();
metadata.policy_config = Some(serde_json::from_str(&policy_json).expect("test policy should parse"));
metadata.policy_config_json = policy_json.clone().into_bytes();
crate::storage::storage_api::set_bucket_metadata(bucket.clone(), metadata)
.await
.expect("publish the RestoreObject policy");
let mut envelope = request(Method::POST).req_info().server_context(&slot).build();
let (authorized_tx, authorized_rx) = tokio::sync::oneshot::channel();
let (resume_tx, resume_rx) = tokio::sync::oneshot::channel();
install_object_generation_test_hook(bucket.clone(), authorized_tx, resume_rx);
let access = {
let slot = slot.clone();
let bucket = bucket.clone();
tokio::spawn(async move {
let result = authorize_restore_object(&slot, &mut envelope, &bucket, "object", None).await;
(result, envelope)
})
};
tokio::time::timeout(std::time::Duration::from_secs(10), authorized_rx)
.await
.expect("RestoreObject should reach the generation snapshot hook")
.expect("RestoreObject access must not fail before reaching the test hook");
store
.delete_bucket(&bucket, &DeleteBucketOptions::default())
.await
.expect("delete the authorized bucket generation");
store
.make_bucket(&bucket, &MakeBucketOptions::default())
.await
.expect("recreate the bucket with the same name");
let mut metadata = (*crate::storage::get_bucket_metadata(&bucket)
.await
.expect("recreated bucket metadata should be cached"))
.clone();
metadata.policy_config = Some(serde_json::from_str(&policy_json).expect("test policy should parse"));
metadata.policy_config_json = policy_json.into_bytes();
crate::storage::storage_api::set_bucket_metadata(bucket.clone(), metadata)
.await
.expect("republish the same policy for admission");
let recreated_incarnation_id = store
.bucket_incarnation_id(&bucket)
.await
.expect("read the recreated bucket incarnation");
assert_ne!(authorized_incarnation_id, recreated_incarnation_id);
let replacement = b"recreated bucket object";
store
.put_object(
&bucket,
"object",
&mut crate::storage::PutObjReader::from_vec(replacement.to_vec()),
&crate::storage::ObjectOptions::default(),
)
.await
.expect("write the recreated bucket's object");
resume_tx
.send(())
.expect("RestoreObject access should still be waiting at the test hook");
let (result, envelope) = access.await.expect("RestoreObject access task should join");
result.expect("the equivalent policy should authorize RestoreObject without refreshing its snapshot");
let mut opts = crate::storage::ObjectOptions::default();
apply_bucket_generation_guard(&envelope, &bucket, &mut opts).expect("apply the RestoreObject authorization guard");
// The RestoreObject usecase future is large enough that heap-pinning a
// single future was not enough on Linux: the whole scenario runs on a
// dedicated large stack (see the `#[test]` wrapper above) so unrelated
// growth in bucket metadata futures cannot tip the test over.
let restore_request = crate::app::storage_api::object_usecase::bucket::RestoreRequest {
days: Some(1),
description: None,
glacier_job_parameters: None,
output_location: None,
select_parameters: None,
tier: None,
type_: None,
};
let result = Box::pin(DefaultObjectUsecase::with_context(Some(app_context)).execute_restore_object_owned(
envelope,
bucket.clone(),
"object".to_string(),
None,
Some(restore_request),
))
.await;
use tokio::io::AsyncReadExt as _;
let mut reader = store
.get_object_reader(&bucket, "object", None, HeaderMap::new(), &crate::storage::ObjectOptions::default())
.await
.expect("read the recreated bucket's object");
let mut bytes = Vec::new();
reader.stream.read_to_end(&mut bytes).await.expect("read replacement bytes");
assert_eq!(
(
opts.expected_bucket_incarnation_id,
result.as_ref().err().map(|error| error.code()),
bytes.as_slice(),
reader.object_info.restore_ongoing
),
(
Some(authorized_incarnation_id),
Some(&S3ErrorCode::NoSuchBucket),
replacement.as_slice(),
false
),
"stale RestoreObject must refuse and preserve the recreated bucket's object"
);
drop(reader);
store
.delete_bucket(
&bucket,
&DeleteBucketOptions {
force: true,
..Default::default()
},
)
.await
.expect("clean up the recreated bucket");
}
#[test]
fn dispatched_put_object_fails_closed_without_generation_guard() {
let envelope = request(Method::PUT).server_context(&ServerContextSlot::new()).build();
let mut opts = crate::storage::ObjectOptions::default();
let err = apply_bucket_generation_guard(&envelope, "generation-guard-bucket", &mut opts)
.expect_err("a dispatched PutObject must not commit without its authorization fence");
assert_eq!(err.code(), &S3ErrorCode::InternalError);
}
#[test]
fn write_offset_bytes_header_detection_is_case_insensitive() {
let mut headers = HeaderMap::new();
headers.insert("X-Amz-Write-Offset-Bytes", http::HeaderValue::from_static("0"));
assert!(has_write_offset_bytes_header(&headers));
}
#[tokio::test]
async fn put_object_rejects_write_offset_bytes_before_authorize_request() {
let mut envelope = request(Method::PUT)
.uri("/test-bucket/test-key")
.header("x-amz-write-offset-bytes", "0")
.extension(ReqInfo {
cred: Some(rustfs_credentials::Credentials::default()),
..ReqInfo::default()
})
.build();
let err = authorize_put_object(&mut envelope, &ServerContextSlot::new(), put_facts("test-bucket", "test-key"), |_| Ok(()))
.await
.expect_err("write-offset-bytes requests should be rejected before authorization");
assert_eq!(err.code(), &S3ErrorCode::NotImplemented);
// The message `ApiError::error_code_to_message` gives `NotImplemented`.
assert_eq!(err.message(), Some("A header you provided implies functionality that is not implemented"));
let req_info = envelope.req_info().expect("req info should remain available");
assert_eq!(req_info.bucket.as_deref(), Some("test-bucket"));
assert_eq!(req_info.object.as_deref(), Some("test-key"));
assert_eq!(req_info.version_id, None);
}
#[tokio::test]
async fn delete_objects_defers_object_authorization_to_usecase() {
let mut envelope = request(Method::POST)
.extension(ReqInfo {
cred: Some(rustfs_credentials::Credentials::default()),
bucket: Some("previous-bucket".to_string()),
object: Some("previous-key".to_string()),
version_id: Some("previous-version".to_string()),
..ReqInfo::default()
})
.build();
authorize_delete_objects(&mut envelope, "test-bucket")
.expect("DeleteObjects access hook should not require bucket-level DeleteObject");
let req_info = envelope.req_info().expect("req info should remain available");
assert_eq!(req_info.bucket.as_deref(), Some("test-bucket"));
assert_eq!(req_info.object, None);
assert_eq!(req_info.version_id, None);
}
#[tokio::test]
async fn abort_multipart_upload_fails_closed_when_policy_metadata_is_unavailable() {
let mut envelope = request(Method::DELETE).req_info().build();
let err = authorize_abort_multipart_upload(&ServerContextSlot::new(), &mut envelope, "bucket", "object")
.await
.expect_err("unavailable policy metadata must fail closed");
assert_eq!(err.code(), &S3ErrorCode::InternalError);
}
#[tokio::test]
async fn complete_multipart_upload_fails_closed_when_policy_metadata_is_unavailable() {
let mut envelope = request(Method::POST).req_info().build();
let err = authorize_complete_multipart_upload(&mut envelope, &ServerContextSlot::new(), "bucket", "object", false)
.await
.expect_err("unavailable policy metadata must fail closed");
assert_eq!(err.code(), &S3ErrorCode::InternalError);
}
#[tokio::test]
async fn upload_part_copy_fails_closed_when_policy_metadata_is_unavailable() {
let mut envelope = request(Method::PUT).req_info().build();
let source = CopySourceObject {
bucket: "src-bucket",
key: "src-object",
version_id: None,
};
let err = authorize_upload_part_copy(&mut envelope, &ServerContextSlot::new(), "dst-bucket", "dst-object", Some(source))
.await
.expect_err("unavailable policy metadata must fail closed");
assert_eq!(err.code(), &S3ErrorCode::InternalError);
}
@@ -1,26 +0,0 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Tests of the per-operation authorization shapes, run on the owned request envelope.
mod copy_tags;
mod generation_guards;
mod listing_multipart;
mod object_commit;
mod object_lock_policy;
mod object_misc;
mod object_read;
mod replication_versions;
mod signed_query_lock;
mod write_admission;
@@ -1,740 +0,0 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Which request values an operation's owned authorization hands the policy engine, and what a
//! stored identity or bucket policy decides on them: versioned tagging actions, the
//! `s3:ObjectCreationOperation` condition, legal hold, tagging, retention and POST form values
//! taken from the applied body instead of the wire headers, the decoded copy source, the
//! remaining-retention-days limit, and the replication proxy's tag-evidence gate.
//!
//! Responsible for: running the `storage::access_owned` functions on a `RequestEnvelope` with the
//! facts the gateway edge decodes, then evaluating the resulting conditions.
//! Not responsible for: decoding those facts from the wire (the gateway handlers own that), or
//! the condition builders in isolation (`storage::authorization` tests).
//! Upstream: `crate::storage::authorization::tests::fixture`. Downstream: `storage::access_owned`,
//! `storage::authorization` conditions, and the identity and bucket policy systems.
use crate::app::RequestEnvelope;
use crate::app::object_meta::tagging::proxy_allowed;
use crate::auth::facts::{AuthzPrincipal, AuthzTarget};
use crate::auth::{OBJECT_CREATION_OPERATION_CONDITION, OBJECT_LOCK_REMAINING_RETENTION_DAYS_CONDITION, get_condition_values};
use crate::storage::access_owned::object_commit::{
CopyObjectAuthorization, CopySourceObject, authorize_copy_object, authorize_upload_part_copy,
};
use crate::storage::access_owned::object_meta::{authorize_put_object_legal_hold, authorize_put_object_tagging};
use crate::storage::access_owned::object_write::{
PostObjectFormPolicy, PostObjectPolicyFields, PutObjectAuthorization, authorize_put_object,
};
use crate::storage::authorization::tests::fixture::request;
use crate::storage::authorization::{
AuthorizationRequest, ObjectTagConditions, authorization_conditions, authorization_conditions_with_internal,
request_authz_facts, set_requested_object_lock_retain_until,
};
use crate::storage::authorization_actions::versioned_tagging_action;
use crate::storage::request_markers::{
MultipartStagingOperation, RequestedObjectLegalHold, RequestedObjectLockRetention, RequestedObjectTags,
};
use crate::storage::storage_api::runtime_sources_consumer::ServerContextSlot;
use http::{HeaderValue, Method};
use rustfs_policy::policy::action::{Action, S3Action};
use rustfs_policy::policy::{Args, BucketPolicy, BucketPolicyArgs, Policy};
use std::collections::HashMap;
use time::OffsetDateTime;
use time::format_description::well_known::Rfc3339;
/// The conditions an anonymous principal's decision about `action` evaluates for `req`.
fn remaining_retention_days_conditions(req: &RequestEnvelope, action: Action) -> HashMap<String, Vec<String>> {
let credentials = rustfs_credentials::Credentials::default();
let principal = AuthzPrincipal::new(&credentials, "");
let facts = request_authz_facts(
req,
AuthzTarget {
action,
bucket: "",
object: "",
version_id: None,
location_constraint: None,
},
);
authorization_conditions(req, &principal, &facts).expect("conditions should build")
}
/// A PUT that asks to retain its object for `retain_for` from now.
fn request_retaining_for(retain_for: time::Duration) -> RequestEnvelope {
let mut req = request(Method::PUT).build();
let retain_until = OffsetDateTime::now_utc() + retain_for;
set_requested_object_lock_retain_until(&mut req, Some(retain_until));
req
}
async fn assert_requested_value_denied(action: Action, key: &str, value: &str, conditions: &HashMap<String, Vec<String>>) {
let document = serde_json::json!({
"Version": "2012-10-17",
"Statement": [
{"Effect": "Allow", "Action": action, "Resource": "arn:aws:s3:::bucket/*"},
{"Effect": "Deny", "Action": action, "Resource": "arn:aws:s3:::bucket/*",
"Condition": {"StringEquals": {key: value}}}
]
});
let policy = Policy::parse_config(&serde_json::to_vec(&document).unwrap()).unwrap();
let claims = HashMap::new();
let args = Args {
account: "writer",
groups: &None,
action,
bucket: "bucket",
conditions,
is_owner: false,
object: "object",
claims: &claims,
deny_only: false,
};
assert!(!policy.is_allowed(&args).await, "the applied request value must trigger Deny");
}
#[tokio::test]
async fn versioned_tagging_requires_version_permissions() {
let policy = Policy::parse_config(br#"{
"Version":"2012-10-17",
"Statement":[{"Effect":"Allow","Action":["s3:GetObjectTagging","s3:PutObjectTagging","s3:DeleteObjectTagging"],"Resource":"arn:aws:s3:::bucket/*"}]
}"#).expect("current-object tagging policy should parse");
let conditions = HashMap::new();
let claims = HashMap::new();
let groups = None;
for (current, versioned) in [
(S3Action::GetObjectTaggingAction, S3Action::GetObjectVersionTaggingAction),
(S3Action::PutObjectTaggingAction, S3Action::PutObjectVersionTaggingAction),
(S3Action::DeleteObjectTaggingAction, S3Action::DeleteObjectVersionTaggingAction),
] {
for version in [None, Some("null"), Some("0194e0f1-0000-7000-8000-000000000000")] {
let action = versioned_tagging_action(current, version);
assert_eq!(action, Action::S3Action(if version.is_some() { versioned } else { current }));
let args = Args {
account: "writer",
groups: &groups,
action,
bucket: "bucket",
conditions: &conditions,
is_owner: false,
object: "object",
claims: &claims,
deny_only: false,
};
assert_eq!(policy.is_allowed(&args).await, version.is_none(), "version selector {version:?}");
}
}
}
#[test]
fn object_creation_operation_classifies_only_multipart_staging() {
for (operation, staging) in [
("CreateMultipartUpload", true),
("UploadPart", true),
("UploadPartCopy", true),
("PutObject", false),
("CopyObject", false),
("CompleteMultipartUpload", false),
("PostObject", false),
("GetObject", false),
("InternalObjectWrite", false),
] {
assert_eq!(MultipartStagingOperation::for_operation(operation).is_some(), staging, "{operation}");
}
}
#[test]
fn object_creation_operation_ignores_request_and_claim_spoofing() {
for forged in ["false", "true"] {
for claim in [
"ObjectCreationOperation",
"objectcreationoperation",
"OBJECTCREATIONOPERATION",
"ldapObjectCreationOperation",
"ldapldapObjectCreationOperation",
] {
let credentials = rustfs_credentials::Credentials {
claims: Some(HashMap::from([(claim.to_string(), serde_json::json!(forged))])),
..Default::default()
};
let spoofed = || {
request(Method::GET)
.uri(&format!("/bucket/key?ObjectCreationOperation={forged}&uploadId=fake&partNumber=1"))
.header("objectcreationoperation", forged)
};
assert!(
!get_condition_values(spoofed().build().headers(), &credentials, None, None, None)
.keys()
.any(|key| key.eq_ignore_ascii_case(OBJECT_CREATION_OPERATION_CONDITION))
);
for staging in [false, true] {
let req = if staging {
spoofed().extension(MultipartStagingOperation).build()
} else {
spoofed().build()
};
// The envelope never carries the legacy internal marker; the internal-object
// edge (`authorize_owned_internal_object_request`) passes the flag explicitly.
for internal in [false, true] {
for action in [
S3Action::PutObjectAction,
S3Action::GetObjectAction,
S3Action::GetObjectVersionAction,
] {
let principal = AuthzPrincipal::new(&credentials, "");
let facts = request_authz_facts(
&req,
AuthzTarget {
action: Action::S3Action(action),
bucket: "",
object: "",
version_id: None,
location_constraint: None,
},
);
let conditions = authorization_conditions_with_internal(&req, &principal, &facts, internal)
.expect("conditions should build");
let values = conditions
.iter()
.filter(|(key, _)| key.eq_ignore_ascii_case(OBJECT_CREATION_OPERATION_CONDITION))
.collect::<Vec<_>>();
if action == S3Action::PutObjectAction {
assert_eq!(values.len(), 1, "one canonical server-derived value");
assert_eq!(values[0].0, OBJECT_CREATION_OPERATION_CONDITION);
assert_eq!(
values[0].1,
&vec![(internal || !staging).to_string()],
"{claim}, internal={internal}, staging={staging}"
);
} else {
assert!(values.is_empty(), "non-PutObject action {action:?} must have no creation condition");
}
}
}
}
}
}
}
#[tokio::test]
async fn object_creation_operation_controls_identity_and_bucket_policies() {
for effect in ["Allow", "Deny"] {
for expected in [false, true] {
let mut statements = vec![serde_json::json!({
"Effect": effect, "Action": "s3:PutObject", "Resource": "arn:aws:s3:::bucket/*",
"Condition": { "Bool": { "s3:ObjectCreationOperation": expected.to_string() } }
})];
if effect == "Deny" {
statements.push(serde_json::json!({
"Effect": "Allow", "Action": "s3:PutObject", "Resource": "arn:aws:s3:::bucket/*"
}));
}
let identity: Policy = serde_json::from_str(
&serde_json::json!({
"Version": "2012-10-17", "Statement": statements
})
.to_string(),
)
.expect("identity policy should accept the AWS Bool key");
for statement in &mut statements {
statement["Principal"] = serde_json::json!("*");
}
let bucket: BucketPolicy = serde_json::from_str(
&serde_json::json!({
"Version": "2012-10-17", "Statement": statements
})
.to_string(),
)
.expect("bucket policy should accept the AWS Bool key");
for actual in [None, Some(false), Some(true)] {
let conditions = actual
.map(|value| HashMap::from([("ObjectCreationOperation".to_string(), vec![value.to_string()])]))
.unwrap_or_default();
let allowed = if effect == "Allow" {
actual == Some(expected)
} else {
actual != Some(expected)
};
assert_eq!(
identity
.is_allowed(&Args {
account: "writer",
groups: &None,
action: Action::S3Action(S3Action::PutObjectAction),
bucket: "bucket",
object: "object",
conditions: &conditions,
is_owner: false,
claims: &HashMap::new(),
deny_only: false,
})
.await,
allowed,
"identity {effect}, expected={expected}, actual={actual:?}"
);
assert_eq!(
bucket
.is_allowed(&BucketPolicyArgs {
account: "writer",
groups: &None,
action: Action::S3Action(S3Action::PutObjectAction),
bucket: "bucket",
object: "object",
conditions: &conditions,
is_owner: false,
})
.await,
allowed,
"bucket {effect}, expected={expected}, actual={actual:?}"
);
}
}
}
}
#[test]
fn legal_hold_policy_checks_body_instead_of_conflicting_headers() {
let mut req = request(Method::PUT).header("x-amz-object-lock-legal-hold", "ON").build();
let action = Action::S3Action(S3Action::PutObjectLegalHoldAction);
// Without the applied body value the header leaks into the decision.
let conditions = remaining_retention_days_conditions(&req, action);
assert_eq!(conditions.get("object-lock-legal-hold"), Some(&vec!["ON".to_string()]));
// The XML body `<LegalHold><Status>OFF</Status></LegalHold>`, as the edge decodes it.
req.install_requested_legal_hold(RequestedObjectLegalHold(Some("OFF".to_string())));
let conditions = remaining_retention_days_conditions(&req, action);
assert_eq!(conditions.get("object-lock-legal-hold"), Some(&vec!["OFF".to_string()]));
// A body without a status stores no hold change; the header must not fill it in.
req.install_requested_legal_hold(RequestedObjectLegalHold::default());
let conditions = remaining_retention_days_conditions(&req, action);
assert!(!conditions.contains_key("object-lock-legal-hold"));
}
#[test]
fn tagging_policy_checks_body_instead_of_conflicting_headers() {
let mut req = request(Method::PUT).header("x-amz-tagging", "classification=public").build();
let action = Action::S3Action(S3Action::PutObjectTaggingAction);
// The XML tag set `classification=confidential`, as the edge decodes it.
req.install_requested_object_tags(RequestedObjectTags(vec![("classification".to_string(), "confidential".to_string())]));
let conditions = remaining_retention_days_conditions(&req, action);
assert_eq!(conditions.get("RequestObjectTag/classification"), Some(&vec!["confidential".to_string()]));
assert_eq!(conditions.get("RequestObjectTagKeys"), Some(&vec!["classification".to_string()]));
req.install_requested_object_tags(RequestedObjectTags::default());
let conditions = remaining_retention_days_conditions(&req, action);
assert!(!conditions.contains_key("RequestObjectTag/classification"));
assert_eq!(conditions.get("RequestObjectTagKeys"), Some(&Vec::new()));
}
#[test]
fn post_object_policy_checks_form_fields_instead_of_headers() {
let mut req = request(Method::POST)
.header("x-amz-object-lock-legal-hold", "OFF")
.header("x-amz-object-lock-mode", "GOVERNANCE")
.header("x-amz-object-lock-retain-until-date", "2029-01-01T00:00:00Z")
.header("x-amz-tagging", "classification=public")
.build();
PostObjectFormPolicy::new(PostObjectPolicyFields {
legal_hold: Some("ON"),
lock_mode: Some("COMPLIANCE"),
tagging: Some("classification=confidential"),
..Default::default()
})
.expect("form fields project onto policy headers")
.install(&mut req);
let action = Action::S3Action(S3Action::PutObjectAction);
let conditions = remaining_retention_days_conditions(&req, action);
assert_eq!(conditions.get("object-lock-legal-hold"), Some(&vec!["ON".to_string()]));
assert_eq!(conditions.get("object-lock-mode"), Some(&vec!["COMPLIANCE".to_string()]));
// The form sets no retain-until date, so the header must not stand in for one.
assert!(!conditions.contains_key("object-lock-retain-until-date"));
assert_eq!(conditions.get("RequestObjectTag/classification"), Some(&vec!["confidential".to_string()]));
// A PUT installs no form projection and keeps reading its headers.
let put = request(Method::PUT).header("x-amz-object-lock-legal-hold", "ON").build();
let conditions = remaining_retention_days_conditions(&put, action);
assert_eq!(conditions.get("object-lock-legal-hold"), Some(&vec!["ON".to_string()]));
}
#[test]
fn gateway_post_policy_projects_effective_write_conditions() {
let mut req = request(Method::POST)
.header("if-none-match", "*")
.header("if-none-match", "\"second\"")
.header("x-amz-server-side-encryption", "wire")
.header("x-amz-server-side-encryption-customer-algorithm", "AES256")
.build();
PostObjectFormPolicy::new(PostObjectPolicyFields {
if_match: Some(HeaderValue::from_static("\"form-only\"")),
if_none_match: Some(HeaderValue::from_static("\"form-only\"")),
encryption: Some("form"),
..Default::default()
})
.expect("valid form policy")
.with_metadata(&[("purpose".to_owned(), "form".to_owned())])
.expect("valid form metadata")
.install_gateway(&mut req);
let projected = &req.post_object_policy_headers().expect("projected headers").0;
assert!(!projected.contains_key("if-match"));
assert_eq!(projected.get("if-none-match"), Some(&HeaderValue::from_static("*")));
assert_eq!(projected.get("x-amz-server-side-encryption"), Some(&HeaderValue::from_static("form")));
assert_eq!(projected.get("x-amz-meta-purpose"), Some(&HeaderValue::from_static("form")));
assert_eq!(
projected.get("x-amz-server-side-encryption-customer-algorithm"),
Some(&HeaderValue::from_static("AES256"))
);
assert_eq!(req.headers().get_all("if-none-match").iter().count(), 2);
assert_eq!(req.headers().get("x-amz-server-side-encryption"), Some(&HeaderValue::from_static("wire")));
}
#[tokio::test]
async fn copy_source_conditions_use_the_decoded_source_for_both_copy_operations() {
let mut document = serde_json::json!({"Version":"2012-10-17","Statement":[
{"Effect":"Allow","Action":"s3:PutObject","Resource":"arn:aws:s3:::bucket/*"},
{"Effect":"Deny","Action":"s3:PutObject","Resource":"arn:aws:s3:::bucket/*",
"Condition":{"StringLike":{"s3:x-amz-copy-source":"secret/*"}}}
]});
let identity = Policy::parse_config(&serde_json::to_vec(&document).expect("encode identity policy")).expect("parse policy");
for statement in document["Statement"].as_array_mut().expect("statements") {
statement["Principal"] = serde_json::json!("*");
}
let bucket: BucketPolicy =
serde_json::from_slice(&serde_json::to_vec(&document).expect("encode bucket policy")).expect("parse bucket policy");
// Each header spelling with the source the request parser decodes from it.
for (spelling, source_bucket, source_key, source_version) in [
("secret/report.txt", "secret", "report.txt", None),
("%73ecret/report.txt", "secret", "report.txt", None),
("secret%2Freport.txt", "secret", "report.txt", None),
("/secret/report.txt", "secret", "report.txt", None),
(
"%2Fsecret%2Freport.txt?versionId=00000000-0000-0000-0000-000000000000",
"secret",
"report.txt",
Some("00000000-0000-0000-0000-000000000000"),
),
("public/report.txt", "public", "report.txt", None),
("/public%2Freport.txt", "public", "report.txt", None),
] {
for multipart in [false, true] {
let source = CopySourceObject {
bucket: source_bucket,
key: source_key,
version_id: source_version,
};
let server = ServerContextSlot::new();
let mut req = request(Method::PUT).req_info().build();
if multipart {
let _ = authorize_upload_part_copy(&mut req, &server, "bucket", "copy", Some(source)).await;
} else {
let facts = CopyObjectAuthorization {
bucket: "bucket",
key: "copy",
source: Some(source),
query_version: None,
tagging: None,
tagging_directive: None,
retain_until: None,
legal_hold: false,
retention: false,
};
let _ = authorize_copy_object(&mut req, &server, facts).await;
}
req.headers_mut()
.insert("x-amz-copy-source", HeaderValue::from_str(spelling).expect("source header"));
let action = Action::S3Action(S3Action::PutObjectAction);
let conditions = remaining_retention_days_conditions(&req, action);
let canonical = &conditions["x-amz-copy-source"];
assert_eq!(canonical.len(), 1, "one semantic source, independent of header spelling");
let expected = if spelling.contains("public") {
"public/report.txt"
} else if spelling.contains('?') {
"secret/report.txt?versionId=null"
} else {
"secret/report.txt"
};
assert_eq!(canonical[0], expected, "{spelling}, multipart={multipart}");
let allowed = expected.starts_with("public/");
assert_eq!(
identity
.is_allowed(&Args {
account: "user",
groups: &None,
action,
bucket: "bucket",
object: "copy",
conditions: &conditions,
claims: &HashMap::new(),
is_owner: false,
deny_only: false
})
.await,
allowed
);
assert_eq!(
bucket
.is_allowed(&BucketPolicyArgs {
account: "user",
groups: &None,
action,
bucket: "bucket",
object: "copy",
conditions: &conditions,
is_owner: false
})
.await,
allowed
);
}
}
}
#[tokio::test]
async fn legal_hold_policy_checks_xml_body() {
let action = Action::S3Action(S3Action::PutObjectLegalHoldAction);
for header in [None, Some("ON")] {
let mut builder = request(Method::PUT).req_info();
if let Some(header) = header {
builder = builder.header("x-amz-object-lock-legal-hold", header);
}
let mut req = builder.build();
let server = ServerContextSlot::new();
// The XML body `<LegalHold><Status>OFF</Status></LegalHold>`, as the edge decodes it.
let _ = authorize_put_object_legal_hold(
&mut req,
&server,
"bucket",
"object",
None,
RequestedObjectLegalHold(Some("OFF".to_string())),
)
.await;
let conditions = remaining_retention_days_conditions(&req, action);
assert_eq!(conditions.get("object-lock-legal-hold"), Some(&vec!["OFF".to_string()]));
assert_requested_value_denied(action, "s3:object-lock-legal-hold", "OFF", &conditions).await;
}
}
#[tokio::test]
async fn tagging_policy_checks_xml_body() {
for version in [None, Some("version")] {
let mut req = request(Method::PUT)
.req_info()
.header("x-amz-tagging", "classification=secret&spoofed=yes")
.build();
// The XML tag set `classification=public`, as the edge decodes it.
let _ = authorize_put_object_tagging(
&mut req,
"bucket",
"object",
version,
RequestedObjectTags(vec![("classification".to_string(), "public".to_string())]),
)
.await;
let action = versioned_tagging_action(S3Action::PutObjectTaggingAction, version);
let conditions = remaining_retention_days_conditions(&req, action);
assert_eq!(conditions.get("RequestObjectTag/classification"), Some(&vec!["public".to_string()]));
assert_eq!(conditions.get("RequestObjectTagKeys"), Some(&vec!["classification".to_string()]));
assert!(!conditions.contains_key("RequestObjectTag/spoofed"));
assert_requested_value_denied(action, "s3:RequestObjectTag/classification", "public", &conditions).await;
}
}
#[tokio::test]
async fn post_object_policy_checks_form_fields() {
let mut req = request(Method::POST)
.req_info()
.header("x-amz-object-lock-mode", "GOVERNANCE")
.header("x-amz-object-lock-legal-hold", "OFF")
.header("x-amz-object-lock-retain-until-date", "2029-01-01T00:00:00Z")
.header("x-amz-storage-class", "STANDARD")
.header("x-amz-tagging", "classification=secret&spoofed=yes")
.header("x-amz-acl", "public-read")
.build();
let retain_until = OffsetDateTime::parse("2035-01-01T00:00:00Z", &Rfc3339).unwrap();
let server = ServerContextSlot::new();
// The form fields as the POST edge decodes them; the form names no ACL.
let input = PutObjectAuthorization {
bucket: "bucket",
key: "object",
version_id: None,
retain_until: Some(retain_until),
specific_etag: false,
legal_hold: true,
retention: true,
post: true,
};
let _ = authorize_put_object(&mut req, &server, input, |req| {
PostObjectFormPolicy::new(PostObjectPolicyFields {
lock_mode: Some("COMPLIANCE"),
legal_hold: Some("ON"),
retain_until: Some(retain_until),
storage_class: Some("STANDARD_IA"),
tagging: Some("classification=public"),
..Default::default()
})?
.install(req);
Ok(())
})
.await;
let action = Action::S3Action(S3Action::PutObjectAction);
let conditions = remaining_retention_days_conditions(&req, action);
for (key, value) in [
("object-lock-mode", "COMPLIANCE"),
("object-lock-legal-hold", "ON"),
("object-lock-retain-until-date", "2035-01-01T00:00:00Z"),
("x-amz-storage-class", "STANDARD_IA"),
("RequestObjectTag/classification", "public"),
] {
assert_eq!(conditions.get(key), Some(&vec![value.to_string()]));
}
assert!(!conditions.contains_key("x-amz-acl"));
assert!(!conditions.contains_key("RequestObjectTag/spoofed"));
assert_requested_value_denied(action, "s3:object-lock-mode", "COMPLIANCE", &conditions).await;
assert_requested_value_denied(action, "s3:RequestObjectTag/classification", "public", &conditions).await;
// The final PostObject check reads the same request, so it sees the same form values.
let post_conditions = remaining_retention_days_conditions(&req, action);
assert_eq!(post_conditions.get("object-lock-mode"), conditions.get("object-lock-mode"));
assert_eq!(
post_conditions.get("RequestObjectTag/classification"),
conditions.get("RequestObjectTag/classification")
);
}
#[tokio::test]
async fn retention_policy_checks_body_instead_of_conflicting_headers() {
let mut req = request(Method::PUT)
.header("x-amz-object-lock-mode", "GOVERNANCE")
.header("x-amz-object-lock-retain-until-date", "2029-01-01T00:00:00Z")
.build();
// The XML body's retention, as the edge decodes it.
let applied = OffsetDateTime::parse("2035-01-02T00:00:00+23:59", &Rfc3339).expect("body date should parse");
let retention = RequestedObjectLockRetention {
mode: Some("COMPLIANCE".to_string()),
retain_until: Some(applied),
};
set_requested_object_lock_retain_until(&mut req, retention.retain_until);
req.install_requested_retention(retention);
let action = Action::S3Action(S3Action::PutObjectRetentionAction);
let conditions = remaining_retention_days_conditions(&req, action);
assert_eq!(conditions.get("object-lock-mode"), Some(&vec!["COMPLIANCE".to_string()]));
let authorized =
OffsetDateTime::parse(&conditions["object-lock-retain-until-date"][0], &Rfc3339).expect("condition date should parse");
assert_eq!(authorized.unix_timestamp(), applied.unix_timestamp());
let policy = Policy::parse_config(
br#"{
"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"s3:PutObjectRetention","Resource":"arn:aws:s3:::bucket/*",
"Condition":{"DateLessThan":{"s3:object-lock-retain-until-date":"2030-01-01T00:00:00Z"}}}]
}"#,
)
.expect("bounded retention policy should parse");
let args = Args {
account: "writer",
groups: &None,
action,
bucket: "bucket",
conditions: &conditions,
is_owner: false,
object: "object",
claims: &HashMap::new(),
deny_only: false,
};
assert!(
!policy.is_allowed(&args).await,
"an in-range header cannot authorize an out-of-range body"
);
// A body without retention applies none; the headers must not fill it in.
let retention = RequestedObjectLockRetention::default();
set_requested_object_lock_retain_until(&mut req, retention.retain_until);
req.install_requested_retention(retention);
let conditions = remaining_retention_days_conditions(&req, action);
assert!(!conditions.contains_key("object-lock-mode"));
assert!(!conditions.contains_key("object-lock-retain-until-date"));
}
#[tokio::test]
async fn bucket_policy_limits_retention_to_thirty_days() {
let policy: BucketPolicy = serde_json::from_str(
r#"{
"Version":"2012-10-17",
"Statement":[
{
"Effect":"Allow",
"Principal":{"AWS":"*"},
"Action":["s3:PutObjectRetention"],
"Resource":["arn:aws:s3:::bucket/*"]
},
{
"Effect":"Deny",
"Principal":{"AWS":"*"},
"Action":["s3:PutObjectRetention"],
"Resource":["arn:aws:s3:::bucket/*"],
"Condition":{"NumericGreaterThan":{"s3:object-lock-remaining-retention-days":"30"}}
}
]
}"#,
)
.expect("bucket policy should parse");
let no_groups: Option<Vec<String>> = None;
let action = Action::S3Action(S3Action::PutObjectRetentionAction);
let spoofed_header = |mut req: RequestEnvelope| {
req.headers_mut()
.insert(OBJECT_LOCK_REMAINING_RETENTION_DAYS_CONDITION, HeaderValue::from_static("1"));
req
};
let cases = [
("29 days", request_retaining_for(time::Duration::days(29)), true),
("30 days", request_retaining_for(time::Duration::days(30)), true),
("31 days", request_retaining_for(time::Duration::days(31)), false),
(
"30 days and a minute",
request_retaining_for(time::Duration::days(30) + time::Duration::MINUTE),
false,
),
("no retention", request(Method::PUT).build(), true),
(
"31 days with spoofed header",
spoofed_header(request_retaining_for(time::Duration::days(31))),
false,
),
];
for (name, req, allowed) in cases {
let conditions = remaining_retention_days_conditions(&req, action);
let args = BucketPolicyArgs {
bucket: "bucket",
action,
is_owner: false,
account: "",
groups: &no_groups,
conditions: &conditions,
object: "obj",
};
assert_eq!(policy.is_allowed(&args).await, allowed, "{name}");
}
}
#[test]
fn replication_proxy_requires_authorization_independent_of_local_tags() {
let mut req = request(Method::PUT).build();
assert!(proxy_allowed(&req));
// A local miss produces an empty tag map, but still establishes that the
// policy needs tags. Neither anonymous nor authenticated callers may proxy.
req.install_object_tag_conditions(ObjectTagConditions::new("bucket", "missing", None, HashMap::new()));
assert!(!proxy_allowed(&req));
req.install_object_tag_conditions(ObjectTagConditions::new(
"bucket",
"missing",
Some("version"),
HashMap::from([("ExistingObjectTag/protected".to_string(), vec!["yes".to_string()])]),
));
assert!(!proxy_allowed(&req));
}
@@ -1,323 +0,0 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Replication requests against the owned per-operation authorization: a replication HEAD needs
//! read permission before replication permission, and a replica write or delete is authorized
//! against the version its replication header names.
//!
//! Responsible for: the allow/deny decisions and the `ReqInfo` version that `authorize_head_object`,
//! `authorize_put_object`, `authorize_copy_object`, `authorize_create_multipart_upload` and
//! `authorize_delete_object` reach for replication requests, against real bucket policies.
//! Not responsible for: decoding the replication headers from the wire, or the storage use cases
//! that run after authorization.
//! Upstream: the owned request fixture and the gating test store. Downstream: `storage::access_owned`.
use crate::storage::access_owned::listing_multipart::{CreateMultipartAuthorization, authorize_create_multipart_upload};
use crate::storage::access_owned::object_commit::{CopyObjectAuthorization, CopySourceObject, authorize_copy_object};
use crate::storage::access_owned::object_misc::authorize_delete_object;
use crate::storage::access_owned::object_read::authorize_head_object;
use crate::storage::access_owned::object_write::{PutObjectAuthorization, authorize_put_object};
use crate::storage::authorization::replication_request_authorized;
use crate::storage::authorization::tests::fixture::{request, server_context};
use crate::storage::storage_api::contract::bucket::{BucketOperations as _, MakeBucketOptions};
use crate::storage::storage_api::contract::object::{ObjectIO as _, ObjectOperations as _};
use http::Method;
use rustfs_s3_types::S3ErrorCode;
#[test]
fn replication_head_checks_read_permissions_before_replication_permissions() {
crate::app::gating_test_env::run_large_stack_test("replication-head-permissions", || async {
let store = crate::app::gating_test_env::shared_gating_ecstore().await;
let bucket = format!("replica-head-{}", uuid::Uuid::new_v4());
store
.make_bucket(&bucket, &MakeBucketOptions::default())
.await
.expect("create bucket");
let slot = server_context(store.clone());
for (grant_read, deny_read) in [(false, false), (true, true), (true, false)] {
let mut statements = vec![
serde_json::json!({"Effect":"Allow","Principal":{"AWS":"*"},"Action":"s3:ReplicateObject","Resource":format!("arn:aws:s3:::{bucket}/*")}),
];
for (enabled, effect) in [(grant_read, "Allow"), (deny_read, "Deny")] {
if enabled {
statements.push(serde_json::json!({"Effect":effect,"Principal":{"AWS":"*"},"Action":["s3:GetObject","s3:GetObjectVersion"],"Resource":format!("arn:aws:s3:::{bucket}/*")}));
}
}
store
.update_bucket_metadata_config(
&bucket,
crate::storage::storage_api::ecstore_bucket::metadata::BUCKET_POLICY_CONFIG,
serde_json::to_vec(&serde_json::json!({"Version":"2012-10-17","Statement":statements})).unwrap(),
)
.await
.expect("publish policy");
for version in [None, Some("null"), Some("12345678-1234-1234-1234-123456789abc")] {
for header in ["x-rustfs-source-replication-check", "x-minio-source-replication-check"] {
let mut envelope = request(Method::HEAD)
.req_info()
.server_context(&slot)
.header(header, "true")
.build();
let result = authorize_head_object(&slot, &mut envelope, &bucket, "object", version).await;
if grant_read && !deny_read {
result.expect("read and replication permissions allow metadata check");
assert!(replication_request_authorized(&envelope));
} else {
assert_eq!(
result.expect_err("replication cannot replace read permission").code(),
&S3ErrorCode::AccessDenied
);
assert!(!replication_request_authorized(&envelope));
}
}
}
}
});
}
#[test]
fn replication_write_hooks_authorize_the_destination_version_and_tags() {
crate::app::gating_test_env::run_large_stack_test("replication-version-permissions", || async {
let store = crate::app::gating_test_env::shared_gating_ecstore().await;
let bucket = format!("replica-write-{}", uuid::Uuid::new_v4());
store
.make_bucket(
&bucket,
&MakeBucketOptions {
versioning_enabled: true,
..Default::default()
},
)
.await
.expect("create versioned bucket");
let slot = server_context(store.clone());
let mut versions = Vec::new();
for (body, tags) in [
(b"secret".to_vec(), "classification=secret"),
(b"public".to_vec(), "classification=public"),
] {
let object = store
.put_object(
&bucket,
"target",
&mut crate::storage::PutObjReader::from_vec(body),
&crate::storage::ObjectOptions {
versioned: true,
..Default::default()
},
)
.await
.expect("seed version");
let version = object.version_id.expect("UUID version").to_string();
store
.put_object_tags(
&bucket,
"target",
tags,
&crate::storage::ObjectOptions {
version_id: Some(version.clone()),
versioned: true,
..Default::default()
},
)
.await
.expect("tag version");
versions.push(version);
}
for condition in [
serde_json::json!({"s3:versionid":versions[0]}),
serde_json::json!({"s3:ExistingObjectTag/classification":"secret"}),
] {
let policy = serde_json::json!({"Version":"2012-10-17","Statement":[
{"Effect":"Allow","Principal":{"AWS":"*"},"Action":"s3:*","Resource":format!("arn:aws:s3:::{bucket}/*")},
{"Effect":"Deny","Principal":{"AWS":"*"},"Action":"s3:PutObject","Resource":format!("arn:aws:s3:::{bucket}/target"),"Condition":{"StringEquals":condition}}
]});
store
.update_bucket_metadata_config(
&bucket,
crate::storage::storage_api::ecstore_bucket::metadata::BUCKET_POLICY_CONFIG,
serde_json::to_vec(&policy).unwrap(),
)
.await
.expect("publish version policy");
for (index, version) in versions.iter().enumerate() {
for header in [None, Some("x-rustfs-source-version-id"), Some("x-minio-source-version-id")] {
for operation in ["put", "copy", "multipart"] {
let query_version = header.is_none().then(|| version.clone());
// A query version alone also requires replication authorization.
let mut builder = request(Method::PUT)
.req_info()
.server_context(&slot)
.header("x-rustfs-source-replication-request", "true");
if let Some(header) = header {
builder = builder.header(header, version);
}
let mut envelope = builder.build();
let result = match operation {
"put" => {
let facts = PutObjectAuthorization {
bucket: &bucket,
key: "target",
version_id: query_version.as_deref(),
retain_until: None,
specific_etag: false,
legal_hold: false,
retention: false,
post: false,
};
authorize_put_object(&mut envelope, &slot, facts, |_| Ok(())).await
}
"copy" => {
// `x-amz-copy-source: /{bucket}/source`, no tagging, no Object Lock.
let facts = CopyObjectAuthorization {
bucket: &bucket,
key: "target",
source: Some(CopySourceObject {
bucket: &bucket,
key: "source",
version_id: None,
}),
query_version: query_version.as_deref(),
tagging: None,
tagging_directive: None,
retain_until: None,
legal_hold: false,
retention: false,
};
authorize_copy_object(&mut envelope, &slot, facts).await
}
_ => {
let facts = CreateMultipartAuthorization {
bucket: &bucket,
key: "target",
query_version: query_version.as_deref(),
retain_until: None,
legal_hold_present: false,
retention_present: false,
};
authorize_create_multipart_upload(&slot, &mut envelope, facts).await
}
};
let info = envelope.req_info().unwrap().clone();
assert_eq!(info.version_id.as_deref(), Some(version.as_str()), "{operation}");
if index == 0 {
assert_eq!(
result.expect_err("protected destination version must be denied").code(),
&S3ErrorCode::AccessDenied,
"{operation}"
);
} else {
result.expect("unprotected destination version remains authorized");
}
}
}
}
}
});
}
#[test]
fn replication_delete_hook_authorizes_the_header_selected_version() {
crate::app::gating_test_env::run_large_stack_test("replication-delete-version-permissions", || async {
let store = crate::app::gating_test_env::shared_gating_ecstore().await;
let bucket = format!("replica-delete-{}", uuid::Uuid::new_v4());
store
.make_bucket(
&bucket,
&MakeBucketOptions {
versioning_enabled: true,
..Default::default()
},
)
.await
.expect("create versioned bucket");
let slot = server_context(store.clone());
let mut versions = Vec::new();
for body in [b"protected".to_vec(), b"disposable".to_vec()] {
let object = store
.put_object(
&bucket,
"target",
&mut crate::storage::PutObjReader::from_vec(body),
&crate::storage::ObjectOptions {
versioned: true,
..Default::default()
},
)
.await
.expect("seed version");
versions.push(object.version_id.expect("UUID version").to_string());
}
let policy = serde_json::json!({"Version":"2012-10-17","Statement":[
{"Effect":"Allow","Principal":{"AWS":"*"},"Action":"s3:*","Resource":format!("arn:aws:s3:::{bucket}/*")},
{"Effect":"Deny","Principal":{"AWS":"*"},"Action":"s3:DeleteObjectVersion","Resource":format!("arn:aws:s3:::{bucket}/target"),"Condition":{"StringEquals":{"s3:versionid":versions[0]}}}
]});
store
.update_bucket_metadata_config(
&bucket,
crate::storage::storage_api::ecstore_bucket::metadata::BUCKET_POLICY_CONFIG,
serde_json::to_vec(&policy).unwrap(),
)
.await
.expect("publish version policy");
// The version named only by the replication header must be the one
// the policy sees: a plain `s3:DeleteObject` evaluation would let a
// replica delete remove the protected version. A request without
// the REPLICA status is authorized for the query version alone,
// because storage ignores the header for it (a stray header must
// not switch the evaluated action to `s3:DeleteObjectVersion`).
for (header, version, replica, expect_denied) in [
("x-rustfs-source-version-id", &versions[0], true, true),
("x-minio-source-version-id", &versions[0], true, true),
("x-rustfs-source-version-id", &versions[0], false, false),
("x-rustfs-source-version-id", &versions[1], true, false),
] {
let mut builder = request(Method::DELETE).req_info().server_context(&slot);
if replica {
builder = builder
.header("x-rustfs-source-replication-request", "true")
.header(rustfs_utils::http::headers::AMZ_BUCKET_REPLICATION_STATUS, "REPLICA");
}
let mut envelope = builder.header(header, version).build();
// DeleteObject with no `versionId` query parameter.
let result = authorize_delete_object(&slot, &mut envelope, &bucket, "target", None).await;
let info = envelope.req_info().unwrap();
let expected_version = replica.then_some(version.as_str());
assert_eq!(info.version_id.as_deref(), expected_version, "{header} replica={replica}");
if expect_denied {
assert_eq!(
result
.expect_err("protected version named by the header must be denied")
.code(),
&S3ErrorCode::AccessDenied,
"{header}"
);
} else {
result.expect("a plain delete, or a replica delete of the unprotected version, succeeds");
}
}
store
.get_object_info(
&bucket,
"target",
&crate::storage::ObjectOptions {
version_id: Some(versions[0].clone()),
versioned: true,
..Default::default()
},
)
.await
.expect("the protected version survives the denied replica deletes");
});
}
@@ -1,159 +0,0 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! A presigned PutObject cannot gain an Object Lock legal hold through its query string.
//!
//! Tests: `authorize_put_object` on the owned request for a presigned PUT whose legal hold is a
//! signed header (refused without `s3:PutObjectLegalHold`) and one whose legal hold and user
//! metadata are query parameters (served as an ordinary PUT: the query hoist restores the user
//! metadata and never the legal hold).
//! Not tested: SigV4 verification of the presigned URL, the PutObject decoder, or storing the
//! object.
//! Upstream: `storage::authorization::tests::fixture`. Downstream: none.
use crate::storage::access_owned::object_write::{PutObjectAuthorization, authorize_put_object};
use crate::storage::authorization::resolve_req_info;
use crate::storage::authorization::tests::fixture::request;
use crate::storage::storage_api::contract::bucket::{BucketOperations as _, MakeBucketOptions};
use crate::storage::storage_api::runtime_sources_consumer::{AppContext, ServerContextSlot};
use http::Method;
use rustfs_s3_types::S3ErrorCode;
use std::sync::Arc;
const ACCESS_KEY: &str = "review-local-access";
const SECRET_KEY: &str = "review-local-secret";
const LEGAL_HOLD: &str = "x-amz-object-lock-legal-hold";
/// The URI of a presigned SigV4 PUT. `hoisted` moves the legal hold, and adds user metadata, into
/// the query; otherwise the legal hold is a signed header. The signature is a placeholder: it was
/// verified before authorization, which only reads the presigned marker set.
fn presigned_lock_put_uri(bucket: &str, key: &str, hoisted: bool) -> String {
let signed_headers = if hoisted {
"host"
} else {
"host;x-amz-object-lock-legal-hold"
};
let mut params = std::collections::BTreeMap::from([
("X-Amz-Algorithm", "AWS4-HMAC-SHA256".to_owned()),
("X-Amz-Credential", format!("{ACCESS_KEY}/20260101/us-east-1/s3/aws4_request")),
("X-Amz-Date", "20260101T000000Z".to_owned()),
("X-Amz-Expires", "300".to_owned()),
("X-Amz-SignedHeaders", signed_headers.to_owned()),
]);
if hoisted {
params.insert(LEGAL_HOLD, "ON".to_owned());
params.insert("x-amz-meta-user-id", "u1".to_owned());
}
let query = params
.into_iter()
.map(|(k, v)| format!("{}={}", urlencoding::encode(k), urlencoding::encode(&v)))
.collect::<Vec<_>>()
.join("&");
format!("/{bucket}/{key}?{query}&X-Amz-Signature={}", "0".repeat(64))
}
#[tokio::test]
async fn signed_query_metadata_cannot_add_legal_hold_without_permission() {
let store = crate::app::gating_test_env::shared_gating_ecstore().await;
let iam = rustfs_iam::init_iam_sys(store.clone()).await.expect("test IAM");
let context = Arc::new(AppContext::with_default_interfaces(
store.clone(),
iam,
Arc::new(rustfs_kms::KmsServiceManager::new()),
));
assert!(context.publish_action_credentials(rustfs_credentials::Credentials {
access_key: ACCESS_KEY.to_owned(),
secret_key: SECRET_KEY.to_owned(),
status: "on".to_owned(),
..Default::default()
}));
let slot = ServerContextSlot::new();
assert!(slot.install(context));
let bucket = format!("review-lock-permission-{}", uuid::Uuid::new_v4().simple());
store
.make_bucket(
&bucket,
&MakeBucketOptions {
lock_enabled: true,
..Default::default()
},
)
.await
.expect("locked bucket");
let policy = format!(
r#"{{"Version":"2012-10-17","Statement":[{{"Effect":"Allow","Principal":{{"AWS":"*"}},"Action":["s3:PutObject"],"Resource":["arn:aws:s3:::{bucket}/*"]}},{{"Effect":"Deny","Principal":{{"AWS":"*"}},"Action":["s3:PutObjectLegalHold","s3:PutObjectRetention"],"Resource":["arn:aws:s3:::{bucket}/*"]}}]}}"#
);
let mut metadata = (*crate::storage::get_bucket_metadata(&bucket).await.expect("bucket metadata")).clone();
metadata.policy_config = Some(serde_json::from_str(&policy).expect("bucket policy"));
metadata.policy_config_json = policy.into_bytes();
crate::storage::storage_api::set_bucket_metadata(bucket.clone(), metadata)
.await
.expect("set bucket policy");
// The caller the access phase resolves for the signing key.
let caller = resolve_req_info(&slot, Some(ACCESS_KEY), "", None)
.await
.expect("resolve the signing caller");
let put = |key: &'static str, hoisted: bool| {
let mut builder = request(Method::PUT)
.uri(&presigned_lock_put_uri(&bucket, key, hoisted))
.header("host", "localhost")
.header("content-length", "4")
.header("content-md5", "CY9rzUYh03PK3k6DJie09g==")
.credentials(ACCESS_KEY, SECRET_KEY)
.extension(caller.clone())
.server_context(&slot);
if !hoisted {
builder = builder.header(LEGAL_HOLD, "ON");
}
builder.build()
};
// The facts the PutObject decoder derives: the legal hold is read from the headers only.
let facts = |key: &'static str, legal_hold: bool| PutObjectAuthorization {
bucket: &bucket,
key,
version_id: None,
retain_until: None,
specific_etag: false,
legal_hold,
retention: false,
post: false,
};
let mut header_control = put("header-control", false);
let legal_hold = header_control.headers().contains_key(LEGAL_HOLD);
assert!(legal_hold, "the control sends its legal hold as a signed header");
let error = authorize_put_object(&mut header_control, &slot, facts("header-control", legal_hold), |_| Ok(()))
.await
.expect_err("a signed legal-hold header must reach the independent permission check");
assert_eq!(error.code(), &S3ErrorCode::AccessDenied, "header control: {error:?}");
let mut query_target = put("query-target", true);
let legal_hold = query_target.headers().contains_key(LEGAL_HOLD);
authorize_put_object(&mut query_target, &slot, facts("query-target", legal_hold), |_| Ok(()))
.await
.expect("ordinary metadata PUT should be authorized");
assert!(
!query_target.headers().contains_key(LEGAL_HOLD),
"query metadata must not grant a separate Object Lock capability"
);
assert_eq!(
query_target
.headers()
.get("x-amz-meta-user-id")
.and_then(|value| value.to_str().ok()),
Some("u1"),
"signed query user metadata is restored as a header"
);
}
@@ -1,520 +0,0 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Write and read admission on the owned request: the action a DeleteObject version selects and
//! the bucket-policy tag hint it implies, the PostObject marker, the write-offset refusal that
//! precedes PutObject authorization, the on-demand-migration source capture the read hooks
//! defer to the use case, and the PutObject order between a policy refusal and a generation
//! read-quorum failure.
//!
//! Not covered here: the policy evaluator itself, request-body decoding, or any legacy request
//! type.
//! Upstream: `access_owned::{object_read, object_write, listing_multipart}`, the authorization
//! guards and `storage::authorization::tests::fixture`; downstream: none.
use crate::app::RequestEnvelope;
use crate::app::bucket_usecase::DefaultBucketUsecase;
use crate::app::object_usecase::{DefaultObjectUsecase, GetObjectFields, HeadObjectFields};
use crate::storage::StorageError;
use crate::storage::access_owned::listing_multipart::authorize_list_objects_v2;
use crate::storage::access_owned::object_read::{authorize_get_object, authorize_head_object};
use crate::storage::access_owned::object_write::{PutObjectAuthorization, authorize_post_object, authorize_put_object};
use crate::storage::authorization::tests::fixture::{request, server_context};
use crate::storage::authorization::{
AMZ_WRITE_OFFSET_BYTES_HEADER, BucketPolicyExistingObjectTagHint, OdmReadGenerationGuard, PUT_GENERATION_READ_QUORUM_FAILURE,
bucket_policy_needs_existing_object_tag_from_hint,
};
use crate::storage::authorization_actions::delete_object_authorize_action;
use crate::storage::req_info::ReqInfo;
use crate::storage::storage_api::contract::bucket::{BucketOperations as _, MakeBucketOptions};
use crate::storage::storage_api::contract::object::{ObjectIO as _, ObjectOperations as _};
use crate::storage::storage_api::runtime_sources_consumer::ServerContextSlot;
use http::Method;
use rustfs_gateway_types::dto as gateway_dto;
use rustfs_gateway_types::{BucketName, ByteRange};
use rustfs_policy::policy::action::{Action, S3Action};
use rustfs_policy::policy::{BucketPolicy, BucketPolicyArgs};
use rustfs_s3_types::S3ErrorCode;
use serial_test::serial;
use std::collections::HashMap;
use std::sync::Arc;
#[test]
fn delete_authorization_selects_the_addressed_version() {
assert_eq!(delete_object_authorize_action(None), Action::S3Action(S3Action::DeleteObjectAction));
for version_id in ["null", "8f418ad0-f9f4-4458-83b2-cc72bc6f1b70"] {
assert_eq!(
delete_object_authorize_action(Some(version_id)),
Action::S3Action(S3Action::DeleteObjectVersionAction)
);
}
}
#[tokio::test]
async fn test_anonymous_version_delete_uses_version_policy_and_tag_hint() {
let policy: BucketPolicy = serde_json::from_str(
r#"{
"Version":"2012-10-17",
"Statement":[
{
"Effect":"Allow",
"Principal":{"AWS":"*"},
"Action":["s3:DeleteObject"],
"Resource":["arn:aws:s3:::bucket/*"]
},
{
"Effect":"Allow",
"Principal":{"AWS":"*"},
"Action":["s3:DeleteObjectVersion"],
"Resource":["arn:aws:s3:::bucket/*"],
"Condition":{"StringEquals":{"s3:ExistingObjectTag/security":"public"}}
}
]
}"#,
)
.expect("bucket policy should parse");
let hint = BucketPolicyExistingObjectTagHint::Parsed(policy.clone());
let no_groups: Option<Vec<String>> = None;
let conditions = HashMap::new();
let args_delete = BucketPolicyArgs {
bucket: "bucket",
action: Action::S3Action(S3Action::DeleteObjectAction),
is_owner: false,
account: "",
groups: &no_groups,
conditions: &conditions,
object: "obj",
};
assert!(
policy.is_allowed(&args_delete).await,
"anonymous DeleteObject can be allowed by bucket policy"
);
let args_delete_version = BucketPolicyArgs {
bucket: "bucket",
action: Action::S3Action(S3Action::DeleteObjectVersionAction),
is_owner: false,
account: "",
groups: &no_groups,
conditions: &conditions,
object: "obj",
};
assert!(
!policy.is_allowed(&args_delete_version).await,
"DeleteObjectVersion should still be denied without matching ExistingObjectTag conditions"
);
let needs_tag_current = bucket_policy_needs_existing_object_tag_from_hint(&hint, &args_delete).await;
let needs_tag_version = bucket_policy_needs_existing_object_tag_from_hint(&hint, &args_delete_version).await;
assert!(!needs_tag_current, "DeleteObject statement itself does not require ExistingObjectTag");
assert!(
needs_tag_version,
"DeleteObjectVersion statement requires ExistingObjectTag when version delete is evaluated"
);
assert!(
needs_tag_version,
"the selected version action must require tag fetch for DeleteObject(versionId)"
);
}
#[tokio::test]
async fn post_object_marks_request_extensions() {
let mut req = request(Method::POST).req_info().build();
let _ = authorize_post_object(&mut req, "test-bucket", "test-key", None).await;
assert!(
req.post_object().is_some(),
"post object request should carry the marker for downstream handling"
);
}
#[tokio::test]
async fn put_object_rejects_write_offset_bytes_before_authorization() {
let mut req = request(Method::PUT)
.uri("/test-bucket/test-key")
.header(AMZ_WRITE_OFFSET_BYTES_HEADER, "0")
.req_info()
.build();
let server = ServerContextSlot::new();
let err = authorize_put_object(&mut req, &server, put_object_authorization("test-bucket", "test-key"), |_| Ok(()))
.await
.expect_err("write-offset-bytes requests should be rejected");
assert_eq!(err.code(), &S3ErrorCode::NotImplemented);
// The generic NotImplemented message (`ApiError::error_code_to_message(NotImplemented)`).
assert_eq!(err.message(), Some("A header you provided implies functionality that is not implemented"));
let req_info = req.req_info().expect("request info should remain available");
assert_eq!(req_info.bucket.as_deref(), Some("test-bucket"));
assert_eq!(req_info.object.as_deref(), Some("test-key"));
}
/// The facts the PutObject hook derives from an input that sets nothing beyond bucket and key.
fn put_object_authorization<'a>(bucket: &'a str, key: &'a str) -> PutObjectAuthorization<'a> {
PutObjectAuthorization {
bucket,
key,
version_id: None,
retain_until: None,
specific_etag: false,
legal_hold: false,
retention: false,
post: false,
}
}
/// An owner request as the ODM read tests authorize it.
fn odm_owner_request(method: Method, slot: &Arc<ServerContextSlot>) -> RequestEnvelope {
request(method)
.extension(ReqInfo {
cred: Some(rustfs_credentials::Credentials::default()),
is_owner: true,
..Default::default()
})
.server_context(slot)
.build()
}
fn get_fields(bucket: &str, key: &str, part_number: Option<i32>) -> GetObjectFields {
GetObjectFields {
bucket: bucket.to_owned(),
key: key.to_owned(),
version_id: None,
part_number,
range: None,
sse_customer_key: None,
sse_customer_key_md5: None,
}
}
fn head_fields(bucket: &str, key: &str, part_number: Option<i32>, range: Option<ByteRange>) -> HeadObjectFields {
HeadObjectFields {
bucket: bucket.to_owned(),
key: key.to_owned(),
version_id: None,
part_number,
range,
sse_customer_key: None,
sse_customer_key_md5: None,
}
}
fn list_input(bucket: &str) -> gateway_dto::ListObjectsV2Input {
gateway_dto::ListObjectsV2Input {
bucket: BucketName::new(bucket).expect("test bucket name"),
..Default::default()
}
}
#[test]
#[serial]
fn odm_read_capture_preserves_access_and_parameter_error_order() {
crate::app::gating_test_env::run_large_stack_test("odm-access-order", || async {
let context = crate::app::gating_test_env::shared_gating_ambient().await;
let server_ctx = ServerContextSlot::new();
assert!(server_ctx.install(Arc::clone(&context)));
let sys = crate::on_demand_migration::OnDemandMigrationSys::get();
let enabled_before = sys.is_module_enabled();
let bucket = format!("odm-access-missing-{}", uuid::Uuid::new_v4());
for enabled in [false, true] {
sys.set_module_enabled(enabled);
let mut get = odm_owner_request(Method::GET, &server_ctx);
let mut head = odm_owner_request(Method::HEAD, &server_ctx);
let mut list = odm_owner_request(Method::GET, &server_ctx);
authorize_get_object(&server_ctx, &mut get, &bucket, "key", None)
.await
.expect("ordinary GET access must not require bucket identity");
authorize_head_object(&server_ctx, &mut head, &bucket, "key", None)
.await
.expect("ordinary HEAD access must not require bucket identity");
authorize_list_objects_v2(&server_ctx, &mut list, &bucket)
.await
.expect("ordinary LIST access must not require bucket identity");
assert!(matches!(get.odm_read_generation(), Some(OdmReadGenerationGuard::Unavailable)));
assert!(matches!(head.odm_read_generation(), Some(OdmReadGenerationGuard::Unavailable)));
assert!(matches!(list.odm_read_generation(), Some(OdmReadGenerationGuard::Unavailable)));
let usecase = DefaultObjectUsecase::with_context(Some(Arc::clone(&context)));
assert_eq!(
crate::app::test_seams::get_object_fields(&usecase, get, get_fields(&bucket, "key", Some(0)))
.await
.expect_err("bad GET part number")
.code(),
&S3ErrorCode::InvalidArgument
);
assert_eq!(
crate::app::test_seams::head_object_fields(
&usecase,
head,
head_fields(&bucket, "key", Some(1), Some(ByteRange::FromTo { first: 0, last: 1 }))
)
.await
.expect_err("range and part number conflict")
.code(),
&S3ErrorCode::InvalidArgument
);
assert_eq!(
crate::app::test_seams::list_objects_v2_owned(
&DefaultBucketUsecase::with_context(Some(Arc::clone(&context))),
list,
gateway_dto::ListObjectsV2Input {
max_keys: Some(-1),
..list_input(&bucket)
}
)
.await
.expect_err("negative max keys")
.code(),
&S3ErrorCode::InvalidArgument
);
}
sys.set_module_enabled(enabled_before);
});
}
#[test]
#[serial]
fn odm_read_capture_failure_is_deferred_until_source_miss() {
crate::app::gating_test_env::run_large_stack_test("odm-access-failure", || async {
let context = crate::app::gating_test_env::shared_gating_ambient().await;
let store = context.object_store();
let server_ctx = ServerContextSlot::new();
assert!(server_ctx.install(Arc::clone(&context)));
let sys = crate::on_demand_migration::OnDemandMigrationSys::get();
let enabled_before = sys.is_module_enabled();
sys.set_module_enabled(true);
let bucket = format!("odm-capture-failure-{}", uuid::Uuid::new_v4());
let mut config: crate::on_demand_migration::OnDemandMigrationConfig = serde_json::from_str(r#"{"source":{"provider":"minio","endpoint":"https://source.example.com","region":"us-east-1","bucket":"source","credentials":{"access_key":"test","secret_key":"test"}}}"#).expect("source config");
config.policy.list_through = true;
sys.apply_for_incarnation(&bucket, uuid::Uuid::new_v4(), Some(&config)).await;
let mut get = odm_owner_request(Method::GET, &server_ctx);
let mut head = odm_owner_request(Method::HEAD, &server_ctx);
let mut list = odm_owner_request(Method::GET, &server_ctx);
authorize_list_objects_v2(&server_ctx, &mut list, &bucket)
.await
.expect("capture failure must not preempt LIST authorization");
authorize_get_object(&server_ctx, &mut get, &bucket, "local", None)
.await
.expect("capture failure must not preempt GET authorization");
authorize_head_object(&server_ctx, &mut head, &bucket, "local", None)
.await
.expect("capture failure must not preempt HEAD authorization");
assert!(matches!(get.odm_read_generation(), Some(OdmReadGenerationGuard::Failed { .. })));
assert!(matches!(head.odm_read_generation(), Some(OdmReadGenerationGuard::Failed { .. })));
store
.make_bucket(
&bucket,
&MakeBucketOptions {
versioning_enabled: true,
..Default::default()
},
)
.await
.expect("create bucket after capture");
store
.put_object(
&bucket,
"local",
&mut crate::storage::PutObjReader::from_vec(b"local".to_vec()),
&crate::storage::ObjectOptions::default(),
)
.await
.expect("create local hit");
// Publishing a usable source later must not repair a failed capture.
let incarnation = store.bucket_incarnation_id(&bucket).await.expect("created identity");
sys.apply_for_incarnation(&bucket, incarnation, Some(&config)).await;
let usecase = DefaultObjectUsecase::with_context(Some(Arc::clone(&context)));
let bucket_usecase = || DefaultBucketUsecase::with_context(Some(Arc::clone(&context)));
crate::app::test_seams::get_object_fields(&usecase, get.fork_for_authorization(), get_fields(&bucket, "local", None))
.await
.expect("a local GET hit ignores source capture failure");
crate::app::test_seams::head_object_fields(
&usecase,
head.fork_for_authorization(),
head_fields(&bucket, "local", None, None),
)
.await
.expect("a local HEAD hit ignores source capture failure");
assert_eq!(
crate::app::test_seams::get_object_fields(
&usecase,
get.fork_for_authorization(),
get_fields(&bucket, "missing", None)
)
.await
.expect_err("failed capture cannot rebind on GET miss")
.code(),
&S3ErrorCode::NoSuchBucket
);
assert_eq!(
crate::app::test_seams::head_object_fields(
&usecase,
head.fork_for_authorization(),
head_fields(&bucket, "missing", None, None)
)
.await
.expect_err("failed capture cannot rebind on HEAD miss")
.code(),
&S3ErrorCode::NoSuchBucket
);
assert_eq!(
crate::app::test_seams::list_objects_v2_owned(&bucket_usecase(), list.fork_for_authorization(), list_input(&bucket))
.await
.expect_err("failed capture cannot rebind on LIST")
.code(),
&S3ErrorCode::NoSuchBucket
);
config.filter.prefix = Some("remote/".into());
sys.apply_for_incarnation(&bucket, incarnation, Some(&config)).await;
assert_eq!(
crate::app::test_seams::get_object_fields(
&usecase,
get.fork_for_authorization(),
get_fields(&bucket, "missing", None)
)
.await
.expect_err("filtered GET remains local")
.code(),
&S3ErrorCode::NoSuchKey
);
assert_eq!(
crate::app::test_seams::head_object_fields(
&usecase,
head.fork_for_authorization(),
head_fields(&bucket, "missing", None, None)
)
.await
.expect_err("filtered HEAD remains local")
.code(),
&S3ErrorCode::NoSuchKey
);
let listing = crate::app::test_seams::list_objects_v2_owned(
&bucket_usecase(),
list.fork_for_authorization(),
gateway_dto::ListObjectsV2Input {
prefix: Some("local/".into()),
..list_input(&bucket)
},
)
.await
.expect("disjoint prefix needs no source identity");
assert_eq!(listing.key_count, 0);
let listing = crate::app::test_seams::list_objects_v2_owned(
&bucket_usecase(),
list,
gateway_dto::ListObjectsV2Input {
prefix: Some("remote/".into()),
max_keys: Some(0),
..list_input(&bucket)
},
)
.await
.expect("an empty page needs no source identity");
assert_eq!(listing.key_count, 0);
config.filter.prefix = None;
config.policy.head = crate::on_demand_migration::HeadPolicy::LocalOnly;
sys.apply_for_incarnation(&bucket, incarnation, Some(&config)).await;
assert_eq!(
crate::app::test_seams::head_object_fields(
&usecase,
head.fork_for_authorization(),
head_fields(&bucket, "missing", None, None)
)
.await
.expect_err("local-only HEAD needs no source identity")
.code(),
&S3ErrorCode::NoSuchKey
);
config.policy.head = crate::on_demand_migration::HeadPolicy::Proxy;
sys.apply_for_incarnation(&bucket, incarnation, Some(&config)).await;
store
.delete_object(
&bucket,
"missing",
crate::storage::ObjectOptions {
versioned: true,
..Default::default()
},
)
.await
.expect("create local delete marker");
assert_eq!(
crate::app::test_seams::get_object_fields(&usecase, get, get_fields(&bucket, "missing", None))
.await
.expect_err("GET respects the delete marker before capture failure")
.code(),
&S3ErrorCode::NoSuchKey
);
assert_eq!(
crate::app::test_seams::head_object_fields(&usecase, head, head_fields(&bucket, "missing", None, None))
.await
.expect_err("HEAD respects the delete marker before capture failure")
.code(),
&S3ErrorCode::NoSuchKey
);
sys.remove(&bucket);
sys.set_module_enabled(enabled_before);
});
}
#[tokio::test]
#[serial]
async fn put_read_quorum_access_keeps_authorization_precedence() {
let store = crate::app::gating_test_env::shared_gating_ecstore().await;
let server_ctx = server_context(Arc::clone(&store));
for effect in ["Allow", "Deny"] {
let bucket = format!("put-quorum-access-{}", uuid::Uuid::new_v4());
store
.make_bucket(&bucket, &MakeBucketOptions::default())
.await
.expect("authorization precedence bucket must be created");
let policy_json = format!(
r#"{{"Version":"2012-10-17","Statement":[{{"Effect":"{effect}","Principal":{{"AWS":"*"}},"Action":["s3:PutObject"],"Resource":["arn:aws:s3:::{bucket}/*"]}}]}}"#
);
let mut metadata = (*crate::storage::get_bucket_metadata(&bucket)
.await
.expect("test bucket metadata must be cached"))
.clone();
metadata.policy_config = Some(serde_json::from_str(&policy_json).expect("test policy must parse"));
metadata.policy_config_json = policy_json.into_bytes();
crate::storage::storage_api::set_bucket_metadata(bucket.clone(), metadata)
.await
.expect("test policy must be installed");
let mut req = request(Method::PUT).req_info().server_context(&server_ctx).build();
let err = PUT_GENERATION_READ_QUORUM_FAILURE
.scope(
(),
authorize_put_object(&mut req, &server_ctx, put_object_authorization(&bucket, "object"), |_| Ok(())),
)
.await
.expect_err("a generation failure must never admit a write");
if effect == "Allow" {
assert_eq!(err.code(), &S3ErrorCode::ServiceUnavailable);
assert_eq!(err.status_code(), Some(http::StatusCode::SERVICE_UNAVAILABLE));
assert!(matches!(
err.source().and_then(|source| source.downcast_ref::<StorageError>()),
Some(StorageError::ErasureReadQuorum)
));
} else {
assert_eq!(err.code(), &S3ErrorCode::AccessDenied);
assert_eq!(err.status_code(), Some(http::StatusCode::FORBIDDEN));
}
assert!(req.bucket_generation().is_none());
}
}
@@ -1,34 +1,15 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! The signing facts a permission probe forked from an account-permissions request carries into
//! its authorization conditions: the caller's verified query-signing family survives, its URL
//! capability does not, and an unauthenticated query never becomes a signing fact.
//!
//! Not covered here: the probe's endpoint handling, or decisions for ordinary requests.
//! Upstream: `RequestEnvelope::fork_for_permission_probe`, `request_authz_facts`, and
//! `build_conditions`; downstream: none.
// Copyright 2026 RustFS Team
// SPDX-License-Identifier: Apache-2.0
use super::{
AccessMarkers, Action, AuthorizationRequest, AuthzPrincipal, AuthzTarget, HeaderMap, ReqInfo, S3Action, VerifiedSignature,
build_args, build_conditions, get_request_auth_type_with_query, request_authz_facts,
};
use crate::app::{RequestCredentials, RequestEnvelope, RequestHead};
use crate::auth::facts::{AuthTypeCondition, AuthzPrincipal, AuthzTarget, build_args, build_conditions};
use crate::auth::get_request_auth_type_with_query;
use crate::storage::authorization::{AccessMarkers, AuthorizationRequest, VerifiedSignature, request_authz_facts};
use crate::storage::req_info::ReqInfo;
use http::{Extensions, HeaderMap, Method};
use crate::auth::facts::AuthTypeCondition;
use http::{Extensions, Method};
use rustfs_credentials::Credentials;
use rustfs_policy::policy::Policy;
use rustfs_policy::policy::action::{Action, S3Action};
fn accepted_query_caller(query: &str) -> (RequestEnvelope, Credentials) {
let credentials = Credentials {
@@ -12,35 +12,25 @@
// See the License for the specific language governing permissions and
// limitations under the License.
//! The state an access phase hands to a use case, read back through the `RequestEnvelope`: every
//! reader and installer gives its pinned answer with the state present and absent, and the
//! request-derived condition values come out of the envelope's request facts.
//! Owned-versus-legacy parity of the state an access hook hands to a use case: every reader and
//! installer must answer the same through an s3s request and through the `RequestEnvelope` built
//! from it, with the state present and absent, and each answer is pinned to its expected value so
//! the parity cannot hold vacuously.
//!
//! Not covered here: policy decisions (`app/owned_authorization_test.rs`), or any parity with a
//! legacy request type.
//! Upstream: the readers and installers in `storage::authorization` and the envelope's
//! `AuthorizationRequest` impl; downstream: none.
//! Not covered here: policy decisions (`app/owned_authorization_test.rs`).
//! Upstream: the readers in `storage/access.rs`; downstream: none.
use crate::app::RequestEnvelope;
use crate::shared_types::RemoteAddr;
use crate::storage::ObjectOptions;
use crate::storage::authorization::tests::fixture::{RequestBuilder, request};
use crate::storage::authorization::{
AuthorizationRequest, BucketConfigMutationSnapshot, BucketGenerationGuard, CopySourceBucketGenerationGuard,
OdmReadGenerationGuard, PendingDeleteBucketGenerationGuard, TableDataPlanePublicationGuards, apply_bucket_generation_guard,
use super::{
Action, AuthorizationRequest, BucketConfigMutationSnapshot, BucketGenerationGuard, CopySourceBucketGenerationGuard, HashMap,
HeaderMap, ObjectOptions, OdmReadGenerationGuard, PendingDeleteBucketGenerationGuard, RemoteAddr, ReqInfo, RequestContext,
S3Action, S3ErrorCode, S3Request, TableDataPlanePublicationGuards, apply_bucket_generation_guard,
apply_copy_source_bucket_generation_guard, authorize_object_request, bucket_config_mutation_incarnation,
capture_odm_read_generation, odm_read_generation, replication_request_authorized, request_context_from_req,
resource_free_condition_values, retained_table_data_plane_publication_guards,
};
use crate::storage::req_info::ReqInfo;
use crate::storage::request_context::RequestContext;
use crate::storage::storage_api::runtime_sources_consumer::ServerContextSlot;
use http::header::{HeaderName, HeaderValue};
use http::{HeaderMap, Method};
use rustfs_policy::policy::action::{Action, S3Action};
use rustfs_s3_types::S3ErrorCode;
use crate::app::RequestEnvelope;
use crate::runtime_sources::ServerContextSlot;
use rustfs_trusted_proxies::ClientInfo;
use std::collections::HashMap;
use std::net::SocketAddr;
const BUCKET: &str = "guarded-bucket";
@@ -56,14 +46,24 @@ fn guard(bucket: &str) -> BucketGenerationGuard {
}
}
/// A GET of `/guarded-bucket/key` carrying what `fill` adds.
fn envelope(fill: impl FnOnce(RequestBuilder) -> RequestBuilder) -> RequestEnvelope {
fill(request(Method::GET).uri("/guarded-bucket/key")).build()
fn legacy(fill: impl FnOnce(&mut http::Extensions)) -> S3Request<()> {
let mut extensions = http::Extensions::new();
fill(&mut extensions);
S3Request {
input: (),
method: http::Method::GET,
uri: "/guarded-bucket/key".parse().expect("static uri"),
headers: HeaderMap::new(),
extensions,
credentials: None,
region: None,
service: None,
trailing_headers: None,
}
}
/// Marks the request as one the server received, as the edge does by carrying its context.
fn server_request(builder: RequestBuilder) -> RequestBuilder {
builder.server_context(&ServerContextSlot::new())
fn server_request(extensions: &mut http::Extensions) {
extensions.insert(ServerContextSlot::new());
}
/// The incarnation a reader bound, or the refusal's code and message.
@@ -72,7 +72,7 @@ type Answer = Result<Option<uuid::Uuid>, (S3ErrorCode, String)>;
fn answer<T>(result: rustfs_s3_types::S3Result<T>, bound: impl FnOnce(T) -> Option<uuid::Uuid>) -> Answer {
result
.map(bound)
.map_err(|err| (err.code().clone(), err.message().unwrap_or_default().to_owned()))
.map_err(|err| (err.code().clone().into(), err.message().unwrap_or_default().to_owned()))
}
fn refused(code: S3ErrorCode, message: &str) -> Answer {
@@ -99,10 +99,16 @@ fn config_mutation<R: AuthorizationRequest + ?Sized>(req: &R) -> Answer {
answer(bucket_config_mutation_incarnation(req, BUCKET), |id| id)
}
/// Asserts that `read` gives `expected` for every case.
fn assert_cases(read: fn(&RequestEnvelope) -> Answer, cases: Vec<(&str, RequestEnvelope, Answer)>) {
/// Asserts that `read` gives `expected` through both request forms of every case.
fn assert_cases(
read: fn(&S3Request<()>) -> Answer,
owned: fn(&RequestEnvelope) -> Answer,
cases: Vec<(&str, S3Request<()>, Answer)>,
) {
for (name, req, expected) in cases {
assert_eq!(read(&req), expected, "{name}");
let envelope = RequestEnvelope::from_s3s(&req);
assert_eq!(read(&req), expected, "legacy form: {name}");
assert_eq!(owned(&envelope), expected, "owned form: {name}");
}
}
@@ -111,43 +117,48 @@ fn bucket_generation_guard_reads_the_same_through_both_forms() {
let missing = "bucket generation guard is missing";
let mismatch = "bucket generation guard does not match request bucket";
assert_cases(
bucket_fence,
bucket_fence,
vec![
(
"captured guard binds",
envelope(|req| req.extension(guard(BUCKET))),
legacy(|ext| {
ext.insert(guard(BUCKET));
}),
Ok(Some(incarnation())),
),
(
"pending-delete guard wins over the captured guard",
envelope(|req| {
req.extension(guard("other-bucket"))
.extension(PendingDeleteBucketGenerationGuard::Ready(guard(BUCKET)))
legacy(|ext| {
ext.insert(guard("other-bucket"));
ext.insert(PendingDeleteBucketGenerationGuard::Ready(guard(BUCKET)));
}),
Ok(Some(incarnation())),
),
(
"guard of another bucket",
envelope(|req| req.extension(guard("other-bucket"))),
legacy(|ext| {
ext.insert(guard("other-bucket"));
}),
refused(S3ErrorCode::InternalError, mismatch),
),
(
"failed pending-delete capture",
envelope(|req| {
req.extension(guard(BUCKET))
.extension(PendingDeleteBucketGenerationGuard::Failed {
code: S3ErrorCode::NoSuchBucket,
message: "bucket is gone".to_owned(),
})
legacy(|ext| {
ext.insert(guard(BUCKET));
ext.insert(PendingDeleteBucketGenerationGuard::Failed {
code: S3ErrorCode::NoSuchBucket.into(),
message: "bucket is gone".to_owned(),
});
}),
refused(S3ErrorCode::NoSuchBucket, "bucket is gone"),
),
(
"server request without a guard",
envelope(server_request),
legacy(server_request),
refused(S3ErrorCode::InternalError, missing),
),
("direct use-case call without a guard", envelope(|req| req), Ok(None)),
("direct use-case call without a guard", legacy(|_| {}), Ok(None)),
],
);
}
@@ -159,16 +170,21 @@ fn copy_source_generation_guard_reads_the_same_through_both_forms() {
incarnation_id: incarnation(),
};
assert_cases(
copy_source_fence,
copy_source_fence,
vec![
(
"captured guard binds",
envelope(|req| req.extension(copy_guard(BUCKET))),
legacy(|ext| {
ext.insert(copy_guard(BUCKET));
}),
Ok(Some(incarnation())),
),
(
"guard of another bucket",
envelope(|req| req.extension(copy_guard("other-bucket"))),
legacy(|ext| {
ext.insert(copy_guard("other-bucket"));
}),
refused(
S3ErrorCode::InternalError,
"copy source bucket generation guard does not match request bucket",
@@ -176,10 +192,13 @@ fn copy_source_generation_guard_reads_the_same_through_both_forms() {
),
(
"the destination guard does not stand in for the source",
envelope(|req| server_request(req).extension(guard(BUCKET))),
legacy(|ext| {
server_request(ext);
ext.insert(guard(BUCKET));
}),
refused(S3ErrorCode::InternalError, "copy source bucket generation guard is missing"),
),
("direct use-case call without a guard", envelope(|req| req), Ok(None)),
("direct use-case call without a guard", legacy(|_| {}), Ok(None)),
],
);
}
@@ -187,34 +206,41 @@ fn copy_source_generation_guard_reads_the_same_through_both_forms() {
#[test]
fn odm_read_generation_reads_the_same_through_both_forms() {
assert_cases(
odm,
odm,
vec![
(
"captured source generation",
envelope(|req| req.extension(OdmReadGenerationGuard::Ready(guard(BUCKET)))),
legacy(|ext| {
ext.insert(OdmReadGenerationGuard::Ready(guard(BUCKET)));
}),
Ok(Some(incarnation())),
),
(
"source generation of another bucket",
envelope(|req| req.extension(OdmReadGenerationGuard::Ready(guard("other-bucket")))),
legacy(|ext| {
ext.insert(OdmReadGenerationGuard::Ready(guard("other-bucket")));
}),
refused(S3ErrorCode::InternalError, "source generation guard does not match request bucket"),
),
(
"failed capture surfaces its error",
envelope(|req| {
req.extension(OdmReadGenerationGuard::Failed {
code: S3ErrorCode::SlowDown,
legacy(|ext| {
ext.insert(OdmReadGenerationGuard::Failed {
code: S3ErrorCode::SlowDown.into(),
message: "capture failed".to_owned(),
})
});
}),
refused(S3ErrorCode::SlowDown, "capture failed"),
),
(
"no source configured",
envelope(|req| req.extension(OdmReadGenerationGuard::Unavailable)),
legacy(|ext| {
ext.insert(OdmReadGenerationGuard::Unavailable);
}),
Ok(None),
),
("nothing captured", envelope(|req| req), Ok(None)),
("nothing captured", legacy(|_| {}), Ok(None)),
],
);
}
@@ -226,16 +252,21 @@ fn bucket_config_mutation_incarnation_reads_the_same_through_both_forms() {
incarnation_id: incarnation(),
};
assert_cases(
config_mutation,
config_mutation,
vec![
(
"captured snapshot",
envelope(|req| req.extension(snapshot(BUCKET))),
legacy(|ext| {
ext.insert(snapshot(BUCKET));
}),
Ok(Some(incarnation())),
),
(
"snapshot of another bucket",
envelope(|req| req.extension(snapshot("other-bucket"))),
legacy(|ext| {
ext.insert(snapshot("other-bucket"));
}),
refused(
S3ErrorCode::InternalError,
"bucket config mutation snapshot does not match request bucket",
@@ -243,17 +274,17 @@ fn bucket_config_mutation_incarnation_reads_the_same_through_both_forms() {
),
(
"server request without a snapshot",
envelope(server_request),
legacy(server_request),
refused(S3ErrorCode::InternalError, "bucket config mutation snapshot is missing"),
),
("direct use-case call without a snapshot", envelope(|req| req), Ok(None)),
("direct use-case call without a snapshot", legacy(|_| {}), Ok(None)),
],
);
}
#[test]
fn installed_state_reads_back_through_the_envelope() {
let mut envelope = envelope(server_request);
let mut envelope = RequestEnvelope::from_s3s(&legacy(server_request));
assert_eq!(
bucket_fence(&envelope),
refused(S3ErrorCode::InternalError, "bucket generation guard is missing")
@@ -261,7 +292,7 @@ fn installed_state_reads_back_through_the_envelope() {
envelope.install_bucket_generation(guard(BUCKET));
assert_eq!(bucket_fence(&envelope), Ok(Some(incarnation())));
envelope.install_pending_delete_bucket_generation(PendingDeleteBucketGenerationGuard::Failed {
code: S3ErrorCode::NoSuchBucket,
code: S3ErrorCode::NoSuchBucket.into(),
message: "bucket is gone".to_owned(),
});
assert_eq!(bucket_fence(&envelope), refused(S3ErrorCode::NoSuchBucket, "bucket is gone"));
@@ -277,27 +308,36 @@ fn installed_state_reads_back_through_the_envelope() {
#[tokio::test]
async fn odm_capture_without_a_configured_source_is_unavailable_in_both_forms() {
let slot = ServerContextSlot::new();
let envelope = envelope(|req| req.extension(guard(BUCKET)));
let captured = capture_odm_read_generation(&slot, &envelope, BUCKET).await;
assert!(matches!(captured, OdmReadGenerationGuard::Unavailable), "{captured:?}");
let req = legacy(|ext| {
ext.insert(guard(BUCKET));
});
let envelope = RequestEnvelope::from_s3s(&req);
for captured in [
capture_odm_read_generation(&slot, &req, BUCKET).await,
capture_odm_read_generation(&slot, &envelope, BUCKET).await,
] {
assert!(matches!(captured, OdmReadGenerationGuard::Unavailable), "{captured:?}");
}
}
#[test]
fn replication_authorization_flag_reads_the_same_through_both_forms() {
let with_flag = |flag: bool| {
envelope(move |req| {
req.extension(ReqInfo {
legacy(move |ext| {
ext.insert(ReqInfo {
replication_request_authorized: flag,
..Default::default()
})
});
})
};
for (name, req, expected) in [
("authorized replication request", with_flag(true), true),
("ordinary request", with_flag(false), false),
("request without ReqInfo", envelope(|req| req), false),
("request without ReqInfo", legacy(|_| {}), false),
] {
assert_eq!(replication_request_authorized(&req), expected, "{name}");
let envelope = RequestEnvelope::from_s3s(&req);
assert_eq!(replication_request_authorized(&req), expected, "legacy form: {name}");
assert_eq!(replication_request_authorized(&envelope), expected, "owned form: {name}");
}
}
@@ -315,23 +355,44 @@ fn publication_guards_and_request_context_read_the_same_through_both_forms() {
..Default::default()
};
for (name, req, expected) in [
("direct context", envelope(|req| req.extension(context("direct"))), Some("direct")),
("context inside ReqInfo", envelope(|req| req.extension(nested("nested"))), Some("nested")),
(
"the direct context wins",
envelope(|req| req.extension(context("direct")).extension(nested("nested"))),
"direct context",
legacy(|ext| {
ext.insert(context("direct"));
}),
Some("direct"),
),
("no context", envelope(|req| req), None),
(
"context inside ReqInfo",
legacy(|ext| {
ext.insert(nested("nested"));
}),
Some("nested"),
),
(
"the direct context wins",
legacy(|ext| {
ext.insert(context("direct"));
ext.insert(nested("nested"));
}),
Some("direct"),
),
("no context", legacy(|_| {}), None),
] {
let envelope = RequestEnvelope::from_s3s(&req);
let id = |found: Option<RequestContext>| found.map(|context| context.request_id);
assert_eq!(id(request_context_from_req(&req)).as_deref(), expected, "{name}");
assert_eq!(id(request_context_from_req(&req)).as_deref(), expected, "legacy form: {name}");
assert_eq!(id(request_context_from_req(&envelope)).as_deref(), expected, "owned form: {name}");
}
let with_guards = envelope(|req| req.extension(TableDataPlanePublicationGuards::default()));
let with_guards = legacy(|ext| {
ext.insert(TableDataPlanePublicationGuards::default());
});
assert!(retained_table_data_plane_publication_guards(&with_guards).is_some());
let without = envelope(|req| req);
assert!(retained_table_data_plane_publication_guards(&RequestEnvelope::from_s3s(&with_guards)).is_some());
let without = legacy(|_| {});
assert!(retained_table_data_plane_publication_guards(&without).is_none());
assert!(retained_table_data_plane_publication_guards(&RequestEnvelope::from_s3s(&without)).is_none());
}
#[test]
@@ -346,38 +407,66 @@ fn resource_free_conditions_read_the_same_client_through_both_forms() {
for (name, req, expected) in [
(
"forwarded client wins over the peer",
envelope(|req| req.extension(Some(RemoteAddr(peer))).extension(ClientInfo::direct(client))),
legacy(|ext| {
ext.insert(Some(RemoteAddr(peer)));
ext.insert(ClientInfo::direct(client));
}),
Some(vec!["198.51.100.7".to_owned()]),
),
(
"peer only",
envelope(|req| req.extension(Some(RemoteAddr(peer)))),
legacy(|ext| {
ext.insert(Some(RemoteAddr(peer)));
}),
Some(vec!["192.0.2.2".to_owned()]),
),
] {
assert_eq!(source_ip(&resource_free_condition_values(&req, &cred)), expected, "{name}");
let envelope = RequestEnvelope::from_s3s(&req);
// The clock values are read at each call; every request-derived value must match.
let without_clock = |mut conditions: HashMap<String, Vec<String>>| {
conditions.remove("CurrentTime");
conditions.remove("EpochTime");
conditions
};
let legacy_conditions = without_clock(resource_free_condition_values(&req, &cred));
assert_eq!(
legacy_conditions,
without_clock(resource_free_condition_values(&envelope, &cred)),
"{name}"
);
assert_eq!(source_ip(&legacy_conditions), expected, "{name}");
}
}
#[tokio::test]
async fn object_request_without_req_info_is_refused_in_both_forms() {
let mut envelope = envelope(|req| req);
let mut req = legacy(|_| {});
let mut envelope = RequestEnvelope::from_s3s(&req);
let action = Action::S3Action(S3Action::GetObjectAclAction);
let err = authorize_object_request(&mut envelope, BUCKET, "key", Some("v1"), action)
.await
.expect_err("a request without ReqInfo is refused");
assert_eq!(err.code(), &S3ErrorCode::InternalError);
assert_eq!(err.message(), Some("ReqInfo not found in request extensions"));
for result in [
authorize_object_request(&mut req, BUCKET, "key", Some("v1"), action)
.await
.map_err(super::S3Error::from),
authorize_object_request(&mut envelope, BUCKET, "key", Some("v1"), action)
.await
.map_err(super::S3Error::from),
] {
let err = result.expect_err("a request without ReqInfo is refused");
assert_eq!(err.code(), &S3ErrorCode::InternalError);
assert_eq!(err.message(), Some("ReqInfo not found in request extensions"));
}
}
#[test]
fn a_forked_envelope_reads_like_the_original_and_installs_alone() {
let original = envelope(|req| {
server_request(req).extension(guard(BUCKET)).extension(ReqInfo {
let original = RequestEnvelope::from_s3s(&legacy(|ext| {
server_request(ext);
ext.insert(guard(BUCKET));
ext.insert(ReqInfo {
replication_request_authorized: true,
..Default::default()
})
});
});
}));
let mut fork = original.fork_for_authorization();
assert_eq!(bucket_fence(&fork), bucket_fence(&original));
assert_eq!(bucket_fence(&fork), Ok(Some(incarnation())));
@@ -402,12 +491,14 @@ fn resource_free_conditions_preserve_legacy_projection_and_erased_keystone_sessi
for original_session in [None, Some(false), Some(true)] {
for forwarded in [false, true] {
let peer: SocketAddr = "192.0.2.9:9000".parse().unwrap();
let client = forwarded.then(|| {
let mut client = ClientInfo::direct("198.51.100.9:443".parse().unwrap());
client.forwarded_proto = Some("https".into());
client
let mut req = legacy(|ext| {
ext.insert(Some(RemoteAddr(peer)));
if forwarded {
let mut client = ClientInfo::direct("198.51.100.9:443".parse().unwrap());
client.forwarded_proto = Some("https".into());
ext.insert(client);
}
});
let mut headers = HeaderMap::new();
for (name, value) in [
("user-agent", "condition-client"),
("referer", "https://example.test/"),
@@ -417,7 +508,7 @@ fn resource_free_conditions_preserve_legacy_projection_and_erased_keystone_sessi
("x-amz-copy-source-server-side-encryption-customer-key", "source-key"),
("x-amz-meta-tag", "retained-value"),
] {
headers.insert(HeaderName::from_static(name), HeaderValue::from_static(value));
req.headers.insert(name, value.parse().unwrap());
}
let original = rustfs_credentials::Credentials {
access_key: "keystone:request-user".into(),
@@ -436,19 +527,11 @@ fn resource_free_conditions_preserve_legacy_projection_and_erased_keystone_sessi
if original_session.is_some() {
credential.session_token.clear();
}
let mut builder = request(Method::GET)
.uri("/guarded-bucket/key")
.headers(headers.clone())
.extension(Some(RemoteAddr(peer)))
.extension(ReqInfo {
cred: Some(credential.clone()),
keystone_session_present: original_session,
..Default::default()
});
if let Some(client) = client.clone() {
builder = builder.extension(client);
}
let req = builder.build();
req.extensions.insert(ReqInfo {
cred: Some(credential.clone()),
keystone_session_present: original_session,
..Default::default()
});
let without_clock = |mut values: HashMap<String, Vec<String>>| {
assert!(
time::OffsetDateTime::parse(
@@ -462,17 +545,20 @@ fn resource_free_conditions_preserve_legacy_projection_and_erased_keystone_sessi
values.remove("EpochTime");
values
};
// The oracle reads the raw request parts, not the envelope.
let expected = without_clock(crate::auth::get_condition_values_with_client_info(
&headers,
&req.headers,
&original,
None,
None,
Some(peer),
client.as_ref(),
req.remote_addr(),
req.client_info(),
));
let actual = without_clock(resource_free_condition_values(&req, &credential));
assert_eq!(actual, expected, "session={original_session:?}, forwarded={forwarded}");
assert_eq!(
without_clock(resource_free_condition_values(&RequestEnvelope::from_s3s(&req), &credential)),
expected
);
assert_eq!(
actual.get("username").unwrap(),
&vec![
-4
View File
@@ -34,12 +34,8 @@ mod guards;
mod permission_probe_test;
mod request;
mod table_data_plane;
#[cfg(test)]
pub(crate) mod tests;
pub(crate) use admission::{AccessMarkers, VerifiedSignature, admit_operation, resolve_req_info};
#[cfg(test)]
pub(crate) use conditions::authorization_conditions_with_internal;
pub(crate) use conditions::existing_object_tag_conditions;
#[cfg(test)]
pub(crate) use conditions::{
@@ -1,75 +0,0 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! The policy action each owned authorization picks for an operation, and the answer a terminal
//! IAM refusal becomes.
//!
//! Responsible for: `storage::authorization_actions` (the action selection both edges share) and
//! `storage::authorization::iam_authorization_error`.
//! Not responsible for: evaluating a policy against the chosen action, or actions an operation's
//! gateway handler names inline.
//! Upstream: none. Downstream: `storage::authorization_actions` and `storage::authorization`.
use crate::storage::authorization::iam_authorization_error;
use crate::storage::authorization_actions::{
complete_multipart_upload_authorize_action, list_parts_authorize_action, post_object_authorize_action, versioned_read_action,
};
use rustfs_policy::policy::action::{Action, S3Action};
use rustfs_s3_types::{S3Error, S3ErrorCode};
#[test]
fn terminal_iam_denial_is_access_denied_and_backend_failure_stays_internal() {
let denied = S3Error::from(iam_authorization_error(rustfs_iam::error::Error::IAMActionNotAllowed));
assert_eq!(denied.code(), &S3ErrorCode::AccessDenied);
let unavailable = S3Error::from(iam_authorization_error(rustfs_iam::error::Error::IamSysNotInitialized));
assert_eq!(unavailable.code(), &S3ErrorCode::InternalError);
}
#[test]
fn ghsa_3ppv_versioned_read_selects_get_object_version_action() {
// Regression (GHSA-3ppv): a read that names an explicit version must
// authorize against s3:GetObjectVersion so that a principal holding only
// s3:GetObject cannot read historical versions. Applies to GetObject and
// the CopyObject / UploadPartCopy sources.
assert_eq!(
versioned_read_action(Some("0194e0f1-0000-7000-8000-000000000000")),
Action::S3Action(S3Action::GetObjectVersionAction),
"an explicit versionId must require GetObjectVersion"
);
assert_eq!(
versioned_read_action(Some("null")),
Action::S3Action(S3Action::GetObjectVersionAction),
"the sentinel `null` version is still an explicit version selector"
);
assert_eq!(
versioned_read_action(None),
Action::S3Action(S3Action::GetObjectAction),
"a current-object read (no versionId) authorizes against GetObject"
);
}
#[test]
fn post_object_uses_put_object_action() {
assert_eq!(post_object_authorize_action(), Action::S3Action(S3Action::PutObjectAction));
}
#[test]
fn complete_multipart_upload_uses_put_object_action() {
assert_eq!(complete_multipart_upload_authorize_action(), Action::S3Action(S3Action::PutObjectAction));
}
#[test]
fn list_parts_uses_list_multipart_upload_parts_action() {
assert_eq!(list_parts_authorize_action(), Action::S3Action(S3Action::ListMultipartUploadPartsAction));
}
@@ -1,495 +0,0 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! The condition values the owned authorization core builds from one `RequestEnvelope`.
//!
//! Responsible for: list query conditions, request object tag conditions, the verified client
//! facts, the internal-object condition filter, the remaining-retention-days condition, the
//! existing-object-tag hint and merge, and the owner's bucket-policy Deny bypass.
//! Not responsible for: running an operation's `authorize_*` function against stored policies
//! (`storage::access_owned` tests), or the legacy request type.
//! Upstream: `crate::storage::authorization::tests::fixture`. Downstream: `storage::authorization`
//! conditions and decision, `crate::auth::facts`, and the bucket policy system.
use crate::app::RequestEnvelope;
use crate::app::object_meta::tagging::proxy_allowed;
use crate::auth::OBJECT_LOCK_REMAINING_RETENTION_DAYS_CONDITION;
use crate::auth::facts::{AuthzPrincipal, AuthzTarget, list_query_pairs, merge_list_query_conditions};
use crate::shared_types::RemoteAddr;
use crate::storage::StorageError;
use crate::storage::authorization::tests::fixture::{request, server_context};
use crate::storage::authorization::{
BucketPolicyExistingObjectTagHint, BucketPolicyRawLoadErrorKind, ObjectTagConditions, authorization_conditions,
authorization_conditions_with_internal, classify_bucket_policy_raw_load_error, load_bucket_policy_existing_object_tag_hint,
maybe_merge_object_tag_conditions, merge_request_object_tag_conditions, owner_can_bypass_policy_deny,
remaining_retention_days, request_authz_facts, set_requested_object_lock_retain_until,
};
use crate::storage::storage_api::contract::bucket::{BucketOperations as _, MakeBucketOptions};
use http::{HeaderMap, HeaderValue, Method};
use rustfs_policy::policy::action::{Action, S3Action};
use rustfs_policy::policy::{BucketPolicy, bucket_policy_uses_existing_object_tag_conditions};
use rustfs_s3_types::S3ErrorCode;
use serial_test::serial;
use std::collections::HashMap;
use time::OffsetDateTime;
/// The conditions an anonymous principal's decision about `action` evaluates for `req`.
fn remaining_retention_days_conditions(req: &RequestEnvelope, action: Action) -> HashMap<String, Vec<String>> {
let credentials = rustfs_credentials::Credentials::default();
let principal = AuthzPrincipal::new(&credentials, "");
let facts = request_authz_facts(
req,
AuthzTarget {
action,
bucket: "",
object: "",
version_id: None,
location_constraint: None,
},
);
authorization_conditions(req, &principal, &facts).expect("conditions should build")
}
/// A PUT that asks to retain its object for `retain_for` from now.
fn request_retaining_for(retain_for: time::Duration) -> RequestEnvelope {
let mut req = request(Method::PUT).build();
let retain_until = OffsetDateTime::now_utc() + retain_for;
set_requested_object_lock_retain_until(&mut req, Some(retain_until));
req
}
/// Object tag conditions must use keys like ExistingObjectTag/<tag-key> so that
/// bucket policy conditions (e.g. s3:ExistingObjectTag/security) are evaluated correctly.
#[test]
fn test_merge_list_bucket_query_conditions_extracts_supported_keys() {
let mut conditions = HashMap::new();
merge_list_query_conditions(
Action::S3Action(S3Action::ListBucketAction),
&list_query_pairs(Some("prefix=photos%2F2024%2F&delimiter=%2F&max-keys=10&encoding-type=url")),
&mut conditions,
);
assert_eq!(conditions.get("prefix"), Some(&vec!["photos/2024/".to_string()]));
assert_eq!(conditions.get("delimiter"), Some(&vec!["/".to_string()]));
assert_eq!(conditions.get("max-keys"), Some(&vec!["10".to_string()]));
assert!(!conditions.contains_key("encoding-type"));
}
#[test]
fn test_merge_list_bucket_query_conditions_preserves_empty_prefix_signal() {
let mut conditions = HashMap::new();
merge_list_query_conditions(
Action::S3Action(S3Action::ListBucketVersionsAction),
&list_query_pairs(Some("prefix=&delimiter=%2F")),
&mut conditions,
);
assert_eq!(conditions.get("prefix"), Some(&vec![String::new()]));
assert_eq!(conditions.get("delimiter"), Some(&vec!["/".to_string()]));
}
#[test]
fn test_merge_list_bucket_query_conditions_ignores_non_list_actions() {
let mut conditions = HashMap::new();
merge_list_query_conditions(
Action::S3Action(S3Action::GetObjectAction),
&list_query_pairs(Some("prefix=photos%2F2024%2F&delimiter=%2F&max-keys=10")),
&mut conditions,
);
assert!(conditions.is_empty());
}
#[test]
fn listing_conditions_reject_repeated_query_and_ignore_headers() {
for action in [
S3Action::ListBucketAction,
S3Action::ListBucketVersionsAction,
S3Action::ListBucketMultipartUploadsAction,
] {
for key in ["prefix", "delimiter", "max-keys"] {
let action = Action::S3Action(action);
let credentials = rustfs_credentials::Credentials::default();
let conditions_of = |req: &RequestEnvelope| {
let principal = AuthzPrincipal::new(&credentials, "");
let facts = request_authz_facts(
req,
AuthzTarget {
action,
bucket: "bucket",
object: "",
version_id: None,
location_constraint: None,
},
);
authorization_conditions(req, &principal, &facts)
};
// Every request carries the header; only the applied query may describe the listing.
let req = request(Method::GET).header(key, "allowed").build();
let conditions = conditions_of(&req).unwrap();
assert!(!conditions.contains_key(key), "{key} must come from the applied query");
let req = request(Method::GET)
.uri(&format!("/bucket?{key}=actual"))
.header(key, "allowed")
.build();
let conditions = conditions_of(&req).unwrap();
assert_eq!(conditions.get(key), Some(&vec!["actual".to_string()]));
let req = request(Method::GET)
.uri(&format!("/bucket?{key}=actual&{key}=allowed"))
.header(key, "allowed")
.build();
let error = conditions_of(&req).unwrap_err();
assert_eq!(error.code(), &S3ErrorCode::InvalidRequest);
}
}
}
#[test]
fn test_merge_request_object_tag_conditions_applies_only_to_put_object() {
let mut headers = HeaderMap::new();
headers.insert("x-amz-tagging", HeaderValue::from_static("classification=restricted&label=copy%20test"));
let mut source_conditions = HashMap::new();
merge_request_object_tag_conditions(Action::S3Action(S3Action::GetObjectAction), &headers, &mut source_conditions)
.expect("GetObject condition construction should succeed");
assert!(
source_conditions.is_empty(),
"destination request tags must not affect CopyObject source authorization"
);
let mut destination_conditions = HashMap::new();
merge_request_object_tag_conditions(Action::S3Action(S3Action::PutObjectAction), &headers, &mut destination_conditions)
.expect("PutObject condition construction should succeed");
assert_eq!(
destination_conditions.get("RequestObjectTag/classification"),
Some(&vec!["restricted".to_string()])
);
assert_eq!(destination_conditions.get("RequestObjectTag/label"), Some(&vec!["copy test".to_string()]));
assert_eq!(
destination_conditions.get("RequestObjectTagKeys"),
Some(&vec!["classification".to_string(), "label".to_string()])
);
}
#[test]
fn access_facts_preserve_verified_client_context() {
use rustfs_trusted_proxies::{ClientInfo, ValidationMode};
let trusted = ClientInfo::from_trusted_proxy(
"203.0.113.9".parse().expect("client IP"),
None,
Some("https".to_owned()),
"10.0.0.2".parse().expect("proxy IP"),
1,
ValidationMode::Lenient,
Vec::new(),
);
for (client_info, expected_ip, expected_secure) in [(None, "192.0.2.10", "false"), (Some(trusted), "203.0.113.9", "true")] {
let mut builder = request(Method::GET)
.header("x-forwarded-for", "198.51.100.1")
.header("x-forwarded-proto", "https")
.extension(Some(RemoteAddr("192.0.2.10:443".parse().expect("peer address"))));
if let Some(client_info) = client_info {
builder = builder.extension(client_info);
}
let req = builder.build();
let conditions = remaining_retention_days_conditions(&req, Action::S3Action(S3Action::GetObjectAction));
assert_eq!(conditions.get("SourceIp"), Some(&vec![expected_ip.to_owned()]));
assert_eq!(conditions.get("SecureTransport"), Some(&vec![expected_secure.to_owned()]));
assert!(req.remote_addr().is_some(), "facts must only borrow the recorded peer");
}
}
/// AWS does not document how it rounds partial days. MinIO computes
/// `ceil(hours / 24)` in `enforceRetentionBypassForPut`
/// (cmd/bucket-object-lock.go), and RustFS keeps that rounding so a policy
/// moved from MinIO evaluates the same way.
#[test]
fn remaining_retention_days_matches_s3_ceil_rounding() {
let now = OffsetDateTime::now_utc();
let cases = [
(time::Duration::NANOSECOND, 1),
(time::Duration::HOUR, 1),
(time::Duration::DAY - time::Duration::SECOND, 1),
(time::Duration::DAY, 1),
(time::Duration::DAY + time::Duration::SECOND, 2),
(time::Duration::days(29) + time::Duration::hours(12), 30),
(time::Duration::days(30), 30),
(time::Duration::days(30) + time::Duration::SECOND, 31),
];
for (retain_for, expected) in cases {
let minio_days = (retain_for.as_seconds_f64() / 3600.0 / 24.0).ceil() as i64;
assert_eq!(minio_days, expected, "MinIO reference for {retain_for}");
assert_eq!(remaining_retention_days(now + retain_for, now), expected, "retain for {retain_for}");
}
}
#[test]
fn remaining_retention_days_is_zero_for_dates_not_in_future() {
let now = OffsetDateTime::now_utc();
assert_eq!(remaining_retention_days(now, now), 0);
assert_eq!(remaining_retention_days(now - time::Duration::NANOSECOND, now), 0);
assert_eq!(remaining_retention_days(now - time::Duration::days(3), now), 0);
}
#[test]
fn requested_retain_until_sets_remaining_retention_days_condition() {
let req = request_retaining_for(time::Duration::days(10) - time::Duration::HOUR);
let conditions = remaining_retention_days_conditions(&req, Action::S3Action(S3Action::PutObjectAction));
assert_eq!(
conditions.get(OBJECT_LOCK_REMAINING_RETENTION_DAYS_CONDITION),
Some(&vec!["10".to_string()])
);
}
#[test]
fn request_without_retention_has_no_remaining_retention_days_condition() {
let action = Action::S3Action(S3Action::PutObjectAction);
let req = request(Method::PUT).build();
assert_eq!(
remaining_retention_days_conditions(&req, action).get(OBJECT_LOCK_REMAINING_RETENTION_DAYS_CONDITION),
None
);
let mut req = request_retaining_for(time::Duration::days(5));
set_requested_object_lock_retain_until(&mut req, None);
assert_eq!(
remaining_retention_days_conditions(&req, action).get(OBJECT_LOCK_REMAINING_RETENTION_DAYS_CONDITION),
None
);
}
#[test]
fn remaining_retention_days_condition_ignores_client_header() {
let action = Action::S3Action(S3Action::PutObjectRetentionAction);
let mut req = request(Method::PUT)
.header(OBJECT_LOCK_REMAINING_RETENTION_DAYS_CONDITION, "1")
.build();
assert_eq!(
remaining_retention_days_conditions(&req, action).get(OBJECT_LOCK_REMAINING_RETENTION_DAYS_CONDITION),
None
);
let retain_until = OffsetDateTime::now_utc() + time::Duration::days(400);
set_requested_object_lock_retain_until(&mut req, Some(retain_until));
assert_eq!(
remaining_retention_days_conditions(&req, action).get(OBJECT_LOCK_REMAINING_RETENTION_DAYS_CONDITION),
Some(&vec!["400".to_string()])
);
}
#[test]
fn internal_object_authorization_ignores_unapplied_request_options() {
let req = request(Method::POST)
.header(
"authorization",
"AWS4-HMAC-SHA256 Credential=test/20260801/us-east-1/s3tables/aws4_request",
)
.header("user-agent", "pyiceberg/test")
.header("x-amz-server-side-encryption", "AES256")
.header("x-amz-tagging", "classification=%ZZ")
.build();
let credentials = rustfs_credentials::Credentials::default();
let principal = AuthzPrincipal::new(&credentials, "");
let facts = request_authz_facts(
&req,
AuthzTarget {
action: Action::S3Action(S3Action::PutObjectAction),
bucket: "",
object: "",
version_id: None,
location_constraint: None,
},
);
// The owned edge reaches this branch through `authorize_owned_internal_object_request`.
let conditions = authorization_conditions_with_internal(&req, &principal, &facts, true)
.expect("internal object authorization conditions should build");
assert_eq!(conditions.get("authType"), Some(&vec!["REST-HEADER".to_string()]));
assert_eq!(conditions.get("signatureversion"), Some(&vec!["AWS4-HMAC-SHA256".to_string()]));
assert_eq!(conditions.get("UserAgent"), Some(&vec!["pyiceberg/test".to_string()]));
assert!(!conditions.contains_key("authorization"));
assert!(!conditions.contains_key("x-amz-server-side-encryption"));
assert!(!conditions.contains_key("RequestObjectTag/classification"));
assert!(!conditions.contains_key("RequestObjectTagKeys"));
}
#[tokio::test]
async fn test_required_existing_object_tags_are_merged_into_authorization_conditions() {
let (_temp_dir, _disk_paths, store) = crate::app::gating_test_env::isolated_multi_pool_ecstore().await;
let slot = server_context(store);
let mut req = request(Method::GET)
.server_context(&slot)
.extension(ObjectTagConditions::new(
"bucket",
"tagged-object",
None,
HashMap::from([
("ExistingObjectTag/security".to_string(), vec!["restricted".to_string()]),
("ExistingObjectTag/project".to_string(), vec!["webapp".to_string()]),
]),
))
.build();
let mut conditions = HashMap::new();
conditions.insert("delimiter".to_string(), vec!["/".to_string()]);
maybe_merge_object_tag_conditions(
&mut req,
Action::S3Action(S3Action::GetObjectAction),
"bucket",
"tagged-object",
None,
&mut conditions,
true,
)
.await
.expect("merge required existing object tags into authorization conditions");
assert_eq!(conditions.get("ExistingObjectTag/security"), Some(&vec!["restricted".to_string()]));
assert!(!proxy_allowed(&req));
assert_eq!(conditions.get("ExistingObjectTag/project"), Some(&vec!["webapp".to_string()]));
assert_eq!(conditions.get("delimiter"), Some(&vec!["/".to_string()]));
}
#[test]
fn test_bucket_policy_existing_object_tag_condition_key_detection() {
let condition_key_policy = r#"{
"Version":"2012-10-17",
"Statement":[{
"Effect":"Allow",
"Principal":"*",
"Action":["s3:GetObject"],
"Resource":["arn:aws:s3:::bucket/*"],
"Condition":{"StringEquals":{"s3:ExistingObjectTag/security":"public"}}
}]
}"#;
let policy: BucketPolicy = serde_json::from_str(condition_key_policy).expect("valid bucket policy JSON");
assert!(
bucket_policy_uses_existing_object_tag_conditions(&policy),
"ExistingObjectTag in condition key must be detected"
);
let value_only_policy = r#"{
"Version":"2012-10-17",
"Statement":[{
"Effect":"Allow",
"Principal":"*",
"Action":["s3:GetObject"],
"Resource":["arn:aws:s3:::bucket/*"],
"Condition":{"StringEquals":{"s3:prefix":"ExistingObjectTag/security"}}
}]
}"#;
let policy: BucketPolicy = serde_json::from_str(value_only_policy).expect("valid bucket policy JSON");
assert!(
!bucket_policy_uses_existing_object_tag_conditions(&policy),
"ExistingObjectTag text in values should not trigger tag dependency"
);
}
#[test]
fn test_unparsable_bucket_policy_json_implies_conservative_existing_object_tag_fetch() {
// Matches `load_bucket_policy_existing_object_tag_hint`: unparsable policy => conservative tag fetch.
let malformed = r#"{"Version":"2012-10-17","Statement":[INVALID]}"#;
assert!(serde_json::from_str::<BucketPolicy>(malformed).is_err());
let conservative_fetch = serde_json::from_str::<BucketPolicy>(malformed)
.map(|p| bucket_policy_uses_existing_object_tag_conditions(&p))
.unwrap_or(true);
assert!(conservative_fetch);
// Invalid JSON that still contains real ExistingObjectTag condition keys (trailing comma).
let malformed_with_tag_keys =
r#"{"Version":"2012-10-17","Statement":[{"Condition":{"StringEquals":{"s3:ExistingObjectTag/security":"public"}}},]}"#;
assert!(serde_json::from_str::<BucketPolicy>(malformed_with_tag_keys).is_err());
let conservative_with_tag_keys = serde_json::from_str::<BucketPolicy>(malformed_with_tag_keys)
.map(|p| bucket_policy_uses_existing_object_tag_conditions(&p))
.unwrap_or(true);
assert!(conservative_with_tag_keys);
}
#[tokio::test]
#[serial]
async fn existing_tag_hint_reuses_parsed_metadata_and_fails_closed_without_it() {
let store = crate::app::gating_test_env::shared_gating_ecstore().await;
let bucket = format!("cached-policy-hint-{}", uuid::Uuid::new_v4());
store
.make_bucket(&bucket, &MakeBucketOptions::default())
.await
.expect("create hint bucket");
let document = serde_json::json!({"Version":"2012-10-17","Statement":[{
"Effect":"Deny","Principal":"*","Action":"s3:GetObject","Resource":format!("arn:aws:s3:::{bucket}/*"),
"Condition":{"StringEquals":{"s3:ExistingObjectTag/classification":"secret"}}
}]});
let mut metadata = (*crate::storage::get_bucket_metadata(&bucket).await.expect("bucket metadata")).clone();
metadata.policy_config =
Some(serde_json::from_slice(&serde_json::to_vec(&document).expect("policy JSON")).expect("parsed policy"));
// A cached parsed snapshot is the authority used by actual policy
// evaluation; the tag hint must consult that same snapshot.
metadata.policy_config_json = b"invalid raw JSON".to_vec();
crate::storage::storage_api::set_bucket_metadata(bucket.clone(), metadata.clone())
.await
.expect("publish parsed snapshot");
assert!(matches!(
load_bucket_policy_existing_object_tag_hint(&store, &bucket, Action::S3Action(S3Action::GetObjectAction)).await,
BucketPolicyExistingObjectTagHint::Parsed(_)
));
metadata.policy_config = None;
crate::storage::storage_api::set_bucket_metadata(bucket.clone(), metadata)
.await
.expect("publish unreadable snapshot");
assert!(matches!(
load_bucket_policy_existing_object_tag_hint(&store, &bucket, Action::S3Action(S3Action::GetObjectAction)).await,
BucketPolicyExistingObjectTagHint::ConservativeTagRequired
));
}
#[test]
fn test_classify_bucket_policy_raw_load_error() {
assert_eq!(
classify_bucket_policy_raw_load_error(&StorageError::ConfigNotFound),
BucketPolicyRawLoadErrorKind::PolicyMissing
);
assert_eq!(
classify_bucket_policy_raw_load_error(&StorageError::BucketNotFound("b".to_string())),
BucketPolicyRawLoadErrorKind::BucketMissing
);
assert_eq!(
classify_bucket_policy_raw_load_error(&StorageError::Io(std::io::Error::other("boom"))),
BucketPolicyRawLoadErrorKind::Other
);
}
/// Owner can bypass bucket policy Deny only for the three policy management APIs (per AWS S3).
#[test]
fn test_owner_can_bypass_policy_deny_only_for_policy_apis() {
// Owner + policy management actions -> bypass allowed
assert!(owner_can_bypass_policy_deny(true, &Action::S3Action(S3Action::GetBucketPolicyAction)));
assert!(owner_can_bypass_policy_deny(true, &Action::S3Action(S3Action::PutBucketPolicyAction)));
assert!(owner_can_bypass_policy_deny(true, &Action::S3Action(S3Action::DeleteBucketPolicyAction)));
// Owner + other actions -> no bypass (still subject to bucket policy Deny)
assert!(!owner_can_bypass_policy_deny(true, &Action::S3Action(S3Action::ListBucketAction)));
assert!(!owner_can_bypass_policy_deny(true, &Action::S3Action(S3Action::GetObjectAction)));
// Non-owner -> no bypass for any action
assert!(!owner_can_bypass_policy_deny(false, &Action::S3Action(S3Action::GetBucketPolicyAction)));
assert!(!owner_can_bypass_policy_deny(
false,
&Action::S3Action(S3Action::DeleteBucketPolicyAction)
));
}
@@ -1,335 +0,0 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Authorization denial auditing on the owned request: the client-facing AccessDenied stays
//! generic, a denial yields one bounded audit entry built from the envelope's audit view, probe
//! and aggregate flags suppress it, and every request-level denial exit without a
//! `DenialContext` still calls `audit_request_denial` in its owning production file.
//!
//! Not covered here: the limiter's own token arithmetic (`storage/denial_audit.rs` tests), the
//! policy decisions that produce a denial, or any legacy request type.
//! Upstream: `DenialContext`, `audit_request_denial`, `DenialAuditor::audit_owned`, `BulkDenialAudit`
//! and the envelope's `AuthorizationRequest::audit_denial`; downstream: none.
use crate::app::RequestEnvelope;
use crate::storage::authorization::tests::fixture::request;
use crate::storage::authorization::{DenialContext, audit_request_denial};
use crate::storage::denial_audit::{BulkDenialAudit, DenialAuditor, DenialRateLimiter};
use crate::storage::helper::DenialScope;
use crate::storage::req_info::ReqInfo;
use crate::storage::request_context::RequestContext;
use http::Method;
use rustfs_policy::policy::action::{Action, S3Action};
use rustfs_s3_types::S3ErrorCode;
use std::sync::{Arc, Mutex};
use std::time::{Duration, Instant};
type CapturedAudit = Arc<Mutex<Vec<rustfs_audit::AuditEntry>>>;
/// An auditor wired like the global one except that entries land in the returned vector.
fn test_auditor(enabled: bool, authenticated_rate: u64, anonymous_rate: u64) -> (DenialAuditor, CapturedAudit) {
let captured: CapturedAudit = Arc::default();
let sink = Arc::clone(&captured);
// A frozen clock: no wall-clock refill can change what the limiter admits.
let t0 = Instant::now();
let limiter = DenialRateLimiter::new(authenticated_rate, anonymous_rate, t0);
let auditor =
DenialAuditor::new(limiter, move || enabled, move |_, entry| sink.lock().unwrap().push(entry)).with_clock(move || t0);
(auditor, captured)
}
fn denial_request(request_id: &str, authenticated: bool) -> RequestEnvelope {
request(Method::GET)
.uri("/audit-bucket/audit-object")
.extension(RequestContext {
request_id: request_id.to_string(),
x_amz_request_id: request_id.to_string(),
trace_id: None,
span_id: None,
start_time: Instant::now(),
})
.extension(ReqInfo {
cred: authenticated.then(|| rustfs_credentials::Credentials {
access_key: "denied-user".to_string(),
..Default::default()
}),
bucket: Some("audit-bucket".to_string()),
object: Some("audit-object".to_string()),
..Default::default()
})
.build()
}
fn denial_context(auditor: &DenialAuditor, quiet: bool) -> DenialContext<'_> {
DenialContext {
quiet,
suppress_audit: false,
auditor,
bucket: "audit-bucket",
object: "audit-object",
version_id: Some("version"),
account: Some("denied-user"),
is_owner: false,
}
}
const GET_OBJECT: Action = Action::S3Action(S3Action::GetObjectAction);
/// Issue #5740: the denial helper must keep the client-facing error identical
/// to the historical bare response — the added diagnostics are log-only.
#[test]
fn test_access_denied_helper_keeps_generic_wire_response() {
let (auditor, captured) = test_auditor(true, 10, 10);
let req = denial_request("denial-wire-response", true);
let err = denial_context(&auditor, false).deny(&req, "bucket_policy_explicit_deny", GET_OBJECT);
assert_eq!(err.code(), &S3ErrorCode::AccessDenied);
assert_eq!(err.message(), Some("Access Denied"));
assert_eq!(captured.lock().unwrap().len(), 1);
}
/// A real denial yields exactly one audit entry naming the account, bucket, object,
/// denied action and internal reason.
#[test]
fn denial_is_audited_once_with_account_resource_action_and_reason() {
let (auditor, captured) = test_auditor(true, 10, 10);
let req = denial_request("denial-audited-once", true);
let _ = denial_context(&auditor, false).deny(&req, "iam_explicit_deny", Action::S3Action(S3Action::PutObjectAction));
let entries = captured.lock().unwrap();
assert_eq!(entries.len(), 1);
let entry = &entries[0];
assert_eq!(entry.request_id.as_deref(), Some("denial-audited-once"));
assert_eq!(entry.access_key.as_deref(), Some("denied-user"));
assert_eq!(entry.api.name.as_deref(), Some("s3:PutObject"));
assert_eq!(entry.api.bucket.as_deref(), Some("audit-bucket"));
assert_eq!(entry.api.object.as_deref(), Some("audit-object"));
assert_eq!(entry.api.status.as_deref(), Some("failure"));
assert_eq!(entry.api.status_code, Some(403));
assert_eq!(entry.error.as_deref(), Some("Access Denied"));
let tags = entry.tags.as_ref().expect("denial entry carries its reason");
assert_eq!(tags.get("denialReason").and_then(|value| value.as_str()), Some("iam_explicit_deny"));
assert!(!tags.contains_key("suppressedDenials"));
}
/// Probe denials (per-bucket ListBuckets fallback, per-object metadata collection) are
/// expected filter outcomes and must not flood the audit stream.
#[test]
fn quiet_probe_denial_is_not_audited() {
let (auditor, captured) = test_auditor(true, 10, 10);
let req = denial_request("denial-quiet-probe", true);
let err = denial_context(&auditor, true).deny(&req, "no_policy_allows_action", Action::S3Action(S3Action::ListBucketAction));
assert_eq!(err.code(), &S3ErrorCode::AccessDenied);
assert!(captured.lock().unwrap().is_empty());
}
#[test]
fn denial_is_not_audited_when_the_audit_module_is_off() {
let (auditor, captured) = test_auditor(false, 10, 10);
let req = denial_request("denial-audit-off", true);
let _ = denial_context(&auditor, false).deny(&req, "no_policy_allows_action", GET_OBJECT);
assert!(captured.lock().unwrap().is_empty());
}
#[test]
fn denial_audited_in_aggregate_by_the_caller_is_not_audited_per_key() {
let (auditor, captured) = test_auditor(true, 10, 10);
let req = denial_request("denial-per-key", true);
let mut denial = denial_context(&auditor, false);
denial.suppress_audit = true;
let _ = denial.deny(&req, "iam_explicit_deny", GET_OBJECT);
assert!(captured.lock().unwrap().is_empty());
}
/// A flood of denials emits a bounded number of entries; the rest are only counted.
#[test]
fn denial_flood_emits_bounded_entries() {
let (auditor, captured) = test_auditor(true, 3, 3);
let req = denial_request("denial-flood", true);
for _ in 0..20 {
let _ = denial_context(&auditor, false).deny(&req, "no_policy_allows_action", GET_OBJECT);
}
assert_eq!(captured.lock().unwrap().len(), 3);
}
/// The entry that follows a suppressed run reports how many denials were dropped.
///
/// The owned auditor reads time only from its clock, so this test drives a settable clock
/// instead of passing an instant per call.
#[test]
fn entry_after_a_flood_reports_the_suppressed_count() {
let captured: CapturedAudit = Arc::default();
let sink = Arc::clone(&captured);
let t0 = Instant::now();
let now = Arc::new(Mutex::new(t0));
let clock = Arc::clone(&now);
let auditor = DenialAuditor::new(
DenialRateLimiter::new(1, 1, t0),
|| true,
move |_, entry| sink.lock().unwrap().push(entry),
)
.with_clock(move || *clock.lock().unwrap());
let req = denial_request("denial-suppressed-count", true);
for _ in 0..5 {
auditor.audit_owned(&req.audit_view(), &GET_OBJECT, "no_policy_allows_action", DenialScope::Request);
}
*now.lock().unwrap() = t0 + Duration::from_secs(2);
auditor.audit_owned(&req.audit_view(), &GET_OBJECT, "no_policy_allows_action", DenialScope::Request);
let entries = captured.lock().unwrap();
assert_eq!(entries.len(), 2);
let tag = |index: usize| {
entries[index]
.tags
.as_ref()
.unwrap()
.get("suppressedDenials")
.and_then(|v| v.as_u64())
};
assert_eq!(tag(0), None);
assert_eq!(tag(1), Some(4));
}
/// Anonymous callers have their own, lower limit, so an unauthenticated scanner cannot
/// use up the budget for authenticated callers.
#[test]
fn anonymous_denials_are_limited_separately_from_authenticated_ones() {
let (auditor, captured) = test_auditor(true, 10, 1);
let anonymous = denial_request("denial-anonymous", false);
let authenticated = denial_request("denial-authenticated", true);
for _ in 0..5 {
let _ = denial_context(&auditor, false).deny(&anonymous, "no_policy_allows_action", GET_OBJECT);
}
for _ in 0..5 {
let _ = denial_context(&auditor, false).deny(&authenticated, "no_policy_allows_action", GET_OBJECT);
}
let entries = captured.lock().unwrap();
let count = |id: &str| entries.iter().filter(|entry| entry.request_id.as_deref() == Some(id)).count();
assert_eq!(count("denial-anonymous"), 1);
assert_eq!(count("denial-authenticated"), 5);
}
/// Exits that have no `DenialContext` of their own audit through the request's own state
/// and honor the same probe flags.
#[test]
fn request_denial_exit_is_audited_and_honors_the_probe_flags() {
let (auditor, captured) = test_auditor(true, 10, 10);
let mut req = denial_request("denial-exit", true);
audit_request_denial(&req, &auditor, "table_resource_policy_denied", GET_OBJECT);
assert_eq!(captured.lock().unwrap().len(), 1);
req.req_info_mut().unwrap().suppress_denial_log = true;
audit_request_denial(&req, &auditor, "table_resource_policy_denied", GET_OBJECT);
assert_eq!(captured.lock().unwrap().len(), 1);
req.req_info_mut().unwrap().suppress_denial_log = false;
req.req_info_mut().unwrap().suppress_denial_audit = true;
audit_request_denial(&req, &auditor, "table_resource_policy_denied", GET_OBJECT);
assert_eq!(captured.lock().unwrap().len(), 1);
}
fn production_sources_audit_reason(sources: &[&str], reason: &str) -> bool {
sources.iter().any(|source| {
let production = source.split("\nmod tests {").next().expect("source has a production section");
production
.match_indices("audit_request_denial(")
.any(|(at, _)| production[at..].chars().take(260).collect::<String>().contains(reason))
})
}
/// Request-level denial exits span the owned authorization modules.
/// Check each exit in its owning file alongside the audit behavior tests.
#[test]
fn denial_exits_without_a_context_call_the_audit_function() {
let access = include_str!("../table_data_plane.rs");
let object_misc = include_str!("../../access_owned/object_misc.rs")
.split("\nmod tests;")
.next()
.expect("owned source has a production section");
for (source, reason) in [
(access, "\"table_resource_policy_denied\""),
(access, "\"anonymous_table_data_plane_access\""),
(object_misc, "\"recursive_force_delete_requires_authenticated_caller\""),
] {
assert!(
production_sources_audit_reason(&[source], reason),
"{reason} exit does not call audit_request_denial"
);
}
}
#[test]
fn denial_source_guard_tracks_moved_calls_and_rejects_removed_calls() {
let reason = "\"recursive_force_delete_requires_authenticated_caller\"";
let original = "fn original_exit() {}";
let moved = format!("fn moved_exit() {{ audit_request_denial(req, auditor, {reason}, action); }}");
assert!(!production_sources_audit_reason(&[original], reason));
assert!(production_sources_audit_reason(&[original, &moved], reason));
let removed = moved.replace("audit_request_denial(", "unrecorded_denial(");
assert!(!production_sources_audit_reason(&[original, &removed], reason));
let test_only = format!("{original}\nmod tests {{\n{moved}\n}}");
assert!(!production_sources_audit_reason(&[original, &test_only], reason));
}
/// DeleteObjects audits its refused keys as one entry per denied action, carrying the count and
/// no single object, and the per-key mark is cleared afterwards.
#[test]
fn bulk_denial_is_one_entry_per_denied_action_with_the_key_count() {
let (auditor, captured) = test_auditor(true, 10, 10);
let mut envelope = denial_request("denial-bulk", true);
let nothing_denied = BulkDenialAudit::start_owned(&mut envelope).unwrap();
assert!(envelope.req_info().unwrap().suppress_denial_audit);
nothing_denied
.finish_owned_with(&mut envelope, RequestEnvelope::audit_view, &auditor)
.unwrap();
assert!(!envelope.req_info().unwrap().suppress_denial_audit);
assert!(captured.lock().unwrap().is_empty());
let mut bulk = BulkDenialAudit::start_owned(&mut envelope).unwrap();
let delete = Action::S3Action(S3Action::DeleteObjectAction);
let bypass = Action::S3Action(S3Action::BypassGovernanceRetentionAction);
for action in [delete, delete, bypass, delete] {
bulk.denied(action);
}
bulk.finish_owned_with(&mut envelope, RequestEnvelope::audit_view, &auditor)
.unwrap();
let entries = captured.lock().unwrap();
assert_eq!(entries.len(), 2);
let summary: Vec<_> = entries
.iter()
.map(|entry| {
assert_eq!(entry.api.object, None);
(
entry.api.name.clone().unwrap(),
entry
.tags
.as_ref()
.unwrap()
.get("deniedKeys")
.and_then(|value| value.as_u64())
.unwrap(),
)
})
.collect();
assert_eq!(
summary,
vec![
("s3:DeleteObject".to_string(), 3),
("s3:BypassGovernanceRetention".to_string(), 1)
]
);
}
@@ -1,141 +0,0 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Owned request fixtures for the authorization tests: a [`RequestEnvelope`] builder and the
//! IAM and KMS stand-ins a server context needs.
//!
//! Responsible for: building the request the gateway edge hands the authorization core, with
//! the same facts the legacy edge copied from its request (headers, URI, credentials and the
//! session token taken from them, extensions).
//! Not responsible for: any authorization decision, or fixtures for the legacy request type.
//! Upstream: none. Downstream: the tests of `storage::authorization` and `storage::access_owned`.
use crate::app::{RequestCredentials, RequestEnvelope, RequestHead};
use crate::auth::get_session_token;
use crate::storage::req_info::ReqInfo;
use crate::storage::storage_api::runtime_sources_consumer::{AppContext, ECStore, IamInterface, KmsInterface, ServerContextSlot};
use http::header::{HeaderName, HeaderValue};
use http::{Extensions, HeaderMap, Method, Uri};
use rustfs_iam::{store::object::ObjectStore, sys::IamSys};
use rustfs_kms::KmsServiceManager;
use std::sync::Arc;
/// An IAM that is never ready: a request authorized against it resolves no identity policy.
pub(crate) struct UnreadyIam;
impl IamInterface for UnreadyIam {
fn handle(&self) -> Arc<IamSys<ObjectStore>> {
panic!("authorization fixtures built on UnreadyIam do not resolve IAM")
}
fn is_ready(&self) -> bool {
false
}
}
/// A KMS with no configured backend.
pub(crate) struct TestKms;
impl KmsInterface for TestKms {
fn handle(&self) -> Arc<KmsServiceManager> {
Arc::new(KmsServiceManager::new())
}
}
/// A server context bound to `store`, with an unready IAM and an unconfigured KMS.
pub(crate) fn server_context(store: Arc<ECStore>) -> Arc<ServerContextSlot> {
let slot = ServerContextSlot::new();
assert!(slot.install(Arc::new(AppContext::new(store, Arc::new(UnreadyIam), Arc::new(TestKms)))));
slot
}
/// Builds the owned request the authorization core reads.
pub(crate) struct RequestBuilder {
method: Method,
uri: Uri,
headers: HeaderMap,
credentials: Option<(String, String)>,
extensions: Extensions,
}
/// A request with no headers, no credentials, no extensions, and the URI `/`.
pub(crate) fn request(method: Method) -> RequestBuilder {
RequestBuilder {
method,
uri: Uri::from_static("/"),
headers: HeaderMap::new(),
credentials: None,
extensions: Extensions::new(),
}
}
impl RequestBuilder {
pub(crate) fn uri(mut self, uri: &str) -> Self {
self.uri = uri.parse().expect("test URI");
self
}
pub(crate) fn header(mut self, name: &str, value: &str) -> Self {
self.headers.append(
HeaderName::from_bytes(name.as_bytes()).expect("test header name"),
HeaderValue::from_str(value).expect("test header value"),
);
self
}
pub(crate) fn headers(mut self, headers: HeaderMap) -> Self {
self.headers = headers;
self
}
/// Signs the request as `access_key`. The session token is read from the request's
/// headers and query when the envelope is built, as the edge reads it.
pub(crate) fn credentials(mut self, access_key: &str, secret_key: &str) -> Self {
self.credentials = Some((access_key.to_owned(), secret_key.to_owned()));
self
}
/// Carries an empty [`ReqInfo`], as the access phase installs before any hook runs.
pub(crate) fn req_info(self) -> Self {
self.extension(ReqInfo::default())
}
pub(crate) fn server_context(self, slot: &Arc<ServerContextSlot>) -> Self {
self.extension(slot.clone())
}
pub(crate) fn extension<T: Clone + Send + Sync + 'static>(mut self, value: T) -> Self {
self.extensions.insert(value);
self
}
pub(crate) fn build(self) -> RequestEnvelope {
let credentials = self.credentials.map(|(access_key, secret_key)| {
let session_token = get_session_token(&self.uri, &self.headers).map(str::to_owned);
RequestCredentials::new(access_key.clone(), secret_key, session_token.clone())
.unwrap_or_else(|_| RequestCredentials::without_secret_key(access_key, session_token))
});
RequestEnvelope::from_parts(
RequestHead {
method: self.method,
uri: self.uri,
headers: self.headers,
credentials,
region: None,
service: None,
},
self.extensions,
)
}
}
@@ -1,23 +0,0 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Tests of the shared authorization core, run on the owned request envelope.
mod actions;
mod conditions;
mod denial_audit;
pub(crate) mod fixture;
mod permission_auth;
mod readers;
mod table_data_plane;
@@ -1,533 +0,0 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! The table data plane's share of S3 authorization, on the owned request: which object actions
//! map to table metadata actions, which ones need the publication fence, how table listings hide
//! unauthorized objects and seal their cursors, how catalog failures are answered, and how a
//! request reuses the table resolution and fences it already holds.
//!
//! Responsible for: `storage::authorization::table_data_plane` and the multipart authorization
//! functions' use of the publication fence, driven through `RequestEnvelope`.
//! Not responsible for: the table catalog's own storage, or the listing use cases that consume
//! the listing snapshot.
//! Upstream: the owned request fixture, the gating test stores, and the table catalog.
//! Downstream: `storage::authorization` and `storage::access_owned`.
use crate::storage::access_owned::object_commit::{
CopySourceObject, authorize_complete_multipart_upload, authorize_upload_part_copy,
};
use crate::storage::access_owned::object_write::authorize_upload_part;
use crate::storage::authorization::tests::fixture::{request, server_context};
use crate::storage::authorization::{
AuthorizationRequest, TABLE_DATA_PLANE_LIST_CURSOR_MAX_ENCODED_LEN, TableDataPlaneListAccess,
TableDataPlaneListCursorPosition, TableDataPlaneListResource, TableDataPlanePublicationGuards,
require_owned_reserved_table_object, retain_table_bucket_publication_guard, table_catalog_backend_for_data_plane,
table_data_plane_admin_action, table_data_plane_content_mutation, table_data_plane_resource_for_request,
table_publication_guard_error,
};
use crate::storage::storage_api::contract::bucket::{BucketOperations as _, MakeBucketOptions};
use http::Method;
use rustfs_policy::policy::action::{Action, S3Action};
use rustfs_s3_types::S3ErrorCode;
use serial_test::serial;
use std::sync::Arc;
#[test]
fn table_data_plane_admin_action_maps_s3_object_operations() {
assert_eq!(
table_data_plane_admin_action(Action::S3Action(S3Action::GetObjectAction)),
Some(rustfs_policy::policy::action::AdminAction::GetTableMetadataAction)
);
assert_eq!(
table_data_plane_admin_action(Action::S3Action(S3Action::PutObjectAction)),
Some(rustfs_policy::policy::action::AdminAction::SetTableMetadataAction)
);
assert_eq!(
table_data_plane_admin_action(Action::S3Action(S3Action::PutObjectTaggingAction)),
Some(rustfs_policy::policy::action::AdminAction::SetTableMetadataAction)
);
assert_eq!(
table_data_plane_admin_action(Action::S3Action(S3Action::DeleteObjectAction)),
Some(rustfs_policy::policy::action::AdminAction::SetTableMetadataAction)
);
assert_eq!(
table_data_plane_admin_action(Action::S3Action(S3Action::GetObjectTaggingAction)),
Some(rustfs_policy::policy::action::AdminAction::GetTableMetadataAction)
);
assert_eq!(table_data_plane_admin_action(Action::S3Action(S3Action::ListBucketAction)), None);
}
fn table_list_access(allowed: bool) -> TableDataPlaneListAccess {
TableDataPlaneListAccess {
bucket: "analytics".to_string(),
principal_fingerprint: "principal".to_string(),
snapshot_fingerprint: "snapshot".to_string(),
cursor_key: [0x42; 32],
resources: vec![TableDataPlaneListResource {
resource: crate::table_catalog::TableDataPlaneResource {
table_bucket: "analytics".to_string(),
namespace: "sales".to_string(),
table: "orders".to_string(),
table_id: "table-id".to_string(),
warehouse_object_prefix: "tables/table-id/".to_string(),
},
allowed,
}],
}
}
/// The listing authorization hands its table snapshot to the use case through the request it
/// was given.
#[test]
fn table_list_access_reaches_the_use_case_from_either_edge() {
let access = TableDataPlaneListAccess::test_listing_fixture("bucket");
let mut envelope = request(Method::GET).build();
assert!(envelope.table_list_access().is_none());
AuthorizationRequest::install_table_list_access(&mut envelope, access);
assert!(envelope.table_list_access().is_some());
}
#[test]
fn table_list_access_hides_unauthorized_table_objects_and_reserved_metadata() {
let denied = table_list_access(false);
assert!(!denied.allows_object("tables/table-id/data/file.parquet"));
assert!(
!denied.allows_object(".rustfs-table/warehouses/default/namespaces/sales/tables/orders/metadata/00001.metadata.json")
);
assert!(!denied.allows_object(".rustfs-table/private/catalog.json"));
assert!(denied.allows_object("ordinary/file.txt"));
let allowed = table_list_access(true);
assert!(allowed.allows_object("tables/table-id/data/file.parquet"));
assert!(
!allowed.allows_object(".rustfs-table/warehouses/default/namespaces/sales/tables/orders/metadata/00001.metadata.json")
);
assert!(!allowed.allows_object(".rustfs-table/private/catalog.json"));
}
#[test]
fn table_list_cursor_is_confidential_and_bound_to_the_request_snapshot() {
let access = table_list_access(true);
let marker = "tables/hidden/data/file.parquet";
let token = access
.encode_cursor(
"ListObjectsV2",
"tables/",
Some("/"),
1000,
TableDataPlaneListCursorPosition {
marker: Some(marker.to_string()),
version_marker: None,
common_prefix: None,
},
)
.expect("cursor should encode");
assert!(!token.contains(marker));
let decoded = access
.decode_cursor("ListObjectsV2", "tables/", Some("/"), 1000, &token)
.expect("cursor should decode");
assert_eq!(decoded.marker(), Some(marker));
let mut changed = access.clone();
changed.snapshot_fingerprint = "changed".to_string();
assert!(
changed
.decode_cursor("ListObjectsV2", "tables/", Some("/"), 1000, &token)
.is_err()
);
assert!(
access
.decode_cursor("ListObjectsV2", "other/", Some("/"), 1000, &token)
.is_err()
);
let mut rotated_key = access.clone();
rotated_key.cursor_key[0] ^= 1;
assert!(
rotated_key
.decode_cursor("ListObjectsV2", "tables/", Some("/"), 1000, &token)
.is_err()
);
assert!(
access
.decode_cursor(
"ListObjectsV2",
"tables/",
Some("/"),
1000,
&"x".repeat(TABLE_DATA_PLANE_LIST_CURSOR_MAX_ENCODED_LEN + 1),
)
.is_err()
);
}
#[test]
fn reserved_table_metadata_requires_an_active_table_owner() {
let error = require_owned_reserved_table_object::<()>(
".rustfs-table/warehouses/default/namespaces/analytics/tables/events/metadata/00001.metadata.json",
None,
)
.expect_err("unowned reserved metadata must fail closed");
assert_eq!(error.code(), &S3ErrorCode::AccessDenied);
assert!(
require_owned_reserved_table_object("objects/data.parquet", None::<()>)
.unwrap()
.is_none()
);
assert!(
require_owned_reserved_table_object(".rustfs-table/metadata.json", Some(()))
.unwrap()
.is_some()
);
}
#[test]
fn table_data_plane_publication_guard_covers_content_mutations_only() {
for action in [
S3Action::PutObjectAction,
S3Action::DeleteObjectAction,
S3Action::DeleteObjectVersionAction,
S3Action::RestoreObjectAction,
S3Action::ReplicateObjectAction,
S3Action::ReplicateDeleteAction,
S3Action::PutObjectFanOutAction,
] {
assert!(table_data_plane_content_mutation(Action::S3Action(action)));
}
for action in [
S3Action::GetObjectAction,
S3Action::PutObjectTaggingAction,
S3Action::DeleteObjectTaggingAction,
S3Action::ListBucketAction,
] {
assert!(!table_data_plane_content_mutation(Action::S3Action(action)));
}
}
/// The message `ApiError::error_code_to_message` writes for `ServiceUnavailable`.
const SERVICE_UNAVAILABLE_MESSAGE: &str = "The service is unavailable. Please retry.";
/// The message `ApiError::error_code_to_message` writes for `InternalError`.
const INTERNAL_ERROR_MESSAGE: &str = "We encountered an internal error, please try again.";
/// Whether the error carries a `Retry-After` response header, which is the only place the
/// gateway's error rendering (`gateway_handlers::error::handler_error`) takes it from.
fn carries_retry_after(err: &rustfs_s3_types::S3Error) -> bool {
err.headers()
.is_some_and(|headers| headers.contains_key(http::header::RETRY_AFTER))
}
#[test]
fn table_publication_guard_unavailable_errors_are_retryable() {
use crate::table_catalog::TableCatalogStoreError;
for message in [
"failed to acquire catalog migration lock: Quorum not reached: required 2, achieved 1",
"failed to acquire catalog migration lock: lock acquisition timed out after 5s",
"peer unavailable",
"",
] {
let error = TableCatalogStoreError::Unavailable(message.to_string());
let err = table_publication_guard_error(error);
assert_eq!(err.code(), &S3ErrorCode::ServiceUnavailable, "{message}");
assert_eq!(
err.status_code().or_else(|| err.code().status_code()),
Some(http::StatusCode::SERVICE_UNAVAILABLE)
);
assert_eq!(err.message(), Some(SERVICE_UNAVAILABLE_MESSAGE));
assert!(!carries_retry_after(&err));
}
}
#[test]
fn table_publication_guard_non_retryable_errors_stay_internal() {
use crate::table_catalog::TableCatalogStoreError;
let message = "temporarily unavailable: timeout: quorum not reached".to_string();
for error in [
TableCatalogStoreError::Internal(message.clone()),
TableCatalogStoreError::Invalid(message.clone()),
TableCatalogStoreError::Unsupported(message.clone()),
TableCatalogStoreError::NotFound(message.clone()),
TableCatalogStoreError::NamespaceNotFound(message.clone()),
TableCatalogStoreError::TableNotFound(message.clone()),
TableCatalogStoreError::AlreadyExists(message.clone()),
TableCatalogStoreError::Conflict(message),
] {
let err = table_publication_guard_error(error);
assert_eq!(err.code(), &S3ErrorCode::InternalError);
assert_eq!(
err.status_code().or_else(|| err.code().status_code()),
Some(http::StatusCode::INTERNAL_SERVER_ERROR)
);
assert_eq!(err.message(), Some(INTERNAL_ERROR_MESSAGE));
assert!(!carries_retry_after(&err));
}
}
#[tokio::test]
#[serial]
async fn table_data_plane_cold_lookup_does_not_wait_for_backup_while_holding_publication() {
use crate::table_catalog::{TableCatalogObjectBackend as _, TableCatalogStore as _};
let (_temp_dir, _disk_paths, object_store) = crate::app::gating_test_env::isolated_multi_pool_ecstore().await;
let bucket = "cold-authorization";
object_store
.make_bucket(bucket, &MakeBucketOptions::default())
.await
.expect("create table bucket");
let server_ctx = server_context(object_store.clone());
let mut request_envelope = request(Method::PUT).server_context(&server_ctx).build();
let backend = table_catalog_backend_for_data_plane(&request_envelope).expect("request backend");
let catalog = crate::table_catalog::ConfiguredTableCatalogStore::new_for_test(
backend.clone(),
crate::table_catalog::TableCatalogBackingMode::ObjectBacked,
);
catalog
.put_table_bucket(crate::table_catalog::TableBucketEntry {
version: crate::table_catalog::TABLE_CATALOG_ENTRY_VERSION,
table_bucket: bucket.to_string(),
catalog_type: crate::table_catalog::TABLE_BUCKET_CATALOG_TYPE.to_string(),
warehouse_root: format!("s3://{bucket}/"),
state: crate::table_catalog::TableCatalogEntryState::Active,
properties: Default::default(),
active_rename_id: None,
created_at: None,
updated_at: None,
})
.await
.expect("newly enabled catalog entry");
retain_table_bucket_publication_guard(&mut request_envelope, bucket)
.await
.expect("S3 owns publication first");
let backup_guard = backend
.acquire_write_lock(
crate::admin::storage_api::RUSTFS_META_BUCKET,
&crate::table_catalog::TableCatalogObjectPaths::default().catalog_backup_fence_lock_path(bucket),
)
.await
.expect("backup owns its catalog fence");
let resource = tokio::time::timeout(
std::time::Duration::from_secs(5),
table_data_plane_resource_for_request(&mut request_envelope, bucket, "ordinary-object", true),
)
.await
.expect("cold S3 lookup must not wait for the backup catalog fence")
.expect("resolve newly enabled table bucket");
assert!(resource.is_none());
assert!(
request_envelope
.table_publication_guards()
.expect("retained publication")
.state
.lock()
.missing_resources
.contains(&(bucket.to_string(), "ordinary-object".to_string()))
);
drop(backup_guard);
drop(request_envelope);
let mut read_request = request(Method::GET).server_context(&server_ctx).build();
table_data_plane_resource_for_request(&mut read_request, bucket, "ordinary-object", true)
.await
.expect("unfenced reads retain normal index initialization");
assert!(
backend
.read_object(
crate::admin::storage_api::RUSTFS_META_BUCKET,
&crate::table_catalog::TableCatalogObjectPaths::default().warehouse_index_state_path(bucket),
)
.await
.expect("read initialized index marker")
.is_some()
);
}
#[tokio::test]
#[serial]
async fn multipart_parts_skip_table_publication_fence_but_completion_retains_it() {
let store = crate::app::gating_test_env::shared_gating_ecstore().await;
let server_ctx = server_context(Arc::clone(&store));
let bucket = format!("multipart-table-fence-{}", uuid::Uuid::new_v4());
store
.make_bucket(&bucket, &MakeBucketOptions::default())
.await
.expect("test bucket should be created");
let policy_json = format!(
r#"{{"Version":"2012-10-17","Statement":[{{"Effect":"Allow","Principal":{{"AWS":"*"}},"Action":["s3:GetObject","s3:PutObject"],"Resource":["arn:aws:s3:::{bucket}/*"]}}]}}"#
);
let mut metadata = (*crate::storage::get_bucket_metadata(&bucket)
.await
.expect("new bucket metadata should be cached"))
.clone();
metadata.policy_config = Some(serde_json::from_str(&policy_json).expect("test bucket policy should parse"));
metadata.policy_config_json = policy_json.into_bytes();
crate::storage::storage_api::set_bucket_metadata(bucket.clone(), metadata)
.await
.expect("test bucket policy should be published");
// UploadPart: upload id `upload-id`, part number 1.
let mut part_req = request(Method::PUT).req_info().server_context(&server_ctx).build();
authorize_upload_part(&mut part_req, &server_ctx, &bucket, "object")
.await
.expect("anonymous UploadPart should be authorized by the test policy");
assert!(
part_req.table_publication_guards().is_none(),
"staged part data must not retain a table publication fence"
);
// UploadPartCopy: upload id `upload-id`, part number 2, source `{bucket}/source`.
let mut copy_part_req = request(Method::PUT).req_info().server_context(&server_ctx).build();
authorize_upload_part_copy(
&mut copy_part_req,
&server_ctx,
&bucket,
"object",
Some(CopySourceObject {
bucket: &bucket,
key: "source",
version_id: None,
}),
)
.await
.expect("anonymous UploadPartCopy should be authorized by the test policy");
assert!(
copy_part_req.table_publication_guards().is_none(),
"staged copied part data must not retain a table publication fence"
);
// CompleteMultipartUpload: upload id `upload-id`, an empty part list, no `If-Match`.
let mut complete_req = request(Method::POST).req_info().server_context(&server_ctx).build();
authorize_complete_multipart_upload(&mut complete_req, &server_ctx, &bucket, "object", false)
.await
.expect("anonymous CompleteMultipartUpload should be authorized by the test policy");
let bucket_fence = (bucket, crate::table_catalog::default_table_bucket_publication_lock_path());
let guards = complete_req
.table_publication_guards()
.expect("completion should retain a table publication fence");
assert!(guards.state.lock().keys.contains(&bucket_fence));
}
#[tokio::test]
async fn table_data_plane_request_reuses_table_resource_for_distinct_object_while_commit_waits() {
let resource = crate::table_catalog::TableDataPlaneResource {
table_bucket: "warehouse".to_string(),
namespace: "analytics".to_string(),
table: "events".to_string(),
table_id: "table-id".to_string(),
warehouse_object_prefix: "tables/table-id/".to_string(),
};
let cached_key = ("warehouse".to_string(), "tables/table-id/".to_string());
let requested_key = ("warehouse".to_string(), "tables/table-id/data/part-00002.parquet".to_string());
let retained = TableDataPlanePublicationGuards::default();
retained.state.lock().resources.insert(cached_key, resource.clone());
let publication_lock = Arc::new(tokio::sync::Mutex::new(()));
let request_publication_guard = Arc::clone(&publication_lock).lock_owned().await;
retained.state.lock().guards.push(Box::new(request_publication_guard));
let mut req = request(Method::PUT).extension(retained.clone()).build();
let catalog_lock = Arc::new(tokio::sync::Mutex::new(()));
let commit_started = Arc::new(tokio::sync::Notify::new());
let commit_catalog_lock = Arc::clone(&catalog_lock);
let commit_publication_lock = Arc::clone(&publication_lock);
let commit_started_signal = Arc::clone(&commit_started);
let commit = tokio::spawn(async move {
let _catalog_guard = commit_catalog_lock.lock().await;
commit_started_signal.notify_one();
let _publication_guard = commit_publication_lock.lock().await;
});
commit_started.notified().await;
let resolved = tokio::time::timeout(
std::time::Duration::from_secs(1),
table_data_plane_resource_for_request(&mut req, &requested_key.0, &requested_key.1, true),
)
.await
.expect("cached table resolution must not wait for the catalog lock")
.expect("cached table resolution should succeed");
assert_eq!(resolved, Some(resource));
retained.state.lock().guards.clear();
tokio::time::timeout(std::time::Duration::from_secs(1), commit)
.await
.expect("commit should continue after the request releases its publication guard")
.expect("commit task should join");
}
#[tokio::test]
async fn table_data_plane_request_reuses_reserved_warehouse_resource() {
let resource = crate::table_catalog::TableDataPlaneResource {
table_bucket: "warehouse".to_string(),
namespace: "analytics".to_string(),
table: "events".to_string(),
table_id: "table-id".to_string(),
warehouse_object_prefix: ".rustfs-table/custom-warehouse/events/".to_string(),
};
let retained = TableDataPlanePublicationGuards::default();
retained.state.lock().resources.insert(
(resource.table_bucket.clone(), resource.warehouse_object_prefix.clone()),
resource.clone(),
);
let mut req = request(Method::GET).extension(retained).build();
let resolved = table_data_plane_resource_for_request(
&mut req,
"warehouse",
".rustfs-table/custom-warehouse/events/data/part-00001.parquet",
true,
)
.await
.expect("a reserved-prefix warehouse object should use ordinary table resolution");
assert_eq!(resolved, Some(resource));
}
#[tokio::test]
async fn table_data_plane_request_reuses_absence_while_bucket_publication_guard_is_held() {
let key = ("warehouse".to_string(), "tables/new-table/data/file.parquet".to_string());
let bucket_fence = (key.0.clone(), crate::table_catalog::default_table_bucket_publication_lock_path());
let retained = TableDataPlanePublicationGuards::default();
{
let mut state = retained.state.lock();
state.keys.insert(bucket_fence);
state.missing_resources.insert(key.clone());
}
let publication_lock = Arc::new(tokio::sync::Mutex::new(()));
let request_publication_guard = Arc::clone(&publication_lock).lock_owned().await;
retained.state.lock().guards.push(Box::new(request_publication_guard));
let mut req = request(Method::PUT).extension(retained.clone()).build();
let commit_publication_lock = Arc::clone(&publication_lock);
let commit = tokio::spawn(async move {
let _publication_guard = commit_publication_lock.lock().await;
});
tokio::task::yield_now().await;
let resolved = tokio::time::timeout(
std::time::Duration::from_secs(1),
table_data_plane_resource_for_request(&mut req, &key.0, &key.1, true),
)
.await
.expect("cached absence must not reread the catalog while publication waits")
.expect("cached absence should resolve successfully");
assert!(resolved.is_none());
retained.state.lock().guards.clear();
tokio::time::timeout(std::time::Duration::from_secs(1), commit)
.await
.expect("publication should continue after the request releases its bucket guard")
.expect("publication task should join");
}
-2
View File
@@ -155,8 +155,6 @@ mod legacy_edge;
#[cfg(test)]
pub(crate) use legacy_edge::complete_multipart_upload_input;
#[cfg(test)]
pub(crate) use legacy_edge::copy_input_from_legacy;
#[cfg(test)]
pub(crate) use legacy_edge::list_objects_v2_through_legacy_edge;
#[async_trait::async_trait]
-2
View File
@@ -30,8 +30,6 @@ mod object_misc;
#[cfg(test)]
pub(crate) use object_misc::complete_multipart_upload_input;
#[cfg(test)]
pub(crate) use object_misc::copy_input_from_legacy;
mod object_read;
mod object_write;
@@ -280,7 +280,7 @@ impl DefaultObjectUsecase {
}
}
pub(crate) fn copy_input_from_legacy(input: CopyObjectInput) -> CopyObjectWriteInput {
fn copy_input_from_legacy(input: CopyObjectInput) -> CopyObjectWriteInput {
let condition = |value: ETagCondition| match value {
ETagCondition::Any => CopySourceCondition::Any,
ETagCondition::ETag(ETag::Strong(value)) => CopySourceCondition::Strong(value),
+28 -10
View File
@@ -33,14 +33,8 @@ impl FS {
req: S3Request<PutObjectInput>,
) -> S3Result<S3Response<PutObjectOutput>> {
crate::hp_guard!("S3::put_object");
record_s3_op(S3Operation::PutObject);
let envelope = RequestEnvelope::from_s3s(&req);
let input = PutObjectWriteInput::from_legacy(req.input, req.trailing_headers);
s3_api::object_usecase_for(self)
.execute_put_object_owned(envelope, input)
.await
.map(put_output_to_legacy)
.map_err(Into::into)
let usecase = s3_api::object_usecase_for(self);
usecase.execute_put_object(self, req).await
}
#[instrument(name = "upload_part", level = "debug", skip(self, req))]
@@ -57,8 +51,8 @@ impl FS {
use crate::app::RequestEnvelope;
use crate::storage::s3_api::{
DefaultMultipartUsecase, PutObjectWriteInput, PutObjectWriteOutput, UploadPartTarget, UploadPartWriteInput,
UploadPartWriteOutput,
DefaultMultipartUsecase, DefaultObjectUsecase, PutObjectWriteInput, PutObjectWriteOutput, UploadPartTarget,
UploadPartWriteInput, UploadPartWriteOutput,
};
impl PutObjectWriteInput {
@@ -125,6 +119,30 @@ fn put_output_to_legacy(output: PutObjectWriteOutput) -> S3Response<PutObjectOut
response
}
impl DefaultObjectUsecase {
#[instrument(name = "execute_put_object", level = "info", skip(self, _fs, req))]
#[hotpath::measure(impl_type = "DefaultObjectUsecase", future = true)]
pub async fn execute_put_object(&self, _fs: &FS, req: S3Request<PutObjectInput>) -> S3Result<S3Response<PutObjectOutput>> {
record_s3_op(S3Operation::PutObject);
let envelope = RequestEnvelope::from_s3s(&req);
let input = PutObjectWriteInput::from_legacy(req.input, req.trailing_headers);
self.execute_put_object_owned(envelope, input)
.await
.map(put_output_to_legacy)
.map_err(Into::into)
}
pub async fn execute_put_object_extract(&self, req: S3Request<PutObjectInput>) -> S3Result<S3Response<PutObjectOutput>> {
record_s3_op(S3Operation::PutObject);
let envelope = RequestEnvelope::from_s3s(&req);
let input = PutObjectWriteInput::from_legacy(req.input, req.trailing_headers);
self.execute_put_object_extract_owned(envelope, input)
.await
.map(put_output_to_legacy)
.map_err(Into::into)
}
}
impl UploadPartWriteInput {
pub(crate) fn from_legacy(input: UploadPartInput, trailers: Option<TrailingHeaders>) -> Self {
Self {
-1
View File
@@ -483,7 +483,6 @@ pub(crate) mod tonic_service_consumer {
#[cfg(test)]
pub(crate) mod test_consumer {
pub(crate) use super::super::access_owned::object_read::authorize_head_object;
pub(crate) mod sse {
pub(crate) use super::super::super::sse::TestAuthorizationHooks;
pub(crate) use super::super::super::sse::tests::{RecordingKmsKeyAuthorizer, sse_kms_object_metadata};
+9 -11
View File
@@ -79,15 +79,13 @@ SEAMS = {
# P2 splits bucket_usecase.rs into bucket_usecase/*.rs; narrow this to the listing file once it exists.
owner_dirs=("rustfs/src/app/bucket_usecase/",),
),
"execute_put_object_owned": Seam(
"execute_put_object": Seam(
batch="D",
seam_file="rustfs/src/app/test_seams/put_object.rs",
owners=(
"rustfs/src/app/object/put.rs",
"rustfs/src/app/lifecycle_transition_api_test/put.rs",
),
# PostObject reaches the PutObject use case; its body tests are PutObject's own.
owner_dirs=("rustfs/src/app/object/post_body/",),
),
}
@@ -398,7 +396,7 @@ SEAM_BODY = {
"execute_get_object": "pub(crate) fn get_object(u: &U, r: R) -> F { u.execute_get_object_fields(r) }\n",
"execute_head_object": "pub(crate) fn head_object(u: &U, r: R) -> F { u.execute_head_object_fields(r) }\n",
"execute_list_objects_v2": "pub(crate) fn list_objects_v2(u: &U, r: R) -> F { u.execute_list_objects_v2(r) }\n",
"execute_put_object_owned": "pub(crate) fn put_object(u: &U, f: &Fs, r: R) -> F { u.execute_put_object_owned(f, r) }\n",
"execute_put_object": "pub(crate) fn put_object(u: &U, f: &Fs, r: R) -> F { u.execute_put_object(f, r) }\n",
}
@@ -508,7 +506,7 @@ class SelfTest(unittest.TestCase):
self.assertEqual(len(self.run_check()), 1)
def test_file_declared_under_cfg_test_fails(self):
self.write("rustfs/src/app/gating_test.rs", "fn t() { u.execute_put_object_owned(&fs, r); }\n")
self.write("rustfs/src/app/gating_test.rs", "fn t() { u.execute_put_object(&fs, r); }\n")
self.assertEqual(len(self.run_check()), 1)
def test_file_whose_ancestor_is_cfg_test_fails(self):
@@ -517,7 +515,7 @@ class SelfTest(unittest.TestCase):
self.assertEqual(len(self.run_check()), 1)
def test_put_from_non_owner_lifecycle_file_fails(self):
self.write("rustfs/src/app/lifecycle_transition_api_test/delete.rs", "#[test]\nfn t() { u.execute_put_object_owned(&fs, r); }\n")
self.write("rustfs/src/app/lifecycle_transition_api_test/delete.rs", "#[test]\nfn t() { u.execute_put_object(&fs, r); }\n")
self.write("rustfs/src/app/mod.rs", self.files["rustfs/src/app/mod.rs"].replace("mod lifecycle", "#[cfg(test)]\nmod lifecycle"))
self.assertEqual(len(self.run_check()), 1)
@@ -535,14 +533,14 @@ class SelfTest(unittest.TestCase):
self.assertRegex(" ".join(self.run_check()), "seam for execute_get_object is missing")
def test_seam_that_stopped_wrapping_fails(self):
self.write(SEAMS["execute_put_object_owned"].seam_file, "pub(crate) fn put_object() { todo!() }\n")
self.assertRegex(" ".join(self.run_check()), "must call execute_put_object_owned exactly once, found 0")
self.write(SEAMS["execute_put_object"].seam_file, "pub(crate) fn put_object() { todo!() }\n")
self.assertRegex(" ".join(self.run_check()), "must call execute_put_object exactly once, found 0")
def test_missing_owner_file_fails(self):
owner = SEAMS["execute_put_object_owned"].owners[0]
owner = SEAMS["execute_put_object"].owners[0]
(self.root / owner).unlink()
del self.files[owner]
self.assertRegex(" ".join(self.run_check()), "owner file of execute_put_object_owned is missing")
self.assertRegex(" ".join(self.run_check()), "owner file of execute_put_object is missing")
def test_undeclared_file_with_direct_call_fails_loudly(self):
self.write("rustfs/src/app/orphan.rs", "fn t() { u.execute_get_object(r); }\n")
@@ -563,7 +561,7 @@ class SelfTest(unittest.TestCase):
self.assertEqual(self.run_check(), [])
def test_owner_test_and_production_calls_pass(self):
self.write("rustfs/src/app/object/put.rs", "fn prod() {}\n#[cfg(test)]\nmod tests {\n fn t() { u.execute_put_object_owned(&fs, r); }\n}\n")
self.write("rustfs/src/app/object/put.rs", "fn prod() {}\n#[cfg(test)]\nmod tests {\n fn t() { u.execute_put_object(&fs, r); }\n}\n")
self.assertEqual(self.run_check(), [])
def path_module(self, attributes: str = "", body: str = "fn prod() { u.execute_get_object(r); }\n"):