Compare commits

..

5 Commits

Author SHA1 Message Date
overtrue fbe3d1c9c8 test(app): add owned test seams and route authorize_head_object through storage_api 2026-11-20 06:05:13 +08:00
overtrue 6bec9362b1 test(storage): delete the legacy access tests the owned core now covers
The tests module of storage/access.rs, its mounted reader and probe
tests, and the signed-query regression module are gone; their ported
copies run on the owned core. The five registered tests are deleted with
their register entries. access.rs keeps the hooks, the parity support
other modules still use, and four pins on legacy-only helpers that no
gateway-side test pins yet.
2026-11-20 05:23:54 +08:00
overtrue e237d268ac test(storage): port the legacy access tests onto the owned authorization core
Each authorization test of storage/access.rs and its mounted files now
runs on the owned RequestEnvelope through the owned function its legacy
hook delegated to, with the same inputs and expected decisions. A shared
owned fixture replaces build_request and TestKms. The four access_owned
test files move under access_owned/tests and no longer build an s3s
request. Five tests that pin only legacy DTO helpers are registered for
retirement with their gateway-side replacements.
2026-11-20 05:23:40 +08:00
overtrue a52b51b764 refactor(app): drop s3s entrypoints and move test seams to owned types 2026-11-20 05:23:37 +08:00
overtrue e62c6112ea chore(ci): lower the s3s footprint ceilings to the measured values
files 50 -> 49 and s3_error_lines 103 -> 95, as scripts/check_s3s_footprint.sh
measures them after the upload drain tests stopped implementing an s3s
S3 service.
2026-11-20 05:23:37 +08:00
53 changed files with 5551 additions and 5416 deletions
+2 -2
View File
@@ -143,7 +143,7 @@ setup = 'ecstore-base-stack'
# The multipart refusal fixture composes real-store setup, abort, quota and legal-hold futures.
[[profile.default.scripts]]
filter = 'package(rustfs-ecstore) | package(rustfs-s3select-api) | package(rustfs-scanner) | (package(rustfs) & test(/^(app::multipart_usecase::complete::tests::completion_absent_binding_refusal_order_baseline|app::multipart_usecase::complete::tests::concurrent_completions_share_durable_bucket_quota_reservations|app::object::delete::tests::compressed_delete_requests_update_observed_usage_without_releasing_quota_floor|app::object::internal_put::tests::internal_multipart_roundtrip_completes_and_abort_leaves_nothing|app::object::restore::tests::execute_restore_object_maps_failures_to_typed_s3_errors|storage::access::tests::(delete_object_access_captures_authorized_bucket_incarnation|copy_operations_reject_recreated_source_bucket_after_authorization|request_slot_keeps_bucket_policy_bound_to_its_store))$/))'
filter = 'package(rustfs-ecstore) | package(rustfs-s3select-api) | package(rustfs-scanner) | (package(rustfs) & test(/^(app::multipart_usecase::complete::tests::completion_absent_binding_refusal_order_baseline|app::multipart_usecase::complete::tests::concurrent_completions_share_durable_bucket_quota_reservations|app::object::delete::tests::compressed_delete_requests_update_observed_usage_without_releasing_quota_floor|app::object::internal_put::tests::internal_multipart_roundtrip_completes_and_abort_leaves_nothing|app::object::restore::tests::execute_restore_object_maps_failures_to_typed_s3_errors|storage::access_owned::tests::generation_guards::(delete_object_access_captures_authorized_bucket_incarnation|copy_operations_reject_recreated_source_bucket_after_authorization|request_slot_keeps_bucket_policy_bound_to_its_store))$/))'
setup = 'ecstore-base-stack'
# These handler/store fixtures and the relocated-pool GET compose deep storage
@@ -466,7 +466,7 @@ filter = '(package(rustfs) & test(/^(admin::handlers::replication::target_repair
setup = 'ecstore-base-stack'
[[profile.ci.scripts]]
filter = 'package(rustfs-ecstore) | package(rustfs-s3select-api) | package(rustfs-scanner) | (package(rustfs) & test(/^(app::multipart_usecase::complete::tests::completion_absent_binding_refusal_order_baseline|app::multipart_usecase::complete::tests::concurrent_completions_share_durable_bucket_quota_reservations|app::object::delete::tests::compressed_delete_requests_update_observed_usage_without_releasing_quota_floor|app::object::internal_put::tests::internal_multipart_roundtrip_completes_and_abort_leaves_nothing|app::object::restore::tests::execute_restore_object_maps_failures_to_typed_s3_errors|storage::access::tests::(delete_object_access_captures_authorized_bucket_incarnation|copy_operations_reject_recreated_source_bucket_after_authorization|request_slot_keeps_bucket_policy_bound_to_its_store))$/))'
filter = 'package(rustfs-ecstore) | package(rustfs-s3select-api) | package(rustfs-scanner) | (package(rustfs) & test(/^(app::multipart_usecase::complete::tests::completion_absent_binding_refusal_order_baseline|app::multipart_usecase::complete::tests::concurrent_completions_share_durable_bucket_quota_reservations|app::object::delete::tests::compressed_delete_requests_update_observed_usage_without_releasing_quota_floor|app::object::internal_put::tests::internal_multipart_roundtrip_completes_and_abort_leaves_nothing|app::object::restore::tests::execute_restore_object_maps_failures_to_typed_s3_errors|storage::access_owned::tests::generation_guards::(delete_object_access_captures_authorized_bucket_incarnation|copy_operations_reject_recreated_source_bucket_after_authorization|request_slot_keeps_bucket_policy_bound_to_its_store))$/))'
setup = 'ecstore-base-stack'
[[profile.ci.scripts]]
+2 -2
View File
@@ -13,6 +13,6 @@
# s3_error_lines lines holding s3_error!, same scope, minus those the script
# header's rule clears (rustfs_s3_types's macro); fails closed
# ecstore_files Rust files referencing s3s paths under crates/ecstore/src
files=50
s3_error_lines=103
files=49
s3_error_lines=95
ecstore_files=0
-88
View File
@@ -1635,94 +1635,6 @@ async fn run_data_scanner_cycle_publishes_activity_for_owner_lifetime() {
global_metrics().set_cycle(None).await;
}
#[tokio::test]
#[serial]
async fn single_drive_cycle_persists_usage_and_advances_with_healing_marker() {
use crate::scanner_io::{clear_dirty_usage_buckets_for_tests, tests::wait_for_namespace_commit_tails};
use crate::storage_api::scan::{HEALING_MARKER_PATH, MakeBucketOptions};
clear_dirty_usage_buckets_for_tests();
let temp_dir = tempfile::tempdir().expect("scanner cycle test directory should be created");
let store = setup_scanner_cycle_store_at_path_with_layout(temp_dir.path(), true, 1, 1, 1).await;
let bucket = "marked-drive-bucket";
store
.make_bucket(bucket, &MakeBucketOptions::default())
.await
.expect("fixture bucket should be created");
let mut reader = PutObjReader::from_vec(vec![7; 16]);
store
.put_object(bucket, "object", &mut reader, &ObjectOptions::default())
.await
.expect("fixture object should persist");
wait_for_namespace_commit_tails(store.as_ref()).await;
let marker = temp_dir
.path()
.join("pool0/disk0")
.join(RUSTFS_META_BUCKET)
.join(HEALING_MARKER_PATH);
let marker_contents = b"leftover marker";
tokio::fs::write(&marker, marker_contents)
.await
.expect("fixture healing marker should persist");
let set = &store.pools[0].disk_set[0];
tokio::time::timeout(Duration::from_secs(10), async {
while !set.get_online_disks_with_healing(true).await.1 {
tokio::time::sleep(Duration::from_millis(100)).await;
}
})
.await
.expect("disk info should observe the marker after its cache expires");
let ctx = CancellationToken::new();
let mut cycle_info = CurrentCycle {
next: 1,
..Default::default()
};
let mut revision = DataUsageCacheRevision::Missing;
let leader_epoch = 11;
let outcome = tokio::time::timeout(
Duration::from_secs(60),
run_data_scanner_cycle(&ctx, &store, &mut cycle_info, &mut revision, leader_epoch),
)
.await
.expect("scanner cycle should finish");
assert!(
matches!(
outcome,
ScannerCycleOutcome::Completed | ScannerCycleOutcome::CompletedWithPendingMaintenance
),
"unexpected cycle outcome: {outcome:?}",
);
let mut set_snapshot = DataUsageCache::default();
set_snapshot
.load_with_revisions(Arc::clone(set), DATA_USAGE_CACHE_NAME)
.await
.expect("set usage snapshot should be readable");
assert!(set_snapshot.info.snapshot_complete);
assert_eq!(set_snapshot.info.next_cycle, 1);
let usage = read_config(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str())
.await
.expect("global usage snapshot should be durable");
let usage: DataUsageInfo = serde_json::from_slice(&usage).expect("global usage snapshot should decode");
assert!(usage.usage_snapshot_complete);
assert!(usage.last_update.is_some());
assert_eq!(usage.buckets_usage[bucket].objects_count, 1);
let persisted_cycle = read_config(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str())
.await
.expect("scanner cycle cursor should be durable");
let (persisted_cycle, persisted_epoch) =
decode_scanner_cycle_state(&persisted_cycle).expect("scanner cycle cursor should decode");
assert_eq!(persisted_epoch, leader_epoch);
assert_eq!(persisted_cycle.next, 2);
assert_eq!(tokio::fs::read(marker).await.expect("healing marker should remain"), marker_contents);
clear_dirty_usage_buckets_for_tests();
global_metrics().set_cycle(None).await;
}
#[tokio::test]
#[serial]
async fn coordinator_publishes_authority_during_pending_put() {
+7 -20
View File
@@ -620,10 +620,7 @@ impl ScannerIOCache for SetDisks {
.ok_or_else(|| StorageError::other("failed to persist empty scanner set scope"));
}
let single_drive = self.set_drive_count == 1;
// A single-drive set has no alternate scan source. A heal marker
// still disables scanner heals, but must not exclude its only usage walk.
let (disks, healing) = self.get_online_disks_with_healing(single_drive).await;
let (disks, healing) = self.get_online_disks_with_healing(false).await;
#[cfg(test)]
let disks = super::tests::usage_remote_drain::walker_disks(&self, &buckets, disks);
if disks.is_empty() {
@@ -760,18 +757,7 @@ impl ScannerIOCache for SetDisks {
let _ = updates.send(incomplete_scope).await;
return Ok(ScannerSourceTurnReceipt::default());
}
// Coordinator workers need the full inventory only for cross-drive
// healing. Remote workers own their disk inventory on the peer.
let heal_disk_inventory = if !single_drive
&& !healing
&& workers
.iter()
.any(|(_, mode)| *mode == NamespaceScannerWorkerMode::Coordinator)
{
Some(Arc::new(scanner_set_disk_inventory(self.as_ref()).await))
} else {
None
};
let set_disk_inventory = Arc::new(scanner_set_disk_inventory(self.as_ref()).await);
if unsupported_remote_disks > 0 {
debug!(
target: "rustfs::scanner::io",
@@ -1024,6 +1010,7 @@ impl ScannerIOCache for SetDisks {
let store_clone_clone = self.clone();
let bucket_result_tx_clone = bucket_result_tx.clone();
let bucket_observations_clone = bucket_observations.clone();
let set_disk_inventory_clone = set_disk_inventory.clone();
let disk_scan_semaphore_clone = disk_scan_semaphore.clone();
let queued_disk_bucket_scans_clone = queued_disk_bucket_scans.clone();
let active_disk_bucket_scans_clone = active_disk_bucket_scans.clone();
@@ -1039,9 +1026,9 @@ impl ScannerIOCache for SetDisks {
let worker_retired_buckets = retired_buckets.clone();
#[cfg(feature = "e2e-test-hooks")]
let worker_incarnations = current_bucket_incarnations.clone();
let (remote_capabilities, heal_disk_inventory) = match worker_mode {
NamespaceScannerWorkerMode::Coordinator => (None, heal_disk_inventory.clone()),
NamespaceScannerWorkerMode::RemoteV4(capabilities) => (Some(capabilities), None),
let remote_capabilities = match worker_mode {
NamespaceScannerWorkerMode::RemoteV4(capabilities) => Some(capabilities),
NamespaceScannerWorkerMode::Coordinator => None,
};
#[cfg(test)]
let worker_index = futs.len();
@@ -1684,7 +1671,7 @@ impl ScannerIOCache for SetDisks {
let scan = crate::workload_admission::with_scanner_node_requests(node_tx, disk.clone().nsscanner_disk(
scan_ctx.clone(),
bucket_budget.clone(),
heal_disk_inventory.as_deref().cloned().unwrap_or_default(),
set_disk_inventory_clone.as_ref().clone(),
cache,
None,
ScannerDiskScanOptions {
-2
View File
@@ -422,8 +422,6 @@ pub(crate) mod scan {
#[cfg(test)]
pub(crate) use rustfs_ecstore::api::bucket::metadata_sys::acquire_scanner_bucket_incarnation_fence as ecstore_scanner_ack_fence;
#[cfg(test)]
pub(crate) use rustfs_ecstore::api::disk::HEALING_MARKER_PATH;
#[cfg(test)]
pub(crate) use rustfs_ecstore::api::notification::new_global_notification_sys;
}
@@ -79,3 +79,12 @@ mod object_lock;
mod put;
mod restore;
mod transition;
/// PutObject through its test seam, from a test's s3s fixture request.
async fn put_through_seam(
usecase: &DefaultObjectUsecase,
req: S3Request<PutObjectInput>,
) -> rustfs_s3_types::S3Result<crate::app::object_usecase::PutObjectWriteOutput> {
let (envelope, input) = crate::app::object_usecase::PutObjectWriteInput::from_test_request(req);
crate::app::test_seams::put_object(usecase, envelope, input).await
}
@@ -16,7 +16,6 @@
use super::common::*;
use super::*;
use crate::app::test_seams;
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
#[serial]
@@ -70,7 +69,7 @@ async fn copy_object_if_none_match_existing_destination_returns_precondition_fai
let mut req = build_request(copy_input, Method::PUT);
req.headers.insert(IF_NONE_MATCH, header.clone());
let err = Box::pin(usecase.execute_copy_object(req))
let err = Box::pin(copy_owned(&usecase, req))
.await
.expect_err("conditional COPY must reject the replacement");
assert_eq!(err.code(), &expected_code, "header={header:?}, has_dto={has_dto}");
@@ -93,7 +92,6 @@ async fn copy_object_if_none_match_existing_destination_returns_precondition_fai
#[ignore = "global-state object-version integration test: runs serialized in the CI ILM Integration (serial) lane"]
async fn undo_copy_requires_the_observed_current_version() {
let (_disk_paths, ecstore) = setup_test_env().await;
let fs = FS::new();
let usecase = DefaultObjectUsecase::from_global();
let bucket = format!("test-undo-copy-{}", &Uuid::new_v4().simple().to_string()[..8]);
let object = "test/object.txt";
@@ -108,16 +106,14 @@ async fn undo_copy_requires_the_observed_current_version() {
.build()
.unwrap()
};
let original = Box::pin(test_seams::put_object(&usecase, &fs, build_request(put(b"original"), Method::PUT)))
let original = Box::pin(put_through_seam(&usecase, build_request(put(b"original"), Method::PUT)))
.await
.expect("initial version should be written")
.output
.version_id
.expect("versioned PUT should return a version ID");
let observed_current = Box::pin(test_seams::put_object(&usecase, &fs, build_request(put(b"replacement"), Method::PUT)))
let observed_current = Box::pin(put_through_seam(&usecase, build_request(put(b"replacement"), Method::PUT)))
.await
.expect("replacement version should be written")
.output
.version_id
.expect("versioned PUT should return a version ID");
@@ -139,7 +135,7 @@ async fn undo_copy_requires_the_observed_current_version() {
HeaderValue::from_str(&Uuid::new_v4().to_string()).unwrap(),
);
let versions_before = live_object_version_count(&ecstore, bucket.as_str(), object).await;
let err = Box::pin(usecase.execute_copy_object(stale_req))
let err = Box::pin(copy_owned(&usecase, stale_req))
.await
.expect_err("stale destination must fail");
assert_eq!(err.code(), &S3ErrorCode::PreconditionFailed);
@@ -150,7 +146,7 @@ async fn undo_copy_requires_the_observed_current_version() {
matching_req
.headers
.insert(RUSTFS_EXPECTED_CURRENT_VERSION_ID, HeaderValue::from_str(&observed_current).unwrap());
Box::pin(usecase.execute_copy_object(matching_req))
Box::pin(copy_owned(&usecase, matching_req))
.await
.expect("matching destination version should permit undo copy");
assert_eq!(read_object_bytes(&ecstore, bucket.as_str(), object).await, b"original");
@@ -175,18 +171,15 @@ async fn undo_copy_serializes_a_concurrent_put_until_after_commit() {
.build()
.unwrap()
};
let fs = FS::new();
let usecase = DefaultObjectUsecase::from_global();
let original = Box::pin(test_seams::put_object(&usecase, &fs, build_request(put(b"original"), Method::PUT)))
let original = Box::pin(put_through_seam(&usecase, build_request(put(b"original"), Method::PUT)))
.await
.expect("original should be written")
.output
.version_id
.expect("original should be versioned");
let observed = Box::pin(test_seams::put_object(&usecase, &fs, build_request(put(b"observed"), Method::PUT)))
let observed = Box::pin(put_through_seam(&usecase, build_request(put(b"observed"), Method::PUT)))
.await
.expect("observed should be written")
.output
.version_id
.expect("observed should be versioned");
let encoded_object = encode_dir_object(object);
@@ -218,7 +211,7 @@ async fn undo_copy_serializes_a_concurrent_put_until_after_commit() {
let undo = tokio::spawn(async move {
let usecase = DefaultObjectUsecase::from_global();
Box::pin(usecase.execute_copy_object(undo_req)).await
Box::pin(copy_owned(&usecase, undo_req)).await
});
tokio::task::yield_now().await;
let put = PutObjectInput::builder()
@@ -229,9 +222,8 @@ async fn undo_copy_serializes_a_concurrent_put_until_after_commit() {
.build()
.unwrap();
let concurrent_put = tokio::spawn(async move {
let fs = FS::new();
let usecase = DefaultObjectUsecase::from_global();
Box::pin(test_seams::put_object(&usecase, &fs, build_request(put, Method::PUT))).await
Box::pin(put_through_seam(&usecase, build_request(put, Method::PUT))).await
});
drop(blocker);
@@ -249,7 +241,6 @@ async fn undo_copy_serializes_a_concurrent_put_until_after_commit() {
#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-1)"]
async fn copy_object_allows_new_version_for_locked_destination_but_blocks_explicit_overwrite() {
let (_disk_paths, ecstore) = setup_test_env().await;
let fs = FS::new();
let usecase = DefaultObjectUsecase::from_global();
let bucket = format!("test-copy-object-lock-{}", &Uuid::new_v4().simple().to_string()[..8]);
@@ -275,7 +266,7 @@ async fn copy_object_allows_new_version_for_locked_destination_but_blocks_explic
.build()
.unwrap();
Box::pin(test_seams::put_object(&usecase, &fs, build_request(locked_input, Method::PUT)))
Box::pin(put_through_seam(&usecase, build_request(locked_input, Method::PUT)))
.await
.expect("Failed to upload locked destination object");
@@ -299,11 +290,10 @@ async fn copy_object_allows_new_version_for_locked_destination_but_blocks_explic
.build()
.unwrap();
let copy_output = Box::pin(usecase.execute_copy_object(build_request(copy_input, Method::PUT)))
let copy_output = Box::pin(copy_owned(&usecase, build_request(copy_input, Method::PUT)))
.await
.expect("CopyObject should create a new version over a locked current version");
let copied_version_id = copy_output
.output
.version_id
.expect("versioned CopyObject should return the created version ID");
@@ -323,7 +313,7 @@ async fn copy_object_allows_new_version_for_locked_destination_but_blocks_explic
.build()
.unwrap();
let err = Box::pin(usecase.execute_copy_object(build_request(explicit_overwrite_input, Method::PUT)))
let err = Box::pin(copy_owned(&usecase, build_request(explicit_overwrite_input, Method::PUT)))
.await
.expect_err("explicit CopyObject overwrite of a locked version should be blocked");
@@ -379,13 +369,13 @@ async fn concurrent_reverse_copy_object_does_not_deadlock_with_reader_locks() {
let a_to_b = tokio::spawn(async move {
start_a.wait().await;
let usecase = DefaultObjectUsecase::from_global();
Box::pin(usecase.execute_copy_object(build_request(a_to_b_input, Method::PUT))).await
Box::pin(copy_owned(&usecase, build_request(a_to_b_input, Method::PUT))).await
});
let start_b = start.clone();
let b_to_a = tokio::spawn(async move {
start_b.wait().await;
let usecase = DefaultObjectUsecase::from_global();
Box::pin(usecase.execute_copy_object(build_request(b_to_a_input, Method::PUT))).await
Box::pin(copy_owned(&usecase, build_request(b_to_a_input, Method::PUT))).await
});
start.wait().await;
@@ -422,7 +412,6 @@ async fn concurrent_reverse_copy_object_does_not_deadlock_with_reader_locks() {
#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-1)"]
async fn put_and_copy_object_transition_immediately_via_usecases() {
let (_disk_paths, ecstore) = setup_test_env().await;
let fs = FS::new();
let usecase = DefaultObjectUsecase::from_global();
let tier_name = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase();
@@ -445,7 +434,7 @@ async fn put_and_copy_object_transition_immediately_via_usecases() {
.build()
.unwrap();
Box::pin(test_seams::put_object(&usecase, &fs, build_request(put_input, Method::PUT)))
Box::pin(put_through_seam(&usecase, build_request(put_input, Method::PUT)))
.await
.expect("Failed to put object through usecase");
@@ -481,7 +470,7 @@ async fn put_and_copy_object_transition_immediately_via_usecases() {
.build()
.unwrap();
Box::pin(usecase.execute_copy_object(build_request(copy_input, Method::PUT)))
Box::pin(copy_owned(&usecase, build_request(copy_input, Method::PUT)))
.await
.expect("Failed to copy object through usecase");
@@ -529,7 +518,7 @@ async fn compensation_driven_copy_still_completes_transition() {
.unwrap();
with_forced_immediate_enqueue_timeout(&backend, || async {
Box::pin(usecase.execute_copy_object(build_request(copy_input, Method::PUT)))
Box::pin(copy_owned(&usecase, build_request(copy_input, Method::PUT)))
.await
.expect("Failed to copy object through usecase");
})
@@ -616,7 +605,7 @@ async fn copy_object_computes_replication_decision_and_strips_stale_status() {
.key(dst_object.to_string())
.build()
.unwrap();
Box::pin(usecase.execute_copy_object(build_request(copy_input, Method::PUT)))
Box::pin(copy_owned(&usecase, build_request(copy_input, Method::PUT)))
.await
.expect("Failed to copy object through usecase");
@@ -652,3 +641,13 @@ async fn copy_object_computes_replication_decision_and_strips_stale_status() {
"destination must not inherit the surfaced x-amz-replication-status header key"
);
}
/// CopyObject through the owned entry point the gateway handler calls, its error in the s3s
/// form these tests compare.
async fn copy_owned(
usecase: &DefaultObjectUsecase,
req: S3Request<CopyObjectInput>,
) -> crate::app::storage_api::s3::S3Result<crate::app::object_usecase::CopyObjectWriteOutput> {
let (envelope, input) = crate::app::object_usecase::CopyObjectWriteInput::from_test_request(req);
usecase.execute_copy_object_owned(envelope, input).await.map_err(Into::into)
}
@@ -43,7 +43,6 @@ async fn wait_for_object_absence(ecstore: &Arc<ECStore>, bucket: &str, object: &
#[ignore = "global-state object-version integration test: runs serialized in the CI ILM Integration (serial) lane"]
async fn undo_delete_only_removes_the_current_matching_delete_marker() {
let (_disk_paths, ecstore) = setup_test_env().await;
let fs = FS::new();
let usecase = DefaultObjectUsecase::from_global();
let bucket = format!("test-undo-delete-{}", &Uuid::new_v4().simple().to_string()[..8]);
let object = "test/object.txt";
@@ -56,7 +55,7 @@ async fn undo_delete_only_removes_the_current_matching_delete_marker() {
.content_length(Some(13))
.build()
.unwrap();
Box::pin(test_seams::put_object(&usecase, &fs, build_request(put, Method::PUT)))
Box::pin(put_through_seam(&usecase, build_request(put, Method::PUT)))
.await
.expect("object version should be written");
let marker = Box::pin(
@@ -22,7 +22,6 @@ use super::*;
#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-1)"]
async fn put_object_if_none_match_existing_object_returns_precondition_failed() {
let (_disk_paths, ecstore) = setup_test_env().await;
let fs = FS::new();
let usecase = DefaultObjectUsecase::from_global();
let bucket = format!("test-put-if-none-match-{}", &Uuid::new_v4().simple().to_string()[..8]);
@@ -39,7 +38,7 @@ async fn put_object_if_none_match_existing_object_returns_precondition_failed()
.content_length(Some(initial_payload.len() as i64))
.build()
.unwrap();
Box::pin(usecase.execute_put_object(&fs, build_request(initial_input, Method::PUT)))
Box::pin(put_owned(&usecase, build_request(initial_input, Method::PUT)))
.await
.expect("Failed to upload initial object through usecase");
@@ -73,7 +72,7 @@ async fn put_object_if_none_match_existing_object_returns_precondition_failed()
let mut req = build_request(replacement_input, Method::PUT);
req.headers.insert(IF_NONE_MATCH, header.clone());
let err = Box::pin(usecase.execute_put_object(&fs, req))
let err = Box::pin(put_owned(&usecase, req))
.await
.expect_err("conditional PUT must reject the replacement");
assert_eq!(err.code(), &expected_code, "header={header:?}, has_dto={has_dto}");
@@ -108,7 +107,6 @@ async fn put_object_computes_replication_decision_exactly_once() {
use std::sync::atomic::Ordering;
let (_disk_paths, ecstore) = setup_test_env().await;
let fs = FS::new();
let usecase = DefaultObjectUsecase::from_global();
let bucket = format!("test-repl-decision-once-{}", &Uuid::new_v4().simple().to_string()[..8]);
@@ -129,7 +127,7 @@ async fn put_object_computes_replication_decision_exactly_once() {
.content_length(Some(payload.len() as i64))
.build()
.unwrap();
Box::pin(usecase.execute_put_object(&fs, build_request(input, Method::PUT)))
Box::pin(put_owned(&usecase, build_request(input, Method::PUT)))
.await
.expect("Failed to upload object through usecase");
@@ -207,7 +205,7 @@ async fn put_object_extract_computes_replication_decision_per_entry() {
.unwrap();
let mut req = build_request(input, Method::PUT);
req.extensions.insert(ReqInfo::default());
Box::pin(usecase.execute_put_object_extract(req))
Box::pin(put_extract_owned(&usecase, req))
.await
.expect("Failed to extract archive through usecase");
@@ -219,3 +217,26 @@ async fn put_object_extract_computes_replication_decision_per_entry() {
PutObjectExtract parity)"
);
}
/// PutObject through the owned entry point the gateway handler calls, its error in the s3s form
/// these tests compare.
async fn put_owned(
usecase: &DefaultObjectUsecase,
req: S3Request<PutObjectInput>,
) -> s3s::S3Result<crate::app::object_usecase::PutObjectWriteOutput> {
let (envelope, input) = crate::app::object_usecase::PutObjectWriteInput::from_test_request(req);
Box::pin(usecase.execute_put_object_owned(envelope, input))
.await
.map_err(Into::into)
}
/// As [`put_owned`], through the archive-extraction entry point.
async fn put_extract_owned(
usecase: &DefaultObjectUsecase,
req: S3Request<PutObjectInput>,
) -> s3s::S3Result<crate::app::object_usecase::PutObjectWriteOutput> {
let (envelope, input) = crate::app::object_usecase::PutObjectWriteInput::from_test_request(req);
Box::pin(usecase.execute_put_object_extract_owned(envelope, input))
.await
.map_err(Into::into)
}
+70 -53
View File
@@ -931,6 +931,16 @@ impl DefaultObjectUsecase {
#[cfg(test)]
mod tests {
use super::*;
/// CopyObject through the owned entry point the gateway handler calls, its error in the s3s
/// form these tests compare.
async fn copy_owned(
usecase: &DefaultObjectUsecase,
req: S3Request<CopyObjectInput>,
) -> crate::app::storage_api::s3::S3Result<crate::app::object_usecase::CopyObjectWriteOutput> {
let (envelope, input) = crate::app::object_usecase::CopyObjectWriteInput::from_test_request(req);
usecase.execute_copy_object_owned(envelope, input).await.map_err(Into::into)
}
use crate::app::storage_api::s3::S3ErrorCode;
use crate::app::storage_api::s3::dto::ServerSideEncryption;
use http::{HeaderValue, Method};
@@ -1061,7 +1071,7 @@ mod tests {
let req = build_request(input, Method::PUT);
let usecase = DefaultObjectUsecase::without_context();
let err = Box::pin(usecase.execute_copy_object(req)).await.unwrap_err();
let err = Box::pin(copy_owned(&usecase, req)).await.unwrap_err();
assert_eq!(err.code(), &S3ErrorCode::InvalidRequest);
}
@@ -1077,8 +1087,7 @@ mod tests {
.key(".rustfs-table/metadata/current.json".to_string())
.build()
.unwrap();
let error = DefaultObjectUsecase::without_context()
.execute_copy_object(build_request(input, Method::PUT))
let error = copy_owned(&DefaultObjectUsecase::without_context(), build_request(input, Method::PUT))
.await
.expect_err("reserved catalog destination must fail closed without bucket metadata");
assert_eq!(error.code(), &S3ErrorCode::InvalidRequest);
@@ -1102,7 +1111,7 @@ mod tests {
let req = build_request(input, Method::PUT);
let usecase = DefaultObjectUsecase::without_context();
let err = Box::pin(usecase.execute_copy_object(req)).await.unwrap_err();
let err = Box::pin(copy_owned(&usecase, req)).await.unwrap_err();
assert_eq!(err.code(), &S3ErrorCode::InvalidStorageClass);
}
@@ -1123,7 +1132,7 @@ mod tests {
let req = build_request(input, Method::PUT);
let usecase = DefaultObjectUsecase::without_context();
let err = Box::pin(usecase.execute_copy_object(req)).await.unwrap_err();
let err = Box::pin(copy_owned(&usecase, req)).await.unwrap_err();
// Self-copy with explicit storage class change must pass the self-copy guard.
assert_ne!(err.code(), &S3ErrorCode::InvalidRequest);
}
@@ -1132,7 +1141,9 @@ mod tests {
#[serial_test::serial]
async fn execute_self_copy_when_object_name_equals_bucket_observes_lock_order() {
use crate::app::storage_api::test::contract::bucket::{BucketOperations as _, DeleteBucketOptions, MakeBucketOptions};
use s3s::access::S3Access as _;
use crate::storage_api::server::http::access::access_owned::object_commit::{
CopyObjectAuthorization, CopySourceObject, authorize_copy_object,
};
let store = crate::app::gating_test_env::shared_gating_ecstore().await;
if current_app_context().is_none() {
@@ -1142,7 +1153,6 @@ mod tests {
let context = Arc::new(AppContext::new(Arc::clone(&store), ambient.iam(), ambient.kms()));
let server_ctx = crate::app::runtime_sources::ServerContextSlot::new();
assert!(server_ctx.install(Arc::clone(&context)));
let fs = FS::with_server_ctx(server_ctx);
let bucket = format!("self-copy-lock-order-{}", Uuid::new_v4());
store
@@ -1187,18 +1197,33 @@ mod tests {
.expect("self-copy input should build");
let mut req = build_request(input, Method::PUT);
req.extensions.insert(crate::storage::req_info::ReqInfo::default());
fs.copy_object(&mut req)
let (mut envelope, input) = CopyObjectWriteInput::from_test_request(req);
let facts = CopyObjectAuthorization {
bucket: &bucket,
key: &bucket,
source: Some(CopySourceObject {
bucket: &bucket,
key: &bucket,
version_id: None,
}),
query_version: None,
tagging: None,
tagging_directive: None,
retain_until: None,
legal_hold: false,
retention: false,
};
authorize_copy_object(&mut envelope, &server_ctx, facts)
.await
.expect("authorize self-copy whose object key equals its bucket");
let response = tokio::time::timeout(
std::time::Duration::from_secs(30),
DefaultObjectUsecase::with_context(Some(context)).execute_copy_object(req),
)
.await
.expect("lifecycle, authority, and exact object locks must not deadlock")
.expect("self-copy whose object key equals its bucket should succeed");
assert!(response.output.copy_object_result.is_some());
let usecase = DefaultObjectUsecase::with_context(Some(context));
let response =
tokio::time::timeout(std::time::Duration::from_secs(30), usecase.execute_copy_object_owned(envelope, input))
.await
.expect("lifecycle, authority, and exact object locks must not deadlock")
.expect("self-copy whose object key equals its bucket should succeed");
assert!(response.e_tag.is_some());
let info = store
.get_object_info(&bucket, &bucket, &ObjectOptions::default())
.await
@@ -1235,7 +1260,7 @@ mod tests {
let req = build_request(input, Method::PUT);
let usecase = DefaultObjectUsecase::without_context();
let err = Box::pin(usecase.execute_copy_object(req)).await.unwrap_err();
let err = Box::pin(copy_owned(&usecase, req)).await.unwrap_err();
// Tiered self-copy with STANDARD storage class must pass all validation checks.
// The call fails at store init (no store in unit tests), not at validation.
assert_ne!(err.code(), &S3ErrorCode::InvalidRequest);
@@ -1260,7 +1285,7 @@ mod tests {
let req = build_request(input, Method::PUT);
let usecase = DefaultObjectUsecase::without_context();
let err = Box::pin(usecase.execute_copy_object(req)).await.unwrap_err();
let err = Box::pin(copy_owned(&usecase, req)).await.unwrap_err();
// Must not be rejected by the self-copy guard; it fails later at store init instead.
assert_ne!(err.code(), &S3ErrorCode::InvalidRequest);
}
@@ -1341,17 +1366,16 @@ mod tests {
[(rustfs_config::ENV_OBJECT_LOCK_OPTIMIZATION_ENABLE, Some("false"))],
tokio::time::timeout(
Duration::from_secs(10),
DefaultObjectUsecase::with_context(Some(context)).execute_copy_object(req),
copy_owned(&DefaultObjectUsecase::with_context(Some(context)), req),
),
)
.await
.expect("nonbuffered source reader must not deadlock behind the self-copy write lock")
.expect("expected-current historical self-copy should succeed");
assert!(response.output.copy_object_result.is_some());
assert!(response.e_tag.is_some());
let source_version_id_text = source_version_id.to_string();
assert_eq!(response.output.copy_source_version_id.as_deref(), Some(source_version_id_text.as_str()));
assert_eq!(response.copy_source_version_id.as_deref(), Some(source_version_id_text.as_str()));
let destination_version_id = response
.output
.version_id
.clone()
.expect("versioned self-copy should return its destination version");
@@ -1437,13 +1461,12 @@ mod tests {
.key(destination.to_string())
.build()
.expect("directory copy input should build");
let response = DefaultObjectUsecase::with_context(Some(context))
.execute_copy_object(build_request(input, Method::PUT))
let response = copy_owned(&DefaultObjectUsecase::with_context(Some(context)), build_request(input, Method::PUT))
.await
.expect("copying to a directory marker should succeed");
assert_eq!(
response.output.version_id.as_deref(),
response.version_id.as_deref(),
Some(NULL_VERSION_ID),
"CopyObject must expose the null destination version without leaking the internal nil UUID"
);
@@ -1477,7 +1500,7 @@ mod tests {
let req = build_request(input, Method::PUT);
let usecase = DefaultObjectUsecase::without_context();
let err = Box::pin(usecase.execute_copy_object(req)).await.unwrap_err();
let err = Box::pin(copy_owned(&usecase, req)).await.unwrap_err();
assert_ne!(err.code(), &S3ErrorCode::InvalidRequest);
}
@@ -1498,7 +1521,7 @@ mod tests {
let req = build_request(input, Method::PUT);
let usecase = DefaultObjectUsecase::without_context();
let err = Box::pin(usecase.execute_copy_object(req)).await.unwrap_err();
let err = Box::pin(copy_owned(&usecase, req)).await.unwrap_err();
assert_eq!(err.code(), &S3ErrorCode::InvalidArgument);
}
@@ -1520,7 +1543,7 @@ mod tests {
HeaderValue::from_str(&Uuid::new_v4().to_string()).unwrap(),
);
let err = Box::pin(DefaultObjectUsecase::without_context().execute_copy_object(req))
let err = Box::pin(copy_owned(&DefaultObjectUsecase::without_context(), req))
.await
.unwrap_err();
assert_eq!(err.code(), &S3ErrorCode::InvalidRequest);
@@ -1529,9 +1552,8 @@ mod tests {
#[tokio::test]
#[serial_test::serial]
async fn metadata_only_copy_escapes_legacy_etags_in_xml() {
use crate::app::storage_api::test::ReqInfo;
use crate::app::storage_api::test::contract::bucket::{BucketOperations as _, MakeBucketOptions};
use s3s::xml::{SerializeContent, Serializer};
use http_body_util::BodyExt as _;
let (store, context) = real_store_test_context().await;
let bucket = format!("copy-legacy-etag-{}", Uuid::new_v4());
let key = "owned.txt";
@@ -1558,30 +1580,25 @@ mod tests {
.await
.expect("legacy object");
assert_eq!(before.etag.as_deref(), Some(etag));
let input = CopyObjectInput::builder()
.bucket(bucket.clone())
.key(key.to_string())
.copy_source(CopySource::Bucket {
bucket: bucket.clone().into(),
key: key.into(),
version_id: None,
})
.metadata_directive(Some(s3s::dto::MetadataDirective::from_static("REPLACE")))
.build()
.expect("self-copy request");
let mut request = build_request(input, Method::PUT);
request.extensions.insert(ReqInfo::default());
let result = Box::pin(DefaultObjectUsecase::with_context(Some(context)).execute_copy_object(request))
// The use case hands the stored ETag on verbatim; the gateway's encoder escapes it.
let slot = crate::app::runtime_sources::ServerContextSlot::new();
assert!(slot.install(Arc::clone(&context)));
let gateway = crate::server::TestGateway::over(&context, slot);
let response = gateway
.signed(
Method::PUT,
&format!("/{bucket}/{key}"),
vec![
("x-amz-copy-source", format!("/{bucket}/{key}")),
("x-amz-metadata-directive", "REPLACE".to_owned()),
],
Bytes::new(),
)
.await
.expect("metadata-only self-copy");
let mut bytes = Vec::new();
result
.output
.copy_object_result
.expect("copy result")
.serialize_content(&mut Serializer::new(&mut bytes))
.expect("serialize copy result");
let xml = String::from_utf8(bytes).expect("XML UTF-8");
assert_eq!(response.status(), http::StatusCode::OK);
let body = response.into_body().collect().await.expect("copy result body").to_bytes();
let xml = String::from_utf8(body.to_vec()).expect("XML UTF-8");
assert!(!xml.contains("<Injected>"), "{xml}");
assert!(xml.contains("&lt;Injected&gt;victim&lt;/Injected&gt;"), "{xml}");
let stored = store
@@ -1623,7 +1640,7 @@ mod tests {
.expect("copy input must build");
let usecase = DefaultObjectUsecase::with_context(Some(Arc::clone(&context)));
let err = Box::pin(usecase.execute_copy_object(build_request(input, Method::PUT)))
let err = Box::pin(copy_owned(&usecase, build_request(input, Method::PUT)))
.await
.expect_err("an unreadable bucket encryption configuration must refuse the copy");
+12
View File
@@ -70,6 +70,7 @@ pub(crate) struct CopyObjectWriteInput {
pub(crate) checksum_algorithm: Option<String>,
}
#[derive(Debug)]
pub(crate) struct CopyObjectWriteOutput {
pub(crate) e_tag: Option<String>,
pub(crate) last_modified: Option<OffsetDateTime>,
@@ -91,3 +92,14 @@ pub(crate) struct CopyObjectWriteOutput {
pub(crate) checksum_xxhash128: Option<String>,
pub(crate) checksum_type: Option<String>,
}
#[cfg(test)]
impl CopyObjectWriteInput {
/// A test's s3s fixture request as the envelope and input a gateway handler hands the use case.
pub(crate) fn from_test_request(
req: crate::app::storage_api::s3::S3Request<crate::app::storage_api::s3::dto::CopyObjectInput>,
) -> (crate::app::RequestEnvelope, Self) {
let envelope = crate::app::RequestEnvelope::from_s3s(&req);
(envelope, crate::storage::ecfs::copy_input_from_legacy(req.input))
}
}
+25 -31
View File
@@ -2082,7 +2082,6 @@ mod tests {
#[serial_test::serial]
fn repeated_versioned_delete_returns_204_with_legacy_residue() {
crate::app::gating_test_env::run_large_stack_test("repeated-versioned-delete-http", || async {
use crate::app::storage_api::s3::{Body, S3ServiceBuilder};
use crate::app::storage_api::test::contract::bucket::{BucketOperations as _, MakeBucketOptions};
use http_body_util::BodyExt;
@@ -2090,8 +2089,8 @@ mod tests {
let ambient = crate::app::gating_test_env::shared_gating_ambient().await;
let context = Arc::new(AppContext::new(Arc::clone(&store), ambient.iam(), ambient.kms()));
let slot = crate::app::runtime_sources::ServerContextSlot::new();
assert!(slot.install(context));
let service = S3ServiceBuilder::new(FS::with_server_ctx(slot)).build();
assert!(slot.install(Arc::clone(&context)));
let gateway = crate::server::TestGateway::over(&context, slot);
let bucket = format!("repeated-delete-http-{}", Uuid::new_v4());
let key = "history";
let payload = b"historical version behind repeated delete markers";
@@ -2117,26 +2116,14 @@ mod tests {
)
.await
.expect("write historical version");
// Exercise HTTP decoding and serialization with the production FS
// dispatcher. Authentication is supplied by the owner fixture.
let request = |method: Method, uri: String| {
http::Request::builder()
.method(method)
.uri(uri)
.header("host", "localhost")
.extension(crate::storage::req_info::ReqInfo {
cred: Some(rustfs_credentials::Credentials::default()),
is_owner: true,
..Default::default()
})
.body(Body::empty())
.expect("HTTP request")
};
// HTTP decoding and serialization through the gateway stack, signed by its root.
let gateway = &gateway;
let request =
|method: Method, uri: String| async move { gateway.signed(method, &uri, Vec::new(), Bytes::new()).await };
let mut marker_ids = Vec::new();
let mut backup = None;
for attempt in 0..2 {
let response = service
.call(request(Method::DELETE, format!("/{bucket}/{key}")))
let response = request(Method::DELETE, format!("/{bucket}/{key}"))
.await
.expect("DELETE HTTP response");
assert_eq!(response.status(), StatusCode::NO_CONTENT);
@@ -2172,13 +2159,12 @@ mod tests {
}
let (path, bytes) = backup.expect("legacy backup fixture");
assert_eq!(tokio::fs::read(path).await.expect("legacy backup remains"), bytes);
let response = service
.call(request(
Method::GET,
format!("/{bucket}/{key}?versionId={}", original.version_id.expect("historical version ID")),
))
.await
.expect("historical GET HTTP response");
let response = request(
Method::GET,
format!("/{bucket}/{key}?versionId={}", original.version_id.expect("historical version ID")),
)
.await
.expect("historical GET HTTP response");
assert_eq!(response.status(), StatusCode::OK);
assert_eq!(
BodyExt::collect(response.into_body())
@@ -3621,9 +3607,8 @@ mod tests {
}
#[test]
fn invalid_legacy_delete_key_still_records_an_audit() {
fn invalid_delete_key_still_records_an_audit() {
crate::app::gating_test_env::run_large_stack_test("invalid-delete-audit", || async {
use s3s::S3 as _;
let bucket = format!("invalid-delete-audit-{}", Uuid::new_v4());
let probe = crate::app::storage_api::object_usecase::helper::MutationCompletionProbe::install(&bucket);
let input = DeleteObjectInput::builder()
@@ -3633,8 +3618,17 @@ mod tests {
.expect("delete input");
let mut request = build_request(input, Method::DELETE);
request.uri = format!("/{bucket}/key").parse().expect("URI");
let error = FS::new().delete_object(request).await.expect_err("invalid key");
assert_eq!(error.code(), &S3ErrorCode::InvalidArgument);
let envelope = RequestEnvelope::from_s3s(&request);
let input = request.input;
let error = Box::pin(DefaultObjectUsecase::from_global().execute_delete_object_fields(
envelope,
input.bucket,
input.key,
input.version_id,
))
.await
.expect_err("invalid key");
assert_eq!(error.code(), &rustfs_s3_types::S3ErrorCode::InvalidArgument);
let audits = probe.audits.lock().expect("audits");
assert_eq!(audits.len(), 1);
assert_eq!(audits[0].api.status.as_deref(), Some("failure"));
+15 -7
View File
@@ -2832,6 +2832,15 @@ pub(super) mod post_archive_preflight;
#[cfg(test)]
mod tests {
use super::*;
/// Archive extraction through the owned entry point the gateway handler calls, its error in
/// the s3s form these tests compare.
async fn extract_owned(usecase: &DefaultObjectUsecase, req: S3Request<PutObjectInput>) -> S3Result<PutObjectWriteOutput> {
let (envelope, input) = PutObjectWriteInput::from_test_request(req);
Box::pin(usecase.execute_put_object_extract_owned(envelope, input))
.await
.map_err(S3Error::from)
}
use crate::app::storage_api::s3::StdError;
use crate::app::storage_api::s3::dto::{ObjectLockRetentionMode, StreamingBlob};
use http::{HeaderMap, HeaderName, HeaderValue};
@@ -3712,8 +3721,7 @@ mod tests {
request
.headers
.insert("x-amz-meta-snowball-auto-extract", HeaderValue::from_static("true"));
let error = DefaultObjectUsecase::with_context(Some(context))
.execute_put_object_extract(request)
let error = extract_owned(&DefaultObjectUsecase::with_context(Some(context)), request)
.await
.expect_err("a bad payload digest must reject the archive request");
assert_eq!(error.code().as_str(), "XAmzContentSHA256Mismatch");
@@ -3796,7 +3804,7 @@ mod tests {
.expect("extract request");
let mut request = build_request(input, Method::PUT);
request.extensions.insert(ReqInfo::default());
let result = Box::pin(usecase.execute_put_object_extract(request)).await;
let result = Box::pin(extract_owned(&usecase, request)).await;
if explicit_version && !replicate {
assert_eq!(
result
@@ -3900,7 +3908,7 @@ mod tests {
.expect("allowed member exists");
assert_eq!(info.size, 1);
} else {
assert_eq!(result.err().expect("IAM must deny the member").code().as_str(), "AccessDenied");
assert_eq!(result.expect_err("IAM must deny the member").code().as_str(), "AccessDenied");
if member == "denied-existing" {
let after = store
.get_object_info(&bucket, member, &options)
@@ -4010,7 +4018,7 @@ mod tests {
result.expect("unchanged public member is allowed");
assert_eq!(after.size, 1);
} else {
assert_eq!(result.err().expect("private member must be denied").code().as_str(), "AccessDenied");
assert_eq!(result.expect_err("private member must be denied").code().as_str(), "AccessDenied");
assert_eq!(after.etag, before.etag, "denied member must not overwrite");
assert_eq!(after.user_tags.as_str(), "classification=private");
}
@@ -4089,7 +4097,7 @@ mod tests {
result.expect("private member is authorized");
assert_eq!(stored.expect("private member stored").user_tags.as_str(), "classification=private");
} else {
assert_eq!(result.err().expect("public member must be denied").code().as_str(), "AccessDenied");
assert_eq!(result.expect_err("public member must be denied").code().as_str(), "AccessDenied");
assert!(is_err_object_not_found(&stored.expect_err("denied member must not be stored")));
}
}
@@ -4190,7 +4198,7 @@ mod tests {
}
} else {
assert_eq!(
result.err().expect("member privilege required").code().as_str(),
result.expect_err("member privilege required").code().as_str(),
"AccessDenied",
"{action}: {permission}"
);
+7 -3
View File
@@ -482,9 +482,13 @@ async fn ssec_wire_contract(usecase: &DefaultObjectUsecase, server: &crate::embe
.sse_customer_key_md5(Some(digest.clone()))
.build()
.unwrap();
crate::app::test_seams::put_object(usecase, &FS::new(), build_request(input, Method::PUT))
.await
.unwrap();
{
let (envelope, input) =
crate::app::object_usecase::PutObjectWriteInput::from_test_request(build_request(input, Method::PUT));
crate::app::test_seams::put_object(usecase, envelope, input)
}
.await
.unwrap();
for (case, valid, wrong_key, condition, range) in [
("ordinary", true, false, false, false),
("missing", false, false, false, false),
+8 -7
View File
@@ -1246,15 +1246,13 @@ mod tests {
crate::app::gating_test_env::run_large_stack_test("head-metadata-permissions", || async {
use crate::app::runtime_sources::{AppContext, ServerContextSlot};
use crate::app::storage_api::test::ReqInfo;
use crate::app::storage_api::test::S3Access as _;
use crate::app::storage_api::test::authorize_head_object;
use crate::app::storage_api::test::contract::bucket::{BucketOperations as _, MakeBucketOptions};
use crate::app::storage_api::test::ecfs::FS;
let ambient = crate::app::gating_test_env::shared_gating_ambient().await;
let store = crate::app::gating_test_env::shared_gating_ecstore().await;
let context = Arc::new(AppContext::new(store.clone(), ambient.iam(), ambient.kms()));
let slot = ServerContextSlot::new();
assert!(slot.install(context.clone()));
let fs = FS::with_server_ctx(slot.clone());
let usecase = DefaultObjectUsecase::with_context(Some(context));
let bucket = format!("head-metadata-{}", Uuid::new_v4().simple());
store
@@ -1310,7 +1308,7 @@ mod tests {
.unwrap();
let version_id = object.version_id.map(|version| version.to_string());
for (version, tagging_directive) in [(None, false), (None, true), (version_id, true)] {
let mut req = head_input(&bucket, "protected", version);
let mut req = head_input(&bucket, "protected", version.clone());
if tagging_directive {
req.headers
.insert(rustfs_utils::http::AMZ_TAGGING_DIRECTIVE_LOWER, HeaderValue::from_static("ACCESS"));
@@ -1321,11 +1319,14 @@ mod tests {
req.headers
.insert("x-rustfs-source-replication-check", HeaderValue::from_static("true"));
}
fs.head_object(&mut req).await.expect("ordinary HEAD read is permitted");
let response = Box::pin(crate::app::test_seams::head_object(&usecase, req))
let mut envelope = RequestEnvelope::from_s3s(&req);
authorize_head_object(&slot, &mut envelope, &bucket, "protected", version.as_deref())
.await
.expect("ordinary HEAD read is permitted");
let response = Box::pin(usecase.execute_head_object_fields(envelope, HeadObjectFields::from_s3s(req.input)))
.await
.expect("HEAD succeeds");
assert_eq!(response.output.content_length, Some(4));
assert_eq!(response.output.dto.content_length, Some(4));
for header in [
AMZ_TAG_COUNT,
AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER,
+70 -76
View File
@@ -2605,6 +2605,23 @@ mod tests {
use http::{HeaderMap, HeaderName, HeaderValue, Method};
use rustfs_s3_types::S3ErrorCode as OwnedS3ErrorCode;
/// PutObject through the owned entry point the gateway handler calls, its error in the s3s
/// form these tests compare.
async fn put_owned(usecase: &DefaultObjectUsecase, req: S3Request<PutObjectInput>) -> S3Result<PutObjectWriteOutput> {
let (envelope, input) = PutObjectWriteInput::from_test_request(req);
Box::pin(usecase.execute_put_object_owned(envelope, input))
.await
.map_err(S3Error::from)
}
/// As [`put_owned`], through the archive-extraction entry point.
async fn put_extract_owned(usecase: &DefaultObjectUsecase, req: S3Request<PutObjectInput>) -> S3Result<PutObjectWriteOutput> {
let (envelope, input) = PutObjectWriteInput::from_test_request(req);
Box::pin(usecase.execute_put_object_extract_owned(envelope, input))
.await
.map_err(S3Error::from)
}
#[tokio::test]
#[serial_test::serial]
async fn s3_put_uses_tail_drained_completion_before_acknowledgement() {
@@ -2626,10 +2643,9 @@ mod tests {
.build()
.expect("PUT completion request");
let usecase = DefaultObjectUsecase::with_context(Some(context));
let fs = FS::new();
let (response, observed) = PUT_STORAGE_TEST_COMPLETION
.scope(std::cell::Cell::new(None), async {
let response = Box::pin(usecase.execute_put_object(&fs, build_request(input, Method::PUT))).await;
let response = Box::pin(put_owned(&usecase, build_request(input, Method::PUT))).await;
(response, PUT_STORAGE_TEST_COMPLETION.with(std::cell::Cell::get))
})
.await;
@@ -2644,7 +2660,9 @@ mod tests {
#[test]
#[serial_test::serial]
fn put_edge_counts_polled_requests_once_before_usecase_refusal() {
/// The PutObject use cases count nothing: the gateway's dispatch counts each request once before
/// the use case runs (`GatewayBackend::serve`), so a count here would count it twice.
fn owned_put_entry_points_never_count_the_operation() {
use crate::app::storage_api::test::contract::bucket::{BucketOperations as _, MakeBucketOptions};
use metrics_util::debugging::{DebugValue, DebuggingRecorder, Snapshotter};
@@ -2690,7 +2708,6 @@ mod tests {
.block_on(Box::pin(store.make_bucket(&bucket, &MakeBucketOptions::default())))
.expect("count test bucket");
let usecase = DefaultObjectUsecase::with_context(Some(context));
let fs = FS::new();
let recorder = DebuggingRecorder::new();
let snapshotter = recorder.snapshotter();
let request = |invalid_class: bool| {
@@ -2707,39 +2724,35 @@ mod tests {
let baseline = official_count();
let mut observed = 0;
metrics::with_local_recorder(&recorder, || {
drop(usecase.execute_put_object(&fs, request(false)));
drop(put_owned(&usecase, request(false)));
assert_eq!(put_count(&snapshotter, &mut observed), 0, "constructing a future does not count a PUT");
assert_eq!(official_count() - baseline, 0, "official operation total");
let error = runtime
.block_on(Box::pin(usecase.execute_put_object(&fs, request(true))))
.block_on(Box::pin(put_owned(&usecase, request(true))))
.expect_err("invalid class");
assert_eq!(error.code(), &S3ErrorCode::InvalidStorageClass);
assert_eq!(put_count(&snapshotter, &mut observed), 1, "the owned edge counts before usecase refusal");
assert_eq!(official_count() - baseline, 1, "official operation total");
assert_eq!(put_count(&snapshotter, &mut observed), 0, "the use case does not count a refused PUT");
assert_eq!(official_count() - baseline, 0, "official operation total");
runtime
.block_on(Box::pin(usecase.execute_put_object(&fs, request(false))))
.block_on(Box::pin(put_owned(&usecase, request(false))))
.expect("successful PUT");
assert_eq!(
put_count(&snapshotter, &mut observed),
2,
"the usecase must not count the accepted PUT again"
);
assert_eq!(official_count() - baseline, 2, "official operation total");
assert_eq!(put_count(&snapshotter, &mut observed), 0, "the use case does not count an accepted PUT");
assert_eq!(official_count() - baseline, 0, "official operation total");
let input = PutObjectInput {
bucket: bucket.clone(),
key: "archive.tar".to_owned(),
..Default::default()
};
let error = runtime
.block_on(Box::pin(usecase.execute_put_object_extract(build_request(input, Method::PUT))))
.block_on(Box::pin(put_extract_owned(&usecase, build_request(input, Method::PUT))))
.expect_err("missing archive body");
assert_eq!(error.code(), &S3ErrorCode::IncompleteBody);
assert_eq!(
put_count(&snapshotter, &mut observed),
3,
"the direct archive edge counts once before validation"
0,
"the archive use case does not count a refused PUT"
);
assert_eq!(official_count() - baseline, 3, "official operation total");
assert_eq!(official_count() - baseline, 0, "official operation total");
});
}
@@ -2772,13 +2785,16 @@ mod tests {
let snapshotter = recorder.snapshotter();
let mut request = S3HttpRequestGuard::new("PUT");
let error = metrics::with_local_recorder(&recorder, || {
let fs = FS::new();
let mut future = Box::pin(PUT_STORAGE_TEST_FAILURE.scope(
(
PutStorageTestStage::Commit,
StorageError::Lock(LockError::timeout("private-bucket/private-key", Duration::from_secs(5))),
),
Box::pin(usecase.execute_put_object(&fs, build_request(input, Method::PUT))),
// The gateway's dispatch names the operation before the use case runs.
Box::pin(async {
rustfs_io_metrics::record_s3_op(S3Operation::PutObject);
put_owned(&usecase, build_request(input, Method::PUT)).await
}),
));
runtime.block_on(std::future::poll_fn(|cx| {
request.in_scope(|| std::future::Future::poll(future.as_mut(), cx))
@@ -2859,7 +2875,7 @@ mod tests {
let err = PUT_STORAGE_TEST_FAILURE
.scope(
(stage, StorageError::InsufficientReadQuorum("bucket".into(), "object".into())),
usecase.execute_put_object(&FS::new(), req),
put_owned(&usecase, req),
)
.await
.expect_err("read quorum failure must refuse the upload");
@@ -2919,7 +2935,7 @@ mod tests {
.expect("capacity PUT input must build");
let request = build_request(input, Method::PUT);
let error = PUT_STORAGE_TEST_FAILURE
.scope((PutStorageTestStage::Commit, failure), usecase.execute_put_object(&FS::new(), request))
.scope((PutStorageTestStage::Commit, failure), put_owned(&usecase, request))
.await
.expect_err("capacity refusal must fail PUT");
assert_eq!(error.code(), &S3ErrorCode::ServiceUnavailable);
@@ -3273,11 +3289,7 @@ mod tests {
.build()
.expect("PUT input must build");
let put_usecase = DefaultObjectUsecase::with_context(Some(context));
let put = tokio::spawn(async move {
put_usecase
.execute_put_object(&FS::new(), build_request(put_input, Method::PUT))
.await
});
let put = tokio::spawn(async move { put_owned(&put_usecase, build_request(put_input, Method::PUT)).await });
tokio::time::timeout(Duration::from_secs(2), async {
while !object_traffic_health.snapshot().write_stalled {
tokio::task::yield_now().await;
@@ -3337,7 +3349,7 @@ mod tests {
let put_usecase = DefaultObjectUsecase::with_context(Some(Arc::clone(&context)));
let mut request = build_request(put_input, Method::PUT);
request.uri = format!("/{bucket}/{object}").parse().expect("PUT URI");
let put = tokio::spawn(async move { put_usecase.execute_put_object(&FS::new(), request).await });
let put = tokio::spawn(async move { put_owned(&put_usecase, request).await });
tokio::time::timeout(Duration::from_secs(10), post_store_entered.wait())
.await
@@ -3380,8 +3392,7 @@ mod tests {
.expect("normal PUT");
let mut normal_request = build_request(normal_input, Method::PUT);
normal_request.uri = format!("/{bucket}/normal.bin").parse().expect("URI");
DefaultObjectUsecase::with_context(Some(context))
.execute_put_object(&FS::new(), normal_request)
put_owned(&DefaultObjectUsecase::with_context(Some(context)), normal_request)
.await
.expect("normal response completion");
let audits = probe.audits.lock().expect("audits");
@@ -3418,8 +3429,7 @@ mod tests {
let extra_body_usecase = DefaultObjectUsecase::with_context(Some(Arc::clone(&context)));
let mut extra_body_request = build_request(extra_body_input, Method::PUT);
extra_body_request.headers = streaming_headers(Some("0"));
let extra_body_err = extra_body_usecase
.execute_put_object(&FS::new(), extra_body_request)
let extra_body_err = put_owned(&extra_body_usecase, extra_body_request)
.await
.expect_err("decoded zero-byte PUT with body data must fail");
assert_eq!(extra_body_err.code(), &S3ErrorCode::UnexpectedContent);
@@ -3455,7 +3465,7 @@ mod tests {
let zero_usecase = DefaultObjectUsecase::with_context(Some(Arc::clone(&context)));
let mut zero_request = build_request(zero_input, Method::PUT);
zero_request.headers = streaming_headers(Some("0"));
let zero_put = tokio::spawn(async move { zero_usecase.execute_put_object(&FS::new(), zero_request).await });
let zero_put = tokio::spawn(async move { put_owned(&zero_usecase, zero_request).await });
tokio::time::timeout(Duration::from_secs(30), body_polled_rx)
.await
@@ -3507,11 +3517,8 @@ mod tests {
.build()
.expect("zero-copy PUT input must build");
let zero_copy_usecase = DefaultObjectUsecase::with_context(Some(context));
let zero_copy_put = tokio::spawn(async move {
zero_copy_usecase
.execute_put_object(&FS::new(), build_request(zero_copy_input, Method::PUT))
.await
});
let zero_copy_put =
tokio::spawn(async move { put_owned(&zero_copy_usecase, build_request(zero_copy_input, Method::PUT)).await });
tokio::time::timeout(Duration::from_secs(30), async {
while !object_traffic_health.snapshot().write_stalled {
@@ -3973,7 +3980,7 @@ mod tests {
let mut request = build_request(input, Method::PUT);
request.extensions.insert(control.clone());
let usecase = DefaultObjectUsecase::with_context(Some(Arc::clone(&context)));
let owner = tokio::spawn(async move { usecase.execute_put_object(&FS::new(), request).await });
let owner = tokio::spawn(async move { put_owned(&usecase, request).await });
control.wait_until_pending_for_test().await;
control.advance_wait_for_test(Duration::from_secs(60));
sender.send(Ok(Frame::data(Bytes::new()))).unwrap();
@@ -4358,7 +4365,7 @@ mod tests {
async fn post_object_metadata_cannot_dispatch_legacy_archive_extraction() {
let mut req = build_request(PutObjectInput::default(), Method::POST);
req.extensions.insert(PostObjectRequestMarker);
let err = Box::pin(DefaultObjectUsecase::without_context().execute_put_object_extract(req))
let err = Box::pin(put_extract_owned(&DefaultObjectUsecase::without_context(), req))
.await
.expect_err("a legacy form cannot authorize archive replay");
assert_eq!(err.code(), &S3ErrorCode::NotImplemented);
@@ -4377,7 +4384,7 @@ mod tests {
.insert(http::header::IF_NONE_MATCH, HeaderValue::from_static("\"different-etag\""));
let usecase = DefaultObjectUsecase::without_context();
let err = Box::pin(usecase.execute_put_object(&FS::new(), req))
let err = Box::pin(put_owned(&usecase, req))
.await
.expect_err("POST header-only ETag precondition must fail before storage access");
assert_eq!(err.code(), &S3ErrorCode::NotImplemented);
@@ -4407,7 +4414,7 @@ mod tests {
} else {
Box::pin(usecase.execute_put_object_owned(envelope, input)).await
};
let error = result.err().expect("unbound gateway archive must be refused");
let error = result.expect_err("unbound gateway archive must be refused");
assert_eq!(error.code().as_str(), "NotImplemented", "direct={direct_extract}");
assert_eq!(
polls.load(std::sync::atomic::Ordering::SeqCst),
@@ -4430,9 +4437,8 @@ mod tests {
req.extensions.insert(PostObjectRequestMarker);
let usecase = DefaultObjectUsecase::without_context();
let fs = FS::new();
let err = Box::pin(usecase.execute_put_object(&fs, req)).await.unwrap_err();
let err = Box::pin(put_owned(&usecase, req)).await.unwrap_err();
assert_eq!(err.code(), &S3ErrorCode::NotImplemented);
}
@@ -4449,9 +4455,8 @@ mod tests {
req.headers.insert(AMZ_SNOWBALL_EXTRACT, HeaderValue::from_static("true"));
let usecase = DefaultObjectUsecase::without_context();
let fs = FS::new();
let err = Box::pin(usecase.execute_put_object(&fs, req)).await.unwrap_err();
let err = Box::pin(put_owned(&usecase, req)).await.unwrap_err();
assert_eq!(err.code(), &S3ErrorCode::NotImplemented);
}
@@ -4468,9 +4473,8 @@ mod tests {
req.headers.insert(AMZ_SNOWBALL_EXTRACT, HeaderValue::from_static("true"));
let usecase = DefaultObjectUsecase::without_context();
let fs = FS::new();
let err = Box::pin(usecase.execute_put_object(&fs, req)).await.unwrap_err();
let err = Box::pin(put_owned(&usecase, req)).await.unwrap_err();
assert_eq!(err.code(), &S3ErrorCode::InvalidStorageClass);
}
@@ -4491,9 +4495,8 @@ mod tests {
let req = build_request(input, Method::PUT);
let usecase = DefaultObjectUsecase::without_context();
let fs = FS::new();
let err = Box::pin(usecase.execute_put_object(&fs, req)).await.unwrap_err();
let err = Box::pin(put_owned(&usecase, req)).await.unwrap_err();
assert_eq!(err.code(), &S3ErrorCode::EntityTooLarge);
assert_eq!(polls.load(std::sync::atomic::Ordering::SeqCst), 0, "body must not be polled");
}
@@ -4529,9 +4532,8 @@ mod tests {
HeaderValue::from_str(&decoded.to_string()).unwrap(),
);
let usecase = DefaultObjectUsecase::without_context();
let fs = FS::new();
let err = Box::pin(usecase.execute_put_object(&fs, req)).await.unwrap_err();
let err = Box::pin(put_owned(&usecase, req)).await.unwrap_err();
if expect_too_large {
assert_eq!(err.code(), &S3ErrorCode::EntityTooLarge, "decoded {decoded}");
assert_eq!(polls.load(std::sync::atomic::Ordering::SeqCst), 0, "body must not be polled");
@@ -4559,9 +4561,8 @@ mod tests {
.insert(AMZ_SERVER_SIDE_ENCRYPTION, HeaderValue::from_static("aws:kms"));
let usecase = DefaultObjectUsecase::without_context();
let fs = FS::new();
let err = Box::pin(usecase.execute_put_object(&fs, req)).await.unwrap_err();
let err = Box::pin(put_owned(&usecase, req)).await.unwrap_err();
assert_eq!(err.code(), &S3ErrorCode::NotImplemented);
}
@@ -4579,9 +4580,8 @@ mod tests {
.insert(AMZ_SERVER_SIDE_ENCRYPTION_KMS_ID, HeaderValue::from_static("test-kms-key-id"));
let usecase = DefaultObjectUsecase::without_context();
let fs = FS::new();
let err = Box::pin(usecase.execute_put_object(&fs, req)).await.unwrap_err();
let err = Box::pin(put_owned(&usecase, req)).await.unwrap_err();
assert_eq!(err.code(), &S3ErrorCode::NotImplemented);
}
@@ -4596,9 +4596,8 @@ mod tests {
let req = build_request(input, Method::PUT);
let usecase = DefaultObjectUsecase::without_context();
let fs = FS::new();
let err = Box::pin(usecase.execute_put_object(&fs, req)).await.unwrap_err();
let err = Box::pin(put_owned(&usecase, req)).await.unwrap_err();
assert_eq!(err.code(), &S3ErrorCode::InvalidStorageClass);
}
@@ -4820,13 +4819,12 @@ mod tests {
.expect("SSE-C PUT input must build");
let usecase = DefaultObjectUsecase::from_global();
let response = usecase
.execute_put_object(&FS::new(), build_request(input, Method::PUT))
let response = put_owned(&usecase, build_request(input, Method::PUT))
.await
.unwrap_or_else(|err| panic!("a {algorithm} default bucket must accept an SSE-C PUT: {err:?}"));
assert_eq!(response.output.server_side_encryption, None);
assert_eq!(response.output.sse_customer_algorithm.as_deref(), Some("AES256"));
assert_eq!(response.output.sse_customer_key_md5.as_deref(), Some(key_md5.as_str()));
assert_eq!(response.server_side_encryption, None);
assert_eq!(response.sse_customer_algorithm.as_deref(), Some("AES256"));
assert_eq!(response.sse_customer_key_md5.as_deref(), Some(key_md5.as_str()));
let encoded_key = base64_simd::STANDARD.encode_to_string(customer_key);
let mut read_headers = HeaderMap::new();
@@ -4940,8 +4938,7 @@ mod tests {
..Default::default()
});
let err = DefaultObjectUsecase::from_global()
.execute_put_object(&FS::new(), request)
let err = put_owned(&DefaultObjectUsecase::from_global(), request)
.await
.expect_err("quota-enabled ciphertext replication should fail at ingress");
assert_eq!(err.code(), &S3ErrorCode::InvalidRequest);
@@ -4991,8 +4988,7 @@ mod tests {
.build()
.expect("legacy quota PUT input should build");
let err = DefaultObjectUsecase::from_global()
.execute_put_object(&FS::new(), build_request(input, Method::PUT))
let err = put_owned(&DefaultObjectUsecase::from_global(), build_request(input, Method::PUT))
.await
.expect_err("4 GiB used plus a 2 GiB PUT must exceed a 5 GiB legacy quota");
assert_eq!(err.code(), &S3ErrorCode::InvalidRequest);
@@ -5038,8 +5034,7 @@ mod tests {
.content_length(Some(i64::try_from(payload.len()).expect("test payload length must fit i64")))
.build()
.expect("PUT input should build");
DefaultObjectUsecase::from_global()
.execute_put_object(&FS::new(), build_request(input, Method::PUT))
put_owned(&DefaultObjectUsecase::from_global(), build_request(input, Method::PUT))
.await
.expect("a ledger-admitted bucket must accept a write without scanner usage");
@@ -5051,8 +5046,7 @@ mod tests {
.content_length(Some(i64::try_from(too_large.len()).expect("test payload length must fit i64")))
.build()
.expect("PUT input should build");
let err = DefaultObjectUsecase::from_global()
.execute_put_object(&FS::new(), build_request(input, Method::PUT))
let err = put_owned(&DefaultObjectUsecase::from_global(), build_request(input, Method::PUT))
.await
.expect_err("the ledger must still reject a write above the quota limit");
assert_eq!(err.code(), &S3ErrorCode::InvalidRequest);
@@ -5455,7 +5449,7 @@ mod tests {
.expect("PUT input must build");
let usecase = DefaultObjectUsecase::with_context(Some(Arc::clone(&context)));
let err = Box::pin(usecase.execute_put_object(&FS::new(), build_request(input, Method::PUT)))
let err = Box::pin(put_owned(&usecase, build_request(input, Method::PUT)))
.await
.expect_err("an unreadable bucket encryption configuration must refuse the write");
@@ -5500,7 +5494,7 @@ mod tests {
metrics::with_local_recorder(&recorder, || {
request.in_scope(|| {
runtime.block_on(async {
Box::pin(usecase.execute_put_object(&FS::new(), build_request(input, Method::PUT)))
Box::pin(put_owned(&usecase, build_request(input, Method::PUT)))
.await
.expect("S3 PUT must commit");
assert_eq!(current_s3_http_operation_name(), Some("s3:PutObject"));
@@ -5567,7 +5561,7 @@ mod tests {
.expect("PUT input must build");
let usecase = DefaultObjectUsecase::with_context(Some(Arc::clone(&context)));
Box::pin(usecase.execute_put_object(&FS::new(), build_request(input, Method::PUT)))
Box::pin(put_owned(&usecase, build_request(input, Method::PUT)))
.await
.expect("a bucket without default encryption must still accept a plaintext write");
@@ -5613,7 +5607,7 @@ mod tests {
req.headers.insert(AMZ_SNOWBALL_EXTRACT, HeaderValue::from_static("true"));
let usecase = DefaultObjectUsecase::with_context(Some(Arc::clone(&context)));
let err = Box::pin(usecase.execute_put_object(&FS::new(), req))
let err = Box::pin(put_owned(&usecase, req))
.await
.expect_err("an unreadable bucket encryption configuration must refuse the extract upload");
+12 -1
View File
@@ -62,7 +62,7 @@ pub(crate) struct PutObjectWriteInput {
pub(crate) checksum_crc64nvme: Option<String>,
}
#[derive(Default)]
#[derive(Debug, Default)]
pub(crate) struct PutObjectWriteOutput {
pub(crate) e_tag: Option<String>,
pub(crate) version_id: Option<String>,
@@ -88,3 +88,14 @@ impl PutObjectWriteInput {
self.trailing_headers = Some(trailers);
}
}
#[cfg(test)]
impl PutObjectWriteInput {
/// A test's s3s fixture request as the envelope and input a gateway handler hands the use case.
pub(crate) fn from_test_request(
req: crate::app::storage_api::s3::S3Request<crate::app::storage_api::s3::PutObjectInput>,
) -> (crate::app::RequestEnvelope, Self) {
let envelope = crate::app::RequestEnvelope::from_s3s(&req);
(envelope, Self::from_legacy(req.input, req.trailing_headers))
}
}
+2 -2
View File
@@ -73,7 +73,7 @@ pub(crate) mod s3 {
#[cfg(test)]
pub(crate) use s3s::dto::{GetObjectInput, HeadObjectInput, ListObjectsV2Input, ListObjectsV2Output};
#[cfg(test)]
pub(crate) use s3s::dto::{GetObjectOutput, HeadObjectOutput, PutObjectInput, PutObjectOutput};
pub(crate) use s3s::dto::{GetObjectOutput, HeadObjectOutput, PutObjectInput};
#[cfg(test)]
pub(crate) use s3s::dto::{ListObjectsInput, StreamingBlob, UploadPartInput, UploadPartOutput};
pub(crate) use s3s::header;
@@ -1951,7 +1951,6 @@ pub(crate) mod context {
#[cfg(test)]
pub(crate) mod test {
pub(crate) use super::EndpointServerPools;
pub(crate) use s3s::access::S3Access;
pub(crate) mod contract {
pub(crate) mod bucket {
pub(crate) use super::super::super::storage_contracts::{
@@ -1982,6 +1981,7 @@ pub(crate) mod test {
pub(crate) use super::access::ReqInfo;
pub(crate) use super::options::VERSIONING_CONFIG_LOOKUPS;
pub(crate) use crate::storage::storage_api::test_consumer::authorize_head_object;
pub(crate) mod sse {
pub(crate) use crate::storage::storage_api::test_consumer::sse::{
RecordingKmsKeyAuthorizer, TestAuthorizationHooks, sse_kms_object_metadata,
+10 -3
View File
@@ -27,11 +27,18 @@ pub(crate) async fn get_object(
use crate::app::{Answered, RequestEnvelope};
rustfs_io_metrics::record_s3_op(rustfs_s3_ops::S3Operation::GetObject);
let envelope = RequestEnvelope::from_s3s(&req);
let Answered { output, status, headers } = usecase
.execute_get_object_fields(envelope, GetObjectFields::from_s3s(req.input))
.await?;
let Answered { output, status, headers } = get_object_fields(usecase, envelope, GetObjectFields::from_s3s(req.input)).await?;
let mut response = S3Response::new(output.into_s3s().map_err(crate::app::dto_values::seam_error)?);
response.status = status;
response.headers = headers;
Ok(response)
}
/// Calls `DefaultObjectUsecase::execute_get_object_fields` with owned inputs.
pub(crate) async fn get_object_fields(
usecase: &DefaultObjectUsecase,
envelope: crate::app::RequestEnvelope,
input: crate::app::object_usecase::GetObjectFields,
) -> rustfs_s3_types::S3Result<crate::app::Answered<crate::app::object_usecase::GetObjectOutputFacts>> {
usecase.execute_get_object_fields(envelope, input).await
}
+11 -3
View File
@@ -27,11 +27,19 @@ pub(crate) async fn head_object(
use crate::app::{Answered, RequestEnvelope};
rustfs_io_metrics::record_s3_op(rustfs_s3_ops::S3Operation::HeadObject);
let envelope = RequestEnvelope::from_s3s(&req);
let Answered { output, status, headers } = usecase
.execute_head_object_fields(envelope, HeadObjectFields::from_s3s(req.input))
.await?;
let Answered { output, status, headers } =
head_object_fields(usecase, envelope, HeadObjectFields::from_s3s(req.input)).await?;
let mut response = S3Response::new(output.into_s3s().map_err(crate::app::dto_values::seam_error)?);
response.status = status;
response.headers = headers;
Ok(response)
}
/// Calls `DefaultObjectUsecase::execute_head_object_fields` with owned inputs.
pub(crate) async fn head_object_fields(
usecase: &DefaultObjectUsecase,
envelope: crate::app::RequestEnvelope,
input: crate::app::object_usecase::HeadObjectFields,
) -> rustfs_s3_types::S3Result<crate::app::Answered<crate::app::object_usecase::HeadObjectOutputFacts>> {
usecase.execute_head_object_fields(envelope, input).await
}
+10 -1
View File
@@ -24,7 +24,16 @@ pub(crate) async fn list_objects_v2(
req: S3Request<ListObjectsV2Input>,
) -> S3Result<S3Response<ListObjectsV2Output>> {
crate::storage::ecfs::list_objects_v2_through_legacy_edge(req, |envelope, input| {
usecase.execute_list_objects_v2(envelope, input)
list_objects_v2_owned(usecase, envelope, input)
})
.await
}
/// Calls `DefaultBucketUsecase::execute_list_objects_v2` with owned inputs.
pub(crate) async fn list_objects_v2_owned(
usecase: &DefaultBucketUsecase,
envelope: crate::app::RequestEnvelope,
input: rustfs_gateway::dto::ListObjectsV2Input,
) -> rustfs_s3_types::S3Result<rustfs_gateway::dto::ListObjectsV2Output> {
usecase.execute_list_objects_v2(envelope, input).await
}
+3 -3
View File
@@ -23,7 +23,7 @@ mod head_object;
mod list_objects_v2;
mod put_object;
pub(crate) use get_object::get_object;
pub(crate) use head_object::head_object;
pub(crate) use list_objects_v2::list_objects_v2;
pub(crate) use get_object::{get_object, get_object_fields};
pub(crate) use head_object::{head_object, head_object_fields};
pub(crate) use list_objects_v2::{list_objects_v2, list_objects_v2_owned};
pub(crate) use put_object::put_object;
+9 -10
View File
@@ -12,19 +12,18 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::app::object_usecase::DefaultObjectUsecase;
use crate::app::storage_api::s3::{PutObjectInput, PutObjectOutput, S3Request, S3Response, S3Result};
use crate::app::storage_api::test::ecfs::FS;
use crate::app::RequestEnvelope;
use crate::app::object_usecase::{DefaultObjectUsecase, PutObjectWriteInput, PutObjectWriteOutput};
use std::future::Future;
/// Calls `DefaultObjectUsecase::execute_put_object` with today's arguments.
/// Calls `DefaultObjectUsecase::execute_put_object_owned`, the entry point the gateway handler calls.
///
/// Owned by the PutObject batch (D). Tests outside that batch's files call this seam instead of the use-case method;
/// when batch D changes the use-case signature it changes only this body.
pub(crate) fn put_object<'a>(
usecase: &'a DefaultObjectUsecase,
fs: &'a FS,
req: S3Request<PutObjectInput>,
) -> impl Future<Output = S3Result<S3Response<PutObjectOutput>>> + 'a {
usecase.execute_put_object(fs, req)
pub(crate) fn put_object(
usecase: &DefaultObjectUsecase,
envelope: RequestEnvelope,
input: PutObjectWriteInput,
) -> impl Future<Output = rustfs_s3_types::S3Result<PutObjectWriteOutput>> + '_ {
Box::pin(usecase.execute_put_object_owned(envelope, input))
}
+80
View File
@@ -347,6 +347,86 @@ impl GatewayStack {
}
}
/// The gateway stack over a server context, with requests signed by a root it publishes on that
/// context: for tests that drive an operation through the wire codec, authentication and
/// authorization instead of calling its use case.
#[cfg(test)]
pub(crate) struct TestGateway(GatewayStack);
#[cfg(test)]
impl TestGateway {
pub(crate) const ACCESS_KEY: &str = "gatewaytestroot";
pub(crate) const SECRET_KEY: &str = "gateway-test-root-secret";
/// Publishes the root on `context` and assembles the stack over `slot`, which holds `context`.
pub(crate) fn over(context: &crate::runtime_sources::AppContext, slot: Arc<ServerContextSlot>) -> Self {
assert!(
context.publish_action_credentials(rustfs_credentials::Credentials {
access_key: Self::ACCESS_KEY.to_owned(),
secret_key: Self::SECRET_KEY.to_owned(),
status: "on".to_owned(),
..Default::default()
}),
"the test context has no root yet"
);
let stack = GatewayStack::assemble(GatewayAssembly {
server_ctx: slot,
root_access_key: Self::ACCESS_KEY.to_owned(),
root_secret_key: Self::SECRET_KEY.to_owned(),
region: None,
server_domains: Vec::new(),
body_read_timeout: Duration::from_secs(300),
keystone_auth: None,
})
.expect("the gateway stack assembles");
Self(stack)
}
/// `method target` with `headers` and `body`, header-signed by the root over an unsigned
/// payload. A `Result`, so a call site reads as a service call does.
pub(crate) async fn signed(
&self,
method: http::Method,
target: &str,
headers: Vec<(&str, String)>,
body: bytes::Bytes,
) -> Result<Response<rustfs_gateway::Body>, std::convert::Infallible> {
use crate::server::gateway_handlers::test_support::{Signed, signed_request_with_headers};
let (path, query) = target.split_once('?').unwrap_or((target, ""));
// The signer takes the canonical spelling of a subresource, `uploads=` for `uploads`.
let query = query
.split('&')
.filter(|pair| !pair.is_empty())
.map(|pair| {
if pair.contains('=') {
pair.to_owned()
} else {
format!("{pair}=")
}
})
.collect::<Vec<_>>()
.join("&");
let mut signed_headers = http::HeaderMap::new();
// The length is the body's own, set below; the signer does not sign it.
for (name, value) in headers
.into_iter()
.filter(|(name, _)| !name.eq_ignore_ascii_case("content-length"))
{
signed_headers.append(
http::HeaderName::from_bytes(name.as_bytes()).expect("a header name"),
value.parse().expect("a header value"),
);
}
let request =
signed_request_with_headers(Signed::Header, method, path, &query, Self::ACCESS_KEY, Self::SECRET_KEY, signed_headers);
let mut request = request.map(|_| http_body_util::Full::new(body.clone()));
request
.headers_mut()
.insert(http::header::CONTENT_LENGTH, http::HeaderValue::from(body.len()));
Ok(self.0.call(request).await)
}
}
/// The early gate over `table` and the two metadata listings: an operation it serves is allowed on
/// to its handler, which decides; every other operation is denied.
fn early_gate(table: OperationTable) -> impl Authorizer {
+2
View File
@@ -66,6 +66,8 @@ pub(crate) use event::{
reconcile_event_notifier_from_store, start_persisted_event_notifier_reconciler, with_notify_runtime_reconcile_lock,
};
#[cfg(test)]
pub(crate) use gateway_stack::TestGateway;
#[cfg(test)]
pub(crate) use health::{
HealthPayloadContext, HealthReadinessSource, build_component_details, build_health_payload, health_check_state,
readiness_source_for_probe,
File diff suppressed because it is too large Load Diff
+3
View File
@@ -25,3 +25,6 @@ pub(crate) mod object_meta;
pub(crate) mod object_misc;
pub(crate) mod object_read;
pub(crate) mod object_write;
#[cfg(test)]
mod tests;
@@ -0,0 +1,655 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Tag- and condition-dependent decisions of the owned object authorization functions.
//!
//! Responsible for: the tag evidence a local miss leaves for the replication proxy, the read
//! permission a specific `If-Match` write requires, the version permission an include-deleted
//! listing requires, and the source and destination tag decisions of CopyObject, including the
//! fresh decision the copy use case makes against the source it actually reads.
//! Not responsible for: bucket-generation snapshots (see `generation_guards.rs`), or the policy
//! evaluation itself.
//! Upstream: the owned request fixture in `storage::authorization::tests::fixture`.
//! Downstream: `storage::access_owned::{object_read, object_meta, object_write, object_commit,
//! listing_multipart}` and the CopyObject use case.
use crate::app::RequestEnvelope;
use crate::app::object_usecase::{CopyObjectWriteInput, DefaultObjectUsecase};
use crate::storage::access_owned::listing_multipart::authorize_bucket_listing;
use crate::storage::access_owned::object_commit::{
CopyObjectAuthorization, CopySourceObject, authorize_complete_multipart_upload, authorize_copy_object,
};
use crate::storage::access_owned::object_meta::{authorize_object_tagging_request, authorize_put_object_tagging};
use crate::storage::access_owned::object_read::{authorize_get_object, authorize_head_object};
use crate::storage::access_owned::object_write::{PutObjectAuthorization, authorize_put_object};
use crate::storage::authorization::tests::fixture::{TestKms, UnreadyIam, request, server_context};
use crate::storage::req_info::ReqInfo;
use crate::storage::request_markers::RequestedObjectTags;
use crate::storage::storage_api::contract::bucket::{BucketOperations as _, MakeBucketOptions};
use crate::storage::storage_api::contract::object::{ObjectIO as _, ObjectOperations as _};
use crate::storage::storage_api::runtime_sources_consumer::{AppContext, IamInterface, ServerContextSlot};
use http::Method;
use rustfs_gateway_types::CopySourceValue;
use rustfs_iam::{store::object::ObjectStore, sys::IamSys};
use rustfs_policy::policy::action::S3Action;
use rustfs_s3_types::S3ErrorCode;
use serial_test::serial;
use std::{collections::HashMap, sync::Arc};
/// The copy use case input the legacy edge built from a CopyObject request that carries only
/// these fields.
fn copy_write_input(
bucket: &str,
key: &str,
copy_source: CopySourceValue,
metadata_directive: Option<&str>,
tagging: Option<&str>,
tagging_directive: Option<&str>,
) -> CopyObjectWriteInput {
CopyObjectWriteInput {
bucket: bucket.to_owned(),
key: key.to_owned(),
copy_source: Some(copy_source),
version_id: None,
specific_if_none_match: false,
server_side_encryption: None,
ssekms_key_id: None,
sse_customer_algorithm: None,
sse_customer_key: None,
sse_customer_key_md5: None,
copy_source_sse_customer_algorithm: None,
copy_source_sse_customer_key: None,
copy_source_sse_customer_key_md5: None,
metadata_directive: metadata_directive.map(str::to_owned),
metadata: None,
tagging: tagging.map(str::to_owned),
tagging_directive: tagging_directive.map(str::to_owned),
copy_source_if_match: None,
copy_source_if_none_match: None,
cache_control: None,
content_disposition: None,
content_encoding: None,
content_language: None,
content_type: None,
expires: None,
website_redirect_location: None,
object_lock_legal_hold_status: None,
object_lock_mode: None,
object_lock_retain_until_date: None,
storage_class: None,
checksum_algorithm: None,
}
}
/// CopyObject's authorization facts for a request naming only a bucket-path source, tags and
/// a tagging directive.
fn copy_facts<'a>(
bucket: &'a str,
key: &'a str,
source: CopySourceObject<'a>,
tagging: Option<&'a str>,
tagging_directive: Option<&'a str>,
) -> CopyObjectAuthorization<'a> {
CopyObjectAuthorization {
bucket,
key,
source: Some(source),
query_version: None,
tagging,
tagging_directive,
retain_until: None,
legal_hold: false,
retention: false,
}
}
#[tokio::test]
#[serial]
async fn tag_dependent_access_hooks_preserve_local_miss_for_replication_proxy() {
let store = crate::app::gating_test_env::shared_gating_ecstore().await;
let slot = server_context(Arc::clone(&store));
let bucket = format!("tag-proxy-boundary-{}", uuid::Uuid::new_v4());
store
.make_bucket(&bucket, &MakeBucketOptions::default())
.await
.expect("create bucket");
let policy_json = format!(
r#"{{"Version":"2012-10-17","Statement":[
{{"Effect":"Allow","Principal":{{"AWS":"*"}},"Action":["s3:GetObject","s3:GetObjectTagging","s3:PutObjectTagging","s3:DeleteObjectTagging"],"Resource":["arn:aws:s3:::{bucket}/*"]}},
{{"Effect":"Deny","Principal":{{"AWS":"*"}},"Action":["s3:GetObject","s3:GetObjectTagging","s3:PutObjectTagging","s3:DeleteObjectTagging"],"Resource":["arn:aws:s3:::{bucket}/*"],"Condition":{{"StringEquals":{{"s3:ExistingObjectTag/protected":"yes"}}}}}}
]}}"#
);
let policy_bytes = policy_json.into_bytes();
store
.update_bucket_metadata_config(
&bucket,
crate::storage::storage_api::ecstore_bucket::metadata::BUCKET_POLICY_CONFIG,
policy_bytes,
)
.await
.expect("publish policy");
let check = |envelope: &RequestEnvelope| {
let evidence = envelope
.object_tag_conditions()
.expect("authorization must retain tag dependency");
assert!(evidence.values.is_empty(), "missing local object has no tags");
assert!(
!crate::app::object_meta::tagging::proxy_allowed(envelope),
"local absence cannot authorize a tagged replica"
);
};
let mut envelope = request(Method::GET).req_info().server_context(&slot).build();
authorize_get_object(&slot, &mut envelope, &bucket, "missing", None)
.await
.expect("local miss does not match the tag deny");
check(&envelope);
let mut envelope = request(Method::HEAD).req_info().server_context(&slot).build();
authorize_head_object(&slot, &mut envelope, &bucket, "missing", None)
.await
.expect("local miss does not match the tag deny");
check(&envelope);
let mut envelope = request(Method::GET).req_info().server_context(&slot).build();
authorize_object_tagging_request(&mut envelope, &bucket, "missing", None, S3Action::GetObjectTaggingAction)
.await
.expect("local miss does not match the tag deny");
check(&envelope);
let mut envelope = request(Method::DELETE).req_info().server_context(&slot).build();
authorize_object_tagging_request(&mut envelope, &bucket, "missing", None, S3Action::DeleteObjectTaggingAction)
.await
.expect("local miss does not match the tag deny");
check(&envelope);
let mut envelope = request(Method::PUT).req_info().server_context(&slot).build();
authorize_put_object_tagging(&mut envelope, &bucket, "missing", None, RequestedObjectTags(Vec::new()))
.await
.expect("local miss does not match the tag deny");
check(&envelope);
}
#[tokio::test]
#[serial]
async fn conditional_writes_require_read_permission_only_for_specific_etags() {
let store = crate::app::gating_test_env::shared_gating_ecstore().await;
let slot = server_context(Arc::clone(&store));
for readable in [false, true] {
let bucket = format!("etag-oracle-{}", uuid::Uuid::new_v4());
store
.make_bucket(&bucket, &MakeBucketOptions::default())
.await
.expect("create bucket");
let read_effect = if readable { "Allow" } else { "Deny" };
let policy_json = format!(
r#"{{"Version":"2012-10-17","Statement":[
{{"Effect":"Allow","Principal":{{"AWS":"*"}},"Action":["s3:PutObject"],"Resource":["arn:aws:s3:::{bucket}/*"]}},
{{"Effect":"{read_effect}","Principal":{{"AWS":"*"}},"Action":["s3:GetObject","s3:GetObjectVersion"],"Resource":["arn:aws:s3:::{bucket}/*"]}}
]}}"#
);
let policy_bytes = policy_json.into_bytes();
store
.update_bucket_metadata_config(
&bucket,
crate::storage::storage_api::ecstore_bucket::metadata::BUCKET_POLICY_CONFIG,
policy_bytes,
)
.await
.expect("publish policy");
for condition in ["absent", "wildcard", "dto", "header"] {
let specific = matches!(condition, "dto" | "header");
// The decoded `If-Match` value is specific only for a strong ETag; the wildcard and
// the header-only case leave the decoded value non-specific.
let specific_dto_etag = condition == "dto";
let mut put = request(Method::PUT).req_info().server_context(&slot);
let mut complete = request(Method::POST).req_info().server_context(&slot);
if condition == "header" {
put = put.header("if-match", "\"guessed-etag\"");
complete = complete.header("if-match", "\"guessed-etag\"");
}
let mut put = put.build();
let mut complete = complete.build();
let put_facts = PutObjectAuthorization {
bucket: &bucket,
key: "object",
version_id: None,
retain_until: None,
specific_etag: specific_dto_etag,
legal_hold: false,
retention: false,
post: false,
};
for result in [
authorize_put_object(&mut put, &slot, put_facts, |_| Ok(())).await,
authorize_complete_multipart_upload(&mut complete, &slot, &bucket, "object", specific_dto_etag).await,
] {
if specific && !readable {
assert_eq!(
result
.expect_err("a write-only caller must not compare content fingerprints")
.code(),
&S3ErrorCode::AccessDenied
);
} else {
result.expect("readable or wildcard/ordinary writes remain authorized");
}
}
}
}
}
#[tokio::test]
#[serial]
async fn include_deleted_listing_enforces_version_permissions() {
use rustfs_iam::store::Store as _;
use rustfs_madmin::{AccountStatus, AddOrUpdateUserReq};
struct ReadyIam(Arc<IamSys<ObjectStore>>);
impl IamInterface for ReadyIam {
fn handle(&self) -> Arc<IamSys<ObjectStore>> {
self.0.clone()
}
fn is_ready(&self) -> bool {
true
}
}
let (_directory, _paths, store) = crate::app::gating_test_env::isolated_multi_pool_ecstore().await;
ObjectStore::new(store.clone())
.save_iam_config(
serde_json::json!({"version":1}),
format!("{}/format.json", *rustfs_iam::store::object::IAM_CONFIG_PREFIX),
)
.await
.expect("seed IAM format");
let iam = rustfs_iam::build_iam_sys(store.clone()).await.expect("initialize IAM");
iam.create_user(
"list-reader",
&AddOrUpdateUserReq {
secret_key: "list-reader-secret".to_string(),
policy: None,
status: AccountStatus::Enabled,
},
)
.await
.expect("create reader");
let user = iam.get_user("list-reader").await.expect("reader identity").credentials;
iam.set_policy(
"list-only",
rustfs_policy::policy::Policy::parse_config(
br#"{"Version":"2012-10-17","Statement":[
{"Effect":"Allow","Action":"s3:ListBucket","Resource":"arn:aws:s3:::*"}]}"#,
)
.expect("list policy"),
)
.await
.expect("store identity policy");
iam.policy_db_set("list-reader", rustfs_iam::store::UserType::Reg, false, "list-only")
.await
.expect("attach list policy");
let bucket = format!("deleted-list-auth-{}", uuid::Uuid::new_v4());
store
.make_bucket(&bucket, &MakeBucketOptions::default())
.await
.expect("create bucket");
let slot = ServerContextSlot::new();
assert!(slot.install(Arc::new(AppContext::new(Arc::clone(&store), Arc::new(ReadyIam(iam)), Arc::new(TestKms)))));
for (grant_versions, deny_versions) in [(false, false), (true, false), (true, true)] {
let mut statements = vec![serde_json::json!({
"Effect": "Allow", "Principal": {"AWS": "*"}, "Action": ["s3:ListBucket"], "Resource": format!("arn:aws:s3:::{bucket}")
})];
if grant_versions {
statements.push(serde_json::json!({
"Effect": "Allow", "Principal": {"AWS": "*"}, "Action": ["s3:ListBucketVersions"], "Resource": format!("arn:aws:s3:::{bucket}")
}));
}
if deny_versions {
statements.push(serde_json::json!({
"Effect": "Deny", "Principal": {"AWS": "*"}, "Action": ["s3:ListBucketVersions"], "Resource": format!("arn:aws:s3:::{bucket}")
}));
}
let policy = serde_json::json!({"Version": "2012-10-17", "Statement": statements});
store
.update_bucket_metadata_config(
&bucket,
crate::storage::storage_api::ecstore_bucket::metadata::BUCKET_POLICY_CONFIG,
serde_json::to_vec(&policy).unwrap(),
)
.await
.expect("publish listing policy");
for header in ["x-rustfs-include-deleted", "x-minio-include-deleted"] {
for (value, authenticated) in [("true", true), ("false", true), ("true", false), ("false", false)] {
let mut envelope = request(Method::GET)
.server_context(&slot)
.extension(ReqInfo {
bucket: Some(bucket.clone()),
cred: authenticated.then(|| user.clone()),
..Default::default()
})
.header(header, value)
.build();
let result = authorize_bucket_listing(&mut envelope, &bucket, S3Action::ListBucketAction).await;
if value == "true" && (deny_versions || (authenticated && !grant_versions)) {
assert_eq!(
result.expect_err("version visibility requires permission").code(),
&S3ErrorCode::AccessDenied,
"{header}={value}, authenticated={authenticated}"
);
} else {
result.expect("authorized listings remain available");
}
}
}
}
}
#[test]
fn copy_tags_authorize_source_and_effective_destination_tags() {
crate::app::gating_test_env::run_large_stack_test("copy-tag-authorization", || async {
let store = crate::app::gating_test_env::shared_gating_ecstore().await;
let slot = ServerContextSlot::new();
let context = Arc::new(AppContext::new(store.clone(), Arc::new(UnreadyIam), Arc::new(TestKms)));
assert!(slot.install(context.clone()));
let bucket = format!("copy-tags-{}", uuid::Uuid::new_v4());
store
.make_bucket(&bucket, &MakeBucketOptions::default())
.await
.expect("bucket");
let mut reader = crate::storage::PutObjReader::from_vec(b"confidential source".to_vec());
let source = store
.put_object(
&bucket,
"source",
&mut reader,
&crate::storage::ObjectOptions {
versioned: true,
user_defined: HashMap::from([(
rustfs_utils::http::AMZ_OBJECT_TAGGING.to_string(),
"visibility=private".to_string(),
)]),
..Default::default()
},
)
.await
.expect("tagged historical source");
let historical = source.version_id.expect("versioned source").to_string();
let mut reader = crate::storage::PutObjReader::from_vec(b"current source".to_vec());
store
.put_object(
&bucket,
"source",
&mut reader,
&crate::storage::ObjectOptions {
versioned: true,
user_defined: HashMap::from([(
rustfs_utils::http::AMZ_OBJECT_TAGGING.to_string(),
"visibility=private".to_string(),
)]),
..Default::default()
},
)
.await
.expect("current tagged source");
for (key, tags) in [("source-null", "visibility=private"), ("source-empty", "")] {
let mut reader = crate::storage::PutObjReader::from_vec(b"unversioned source".to_vec());
store
.put_object(
&bucket,
key,
&mut reader,
&crate::storage::ObjectOptions {
user_defined: HashMap::from([(rustfs_utils::http::AMZ_OBJECT_TAGGING.to_string(), tags.to_string())]),
..Default::default()
},
)
.await
.expect("unversioned source");
}
for (index, (directive, replacement, explicit_version, tag_permission, deny_public, expected)) in [
(None, None, None, "missing", false, false),
(Some("COPY"), None, None, "deny", false, false),
(None, None, None, "current", false, true),
(Some("COPY"), None, Some("historical"), "current", false, false),
(Some("COPY"), None, Some("historical"), "version", false, true),
(Some("COPY"), None, Some("null"), "current", false, false),
(Some("COPY"), None, Some("null"), "version", false, true),
(None, None, None, "empty", false, true),
(Some("REPLACE"), Some("visibility=private"), None, "deny", false, true),
(Some("REPLACE"), Some(""), None, "deny", false, true),
(Some("REPLACE"), Some("visibility=public"), None, "deny", true, false),
]
.into_iter()
.enumerate()
{
let mut statements = vec![
serde_json::json!({"Effect":"Allow","Principal":{"AWS":"*"},"Action":["s3:GetObject","s3:GetObjectVersion"],"Resource":format!("arn:aws:s3:::{bucket}/source*")}),
serde_json::json!({"Effect":"Allow","Principal":{"AWS":"*"},"Action":"s3:PutObject","Resource":format!("arn:aws:s3:::{bucket}/copied-*")}),
];
if !matches!(tag_permission, "missing" | "empty") {
let action = if tag_permission == "version" {
"s3:GetObjectVersionTagging"
} else {
"s3:GetObjectTagging"
};
statements.push(serde_json::json!({"Effect":if tag_permission == "deny" { "Deny" } else { "Allow" },"Principal":{"AWS":"*"},"Action":action,"Resource":format!("arn:aws:s3:::{bucket}/source*")}));
}
if deny_public {
statements.push(serde_json::json!({"Effect":"Deny","Principal":{"AWS":"*"},"Action":"s3:PutObject","Resource":format!("arn:aws:s3:::{bucket}/*"),"Condition":{"StringEquals":{"s3:RequestObjectTag/visibility":"public"}}}));
}
let policy = serde_json::to_vec(&serde_json::json!({"Version":"2012-10-17","Statement":statements})).expect("policy");
store
.update_bucket_metadata_config(
&bucket,
crate::storage::storage_api::ecstore_bucket::metadata::BUCKET_POLICY_CONFIG,
policy,
)
.await
.expect("policy publication");
let destination = format!("copied-{index}");
let source_key = if tag_permission == "empty" {
"source-empty"
} else if explicit_version == Some("null") {
"source-null"
} else {
"source"
};
let source_version = explicit_version.map(|version| {
if version == "historical" {
historical.clone()
} else {
"null".to_owned()
}
});
let mut envelope = request(Method::PUT).req_info().server_context(&slot).build();
let facts = copy_facts(
&bucket,
&destination,
CopySourceObject {
bucket: &bucket,
key: source_key,
version_id: source_version.as_deref(),
},
replacement,
directive,
);
let access = authorize_copy_object(&mut envelope, &slot, facts).await;
if !expected {
assert_eq!(
access.expect_err("copy must enforce tagging and destination policy").code(),
&S3ErrorCode::AccessDenied,
"case {index}"
);
assert!(
store
.get_object_info(&bucket, &destination, &crate::storage::ObjectOptions::default())
.await
.is_err()
);
continue;
}
access.expect("authorized copy");
let input = copy_write_input(
&bucket,
&destination,
CopySourceValue::Bucket {
bucket: bucket.clone(),
key: source_key.to_owned(),
version_id: source_version,
},
None,
replacement,
directive,
);
DefaultObjectUsecase::with_context(Some(context.clone()))
.execute_copy_object_owned(envelope, input)
.await
.expect("copy preserves allowed tags");
assert_eq!(
store
.get_object_tags(&bucket, &destination, &crate::storage::ObjectOptions::default())
.await
.expect("destination tags"),
replacement.unwrap_or(if tag_permission == "empty" { "" } else { "visibility=private" })
);
}
});
}
#[test]
fn copy_tags_rechecks_actual_source_after_access_snapshot() {
crate::app::gating_test_env::run_large_stack_test("copy-tag-snapshot", || async {
let store = crate::app::gating_test_env::shared_gating_ecstore().await;
let slot = ServerContextSlot::new();
let context = Arc::new(AppContext::new(store.clone(), Arc::new(UnreadyIam), Arc::new(TestKms)));
assert!(slot.install(context.clone()));
for mode in ["tag-change", "new-tags", "allowed-tags", "source-tag-change", "self-copy"] {
let bucket = format!("copy-snapshot-{}", uuid::Uuid::new_v4());
store
.make_bucket(&bucket, &MakeBucketOptions::default())
.await
.expect("bucket");
let destination = if mode == "self-copy" { "source" } else { "destination" };
let initial = if mode == "new-tags" { "" } else { "visibility=private" };
let mut reader = crate::storage::PutObjReader::from_vec(b"source".to_vec());
store
.put_object(
&bucket,
"source",
&mut reader,
&crate::storage::ObjectOptions {
user_defined: HashMap::from([(rustfs_utils::http::AMZ_OBJECT_TAGGING.to_string(), initial.to_string())]),
..Default::default()
},
)
.await
.expect("source");
let mut statements = vec![
serde_json::json!({"Effect":"Allow","Principal":{"AWS":"*"},"Action":"s3:GetObject","Resource":format!("arn:aws:s3:::{bucket}/source")}),
serde_json::json!({"Effect":"Allow","Principal":{"AWS":"*"},"Action":"s3:PutObject","Resource":format!("arn:aws:s3:::{bucket}/{destination}")}),
serde_json::json!({"Effect":"Deny","Principal":{"AWS":"*"},"Action":"s3:PutObject","Resource":format!("arn:aws:s3:::{bucket}/*"),"Condition":{"StringEquals":{"s3:RequestObjectTag/visibility":"public"}}}),
];
if mode != "new-tags" {
statements.push(serde_json::json!({"Effect":"Allow","Principal":{"AWS":"*"},"Action":"s3:GetObjectTagging","Resource":format!("arn:aws:s3:::{bucket}/source")}));
// A conditional Allow must work at the preliminary hook as well.
statements[1]["Condition"] = serde_json::json!({"StringEquals":{"s3:RequestObjectTag/visibility":"private"}});
if mode == "source-tag-change" {
statements[1].as_object_mut().expect("statement").remove("Condition");
statements.remove(2);
statements.last_mut().expect("tag read statement")["Condition"] =
serde_json::json!({"StringEquals":{"s3:ExistingObjectTag/visibility":"private"}});
} else if mode == "self-copy" {
statements[1]["Condition"]["StringEquals"]["s3:ExistingObjectTag/visibility"] = serde_json::json!("private");
statements.last_mut().expect("tag read statement")["Condition"] =
serde_json::json!({"StringEquals":{"s3:ExistingObjectTag/visibility":"private"}});
}
}
store
.update_bucket_metadata_config(
&bucket,
crate::storage::storage_api::ecstore_bucket::metadata::BUCKET_POLICY_CONFIG,
serde_json::to_vec(&serde_json::json!({"Version":"2012-10-17","Statement":statements})).expect("policy"),
)
.await
.expect("publish policy");
let mut envelope = request(Method::PUT).req_info().server_context(&slot).build();
let facts = copy_facts(
&bucket,
destination,
CopySourceObject {
bucket: &bucket,
key: "source",
version_id: None,
},
None,
None,
);
authorize_copy_object(&mut envelope, &slot, facts)
.await
.expect("preliminary snapshot permits copy");
if !matches!(mode, "allowed-tags" | "self-copy") {
store
.put_object_tags(&bucket, "source", "visibility=public", &crate::storage::ObjectOptions::default())
.await
.expect("change source after access hook");
}
let input = copy_write_input(
&bucket,
destination,
CopySourceValue::Bucket {
bucket: bucket.clone(),
key: "source".to_owned(),
version_id: None,
},
(mode == "self-copy").then_some("REPLACE"),
None,
None,
);
let probe = crate::storage::helper::MutationCompletionProbe::install(&bucket);
let result = tokio::time::timeout(
std::time::Duration::from_secs(10),
DefaultObjectUsecase::with_context(Some(context.clone())).execute_copy_object_owned(envelope, input),
)
.await
.expect("copy authorization must not reacquire its own object lock");
if matches!(mode, "allowed-tags" | "self-copy") {
result.expect("unchanged allowed tags complete");
{
let events = probe.events.lock().unwrap();
assert_eq!(events.len(), 1, "one committed copy event");
assert_eq!(events[0].event_name, rustfs_s3_types::EventName::ObjectCreatedCopy);
assert_eq!(events[0].object.name, destination);
}
assert_eq!(
store
.get_object_tags(&bucket, destination, &crate::storage::ObjectOptions::default())
.await
.expect("copied tags"),
"visibility=private"
);
} else {
assert_eq!(
result.expect_err("actual source tags require fresh authorization").code(),
&S3ErrorCode::AccessDenied
);
assert!(
store
.get_object_info(&bucket, "destination", &crate::storage::ObjectOptions::default())
.await
.is_err(),
"denial must precede destination publication"
);
assert!(probe.events.lock().unwrap().is_empty(), "refused copy cannot emit a success event");
}
}
});
}
@@ -0,0 +1,757 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Bucket-generation snapshots and early refusals of the owned object authorization functions.
//!
//! Responsible for: the destination and copy-source bucket generations an authorization captures
//! and keeps across a recreation of the same bucket name, the use case refusing to act on the
//! recreated bucket, the fail-closed refusal of a request without its generation guard or whose
//! policy metadata is unavailable, the write-offset refusal ahead of authorization, and the
//! request facts DeleteObjects leaves for its use case.
//! Not responsible for: tag- or condition-dependent decisions (see `copy_tags.rs`), or the policy
//! evaluation itself.
//! Upstream: the owned request fixture in `storage::authorization::tests::fixture`.
//! Downstream: `storage::access_owned::{object_write, object_misc, object_commit, listing_multipart}`,
//! the generation guards in `storage::authorization`, and the Delete, Restore, Copy and
//! UploadPartCopy use cases.
use crate::app::multipart_usecase::{DefaultMultipartUsecase, UploadPartCopyWriteInput, UploadPartTarget};
use crate::app::object_usecase::{CopyObjectWriteInput, DefaultObjectUsecase};
use crate::storage::access_owned::listing_multipart::authorize_abort_multipart_upload;
use crate::storage::access_owned::object_commit::{
CopyObjectAuthorization, CopySourceObject, authorize_complete_multipart_upload, authorize_copy_object,
authorize_upload_part_copy,
};
use crate::storage::access_owned::object_misc::{authorize_delete_object, authorize_delete_objects, authorize_restore_object};
use crate::storage::access_owned::object_write::{PutObjectAuthorization, authorize_put_object};
use crate::storage::authorization::tests::fixture::{TestKms, UnreadyIam, request, server_context};
use crate::storage::authorization::{
apply_bucket_generation_guard, apply_copy_source_bucket_generation_guard, existing_object_tag_conditions,
has_write_offset_bytes_header, install_object_generation_test_hook,
};
use crate::storage::req_info::ReqInfo;
use crate::storage::storage_api::contract::bucket::{BucketOperations as _, DeleteBucketOptions, MakeBucketOptions};
use crate::storage::storage_api::contract::multipart::MultipartOperations as _;
use crate::storage::storage_api::contract::object::{ObjectIO as _, ObjectOperations as _};
use crate::storage::storage_api::runtime_sources_consumer::{AppContext, ServerContextSlot};
use http::{HeaderMap, Method};
use rustfs_gateway_types::CopySourceValue;
use rustfs_s3_types::S3ErrorCode;
use serial_test::serial;
use std::sync::Arc;
/// The copy use case input the legacy edge built from a CopyObject request that names only a
/// destination and a bucket-path source.
fn copy_write_input(bucket: &str, key: &str, copy_source: CopySourceValue) -> CopyObjectWriteInput {
CopyObjectWriteInput {
bucket: bucket.to_owned(),
key: key.to_owned(),
copy_source: Some(copy_source),
version_id: None,
specific_if_none_match: false,
server_side_encryption: None,
ssekms_key_id: None,
sse_customer_algorithm: None,
sse_customer_key: None,
sse_customer_key_md5: None,
copy_source_sse_customer_algorithm: None,
copy_source_sse_customer_key: None,
copy_source_sse_customer_key_md5: None,
metadata_directive: None,
metadata: None,
tagging: None,
tagging_directive: None,
copy_source_if_match: None,
copy_source_if_none_match: None,
cache_control: None,
content_disposition: None,
content_encoding: None,
content_language: None,
content_type: None,
expires: None,
website_redirect_location: None,
object_lock_legal_hold_status: None,
object_lock_mode: None,
object_lock_retain_until_date: None,
storage_class: None,
checksum_algorithm: None,
}
}
/// CopyObject's authorization facts for a request naming only a destination and an unversioned
/// bucket-path source.
fn copy_facts<'a>(bucket: &'a str, key: &'a str, source_bucket: &'a str, source_key: &'a str) -> CopyObjectAuthorization<'a> {
CopyObjectAuthorization {
bucket,
key,
source: Some(CopySourceObject {
bucket: source_bucket,
key: source_key,
version_id: None,
}),
query_version: None,
tagging: None,
tagging_directive: None,
retain_until: None,
legal_hold: false,
retention: false,
}
}
/// PutObject's authorization facts for a request naming only a bucket and key.
fn put_facts<'a>(bucket: &'a str, key: &'a str) -> PutObjectAuthorization<'a> {
PutObjectAuthorization {
bucket,
key,
version_id: None,
retain_until: None,
specific_etag: false,
legal_hold: false,
retention: false,
post: false,
}
}
#[tokio::test]
#[serial]
async fn put_object_access_captures_authorized_bucket_incarnation() {
let store = crate::app::gating_test_env::shared_gating_ecstore().await;
let slot = server_context(Arc::clone(&store));
let bucket = "generation-guard-collision";
store
.make_bucket(bucket, &MakeBucketOptions::default())
.await
.expect("test bucket should be created");
let policy_json = format!(
r#"{{"Version":"2012-10-17","Statement":[{{"Effect":"Allow","Principal":{{"AWS":"*"}},"Action":["s3:PutObject"],"Resource":["arn:aws:s3:::{bucket}/*"]}}]}}"#
);
let mut metadata = (*crate::storage::get_bucket_metadata(bucket)
.await
.expect("new bucket metadata should be cached"))
.clone();
metadata.policy_config = Some(serde_json::from_str(&policy_json).expect("test bucket policy should parse"));
metadata.policy_config_json = policy_json.into_bytes();
crate::storage::storage_api::set_bucket_metadata(bucket.to_string(), metadata)
.await
.expect("test bucket policy should be published");
let mut envelope = request(Method::PUT).req_info().server_context(&slot).build();
authorize_put_object(&mut envelope, &slot, put_facts(bucket, bucket), |_| Ok(()))
.await
.expect("anonymous PutObject should be authorized by the test policy");
let mut opts = crate::storage::ObjectOptions::default();
apply_bucket_generation_guard(&envelope, bucket, &mut opts).expect("request snapshot should apply to PutObject options");
assert_eq!(
opts.expected_bucket_incarnation_id,
Some(
store
.bucket_incarnation_id(bucket)
.await
.expect("bucket incarnation should remain readable")
)
);
}
#[test]
#[serial]
fn delete_object_access_keeps_snapshot_across_recreation_before_admission() {
crate::app::gating_test_env::run_large_stack_test(
"delete-object-generation-guard",
delete_object_access_keeps_snapshot_across_recreation_before_admission_inner,
);
}
async fn delete_object_access_keeps_snapshot_across_recreation_before_admission_inner() {
let store = crate::app::gating_test_env::shared_gating_ecstore().await;
let slot = ServerContextSlot::new();
let app_context = Arc::new(AppContext::new(Arc::clone(&store), Arc::new(UnreadyIam), Arc::new(TestKms)));
assert!(slot.install(Arc::clone(&app_context)));
let bucket = format!("delete-generation-guard-{}", uuid::Uuid::new_v4());
store
.make_bucket(&bucket, &MakeBucketOptions::default())
.await
.expect("test bucket should be created");
assert!(
existing_object_tag_conditions(store.as_ref(), &bucket, "missing-object", None)
.await
.expect("a missing object must be treated as having no policy tags")
.is_empty()
);
assert!(
existing_object_tag_conditions(store.as_ref(), &format!("missing-{bucket}"), "missing-object", None)
.await
.expect("a missing bucket must be indistinguishable during pre-authorization tag lookup")
.is_empty()
);
let policy_json = format!(
r#"{{"Version":"2012-10-17","Statement":[{{"Effect":"Allow","Principal":{{"AWS":"*"}},"Action":["s3:DeleteObject"],"Resource":["arn:aws:s3:::{bucket}/*"]}}]}}"#
);
let mut metadata = (*crate::storage::get_bucket_metadata(&bucket)
.await
.expect("new bucket metadata should be cached"))
.clone();
metadata.policy_config = Some(serde_json::from_str(&policy_json).expect("test bucket policy should parse"));
metadata.policy_config_json = policy_json.clone().into_bytes();
crate::storage::storage_api::set_bucket_metadata(bucket.clone(), metadata)
.await
.expect("test bucket policy should be published");
let mut envelope = request(Method::DELETE).req_info().server_context(&slot).build();
let captured_incarnation_id = store
.bucket_incarnation_id(&bucket)
.await
.expect("read the bucket generation before admission");
let (loaded_tx, loaded_rx) = tokio::sync::oneshot::channel();
let (resume_tx, resume_rx) = tokio::sync::oneshot::channel();
install_object_generation_test_hook(bucket.clone(), loaded_tx, resume_rx);
let access = {
let slot = slot.clone();
let bucket = bucket.clone();
tokio::spawn(async move {
let result = authorize_delete_object(&slot, &mut envelope, &bucket, "object", None).await;
(result, envelope)
})
};
tokio::time::timeout(std::time::Duration::from_secs(10), loaded_rx)
.await
.expect("DeleteObject should reach the generation snapshot hook")
.expect("DeleteObject access should remain paused before admission");
store
.delete_bucket(&bucket, &DeleteBucketOptions::default())
.await
.expect("delete the authorized bucket generation");
store
.make_bucket(&bucket, &MakeBucketOptions::default())
.await
.expect("recreate the same bucket name");
let mut metadata = (*crate::storage::get_bucket_metadata(&bucket)
.await
.expect("recreated bucket metadata should be cached"))
.clone();
metadata.policy_config = Some(serde_json::from_str(&policy_json).expect("test policy should parse"));
metadata.policy_config_json = policy_json.into_bytes();
crate::storage::storage_api::set_bucket_metadata(bucket.clone(), metadata)
.await
.expect("republish the same policy for admission");
let mut reader = crate::storage::PutObjReader::from_vec(b"new generation".to_vec());
store
.put_object(&bucket, "object", &mut reader, &crate::storage::ObjectOptions::default())
.await
.expect("put the new-generation object");
assert_ne!(
captured_incarnation_id,
store
.bucket_incarnation_id(&bucket)
.await
.expect("read the recreated bucket generation")
);
resume_tx
.send(())
.expect("DeleteObject should still be paused before admission");
let (result, envelope) = access.await.expect("DeleteObject access task should join");
result.expect("the equivalent policy should authorize DeleteObject");
let mut opts = crate::storage::ObjectOptions::default();
apply_bucket_generation_guard(&envelope, &bucket, &mut opts).expect("request snapshot should apply to DeleteObject options");
assert_eq!(opts.expected_bucket_incarnation_id, Some(captured_incarnation_id));
let err = Box::pin(DefaultObjectUsecase::with_context(Some(app_context)).execute_delete_object_fields(
envelope,
bucket.clone(),
"object".to_string(),
None,
))
.await
.expect_err("the old authorization must not delete from the recreated bucket");
assert_eq!(err.code(), &S3ErrorCode::NoSuchBucket);
store
.get_object_info(&bucket, "object", &crate::storage::ObjectOptions::default())
.await
.expect("the new-generation object must survive the stale request");
}
#[tokio::test]
#[serial]
async fn copy_operations_reject_recreated_source_bucket_after_authorization() {
let store = crate::app::gating_test_env::shared_gating_ecstore().await;
let slot = ServerContextSlot::new();
let app_context = Arc::new(AppContext::new(Arc::clone(&store), Arc::new(UnreadyIam), Arc::new(TestKms)));
assert!(slot.install(Arc::clone(&app_context)));
let suffix = uuid::Uuid::new_v4();
let src_bucket = format!("copy-src-generation-{suffix}");
let dst_bucket = format!("copy-dst-generation-{suffix}");
for bucket in [&src_bucket, &dst_bucket] {
store
.make_bucket(bucket, &MakeBucketOptions::default())
.await
.expect("create copy test bucket");
}
let src_policy_json = format!(
r#"{{"Version":"2012-10-17","Statement":[{{"Effect":"Allow","Principal":{{"AWS":"*"}},"Action":["s3:GetObject"],"Resource":["arn:aws:s3:::{src_bucket}/*"]}}]}}"#
);
let mut src_metadata = (*crate::storage::get_bucket_metadata(&src_bucket)
.await
.expect("source bucket metadata should be cached"))
.clone();
src_metadata.policy_config = Some(serde_json::from_str(&src_policy_json).expect("source policy should parse"));
src_metadata.policy_config_json = src_policy_json.into_bytes();
crate::storage::storage_api::set_bucket_metadata(src_bucket.clone(), src_metadata)
.await
.expect("publish source policy");
let dst_policy_json = format!(
r#"{{"Version":"2012-10-17","Statement":[{{"Effect":"Allow","Principal":{{"AWS":"*"}},"Action":["s3:PutObject"],"Resource":["arn:aws:s3:::{dst_bucket}/*"]}}]}}"#
);
let mut dst_metadata = (*crate::storage::get_bucket_metadata(&dst_bucket)
.await
.expect("destination bucket metadata should be cached"))
.clone();
dst_metadata.policy_config = Some(serde_json::from_str(&dst_policy_json).expect("destination policy should parse"));
dst_metadata.policy_config_json = dst_policy_json.into_bytes();
crate::storage::storage_api::set_bucket_metadata(dst_bucket.clone(), dst_metadata)
.await
.expect("publish destination policy");
let mut envelope = request(Method::PUT).req_info().server_context(&slot).build();
authorize_copy_object(&mut envelope, &slot, copy_facts(&dst_bucket, "copied", &src_bucket, "secret"))
.await
.expect("test policies should authorize the copy request");
let mut source_opts = crate::storage::ObjectOptions::default();
apply_copy_source_bucket_generation_guard(&envelope, &src_bucket, &mut source_opts)
.expect("authorized source generation should be attached");
let authorized_source_incarnation_id = source_opts
.expected_bucket_incarnation_id
.expect("source incarnation should be captured");
let upload = store
.new_multipart_upload(&dst_bucket, "multipart-copy", &crate::storage::ObjectOptions::default())
.await
.expect("create destination multipart upload");
let upload_id = upload.upload_id;
let mut upload_envelope = request(Method::PUT).req_info().server_context(&slot).build();
authorize_upload_part_copy(
&mut upload_envelope,
&slot,
&dst_bucket,
"multipart-copy",
Some(CopySourceObject {
bucket: &src_bucket,
key: "secret",
version_id: None,
}),
)
.await
.expect("test policies should authorize the part copy request");
store
.delete_bucket(&src_bucket, &DeleteBucketOptions::default())
.await
.expect("delete the authorized source generation");
store
.make_bucket(&src_bucket, &MakeBucketOptions::default())
.await
.expect("recreate the source bucket name");
assert_ne!(
authorized_source_incarnation_id,
store
.bucket_incarnation_id(&src_bucket)
.await
.expect("read recreated source incarnation")
);
let mut reader = crate::storage::PutObjReader::from_vec(b"new generation secret".to_vec());
store
.put_object(&src_bucket, "secret", &mut reader, &crate::storage::ObjectOptions::default())
.await
.expect("write the new-generation source object");
let source = || CopySourceValue::Bucket {
bucket: src_bucket.clone(),
key: "secret".to_owned(),
version_id: None,
};
let err = DefaultObjectUsecase::with_context(Some(Arc::clone(&app_context)))
.execute_copy_object_owned(envelope, copy_write_input(&dst_bucket, "copied", source()))
.await
.expect_err("stale authorization must not read from the recreated source bucket");
assert_eq!(err.code(), &S3ErrorCode::NoSuchBucket);
let upload_input = UploadPartCopyWriteInput {
target: UploadPartTarget::Legacy {
bucket: dst_bucket.clone(),
key: "multipart-copy".to_owned(),
upload_id,
},
part_number: 1,
copy_source: source(),
copy_source_range: None,
copy_source_if_match: None,
copy_source_if_none_match: None,
sse_customer_algorithm: None,
sse_customer_key: None,
sse_customer_key_md5: None,
copy_source_sse_customer_algorithm: None,
copy_source_sse_customer_key: None,
copy_source_sse_customer_key_md5: None,
};
let err = DefaultMultipartUsecase::with_context(Some(app_context))
.execute_upload_part_copy_owned(upload_envelope, upload_input)
.await
.err()
.expect("stale authorization must not read the recreated source into a part");
assert_eq!(err.code(), &S3ErrorCode::NoSuchBucket);
store
.get_object_info(&dst_bucket, "copied", &crate::storage::ObjectOptions::default())
.await
.expect_err("the new-generation source object must not be copied");
}
#[tokio::test]
#[serial]
async fn copy_object_rejects_recreated_destination_bucket_after_authorization() {
let store = crate::app::gating_test_env::shared_gating_ecstore().await;
let slot = ServerContextSlot::new();
let context = Arc::new(AppContext::new(store.clone(), Arc::new(UnreadyIam), Arc::new(TestKms)));
assert!(slot.install(context.clone()));
let src = format!("copy-src-{}", uuid::Uuid::new_v4());
let dst = format!("copy-dst-{}", uuid::Uuid::new_v4());
let policy = |bucket: &str, action: &str| {
serde_json::to_vec(&serde_json::json!({"Version":"2012-10-17","Statement":[{
"Effect":"Allow","Principal":{"AWS":"*"},"Action":action,"Resource":format!("arn:aws:s3:::{bucket}/*")
}]}))
.unwrap()
};
for (bucket, action) in [(&src, "s3:GetObject"), (&dst, "s3:PutObject")] {
store.make_bucket(bucket, &MakeBucketOptions::default()).await.unwrap();
store
.update_bucket_metadata_config(
bucket,
crate::storage::storage_api::ecstore_bucket::metadata::BUCKET_POLICY_CONFIG,
policy(bucket, action),
)
.await
.unwrap();
}
let opts = crate::storage::ObjectOptions::default();
let mut body = crate::storage::PutObjReader::from_vec(b"source contents".to_vec());
store.put_object(&src, "source", &mut body, &opts).await.unwrap();
let mut envelope = request(Method::PUT).req_info().server_context(&slot).build();
let captured_incarnation = store.bucket_incarnation_id(&dst).await.unwrap();
let (loaded_tx, loaded_rx) = tokio::sync::oneshot::channel();
let (resume_tx, resume_rx) = tokio::sync::oneshot::channel();
install_object_generation_test_hook(dst.clone(), loaded_tx, resume_rx);
let access = {
let (slot, src, dst) = (slot.clone(), src.clone(), dst.clone());
tokio::spawn(async move {
let result = authorize_copy_object(&mut envelope, &slot, copy_facts(&dst, "copied", &src, "source")).await;
(result, envelope)
})
};
tokio::time::timeout(std::time::Duration::from_secs(10), loaded_rx)
.await
.expect("destination generation capture must be reached")
.expect("copy access ended before capture");
store.delete_bucket(&dst, &DeleteBucketOptions::default()).await.unwrap();
store.make_bucket(&dst, &MakeBucketOptions::default()).await.unwrap();
store
.update_bucket_metadata_config(
&dst,
crate::storage::storage_api::ecstore_bucket::metadata::BUCKET_POLICY_CONFIG,
policy(&dst, "s3:PutObject"),
)
.await
.unwrap();
assert_ne!(captured_incarnation, store.bucket_incarnation_id(&dst).await.unwrap());
resume_tx.send(()).expect("resume copy before admission");
let (result, envelope) = access.await.unwrap();
result.expect("the equivalent policy should admit the copy");
let mut captured = crate::storage::ObjectOptions::default();
apply_bucket_generation_guard(&envelope, &dst, &mut captured).expect("copy carries the destination snapshot");
assert_eq!(captured.expected_bucket_incarnation_id, Some(captured_incarnation));
let input = copy_write_input(
&dst,
"copied",
CopySourceValue::Bucket {
bucket: src.clone(),
key: "source".to_owned(),
version_id: None,
},
);
let error = DefaultObjectUsecase::with_context(Some(context))
.execute_copy_object_owned(envelope, input)
.await
.expect_err("must refuse before publishing to the recreated bucket");
assert_eq!(error.code(), &S3ErrorCode::NoSuchBucket);
let missing = store
.get_object_info(&dst, "copied", &opts)
.await
.expect_err("stale copy wrote an object");
assert!(crate::storage::is_err_object_not_found(&missing), "{missing}");
}
#[test]
#[serial]
fn restore_object_access_keeps_snapshot_across_recreation_before_admission() {
crate::app::gating_test_env::run_large_stack_test(
"restore-object-generation-guard",
restore_object_access_keeps_snapshot_across_recreation_before_admission_inner,
);
}
async fn restore_object_access_keeps_snapshot_across_recreation_before_admission_inner() {
let store = crate::app::gating_test_env::shared_gating_ecstore().await;
let slot = ServerContextSlot::new();
let app_context = Arc::new(AppContext::new(Arc::clone(&store), Arc::new(UnreadyIam), Arc::new(TestKms)));
assert!(slot.install(Arc::clone(&app_context)));
let bucket = format!("restore-generation-guard-{}", uuid::Uuid::new_v4());
store
.make_bucket(&bucket, &MakeBucketOptions::default())
.await
.expect("create the bucket generation being authorized");
let authorized_incarnation_id = store
.bucket_incarnation_id(&bucket)
.await
.expect("read the authorized bucket incarnation");
let policy_json = format!(
r#"{{"Version":"2012-10-17","Statement":[{{"Effect":"Allow","Principal":{{"AWS":"*"}},"Action":["s3:RestoreObject"],"Resource":["arn:aws:s3:::{bucket}/*"]}}]}}"#
);
let mut metadata = (*crate::storage::get_bucket_metadata(&bucket)
.await
.expect("authorized bucket metadata should be cached"))
.clone();
metadata.policy_config = Some(serde_json::from_str(&policy_json).expect("test policy should parse"));
metadata.policy_config_json = policy_json.clone().into_bytes();
crate::storage::storage_api::set_bucket_metadata(bucket.clone(), metadata)
.await
.expect("publish the RestoreObject policy");
let mut envelope = request(Method::POST).req_info().server_context(&slot).build();
let (authorized_tx, authorized_rx) = tokio::sync::oneshot::channel();
let (resume_tx, resume_rx) = tokio::sync::oneshot::channel();
install_object_generation_test_hook(bucket.clone(), authorized_tx, resume_rx);
let access = {
let slot = slot.clone();
let bucket = bucket.clone();
tokio::spawn(async move {
let result = authorize_restore_object(&slot, &mut envelope, &bucket, "object", None).await;
(result, envelope)
})
};
tokio::time::timeout(std::time::Duration::from_secs(10), authorized_rx)
.await
.expect("RestoreObject should reach the generation snapshot hook")
.expect("RestoreObject access must not fail before reaching the test hook");
store
.delete_bucket(&bucket, &DeleteBucketOptions::default())
.await
.expect("delete the authorized bucket generation");
store
.make_bucket(&bucket, &MakeBucketOptions::default())
.await
.expect("recreate the bucket with the same name");
let mut metadata = (*crate::storage::get_bucket_metadata(&bucket)
.await
.expect("recreated bucket metadata should be cached"))
.clone();
metadata.policy_config = Some(serde_json::from_str(&policy_json).expect("test policy should parse"));
metadata.policy_config_json = policy_json.into_bytes();
crate::storage::storage_api::set_bucket_metadata(bucket.clone(), metadata)
.await
.expect("republish the same policy for admission");
let recreated_incarnation_id = store
.bucket_incarnation_id(&bucket)
.await
.expect("read the recreated bucket incarnation");
assert_ne!(authorized_incarnation_id, recreated_incarnation_id);
let replacement = b"recreated bucket object";
store
.put_object(
&bucket,
"object",
&mut crate::storage::PutObjReader::from_vec(replacement.to_vec()),
&crate::storage::ObjectOptions::default(),
)
.await
.expect("write the recreated bucket's object");
resume_tx
.send(())
.expect("RestoreObject access should still be waiting at the test hook");
let (result, envelope) = access.await.expect("RestoreObject access task should join");
result.expect("the equivalent policy should authorize RestoreObject without refreshing its snapshot");
let mut opts = crate::storage::ObjectOptions::default();
apply_bucket_generation_guard(&envelope, &bucket, &mut opts).expect("apply the RestoreObject authorization guard");
// The RestoreObject usecase future is large enough that heap-pinning a
// single future was not enough on Linux: the whole scenario runs on a
// dedicated large stack (see the `#[test]` wrapper above) so unrelated
// growth in bucket metadata futures cannot tip the test over.
let restore_request = crate::app::storage_api::object_usecase::bucket::RestoreRequest {
days: Some(1),
description: None,
glacier_job_parameters: None,
output_location: None,
select_parameters: None,
tier: None,
type_: None,
};
let result = Box::pin(DefaultObjectUsecase::with_context(Some(app_context)).execute_restore_object_owned(
envelope,
bucket.clone(),
"object".to_string(),
None,
Some(restore_request),
))
.await;
use tokio::io::AsyncReadExt as _;
let mut reader = store
.get_object_reader(&bucket, "object", None, HeaderMap::new(), &crate::storage::ObjectOptions::default())
.await
.expect("read the recreated bucket's object");
let mut bytes = Vec::new();
reader.stream.read_to_end(&mut bytes).await.expect("read replacement bytes");
assert_eq!(
(
opts.expected_bucket_incarnation_id,
result.as_ref().err().map(|error| error.code()),
bytes.as_slice(),
reader.object_info.restore_ongoing
),
(
Some(authorized_incarnation_id),
Some(&S3ErrorCode::NoSuchBucket),
replacement.as_slice(),
false
),
"stale RestoreObject must refuse and preserve the recreated bucket's object"
);
drop(reader);
store
.delete_bucket(
&bucket,
&DeleteBucketOptions {
force: true,
..Default::default()
},
)
.await
.expect("clean up the recreated bucket");
}
#[test]
fn dispatched_put_object_fails_closed_without_generation_guard() {
let envelope = request(Method::PUT).server_context(&ServerContextSlot::new()).build();
let mut opts = crate::storage::ObjectOptions::default();
let err = apply_bucket_generation_guard(&envelope, "generation-guard-bucket", &mut opts)
.expect_err("a dispatched PutObject must not commit without its authorization fence");
assert_eq!(err.code(), &S3ErrorCode::InternalError);
}
#[test]
fn write_offset_bytes_header_detection_is_case_insensitive() {
let mut headers = HeaderMap::new();
headers.insert("X-Amz-Write-Offset-Bytes", http::HeaderValue::from_static("0"));
assert!(has_write_offset_bytes_header(&headers));
}
#[tokio::test]
async fn put_object_rejects_write_offset_bytes_before_authorize_request() {
let mut envelope = request(Method::PUT)
.uri("/test-bucket/test-key")
.header("x-amz-write-offset-bytes", "0")
.extension(ReqInfo {
cred: Some(rustfs_credentials::Credentials::default()),
..ReqInfo::default()
})
.build();
let err = authorize_put_object(&mut envelope, &ServerContextSlot::new(), put_facts("test-bucket", "test-key"), |_| Ok(()))
.await
.expect_err("write-offset-bytes requests should be rejected before authorization");
assert_eq!(err.code(), &S3ErrorCode::NotImplemented);
// The message `ApiError::error_code_to_message` gives `NotImplemented`.
assert_eq!(err.message(), Some("A header you provided implies functionality that is not implemented"));
let req_info = envelope.req_info().expect("req info should remain available");
assert_eq!(req_info.bucket.as_deref(), Some("test-bucket"));
assert_eq!(req_info.object.as_deref(), Some("test-key"));
assert_eq!(req_info.version_id, None);
}
#[tokio::test]
async fn delete_objects_defers_object_authorization_to_usecase() {
let mut envelope = request(Method::POST)
.extension(ReqInfo {
cred: Some(rustfs_credentials::Credentials::default()),
bucket: Some("previous-bucket".to_string()),
object: Some("previous-key".to_string()),
version_id: Some("previous-version".to_string()),
..ReqInfo::default()
})
.build();
authorize_delete_objects(&mut envelope, "test-bucket")
.expect("DeleteObjects access hook should not require bucket-level DeleteObject");
let req_info = envelope.req_info().expect("req info should remain available");
assert_eq!(req_info.bucket.as_deref(), Some("test-bucket"));
assert_eq!(req_info.object, None);
assert_eq!(req_info.version_id, None);
}
#[tokio::test]
async fn abort_multipart_upload_fails_closed_when_policy_metadata_is_unavailable() {
let mut envelope = request(Method::DELETE).req_info().build();
let err = authorize_abort_multipart_upload(&ServerContextSlot::new(), &mut envelope, "bucket", "object")
.await
.expect_err("unavailable policy metadata must fail closed");
assert_eq!(err.code(), &S3ErrorCode::InternalError);
}
#[tokio::test]
async fn complete_multipart_upload_fails_closed_when_policy_metadata_is_unavailable() {
let mut envelope = request(Method::POST).req_info().build();
let err = authorize_complete_multipart_upload(&mut envelope, &ServerContextSlot::new(), "bucket", "object", false)
.await
.expect_err("unavailable policy metadata must fail closed");
assert_eq!(err.code(), &S3ErrorCode::InternalError);
}
#[tokio::test]
async fn upload_part_copy_fails_closed_when_policy_metadata_is_unavailable() {
let mut envelope = request(Method::PUT).req_info().build();
let source = CopySourceObject {
bucket: "src-bucket",
key: "src-object",
version_id: None,
};
let err = authorize_upload_part_copy(&mut envelope, &ServerContextSlot::new(), "dst-bucket", "dst-object", Some(source))
.await
.expect_err("unavailable policy metadata must fail closed");
assert_eq!(err.code(), &S3ErrorCode::InternalError);
}
@@ -13,18 +13,16 @@
// limitations under the License.
//! Multipart initiation policy decisions take precedence over generation-read failures.
//!
//! Tests: `authorize_create_multipart_upload` on the owned request, with every write action the
//! initiation can need excluded in turn. Not tested: the listing hooks or the request decoder.
//! Upstream: `storage::authorization::tests::fixture`. Downstream: none.
use super::access_owned::listing_multipart::{CreateMultipartAuthorization, authorize_create_multipart_upload};
use crate::app::RequestEnvelope;
use crate::runtime_sources::{AppContext, ServerContextSlot};
use crate::storage::access::tests::{TestKms, UnreadyIam, build_request, ensure_req_info};
use crate::storage::access::{
AuthorizationRequest, BucketGenerationGuard, CreateMultipartUploadInput, FS, ObjectLockLegalHoldStatus, ObjectLockMode,
S3Access as _,
};
use crate::storage::access_owned::listing_multipart::{CreateMultipartAuthorization, authorize_create_multipart_upload};
use crate::storage::authorization::tests::fixture::{request, server_context};
use crate::storage::authorization::{AuthorizationRequest as _, BucketGenerationGuard};
use crate::storage::storage_api::contract::bucket::{BucketOperations as _, MakeBucketOptions};
use http::{HeaderValue, Method};
use std::sync::Arc;
use http::Method;
use time::OffsetDateTime;
#[test]
@@ -38,9 +36,7 @@ fn create_multipart_authorization_surfaces_generation_errors_after_policy_checks
"s3:PutObjectRetention",
];
let store = crate::app::gating_test_env::shared_gating_ecstore().await;
let slot = ServerContextSlot::new();
let fs = FS::with_server_ctx(slot.clone());
assert!(slot.install(Arc::new(AppContext::new(store.clone(), Arc::new(UnreadyIam), Arc::new(TestKms)))));
let slot = server_context(store.clone());
let bucket = format!("create-auth-order-{}", uuid::Uuid::new_v4().simple());
store
.make_bucket(&bucket, &MakeBucketOptions::default())
@@ -61,31 +57,15 @@ fn create_multipart_authorization_surfaces_generation_errors_after_policy_checks
.await
.expect("publish policy");
let retain_until = OffsetDateTime::now_utc() + time::Duration::days(1);
let mut legacy = build_request(
CreateMultipartUploadInput {
bucket: bucket.clone(),
key: "object".to_owned(),
object_lock_legal_hold_status: Some(ObjectLockLegalHoldStatus::from_static("OFF")),
object_lock_mode: Some(ObjectLockMode::from_static("GOVERNANCE")),
object_lock_retain_until_date: Some(retain_until.into()),
..Default::default()
},
Method::POST,
);
legacy
.headers
.insert("x-rustfs-source-replication-request", HeaderValue::from_static("true"));
legacy.extensions.insert(slot.clone());
ensure_req_info(&mut legacy);
legacy.install_bucket_generation(BucketGenerationGuard {
let mut owned = request(Method::POST)
.header("x-rustfs-source-replication-request", "true")
.server_context(&slot)
.req_info()
.build();
owned.install_bucket_generation(BucketGenerationGuard {
bucket: "other-bucket".to_owned(),
incarnation_id: uuid::Uuid::new_v4(),
});
let mut owned = RequestEnvelope::from_s3s(&legacy);
let legacy = fs
.create_multipart_upload(&mut legacy)
.await
.expect_err("policy or generation refusal");
let owned = authorize_create_multipart_upload(
&slot,
&mut owned,
@@ -101,7 +81,6 @@ fn create_multipart_authorization_surfaces_generation_errors_after_policy_checks
.await
.expect_err("policy or generation refusal");
let expected = if excluded.is_some() { "AccessDenied" } else { "InternalError" };
assert_eq!(legacy.code().as_str(), expected, "legacy {excluded:?}");
assert_eq!(owned.code().as_str(), expected, "owned {excluded:?}");
}
});
@@ -0,0 +1,26 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Tests of the per-operation authorization shapes, run on the owned request envelope.
mod copy_tags;
mod generation_guards;
mod listing_multipart;
mod object_commit;
mod object_lock_policy;
mod object_misc;
mod object_read;
mod replication_versions;
mod signed_query_lock;
mod write_admission;
@@ -12,19 +12,23 @@
// See the License for the specific language governing permissions and
// limitations under the License.
//! Parity of the copy-source authorization across both request edges: the same decision and the
//! same state handed on, for an `S3Request` and for a `RequestEnvelope`, in both directions.
//! The copy-source, copy and completion authorization on the owned request envelope: the
//! decision and the state handed on, allowed and refused.
//!
//! Tests: `authorize_copy_source`, `authorize_copy_object`, `authorize_upload_part_copy` and
//! `authorize_complete_multipart_upload`. Not tested: decoding `x-amz-copy-source` (the facts are
//! written out as the gateway handler derives them) or the copy body.
//! Upstream: `storage::authorization::tests::fixture`. Downstream: none.
use super::access_owned::object_commit::{CopySourceObject, authorize_copy_source};
use crate::app::{RequestEnvelope, RequestHead};
use crate::storage::access::tests::{TestKms, UnreadyIam, build_request, ensure_req_info};
use crate::storage::access::{
AuthorizationRequest, CopyObjectInput, CopySource, CopySourceBucketGenerationGuard, ECStore, FS, ReqInfo, S3Access as _,
UploadPartCopyInput,
use crate::storage::access_owned::object_commit::{
CopyObjectAuthorization, CopySourceObject, authorize_complete_multipart_upload, authorize_copy_object, authorize_copy_source,
authorize_upload_part_copy,
};
use crate::storage::authorization::tests::fixture::{request, server_context};
use crate::storage::authorization::{AuthorizationRequest, CopySourceBucketGenerationGuard};
use crate::storage::storage_api::contract::bucket::{BucketOperations as _, MakeBucketOptions};
use crate::storage::storage_api::runtime_sources_consumer::{AppContext, ServerContextSlot};
use http::{Extensions, HeaderMap, Method, Uri};
use crate::storage::storage_api::runtime_sources_consumer::{ECStore, ServerContextSlot};
use http::Method;
use rustfs_s3_types::{S3ErrorCode, S3Result};
use serial_test::serial;
use std::sync::Arc;
@@ -55,42 +59,22 @@ fn observe<R: AuthorizationRequest>(
}
}
/// Runs the copy-source authorization once on each edge with the same facts: `served` is the
/// request's server context, `capture` the one the source incarnation is read from.
async fn on_both_edges(
/// Runs the copy-source authorization once with the given facts: `served` is the request's
/// server context, `capture` the one the source incarnation is read from.
async fn on_owned_edge(
served: &Arc<ServerContextSlot>,
capture: &ServerContextSlot,
with_req_info: bool,
source: Option<(&str, &str, Option<&str>)>,
) -> (Observed, Observed) {
let extensions = || {
let mut extensions = Extensions::new();
if with_req_info {
extensions.insert(ReqInfo::default());
}
extensions.insert(Arc::clone(served));
extensions
};
let object = || source.map(|(bucket, key, version_id)| CopySourceObject { bucket, key, version_id });
let mut legacy = build_request((), Method::PUT);
legacy.extensions = extensions();
let result = authorize_copy_source(&mut legacy, capture, object()).await;
let generation = legacy.extensions.get::<CopySourceBucketGenerationGuard>();
let legacy = observe(&legacy, result, generation);
let head = RequestHead {
method: Method::PUT,
uri: Uri::from_static("/"),
headers: HeaderMap::new(),
credentials: None,
region: None,
service: None,
};
let mut envelope = RequestEnvelope::from_parts(head, extensions());
let result = authorize_copy_source(&mut envelope, capture, object()).await;
let owned = observe(&envelope, result, envelope.copy_source_bucket_generation());
(legacy, owned)
) -> Observed {
let object = source.map(|(bucket, key, version_id)| CopySourceObject { bucket, key, version_id });
let mut builder = request(Method::PUT);
if with_req_info {
builder = builder.req_info();
}
let mut envelope = builder.server_context(served).build();
let result = authorize_copy_source(&mut envelope, capture, object).await;
observe(&envelope, result, envelope.copy_source_bucket_generation())
}
async fn source_bucket(store: &ECStore, actions: &[&str]) -> String {
@@ -125,9 +109,8 @@ fn copy_source_authorization_decides_and_hands_on_the_same_on_both_edges() {
async fn copy_source_parity_inner() {
let store = crate::app::gating_test_env::shared_gating_ecstore().await;
let server = Arc::new(ServerContextSlot::new());
assert!(server.install(Arc::new(AppContext::new(Arc::clone(&store), Arc::new(UnreadyIam), Arc::new(TestKms)))));
let unready = Arc::new(ServerContextSlot::new());
let server = server_context(Arc::clone(&store));
let unready = ServerContextSlot::new();
let readable = source_bucket(&store, &["s3:GetObject"]).await;
let version_readable = source_bucket(&store, &["s3:GetObjectVersion"]).await;
@@ -238,9 +221,9 @@ async fn copy_source_parity_inner() {
// Every case is checked before failing, so one run names every case a defect breaks.
let mut mismatches = Vec::new();
for (case, slot, with_req_info, source, expected) in cases {
let (legacy, owned) = on_both_edges(&server, &slot, with_req_info, source).await;
if legacy != expected || owned != expected {
mismatches.push(format!("{case}: expected {expected:?}, legacy {legacy:?}, owned {owned:?}"));
let owned = on_owned_edge(&server, &slot, with_req_info, source).await;
if owned != expected {
mismatches.push(format!("{case}: expected {expected:?}, owned {owned:?}"));
}
}
assert!(mismatches.is_empty(), "{}", mismatches.join("\n"));
@@ -256,57 +239,62 @@ fn copy_hooks_decide_the_source_before_the_destination() {
/// `InvalidRequest`; that refusal must never be reached for a source the caller may not read.
async fn copy_hooks_source_first_inner() {
let store = crate::app::gating_test_env::shared_gating_ecstore().await;
let server = ServerContextSlot::new();
assert!(server.install(Arc::new(AppContext::new(Arc::clone(&store), Arc::new(UnreadyIam), Arc::new(TestKms)))));
let fs = FS::with_server_ctx(server);
let server = server_context(Arc::clone(&store));
let closed = source_bucket(&store, &[]).await;
let readable = source_bucket(&store, &["s3:GetObject"]).await;
let destination = source_bucket(&store, &["s3:PutObject"]).await;
let arn = "arn:aws:s3:us-east-1:123456789012:accesspoint/my-access-point/object/key";
for (case, spelling, copy_expected, part_expected) in [
// The copy source bucket as the request decoder hands it on (key `key`, no version); `None`
// is an access-point ARN such as
// `arn:aws:s3:us-east-1:123456789012:accesspoint/my-access-point/object/key`.
for (case, source_bucket_name, copy_expected, part_expected) in [
(
"a denied source",
format!("{closed}/key"),
Some(closed.as_str()),
S3ErrorCode::AccessDenied,
S3ErrorCode::AccessDenied,
),
("an ARN source", arn.to_owned(), S3ErrorCode::NotImplemented, S3ErrorCode::NotImplemented),
("an ARN source", None, S3ErrorCode::NotImplemented, S3ErrorCode::NotImplemented),
// Control: with the source readable, the destination half is reached and refuses.
(
"a readable source",
format!("{readable}/key"),
Some(readable.as_str()),
S3ErrorCode::InvalidRequest,
S3ErrorCode::AccessDenied,
),
] {
let source = || CopySource::parse(&spelling).expect("the request parser accepts the source");
let copy = CopyObjectInput::builder()
.bucket(destination.clone())
.key("copy".to_string())
.copy_source(source())
.tagging(Some("a=b".to_string()))
.build()
.expect("copy input");
let mut copy = build_request(copy, Method::PUT);
ensure_req_info(&mut copy);
copy.extensions.insert(fs.server_ctx().clone());
let copy_error = fs.copy_object(&mut copy).await.expect_err(case);
assert_eq!(S3ErrorCode::from(copy_error.code().clone()), copy_expected, "CopyObject: {case}");
let source = || {
source_bucket_name.map(|bucket| CopySourceObject {
bucket,
key: "key",
version_id: None,
})
};
let mut copy = request(Method::PUT).req_info().server_context(&server).build();
let copy_error = authorize_copy_object(
&mut copy,
&server,
CopyObjectAuthorization {
bucket: &destination,
key: "copy",
source: source(),
query_version: None,
tagging: Some("a=b"),
tagging_directive: None,
retain_until: None,
legal_hold: false,
retention: false,
},
)
.await
.expect_err(case);
assert_eq!(copy_error.code(), &copy_expected, "CopyObject: {case}");
let part = UploadPartCopyInput::builder()
.bucket(closed.clone())
.key("copy".to_string())
.copy_source(source())
.upload_id("unused".to_string())
.part_number(1)
.build()
.expect("part copy input");
let mut part = build_request(part, Method::PUT);
ensure_req_info(&mut part);
part.extensions.insert(fs.server_ctx().clone());
let part_error = fs.upload_part_copy(&mut part).await.expect_err(case);
assert_eq!(S3ErrorCode::from(part_error.code().clone()), part_expected, "UploadPartCopy: {case}");
let mut part = request(Method::PUT).req_info().server_context(&server).build();
let part_error = authorize_upload_part_copy(&mut part, &server, &closed, "copy", source())
.await
.expect_err(case);
assert_eq!(part_error.code(), &part_expected, "UploadPartCopy: {case}");
}
}
@@ -314,8 +302,7 @@ async fn copy_hooks_source_first_inner() {
#[serial]
async fn completion_authorization_requires_write_read_and_replication_permissions() {
let store = crate::app::gating_test_env::shared_gating_ecstore().await;
let server = ServerContextSlot::new();
assert!(server.install(Arc::new(AppContext::new(Arc::clone(&store), Arc::new(UnreadyIam), Arc::new(TestKms)))));
let server = server_context(Arc::clone(&store));
for (actions, specific_etag, replica, allowed) in [
(vec!["s3:PutObject"], false, false, true),
(vec![], false, false, false),
@@ -325,40 +312,16 @@ async fn completion_authorization_requires_write_read_and_replication_permission
(vec!["s3:GetObject"], false, false, false),
] {
let bucket = source_bucket(&store, &actions).await;
let mut request = build_request((), Method::POST);
request.extensions.insert(Arc::clone(&server));
request.extensions.insert(ReqInfo::default());
let mut builder = request(Method::POST).server_context(&server).req_info();
if replica {
request
.headers
.insert("x-amz-replication-status", http::HeaderValue::from_static("REPLICA"));
builder = builder.header("x-amz-replication-status", "REPLICA");
}
let mut envelope = RequestEnvelope::from_s3s(&request);
for owned in [false, true] {
let result = if owned {
super::access_owned::object_commit::authorize_complete_multipart_upload(
&mut envelope,
&server,
&bucket,
"object",
specific_etag,
)
.await
} else {
super::access_owned::object_commit::authorize_complete_multipart_upload(
&mut request,
&server,
&bucket,
"object",
specific_etag,
)
.await
};
if allowed {
result.expect("permitted completion");
} else {
assert_eq!(result.expect_err("completion refused").code(), &S3ErrorCode::AccessDenied);
}
let mut envelope = builder.build();
let result = authorize_complete_multipart_upload(&mut envelope, &server, &bucket, "object", specific_etag).await;
if allowed {
result.expect("permitted completion");
} else {
assert_eq!(result.expect_err("completion refused").code(), &S3ErrorCode::AccessDenied);
}
assert_eq!(envelope.bucket_generation().is_some(), allowed);
assert_eq!(envelope.req_info().unwrap().bucket.as_deref(), Some(bucket.as_str()));
@@ -0,0 +1,740 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Which request values an operation's owned authorization hands the policy engine, and what a
//! stored identity or bucket policy decides on them: versioned tagging actions, the
//! `s3:ObjectCreationOperation` condition, legal hold, tagging, retention and POST form values
//! taken from the applied body instead of the wire headers, the decoded copy source, the
//! remaining-retention-days limit, and the replication proxy's tag-evidence gate.
//!
//! Responsible for: running the `storage::access_owned` functions on a `RequestEnvelope` with the
//! facts the gateway edge decodes, then evaluating the resulting conditions.
//! Not responsible for: decoding those facts from the wire (the gateway handlers own that), or
//! the condition builders in isolation (`storage::authorization` tests).
//! Upstream: `crate::storage::authorization::tests::fixture`. Downstream: `storage::access_owned`,
//! `storage::authorization` conditions, and the identity and bucket policy systems.
use crate::app::RequestEnvelope;
use crate::app::object_meta::tagging::proxy_allowed;
use crate::auth::facts::{AuthzPrincipal, AuthzTarget};
use crate::auth::{OBJECT_CREATION_OPERATION_CONDITION, OBJECT_LOCK_REMAINING_RETENTION_DAYS_CONDITION, get_condition_values};
use crate::storage::access_owned::object_commit::{
CopyObjectAuthorization, CopySourceObject, authorize_copy_object, authorize_upload_part_copy,
};
use crate::storage::access_owned::object_meta::{authorize_put_object_legal_hold, authorize_put_object_tagging};
use crate::storage::access_owned::object_write::{
PostObjectFormPolicy, PostObjectPolicyFields, PutObjectAuthorization, authorize_put_object,
};
use crate::storage::authorization::tests::fixture::request;
use crate::storage::authorization::{
AuthorizationRequest, ObjectTagConditions, authorization_conditions, authorization_conditions_with_internal,
request_authz_facts, set_requested_object_lock_retain_until,
};
use crate::storage::authorization_actions::versioned_tagging_action;
use crate::storage::request_markers::{
MultipartStagingOperation, RequestedObjectLegalHold, RequestedObjectLockRetention, RequestedObjectTags,
};
use crate::storage::storage_api::runtime_sources_consumer::ServerContextSlot;
use http::{HeaderValue, Method};
use rustfs_policy::policy::action::{Action, S3Action};
use rustfs_policy::policy::{Args, BucketPolicy, BucketPolicyArgs, Policy};
use std::collections::HashMap;
use time::OffsetDateTime;
use time::format_description::well_known::Rfc3339;
/// The conditions an anonymous principal's decision about `action` evaluates for `req`.
fn remaining_retention_days_conditions(req: &RequestEnvelope, action: Action) -> HashMap<String, Vec<String>> {
let credentials = rustfs_credentials::Credentials::default();
let principal = AuthzPrincipal::new(&credentials, "");
let facts = request_authz_facts(
req,
AuthzTarget {
action,
bucket: "",
object: "",
version_id: None,
location_constraint: None,
},
);
authorization_conditions(req, &principal, &facts).expect("conditions should build")
}
/// A PUT that asks to retain its object for `retain_for` from now.
fn request_retaining_for(retain_for: time::Duration) -> RequestEnvelope {
let mut req = request(Method::PUT).build();
let retain_until = OffsetDateTime::now_utc() + retain_for;
set_requested_object_lock_retain_until(&mut req, Some(retain_until));
req
}
async fn assert_requested_value_denied(action: Action, key: &str, value: &str, conditions: &HashMap<String, Vec<String>>) {
let document = serde_json::json!({
"Version": "2012-10-17",
"Statement": [
{"Effect": "Allow", "Action": action, "Resource": "arn:aws:s3:::bucket/*"},
{"Effect": "Deny", "Action": action, "Resource": "arn:aws:s3:::bucket/*",
"Condition": {"StringEquals": {key: value}}}
]
});
let policy = Policy::parse_config(&serde_json::to_vec(&document).unwrap()).unwrap();
let claims = HashMap::new();
let args = Args {
account: "writer",
groups: &None,
action,
bucket: "bucket",
conditions,
is_owner: false,
object: "object",
claims: &claims,
deny_only: false,
};
assert!(!policy.is_allowed(&args).await, "the applied request value must trigger Deny");
}
#[tokio::test]
async fn versioned_tagging_requires_version_permissions() {
let policy = Policy::parse_config(br#"{
"Version":"2012-10-17",
"Statement":[{"Effect":"Allow","Action":["s3:GetObjectTagging","s3:PutObjectTagging","s3:DeleteObjectTagging"],"Resource":"arn:aws:s3:::bucket/*"}]
}"#).expect("current-object tagging policy should parse");
let conditions = HashMap::new();
let claims = HashMap::new();
let groups = None;
for (current, versioned) in [
(S3Action::GetObjectTaggingAction, S3Action::GetObjectVersionTaggingAction),
(S3Action::PutObjectTaggingAction, S3Action::PutObjectVersionTaggingAction),
(S3Action::DeleteObjectTaggingAction, S3Action::DeleteObjectVersionTaggingAction),
] {
for version in [None, Some("null"), Some("0194e0f1-0000-7000-8000-000000000000")] {
let action = versioned_tagging_action(current, version);
assert_eq!(action, Action::S3Action(if version.is_some() { versioned } else { current }));
let args = Args {
account: "writer",
groups: &groups,
action,
bucket: "bucket",
conditions: &conditions,
is_owner: false,
object: "object",
claims: &claims,
deny_only: false,
};
assert_eq!(policy.is_allowed(&args).await, version.is_none(), "version selector {version:?}");
}
}
}
#[test]
fn object_creation_operation_classifies_only_multipart_staging() {
for (operation, staging) in [
("CreateMultipartUpload", true),
("UploadPart", true),
("UploadPartCopy", true),
("PutObject", false),
("CopyObject", false),
("CompleteMultipartUpload", false),
("PostObject", false),
("GetObject", false),
("InternalObjectWrite", false),
] {
assert_eq!(MultipartStagingOperation::for_operation(operation).is_some(), staging, "{operation}");
}
}
#[test]
fn object_creation_operation_ignores_request_and_claim_spoofing() {
for forged in ["false", "true"] {
for claim in [
"ObjectCreationOperation",
"objectcreationoperation",
"OBJECTCREATIONOPERATION",
"ldapObjectCreationOperation",
"ldapldapObjectCreationOperation",
] {
let credentials = rustfs_credentials::Credentials {
claims: Some(HashMap::from([(claim.to_string(), serde_json::json!(forged))])),
..Default::default()
};
let spoofed = || {
request(Method::GET)
.uri(&format!("/bucket/key?ObjectCreationOperation={forged}&uploadId=fake&partNumber=1"))
.header("objectcreationoperation", forged)
};
assert!(
!get_condition_values(spoofed().build().headers(), &credentials, None, None, None)
.keys()
.any(|key| key.eq_ignore_ascii_case(OBJECT_CREATION_OPERATION_CONDITION))
);
for staging in [false, true] {
let req = if staging {
spoofed().extension(MultipartStagingOperation).build()
} else {
spoofed().build()
};
// The envelope never carries the legacy internal marker; the internal-object
// edge (`authorize_owned_internal_object_request`) passes the flag explicitly.
for internal in [false, true] {
for action in [
S3Action::PutObjectAction,
S3Action::GetObjectAction,
S3Action::GetObjectVersionAction,
] {
let principal = AuthzPrincipal::new(&credentials, "");
let facts = request_authz_facts(
&req,
AuthzTarget {
action: Action::S3Action(action),
bucket: "",
object: "",
version_id: None,
location_constraint: None,
},
);
let conditions = authorization_conditions_with_internal(&req, &principal, &facts, internal)
.expect("conditions should build");
let values = conditions
.iter()
.filter(|(key, _)| key.eq_ignore_ascii_case(OBJECT_CREATION_OPERATION_CONDITION))
.collect::<Vec<_>>();
if action == S3Action::PutObjectAction {
assert_eq!(values.len(), 1, "one canonical server-derived value");
assert_eq!(values[0].0, OBJECT_CREATION_OPERATION_CONDITION);
assert_eq!(
values[0].1,
&vec![(internal || !staging).to_string()],
"{claim}, internal={internal}, staging={staging}"
);
} else {
assert!(values.is_empty(), "non-PutObject action {action:?} must have no creation condition");
}
}
}
}
}
}
}
#[tokio::test]
async fn object_creation_operation_controls_identity_and_bucket_policies() {
for effect in ["Allow", "Deny"] {
for expected in [false, true] {
let mut statements = vec![serde_json::json!({
"Effect": effect, "Action": "s3:PutObject", "Resource": "arn:aws:s3:::bucket/*",
"Condition": { "Bool": { "s3:ObjectCreationOperation": expected.to_string() } }
})];
if effect == "Deny" {
statements.push(serde_json::json!({
"Effect": "Allow", "Action": "s3:PutObject", "Resource": "arn:aws:s3:::bucket/*"
}));
}
let identity: Policy = serde_json::from_str(
&serde_json::json!({
"Version": "2012-10-17", "Statement": statements
})
.to_string(),
)
.expect("identity policy should accept the AWS Bool key");
for statement in &mut statements {
statement["Principal"] = serde_json::json!("*");
}
let bucket: BucketPolicy = serde_json::from_str(
&serde_json::json!({
"Version": "2012-10-17", "Statement": statements
})
.to_string(),
)
.expect("bucket policy should accept the AWS Bool key");
for actual in [None, Some(false), Some(true)] {
let conditions = actual
.map(|value| HashMap::from([("ObjectCreationOperation".to_string(), vec![value.to_string()])]))
.unwrap_or_default();
let allowed = if effect == "Allow" {
actual == Some(expected)
} else {
actual != Some(expected)
};
assert_eq!(
identity
.is_allowed(&Args {
account: "writer",
groups: &None,
action: Action::S3Action(S3Action::PutObjectAction),
bucket: "bucket",
object: "object",
conditions: &conditions,
is_owner: false,
claims: &HashMap::new(),
deny_only: false,
})
.await,
allowed,
"identity {effect}, expected={expected}, actual={actual:?}"
);
assert_eq!(
bucket
.is_allowed(&BucketPolicyArgs {
account: "writer",
groups: &None,
action: Action::S3Action(S3Action::PutObjectAction),
bucket: "bucket",
object: "object",
conditions: &conditions,
is_owner: false,
})
.await,
allowed,
"bucket {effect}, expected={expected}, actual={actual:?}"
);
}
}
}
}
#[test]
fn legal_hold_policy_checks_body_instead_of_conflicting_headers() {
let mut req = request(Method::PUT).header("x-amz-object-lock-legal-hold", "ON").build();
let action = Action::S3Action(S3Action::PutObjectLegalHoldAction);
// Without the applied body value the header leaks into the decision.
let conditions = remaining_retention_days_conditions(&req, action);
assert_eq!(conditions.get("object-lock-legal-hold"), Some(&vec!["ON".to_string()]));
// The XML body `<LegalHold><Status>OFF</Status></LegalHold>`, as the edge decodes it.
req.install_requested_legal_hold(RequestedObjectLegalHold(Some("OFF".to_string())));
let conditions = remaining_retention_days_conditions(&req, action);
assert_eq!(conditions.get("object-lock-legal-hold"), Some(&vec!["OFF".to_string()]));
// A body without a status stores no hold change; the header must not fill it in.
req.install_requested_legal_hold(RequestedObjectLegalHold::default());
let conditions = remaining_retention_days_conditions(&req, action);
assert!(!conditions.contains_key("object-lock-legal-hold"));
}
#[test]
fn tagging_policy_checks_body_instead_of_conflicting_headers() {
let mut req = request(Method::PUT).header("x-amz-tagging", "classification=public").build();
let action = Action::S3Action(S3Action::PutObjectTaggingAction);
// The XML tag set `classification=confidential`, as the edge decodes it.
req.install_requested_object_tags(RequestedObjectTags(vec![("classification".to_string(), "confidential".to_string())]));
let conditions = remaining_retention_days_conditions(&req, action);
assert_eq!(conditions.get("RequestObjectTag/classification"), Some(&vec!["confidential".to_string()]));
assert_eq!(conditions.get("RequestObjectTagKeys"), Some(&vec!["classification".to_string()]));
req.install_requested_object_tags(RequestedObjectTags::default());
let conditions = remaining_retention_days_conditions(&req, action);
assert!(!conditions.contains_key("RequestObjectTag/classification"));
assert_eq!(conditions.get("RequestObjectTagKeys"), Some(&Vec::new()));
}
#[test]
fn post_object_policy_checks_form_fields_instead_of_headers() {
let mut req = request(Method::POST)
.header("x-amz-object-lock-legal-hold", "OFF")
.header("x-amz-object-lock-mode", "GOVERNANCE")
.header("x-amz-object-lock-retain-until-date", "2029-01-01T00:00:00Z")
.header("x-amz-tagging", "classification=public")
.build();
PostObjectFormPolicy::new(PostObjectPolicyFields {
legal_hold: Some("ON"),
lock_mode: Some("COMPLIANCE"),
tagging: Some("classification=confidential"),
..Default::default()
})
.expect("form fields project onto policy headers")
.install(&mut req);
let action = Action::S3Action(S3Action::PutObjectAction);
let conditions = remaining_retention_days_conditions(&req, action);
assert_eq!(conditions.get("object-lock-legal-hold"), Some(&vec!["ON".to_string()]));
assert_eq!(conditions.get("object-lock-mode"), Some(&vec!["COMPLIANCE".to_string()]));
// The form sets no retain-until date, so the header must not stand in for one.
assert!(!conditions.contains_key("object-lock-retain-until-date"));
assert_eq!(conditions.get("RequestObjectTag/classification"), Some(&vec!["confidential".to_string()]));
// A PUT installs no form projection and keeps reading its headers.
let put = request(Method::PUT).header("x-amz-object-lock-legal-hold", "ON").build();
let conditions = remaining_retention_days_conditions(&put, action);
assert_eq!(conditions.get("object-lock-legal-hold"), Some(&vec!["ON".to_string()]));
}
#[test]
fn gateway_post_policy_projects_effective_write_conditions() {
let mut req = request(Method::POST)
.header("if-none-match", "*")
.header("if-none-match", "\"second\"")
.header("x-amz-server-side-encryption", "wire")
.header("x-amz-server-side-encryption-customer-algorithm", "AES256")
.build();
PostObjectFormPolicy::new(PostObjectPolicyFields {
if_match: Some(HeaderValue::from_static("\"form-only\"")),
if_none_match: Some(HeaderValue::from_static("\"form-only\"")),
encryption: Some("form"),
..Default::default()
})
.expect("valid form policy")
.with_metadata(&[("purpose".to_owned(), "form".to_owned())])
.expect("valid form metadata")
.install_gateway(&mut req);
let projected = &req.post_object_policy_headers().expect("projected headers").0;
assert!(!projected.contains_key("if-match"));
assert_eq!(projected.get("if-none-match"), Some(&HeaderValue::from_static("*")));
assert_eq!(projected.get("x-amz-server-side-encryption"), Some(&HeaderValue::from_static("form")));
assert_eq!(projected.get("x-amz-meta-purpose"), Some(&HeaderValue::from_static("form")));
assert_eq!(
projected.get("x-amz-server-side-encryption-customer-algorithm"),
Some(&HeaderValue::from_static("AES256"))
);
assert_eq!(req.headers().get_all("if-none-match").iter().count(), 2);
assert_eq!(req.headers().get("x-amz-server-side-encryption"), Some(&HeaderValue::from_static("wire")));
}
#[tokio::test]
async fn copy_source_conditions_use_the_decoded_source_for_both_copy_operations() {
let mut document = serde_json::json!({"Version":"2012-10-17","Statement":[
{"Effect":"Allow","Action":"s3:PutObject","Resource":"arn:aws:s3:::bucket/*"},
{"Effect":"Deny","Action":"s3:PutObject","Resource":"arn:aws:s3:::bucket/*",
"Condition":{"StringLike":{"s3:x-amz-copy-source":"secret/*"}}}
]});
let identity = Policy::parse_config(&serde_json::to_vec(&document).expect("encode identity policy")).expect("parse policy");
for statement in document["Statement"].as_array_mut().expect("statements") {
statement["Principal"] = serde_json::json!("*");
}
let bucket: BucketPolicy =
serde_json::from_slice(&serde_json::to_vec(&document).expect("encode bucket policy")).expect("parse bucket policy");
// Each header spelling with the source the request parser decodes from it.
for (spelling, source_bucket, source_key, source_version) in [
("secret/report.txt", "secret", "report.txt", None),
("%73ecret/report.txt", "secret", "report.txt", None),
("secret%2Freport.txt", "secret", "report.txt", None),
("/secret/report.txt", "secret", "report.txt", None),
(
"%2Fsecret%2Freport.txt?versionId=00000000-0000-0000-0000-000000000000",
"secret",
"report.txt",
Some("00000000-0000-0000-0000-000000000000"),
),
("public/report.txt", "public", "report.txt", None),
("/public%2Freport.txt", "public", "report.txt", None),
] {
for multipart in [false, true] {
let source = CopySourceObject {
bucket: source_bucket,
key: source_key,
version_id: source_version,
};
let server = ServerContextSlot::new();
let mut req = request(Method::PUT).req_info().build();
if multipart {
let _ = authorize_upload_part_copy(&mut req, &server, "bucket", "copy", Some(source)).await;
} else {
let facts = CopyObjectAuthorization {
bucket: "bucket",
key: "copy",
source: Some(source),
query_version: None,
tagging: None,
tagging_directive: None,
retain_until: None,
legal_hold: false,
retention: false,
};
let _ = authorize_copy_object(&mut req, &server, facts).await;
}
req.headers_mut()
.insert("x-amz-copy-source", HeaderValue::from_str(spelling).expect("source header"));
let action = Action::S3Action(S3Action::PutObjectAction);
let conditions = remaining_retention_days_conditions(&req, action);
let canonical = &conditions["x-amz-copy-source"];
assert_eq!(canonical.len(), 1, "one semantic source, independent of header spelling");
let expected = if spelling.contains("public") {
"public/report.txt"
} else if spelling.contains('?') {
"secret/report.txt?versionId=null"
} else {
"secret/report.txt"
};
assert_eq!(canonical[0], expected, "{spelling}, multipart={multipart}");
let allowed = expected.starts_with("public/");
assert_eq!(
identity
.is_allowed(&Args {
account: "user",
groups: &None,
action,
bucket: "bucket",
object: "copy",
conditions: &conditions,
claims: &HashMap::new(),
is_owner: false,
deny_only: false
})
.await,
allowed
);
assert_eq!(
bucket
.is_allowed(&BucketPolicyArgs {
account: "user",
groups: &None,
action,
bucket: "bucket",
object: "copy",
conditions: &conditions,
is_owner: false
})
.await,
allowed
);
}
}
}
#[tokio::test]
async fn legal_hold_policy_checks_xml_body() {
let action = Action::S3Action(S3Action::PutObjectLegalHoldAction);
for header in [None, Some("ON")] {
let mut builder = request(Method::PUT).req_info();
if let Some(header) = header {
builder = builder.header("x-amz-object-lock-legal-hold", header);
}
let mut req = builder.build();
let server = ServerContextSlot::new();
// The XML body `<LegalHold><Status>OFF</Status></LegalHold>`, as the edge decodes it.
let _ = authorize_put_object_legal_hold(
&mut req,
&server,
"bucket",
"object",
None,
RequestedObjectLegalHold(Some("OFF".to_string())),
)
.await;
let conditions = remaining_retention_days_conditions(&req, action);
assert_eq!(conditions.get("object-lock-legal-hold"), Some(&vec!["OFF".to_string()]));
assert_requested_value_denied(action, "s3:object-lock-legal-hold", "OFF", &conditions).await;
}
}
#[tokio::test]
async fn tagging_policy_checks_xml_body() {
for version in [None, Some("version")] {
let mut req = request(Method::PUT)
.req_info()
.header("x-amz-tagging", "classification=secret&spoofed=yes")
.build();
// The XML tag set `classification=public`, as the edge decodes it.
let _ = authorize_put_object_tagging(
&mut req,
"bucket",
"object",
version,
RequestedObjectTags(vec![("classification".to_string(), "public".to_string())]),
)
.await;
let action = versioned_tagging_action(S3Action::PutObjectTaggingAction, version);
let conditions = remaining_retention_days_conditions(&req, action);
assert_eq!(conditions.get("RequestObjectTag/classification"), Some(&vec!["public".to_string()]));
assert_eq!(conditions.get("RequestObjectTagKeys"), Some(&vec!["classification".to_string()]));
assert!(!conditions.contains_key("RequestObjectTag/spoofed"));
assert_requested_value_denied(action, "s3:RequestObjectTag/classification", "public", &conditions).await;
}
}
#[tokio::test]
async fn post_object_policy_checks_form_fields() {
let mut req = request(Method::POST)
.req_info()
.header("x-amz-object-lock-mode", "GOVERNANCE")
.header("x-amz-object-lock-legal-hold", "OFF")
.header("x-amz-object-lock-retain-until-date", "2029-01-01T00:00:00Z")
.header("x-amz-storage-class", "STANDARD")
.header("x-amz-tagging", "classification=secret&spoofed=yes")
.header("x-amz-acl", "public-read")
.build();
let retain_until = OffsetDateTime::parse("2035-01-01T00:00:00Z", &Rfc3339).unwrap();
let server = ServerContextSlot::new();
// The form fields as the POST edge decodes them; the form names no ACL.
let input = PutObjectAuthorization {
bucket: "bucket",
key: "object",
version_id: None,
retain_until: Some(retain_until),
specific_etag: false,
legal_hold: true,
retention: true,
post: true,
};
let _ = authorize_put_object(&mut req, &server, input, |req| {
PostObjectFormPolicy::new(PostObjectPolicyFields {
lock_mode: Some("COMPLIANCE"),
legal_hold: Some("ON"),
retain_until: Some(retain_until),
storage_class: Some("STANDARD_IA"),
tagging: Some("classification=public"),
..Default::default()
})?
.install(req);
Ok(())
})
.await;
let action = Action::S3Action(S3Action::PutObjectAction);
let conditions = remaining_retention_days_conditions(&req, action);
for (key, value) in [
("object-lock-mode", "COMPLIANCE"),
("object-lock-legal-hold", "ON"),
("object-lock-retain-until-date", "2035-01-01T00:00:00Z"),
("x-amz-storage-class", "STANDARD_IA"),
("RequestObjectTag/classification", "public"),
] {
assert_eq!(conditions.get(key), Some(&vec![value.to_string()]));
}
assert!(!conditions.contains_key("x-amz-acl"));
assert!(!conditions.contains_key("RequestObjectTag/spoofed"));
assert_requested_value_denied(action, "s3:object-lock-mode", "COMPLIANCE", &conditions).await;
assert_requested_value_denied(action, "s3:RequestObjectTag/classification", "public", &conditions).await;
// The final PostObject check reads the same request, so it sees the same form values.
let post_conditions = remaining_retention_days_conditions(&req, action);
assert_eq!(post_conditions.get("object-lock-mode"), conditions.get("object-lock-mode"));
assert_eq!(
post_conditions.get("RequestObjectTag/classification"),
conditions.get("RequestObjectTag/classification")
);
}
#[tokio::test]
async fn retention_policy_checks_body_instead_of_conflicting_headers() {
let mut req = request(Method::PUT)
.header("x-amz-object-lock-mode", "GOVERNANCE")
.header("x-amz-object-lock-retain-until-date", "2029-01-01T00:00:00Z")
.build();
// The XML body's retention, as the edge decodes it.
let applied = OffsetDateTime::parse("2035-01-02T00:00:00+23:59", &Rfc3339).expect("body date should parse");
let retention = RequestedObjectLockRetention {
mode: Some("COMPLIANCE".to_string()),
retain_until: Some(applied),
};
set_requested_object_lock_retain_until(&mut req, retention.retain_until);
req.install_requested_retention(retention);
let action = Action::S3Action(S3Action::PutObjectRetentionAction);
let conditions = remaining_retention_days_conditions(&req, action);
assert_eq!(conditions.get("object-lock-mode"), Some(&vec!["COMPLIANCE".to_string()]));
let authorized =
OffsetDateTime::parse(&conditions["object-lock-retain-until-date"][0], &Rfc3339).expect("condition date should parse");
assert_eq!(authorized.unix_timestamp(), applied.unix_timestamp());
let policy = Policy::parse_config(
br#"{
"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"s3:PutObjectRetention","Resource":"arn:aws:s3:::bucket/*",
"Condition":{"DateLessThan":{"s3:object-lock-retain-until-date":"2030-01-01T00:00:00Z"}}}]
}"#,
)
.expect("bounded retention policy should parse");
let args = Args {
account: "writer",
groups: &None,
action,
bucket: "bucket",
conditions: &conditions,
is_owner: false,
object: "object",
claims: &HashMap::new(),
deny_only: false,
};
assert!(
!policy.is_allowed(&args).await,
"an in-range header cannot authorize an out-of-range body"
);
// A body without retention applies none; the headers must not fill it in.
let retention = RequestedObjectLockRetention::default();
set_requested_object_lock_retain_until(&mut req, retention.retain_until);
req.install_requested_retention(retention);
let conditions = remaining_retention_days_conditions(&req, action);
assert!(!conditions.contains_key("object-lock-mode"));
assert!(!conditions.contains_key("object-lock-retain-until-date"));
}
#[tokio::test]
async fn bucket_policy_limits_retention_to_thirty_days() {
let policy: BucketPolicy = serde_json::from_str(
r#"{
"Version":"2012-10-17",
"Statement":[
{
"Effect":"Allow",
"Principal":{"AWS":"*"},
"Action":["s3:PutObjectRetention"],
"Resource":["arn:aws:s3:::bucket/*"]
},
{
"Effect":"Deny",
"Principal":{"AWS":"*"},
"Action":["s3:PutObjectRetention"],
"Resource":["arn:aws:s3:::bucket/*"],
"Condition":{"NumericGreaterThan":{"s3:object-lock-remaining-retention-days":"30"}}
}
]
}"#,
)
.expect("bucket policy should parse");
let no_groups: Option<Vec<String>> = None;
let action = Action::S3Action(S3Action::PutObjectRetentionAction);
let spoofed_header = |mut req: RequestEnvelope| {
req.headers_mut()
.insert(OBJECT_LOCK_REMAINING_RETENTION_DAYS_CONDITION, HeaderValue::from_static("1"));
req
};
let cases = [
("29 days", request_retaining_for(time::Duration::days(29)), true),
("30 days", request_retaining_for(time::Duration::days(30)), true),
("31 days", request_retaining_for(time::Duration::days(31)), false),
(
"30 days and a minute",
request_retaining_for(time::Duration::days(30) + time::Duration::MINUTE),
false,
),
("no retention", request(Method::PUT).build(), true),
(
"31 days with spoofed header",
spoofed_header(request_retaining_for(time::Duration::days(31))),
false,
),
];
for (name, req, allowed) in cases {
let conditions = remaining_retention_days_conditions(&req, action);
let args = BucketPolicyArgs {
bucket: "bucket",
action,
is_owner: false,
account: "",
groups: &no_groups,
conditions: &conditions,
object: "obj",
};
assert_eq!(policy.is_allowed(&args).await, allowed, "{name}");
}
}
#[test]
fn replication_proxy_requires_authorization_independent_of_local_tags() {
let mut req = request(Method::PUT).build();
assert!(proxy_allowed(&req));
// A local miss produces an empty tag map, but still establishes that the
// policy needs tags. Neither anonymous nor authenticated callers may proxy.
req.install_object_tag_conditions(ObjectTagConditions::new("bucket", "missing", None, HashMap::new()));
assert!(!proxy_allowed(&req));
req.install_object_tag_conditions(ObjectTagConditions::new(
"bucket",
"missing",
Some("version"),
HashMap::from([("ExistingObjectTag/protected".to_string(), vec!["yes".to_string()])]),
));
assert!(!proxy_allowed(&req));
}
@@ -12,20 +12,22 @@
// See the License for the specific language governing permissions and
// limitations under the License.
//! Both request forms through the one decision: the same answer and the same state handed on
//! The DeleteObject decision on the owned request envelope: the answer and the state handed on
//! (the caller's `ReqInfo` and the guard), allowed and refused.
//!
//! Tests: `decide_delete_object` for anonymous, policy-less, IAM and owner callers, the headers
//! that add actions, and replica deletes. Not tested: the delete handler or the request decoder.
//! Upstream: `storage::authorization::tests::fixture`. Downstream: none.
use super::access_owned::object_misc::decide_delete_object;
use crate::app::RequestEnvelope;
use crate::storage::access::{
AuthorizationRequest, BucketGenerationGuard, PendingDeleteBucketGenerationGuard, ReqInfo, S3Request,
};
use crate::storage::access_owned::object_misc::decide_delete_object;
use crate::storage::authorization::tests::fixture::{TestKms, UnreadyIam, request};
use crate::storage::authorization::{AuthorizationRequest, BucketGenerationGuard, PendingDeleteBucketGenerationGuard};
use crate::storage::req_info::ReqInfo;
use crate::storage::storage_api::contract::bucket::{BucketOperations as _, MakeBucketOptions};
use crate::storage::storage_api::runtime_sources_consumer::{AppContext, IamInterface, KmsInterface, ServerContextSlot};
use http::{Extensions, HeaderMap, HeaderValue, Method, Uri};
use crate::storage::storage_api::runtime_sources_consumer::{AppContext, IamInterface, ServerContextSlot};
use http::{HeaderMap, Method};
use rustfs_iam::store::{Store as _, UserType};
use rustfs_iam::{store::object::ObjectStore, sys::IamSys};
use rustfs_kms::KmsServiceManager;
use rustfs_madmin::{AccountStatus, AddOrUpdateUserReq};
use rustfs_policy::policy::Policy;
use rustfs_s3_types::{S3Error, S3ErrorCode, S3Result};
@@ -167,28 +169,20 @@ fn cases() -> Vec<Case> {
]
}
fn request(case: &Case, server_ctx: &Arc<ServerContextSlot>) -> S3Request<()> {
fn case_request(case: &Case, server_ctx: &Arc<ServerContextSlot>) -> crate::app::RequestEnvelope {
let mut headers = HeaderMap::new();
insert_header(&mut headers, SUFFIX_SOURCE_VERSION_ID, SOURCE_VERSION);
let mut builder = request(Method::DELETE)
.uri("/e2-delete-parity/key")
.headers(headers)
.server_context(server_ctx);
for (name, value) in case.headers {
headers.insert(*name, HeaderValue::from_static(value));
builder = builder.header(name, value);
}
let mut extensions = Extensions::new();
if let Some(caller) = &case.caller {
extensions.insert(caller.clone());
}
extensions.insert(Arc::clone(server_ctx));
S3Request {
input: (),
method: Method::DELETE,
uri: Uri::from_static("/e2-delete-parity/key"),
headers,
extensions,
credentials: None,
region: None,
service: None,
trailing_headers: None,
builder = builder.extension(caller.clone());
}
builder.build()
}
async fn decide<R: AuthorizationRequest + ?Sized>(req: &mut R, bucket: &str, case: &Case) -> Result<Handed, S3ErrorCode> {
@@ -203,19 +197,14 @@ async fn decide<R: AuthorizationRequest + ?Sized>(req: &mut R, bucket: &str, cas
})
}
/// Runs every case in both request forms; one run names every case a defect breaks.
/// Runs every case; one run names every case a defect breaks.
async fn assert_cases(cases: Vec<Case>, bucket: &str, server_ctx: &Arc<ServerContextSlot>) {
let mut wrong = Vec::new();
for case in cases {
let mut legacy = request(&case, server_ctx);
let mut envelope = RequestEnvelope::from_s3s(&legacy);
for (form, answer) in [
("legacy request", decide(&mut legacy, bucket, &case).await),
("envelope", decide(&mut envelope, bucket, &case).await),
] {
if answer != case.expected {
wrong.push(format!("{} ({form}): {answer:?}", case.name));
}
let mut envelope = case_request(&case, server_ctx);
let answer = decide(&mut envelope, bucket, &case).await;
if answer != case.expected {
wrong.push(format!("{}: {answer:?}", case.name));
}
}
assert!(wrong.is_empty(), "{wrong:#?}");
@@ -375,23 +364,3 @@ impl IamInterface for ReadyIam {
true
}
}
struct UnreadyIam;
impl IamInterface for UnreadyIam {
fn handle(&self) -> Arc<IamSys<ObjectStore>> {
panic!("a bucket-policy decision does not resolve IAM")
}
fn is_ready(&self) -> bool {
false
}
}
struct TestKms;
impl KmsInterface for TestKms {
fn handle(&self) -> Arc<KmsServiceManager> {
Arc::new(KmsServiceManager::new())
}
}
@@ -12,13 +12,18 @@
// See the License for the specific language governing permissions and
// limitations under the License.
//! Read authorization tests for the legacy request and owned envelope.
//! Read authorization tests on the owned request envelope.
//!
//! Tests: `authorize_head_object`, `authorize_get_object` and `authorize_get_object_attributes`
//! against bucket policies, and the resource each leaves in `ReqInfo`. Not tested: the read
//! handlers or the request decoder.
//! Upstream: `storage::authorization::tests::fixture`. Downstream: none.
use super::access_owned::object_read::authorize_head_object;
use crate::app::RequestEnvelope;
use crate::storage::access::{ReqInfo, S3Request, ServerContextSlot};
use crate::storage::access_owned::object_read::{authorize_get_object, authorize_get_object_attributes, authorize_head_object};
use crate::storage::authorization::tests::fixture::request;
use crate::storage::storage_api::contract::bucket::{BucketOperations as _, MakeBucketOptions};
use http::{Extensions, HeaderMap, HeaderValue, Method, Uri};
use crate::storage::storage_api::runtime_sources_consumer::ServerContextSlot;
use http::Method;
use rustfs_s3_types::S3ErrorCode;
use std::sync::Arc;
@@ -62,23 +67,11 @@ fn head_replication_marker_requires_its_own_permission() {
]
};
for header in headers {
let mut request = S3Request {
input: (),
method: Method::HEAD,
uri: Uri::from_static("/bucket/key"),
headers: HeaderMap::new(),
extensions: Extensions::new(),
credentials: None,
region: None,
service: None,
trailing_headers: None,
};
request.extensions.insert(ReqInfo::default());
request.extensions.insert(Arc::clone(&slot));
let mut builder = request(Method::HEAD).uri("/bucket/key").req_info().server_context(&slot);
if let Some(header) = header {
request.headers.insert(header, HeaderValue::from_static("true"));
builder = builder.header(header, "true");
}
let mut envelope = RequestEnvelope::from_s3s(&request);
let mut envelope = builder.build();
let result = authorize_head_object(&slot, &mut envelope, &bucket, "key", None).await;
let permitted = header.is_none() || (allow && !deny);
if permitted {
@@ -128,41 +121,23 @@ fn get_authorization_binds_version_and_explicit_denial() {
.into_iter()
.flat_map(|version| ["key", "other"].map(|key| (version, key)))
{
let mut request = S3Request {
input: (),
method: Method::GET,
uri: Uri::from_static("/bucket/key"),
headers: HeaderMap::new(),
extensions: Extensions::new(),
credentials: None,
region: None,
service: None,
trailing_headers: None,
};
request.extensions.insert(ReqInfo::default());
request.extensions.insert(Arc::clone(&slot));
let mut envelope = RequestEnvelope::from_s3s(&request);
let legacy =
super::access_owned::object_read::authorize_get_object(&slot, &mut request, &bucket, key, version).await;
let native =
super::access_owned::object_read::authorize_get_object(&slot, &mut envelope, &bucket, key, version).await;
let mut envelope = request(Method::GET)
.uri("/bucket/key")
.req_info()
.server_context(&slot)
.build();
let result = authorize_get_object(&slot, &mut envelope, &bucket, key, version).await;
let allowed = key == "key" && if version.is_some() { version_allowed } else { current_allowed };
for (edge, result) in [("legacy", legacy), ("native", native)] {
let label = format!("actions={actions:?} deny={explicit_deny} version={version:?} key={key} edge={edge}");
let expected = if allowed { None } else { Some(&S3ErrorCode::AccessDenied) };
if result.as_ref().err().map(|error| error.code()) != expected {
failures.push(format!("{label}: expected {expected:?}, got {result:?}"));
}
let label = format!("actions={actions:?} deny={explicit_deny} version={version:?} key={key}");
let expected = if allowed { None } else { Some(&S3ErrorCode::AccessDenied) };
if result.as_ref().err().map(|error| error.code()) != expected {
failures.push(format!("{label}: expected {expected:?}, got {result:?}"));
}
for (edge, info) in [
("legacy", request.extensions.get::<ReqInfo>().unwrap()),
("native", envelope.req_info().unwrap()),
] {
let actual = (info.bucket.as_deref(), info.object.as_deref(), info.version_id.as_deref());
let expected = (Some(bucket.as_str()), Some(key), version);
if actual != expected {
failures.push(format!("{edge}: expected resource {expected:?}, got {actual:?}"));
}
let info = envelope.req_info().unwrap();
let actual = (info.bucket.as_deref(), info.object.as_deref(), info.version_id.as_deref());
let expected = (Some(bucket.as_str()), Some(key), version);
if actual != expected {
failures.push(format!("{label}: expected resource {expected:?}, got {actual:?}"));
}
}
}
@@ -173,12 +148,10 @@ fn get_authorization_binds_version_and_explicit_denial() {
#[test]
#[serial_test::serial]
fn attributes_authorization_requires_each_action_on_the_original_version() {
use super::super::access_owned::object_read::authorize_get_object_attributes;
crate::app::gating_test_env::run_large_stack_test("attributes-authorization", || async {
let context = crate::app::gating_test_env::shared_gating_ambient().await;
let store = context.object_store();
let slot = std::sync::Arc::new(ServerContextSlot::new());
let slot = ServerContextSlot::new();
assert!(slot.install(Arc::clone(&context)));
for (version, required) in [
(None, ["s3:GetObjectAttributes", "s3:GetObject"]),
@@ -222,37 +195,23 @@ fn attributes_authorization_requires_each_action_on_the_original_version() {
)
.await
.unwrap();
let mut request = S3Request {
input: (),
method: Method::GET,
uri: Uri::from_static("/bucket/key?attributes"),
headers: HeaderMap::new(),
extensions: Extensions::new(),
credentials: None,
region: None,
service: None,
trailing_headers: None,
};
request.extensions.insert(ReqInfo::default());
request.extensions.insert(Arc::clone(&slot));
let mut envelope = RequestEnvelope::from_s3s(&request);
let legacy = authorize_get_object_attributes(&mut request, &bucket, key, version).await;
let native = authorize_get_object_attributes(&mut envelope, &bucket, key, version).await;
let mut envelope = request(Method::GET)
.uri("/bucket/key?attributes")
.req_info()
.server_context(&slot)
.build();
let result = authorize_get_object_attributes(&mut envelope, &bucket, key, version).await;
let expected = (!permitted).then_some(&S3ErrorCode::AccessDenied);
for (edge, result, info) in [
("legacy", legacy, request.extensions.get::<ReqInfo>().unwrap()),
("native", native, envelope.req_info().unwrap()),
] {
assert_eq!(
result.as_ref().err().map(|error| error.code()),
expected,
"edge={edge}, allow={allowed:?}, deny={denied:?}, key={key}, version={version:?}",
);
assert_eq!(
(info.bucket.as_deref(), info.object.as_deref(), info.version_id.as_deref()),
(Some(bucket.as_str()), Some(key), version)
);
}
let info = envelope.req_info().unwrap();
assert_eq!(
result.as_ref().err().map(|error| error.code()),
expected,
"allow={allowed:?}, deny={denied:?}, key={key}, version={version:?}",
);
assert_eq!(
(info.bucket.as_deref(), info.object.as_deref(), info.version_id.as_deref()),
(Some(bucket.as_str()), Some(key), version)
);
}
}
});
@@ -0,0 +1,323 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Replication requests against the owned per-operation authorization: a replication HEAD needs
//! read permission before replication permission, and a replica write or delete is authorized
//! against the version its replication header names.
//!
//! Responsible for: the allow/deny decisions and the `ReqInfo` version that `authorize_head_object`,
//! `authorize_put_object`, `authorize_copy_object`, `authorize_create_multipart_upload` and
//! `authorize_delete_object` reach for replication requests, against real bucket policies.
//! Not responsible for: decoding the replication headers from the wire, or the storage use cases
//! that run after authorization.
//! Upstream: the owned request fixture and the gating test store. Downstream: `storage::access_owned`.
use crate::storage::access_owned::listing_multipart::{CreateMultipartAuthorization, authorize_create_multipart_upload};
use crate::storage::access_owned::object_commit::{CopyObjectAuthorization, CopySourceObject, authorize_copy_object};
use crate::storage::access_owned::object_misc::authorize_delete_object;
use crate::storage::access_owned::object_read::authorize_head_object;
use crate::storage::access_owned::object_write::{PutObjectAuthorization, authorize_put_object};
use crate::storage::authorization::replication_request_authorized;
use crate::storage::authorization::tests::fixture::{request, server_context};
use crate::storage::storage_api::contract::bucket::{BucketOperations as _, MakeBucketOptions};
use crate::storage::storage_api::contract::object::{ObjectIO as _, ObjectOperations as _};
use http::Method;
use rustfs_s3_types::S3ErrorCode;
#[test]
fn replication_head_checks_read_permissions_before_replication_permissions() {
crate::app::gating_test_env::run_large_stack_test("replication-head-permissions", || async {
let store = crate::app::gating_test_env::shared_gating_ecstore().await;
let bucket = format!("replica-head-{}", uuid::Uuid::new_v4());
store
.make_bucket(&bucket, &MakeBucketOptions::default())
.await
.expect("create bucket");
let slot = server_context(store.clone());
for (grant_read, deny_read) in [(false, false), (true, true), (true, false)] {
let mut statements = vec![
serde_json::json!({"Effect":"Allow","Principal":{"AWS":"*"},"Action":"s3:ReplicateObject","Resource":format!("arn:aws:s3:::{bucket}/*")}),
];
for (enabled, effect) in [(grant_read, "Allow"), (deny_read, "Deny")] {
if enabled {
statements.push(serde_json::json!({"Effect":effect,"Principal":{"AWS":"*"},"Action":["s3:GetObject","s3:GetObjectVersion"],"Resource":format!("arn:aws:s3:::{bucket}/*")}));
}
}
store
.update_bucket_metadata_config(
&bucket,
crate::storage::storage_api::ecstore_bucket::metadata::BUCKET_POLICY_CONFIG,
serde_json::to_vec(&serde_json::json!({"Version":"2012-10-17","Statement":statements})).unwrap(),
)
.await
.expect("publish policy");
for version in [None, Some("null"), Some("12345678-1234-1234-1234-123456789abc")] {
for header in ["x-rustfs-source-replication-check", "x-minio-source-replication-check"] {
let mut envelope = request(Method::HEAD)
.req_info()
.server_context(&slot)
.header(header, "true")
.build();
let result = authorize_head_object(&slot, &mut envelope, &bucket, "object", version).await;
if grant_read && !deny_read {
result.expect("read and replication permissions allow metadata check");
assert!(replication_request_authorized(&envelope));
} else {
assert_eq!(
result.expect_err("replication cannot replace read permission").code(),
&S3ErrorCode::AccessDenied
);
assert!(!replication_request_authorized(&envelope));
}
}
}
}
});
}
#[test]
fn replication_write_hooks_authorize_the_destination_version_and_tags() {
crate::app::gating_test_env::run_large_stack_test("replication-version-permissions", || async {
let store = crate::app::gating_test_env::shared_gating_ecstore().await;
let bucket = format!("replica-write-{}", uuid::Uuid::new_v4());
store
.make_bucket(
&bucket,
&MakeBucketOptions {
versioning_enabled: true,
..Default::default()
},
)
.await
.expect("create versioned bucket");
let slot = server_context(store.clone());
let mut versions = Vec::new();
for (body, tags) in [
(b"secret".to_vec(), "classification=secret"),
(b"public".to_vec(), "classification=public"),
] {
let object = store
.put_object(
&bucket,
"target",
&mut crate::storage::PutObjReader::from_vec(body),
&crate::storage::ObjectOptions {
versioned: true,
..Default::default()
},
)
.await
.expect("seed version");
let version = object.version_id.expect("UUID version").to_string();
store
.put_object_tags(
&bucket,
"target",
tags,
&crate::storage::ObjectOptions {
version_id: Some(version.clone()),
versioned: true,
..Default::default()
},
)
.await
.expect("tag version");
versions.push(version);
}
for condition in [
serde_json::json!({"s3:versionid":versions[0]}),
serde_json::json!({"s3:ExistingObjectTag/classification":"secret"}),
] {
let policy = serde_json::json!({"Version":"2012-10-17","Statement":[
{"Effect":"Allow","Principal":{"AWS":"*"},"Action":"s3:*","Resource":format!("arn:aws:s3:::{bucket}/*")},
{"Effect":"Deny","Principal":{"AWS":"*"},"Action":"s3:PutObject","Resource":format!("arn:aws:s3:::{bucket}/target"),"Condition":{"StringEquals":condition}}
]});
store
.update_bucket_metadata_config(
&bucket,
crate::storage::storage_api::ecstore_bucket::metadata::BUCKET_POLICY_CONFIG,
serde_json::to_vec(&policy).unwrap(),
)
.await
.expect("publish version policy");
for (index, version) in versions.iter().enumerate() {
for header in [None, Some("x-rustfs-source-version-id"), Some("x-minio-source-version-id")] {
for operation in ["put", "copy", "multipart"] {
let query_version = header.is_none().then(|| version.clone());
// A query version alone also requires replication authorization.
let mut builder = request(Method::PUT)
.req_info()
.server_context(&slot)
.header("x-rustfs-source-replication-request", "true");
if let Some(header) = header {
builder = builder.header(header, version);
}
let mut envelope = builder.build();
let result = match operation {
"put" => {
let facts = PutObjectAuthorization {
bucket: &bucket,
key: "target",
version_id: query_version.as_deref(),
retain_until: None,
specific_etag: false,
legal_hold: false,
retention: false,
post: false,
};
authorize_put_object(&mut envelope, &slot, facts, |_| Ok(())).await
}
"copy" => {
// `x-amz-copy-source: /{bucket}/source`, no tagging, no Object Lock.
let facts = CopyObjectAuthorization {
bucket: &bucket,
key: "target",
source: Some(CopySourceObject {
bucket: &bucket,
key: "source",
version_id: None,
}),
query_version: query_version.as_deref(),
tagging: None,
tagging_directive: None,
retain_until: None,
legal_hold: false,
retention: false,
};
authorize_copy_object(&mut envelope, &slot, facts).await
}
_ => {
let facts = CreateMultipartAuthorization {
bucket: &bucket,
key: "target",
query_version: query_version.as_deref(),
retain_until: None,
legal_hold_present: false,
retention_present: false,
};
authorize_create_multipart_upload(&slot, &mut envelope, facts).await
}
};
let info = envelope.req_info().unwrap().clone();
assert_eq!(info.version_id.as_deref(), Some(version.as_str()), "{operation}");
if index == 0 {
assert_eq!(
result.expect_err("protected destination version must be denied").code(),
&S3ErrorCode::AccessDenied,
"{operation}"
);
} else {
result.expect("unprotected destination version remains authorized");
}
}
}
}
}
});
}
#[test]
fn replication_delete_hook_authorizes_the_header_selected_version() {
crate::app::gating_test_env::run_large_stack_test("replication-delete-version-permissions", || async {
let store = crate::app::gating_test_env::shared_gating_ecstore().await;
let bucket = format!("replica-delete-{}", uuid::Uuid::new_v4());
store
.make_bucket(
&bucket,
&MakeBucketOptions {
versioning_enabled: true,
..Default::default()
},
)
.await
.expect("create versioned bucket");
let slot = server_context(store.clone());
let mut versions = Vec::new();
for body in [b"protected".to_vec(), b"disposable".to_vec()] {
let object = store
.put_object(
&bucket,
"target",
&mut crate::storage::PutObjReader::from_vec(body),
&crate::storage::ObjectOptions {
versioned: true,
..Default::default()
},
)
.await
.expect("seed version");
versions.push(object.version_id.expect("UUID version").to_string());
}
let policy = serde_json::json!({"Version":"2012-10-17","Statement":[
{"Effect":"Allow","Principal":{"AWS":"*"},"Action":"s3:*","Resource":format!("arn:aws:s3:::{bucket}/*")},
{"Effect":"Deny","Principal":{"AWS":"*"},"Action":"s3:DeleteObjectVersion","Resource":format!("arn:aws:s3:::{bucket}/target"),"Condition":{"StringEquals":{"s3:versionid":versions[0]}}}
]});
store
.update_bucket_metadata_config(
&bucket,
crate::storage::storage_api::ecstore_bucket::metadata::BUCKET_POLICY_CONFIG,
serde_json::to_vec(&policy).unwrap(),
)
.await
.expect("publish version policy");
// The version named only by the replication header must be the one
// the policy sees: a plain `s3:DeleteObject` evaluation would let a
// replica delete remove the protected version. A request without
// the REPLICA status is authorized for the query version alone,
// because storage ignores the header for it (a stray header must
// not switch the evaluated action to `s3:DeleteObjectVersion`).
for (header, version, replica, expect_denied) in [
("x-rustfs-source-version-id", &versions[0], true, true),
("x-minio-source-version-id", &versions[0], true, true),
("x-rustfs-source-version-id", &versions[0], false, false),
("x-rustfs-source-version-id", &versions[1], true, false),
] {
let mut builder = request(Method::DELETE).req_info().server_context(&slot);
if replica {
builder = builder
.header("x-rustfs-source-replication-request", "true")
.header(rustfs_utils::http::headers::AMZ_BUCKET_REPLICATION_STATUS, "REPLICA");
}
let mut envelope = builder.header(header, version).build();
// DeleteObject with no `versionId` query parameter.
let result = authorize_delete_object(&slot, &mut envelope, &bucket, "target", None).await;
let info = envelope.req_info().unwrap();
let expected_version = replica.then_some(version.as_str());
assert_eq!(info.version_id.as_deref(), expected_version, "{header} replica={replica}");
if expect_denied {
assert_eq!(
result
.expect_err("protected version named by the header must be denied")
.code(),
&S3ErrorCode::AccessDenied,
"{header}"
);
} else {
result.expect("a plain delete, or a replica delete of the unprotected version, succeeds");
}
}
store
.get_object_info(
&bucket,
"target",
&crate::storage::ObjectOptions {
version_id: Some(versions[0].clone()),
versioned: true,
..Default::default()
},
)
.await
.expect("the protected version survives the denied replica deletes");
});
}
@@ -0,0 +1,159 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! A presigned PutObject cannot gain an Object Lock legal hold through its query string.
//!
//! Tests: `authorize_put_object` on the owned request for a presigned PUT whose legal hold is a
//! signed header (refused without `s3:PutObjectLegalHold`) and one whose legal hold and user
//! metadata are query parameters (served as an ordinary PUT: the query hoist restores the user
//! metadata and never the legal hold).
//! Not tested: SigV4 verification of the presigned URL, the PutObject decoder, or storing the
//! object.
//! Upstream: `storage::authorization::tests::fixture`. Downstream: none.
use crate::storage::access_owned::object_write::{PutObjectAuthorization, authorize_put_object};
use crate::storage::authorization::resolve_req_info;
use crate::storage::authorization::tests::fixture::request;
use crate::storage::storage_api::contract::bucket::{BucketOperations as _, MakeBucketOptions};
use crate::storage::storage_api::runtime_sources_consumer::{AppContext, ServerContextSlot};
use http::Method;
use rustfs_s3_types::S3ErrorCode;
use std::sync::Arc;
const ACCESS_KEY: &str = "review-local-access";
const SECRET_KEY: &str = "review-local-secret";
const LEGAL_HOLD: &str = "x-amz-object-lock-legal-hold";
/// The URI of a presigned SigV4 PUT. `hoisted` moves the legal hold, and adds user metadata, into
/// the query; otherwise the legal hold is a signed header. The signature is a placeholder: it was
/// verified before authorization, which only reads the presigned marker set.
fn presigned_lock_put_uri(bucket: &str, key: &str, hoisted: bool) -> String {
let signed_headers = if hoisted {
"host"
} else {
"host;x-amz-object-lock-legal-hold"
};
let mut params = std::collections::BTreeMap::from([
("X-Amz-Algorithm", "AWS4-HMAC-SHA256".to_owned()),
("X-Amz-Credential", format!("{ACCESS_KEY}/20260101/us-east-1/s3/aws4_request")),
("X-Amz-Date", "20260101T000000Z".to_owned()),
("X-Amz-Expires", "300".to_owned()),
("X-Amz-SignedHeaders", signed_headers.to_owned()),
]);
if hoisted {
params.insert(LEGAL_HOLD, "ON".to_owned());
params.insert("x-amz-meta-user-id", "u1".to_owned());
}
let query = params
.into_iter()
.map(|(k, v)| format!("{}={}", urlencoding::encode(k), urlencoding::encode(&v)))
.collect::<Vec<_>>()
.join("&");
format!("/{bucket}/{key}?{query}&X-Amz-Signature={}", "0".repeat(64))
}
#[tokio::test]
async fn signed_query_metadata_cannot_add_legal_hold_without_permission() {
let store = crate::app::gating_test_env::shared_gating_ecstore().await;
let iam = rustfs_iam::init_iam_sys(store.clone()).await.expect("test IAM");
let context = Arc::new(AppContext::with_default_interfaces(
store.clone(),
iam,
Arc::new(rustfs_kms::KmsServiceManager::new()),
));
assert!(context.publish_action_credentials(rustfs_credentials::Credentials {
access_key: ACCESS_KEY.to_owned(),
secret_key: SECRET_KEY.to_owned(),
status: "on".to_owned(),
..Default::default()
}));
let slot = ServerContextSlot::new();
assert!(slot.install(context));
let bucket = format!("review-lock-permission-{}", uuid::Uuid::new_v4().simple());
store
.make_bucket(
&bucket,
&MakeBucketOptions {
lock_enabled: true,
..Default::default()
},
)
.await
.expect("locked bucket");
let policy = format!(
r#"{{"Version":"2012-10-17","Statement":[{{"Effect":"Allow","Principal":{{"AWS":"*"}},"Action":["s3:PutObject"],"Resource":["arn:aws:s3:::{bucket}/*"]}},{{"Effect":"Deny","Principal":{{"AWS":"*"}},"Action":["s3:PutObjectLegalHold","s3:PutObjectRetention"],"Resource":["arn:aws:s3:::{bucket}/*"]}}]}}"#
);
let mut metadata = (*crate::storage::get_bucket_metadata(&bucket).await.expect("bucket metadata")).clone();
metadata.policy_config = Some(serde_json::from_str(&policy).expect("bucket policy"));
metadata.policy_config_json = policy.into_bytes();
crate::storage::storage_api::set_bucket_metadata(bucket.clone(), metadata)
.await
.expect("set bucket policy");
// The caller the access phase resolves for the signing key.
let caller = resolve_req_info(&slot, Some(ACCESS_KEY), "", None)
.await
.expect("resolve the signing caller");
let put = |key: &'static str, hoisted: bool| {
let mut builder = request(Method::PUT)
.uri(&presigned_lock_put_uri(&bucket, key, hoisted))
.header("host", "localhost")
.header("content-length", "4")
.header("content-md5", "CY9rzUYh03PK3k6DJie09g==")
.credentials(ACCESS_KEY, SECRET_KEY)
.extension(caller.clone())
.server_context(&slot);
if !hoisted {
builder = builder.header(LEGAL_HOLD, "ON");
}
builder.build()
};
// The facts the PutObject decoder derives: the legal hold is read from the headers only.
let facts = |key: &'static str, legal_hold: bool| PutObjectAuthorization {
bucket: &bucket,
key,
version_id: None,
retain_until: None,
specific_etag: false,
legal_hold,
retention: false,
post: false,
};
let mut header_control = put("header-control", false);
let legal_hold = header_control.headers().contains_key(LEGAL_HOLD);
assert!(legal_hold, "the control sends its legal hold as a signed header");
let error = authorize_put_object(&mut header_control, &slot, facts("header-control", legal_hold), |_| Ok(()))
.await
.expect_err("a signed legal-hold header must reach the independent permission check");
assert_eq!(error.code(), &S3ErrorCode::AccessDenied, "header control: {error:?}");
let mut query_target = put("query-target", true);
let legal_hold = query_target.headers().contains_key(LEGAL_HOLD);
authorize_put_object(&mut query_target, &slot, facts("query-target", legal_hold), |_| Ok(()))
.await
.expect("ordinary metadata PUT should be authorized");
assert!(
!query_target.headers().contains_key(LEGAL_HOLD),
"query metadata must not grant a separate Object Lock capability"
);
assert_eq!(
query_target
.headers()
.get("x-amz-meta-user-id")
.and_then(|value| value.to_str().ok()),
Some("u1"),
"signed query user metadata is restored as a header"
);
}
@@ -0,0 +1,520 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Write and read admission on the owned request: the action a DeleteObject version selects and
//! the bucket-policy tag hint it implies, the PostObject marker, the write-offset refusal that
//! precedes PutObject authorization, the on-demand-migration source capture the read hooks
//! defer to the use case, and the PutObject order between a policy refusal and a generation
//! read-quorum failure.
//!
//! Not covered here: the policy evaluator itself, request-body decoding, or any legacy request
//! type.
//! Upstream: `access_owned::{object_read, object_write, listing_multipart}`, the authorization
//! guards and `storage::authorization::tests::fixture`; downstream: none.
use crate::app::RequestEnvelope;
use crate::app::bucket_usecase::DefaultBucketUsecase;
use crate::app::object_usecase::{DefaultObjectUsecase, GetObjectFields, HeadObjectFields};
use crate::storage::StorageError;
use crate::storage::access_owned::listing_multipart::authorize_list_objects_v2;
use crate::storage::access_owned::object_read::{authorize_get_object, authorize_head_object};
use crate::storage::access_owned::object_write::{PutObjectAuthorization, authorize_post_object, authorize_put_object};
use crate::storage::authorization::tests::fixture::{request, server_context};
use crate::storage::authorization::{
AMZ_WRITE_OFFSET_BYTES_HEADER, BucketPolicyExistingObjectTagHint, OdmReadGenerationGuard, PUT_GENERATION_READ_QUORUM_FAILURE,
bucket_policy_needs_existing_object_tag_from_hint,
};
use crate::storage::authorization_actions::delete_object_authorize_action;
use crate::storage::req_info::ReqInfo;
use crate::storage::storage_api::contract::bucket::{BucketOperations as _, MakeBucketOptions};
use crate::storage::storage_api::contract::object::{ObjectIO as _, ObjectOperations as _};
use crate::storage::storage_api::runtime_sources_consumer::ServerContextSlot;
use http::Method;
use rustfs_gateway_types::dto as gateway_dto;
use rustfs_gateway_types::{BucketName, ByteRange};
use rustfs_policy::policy::action::{Action, S3Action};
use rustfs_policy::policy::{BucketPolicy, BucketPolicyArgs};
use rustfs_s3_types::S3ErrorCode;
use serial_test::serial;
use std::collections::HashMap;
use std::sync::Arc;
#[test]
fn delete_authorization_selects_the_addressed_version() {
assert_eq!(delete_object_authorize_action(None), Action::S3Action(S3Action::DeleteObjectAction));
for version_id in ["null", "8f418ad0-f9f4-4458-83b2-cc72bc6f1b70"] {
assert_eq!(
delete_object_authorize_action(Some(version_id)),
Action::S3Action(S3Action::DeleteObjectVersionAction)
);
}
}
#[tokio::test]
async fn test_anonymous_version_delete_uses_version_policy_and_tag_hint() {
let policy: BucketPolicy = serde_json::from_str(
r#"{
"Version":"2012-10-17",
"Statement":[
{
"Effect":"Allow",
"Principal":{"AWS":"*"},
"Action":["s3:DeleteObject"],
"Resource":["arn:aws:s3:::bucket/*"]
},
{
"Effect":"Allow",
"Principal":{"AWS":"*"},
"Action":["s3:DeleteObjectVersion"],
"Resource":["arn:aws:s3:::bucket/*"],
"Condition":{"StringEquals":{"s3:ExistingObjectTag/security":"public"}}
}
]
}"#,
)
.expect("bucket policy should parse");
let hint = BucketPolicyExistingObjectTagHint::Parsed(policy.clone());
let no_groups: Option<Vec<String>> = None;
let conditions = HashMap::new();
let args_delete = BucketPolicyArgs {
bucket: "bucket",
action: Action::S3Action(S3Action::DeleteObjectAction),
is_owner: false,
account: "",
groups: &no_groups,
conditions: &conditions,
object: "obj",
};
assert!(
policy.is_allowed(&args_delete).await,
"anonymous DeleteObject can be allowed by bucket policy"
);
let args_delete_version = BucketPolicyArgs {
bucket: "bucket",
action: Action::S3Action(S3Action::DeleteObjectVersionAction),
is_owner: false,
account: "",
groups: &no_groups,
conditions: &conditions,
object: "obj",
};
assert!(
!policy.is_allowed(&args_delete_version).await,
"DeleteObjectVersion should still be denied without matching ExistingObjectTag conditions"
);
let needs_tag_current = bucket_policy_needs_existing_object_tag_from_hint(&hint, &args_delete).await;
let needs_tag_version = bucket_policy_needs_existing_object_tag_from_hint(&hint, &args_delete_version).await;
assert!(!needs_tag_current, "DeleteObject statement itself does not require ExistingObjectTag");
assert!(
needs_tag_version,
"DeleteObjectVersion statement requires ExistingObjectTag when version delete is evaluated"
);
assert!(
needs_tag_version,
"the selected version action must require tag fetch for DeleteObject(versionId)"
);
}
#[tokio::test]
async fn post_object_marks_request_extensions() {
let mut req = request(Method::POST).req_info().build();
let _ = authorize_post_object(&mut req, "test-bucket", "test-key", None).await;
assert!(
req.post_object().is_some(),
"post object request should carry the marker for downstream handling"
);
}
#[tokio::test]
async fn put_object_rejects_write_offset_bytes_before_authorization() {
let mut req = request(Method::PUT)
.uri("/test-bucket/test-key")
.header(AMZ_WRITE_OFFSET_BYTES_HEADER, "0")
.req_info()
.build();
let server = ServerContextSlot::new();
let err = authorize_put_object(&mut req, &server, put_object_authorization("test-bucket", "test-key"), |_| Ok(()))
.await
.expect_err("write-offset-bytes requests should be rejected");
assert_eq!(err.code(), &S3ErrorCode::NotImplemented);
// The generic NotImplemented message (`ApiError::error_code_to_message(NotImplemented)`).
assert_eq!(err.message(), Some("A header you provided implies functionality that is not implemented"));
let req_info = req.req_info().expect("request info should remain available");
assert_eq!(req_info.bucket.as_deref(), Some("test-bucket"));
assert_eq!(req_info.object.as_deref(), Some("test-key"));
}
/// The facts the PutObject hook derives from an input that sets nothing beyond bucket and key.
fn put_object_authorization<'a>(bucket: &'a str, key: &'a str) -> PutObjectAuthorization<'a> {
PutObjectAuthorization {
bucket,
key,
version_id: None,
retain_until: None,
specific_etag: false,
legal_hold: false,
retention: false,
post: false,
}
}
/// An owner request as the ODM read tests authorize it.
fn odm_owner_request(method: Method, slot: &Arc<ServerContextSlot>) -> RequestEnvelope {
request(method)
.extension(ReqInfo {
cred: Some(rustfs_credentials::Credentials::default()),
is_owner: true,
..Default::default()
})
.server_context(slot)
.build()
}
fn get_fields(bucket: &str, key: &str, part_number: Option<i32>) -> GetObjectFields {
GetObjectFields {
bucket: bucket.to_owned(),
key: key.to_owned(),
version_id: None,
part_number,
range: None,
sse_customer_key: None,
sse_customer_key_md5: None,
}
}
fn head_fields(bucket: &str, key: &str, part_number: Option<i32>, range: Option<ByteRange>) -> HeadObjectFields {
HeadObjectFields {
bucket: bucket.to_owned(),
key: key.to_owned(),
version_id: None,
part_number,
range,
sse_customer_key: None,
sse_customer_key_md5: None,
}
}
fn list_input(bucket: &str) -> gateway_dto::ListObjectsV2Input {
gateway_dto::ListObjectsV2Input {
bucket: BucketName::new(bucket).expect("test bucket name"),
..Default::default()
}
}
#[test]
#[serial]
fn odm_read_capture_preserves_access_and_parameter_error_order() {
crate::app::gating_test_env::run_large_stack_test("odm-access-order", || async {
let context = crate::app::gating_test_env::shared_gating_ambient().await;
let server_ctx = ServerContextSlot::new();
assert!(server_ctx.install(Arc::clone(&context)));
let sys = crate::on_demand_migration::OnDemandMigrationSys::get();
let enabled_before = sys.is_module_enabled();
let bucket = format!("odm-access-missing-{}", uuid::Uuid::new_v4());
for enabled in [false, true] {
sys.set_module_enabled(enabled);
let mut get = odm_owner_request(Method::GET, &server_ctx);
let mut head = odm_owner_request(Method::HEAD, &server_ctx);
let mut list = odm_owner_request(Method::GET, &server_ctx);
authorize_get_object(&server_ctx, &mut get, &bucket, "key", None)
.await
.expect("ordinary GET access must not require bucket identity");
authorize_head_object(&server_ctx, &mut head, &bucket, "key", None)
.await
.expect("ordinary HEAD access must not require bucket identity");
authorize_list_objects_v2(&server_ctx, &mut list, &bucket)
.await
.expect("ordinary LIST access must not require bucket identity");
assert!(matches!(get.odm_read_generation(), Some(OdmReadGenerationGuard::Unavailable)));
assert!(matches!(head.odm_read_generation(), Some(OdmReadGenerationGuard::Unavailable)));
assert!(matches!(list.odm_read_generation(), Some(OdmReadGenerationGuard::Unavailable)));
let usecase = DefaultObjectUsecase::with_context(Some(Arc::clone(&context)));
assert_eq!(
crate::app::test_seams::get_object_fields(&usecase, get, get_fields(&bucket, "key", Some(0)))
.await
.expect_err("bad GET part number")
.code(),
&S3ErrorCode::InvalidArgument
);
assert_eq!(
crate::app::test_seams::head_object_fields(
&usecase,
head,
head_fields(&bucket, "key", Some(1), Some(ByteRange::FromTo { first: 0, last: 1 }))
)
.await
.expect_err("range and part number conflict")
.code(),
&S3ErrorCode::InvalidArgument
);
assert_eq!(
crate::app::test_seams::list_objects_v2_owned(
&DefaultBucketUsecase::with_context(Some(Arc::clone(&context))),
list,
gateway_dto::ListObjectsV2Input {
max_keys: Some(-1),
..list_input(&bucket)
}
)
.await
.expect_err("negative max keys")
.code(),
&S3ErrorCode::InvalidArgument
);
}
sys.set_module_enabled(enabled_before);
});
}
#[test]
#[serial]
fn odm_read_capture_failure_is_deferred_until_source_miss() {
crate::app::gating_test_env::run_large_stack_test("odm-access-failure", || async {
let context = crate::app::gating_test_env::shared_gating_ambient().await;
let store = context.object_store();
let server_ctx = ServerContextSlot::new();
assert!(server_ctx.install(Arc::clone(&context)));
let sys = crate::on_demand_migration::OnDemandMigrationSys::get();
let enabled_before = sys.is_module_enabled();
sys.set_module_enabled(true);
let bucket = format!("odm-capture-failure-{}", uuid::Uuid::new_v4());
let mut config: crate::on_demand_migration::OnDemandMigrationConfig = serde_json::from_str(r#"{"source":{"provider":"minio","endpoint":"https://source.example.com","region":"us-east-1","bucket":"source","credentials":{"access_key":"test","secret_key":"test"}}}"#).expect("source config");
config.policy.list_through = true;
sys.apply_for_incarnation(&bucket, uuid::Uuid::new_v4(), Some(&config)).await;
let mut get = odm_owner_request(Method::GET, &server_ctx);
let mut head = odm_owner_request(Method::HEAD, &server_ctx);
let mut list = odm_owner_request(Method::GET, &server_ctx);
authorize_list_objects_v2(&server_ctx, &mut list, &bucket)
.await
.expect("capture failure must not preempt LIST authorization");
authorize_get_object(&server_ctx, &mut get, &bucket, "local", None)
.await
.expect("capture failure must not preempt GET authorization");
authorize_head_object(&server_ctx, &mut head, &bucket, "local", None)
.await
.expect("capture failure must not preempt HEAD authorization");
assert!(matches!(get.odm_read_generation(), Some(OdmReadGenerationGuard::Failed { .. })));
assert!(matches!(head.odm_read_generation(), Some(OdmReadGenerationGuard::Failed { .. })));
store
.make_bucket(
&bucket,
&MakeBucketOptions {
versioning_enabled: true,
..Default::default()
},
)
.await
.expect("create bucket after capture");
store
.put_object(
&bucket,
"local",
&mut crate::storage::PutObjReader::from_vec(b"local".to_vec()),
&crate::storage::ObjectOptions::default(),
)
.await
.expect("create local hit");
// Publishing a usable source later must not repair a failed capture.
let incarnation = store.bucket_incarnation_id(&bucket).await.expect("created identity");
sys.apply_for_incarnation(&bucket, incarnation, Some(&config)).await;
let usecase = DefaultObjectUsecase::with_context(Some(Arc::clone(&context)));
let bucket_usecase = || DefaultBucketUsecase::with_context(Some(Arc::clone(&context)));
crate::app::test_seams::get_object_fields(&usecase, get.fork_for_authorization(), get_fields(&bucket, "local", None))
.await
.expect("a local GET hit ignores source capture failure");
crate::app::test_seams::head_object_fields(
&usecase,
head.fork_for_authorization(),
head_fields(&bucket, "local", None, None),
)
.await
.expect("a local HEAD hit ignores source capture failure");
assert_eq!(
crate::app::test_seams::get_object_fields(
&usecase,
get.fork_for_authorization(),
get_fields(&bucket, "missing", None)
)
.await
.expect_err("failed capture cannot rebind on GET miss")
.code(),
&S3ErrorCode::NoSuchBucket
);
assert_eq!(
crate::app::test_seams::head_object_fields(
&usecase,
head.fork_for_authorization(),
head_fields(&bucket, "missing", None, None)
)
.await
.expect_err("failed capture cannot rebind on HEAD miss")
.code(),
&S3ErrorCode::NoSuchBucket
);
assert_eq!(
crate::app::test_seams::list_objects_v2_owned(&bucket_usecase(), list.fork_for_authorization(), list_input(&bucket))
.await
.expect_err("failed capture cannot rebind on LIST")
.code(),
&S3ErrorCode::NoSuchBucket
);
config.filter.prefix = Some("remote/".into());
sys.apply_for_incarnation(&bucket, incarnation, Some(&config)).await;
assert_eq!(
crate::app::test_seams::get_object_fields(
&usecase,
get.fork_for_authorization(),
get_fields(&bucket, "missing", None)
)
.await
.expect_err("filtered GET remains local")
.code(),
&S3ErrorCode::NoSuchKey
);
assert_eq!(
crate::app::test_seams::head_object_fields(
&usecase,
head.fork_for_authorization(),
head_fields(&bucket, "missing", None, None)
)
.await
.expect_err("filtered HEAD remains local")
.code(),
&S3ErrorCode::NoSuchKey
);
let listing = crate::app::test_seams::list_objects_v2_owned(
&bucket_usecase(),
list.fork_for_authorization(),
gateway_dto::ListObjectsV2Input {
prefix: Some("local/".into()),
..list_input(&bucket)
},
)
.await
.expect("disjoint prefix needs no source identity");
assert_eq!(listing.key_count, 0);
let listing = crate::app::test_seams::list_objects_v2_owned(
&bucket_usecase(),
list,
gateway_dto::ListObjectsV2Input {
prefix: Some("remote/".into()),
max_keys: Some(0),
..list_input(&bucket)
},
)
.await
.expect("an empty page needs no source identity");
assert_eq!(listing.key_count, 0);
config.filter.prefix = None;
config.policy.head = crate::on_demand_migration::HeadPolicy::LocalOnly;
sys.apply_for_incarnation(&bucket, incarnation, Some(&config)).await;
assert_eq!(
crate::app::test_seams::head_object_fields(
&usecase,
head.fork_for_authorization(),
head_fields(&bucket, "missing", None, None)
)
.await
.expect_err("local-only HEAD needs no source identity")
.code(),
&S3ErrorCode::NoSuchKey
);
config.policy.head = crate::on_demand_migration::HeadPolicy::Proxy;
sys.apply_for_incarnation(&bucket, incarnation, Some(&config)).await;
store
.delete_object(
&bucket,
"missing",
crate::storage::ObjectOptions {
versioned: true,
..Default::default()
},
)
.await
.expect("create local delete marker");
assert_eq!(
crate::app::test_seams::get_object_fields(&usecase, get, get_fields(&bucket, "missing", None))
.await
.expect_err("GET respects the delete marker before capture failure")
.code(),
&S3ErrorCode::NoSuchKey
);
assert_eq!(
crate::app::test_seams::head_object_fields(&usecase, head, head_fields(&bucket, "missing", None, None))
.await
.expect_err("HEAD respects the delete marker before capture failure")
.code(),
&S3ErrorCode::NoSuchKey
);
sys.remove(&bucket);
sys.set_module_enabled(enabled_before);
});
}
#[tokio::test]
#[serial]
async fn put_read_quorum_access_keeps_authorization_precedence() {
let store = crate::app::gating_test_env::shared_gating_ecstore().await;
let server_ctx = server_context(Arc::clone(&store));
for effect in ["Allow", "Deny"] {
let bucket = format!("put-quorum-access-{}", uuid::Uuid::new_v4());
store
.make_bucket(&bucket, &MakeBucketOptions::default())
.await
.expect("authorization precedence bucket must be created");
let policy_json = format!(
r#"{{"Version":"2012-10-17","Statement":[{{"Effect":"{effect}","Principal":{{"AWS":"*"}},"Action":["s3:PutObject"],"Resource":["arn:aws:s3:::{bucket}/*"]}}]}}"#
);
let mut metadata = (*crate::storage::get_bucket_metadata(&bucket)
.await
.expect("test bucket metadata must be cached"))
.clone();
metadata.policy_config = Some(serde_json::from_str(&policy_json).expect("test policy must parse"));
metadata.policy_config_json = policy_json.into_bytes();
crate::storage::storage_api::set_bucket_metadata(bucket.clone(), metadata)
.await
.expect("test policy must be installed");
let mut req = request(Method::PUT).req_info().server_context(&server_ctx).build();
let err = PUT_GENERATION_READ_QUORUM_FAILURE
.scope(
(),
authorize_put_object(&mut req, &server_ctx, put_object_authorization(&bucket, "object"), |_| Ok(())),
)
.await
.expect_err("a generation failure must never admit a write");
if effect == "Allow" {
assert_eq!(err.code(), &S3ErrorCode::ServiceUnavailable);
assert_eq!(err.status_code(), Some(http::StatusCode::SERVICE_UNAVAILABLE));
assert!(matches!(
err.source().and_then(|source| source.downcast_ref::<StorageError>()),
Some(StorageError::ErasureReadQuorum)
));
} else {
assert_eq!(err.code(), &S3ErrorCode::AccessDenied);
assert_eq!(err.status_code(), Some(http::StatusCode::FORBIDDEN));
}
assert!(req.bucket_generation().is_none());
}
}
+4
View File
@@ -34,8 +34,12 @@ mod guards;
mod permission_probe_test;
mod request;
mod table_data_plane;
#[cfg(test)]
pub(crate) mod tests;
pub(crate) use admission::{AccessMarkers, VerifiedSignature, admit_operation, resolve_req_info};
#[cfg(test)]
pub(crate) use conditions::authorization_conditions_with_internal;
pub(crate) use conditions::existing_object_tag_conditions;
#[cfg(test)]
pub(crate) use conditions::{
@@ -0,0 +1,75 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! The policy action each owned authorization picks for an operation, and the answer a terminal
//! IAM refusal becomes.
//!
//! Responsible for: `storage::authorization_actions` (the action selection both edges share) and
//! `storage::authorization::iam_authorization_error`.
//! Not responsible for: evaluating a policy against the chosen action, or actions an operation's
//! gateway handler names inline.
//! Upstream: none. Downstream: `storage::authorization_actions` and `storage::authorization`.
use crate::storage::authorization::iam_authorization_error;
use crate::storage::authorization_actions::{
complete_multipart_upload_authorize_action, list_parts_authorize_action, post_object_authorize_action, versioned_read_action,
};
use rustfs_policy::policy::action::{Action, S3Action};
use rustfs_s3_types::{S3Error, S3ErrorCode};
#[test]
fn terminal_iam_denial_is_access_denied_and_backend_failure_stays_internal() {
let denied = S3Error::from(iam_authorization_error(rustfs_iam::error::Error::IAMActionNotAllowed));
assert_eq!(denied.code(), &S3ErrorCode::AccessDenied);
let unavailable = S3Error::from(iam_authorization_error(rustfs_iam::error::Error::IamSysNotInitialized));
assert_eq!(unavailable.code(), &S3ErrorCode::InternalError);
}
#[test]
fn ghsa_3ppv_versioned_read_selects_get_object_version_action() {
// Regression (GHSA-3ppv): a read that names an explicit version must
// authorize against s3:GetObjectVersion so that a principal holding only
// s3:GetObject cannot read historical versions. Applies to GetObject and
// the CopyObject / UploadPartCopy sources.
assert_eq!(
versioned_read_action(Some("0194e0f1-0000-7000-8000-000000000000")),
Action::S3Action(S3Action::GetObjectVersionAction),
"an explicit versionId must require GetObjectVersion"
);
assert_eq!(
versioned_read_action(Some("null")),
Action::S3Action(S3Action::GetObjectVersionAction),
"the sentinel `null` version is still an explicit version selector"
);
assert_eq!(
versioned_read_action(None),
Action::S3Action(S3Action::GetObjectAction),
"a current-object read (no versionId) authorizes against GetObject"
);
}
#[test]
fn post_object_uses_put_object_action() {
assert_eq!(post_object_authorize_action(), Action::S3Action(S3Action::PutObjectAction));
}
#[test]
fn complete_multipart_upload_uses_put_object_action() {
assert_eq!(complete_multipart_upload_authorize_action(), Action::S3Action(S3Action::PutObjectAction));
}
#[test]
fn list_parts_uses_list_multipart_upload_parts_action() {
assert_eq!(list_parts_authorize_action(), Action::S3Action(S3Action::ListMultipartUploadPartsAction));
}
@@ -0,0 +1,495 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! The condition values the owned authorization core builds from one `RequestEnvelope`.
//!
//! Responsible for: list query conditions, request object tag conditions, the verified client
//! facts, the internal-object condition filter, the remaining-retention-days condition, the
//! existing-object-tag hint and merge, and the owner's bucket-policy Deny bypass.
//! Not responsible for: running an operation's `authorize_*` function against stored policies
//! (`storage::access_owned` tests), or the legacy request type.
//! Upstream: `crate::storage::authorization::tests::fixture`. Downstream: `storage::authorization`
//! conditions and decision, `crate::auth::facts`, and the bucket policy system.
use crate::app::RequestEnvelope;
use crate::app::object_meta::tagging::proxy_allowed;
use crate::auth::OBJECT_LOCK_REMAINING_RETENTION_DAYS_CONDITION;
use crate::auth::facts::{AuthzPrincipal, AuthzTarget, list_query_pairs, merge_list_query_conditions};
use crate::shared_types::RemoteAddr;
use crate::storage::StorageError;
use crate::storage::authorization::tests::fixture::{request, server_context};
use crate::storage::authorization::{
BucketPolicyExistingObjectTagHint, BucketPolicyRawLoadErrorKind, ObjectTagConditions, authorization_conditions,
authorization_conditions_with_internal, classify_bucket_policy_raw_load_error, load_bucket_policy_existing_object_tag_hint,
maybe_merge_object_tag_conditions, merge_request_object_tag_conditions, owner_can_bypass_policy_deny,
remaining_retention_days, request_authz_facts, set_requested_object_lock_retain_until,
};
use crate::storage::storage_api::contract::bucket::{BucketOperations as _, MakeBucketOptions};
use http::{HeaderMap, HeaderValue, Method};
use rustfs_policy::policy::action::{Action, S3Action};
use rustfs_policy::policy::{BucketPolicy, bucket_policy_uses_existing_object_tag_conditions};
use rustfs_s3_types::S3ErrorCode;
use serial_test::serial;
use std::collections::HashMap;
use time::OffsetDateTime;
/// The conditions an anonymous principal's decision about `action` evaluates for `req`.
fn remaining_retention_days_conditions(req: &RequestEnvelope, action: Action) -> HashMap<String, Vec<String>> {
let credentials = rustfs_credentials::Credentials::default();
let principal = AuthzPrincipal::new(&credentials, "");
let facts = request_authz_facts(
req,
AuthzTarget {
action,
bucket: "",
object: "",
version_id: None,
location_constraint: None,
},
);
authorization_conditions(req, &principal, &facts).expect("conditions should build")
}
/// A PUT that asks to retain its object for `retain_for` from now.
fn request_retaining_for(retain_for: time::Duration) -> RequestEnvelope {
let mut req = request(Method::PUT).build();
let retain_until = OffsetDateTime::now_utc() + retain_for;
set_requested_object_lock_retain_until(&mut req, Some(retain_until));
req
}
/// Object tag conditions must use keys like ExistingObjectTag/<tag-key> so that
/// bucket policy conditions (e.g. s3:ExistingObjectTag/security) are evaluated correctly.
#[test]
fn test_merge_list_bucket_query_conditions_extracts_supported_keys() {
let mut conditions = HashMap::new();
merge_list_query_conditions(
Action::S3Action(S3Action::ListBucketAction),
&list_query_pairs(Some("prefix=photos%2F2024%2F&delimiter=%2F&max-keys=10&encoding-type=url")),
&mut conditions,
);
assert_eq!(conditions.get("prefix"), Some(&vec!["photos/2024/".to_string()]));
assert_eq!(conditions.get("delimiter"), Some(&vec!["/".to_string()]));
assert_eq!(conditions.get("max-keys"), Some(&vec!["10".to_string()]));
assert!(!conditions.contains_key("encoding-type"));
}
#[test]
fn test_merge_list_bucket_query_conditions_preserves_empty_prefix_signal() {
let mut conditions = HashMap::new();
merge_list_query_conditions(
Action::S3Action(S3Action::ListBucketVersionsAction),
&list_query_pairs(Some("prefix=&delimiter=%2F")),
&mut conditions,
);
assert_eq!(conditions.get("prefix"), Some(&vec![String::new()]));
assert_eq!(conditions.get("delimiter"), Some(&vec!["/".to_string()]));
}
#[test]
fn test_merge_list_bucket_query_conditions_ignores_non_list_actions() {
let mut conditions = HashMap::new();
merge_list_query_conditions(
Action::S3Action(S3Action::GetObjectAction),
&list_query_pairs(Some("prefix=photos%2F2024%2F&delimiter=%2F&max-keys=10")),
&mut conditions,
);
assert!(conditions.is_empty());
}
#[test]
fn listing_conditions_reject_repeated_query_and_ignore_headers() {
for action in [
S3Action::ListBucketAction,
S3Action::ListBucketVersionsAction,
S3Action::ListBucketMultipartUploadsAction,
] {
for key in ["prefix", "delimiter", "max-keys"] {
let action = Action::S3Action(action);
let credentials = rustfs_credentials::Credentials::default();
let conditions_of = |req: &RequestEnvelope| {
let principal = AuthzPrincipal::new(&credentials, "");
let facts = request_authz_facts(
req,
AuthzTarget {
action,
bucket: "bucket",
object: "",
version_id: None,
location_constraint: None,
},
);
authorization_conditions(req, &principal, &facts)
};
// Every request carries the header; only the applied query may describe the listing.
let req = request(Method::GET).header(key, "allowed").build();
let conditions = conditions_of(&req).unwrap();
assert!(!conditions.contains_key(key), "{key} must come from the applied query");
let req = request(Method::GET)
.uri(&format!("/bucket?{key}=actual"))
.header(key, "allowed")
.build();
let conditions = conditions_of(&req).unwrap();
assert_eq!(conditions.get(key), Some(&vec!["actual".to_string()]));
let req = request(Method::GET)
.uri(&format!("/bucket?{key}=actual&{key}=allowed"))
.header(key, "allowed")
.build();
let error = conditions_of(&req).unwrap_err();
assert_eq!(error.code(), &S3ErrorCode::InvalidRequest);
}
}
}
#[test]
fn test_merge_request_object_tag_conditions_applies_only_to_put_object() {
let mut headers = HeaderMap::new();
headers.insert("x-amz-tagging", HeaderValue::from_static("classification=restricted&label=copy%20test"));
let mut source_conditions = HashMap::new();
merge_request_object_tag_conditions(Action::S3Action(S3Action::GetObjectAction), &headers, &mut source_conditions)
.expect("GetObject condition construction should succeed");
assert!(
source_conditions.is_empty(),
"destination request tags must not affect CopyObject source authorization"
);
let mut destination_conditions = HashMap::new();
merge_request_object_tag_conditions(Action::S3Action(S3Action::PutObjectAction), &headers, &mut destination_conditions)
.expect("PutObject condition construction should succeed");
assert_eq!(
destination_conditions.get("RequestObjectTag/classification"),
Some(&vec!["restricted".to_string()])
);
assert_eq!(destination_conditions.get("RequestObjectTag/label"), Some(&vec!["copy test".to_string()]));
assert_eq!(
destination_conditions.get("RequestObjectTagKeys"),
Some(&vec!["classification".to_string(), "label".to_string()])
);
}
#[test]
fn access_facts_preserve_verified_client_context() {
use rustfs_trusted_proxies::{ClientInfo, ValidationMode};
let trusted = ClientInfo::from_trusted_proxy(
"203.0.113.9".parse().expect("client IP"),
None,
Some("https".to_owned()),
"10.0.0.2".parse().expect("proxy IP"),
1,
ValidationMode::Lenient,
Vec::new(),
);
for (client_info, expected_ip, expected_secure) in [(None, "192.0.2.10", "false"), (Some(trusted), "203.0.113.9", "true")] {
let mut builder = request(Method::GET)
.header("x-forwarded-for", "198.51.100.1")
.header("x-forwarded-proto", "https")
.extension(Some(RemoteAddr("192.0.2.10:443".parse().expect("peer address"))));
if let Some(client_info) = client_info {
builder = builder.extension(client_info);
}
let req = builder.build();
let conditions = remaining_retention_days_conditions(&req, Action::S3Action(S3Action::GetObjectAction));
assert_eq!(conditions.get("SourceIp"), Some(&vec![expected_ip.to_owned()]));
assert_eq!(conditions.get("SecureTransport"), Some(&vec![expected_secure.to_owned()]));
assert!(req.remote_addr().is_some(), "facts must only borrow the recorded peer");
}
}
/// AWS does not document how it rounds partial days. MinIO computes
/// `ceil(hours / 24)` in `enforceRetentionBypassForPut`
/// (cmd/bucket-object-lock.go), and RustFS keeps that rounding so a policy
/// moved from MinIO evaluates the same way.
#[test]
fn remaining_retention_days_matches_s3_ceil_rounding() {
let now = OffsetDateTime::now_utc();
let cases = [
(time::Duration::NANOSECOND, 1),
(time::Duration::HOUR, 1),
(time::Duration::DAY - time::Duration::SECOND, 1),
(time::Duration::DAY, 1),
(time::Duration::DAY + time::Duration::SECOND, 2),
(time::Duration::days(29) + time::Duration::hours(12), 30),
(time::Duration::days(30), 30),
(time::Duration::days(30) + time::Duration::SECOND, 31),
];
for (retain_for, expected) in cases {
let minio_days = (retain_for.as_seconds_f64() / 3600.0 / 24.0).ceil() as i64;
assert_eq!(minio_days, expected, "MinIO reference for {retain_for}");
assert_eq!(remaining_retention_days(now + retain_for, now), expected, "retain for {retain_for}");
}
}
#[test]
fn remaining_retention_days_is_zero_for_dates_not_in_future() {
let now = OffsetDateTime::now_utc();
assert_eq!(remaining_retention_days(now, now), 0);
assert_eq!(remaining_retention_days(now - time::Duration::NANOSECOND, now), 0);
assert_eq!(remaining_retention_days(now - time::Duration::days(3), now), 0);
}
#[test]
fn requested_retain_until_sets_remaining_retention_days_condition() {
let req = request_retaining_for(time::Duration::days(10) - time::Duration::HOUR);
let conditions = remaining_retention_days_conditions(&req, Action::S3Action(S3Action::PutObjectAction));
assert_eq!(
conditions.get(OBJECT_LOCK_REMAINING_RETENTION_DAYS_CONDITION),
Some(&vec!["10".to_string()])
);
}
#[test]
fn request_without_retention_has_no_remaining_retention_days_condition() {
let action = Action::S3Action(S3Action::PutObjectAction);
let req = request(Method::PUT).build();
assert_eq!(
remaining_retention_days_conditions(&req, action).get(OBJECT_LOCK_REMAINING_RETENTION_DAYS_CONDITION),
None
);
let mut req = request_retaining_for(time::Duration::days(5));
set_requested_object_lock_retain_until(&mut req, None);
assert_eq!(
remaining_retention_days_conditions(&req, action).get(OBJECT_LOCK_REMAINING_RETENTION_DAYS_CONDITION),
None
);
}
#[test]
fn remaining_retention_days_condition_ignores_client_header() {
let action = Action::S3Action(S3Action::PutObjectRetentionAction);
let mut req = request(Method::PUT)
.header(OBJECT_LOCK_REMAINING_RETENTION_DAYS_CONDITION, "1")
.build();
assert_eq!(
remaining_retention_days_conditions(&req, action).get(OBJECT_LOCK_REMAINING_RETENTION_DAYS_CONDITION),
None
);
let retain_until = OffsetDateTime::now_utc() + time::Duration::days(400);
set_requested_object_lock_retain_until(&mut req, Some(retain_until));
assert_eq!(
remaining_retention_days_conditions(&req, action).get(OBJECT_LOCK_REMAINING_RETENTION_DAYS_CONDITION),
Some(&vec!["400".to_string()])
);
}
#[test]
fn internal_object_authorization_ignores_unapplied_request_options() {
let req = request(Method::POST)
.header(
"authorization",
"AWS4-HMAC-SHA256 Credential=test/20260801/us-east-1/s3tables/aws4_request",
)
.header("user-agent", "pyiceberg/test")
.header("x-amz-server-side-encryption", "AES256")
.header("x-amz-tagging", "classification=%ZZ")
.build();
let credentials = rustfs_credentials::Credentials::default();
let principal = AuthzPrincipal::new(&credentials, "");
let facts = request_authz_facts(
&req,
AuthzTarget {
action: Action::S3Action(S3Action::PutObjectAction),
bucket: "",
object: "",
version_id: None,
location_constraint: None,
},
);
// The owned edge reaches this branch through `authorize_owned_internal_object_request`.
let conditions = authorization_conditions_with_internal(&req, &principal, &facts, true)
.expect("internal object authorization conditions should build");
assert_eq!(conditions.get("authType"), Some(&vec!["REST-HEADER".to_string()]));
assert_eq!(conditions.get("signatureversion"), Some(&vec!["AWS4-HMAC-SHA256".to_string()]));
assert_eq!(conditions.get("UserAgent"), Some(&vec!["pyiceberg/test".to_string()]));
assert!(!conditions.contains_key("authorization"));
assert!(!conditions.contains_key("x-amz-server-side-encryption"));
assert!(!conditions.contains_key("RequestObjectTag/classification"));
assert!(!conditions.contains_key("RequestObjectTagKeys"));
}
#[tokio::test]
async fn test_required_existing_object_tags_are_merged_into_authorization_conditions() {
let (_temp_dir, _disk_paths, store) = crate::app::gating_test_env::isolated_multi_pool_ecstore().await;
let slot = server_context(store);
let mut req = request(Method::GET)
.server_context(&slot)
.extension(ObjectTagConditions::new(
"bucket",
"tagged-object",
None,
HashMap::from([
("ExistingObjectTag/security".to_string(), vec!["restricted".to_string()]),
("ExistingObjectTag/project".to_string(), vec!["webapp".to_string()]),
]),
))
.build();
let mut conditions = HashMap::new();
conditions.insert("delimiter".to_string(), vec!["/".to_string()]);
maybe_merge_object_tag_conditions(
&mut req,
Action::S3Action(S3Action::GetObjectAction),
"bucket",
"tagged-object",
None,
&mut conditions,
true,
)
.await
.expect("merge required existing object tags into authorization conditions");
assert_eq!(conditions.get("ExistingObjectTag/security"), Some(&vec!["restricted".to_string()]));
assert!(!proxy_allowed(&req));
assert_eq!(conditions.get("ExistingObjectTag/project"), Some(&vec!["webapp".to_string()]));
assert_eq!(conditions.get("delimiter"), Some(&vec!["/".to_string()]));
}
#[test]
fn test_bucket_policy_existing_object_tag_condition_key_detection() {
let condition_key_policy = r#"{
"Version":"2012-10-17",
"Statement":[{
"Effect":"Allow",
"Principal":"*",
"Action":["s3:GetObject"],
"Resource":["arn:aws:s3:::bucket/*"],
"Condition":{"StringEquals":{"s3:ExistingObjectTag/security":"public"}}
}]
}"#;
let policy: BucketPolicy = serde_json::from_str(condition_key_policy).expect("valid bucket policy JSON");
assert!(
bucket_policy_uses_existing_object_tag_conditions(&policy),
"ExistingObjectTag in condition key must be detected"
);
let value_only_policy = r#"{
"Version":"2012-10-17",
"Statement":[{
"Effect":"Allow",
"Principal":"*",
"Action":["s3:GetObject"],
"Resource":["arn:aws:s3:::bucket/*"],
"Condition":{"StringEquals":{"s3:prefix":"ExistingObjectTag/security"}}
}]
}"#;
let policy: BucketPolicy = serde_json::from_str(value_only_policy).expect("valid bucket policy JSON");
assert!(
!bucket_policy_uses_existing_object_tag_conditions(&policy),
"ExistingObjectTag text in values should not trigger tag dependency"
);
}
#[test]
fn test_unparsable_bucket_policy_json_implies_conservative_existing_object_tag_fetch() {
// Matches `load_bucket_policy_existing_object_tag_hint`: unparsable policy => conservative tag fetch.
let malformed = r#"{"Version":"2012-10-17","Statement":[INVALID]}"#;
assert!(serde_json::from_str::<BucketPolicy>(malformed).is_err());
let conservative_fetch = serde_json::from_str::<BucketPolicy>(malformed)
.map(|p| bucket_policy_uses_existing_object_tag_conditions(&p))
.unwrap_or(true);
assert!(conservative_fetch);
// Invalid JSON that still contains real ExistingObjectTag condition keys (trailing comma).
let malformed_with_tag_keys =
r#"{"Version":"2012-10-17","Statement":[{"Condition":{"StringEquals":{"s3:ExistingObjectTag/security":"public"}}},]}"#;
assert!(serde_json::from_str::<BucketPolicy>(malformed_with_tag_keys).is_err());
let conservative_with_tag_keys = serde_json::from_str::<BucketPolicy>(malformed_with_tag_keys)
.map(|p| bucket_policy_uses_existing_object_tag_conditions(&p))
.unwrap_or(true);
assert!(conservative_with_tag_keys);
}
#[tokio::test]
#[serial]
async fn existing_tag_hint_reuses_parsed_metadata_and_fails_closed_without_it() {
let store = crate::app::gating_test_env::shared_gating_ecstore().await;
let bucket = format!("cached-policy-hint-{}", uuid::Uuid::new_v4());
store
.make_bucket(&bucket, &MakeBucketOptions::default())
.await
.expect("create hint bucket");
let document = serde_json::json!({"Version":"2012-10-17","Statement":[{
"Effect":"Deny","Principal":"*","Action":"s3:GetObject","Resource":format!("arn:aws:s3:::{bucket}/*"),
"Condition":{"StringEquals":{"s3:ExistingObjectTag/classification":"secret"}}
}]});
let mut metadata = (*crate::storage::get_bucket_metadata(&bucket).await.expect("bucket metadata")).clone();
metadata.policy_config =
Some(serde_json::from_slice(&serde_json::to_vec(&document).expect("policy JSON")).expect("parsed policy"));
// A cached parsed snapshot is the authority used by actual policy
// evaluation; the tag hint must consult that same snapshot.
metadata.policy_config_json = b"invalid raw JSON".to_vec();
crate::storage::storage_api::set_bucket_metadata(bucket.clone(), metadata.clone())
.await
.expect("publish parsed snapshot");
assert!(matches!(
load_bucket_policy_existing_object_tag_hint(&store, &bucket, Action::S3Action(S3Action::GetObjectAction)).await,
BucketPolicyExistingObjectTagHint::Parsed(_)
));
metadata.policy_config = None;
crate::storage::storage_api::set_bucket_metadata(bucket.clone(), metadata)
.await
.expect("publish unreadable snapshot");
assert!(matches!(
load_bucket_policy_existing_object_tag_hint(&store, &bucket, Action::S3Action(S3Action::GetObjectAction)).await,
BucketPolicyExistingObjectTagHint::ConservativeTagRequired
));
}
#[test]
fn test_classify_bucket_policy_raw_load_error() {
assert_eq!(
classify_bucket_policy_raw_load_error(&StorageError::ConfigNotFound),
BucketPolicyRawLoadErrorKind::PolicyMissing
);
assert_eq!(
classify_bucket_policy_raw_load_error(&StorageError::BucketNotFound("b".to_string())),
BucketPolicyRawLoadErrorKind::BucketMissing
);
assert_eq!(
classify_bucket_policy_raw_load_error(&StorageError::Io(std::io::Error::other("boom"))),
BucketPolicyRawLoadErrorKind::Other
);
}
/// Owner can bypass bucket policy Deny only for the three policy management APIs (per AWS S3).
#[test]
fn test_owner_can_bypass_policy_deny_only_for_policy_apis() {
// Owner + policy management actions -> bypass allowed
assert!(owner_can_bypass_policy_deny(true, &Action::S3Action(S3Action::GetBucketPolicyAction)));
assert!(owner_can_bypass_policy_deny(true, &Action::S3Action(S3Action::PutBucketPolicyAction)));
assert!(owner_can_bypass_policy_deny(true, &Action::S3Action(S3Action::DeleteBucketPolicyAction)));
// Owner + other actions -> no bypass (still subject to bucket policy Deny)
assert!(!owner_can_bypass_policy_deny(true, &Action::S3Action(S3Action::ListBucketAction)));
assert!(!owner_can_bypass_policy_deny(true, &Action::S3Action(S3Action::GetObjectAction)));
// Non-owner -> no bypass for any action
assert!(!owner_can_bypass_policy_deny(false, &Action::S3Action(S3Action::GetBucketPolicyAction)));
assert!(!owner_can_bypass_policy_deny(
false,
&Action::S3Action(S3Action::DeleteBucketPolicyAction)
));
}
@@ -0,0 +1,335 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Authorization denial auditing on the owned request: the client-facing AccessDenied stays
//! generic, a denial yields one bounded audit entry built from the envelope's audit view, probe
//! and aggregate flags suppress it, and every request-level denial exit without a
//! `DenialContext` still calls `audit_request_denial` in its owning production file.
//!
//! Not covered here: the limiter's own token arithmetic (`storage/denial_audit.rs` tests), the
//! policy decisions that produce a denial, or any legacy request type.
//! Upstream: `DenialContext`, `audit_request_denial`, `DenialAuditor::audit_owned`, `BulkDenialAudit`
//! and the envelope's `AuthorizationRequest::audit_denial`; downstream: none.
use crate::app::RequestEnvelope;
use crate::storage::authorization::tests::fixture::request;
use crate::storage::authorization::{DenialContext, audit_request_denial};
use crate::storage::denial_audit::{BulkDenialAudit, DenialAuditor, DenialRateLimiter};
use crate::storage::helper::DenialScope;
use crate::storage::req_info::ReqInfo;
use crate::storage::request_context::RequestContext;
use http::Method;
use rustfs_policy::policy::action::{Action, S3Action};
use rustfs_s3_types::S3ErrorCode;
use std::sync::{Arc, Mutex};
use std::time::{Duration, Instant};
type CapturedAudit = Arc<Mutex<Vec<rustfs_audit::AuditEntry>>>;
/// An auditor wired like the global one except that entries land in the returned vector.
fn test_auditor(enabled: bool, authenticated_rate: u64, anonymous_rate: u64) -> (DenialAuditor, CapturedAudit) {
let captured: CapturedAudit = Arc::default();
let sink = Arc::clone(&captured);
// A frozen clock: no wall-clock refill can change what the limiter admits.
let t0 = Instant::now();
let limiter = DenialRateLimiter::new(authenticated_rate, anonymous_rate, t0);
let auditor =
DenialAuditor::new(limiter, move || enabled, move |_, entry| sink.lock().unwrap().push(entry)).with_clock(move || t0);
(auditor, captured)
}
fn denial_request(request_id: &str, authenticated: bool) -> RequestEnvelope {
request(Method::GET)
.uri("/audit-bucket/audit-object")
.extension(RequestContext {
request_id: request_id.to_string(),
x_amz_request_id: request_id.to_string(),
trace_id: None,
span_id: None,
start_time: Instant::now(),
})
.extension(ReqInfo {
cred: authenticated.then(|| rustfs_credentials::Credentials {
access_key: "denied-user".to_string(),
..Default::default()
}),
bucket: Some("audit-bucket".to_string()),
object: Some("audit-object".to_string()),
..Default::default()
})
.build()
}
fn denial_context(auditor: &DenialAuditor, quiet: bool) -> DenialContext<'_> {
DenialContext {
quiet,
suppress_audit: false,
auditor,
bucket: "audit-bucket",
object: "audit-object",
version_id: Some("version"),
account: Some("denied-user"),
is_owner: false,
}
}
const GET_OBJECT: Action = Action::S3Action(S3Action::GetObjectAction);
/// Issue #5740: the denial helper must keep the client-facing error identical
/// to the historical bare response — the added diagnostics are log-only.
#[test]
fn test_access_denied_helper_keeps_generic_wire_response() {
let (auditor, captured) = test_auditor(true, 10, 10);
let req = denial_request("denial-wire-response", true);
let err = denial_context(&auditor, false).deny(&req, "bucket_policy_explicit_deny", GET_OBJECT);
assert_eq!(err.code(), &S3ErrorCode::AccessDenied);
assert_eq!(err.message(), Some("Access Denied"));
assert_eq!(captured.lock().unwrap().len(), 1);
}
/// A real denial yields exactly one audit entry naming the account, bucket, object,
/// denied action and internal reason.
#[test]
fn denial_is_audited_once_with_account_resource_action_and_reason() {
let (auditor, captured) = test_auditor(true, 10, 10);
let req = denial_request("denial-audited-once", true);
let _ = denial_context(&auditor, false).deny(&req, "iam_explicit_deny", Action::S3Action(S3Action::PutObjectAction));
let entries = captured.lock().unwrap();
assert_eq!(entries.len(), 1);
let entry = &entries[0];
assert_eq!(entry.request_id.as_deref(), Some("denial-audited-once"));
assert_eq!(entry.access_key.as_deref(), Some("denied-user"));
assert_eq!(entry.api.name.as_deref(), Some("s3:PutObject"));
assert_eq!(entry.api.bucket.as_deref(), Some("audit-bucket"));
assert_eq!(entry.api.object.as_deref(), Some("audit-object"));
assert_eq!(entry.api.status.as_deref(), Some("failure"));
assert_eq!(entry.api.status_code, Some(403));
assert_eq!(entry.error.as_deref(), Some("Access Denied"));
let tags = entry.tags.as_ref().expect("denial entry carries its reason");
assert_eq!(tags.get("denialReason").and_then(|value| value.as_str()), Some("iam_explicit_deny"));
assert!(!tags.contains_key("suppressedDenials"));
}
/// Probe denials (per-bucket ListBuckets fallback, per-object metadata collection) are
/// expected filter outcomes and must not flood the audit stream.
#[test]
fn quiet_probe_denial_is_not_audited() {
let (auditor, captured) = test_auditor(true, 10, 10);
let req = denial_request("denial-quiet-probe", true);
let err = denial_context(&auditor, true).deny(&req, "no_policy_allows_action", Action::S3Action(S3Action::ListBucketAction));
assert_eq!(err.code(), &S3ErrorCode::AccessDenied);
assert!(captured.lock().unwrap().is_empty());
}
#[test]
fn denial_is_not_audited_when_the_audit_module_is_off() {
let (auditor, captured) = test_auditor(false, 10, 10);
let req = denial_request("denial-audit-off", true);
let _ = denial_context(&auditor, false).deny(&req, "no_policy_allows_action", GET_OBJECT);
assert!(captured.lock().unwrap().is_empty());
}
#[test]
fn denial_audited_in_aggregate_by_the_caller_is_not_audited_per_key() {
let (auditor, captured) = test_auditor(true, 10, 10);
let req = denial_request("denial-per-key", true);
let mut denial = denial_context(&auditor, false);
denial.suppress_audit = true;
let _ = denial.deny(&req, "iam_explicit_deny", GET_OBJECT);
assert!(captured.lock().unwrap().is_empty());
}
/// A flood of denials emits a bounded number of entries; the rest are only counted.
#[test]
fn denial_flood_emits_bounded_entries() {
let (auditor, captured) = test_auditor(true, 3, 3);
let req = denial_request("denial-flood", true);
for _ in 0..20 {
let _ = denial_context(&auditor, false).deny(&req, "no_policy_allows_action", GET_OBJECT);
}
assert_eq!(captured.lock().unwrap().len(), 3);
}
/// The entry that follows a suppressed run reports how many denials were dropped.
///
/// The owned auditor reads time only from its clock, so this test drives a settable clock
/// instead of passing an instant per call.
#[test]
fn entry_after_a_flood_reports_the_suppressed_count() {
let captured: CapturedAudit = Arc::default();
let sink = Arc::clone(&captured);
let t0 = Instant::now();
let now = Arc::new(Mutex::new(t0));
let clock = Arc::clone(&now);
let auditor = DenialAuditor::new(
DenialRateLimiter::new(1, 1, t0),
|| true,
move |_, entry| sink.lock().unwrap().push(entry),
)
.with_clock(move || *clock.lock().unwrap());
let req = denial_request("denial-suppressed-count", true);
for _ in 0..5 {
auditor.audit_owned(&req.audit_view(), &GET_OBJECT, "no_policy_allows_action", DenialScope::Request);
}
*now.lock().unwrap() = t0 + Duration::from_secs(2);
auditor.audit_owned(&req.audit_view(), &GET_OBJECT, "no_policy_allows_action", DenialScope::Request);
let entries = captured.lock().unwrap();
assert_eq!(entries.len(), 2);
let tag = |index: usize| {
entries[index]
.tags
.as_ref()
.unwrap()
.get("suppressedDenials")
.and_then(|v| v.as_u64())
};
assert_eq!(tag(0), None);
assert_eq!(tag(1), Some(4));
}
/// Anonymous callers have their own, lower limit, so an unauthenticated scanner cannot
/// use up the budget for authenticated callers.
#[test]
fn anonymous_denials_are_limited_separately_from_authenticated_ones() {
let (auditor, captured) = test_auditor(true, 10, 1);
let anonymous = denial_request("denial-anonymous", false);
let authenticated = denial_request("denial-authenticated", true);
for _ in 0..5 {
let _ = denial_context(&auditor, false).deny(&anonymous, "no_policy_allows_action", GET_OBJECT);
}
for _ in 0..5 {
let _ = denial_context(&auditor, false).deny(&authenticated, "no_policy_allows_action", GET_OBJECT);
}
let entries = captured.lock().unwrap();
let count = |id: &str| entries.iter().filter(|entry| entry.request_id.as_deref() == Some(id)).count();
assert_eq!(count("denial-anonymous"), 1);
assert_eq!(count("denial-authenticated"), 5);
}
/// Exits that have no `DenialContext` of their own audit through the request's own state
/// and honor the same probe flags.
#[test]
fn request_denial_exit_is_audited_and_honors_the_probe_flags() {
let (auditor, captured) = test_auditor(true, 10, 10);
let mut req = denial_request("denial-exit", true);
audit_request_denial(&req, &auditor, "table_resource_policy_denied", GET_OBJECT);
assert_eq!(captured.lock().unwrap().len(), 1);
req.req_info_mut().unwrap().suppress_denial_log = true;
audit_request_denial(&req, &auditor, "table_resource_policy_denied", GET_OBJECT);
assert_eq!(captured.lock().unwrap().len(), 1);
req.req_info_mut().unwrap().suppress_denial_log = false;
req.req_info_mut().unwrap().suppress_denial_audit = true;
audit_request_denial(&req, &auditor, "table_resource_policy_denied", GET_OBJECT);
assert_eq!(captured.lock().unwrap().len(), 1);
}
fn production_sources_audit_reason(sources: &[&str], reason: &str) -> bool {
sources.iter().any(|source| {
let production = source.split("\nmod tests {").next().expect("source has a production section");
production
.match_indices("audit_request_denial(")
.any(|(at, _)| production[at..].chars().take(260).collect::<String>().contains(reason))
})
}
/// Request-level denial exits span the owned authorization modules.
/// Check each exit in its owning file alongside the audit behavior tests.
#[test]
fn denial_exits_without_a_context_call_the_audit_function() {
let access = include_str!("../table_data_plane.rs");
let object_misc = include_str!("../../access_owned/object_misc.rs")
.split("\nmod tests;")
.next()
.expect("owned source has a production section");
for (source, reason) in [
(access, "\"table_resource_policy_denied\""),
(access, "\"anonymous_table_data_plane_access\""),
(object_misc, "\"recursive_force_delete_requires_authenticated_caller\""),
] {
assert!(
production_sources_audit_reason(&[source], reason),
"{reason} exit does not call audit_request_denial"
);
}
}
#[test]
fn denial_source_guard_tracks_moved_calls_and_rejects_removed_calls() {
let reason = "\"recursive_force_delete_requires_authenticated_caller\"";
let original = "fn original_exit() {}";
let moved = format!("fn moved_exit() {{ audit_request_denial(req, auditor, {reason}, action); }}");
assert!(!production_sources_audit_reason(&[original], reason));
assert!(production_sources_audit_reason(&[original, &moved], reason));
let removed = moved.replace("audit_request_denial(", "unrecorded_denial(");
assert!(!production_sources_audit_reason(&[original, &removed], reason));
let test_only = format!("{original}\nmod tests {{\n{moved}\n}}");
assert!(!production_sources_audit_reason(&[original, &test_only], reason));
}
/// DeleteObjects audits its refused keys as one entry per denied action, carrying the count and
/// no single object, and the per-key mark is cleared afterwards.
#[test]
fn bulk_denial_is_one_entry_per_denied_action_with_the_key_count() {
let (auditor, captured) = test_auditor(true, 10, 10);
let mut envelope = denial_request("denial-bulk", true);
let nothing_denied = BulkDenialAudit::start_owned(&mut envelope).unwrap();
assert!(envelope.req_info().unwrap().suppress_denial_audit);
nothing_denied
.finish_owned_with(&mut envelope, RequestEnvelope::audit_view, &auditor)
.unwrap();
assert!(!envelope.req_info().unwrap().suppress_denial_audit);
assert!(captured.lock().unwrap().is_empty());
let mut bulk = BulkDenialAudit::start_owned(&mut envelope).unwrap();
let delete = Action::S3Action(S3Action::DeleteObjectAction);
let bypass = Action::S3Action(S3Action::BypassGovernanceRetentionAction);
for action in [delete, delete, bypass, delete] {
bulk.denied(action);
}
bulk.finish_owned_with(&mut envelope, RequestEnvelope::audit_view, &auditor)
.unwrap();
let entries = captured.lock().unwrap();
assert_eq!(entries.len(), 2);
let summary: Vec<_> = entries
.iter()
.map(|entry| {
assert_eq!(entry.api.object, None);
(
entry.api.name.clone().unwrap(),
entry
.tags
.as_ref()
.unwrap()
.get("deniedKeys")
.and_then(|value| value.as_u64())
.unwrap(),
)
})
.collect();
assert_eq!(
summary,
vec![
("s3:DeleteObject".to_string(), 3),
("s3:BypassGovernanceRetention".to_string(), 1)
]
);
}
@@ -0,0 +1,141 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Owned request fixtures for the authorization tests: a [`RequestEnvelope`] builder and the
//! IAM and KMS stand-ins a server context needs.
//!
//! Responsible for: building the request the gateway edge hands the authorization core, with
//! the same facts the legacy edge copied from its request (headers, URI, credentials and the
//! session token taken from them, extensions).
//! Not responsible for: any authorization decision, or fixtures for the legacy request type.
//! Upstream: none. Downstream: the tests of `storage::authorization` and `storage::access_owned`.
use crate::app::{RequestCredentials, RequestEnvelope, RequestHead};
use crate::auth::get_session_token;
use crate::storage::req_info::ReqInfo;
use crate::storage::storage_api::runtime_sources_consumer::{AppContext, ECStore, IamInterface, KmsInterface, ServerContextSlot};
use http::header::{HeaderName, HeaderValue};
use http::{Extensions, HeaderMap, Method, Uri};
use rustfs_iam::{store::object::ObjectStore, sys::IamSys};
use rustfs_kms::KmsServiceManager;
use std::sync::Arc;
/// An IAM that is never ready: a request authorized against it resolves no identity policy.
pub(crate) struct UnreadyIam;
impl IamInterface for UnreadyIam {
fn handle(&self) -> Arc<IamSys<ObjectStore>> {
panic!("authorization fixtures built on UnreadyIam do not resolve IAM")
}
fn is_ready(&self) -> bool {
false
}
}
/// A KMS with no configured backend.
pub(crate) struct TestKms;
impl KmsInterface for TestKms {
fn handle(&self) -> Arc<KmsServiceManager> {
Arc::new(KmsServiceManager::new())
}
}
/// A server context bound to `store`, with an unready IAM and an unconfigured KMS.
pub(crate) fn server_context(store: Arc<ECStore>) -> Arc<ServerContextSlot> {
let slot = ServerContextSlot::new();
assert!(slot.install(Arc::new(AppContext::new(store, Arc::new(UnreadyIam), Arc::new(TestKms)))));
slot
}
/// Builds the owned request the authorization core reads.
pub(crate) struct RequestBuilder {
method: Method,
uri: Uri,
headers: HeaderMap,
credentials: Option<(String, String)>,
extensions: Extensions,
}
/// A request with no headers, no credentials, no extensions, and the URI `/`.
pub(crate) fn request(method: Method) -> RequestBuilder {
RequestBuilder {
method,
uri: Uri::from_static("/"),
headers: HeaderMap::new(),
credentials: None,
extensions: Extensions::new(),
}
}
impl RequestBuilder {
pub(crate) fn uri(mut self, uri: &str) -> Self {
self.uri = uri.parse().expect("test URI");
self
}
pub(crate) fn header(mut self, name: &str, value: &str) -> Self {
self.headers.append(
HeaderName::from_bytes(name.as_bytes()).expect("test header name"),
HeaderValue::from_str(value).expect("test header value"),
);
self
}
pub(crate) fn headers(mut self, headers: HeaderMap) -> Self {
self.headers = headers;
self
}
/// Signs the request as `access_key`. The session token is read from the request's
/// headers and query when the envelope is built, as the edge reads it.
pub(crate) fn credentials(mut self, access_key: &str, secret_key: &str) -> Self {
self.credentials = Some((access_key.to_owned(), secret_key.to_owned()));
self
}
/// Carries an empty [`ReqInfo`], as the access phase installs before any hook runs.
pub(crate) fn req_info(self) -> Self {
self.extension(ReqInfo::default())
}
pub(crate) fn server_context(self, slot: &Arc<ServerContextSlot>) -> Self {
self.extension(slot.clone())
}
pub(crate) fn extension<T: Clone + Send + Sync + 'static>(mut self, value: T) -> Self {
self.extensions.insert(value);
self
}
pub(crate) fn build(self) -> RequestEnvelope {
let credentials = self.credentials.map(|(access_key, secret_key)| {
let session_token = get_session_token(&self.uri, &self.headers).map(str::to_owned);
RequestCredentials::new(access_key.clone(), secret_key, session_token.clone())
.unwrap_or_else(|_| RequestCredentials::without_secret_key(access_key, session_token))
});
RequestEnvelope::from_parts(
RequestHead {
method: self.method,
uri: self.uri,
headers: self.headers,
credentials,
region: None,
service: None,
},
self.extensions,
)
}
}
@@ -0,0 +1,23 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Tests of the shared authorization core, run on the owned request envelope.
mod actions;
mod conditions;
mod denial_audit;
pub(crate) mod fixture;
mod permission_auth;
mod readers;
mod table_data_plane;
@@ -1,15 +1,34 @@
// Copyright 2026 RustFS Team
// SPDX-License-Identifier: Apache-2.0
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! The signing facts a permission probe forked from an account-permissions request carries into
//! its authorization conditions: the caller's verified query-signing family survives, its URL
//! capability does not, and an unauthenticated query never becomes a signing fact.
//!
//! Not covered here: the probe's endpoint handling, or decisions for ordinary requests.
//! Upstream: `RequestEnvelope::fork_for_permission_probe`, `request_authz_facts`, and
//! `build_conditions`; downstream: none.
use super::{
AccessMarkers, Action, AuthorizationRequest, AuthzPrincipal, AuthzTarget, HeaderMap, ReqInfo, S3Action, VerifiedSignature,
build_args, build_conditions, get_request_auth_type_with_query, request_authz_facts,
};
use crate::app::{RequestCredentials, RequestEnvelope, RequestHead};
use crate::auth::facts::AuthTypeCondition;
use http::{Extensions, Method};
use crate::auth::facts::{AuthTypeCondition, AuthzPrincipal, AuthzTarget, build_args, build_conditions};
use crate::auth::get_request_auth_type_with_query;
use crate::storage::authorization::{AccessMarkers, AuthorizationRequest, VerifiedSignature, request_authz_facts};
use crate::storage::req_info::ReqInfo;
use http::{Extensions, HeaderMap, Method};
use rustfs_credentials::Credentials;
use rustfs_policy::policy::Policy;
use rustfs_policy::policy::action::{Action, S3Action};
fn accepted_query_caller(query: &str) -> (RequestEnvelope, Credentials) {
let credentials = Credentials {
@@ -12,25 +12,35 @@
// See the License for the specific language governing permissions and
// limitations under the License.
//! Owned-versus-legacy parity of the state an access hook hands to a use case: every reader and
//! installer must answer the same through an s3s request and through the `RequestEnvelope` built
//! from it, with the state present and absent, and each answer is pinned to its expected value so
//! the parity cannot hold vacuously.
//! The state an access phase hands to a use case, read back through the `RequestEnvelope`: every
//! reader and installer gives its pinned answer with the state present and absent, and the
//! request-derived condition values come out of the envelope's request facts.
//!
//! Not covered here: policy decisions (`app/owned_authorization_test.rs`).
//! Upstream: the readers in `storage/access.rs`; downstream: none.
//! Not covered here: policy decisions (`app/owned_authorization_test.rs`), or any parity with a
//! legacy request type.
//! Upstream: the readers and installers in `storage::authorization` and the envelope's
//! `AuthorizationRequest` impl; downstream: none.
use super::{
Action, AuthorizationRequest, BucketConfigMutationSnapshot, BucketGenerationGuard, CopySourceBucketGenerationGuard, HashMap,
HeaderMap, ObjectOptions, OdmReadGenerationGuard, PendingDeleteBucketGenerationGuard, RemoteAddr, ReqInfo, RequestContext,
S3Action, S3ErrorCode, S3Request, TableDataPlanePublicationGuards, apply_bucket_generation_guard,
use crate::app::RequestEnvelope;
use crate::shared_types::RemoteAddr;
use crate::storage::ObjectOptions;
use crate::storage::authorization::tests::fixture::{RequestBuilder, request};
use crate::storage::authorization::{
AuthorizationRequest, BucketConfigMutationSnapshot, BucketGenerationGuard, CopySourceBucketGenerationGuard,
OdmReadGenerationGuard, PendingDeleteBucketGenerationGuard, TableDataPlanePublicationGuards, apply_bucket_generation_guard,
apply_copy_source_bucket_generation_guard, authorize_object_request, bucket_config_mutation_incarnation,
capture_odm_read_generation, odm_read_generation, replication_request_authorized, request_context_from_req,
resource_free_condition_values, retained_table_data_plane_publication_guards,
};
use crate::app::RequestEnvelope;
use crate::runtime_sources::ServerContextSlot;
use crate::storage::req_info::ReqInfo;
use crate::storage::request_context::RequestContext;
use crate::storage::storage_api::runtime_sources_consumer::ServerContextSlot;
use http::header::{HeaderName, HeaderValue};
use http::{HeaderMap, Method};
use rustfs_policy::policy::action::{Action, S3Action};
use rustfs_s3_types::S3ErrorCode;
use rustfs_trusted_proxies::ClientInfo;
use std::collections::HashMap;
use std::net::SocketAddr;
const BUCKET: &str = "guarded-bucket";
@@ -46,24 +56,14 @@ fn guard(bucket: &str) -> BucketGenerationGuard {
}
}
fn legacy(fill: impl FnOnce(&mut http::Extensions)) -> S3Request<()> {
let mut extensions = http::Extensions::new();
fill(&mut extensions);
S3Request {
input: (),
method: http::Method::GET,
uri: "/guarded-bucket/key".parse().expect("static uri"),
headers: HeaderMap::new(),
extensions,
credentials: None,
region: None,
service: None,
trailing_headers: None,
}
/// A GET of `/guarded-bucket/key` carrying what `fill` adds.
fn envelope(fill: impl FnOnce(RequestBuilder) -> RequestBuilder) -> RequestEnvelope {
fill(request(Method::GET).uri("/guarded-bucket/key")).build()
}
fn server_request(extensions: &mut http::Extensions) {
extensions.insert(ServerContextSlot::new());
/// Marks the request as one the server received, as the edge does by carrying its context.
fn server_request(builder: RequestBuilder) -> RequestBuilder {
builder.server_context(&ServerContextSlot::new())
}
/// The incarnation a reader bound, or the refusal's code and message.
@@ -72,7 +72,7 @@ type Answer = Result<Option<uuid::Uuid>, (S3ErrorCode, String)>;
fn answer<T>(result: rustfs_s3_types::S3Result<T>, bound: impl FnOnce(T) -> Option<uuid::Uuid>) -> Answer {
result
.map(bound)
.map_err(|err| (err.code().clone().into(), err.message().unwrap_or_default().to_owned()))
.map_err(|err| (err.code().clone(), err.message().unwrap_or_default().to_owned()))
}
fn refused(code: S3ErrorCode, message: &str) -> Answer {
@@ -99,16 +99,10 @@ fn config_mutation<R: AuthorizationRequest + ?Sized>(req: &R) -> Answer {
answer(bucket_config_mutation_incarnation(req, BUCKET), |id| id)
}
/// Asserts that `read` gives `expected` through both request forms of every case.
fn assert_cases(
read: fn(&S3Request<()>) -> Answer,
owned: fn(&RequestEnvelope) -> Answer,
cases: Vec<(&str, S3Request<()>, Answer)>,
) {
/// Asserts that `read` gives `expected` for every case.
fn assert_cases(read: fn(&RequestEnvelope) -> Answer, cases: Vec<(&str, RequestEnvelope, Answer)>) {
for (name, req, expected) in cases {
let envelope = RequestEnvelope::from_s3s(&req);
assert_eq!(read(&req), expected, "legacy form: {name}");
assert_eq!(owned(&envelope), expected, "owned form: {name}");
assert_eq!(read(&req), expected, "{name}");
}
}
@@ -117,48 +111,43 @@ fn bucket_generation_guard_reads_the_same_through_both_forms() {
let missing = "bucket generation guard is missing";
let mismatch = "bucket generation guard does not match request bucket";
assert_cases(
bucket_fence,
bucket_fence,
vec![
(
"captured guard binds",
legacy(|ext| {
ext.insert(guard(BUCKET));
}),
envelope(|req| req.extension(guard(BUCKET))),
Ok(Some(incarnation())),
),
(
"pending-delete guard wins over the captured guard",
legacy(|ext| {
ext.insert(guard("other-bucket"));
ext.insert(PendingDeleteBucketGenerationGuard::Ready(guard(BUCKET)));
envelope(|req| {
req.extension(guard("other-bucket"))
.extension(PendingDeleteBucketGenerationGuard::Ready(guard(BUCKET)))
}),
Ok(Some(incarnation())),
),
(
"guard of another bucket",
legacy(|ext| {
ext.insert(guard("other-bucket"));
}),
envelope(|req| req.extension(guard("other-bucket"))),
refused(S3ErrorCode::InternalError, mismatch),
),
(
"failed pending-delete capture",
legacy(|ext| {
ext.insert(guard(BUCKET));
ext.insert(PendingDeleteBucketGenerationGuard::Failed {
code: S3ErrorCode::NoSuchBucket.into(),
message: "bucket is gone".to_owned(),
});
envelope(|req| {
req.extension(guard(BUCKET))
.extension(PendingDeleteBucketGenerationGuard::Failed {
code: S3ErrorCode::NoSuchBucket,
message: "bucket is gone".to_owned(),
})
}),
refused(S3ErrorCode::NoSuchBucket, "bucket is gone"),
),
(
"server request without a guard",
legacy(server_request),
envelope(server_request),
refused(S3ErrorCode::InternalError, missing),
),
("direct use-case call without a guard", legacy(|_| {}), Ok(None)),
("direct use-case call without a guard", envelope(|req| req), Ok(None)),
],
);
}
@@ -170,21 +159,16 @@ fn copy_source_generation_guard_reads_the_same_through_both_forms() {
incarnation_id: incarnation(),
};
assert_cases(
copy_source_fence,
copy_source_fence,
vec![
(
"captured guard binds",
legacy(|ext| {
ext.insert(copy_guard(BUCKET));
}),
envelope(|req| req.extension(copy_guard(BUCKET))),
Ok(Some(incarnation())),
),
(
"guard of another bucket",
legacy(|ext| {
ext.insert(copy_guard("other-bucket"));
}),
envelope(|req| req.extension(copy_guard("other-bucket"))),
refused(
S3ErrorCode::InternalError,
"copy source bucket generation guard does not match request bucket",
@@ -192,13 +176,10 @@ fn copy_source_generation_guard_reads_the_same_through_both_forms() {
),
(
"the destination guard does not stand in for the source",
legacy(|ext| {
server_request(ext);
ext.insert(guard(BUCKET));
}),
envelope(|req| server_request(req).extension(guard(BUCKET))),
refused(S3ErrorCode::InternalError, "copy source bucket generation guard is missing"),
),
("direct use-case call without a guard", legacy(|_| {}), Ok(None)),
("direct use-case call without a guard", envelope(|req| req), Ok(None)),
],
);
}
@@ -206,41 +187,34 @@ fn copy_source_generation_guard_reads_the_same_through_both_forms() {
#[test]
fn odm_read_generation_reads_the_same_through_both_forms() {
assert_cases(
odm,
odm,
vec![
(
"captured source generation",
legacy(|ext| {
ext.insert(OdmReadGenerationGuard::Ready(guard(BUCKET)));
}),
envelope(|req| req.extension(OdmReadGenerationGuard::Ready(guard(BUCKET)))),
Ok(Some(incarnation())),
),
(
"source generation of another bucket",
legacy(|ext| {
ext.insert(OdmReadGenerationGuard::Ready(guard("other-bucket")));
}),
envelope(|req| req.extension(OdmReadGenerationGuard::Ready(guard("other-bucket")))),
refused(S3ErrorCode::InternalError, "source generation guard does not match request bucket"),
),
(
"failed capture surfaces its error",
legacy(|ext| {
ext.insert(OdmReadGenerationGuard::Failed {
code: S3ErrorCode::SlowDown.into(),
envelope(|req| {
req.extension(OdmReadGenerationGuard::Failed {
code: S3ErrorCode::SlowDown,
message: "capture failed".to_owned(),
});
})
}),
refused(S3ErrorCode::SlowDown, "capture failed"),
),
(
"no source configured",
legacy(|ext| {
ext.insert(OdmReadGenerationGuard::Unavailable);
}),
envelope(|req| req.extension(OdmReadGenerationGuard::Unavailable)),
Ok(None),
),
("nothing captured", legacy(|_| {}), Ok(None)),
("nothing captured", envelope(|req| req), Ok(None)),
],
);
}
@@ -252,21 +226,16 @@ fn bucket_config_mutation_incarnation_reads_the_same_through_both_forms() {
incarnation_id: incarnation(),
};
assert_cases(
config_mutation,
config_mutation,
vec![
(
"captured snapshot",
legacy(|ext| {
ext.insert(snapshot(BUCKET));
}),
envelope(|req| req.extension(snapshot(BUCKET))),
Ok(Some(incarnation())),
),
(
"snapshot of another bucket",
legacy(|ext| {
ext.insert(snapshot("other-bucket"));
}),
envelope(|req| req.extension(snapshot("other-bucket"))),
refused(
S3ErrorCode::InternalError,
"bucket config mutation snapshot does not match request bucket",
@@ -274,17 +243,17 @@ fn bucket_config_mutation_incarnation_reads_the_same_through_both_forms() {
),
(
"server request without a snapshot",
legacy(server_request),
envelope(server_request),
refused(S3ErrorCode::InternalError, "bucket config mutation snapshot is missing"),
),
("direct use-case call without a snapshot", legacy(|_| {}), Ok(None)),
("direct use-case call without a snapshot", envelope(|req| req), Ok(None)),
],
);
}
#[test]
fn installed_state_reads_back_through_the_envelope() {
let mut envelope = RequestEnvelope::from_s3s(&legacy(server_request));
let mut envelope = envelope(server_request);
assert_eq!(
bucket_fence(&envelope),
refused(S3ErrorCode::InternalError, "bucket generation guard is missing")
@@ -292,7 +261,7 @@ fn installed_state_reads_back_through_the_envelope() {
envelope.install_bucket_generation(guard(BUCKET));
assert_eq!(bucket_fence(&envelope), Ok(Some(incarnation())));
envelope.install_pending_delete_bucket_generation(PendingDeleteBucketGenerationGuard::Failed {
code: S3ErrorCode::NoSuchBucket.into(),
code: S3ErrorCode::NoSuchBucket,
message: "bucket is gone".to_owned(),
});
assert_eq!(bucket_fence(&envelope), refused(S3ErrorCode::NoSuchBucket, "bucket is gone"));
@@ -308,36 +277,27 @@ fn installed_state_reads_back_through_the_envelope() {
#[tokio::test]
async fn odm_capture_without_a_configured_source_is_unavailable_in_both_forms() {
let slot = ServerContextSlot::new();
let req = legacy(|ext| {
ext.insert(guard(BUCKET));
});
let envelope = RequestEnvelope::from_s3s(&req);
for captured in [
capture_odm_read_generation(&slot, &req, BUCKET).await,
capture_odm_read_generation(&slot, &envelope, BUCKET).await,
] {
assert!(matches!(captured, OdmReadGenerationGuard::Unavailable), "{captured:?}");
}
let envelope = envelope(|req| req.extension(guard(BUCKET)));
let captured = capture_odm_read_generation(&slot, &envelope, BUCKET).await;
assert!(matches!(captured, OdmReadGenerationGuard::Unavailable), "{captured:?}");
}
#[test]
fn replication_authorization_flag_reads_the_same_through_both_forms() {
let with_flag = |flag: bool| {
legacy(move |ext| {
ext.insert(ReqInfo {
envelope(move |req| {
req.extension(ReqInfo {
replication_request_authorized: flag,
..Default::default()
});
})
})
};
for (name, req, expected) in [
("authorized replication request", with_flag(true), true),
("ordinary request", with_flag(false), false),
("request without ReqInfo", legacy(|_| {}), false),
("request without ReqInfo", envelope(|req| req), false),
] {
let envelope = RequestEnvelope::from_s3s(&req);
assert_eq!(replication_request_authorized(&req), expected, "legacy form: {name}");
assert_eq!(replication_request_authorized(&envelope), expected, "owned form: {name}");
assert_eq!(replication_request_authorized(&req), expected, "{name}");
}
}
@@ -355,44 +315,23 @@ fn publication_guards_and_request_context_read_the_same_through_both_forms() {
..Default::default()
};
for (name, req, expected) in [
(
"direct context",
legacy(|ext| {
ext.insert(context("direct"));
}),
Some("direct"),
),
(
"context inside ReqInfo",
legacy(|ext| {
ext.insert(nested("nested"));
}),
Some("nested"),
),
("direct context", envelope(|req| req.extension(context("direct"))), Some("direct")),
("context inside ReqInfo", envelope(|req| req.extension(nested("nested"))), Some("nested")),
(
"the direct context wins",
legacy(|ext| {
ext.insert(context("direct"));
ext.insert(nested("nested"));
}),
envelope(|req| req.extension(context("direct")).extension(nested("nested"))),
Some("direct"),
),
("no context", legacy(|_| {}), None),
("no context", envelope(|req| req), None),
] {
let envelope = RequestEnvelope::from_s3s(&req);
let id = |found: Option<RequestContext>| found.map(|context| context.request_id);
assert_eq!(id(request_context_from_req(&req)).as_deref(), expected, "legacy form: {name}");
assert_eq!(id(request_context_from_req(&envelope)).as_deref(), expected, "owned form: {name}");
assert_eq!(id(request_context_from_req(&req)).as_deref(), expected, "{name}");
}
let with_guards = legacy(|ext| {
ext.insert(TableDataPlanePublicationGuards::default());
});
let with_guards = envelope(|req| req.extension(TableDataPlanePublicationGuards::default()));
assert!(retained_table_data_plane_publication_guards(&with_guards).is_some());
assert!(retained_table_data_plane_publication_guards(&RequestEnvelope::from_s3s(&with_guards)).is_some());
let without = legacy(|_| {});
let without = envelope(|req| req);
assert!(retained_table_data_plane_publication_guards(&without).is_none());
assert!(retained_table_data_plane_publication_guards(&RequestEnvelope::from_s3s(&without)).is_none());
}
#[test]
@@ -407,66 +346,38 @@ fn resource_free_conditions_read_the_same_client_through_both_forms() {
for (name, req, expected) in [
(
"forwarded client wins over the peer",
legacy(|ext| {
ext.insert(Some(RemoteAddr(peer)));
ext.insert(ClientInfo::direct(client));
}),
envelope(|req| req.extension(Some(RemoteAddr(peer))).extension(ClientInfo::direct(client))),
Some(vec!["198.51.100.7".to_owned()]),
),
(
"peer only",
legacy(|ext| {
ext.insert(Some(RemoteAddr(peer)));
}),
envelope(|req| req.extension(Some(RemoteAddr(peer)))),
Some(vec!["192.0.2.2".to_owned()]),
),
] {
let envelope = RequestEnvelope::from_s3s(&req);
// The clock values are read at each call; every request-derived value must match.
let without_clock = |mut conditions: HashMap<String, Vec<String>>| {
conditions.remove("CurrentTime");
conditions.remove("EpochTime");
conditions
};
let legacy_conditions = without_clock(resource_free_condition_values(&req, &cred));
assert_eq!(
legacy_conditions,
without_clock(resource_free_condition_values(&envelope, &cred)),
"{name}"
);
assert_eq!(source_ip(&legacy_conditions), expected, "{name}");
assert_eq!(source_ip(&resource_free_condition_values(&req, &cred)), expected, "{name}");
}
}
#[tokio::test]
async fn object_request_without_req_info_is_refused_in_both_forms() {
let mut req = legacy(|_| {});
let mut envelope = RequestEnvelope::from_s3s(&req);
let mut envelope = envelope(|req| req);
let action = Action::S3Action(S3Action::GetObjectAclAction);
for result in [
authorize_object_request(&mut req, BUCKET, "key", Some("v1"), action)
.await
.map_err(super::S3Error::from),
authorize_object_request(&mut envelope, BUCKET, "key", Some("v1"), action)
.await
.map_err(super::S3Error::from),
] {
let err = result.expect_err("a request without ReqInfo is refused");
assert_eq!(err.code(), &S3ErrorCode::InternalError);
assert_eq!(err.message(), Some("ReqInfo not found in request extensions"));
}
let err = authorize_object_request(&mut envelope, BUCKET, "key", Some("v1"), action)
.await
.expect_err("a request without ReqInfo is refused");
assert_eq!(err.code(), &S3ErrorCode::InternalError);
assert_eq!(err.message(), Some("ReqInfo not found in request extensions"));
}
#[test]
fn a_forked_envelope_reads_like_the_original_and_installs_alone() {
let original = RequestEnvelope::from_s3s(&legacy(|ext| {
server_request(ext);
ext.insert(guard(BUCKET));
ext.insert(ReqInfo {
let original = envelope(|req| {
server_request(req).extension(guard(BUCKET)).extension(ReqInfo {
replication_request_authorized: true,
..Default::default()
});
}));
})
});
let mut fork = original.fork_for_authorization();
assert_eq!(bucket_fence(&fork), bucket_fence(&original));
assert_eq!(bucket_fence(&fork), Ok(Some(incarnation())));
@@ -491,14 +402,12 @@ fn resource_free_conditions_preserve_legacy_projection_and_erased_keystone_sessi
for original_session in [None, Some(false), Some(true)] {
for forwarded in [false, true] {
let peer: SocketAddr = "192.0.2.9:9000".parse().unwrap();
let mut req = legacy(|ext| {
ext.insert(Some(RemoteAddr(peer)));
if forwarded {
let mut client = ClientInfo::direct("198.51.100.9:443".parse().unwrap());
client.forwarded_proto = Some("https".into());
ext.insert(client);
}
let client = forwarded.then(|| {
let mut client = ClientInfo::direct("198.51.100.9:443".parse().unwrap());
client.forwarded_proto = Some("https".into());
client
});
let mut headers = HeaderMap::new();
for (name, value) in [
("user-agent", "condition-client"),
("referer", "https://example.test/"),
@@ -508,7 +417,7 @@ fn resource_free_conditions_preserve_legacy_projection_and_erased_keystone_sessi
("x-amz-copy-source-server-side-encryption-customer-key", "source-key"),
("x-amz-meta-tag", "retained-value"),
] {
req.headers.insert(name, value.parse().unwrap());
headers.insert(HeaderName::from_static(name), HeaderValue::from_static(value));
}
let original = rustfs_credentials::Credentials {
access_key: "keystone:request-user".into(),
@@ -527,11 +436,19 @@ fn resource_free_conditions_preserve_legacy_projection_and_erased_keystone_sessi
if original_session.is_some() {
credential.session_token.clear();
}
req.extensions.insert(ReqInfo {
cred: Some(credential.clone()),
keystone_session_present: original_session,
..Default::default()
});
let mut builder = request(Method::GET)
.uri("/guarded-bucket/key")
.headers(headers.clone())
.extension(Some(RemoteAddr(peer)))
.extension(ReqInfo {
cred: Some(credential.clone()),
keystone_session_present: original_session,
..Default::default()
});
if let Some(client) = client.clone() {
builder = builder.extension(client);
}
let req = builder.build();
let without_clock = |mut values: HashMap<String, Vec<String>>| {
assert!(
time::OffsetDateTime::parse(
@@ -545,20 +462,17 @@ fn resource_free_conditions_preserve_legacy_projection_and_erased_keystone_sessi
values.remove("EpochTime");
values
};
// The oracle reads the raw request parts, not the envelope.
let expected = without_clock(crate::auth::get_condition_values_with_client_info(
&req.headers,
&headers,
&original,
None,
None,
req.remote_addr(),
req.client_info(),
Some(peer),
client.as_ref(),
));
let actual = without_clock(resource_free_condition_values(&req, &credential));
assert_eq!(actual, expected, "session={original_session:?}, forwarded={forwarded}");
assert_eq!(
without_clock(resource_free_condition_values(&RequestEnvelope::from_s3s(&req), &credential)),
expected
);
assert_eq!(
actual.get("username").unwrap(),
&vec![
@@ -0,0 +1,533 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! The table data plane's share of S3 authorization, on the owned request: which object actions
//! map to table metadata actions, which ones need the publication fence, how table listings hide
//! unauthorized objects and seal their cursors, how catalog failures are answered, and how a
//! request reuses the table resolution and fences it already holds.
//!
//! Responsible for: `storage::authorization::table_data_plane` and the multipart authorization
//! functions' use of the publication fence, driven through `RequestEnvelope`.
//! Not responsible for: the table catalog's own storage, or the listing use cases that consume
//! the listing snapshot.
//! Upstream: the owned request fixture, the gating test stores, and the table catalog.
//! Downstream: `storage::authorization` and `storage::access_owned`.
use crate::storage::access_owned::object_commit::{
CopySourceObject, authorize_complete_multipart_upload, authorize_upload_part_copy,
};
use crate::storage::access_owned::object_write::authorize_upload_part;
use crate::storage::authorization::tests::fixture::{request, server_context};
use crate::storage::authorization::{
AuthorizationRequest, TABLE_DATA_PLANE_LIST_CURSOR_MAX_ENCODED_LEN, TableDataPlaneListAccess,
TableDataPlaneListCursorPosition, TableDataPlaneListResource, TableDataPlanePublicationGuards,
require_owned_reserved_table_object, retain_table_bucket_publication_guard, table_catalog_backend_for_data_plane,
table_data_plane_admin_action, table_data_plane_content_mutation, table_data_plane_resource_for_request,
table_publication_guard_error,
};
use crate::storage::storage_api::contract::bucket::{BucketOperations as _, MakeBucketOptions};
use http::Method;
use rustfs_policy::policy::action::{Action, S3Action};
use rustfs_s3_types::S3ErrorCode;
use serial_test::serial;
use std::sync::Arc;
#[test]
fn table_data_plane_admin_action_maps_s3_object_operations() {
assert_eq!(
table_data_plane_admin_action(Action::S3Action(S3Action::GetObjectAction)),
Some(rustfs_policy::policy::action::AdminAction::GetTableMetadataAction)
);
assert_eq!(
table_data_plane_admin_action(Action::S3Action(S3Action::PutObjectAction)),
Some(rustfs_policy::policy::action::AdminAction::SetTableMetadataAction)
);
assert_eq!(
table_data_plane_admin_action(Action::S3Action(S3Action::PutObjectTaggingAction)),
Some(rustfs_policy::policy::action::AdminAction::SetTableMetadataAction)
);
assert_eq!(
table_data_plane_admin_action(Action::S3Action(S3Action::DeleteObjectAction)),
Some(rustfs_policy::policy::action::AdminAction::SetTableMetadataAction)
);
assert_eq!(
table_data_plane_admin_action(Action::S3Action(S3Action::GetObjectTaggingAction)),
Some(rustfs_policy::policy::action::AdminAction::GetTableMetadataAction)
);
assert_eq!(table_data_plane_admin_action(Action::S3Action(S3Action::ListBucketAction)), None);
}
fn table_list_access(allowed: bool) -> TableDataPlaneListAccess {
TableDataPlaneListAccess {
bucket: "analytics".to_string(),
principal_fingerprint: "principal".to_string(),
snapshot_fingerprint: "snapshot".to_string(),
cursor_key: [0x42; 32],
resources: vec![TableDataPlaneListResource {
resource: crate::table_catalog::TableDataPlaneResource {
table_bucket: "analytics".to_string(),
namespace: "sales".to_string(),
table: "orders".to_string(),
table_id: "table-id".to_string(),
warehouse_object_prefix: "tables/table-id/".to_string(),
},
allowed,
}],
}
}
/// The listing authorization hands its table snapshot to the use case through the request it
/// was given.
#[test]
fn table_list_access_reaches_the_use_case_from_either_edge() {
let access = TableDataPlaneListAccess::test_listing_fixture("bucket");
let mut envelope = request(Method::GET).build();
assert!(envelope.table_list_access().is_none());
AuthorizationRequest::install_table_list_access(&mut envelope, access);
assert!(envelope.table_list_access().is_some());
}
#[test]
fn table_list_access_hides_unauthorized_table_objects_and_reserved_metadata() {
let denied = table_list_access(false);
assert!(!denied.allows_object("tables/table-id/data/file.parquet"));
assert!(
!denied.allows_object(".rustfs-table/warehouses/default/namespaces/sales/tables/orders/metadata/00001.metadata.json")
);
assert!(!denied.allows_object(".rustfs-table/private/catalog.json"));
assert!(denied.allows_object("ordinary/file.txt"));
let allowed = table_list_access(true);
assert!(allowed.allows_object("tables/table-id/data/file.parquet"));
assert!(
!allowed.allows_object(".rustfs-table/warehouses/default/namespaces/sales/tables/orders/metadata/00001.metadata.json")
);
assert!(!allowed.allows_object(".rustfs-table/private/catalog.json"));
}
#[test]
fn table_list_cursor_is_confidential_and_bound_to_the_request_snapshot() {
let access = table_list_access(true);
let marker = "tables/hidden/data/file.parquet";
let token = access
.encode_cursor(
"ListObjectsV2",
"tables/",
Some("/"),
1000,
TableDataPlaneListCursorPosition {
marker: Some(marker.to_string()),
version_marker: None,
common_prefix: None,
},
)
.expect("cursor should encode");
assert!(!token.contains(marker));
let decoded = access
.decode_cursor("ListObjectsV2", "tables/", Some("/"), 1000, &token)
.expect("cursor should decode");
assert_eq!(decoded.marker(), Some(marker));
let mut changed = access.clone();
changed.snapshot_fingerprint = "changed".to_string();
assert!(
changed
.decode_cursor("ListObjectsV2", "tables/", Some("/"), 1000, &token)
.is_err()
);
assert!(
access
.decode_cursor("ListObjectsV2", "other/", Some("/"), 1000, &token)
.is_err()
);
let mut rotated_key = access.clone();
rotated_key.cursor_key[0] ^= 1;
assert!(
rotated_key
.decode_cursor("ListObjectsV2", "tables/", Some("/"), 1000, &token)
.is_err()
);
assert!(
access
.decode_cursor(
"ListObjectsV2",
"tables/",
Some("/"),
1000,
&"x".repeat(TABLE_DATA_PLANE_LIST_CURSOR_MAX_ENCODED_LEN + 1),
)
.is_err()
);
}
#[test]
fn reserved_table_metadata_requires_an_active_table_owner() {
let error = require_owned_reserved_table_object::<()>(
".rustfs-table/warehouses/default/namespaces/analytics/tables/events/metadata/00001.metadata.json",
None,
)
.expect_err("unowned reserved metadata must fail closed");
assert_eq!(error.code(), &S3ErrorCode::AccessDenied);
assert!(
require_owned_reserved_table_object("objects/data.parquet", None::<()>)
.unwrap()
.is_none()
);
assert!(
require_owned_reserved_table_object(".rustfs-table/metadata.json", Some(()))
.unwrap()
.is_some()
);
}
#[test]
fn table_data_plane_publication_guard_covers_content_mutations_only() {
for action in [
S3Action::PutObjectAction,
S3Action::DeleteObjectAction,
S3Action::DeleteObjectVersionAction,
S3Action::RestoreObjectAction,
S3Action::ReplicateObjectAction,
S3Action::ReplicateDeleteAction,
S3Action::PutObjectFanOutAction,
] {
assert!(table_data_plane_content_mutation(Action::S3Action(action)));
}
for action in [
S3Action::GetObjectAction,
S3Action::PutObjectTaggingAction,
S3Action::DeleteObjectTaggingAction,
S3Action::ListBucketAction,
] {
assert!(!table_data_plane_content_mutation(Action::S3Action(action)));
}
}
/// The message `ApiError::error_code_to_message` writes for `ServiceUnavailable`.
const SERVICE_UNAVAILABLE_MESSAGE: &str = "The service is unavailable. Please retry.";
/// The message `ApiError::error_code_to_message` writes for `InternalError`.
const INTERNAL_ERROR_MESSAGE: &str = "We encountered an internal error, please try again.";
/// Whether the error carries a `Retry-After` response header, which is the only place the
/// gateway's error rendering (`gateway_handlers::error::handler_error`) takes it from.
fn carries_retry_after(err: &rustfs_s3_types::S3Error) -> bool {
err.headers()
.is_some_and(|headers| headers.contains_key(http::header::RETRY_AFTER))
}
#[test]
fn table_publication_guard_unavailable_errors_are_retryable() {
use crate::table_catalog::TableCatalogStoreError;
for message in [
"failed to acquire catalog migration lock: Quorum not reached: required 2, achieved 1",
"failed to acquire catalog migration lock: lock acquisition timed out after 5s",
"peer unavailable",
"",
] {
let error = TableCatalogStoreError::Unavailable(message.to_string());
let err = table_publication_guard_error(error);
assert_eq!(err.code(), &S3ErrorCode::ServiceUnavailable, "{message}");
assert_eq!(
err.status_code().or_else(|| err.code().status_code()),
Some(http::StatusCode::SERVICE_UNAVAILABLE)
);
assert_eq!(err.message(), Some(SERVICE_UNAVAILABLE_MESSAGE));
assert!(!carries_retry_after(&err));
}
}
#[test]
fn table_publication_guard_non_retryable_errors_stay_internal() {
use crate::table_catalog::TableCatalogStoreError;
let message = "temporarily unavailable: timeout: quorum not reached".to_string();
for error in [
TableCatalogStoreError::Internal(message.clone()),
TableCatalogStoreError::Invalid(message.clone()),
TableCatalogStoreError::Unsupported(message.clone()),
TableCatalogStoreError::NotFound(message.clone()),
TableCatalogStoreError::NamespaceNotFound(message.clone()),
TableCatalogStoreError::TableNotFound(message.clone()),
TableCatalogStoreError::AlreadyExists(message.clone()),
TableCatalogStoreError::Conflict(message),
] {
let err = table_publication_guard_error(error);
assert_eq!(err.code(), &S3ErrorCode::InternalError);
assert_eq!(
err.status_code().or_else(|| err.code().status_code()),
Some(http::StatusCode::INTERNAL_SERVER_ERROR)
);
assert_eq!(err.message(), Some(INTERNAL_ERROR_MESSAGE));
assert!(!carries_retry_after(&err));
}
}
#[tokio::test]
#[serial]
async fn table_data_plane_cold_lookup_does_not_wait_for_backup_while_holding_publication() {
use crate::table_catalog::{TableCatalogObjectBackend as _, TableCatalogStore as _};
let (_temp_dir, _disk_paths, object_store) = crate::app::gating_test_env::isolated_multi_pool_ecstore().await;
let bucket = "cold-authorization";
object_store
.make_bucket(bucket, &MakeBucketOptions::default())
.await
.expect("create table bucket");
let server_ctx = server_context(object_store.clone());
let mut request_envelope = request(Method::PUT).server_context(&server_ctx).build();
let backend = table_catalog_backend_for_data_plane(&request_envelope).expect("request backend");
let catalog = crate::table_catalog::ConfiguredTableCatalogStore::new_for_test(
backend.clone(),
crate::table_catalog::TableCatalogBackingMode::ObjectBacked,
);
catalog
.put_table_bucket(crate::table_catalog::TableBucketEntry {
version: crate::table_catalog::TABLE_CATALOG_ENTRY_VERSION,
table_bucket: bucket.to_string(),
catalog_type: crate::table_catalog::TABLE_BUCKET_CATALOG_TYPE.to_string(),
warehouse_root: format!("s3://{bucket}/"),
state: crate::table_catalog::TableCatalogEntryState::Active,
properties: Default::default(),
active_rename_id: None,
created_at: None,
updated_at: None,
})
.await
.expect("newly enabled catalog entry");
retain_table_bucket_publication_guard(&mut request_envelope, bucket)
.await
.expect("S3 owns publication first");
let backup_guard = backend
.acquire_write_lock(
crate::admin::storage_api::RUSTFS_META_BUCKET,
&crate::table_catalog::TableCatalogObjectPaths::default().catalog_backup_fence_lock_path(bucket),
)
.await
.expect("backup owns its catalog fence");
let resource = tokio::time::timeout(
std::time::Duration::from_secs(5),
table_data_plane_resource_for_request(&mut request_envelope, bucket, "ordinary-object", true),
)
.await
.expect("cold S3 lookup must not wait for the backup catalog fence")
.expect("resolve newly enabled table bucket");
assert!(resource.is_none());
assert!(
request_envelope
.table_publication_guards()
.expect("retained publication")
.state
.lock()
.missing_resources
.contains(&(bucket.to_string(), "ordinary-object".to_string()))
);
drop(backup_guard);
drop(request_envelope);
let mut read_request = request(Method::GET).server_context(&server_ctx).build();
table_data_plane_resource_for_request(&mut read_request, bucket, "ordinary-object", true)
.await
.expect("unfenced reads retain normal index initialization");
assert!(
backend
.read_object(
crate::admin::storage_api::RUSTFS_META_BUCKET,
&crate::table_catalog::TableCatalogObjectPaths::default().warehouse_index_state_path(bucket),
)
.await
.expect("read initialized index marker")
.is_some()
);
}
#[tokio::test]
#[serial]
async fn multipart_parts_skip_table_publication_fence_but_completion_retains_it() {
let store = crate::app::gating_test_env::shared_gating_ecstore().await;
let server_ctx = server_context(Arc::clone(&store));
let bucket = format!("multipart-table-fence-{}", uuid::Uuid::new_v4());
store
.make_bucket(&bucket, &MakeBucketOptions::default())
.await
.expect("test bucket should be created");
let policy_json = format!(
r#"{{"Version":"2012-10-17","Statement":[{{"Effect":"Allow","Principal":{{"AWS":"*"}},"Action":["s3:GetObject","s3:PutObject"],"Resource":["arn:aws:s3:::{bucket}/*"]}}]}}"#
);
let mut metadata = (*crate::storage::get_bucket_metadata(&bucket)
.await
.expect("new bucket metadata should be cached"))
.clone();
metadata.policy_config = Some(serde_json::from_str(&policy_json).expect("test bucket policy should parse"));
metadata.policy_config_json = policy_json.into_bytes();
crate::storage::storage_api::set_bucket_metadata(bucket.clone(), metadata)
.await
.expect("test bucket policy should be published");
// UploadPart: upload id `upload-id`, part number 1.
let mut part_req = request(Method::PUT).req_info().server_context(&server_ctx).build();
authorize_upload_part(&mut part_req, &server_ctx, &bucket, "object")
.await
.expect("anonymous UploadPart should be authorized by the test policy");
assert!(
part_req.table_publication_guards().is_none(),
"staged part data must not retain a table publication fence"
);
// UploadPartCopy: upload id `upload-id`, part number 2, source `{bucket}/source`.
let mut copy_part_req = request(Method::PUT).req_info().server_context(&server_ctx).build();
authorize_upload_part_copy(
&mut copy_part_req,
&server_ctx,
&bucket,
"object",
Some(CopySourceObject {
bucket: &bucket,
key: "source",
version_id: None,
}),
)
.await
.expect("anonymous UploadPartCopy should be authorized by the test policy");
assert!(
copy_part_req.table_publication_guards().is_none(),
"staged copied part data must not retain a table publication fence"
);
// CompleteMultipartUpload: upload id `upload-id`, an empty part list, no `If-Match`.
let mut complete_req = request(Method::POST).req_info().server_context(&server_ctx).build();
authorize_complete_multipart_upload(&mut complete_req, &server_ctx, &bucket, "object", false)
.await
.expect("anonymous CompleteMultipartUpload should be authorized by the test policy");
let bucket_fence = (bucket, crate::table_catalog::default_table_bucket_publication_lock_path());
let guards = complete_req
.table_publication_guards()
.expect("completion should retain a table publication fence");
assert!(guards.state.lock().keys.contains(&bucket_fence));
}
#[tokio::test]
async fn table_data_plane_request_reuses_table_resource_for_distinct_object_while_commit_waits() {
let resource = crate::table_catalog::TableDataPlaneResource {
table_bucket: "warehouse".to_string(),
namespace: "analytics".to_string(),
table: "events".to_string(),
table_id: "table-id".to_string(),
warehouse_object_prefix: "tables/table-id/".to_string(),
};
let cached_key = ("warehouse".to_string(), "tables/table-id/".to_string());
let requested_key = ("warehouse".to_string(), "tables/table-id/data/part-00002.parquet".to_string());
let retained = TableDataPlanePublicationGuards::default();
retained.state.lock().resources.insert(cached_key, resource.clone());
let publication_lock = Arc::new(tokio::sync::Mutex::new(()));
let request_publication_guard = Arc::clone(&publication_lock).lock_owned().await;
retained.state.lock().guards.push(Box::new(request_publication_guard));
let mut req = request(Method::PUT).extension(retained.clone()).build();
let catalog_lock = Arc::new(tokio::sync::Mutex::new(()));
let commit_started = Arc::new(tokio::sync::Notify::new());
let commit_catalog_lock = Arc::clone(&catalog_lock);
let commit_publication_lock = Arc::clone(&publication_lock);
let commit_started_signal = Arc::clone(&commit_started);
let commit = tokio::spawn(async move {
let _catalog_guard = commit_catalog_lock.lock().await;
commit_started_signal.notify_one();
let _publication_guard = commit_publication_lock.lock().await;
});
commit_started.notified().await;
let resolved = tokio::time::timeout(
std::time::Duration::from_secs(1),
table_data_plane_resource_for_request(&mut req, &requested_key.0, &requested_key.1, true),
)
.await
.expect("cached table resolution must not wait for the catalog lock")
.expect("cached table resolution should succeed");
assert_eq!(resolved, Some(resource));
retained.state.lock().guards.clear();
tokio::time::timeout(std::time::Duration::from_secs(1), commit)
.await
.expect("commit should continue after the request releases its publication guard")
.expect("commit task should join");
}
#[tokio::test]
async fn table_data_plane_request_reuses_reserved_warehouse_resource() {
let resource = crate::table_catalog::TableDataPlaneResource {
table_bucket: "warehouse".to_string(),
namespace: "analytics".to_string(),
table: "events".to_string(),
table_id: "table-id".to_string(),
warehouse_object_prefix: ".rustfs-table/custom-warehouse/events/".to_string(),
};
let retained = TableDataPlanePublicationGuards::default();
retained.state.lock().resources.insert(
(resource.table_bucket.clone(), resource.warehouse_object_prefix.clone()),
resource.clone(),
);
let mut req = request(Method::GET).extension(retained).build();
let resolved = table_data_plane_resource_for_request(
&mut req,
"warehouse",
".rustfs-table/custom-warehouse/events/data/part-00001.parquet",
true,
)
.await
.expect("a reserved-prefix warehouse object should use ordinary table resolution");
assert_eq!(resolved, Some(resource));
}
#[tokio::test]
async fn table_data_plane_request_reuses_absence_while_bucket_publication_guard_is_held() {
let key = ("warehouse".to_string(), "tables/new-table/data/file.parquet".to_string());
let bucket_fence = (key.0.clone(), crate::table_catalog::default_table_bucket_publication_lock_path());
let retained = TableDataPlanePublicationGuards::default();
{
let mut state = retained.state.lock();
state.keys.insert(bucket_fence);
state.missing_resources.insert(key.clone());
}
let publication_lock = Arc::new(tokio::sync::Mutex::new(()));
let request_publication_guard = Arc::clone(&publication_lock).lock_owned().await;
retained.state.lock().guards.push(Box::new(request_publication_guard));
let mut req = request(Method::PUT).extension(retained.clone()).build();
let commit_publication_lock = Arc::clone(&publication_lock);
let commit = tokio::spawn(async move {
let _publication_guard = commit_publication_lock.lock().await;
});
tokio::task::yield_now().await;
let resolved = tokio::time::timeout(
std::time::Duration::from_secs(1),
table_data_plane_resource_for_request(&mut req, &key.0, &key.1, true),
)
.await
.expect("cached absence must not reread the catalog while publication waits")
.expect("cached absence should resolve successfully");
assert!(resolved.is_none());
retained.state.lock().guards.clear();
tokio::time::timeout(std::time::Duration::from_secs(1), commit)
.await
.expect("publication should continue after the request releases its bucket guard")
.expect("publication task should join");
}
+2
View File
@@ -155,6 +155,8 @@ mod legacy_edge;
#[cfg(test)]
pub(crate) use legacy_edge::complete_multipart_upload_input;
#[cfg(test)]
pub(crate) use legacy_edge::copy_input_from_legacy;
#[cfg(test)]
pub(crate) use legacy_edge::list_objects_v2_through_legacy_edge;
#[async_trait::async_trait]
+2
View File
@@ -30,6 +30,8 @@ mod object_misc;
#[cfg(test)]
pub(crate) use object_misc::complete_multipart_upload_input;
#[cfg(test)]
pub(crate) use object_misc::copy_input_from_legacy;
mod object_read;
mod object_write;
@@ -280,7 +280,7 @@ impl DefaultObjectUsecase {
}
}
fn copy_input_from_legacy(input: CopyObjectInput) -> CopyObjectWriteInput {
pub(crate) fn copy_input_from_legacy(input: CopyObjectInput) -> CopyObjectWriteInput {
let condition = |value: ETagCondition| match value {
ETagCondition::Any => CopySourceCondition::Any,
ETagCondition::ETag(ETag::Strong(value)) => CopySourceCondition::Strong(value),
+10 -28
View File
@@ -33,8 +33,14 @@ impl FS {
req: S3Request<PutObjectInput>,
) -> S3Result<S3Response<PutObjectOutput>> {
crate::hp_guard!("S3::put_object");
let usecase = s3_api::object_usecase_for(self);
usecase.execute_put_object(self, req).await
record_s3_op(S3Operation::PutObject);
let envelope = RequestEnvelope::from_s3s(&req);
let input = PutObjectWriteInput::from_legacy(req.input, req.trailing_headers);
s3_api::object_usecase_for(self)
.execute_put_object_owned(envelope, input)
.await
.map(put_output_to_legacy)
.map_err(Into::into)
}
#[instrument(name = "upload_part", level = "debug", skip(self, req))]
@@ -51,8 +57,8 @@ impl FS {
use crate::app::RequestEnvelope;
use crate::storage::s3_api::{
DefaultMultipartUsecase, DefaultObjectUsecase, PutObjectWriteInput, PutObjectWriteOutput, UploadPartTarget,
UploadPartWriteInput, UploadPartWriteOutput,
DefaultMultipartUsecase, PutObjectWriteInput, PutObjectWriteOutput, UploadPartTarget, UploadPartWriteInput,
UploadPartWriteOutput,
};
impl PutObjectWriteInput {
@@ -119,30 +125,6 @@ fn put_output_to_legacy(output: PutObjectWriteOutput) -> S3Response<PutObjectOut
response
}
impl DefaultObjectUsecase {
#[instrument(name = "execute_put_object", level = "info", skip(self, _fs, req))]
#[hotpath::measure(impl_type = "DefaultObjectUsecase", future = true)]
pub async fn execute_put_object(&self, _fs: &FS, req: S3Request<PutObjectInput>) -> S3Result<S3Response<PutObjectOutput>> {
record_s3_op(S3Operation::PutObject);
let envelope = RequestEnvelope::from_s3s(&req);
let input = PutObjectWriteInput::from_legacy(req.input, req.trailing_headers);
self.execute_put_object_owned(envelope, input)
.await
.map(put_output_to_legacy)
.map_err(Into::into)
}
pub async fn execute_put_object_extract(&self, req: S3Request<PutObjectInput>) -> S3Result<S3Response<PutObjectOutput>> {
record_s3_op(S3Operation::PutObject);
let envelope = RequestEnvelope::from_s3s(&req);
let input = PutObjectWriteInput::from_legacy(req.input, req.trailing_headers);
self.execute_put_object_extract_owned(envelope, input)
.await
.map(put_output_to_legacy)
.map_err(Into::into)
}
}
impl UploadPartWriteInput {
pub(crate) fn from_legacy(input: UploadPartInput, trailers: Option<TrailingHeaders>) -> Self {
Self {
+1
View File
@@ -483,6 +483,7 @@ pub(crate) mod tonic_service_consumer {
#[cfg(test)]
pub(crate) mod test_consumer {
pub(crate) use super::super::access_owned::object_read::authorize_head_object;
pub(crate) mod sse {
pub(crate) use super::super::super::sse::TestAuthorizationHooks;
pub(crate) use super::super::super::sse::tests::{RecordingKmsKeyAuthorizer, sse_kms_object_metadata};
+11 -9
View File
@@ -79,13 +79,15 @@ SEAMS = {
# P2 splits bucket_usecase.rs into bucket_usecase/*.rs; narrow this to the listing file once it exists.
owner_dirs=("rustfs/src/app/bucket_usecase/",),
),
"execute_put_object": Seam(
"execute_put_object_owned": Seam(
batch="D",
seam_file="rustfs/src/app/test_seams/put_object.rs",
owners=(
"rustfs/src/app/object/put.rs",
"rustfs/src/app/lifecycle_transition_api_test/put.rs",
),
# PostObject reaches the PutObject use case; its body tests are PutObject's own.
owner_dirs=("rustfs/src/app/object/post_body/",),
),
}
@@ -396,7 +398,7 @@ SEAM_BODY = {
"execute_get_object": "pub(crate) fn get_object(u: &U, r: R) -> F { u.execute_get_object_fields(r) }\n",
"execute_head_object": "pub(crate) fn head_object(u: &U, r: R) -> F { u.execute_head_object_fields(r) }\n",
"execute_list_objects_v2": "pub(crate) fn list_objects_v2(u: &U, r: R) -> F { u.execute_list_objects_v2(r) }\n",
"execute_put_object": "pub(crate) fn put_object(u: &U, f: &Fs, r: R) -> F { u.execute_put_object(f, r) }\n",
"execute_put_object_owned": "pub(crate) fn put_object(u: &U, f: &Fs, r: R) -> F { u.execute_put_object_owned(f, r) }\n",
}
@@ -506,7 +508,7 @@ class SelfTest(unittest.TestCase):
self.assertEqual(len(self.run_check()), 1)
def test_file_declared_under_cfg_test_fails(self):
self.write("rustfs/src/app/gating_test.rs", "fn t() { u.execute_put_object(&fs, r); }\n")
self.write("rustfs/src/app/gating_test.rs", "fn t() { u.execute_put_object_owned(&fs, r); }\n")
self.assertEqual(len(self.run_check()), 1)
def test_file_whose_ancestor_is_cfg_test_fails(self):
@@ -515,7 +517,7 @@ class SelfTest(unittest.TestCase):
self.assertEqual(len(self.run_check()), 1)
def test_put_from_non_owner_lifecycle_file_fails(self):
self.write("rustfs/src/app/lifecycle_transition_api_test/delete.rs", "#[test]\nfn t() { u.execute_put_object(&fs, r); }\n")
self.write("rustfs/src/app/lifecycle_transition_api_test/delete.rs", "#[test]\nfn t() { u.execute_put_object_owned(&fs, r); }\n")
self.write("rustfs/src/app/mod.rs", self.files["rustfs/src/app/mod.rs"].replace("mod lifecycle", "#[cfg(test)]\nmod lifecycle"))
self.assertEqual(len(self.run_check()), 1)
@@ -533,14 +535,14 @@ class SelfTest(unittest.TestCase):
self.assertRegex(" ".join(self.run_check()), "seam for execute_get_object is missing")
def test_seam_that_stopped_wrapping_fails(self):
self.write(SEAMS["execute_put_object"].seam_file, "pub(crate) fn put_object() { todo!() }\n")
self.assertRegex(" ".join(self.run_check()), "must call execute_put_object exactly once, found 0")
self.write(SEAMS["execute_put_object_owned"].seam_file, "pub(crate) fn put_object() { todo!() }\n")
self.assertRegex(" ".join(self.run_check()), "must call execute_put_object_owned exactly once, found 0")
def test_missing_owner_file_fails(self):
owner = SEAMS["execute_put_object"].owners[0]
owner = SEAMS["execute_put_object_owned"].owners[0]
(self.root / owner).unlink()
del self.files[owner]
self.assertRegex(" ".join(self.run_check()), "owner file of execute_put_object is missing")
self.assertRegex(" ".join(self.run_check()), "owner file of execute_put_object_owned is missing")
def test_undeclared_file_with_direct_call_fails_loudly(self):
self.write("rustfs/src/app/orphan.rs", "fn t() { u.execute_get_object(r); }\n")
@@ -561,7 +563,7 @@ class SelfTest(unittest.TestCase):
self.assertEqual(self.run_check(), [])
def test_owner_test_and_production_calls_pass(self):
self.write("rustfs/src/app/object/put.rs", "fn prod() {}\n#[cfg(test)]\nmod tests {\n fn t() { u.execute_put_object(&fs, r); }\n}\n")
self.write("rustfs/src/app/object/put.rs", "fn prod() {}\n#[cfg(test)]\nmod tests {\n fn t() { u.execute_put_object_owned(&fs, r); }\n}\n")
self.assertEqual(self.run_check(), [])
def path_module(self, attributes: str = "", body: str = "fn prod() { u.execute_get_object(r); }\n"):