mirror of
https://github.com/rustfs/rustfs.git
synced 2026-11-19 06:24:17 +00:00
Compare commits
2 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 7b7f96924b | |||
| 4c869cd08c |
@@ -1405,7 +1405,7 @@ impl BucketMetadata {
|
||||
/// | public access block | Fails closed in `get_public_access_block_config`: degrading grants the anonymous access the operator asked to block. |
|
||||
/// | quota | Fails closed in `get_quota_config`; the enforcement path in `quota::checker` already re-parses the raw JSON and refuses on error. |
|
||||
/// | lifecycle | `get_lifecycle_config` fails closed, so GetBucketLifecycle reports the fault instead of NoSuchLifecycleConfiguration. ILM, scanner and expiry-header consumers still degrade to "no rules": nothing is deleted or moved on the strength of an unreadable rule set, and the bucket keeps serving reads and writes. |
|
||||
/// | notification | `get_notification_config` fails closed, so GetBucketNotificationConfiguration reports the fault and startup skips that bucket's rule registration; metadata-bound server dispatch suppresses external delivery for the unreadable snapshot. Other buckets are unaffected. |
|
||||
/// | notification | `get_notification_config` fails closed, so GetBucketNotificationConfiguration reports the fault and startup leaves that one bucket's rules unchanged instead of clearing them; other buckets are unaffected. |
|
||||
/// | tagging, CORS, logging, website, accelerate, request payment | The getter fails closed so the matching GET API reports the fault instead of "not configured". Per-request consumers (CORS response headers) still degrade to "not configured", which is the restrictive direction. |
|
||||
/// | bucket ACL | Safe to degrade: it only shapes an optional response. |
|
||||
///
|
||||
|
||||
@@ -2612,10 +2612,6 @@ impl BucketMetadataSys {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn cached_metadata(&self) -> Vec<Arc<BucketMetadata>> {
|
||||
self.metadata_map.read().await.values().cloned().collect()
|
||||
}
|
||||
|
||||
pub async fn set(&self, bucket: String, bm: Arc<BucketMetadata>) {
|
||||
self.revoke_pending_bucket_creation(&bucket).await;
|
||||
bump_local_meta_mutation_gen();
|
||||
@@ -5764,64 +5760,6 @@ mod tests {
|
||||
assert_eq!(tags.tag_set.len(), WRITERS, "every concurrent rewrite must survive: {tags:?}");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn notification_snapshots_follow_periodic_refresh_after_lost_peer_signals() {
|
||||
let (dirs, store) = isolated_store_over_temp_disks().await;
|
||||
let writer = BucketMetadataSys::new(store.clone());
|
||||
let peer = Arc::new(RwLock::new(BucketMetadataSys::new(store.clone())));
|
||||
let bucket = "notification-periodic-snapshots";
|
||||
for dir in &dirs {
|
||||
std::fs::create_dir_all(dir.path().join(bucket)).expect("create empty fixture bucket");
|
||||
}
|
||||
let mut original = BucketMetadata::new(bucket);
|
||||
original.notification_config_xml = concat!(
|
||||
"<NotificationConfiguration><QueueConfiguration><Event>s3:ObjectCreated:Put</Event>",
|
||||
"<Queue>arn:rustfs:sqs:us-east-1:inert:webhook</Queue></QueueConfiguration></NotificationConfiguration>"
|
||||
)
|
||||
.as_bytes()
|
||||
.to_vec();
|
||||
original.parse_all_configs().expect("parse inert subscription");
|
||||
writer
|
||||
.persist_new_and_set(original)
|
||||
.await
|
||||
.expect("persist initial subscription");
|
||||
peer.read().await.reload_from_store(bucket).await.expect("prime peer cache");
|
||||
let old = peer.read().await.get(bucket).await.expect("cached original snapshot");
|
||||
|
||||
let mut removed = (*old).clone();
|
||||
removed.notification_config_xml = b"<NotificationConfiguration/>".to_vec();
|
||||
removed.parse_all_configs().expect("parse removed subscription");
|
||||
writer
|
||||
.persist_new_and_set(removed)
|
||||
.await
|
||||
.expect("persist removal without peer callback");
|
||||
assert!(Arc::ptr_eq(&old, &peer.read().await.get(bucket).await.expect("peer remains stale")));
|
||||
|
||||
// Exercise the timer's actual one-pass entry point, not the peer RPC.
|
||||
refresh_buckets_metadata_once(peer.clone()).await;
|
||||
let snapshots = peer.read().await.cached_metadata().await;
|
||||
assert_eq!(snapshots.len(), 1);
|
||||
assert!(!Arc::ptr_eq(&old, &snapshots[0]));
|
||||
assert_eq!(old.bucket_incarnation_id, snapshots[0].bucket_incarnation_id);
|
||||
assert_eq!(snapshots[0].notification_config_xml, b"<NotificationConfiguration/>");
|
||||
assert!(
|
||||
snapshots[0]
|
||||
.notification_config
|
||||
.as_ref()
|
||||
.expect("parsed removal")
|
||||
.behavior()
|
||||
.queue_routes
|
||||
.is_empty()
|
||||
);
|
||||
|
||||
for dir in &dirs {
|
||||
std::fs::remove_dir(dir.path().join(bucket)).expect("remove empty fixture bucket without peer callback");
|
||||
}
|
||||
refresh_buckets_metadata_once(peer.clone()).await;
|
||||
assert!(peer.read().await.cached_metadata().await.is_empty());
|
||||
assert!(peer.read().await.get(bucket).await.is_err());
|
||||
}
|
||||
|
||||
/// Pins the peer reload-notification contract (`reload_from_store`, the
|
||||
/// LoadBucketMetadata RPC path): only metadata actually read from
|
||||
/// persisted storage enters the cache. A load miss errors out and leaves
|
||||
|
||||
@@ -775,15 +775,6 @@ impl ECStore {
|
||||
sys.read().await.get(bucket).await
|
||||
}
|
||||
|
||||
/// Snapshots this instance's cached bucket metadata without disk IO.
|
||||
///
|
||||
/// # Errors
|
||||
/// Returns an error when this instance's metadata system is not initialized.
|
||||
pub async fn cached_bucket_metadata(&self) -> Result<Vec<Arc<BucketMetadata>>> {
|
||||
let sys = metadata_sys::require_bucket_metadata_sys_in(&self.ctx)?;
|
||||
Ok(sys.read().await.cached_metadata().await)
|
||||
}
|
||||
|
||||
/// Resolve stored tags for authorization. Only confirmed absence returns an
|
||||
/// empty map; unavailable, fabricated or malformed metadata must not grant access.
|
||||
pub async fn get_bucket_tags_for_policy(&self, bucket: &str) -> Result<HashMap<String, String>> {
|
||||
|
||||
@@ -47,18 +47,12 @@ impl NotifyBucketConfigManager {
|
||||
}
|
||||
|
||||
pub async fn has_subscriber(&self, bucket: &str, event: &EventName) -> bool {
|
||||
// The standalone index is only a fast negative check for name-only
|
||||
// rules. Metadata-bound rules can change without an index callback.
|
||||
if !self.rule_engine.uses_bucket_metadata_source() && !self.subscriber_view.has_subscriber(bucket, event) {
|
||||
if !self.subscriber_view.has_subscriber(bucket, event) {
|
||||
return false;
|
||||
}
|
||||
self.rule_engine.has_subscriber(bucket, event).await
|
||||
}
|
||||
|
||||
pub(crate) fn bind_bucket_metadata_source(&self, store: &Arc<crate::NotifyStore>) {
|
||||
self.rule_engine.bind_bucket_metadata_source(store);
|
||||
}
|
||||
|
||||
/// Validates that the runtime can accept `cfg` for `bucket` without
|
||||
/// mutating any state.
|
||||
pub async fn validate_bucket_notification_config(
|
||||
|
||||
@@ -249,16 +249,6 @@ impl NotificationSystem {
|
||||
self.services.bucket_config_manager.has_subscriber(bucket, event).await
|
||||
}
|
||||
|
||||
/// Binds server dispatch to the store-owned metadata cache. Local writes,
|
||||
/// peer reloads, deletes and periodic metadata refreshes then select rules
|
||||
/// from the same snapshot; delayed name-only rule replacements are ignored
|
||||
/// and clears only evict compiled snapshots. Missing/unreadable metadata or
|
||||
/// a dropped store suppresses external dispatch without falling back to the
|
||||
/// standalone rules. Already queued or in-flight deliveries are not recalled.
|
||||
pub fn bind_bucket_metadata_source(&self, store: &Arc<crate::NotifyStore>) {
|
||||
self.services.bucket_config_manager.bind_bucket_metadata_source(store);
|
||||
}
|
||||
|
||||
/// Returns true when at least one in-process consumer is subscribed to live events.
|
||||
pub fn has_live_listeners(&self) -> bool {
|
||||
self.services.pipeline.has_live_listeners()
|
||||
|
||||
@@ -12,14 +12,12 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::rules::xml_config::{FilterRule, S3KeyFilter};
|
||||
use crate::rules::{RulesMap, TargetIdSet};
|
||||
use crate::storage_api::{BucketMetadata, NotifyStore};
|
||||
use percent_encoding::percent_decode_str;
|
||||
use rustfs_s3_types::EventName;
|
||||
use rustfs_targets::arn::{ARN, TargetID};
|
||||
use rustfs_targets::arn::TargetID;
|
||||
use starshard::{AsyncShardedHashMap, DEFAULT_SHARDS, SnapshotMode};
|
||||
use std::sync::{Arc, RwLock, Weak};
|
||||
use std::sync::Arc;
|
||||
use tracing::info;
|
||||
|
||||
const LOG_COMPONENT_NOTIFY: &str = "notify";
|
||||
@@ -37,82 +35,20 @@ fn decoded_object_key_for_matching(object_key: &str) -> Option<String> {
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct NotifyRuleEngine {
|
||||
bucket_rules_map: Arc<AsyncShardedHashMap<String, CachedBucketRules, rustc_hash::FxBuildHasher>>,
|
||||
// Never hold this short synchronous guard across an await. Metadata cache
|
||||
// reads precede the compiled-rules cache, so there is no reverse lock order.
|
||||
metadata_source: Arc<RwLock<Option<Weak<NotifyStore>>>>,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
struct CachedBucketRules {
|
||||
metadata: Option<Weak<BucketMetadata>>,
|
||||
// Cache rejected snapshots too, so corrupt persisted routes are not
|
||||
// repeatedly compiled on the event hot path.
|
||||
rules: Option<RulesMap>,
|
||||
}
|
||||
|
||||
fn rules_from_metadata(metadata: &BucketMetadata) -> Option<RulesMap> {
|
||||
let mut rules = RulesMap::new();
|
||||
let Some(config) = metadata.notification_config.as_ref() else {
|
||||
// Unreadable metadata is not authority to keep an old subscription.
|
||||
return metadata.notification_config_xml.is_empty().then_some(rules);
|
||||
};
|
||||
let config = config.behavior();
|
||||
for route in config
|
||||
.queue_routes
|
||||
.into_iter()
|
||||
.chain(config.topic_routes)
|
||||
.chain(config.lambda_routes)
|
||||
{
|
||||
let target = ARN::parse(&route.destination).ok()?.target_id;
|
||||
let events: Vec<_> = route.events.iter().filter_map(|event| EventName::parse(event).ok()).collect();
|
||||
let mut filter = S3KeyFilter::default();
|
||||
for (name, value) in route.filter_rules.into_iter().flatten() {
|
||||
filter.filter_rule_list.push(FilterRule {
|
||||
name: name?,
|
||||
value: value?,
|
||||
});
|
||||
}
|
||||
filter.validate().ok()?;
|
||||
rules.add_rule_config(&events, filter.pattern(), target);
|
||||
}
|
||||
Some(rules)
|
||||
bucket_rules_map: Arc<AsyncShardedHashMap<String, RulesMap, rustc_hash::FxBuildHasher>>,
|
||||
}
|
||||
|
||||
impl NotifyRuleEngine {
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
bucket_rules_map: Arc::new(AsyncShardedHashMap::with_snapshot_mode(DEFAULT_SHARDS, SnapshotMode::Cached)),
|
||||
metadata_source: Arc::new(RwLock::new(None)),
|
||||
}
|
||||
}
|
||||
|
||||
/// Uses this store's immutable metadata snapshots as rule authority. Once
|
||||
/// bound, missing metadata or a dropped store fails closed, never falling
|
||||
/// back to the standalone name-only table. The weak handle avoids cycles.
|
||||
pub fn bind_bucket_metadata_source(&self, store: &Arc<NotifyStore>) {
|
||||
*self.metadata_source.write().unwrap_or_else(|err| err.into_inner()) = Some(Arc::downgrade(store));
|
||||
}
|
||||
|
||||
fn metadata_source(&self) -> Option<Weak<NotifyStore>> {
|
||||
self.metadata_source.read().unwrap_or_else(|err| err.into_inner()).clone()
|
||||
}
|
||||
|
||||
pub(crate) fn uses_bucket_metadata_source(&self) -> bool {
|
||||
self.metadata_source().is_some()
|
||||
}
|
||||
|
||||
pub async fn is_target_bound_to_any_bucket(&self, target_id: &TargetID) -> bool {
|
||||
if let Some(source) = self.metadata_source() {
|
||||
let Some(store) = source.upgrade() else { return true };
|
||||
let Ok(metadata) = store.cached_bucket_metadata().await else { return true };
|
||||
return metadata
|
||||
.iter()
|
||||
.any(|metadata| rules_from_metadata(metadata).is_none_or(|rules| rules.contains_target_id(target_id)));
|
||||
}
|
||||
let items = self.bucket_rules_map.iter().await;
|
||||
for (_bucket, rules_map) in items {
|
||||
if rules_map.rules.is_some_and(|rules| rules.contains_target_id(target_id)) {
|
||||
if rules_map.contains_target_id(target_id) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
@@ -120,24 +56,11 @@ impl NotifyRuleEngine {
|
||||
}
|
||||
|
||||
pub async fn set_bucket_rules(&self, bucket: &str, rules_map: RulesMap) {
|
||||
if self.uses_bucket_metadata_source() {
|
||||
// Delayed PUT/startup callbacks cannot override a newer revision or
|
||||
// a same-name replacement. Only published metadata is authoritative.
|
||||
return;
|
||||
}
|
||||
let event_count = rules_map.iter_events().count();
|
||||
if rules_map.is_empty() {
|
||||
self.bucket_rules_map.remove(&bucket.to_string()).await;
|
||||
} else {
|
||||
self.bucket_rules_map
|
||||
.insert(
|
||||
bucket.to_string(),
|
||||
CachedBucketRules {
|
||||
metadata: None,
|
||||
rules: Some(rules_map),
|
||||
},
|
||||
)
|
||||
.await;
|
||||
self.bucket_rules_map.insert(bucket.to_string(), rules_map).await;
|
||||
}
|
||||
info!(
|
||||
event = EVENT_NOTIFY_RULES_UPDATED,
|
||||
@@ -151,47 +74,7 @@ impl NotifyRuleEngine {
|
||||
}
|
||||
|
||||
pub async fn get_bucket_rules(&self, bucket: &str) -> Option<RulesMap> {
|
||||
if let Some(source) = self.metadata_source() {
|
||||
let metadata = match source.upgrade() {
|
||||
Some(store) => store.get_bucket_metadata(bucket).await.ok(),
|
||||
None => None,
|
||||
};
|
||||
return self.rules_for_metadata(bucket, metadata).await;
|
||||
}
|
||||
self.bucket_rules_map
|
||||
.get(&bucket.to_string())
|
||||
.await
|
||||
.and_then(|cached| cached.rules)
|
||||
}
|
||||
|
||||
async fn rules_for_metadata(&self, bucket: &str, metadata: Option<Arc<BucketMetadata>>) -> Option<RulesMap> {
|
||||
let key = bucket.to_owned();
|
||||
let Some(metadata) = metadata else {
|
||||
self.bucket_rules_map.remove(&key).await;
|
||||
return None;
|
||||
};
|
||||
if let Some(cached) = self.bucket_rules_map.get(&key).await
|
||||
&& cached
|
||||
.metadata
|
||||
.as_ref()
|
||||
.is_some_and(|snapshot| Weak::ptr_eq(snapshot, &Arc::downgrade(&metadata)))
|
||||
{
|
||||
return cached.rules;
|
||||
}
|
||||
// Snapshot identity covers incarnation, revision AND bytes, without
|
||||
// trusting wall-clock ordering. A late compilation may evict a newer
|
||||
// cache entry, but cannot be reused by a reader of a different snapshot.
|
||||
let rules = rules_from_metadata(&metadata);
|
||||
self.bucket_rules_map
|
||||
.insert(
|
||||
key,
|
||||
CachedBucketRules {
|
||||
metadata: Some(Arc::downgrade(&metadata)),
|
||||
rules: rules.clone(),
|
||||
},
|
||||
)
|
||||
.await;
|
||||
rules
|
||||
self.bucket_rules_map.get(&bucket.to_string()).await
|
||||
}
|
||||
|
||||
pub async fn clear_bucket_rules(&self, bucket: &str) {
|
||||
@@ -236,179 +119,8 @@ impl Default for NotifyRuleEngine {
|
||||
mod tests {
|
||||
use super::NotifyRuleEngine;
|
||||
use crate::rules::RulesMap;
|
||||
use crate::storage_api::BucketMetadata;
|
||||
use rustfs_s3_types::EventName;
|
||||
use rustfs_targets::arn::TargetID;
|
||||
use std::sync::{Arc, Weak};
|
||||
|
||||
fn metadata(target: &str) -> Arc<BucketMetadata> {
|
||||
let mut metadata = BucketMetadata::new("bucket");
|
||||
metadata.notification_config_xml = format!(
|
||||
"<NotificationConfiguration><QueueConfiguration><Event>s3:ObjectCreated:Put</Event>\
|
||||
<Queue>arn:rustfs:sqs:us-east-1:{target}:webhook</Queue></QueueConfiguration></NotificationConfiguration>"
|
||||
)
|
||||
.into_bytes();
|
||||
metadata.notification_config = Some(
|
||||
rustfs_ecstore::api::bucket::utils::deserialize(&metadata.notification_config_xml)
|
||||
.expect("parse inert notification fixture"),
|
||||
);
|
||||
Arc::new(metadata)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn notification_snapshot_cache_never_reuses_replaced_incarnation_or_revision() {
|
||||
let engine = NotifyRuleEngine::new();
|
||||
let old = metadata("old");
|
||||
let mut replacement = (*metadata("new")).clone();
|
||||
// Identical timestamps must not make different configurations equal.
|
||||
replacement.notification_config_updated_at = old.notification_config_updated_at;
|
||||
let replacement = Arc::new(replacement);
|
||||
assert_ne!(old.bucket_incarnation_id, replacement.bucket_incarnation_id);
|
||||
let old_target = TargetID::new("old".into(), "webhook".into());
|
||||
let new_target = TargetID::new("new".into(), "webhook".into());
|
||||
|
||||
for (snapshot, wanted, unwanted) in [
|
||||
(old.clone(), &old_target, &new_target),
|
||||
(replacement.clone(), &new_target, &old_target),
|
||||
// Simulate a late compilation of a captured, older snapshot.
|
||||
(old.clone(), &old_target, &new_target),
|
||||
(replacement.clone(), &new_target, &old_target),
|
||||
] {
|
||||
let rules = engine
|
||||
.rules_for_metadata("bucket", Some(snapshot))
|
||||
.await
|
||||
.expect("readable snapshot");
|
||||
let targets = rules.match_rules(EventName::ObjectCreatedPut, "object");
|
||||
assert!(targets.contains(wanted));
|
||||
assert!(!targets.contains(unwanted));
|
||||
}
|
||||
|
||||
let mut next_revision = (*old).clone();
|
||||
next_revision.notification_config = replacement.notification_config.clone();
|
||||
next_revision.notification_config_xml = replacement.notification_config_xml.clone();
|
||||
let rules = engine
|
||||
.rules_for_metadata("bucket", Some(Arc::new(next_revision)))
|
||||
.await
|
||||
.expect("same-incarnation revision");
|
||||
assert!(!rules.contains_target_id(&old_target));
|
||||
assert!(rules.contains_target_id(&new_target));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn notification_snapshot_absence_and_corruption_never_fall_back_to_old_rules() {
|
||||
let engine = NotifyRuleEngine::new();
|
||||
let old = metadata("old");
|
||||
assert!(engine.rules_for_metadata("bucket", Some(old.clone())).await.is_some());
|
||||
|
||||
let mut unreadable = (*old).clone();
|
||||
unreadable.notification_config = None;
|
||||
unreadable.notification_config_xml = b"<broken>".to_vec();
|
||||
assert!(
|
||||
engine
|
||||
.rules_for_metadata("bucket", Some(Arc::new(unreadable)))
|
||||
.await
|
||||
.is_none()
|
||||
);
|
||||
assert!(
|
||||
engine
|
||||
.bucket_rules_map
|
||||
.get(&"bucket".to_owned())
|
||||
.await
|
||||
.expect("cache rejected snapshot")
|
||||
.rules
|
||||
.is_none()
|
||||
);
|
||||
assert!(engine.rules_for_metadata("bucket", None).await.is_none());
|
||||
assert!(engine.bucket_rules_map.get(&"bucket".to_owned()).await.is_none());
|
||||
|
||||
// A recreated bucket without notifications has no inherited targets.
|
||||
let replacement = Arc::new(BucketMetadata::new("bucket"));
|
||||
assert!(
|
||||
engine
|
||||
.rules_for_metadata("bucket", Some(replacement))
|
||||
.await
|
||||
.expect("empty snapshot")
|
||||
.is_empty()
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn notification_snapshot_binding_never_falls_back_after_store_loss() {
|
||||
let engine = NotifyRuleEngine::new();
|
||||
let target = TargetID::new("old".into(), "webhook".into());
|
||||
let mut rules = RulesMap::new();
|
||||
rules.add_rule_config(&[EventName::ObjectCreatedPut], "*".into(), target.clone());
|
||||
engine.set_bucket_rules("bucket", rules.clone()).await;
|
||||
assert!(engine.has_subscriber("bucket", &EventName::ObjectCreatedPut).await);
|
||||
|
||||
*engine.metadata_source.write().expect("metadata source lock") = Some(Weak::new());
|
||||
// A delayed validated PUT must not revive the standalone table, even
|
||||
// when its owning store is gone. Target removal also fails closed.
|
||||
engine.set_bucket_rules("bucket", rules).await;
|
||||
assert!(!engine.has_subscriber("bucket", &EventName::ObjectCreatedPut).await);
|
||||
assert!(
|
||||
engine
|
||||
.match_targets("bucket", EventName::ObjectCreatedPut, "object")
|
||||
.await
|
||||
.is_empty()
|
||||
);
|
||||
assert!(engine.is_target_bound_to_any_bucket(&target).await);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn notification_snapshot_compilation_preserves_destinations_events_and_filters() {
|
||||
let mut metadata = (*metadata("queue")).clone();
|
||||
metadata.notification_config_xml = concat!(
|
||||
"<NotificationConfiguration>",
|
||||
"<QueueConfiguration><Event>s3:ObjectCreated:*</Event><Event>s3:Future:Unrecognized</Event>",
|
||||
"<Queue>arn:rustfs:sqs:us-east-1:queue:webhook</Queue>",
|
||||
"<Filter><S3Key><FilterRule><Name>Prefix</Name><Value>logs/</Value></FilterRule>",
|
||||
"<FilterRule><Name>Suffix</Name><Value>.csv</Value></FilterRule></S3Key></Filter></QueueConfiguration>",
|
||||
"<TopicConfiguration><Event>s3:ObjectRemoved:*</Event><Topic>arn:rustfs:sqs:us-east-1:topic:webhook</Topic></TopicConfiguration>",
|
||||
"<CloudFunctionConfiguration><Event>s3:ObjectCreated:Put</Event>",
|
||||
"<CloudFunction>arn:rustfs:sqs:us-east-1:lambda:webhook</CloudFunction></CloudFunctionConfiguration>",
|
||||
"</NotificationConfiguration>"
|
||||
).as_bytes().to_vec();
|
||||
metadata.notification_config =
|
||||
Some(rustfs_ecstore::api::bucket::utils::deserialize(&metadata.notification_config_xml).expect("parse destinations"));
|
||||
let rules = super::rules_from_metadata(&metadata).expect("compile destinations");
|
||||
assert_eq!(rules.match_rules(EventName::ObjectCreatedPut, "logs/data.csv").len(), 2);
|
||||
assert_eq!(rules.match_rules(EventName::ObjectCreatedPut, "other/data.csv").len(), 1);
|
||||
assert_eq!(rules.match_rules(EventName::ObjectCreatedPut, "logs/data.txt").len(), 1);
|
||||
let removed = rules.match_rules(EventName::ObjectRemovedDelete, "object");
|
||||
assert_eq!(removed.len(), 1);
|
||||
assert!(removed.contains(&TargetID::new("topic".into(), "webhook".into())));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn notification_snapshot_compilation_rejects_invalid_persisted_routes() {
|
||||
for (destination, filter) in [
|
||||
("not-an-arn", ""),
|
||||
(
|
||||
"arn:rustfs:sqs:us-east-1:inert:webhook",
|
||||
"<Filter><S3Key><FilterRule><Value>logs/</Value></FilterRule></S3Key></Filter>",
|
||||
),
|
||||
(
|
||||
"arn:rustfs:sqs:us-east-1:inert:webhook",
|
||||
concat!(
|
||||
"<Filter><S3Key><FilterRule><Name>prefix</Name><Value>logs/</Value></FilterRule>",
|
||||
"<FilterRule><Name>Prefix</Name><Value>private/</Value></FilterRule></S3Key></Filter>"
|
||||
),
|
||||
),
|
||||
] {
|
||||
let mut metadata = BucketMetadata::new("bucket");
|
||||
metadata.notification_config_xml = format!(
|
||||
"<NotificationConfiguration><QueueConfiguration><Event>s3:ObjectCreated:Put</Event>\
|
||||
<Queue>{destination}</Queue>{filter}</QueueConfiguration></NotificationConfiguration>"
|
||||
)
|
||||
.into_bytes();
|
||||
metadata.notification_config = Some(
|
||||
rustfs_ecstore::api::bucket::utils::deserialize(&metadata.notification_config_xml)
|
||||
.expect("parse invalid but well-formed persisted route"),
|
||||
);
|
||||
assert!(super::rules_from_metadata(&metadata).is_none(), "invalid route must not dispatch");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rule_engine_tracks_bucket_rule_lifecycle() {
|
||||
|
||||
@@ -14,8 +14,6 @@
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
pub(crate) use rustfs_ecstore::api::bucket::metadata::BucketMetadata;
|
||||
|
||||
use rustfs_ecstore::api::config::com::{
|
||||
read_existing_server_config_no_lock as read_existing_notify_config_from_backend_no_lock,
|
||||
read_server_config_snapshot as read_notify_server_config_snapshot_from_backend,
|
||||
|
||||
@@ -22,7 +22,7 @@ use super::storage_api::bucket_usecase::StorageObjectInfo as ObjectInfo;
|
||||
#[cfg(test)]
|
||||
use super::storage_api::bucket_usecase::access::ReqInfo;
|
||||
use super::storage_api::bucket_usecase::access::{
|
||||
authorize_request, bucket_config_mutation_incarnation, log_list_buckets_iam_implicit_deny,
|
||||
TableDataPlaneListAccess, authorize_request, bucket_config_mutation_incarnation, log_list_buckets_iam_implicit_deny,
|
||||
prepare_list_buckets_iam_authorization, prepare_odm_read_generation, req_info_ref,
|
||||
};
|
||||
#[cfg(test)]
|
||||
|
||||
@@ -22,9 +22,7 @@ use crate::app::storage_api::s3::dto::{
|
||||
UserMetadata,
|
||||
};
|
||||
use crate::app::storage_api::s3::xml::{SerResult, SerializeContent};
|
||||
#[cfg(test)]
|
||||
use crate::app::storage_api::s3::{S3ErrorCode, S3Request, S3Response};
|
||||
use crate::app::storage_api::s3::{S3Result, xml};
|
||||
use crate::app::storage_api::s3::{S3Request, S3Response, S3Result, xml};
|
||||
|
||||
type ListObjectVersionsInfo = StorageListObjectVersionsInfo<ObjectInfo>;
|
||||
|
||||
@@ -322,20 +320,31 @@ fn metadata_count_to_i32(value: usize) -> i32 {
|
||||
i32::try_from(value).unwrap_or(i32::MAX)
|
||||
}
|
||||
|
||||
async fn is_list_objects_metadata_action_allowed(
|
||||
envelope: &RequestEnvelope,
|
||||
async fn is_list_objects_metadata_action_allowed<T>(
|
||||
req: &S3Request<T>,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
action: S3Action,
|
||||
) -> S3Result<bool> {
|
||||
let mut auth_req = envelope.fork_for_authorization();
|
||||
let mut req_info = req_info_ref(envelope)?.clone();
|
||||
let mut auth_req = S3Request {
|
||||
input: (),
|
||||
method: req.method.clone(),
|
||||
uri: req.uri.clone(),
|
||||
headers: req.headers.clone(),
|
||||
extensions: req.extensions.clone(),
|
||||
credentials: req.credentials.clone(),
|
||||
region: req.region.clone(),
|
||||
service: req.service.clone(),
|
||||
trailing_headers: req.trailing_headers.clone(),
|
||||
};
|
||||
|
||||
let mut req_info = req_info_ref(req)?.clone();
|
||||
req_info.bucket = Some(bucket.to_string());
|
||||
req_info.object = Some(object.to_string());
|
||||
req_info.version_id = None;
|
||||
// Denial here is an expected filter outcome, not an error (issue #5740).
|
||||
req_info.suppress_denial_log = true;
|
||||
auth_req.install_req_info(req_info);
|
||||
auth_req.extensions.insert(req_info);
|
||||
|
||||
match authorize_request(&mut auth_req, Action::S3Action(action)).await {
|
||||
Ok(()) => Ok(true),
|
||||
@@ -344,8 +353,8 @@ async fn is_list_objects_metadata_action_allowed(
|
||||
}
|
||||
}
|
||||
|
||||
async fn collect_list_objects_metadata_permissions(
|
||||
envelope: &RequestEnvelope,
|
||||
async fn collect_list_objects_metadata_permissions<T>(
|
||||
req: &S3Request<T>,
|
||||
bucket: &str,
|
||||
objects: &[ObjectInfo],
|
||||
) -> S3Result<HashMap<String, ObjectMetadataPermissions>> {
|
||||
@@ -357,9 +366,9 @@ async fn collect_list_objects_metadata_permissions(
|
||||
}
|
||||
|
||||
let metadata_allowed =
|
||||
is_list_objects_metadata_action_allowed(envelope, bucket, &object.name, S3Action::GetObjectAction).await?;
|
||||
is_list_objects_metadata_action_allowed(req, bucket, &object.name, S3Action::GetObjectAction).await?;
|
||||
let tags_allowed =
|
||||
is_list_objects_metadata_action_allowed(envelope, bucket, &object.name, S3Action::GetObjectTaggingAction).await?;
|
||||
is_list_objects_metadata_action_allowed(req, bucket, &object.name, S3Action::GetObjectTaggingAction).await?;
|
||||
|
||||
permissions.insert(
|
||||
object.name.clone(),
|
||||
@@ -596,22 +605,11 @@ fn build_list_objects_v2_metadata_output(
|
||||
}
|
||||
|
||||
impl DefaultBucketUsecase {
|
||||
#[cfg(test)]
|
||||
pub(crate) async fn execute_list_objects_v2m(
|
||||
&self,
|
||||
req: S3Request<ListObjectsV2Input>,
|
||||
) -> S3Result<S3Response<ListObjectsV2MetadataOutput>> {
|
||||
let envelope = RequestEnvelope::from_s3s(&req);
|
||||
self.list_objects_v2_metadata(&envelope, req.input).await.map(S3Response::new)
|
||||
}
|
||||
|
||||
/// One page of `GET /{bucket}?list-type=2&metadata=true`, for the caller the envelope's
|
||||
/// authorization installed: each entry's metadata and tags as far as that caller may read them.
|
||||
pub(crate) async fn list_objects_v2_metadata(
|
||||
&self,
|
||||
envelope: &RequestEnvelope,
|
||||
input: ListObjectsV2Input,
|
||||
) -> S3Result<ListObjectsV2MetadataOutput> {
|
||||
let input = req.input.clone();
|
||||
let ListObjectsV2Input {
|
||||
bucket,
|
||||
continuation_token,
|
||||
@@ -626,14 +624,14 @@ impl DefaultBucketUsecase {
|
||||
|
||||
let params = parse_list_objects_v2_params(prefix, delimiter, max_keys, continuation_token, start_after)?;
|
||||
|
||||
validate_list_object_unordered_with_delimiter(params.delimiter.as_ref(), envelope.uri().query())?;
|
||||
validate_list_object_unordered_with_delimiter(params.delimiter.as_ref(), req.uri.query())?;
|
||||
|
||||
let store = get_validated_store(&bucket).await?;
|
||||
let incl_deleted = get_header(envelope.headers(), rustfs_utils::http::SUFFIX_INCLUDE_DELETED)
|
||||
let incl_deleted = get_header(&req.headers, rustfs_utils::http::SUFFIX_INCLUDE_DELETED)
|
||||
.map(|value| value.as_ref() == "true")
|
||||
.unwrap_or_default();
|
||||
|
||||
let object_infos = match envelope.table_list_access() {
|
||||
let object_infos = match req.extensions.get::<TableDataPlaneListAccess>() {
|
||||
Some(access) => {
|
||||
table_list_isolation::list_objects_v2(
|
||||
store.clone(),
|
||||
@@ -666,34 +664,24 @@ impl DefaultBucketUsecase {
|
||||
.map_err(ApiError::from)?,
|
||||
};
|
||||
|
||||
let permissions = collect_list_objects_metadata_permissions(envelope, &bucket, &object_infos.objects).await?;
|
||||
Ok(build_list_objects_v2_metadata_output(
|
||||
let permissions = collect_list_objects_metadata_permissions(&req, &bucket, &object_infos.objects).await?;
|
||||
let output = build_list_objects_v2_metadata_output(
|
||||
object_infos,
|
||||
&bucket,
|
||||
¶ms,
|
||||
encoding_type.as_ref(),
|
||||
fetch_owner.unwrap_or_default(),
|
||||
&permissions,
|
||||
))
|
||||
);
|
||||
|
||||
Ok(S3Response::new(output))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) async fn execute_list_object_versions_m(
|
||||
&self,
|
||||
req: S3Request<ListObjectVersionsInput>,
|
||||
) -> S3Result<S3Response<ListObjectVersionsMetadataOutput>> {
|
||||
let envelope = RequestEnvelope::from_s3s(&req);
|
||||
self.list_object_versions_metadata(&envelope, req.input)
|
||||
.await
|
||||
.map(S3Response::new)
|
||||
}
|
||||
|
||||
/// One page of `GET /{bucket}?versions&metadata=true`, as [`Self::list_objects_v2_metadata`].
|
||||
pub(crate) async fn list_object_versions_metadata(
|
||||
&self,
|
||||
envelope: &RequestEnvelope,
|
||||
input: ListObjectVersionsInput,
|
||||
) -> S3Result<ListObjectVersionsMetadataOutput> {
|
||||
let input = req.input.clone();
|
||||
let ListObjectVersionsInput {
|
||||
bucket,
|
||||
delimiter,
|
||||
@@ -708,7 +696,7 @@ impl DefaultBucketUsecase {
|
||||
let params = parse_list_object_versions_params(prefix, delimiter, key_marker, version_id_marker, max_keys)?;
|
||||
|
||||
let store = get_validated_store(&bucket).await?;
|
||||
let object_infos = match envelope.table_list_access() {
|
||||
let object_infos = match req.extensions.get::<TableDataPlaneListAccess>() {
|
||||
Some(access) => {
|
||||
table_list_isolation::list_object_versions(
|
||||
store.clone(),
|
||||
@@ -737,14 +725,11 @@ impl DefaultBucketUsecase {
|
||||
.map_err(ApiError::from)?,
|
||||
};
|
||||
|
||||
let permissions = collect_list_objects_metadata_permissions(envelope, &bucket, &object_infos.objects).await?;
|
||||
Ok(build_list_object_versions_metadata_output(
|
||||
object_infos,
|
||||
&bucket,
|
||||
¶ms,
|
||||
encoding_type.as_ref(),
|
||||
&permissions,
|
||||
))
|
||||
let permissions = collect_list_objects_metadata_permissions(&req, &bucket, &object_infos.objects).await?;
|
||||
let output =
|
||||
build_list_object_versions_metadata_output(object_infos, &bucket, ¶ms, encoding_type.as_ref(), &permissions);
|
||||
|
||||
Ok(S3Response::new(output))
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -457,130 +457,3 @@ fn legacy_put_notification_does_not_write_after_the_bucket_is_recreated() {
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn notification_dispatch_follows_peer_reload_delete_and_recreation() {
|
||||
run_large_stack_test("notification-metadata-authority", || async {
|
||||
let ambient = shared_gating_ambient().await;
|
||||
let store = ambient.object_store();
|
||||
let bucket = format!("notify-authority-{}", uuid::Uuid::new_v4().simple());
|
||||
store
|
||||
.make_bucket(&bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create bucket");
|
||||
let xml = |event: &str| {
|
||||
format!(
|
||||
"<NotificationConfiguration><QueueConfiguration><Event>{event}</Event>\
|
||||
<Queue>arn:rustfs:sqs:us-east-1:inert:webhook</Queue></QueueConfiguration></NotificationConfiguration>"
|
||||
)
|
||||
.into_bytes()
|
||||
};
|
||||
store
|
||||
.update_bucket_metadata_config(&bucket, BUCKET_NOTIFICATION_CONFIG, xml("s3:ObjectCreated:Put"))
|
||||
.await
|
||||
.expect("persist initial subscription");
|
||||
let old = store.get_bucket_metadata(&bucket).await.expect("initial cached metadata");
|
||||
let system = rustfs_notify::NotificationSystem::new(Default::default());
|
||||
system.bind_bucket_metadata_source(&store);
|
||||
let engine = rustfs_notify::NotifyRuleEngine::new();
|
||||
engine.bind_bucket_metadata_source(&store);
|
||||
let inert_target = TargetID::new("inert".into(), "webhook".into());
|
||||
assert!(system.has_subscriber(&bucket, &EventName::ObjectCreatedPut).await);
|
||||
assert!(engine.is_target_bound_to_any_bucket(&inert_target).await);
|
||||
|
||||
store
|
||||
.update_bucket_metadata_config(&bucket, BUCKET_NOTIFICATION_CONFIG, EMPTY_XML.to_vec())
|
||||
.await
|
||||
.expect("remove subscription durably");
|
||||
// Stand in for a peer whose cache has not received the reload yet.
|
||||
crate::storage::storage_api::set_bucket_metadata(bucket.clone(), (*old).clone())
|
||||
.await
|
||||
.expect("install stale peer cache");
|
||||
assert!(system.has_subscriber(&bucket, &EventName::ObjectCreatedPut).await);
|
||||
crate::storage::storage_api::reload_bucket_metadata(store.clone(), &bucket)
|
||||
.await
|
||||
.expect("peer reload");
|
||||
assert!(!system.has_subscriber(&bucket, &EventName::ObjectCreatedPut).await);
|
||||
assert!(
|
||||
engine
|
||||
.match_targets(&bucket, EventName::ObjectCreatedPut, "object")
|
||||
.await
|
||||
.is_empty()
|
||||
);
|
||||
assert!(!engine.is_target_bound_to_any_bucket(&inert_target).await);
|
||||
|
||||
store
|
||||
.update_bucket_metadata_config(&bucket, BUCKET_NOTIFICATION_CONFIG, xml("s3:ObjectCreated:Put"))
|
||||
.await
|
||||
.expect("restore initial subscription");
|
||||
assert!(system.has_subscriber(&bucket, &EventName::ObjectCreatedPut).await);
|
||||
store
|
||||
.delete_bucket(&bucket, &DeleteBucketOptions::default())
|
||||
.await
|
||||
.expect("delete bucket");
|
||||
assert!(!system.has_subscriber(&bucket, &EventName::ObjectCreatedPut).await);
|
||||
// A peer may still have the deleted incarnation cached until its delete
|
||||
// callback (or the periodic metadata refresh) establishes absence.
|
||||
crate::storage::storage_api::set_bucket_metadata(bucket.clone(), (*old).clone())
|
||||
.await
|
||||
.expect("install deleted peer snapshot");
|
||||
assert!(system.has_subscriber(&bucket, &EventName::ObjectCreatedPut).await);
|
||||
assert!(
|
||||
crate::storage::storage_api::remove_bucket_metadata_if_absent(store.clone(), &bucket)
|
||||
.await
|
||||
.expect("peer deletion")
|
||||
);
|
||||
assert!(!system.has_subscriber(&bucket, &EventName::ObjectCreatedPut).await);
|
||||
assert!(
|
||||
engine
|
||||
.match_targets(&bucket, EventName::ObjectCreatedPut, "object")
|
||||
.await
|
||||
.is_empty()
|
||||
);
|
||||
store
|
||||
.make_bucket(&bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("recreate bucket");
|
||||
assert_ne!(
|
||||
old.bucket_incarnation_id,
|
||||
store.bucket_incarnation_id(&bucket).await.expect("replacement identity")
|
||||
);
|
||||
assert!(!system.has_subscriber(&bucket, &EventName::ObjectCreatedPut).await);
|
||||
store
|
||||
.update_bucket_metadata_config(&bucket, BUCKET_NOTIFICATION_CONFIG, xml("s3:ObjectRemoved:Delete"))
|
||||
.await
|
||||
.expect("replacement subscription");
|
||||
assert!(system.has_subscriber(&bucket, &EventName::ObjectRemovedDelete).await);
|
||||
assert!(engine.has_subscriber(&bucket, &EventName::ObjectRemovedDelete).await);
|
||||
|
||||
assert!(
|
||||
!crate::storage::storage_api::remove_bucket_metadata_if_absent(store.clone(), &bucket)
|
||||
.await
|
||||
.expect("delayed peer deletion")
|
||||
);
|
||||
let mut stale = rustfs_notify::BucketNotificationConfig::new("us-east-1");
|
||||
stale.add_rule(&[EventName::ObjectCreatedPut], "*".into(), TargetID::new("old".into(), "webhook".into()));
|
||||
// No runtime targets are activated: all assertions inspect inert rules.
|
||||
system.remove_bucket_notification_config(&bucket).await;
|
||||
// Applying already-validated rules models a delayed PUT completion.
|
||||
engine.set_bucket_rules(&bucket, stale.get_rules_map().clone()).await;
|
||||
assert!(engine.has_subscriber(&bucket, &EventName::ObjectRemovedDelete).await);
|
||||
assert!(!engine.has_subscriber(&bucket, &EventName::ObjectCreatedPut).await);
|
||||
assert!(
|
||||
engine
|
||||
.match_targets(&bucket, EventName::ObjectCreatedPut, "object")
|
||||
.await
|
||||
.is_empty()
|
||||
);
|
||||
assert_eq!(
|
||||
engine
|
||||
.match_targets(&bucket, EventName::ObjectRemovedDelete, "object")
|
||||
.await
|
||||
.into_iter()
|
||||
.collect::<Vec<_>>(),
|
||||
vec![inert_target]
|
||||
);
|
||||
assert!(system.has_subscriber(&bucket, &EventName::ObjectRemovedDelete).await);
|
||||
assert!(!system.has_subscriber(&bucket, &EventName::ObjectCreatedPut).await);
|
||||
});
|
||||
}
|
||||
|
||||
@@ -1,272 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! RustFS's two metadata listings, `GET /{bucket}?list-type=2&metadata=true` and
|
||||
//! `GET /{bucket}?versions&metadata=true`: their authorization and their answer.
|
||||
//!
|
||||
//! Responsible for: [`authorize`], the listing check both stacks run once the caller is
|
||||
//! installed, and [`answer`], which reads the listing's parameters from the request, runs the
|
||||
//! owned listing use case and renders the listing document with each entry's metadata, tags and
|
||||
//! internal fields.
|
||||
//! NOT responsible for: which requests are metadata listings (the legacy `MetadataRoute` and the
|
||||
//! gateway's RustFS dialect each select them), resolving the caller, or the listing itself
|
||||
//! (`bucket_usecase/listing.rs`).
|
||||
//! Upstream: the legacy `MetadataRoute` and the gateway stack's metadata listing handler.
|
||||
//! Downstream: the bucket listing authorization and the listing use case.
|
||||
|
||||
use crate::app::RequestEnvelope;
|
||||
use crate::app::bucket_usecase::DefaultBucketUsecase;
|
||||
use crate::app::storage_api::s3::dto::{EncodingType, ListObjectVersionsInput, ListObjectsV2Input};
|
||||
use crate::app::storage_api::s3::xml;
|
||||
use crate::storage::access::access_owned::listing_multipart::authorize_bucket_listing;
|
||||
use crate::storage::authorization::AuthorizationRequest;
|
||||
use http::{HeaderMap, Uri};
|
||||
use rustfs_policy::policy::action::S3Action;
|
||||
use rustfs_s3_types::{S3Error, S3Result, s3_error};
|
||||
use url::form_urlencoded;
|
||||
|
||||
/// Which of the two metadata listings a request is.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub(crate) enum MetadataListing {
|
||||
ListObjectVersions,
|
||||
ListObjectsV2,
|
||||
}
|
||||
|
||||
impl MetadataListing {
|
||||
/// The action the listing is authorized by on its bucket.
|
||||
fn action(self) -> S3Action {
|
||||
match self {
|
||||
Self::ListObjectVersions => S3Action::ListBucketVersionsAction,
|
||||
Self::ListObjectsV2 => S3Action::ListBucketAction,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Authorizes `listing` of `bucket` for the caller `req` carries: the bucket listing check of
|
||||
/// the standard listing with the same action, the table snapshot included.
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// What the bucket listing authorization refuses.
|
||||
pub(crate) async fn authorize<R: AuthorizationRequest + ?Sized>(
|
||||
req: &mut R,
|
||||
listing: MetadataListing,
|
||||
bucket: &str,
|
||||
) -> S3Result<()> {
|
||||
authorize_bucket_listing(req, bucket, listing.action()).await
|
||||
}
|
||||
|
||||
/// The listing document of `listing` over `bucket`, after [`authorize`] ran on `envelope`.
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// `InvalidArgument` for a malformed listing parameter; otherwise what the listing use case
|
||||
/// refuses.
|
||||
pub(crate) async fn answer(envelope: &RequestEnvelope, listing: MetadataListing, bucket: String) -> S3Result<Vec<u8>> {
|
||||
let usecase = DefaultBucketUsecase::from_global();
|
||||
match listing {
|
||||
MetadataListing::ListObjectVersions => {
|
||||
let input = list_object_versions_input(bucket, envelope.uri(), envelope.headers())?;
|
||||
render(&usecase.list_object_versions_metadata(envelope, input).await?)
|
||||
}
|
||||
MetadataListing::ListObjectsV2 => {
|
||||
let input = list_objects_v2_input(bucket, envelope.uri(), envelope.headers())?;
|
||||
render(&usecase.list_objects_v2_metadata(envelope, input).await?)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn render<T: xml::Serialize>(output: &T) -> S3Result<Vec<u8>> {
|
||||
let mut body = Vec::with_capacity(1024);
|
||||
let mut serializer = xml::Serializer::new(&mut body);
|
||||
serializer
|
||||
.decl()
|
||||
.and_then(|()| output.serialize(&mut serializer))
|
||||
.map_err(S3Error::internal_error)?;
|
||||
Ok(body)
|
||||
}
|
||||
|
||||
fn list_object_versions_input(bucket: String, uri: &Uri, headers: &HeaderMap) -> S3Result<ListObjectVersionsInput> {
|
||||
let query = uri.query().unwrap_or_default();
|
||||
|
||||
Ok(ListObjectVersionsInput {
|
||||
bucket,
|
||||
delimiter: query_value(query, "delimiter"),
|
||||
encoding_type: parse_encoding_type(query_value(query, "encoding-type"))?,
|
||||
expected_bucket_owner: header_value(headers, "x-amz-expected-bucket-owner")?,
|
||||
key_marker: query_value(query, "key-marker"),
|
||||
max_keys: query_i32(query, "max-keys")?,
|
||||
optional_object_attributes: None,
|
||||
prefix: query_value(query, "prefix"),
|
||||
request_payer: header_value(headers, "x-amz-request-payer")?.map(Into::into),
|
||||
version_id_marker: query_value(query, "version-id-marker"),
|
||||
})
|
||||
}
|
||||
|
||||
fn list_objects_v2_input(bucket: String, uri: &Uri, headers: &HeaderMap) -> S3Result<ListObjectsV2Input> {
|
||||
let query = uri.query().unwrap_or_default();
|
||||
|
||||
Ok(ListObjectsV2Input {
|
||||
bucket,
|
||||
continuation_token: query_value(query, "continuation-token"),
|
||||
delimiter: query_value(query, "delimiter"),
|
||||
encoding_type: parse_encoding_type(query_value(query, "encoding-type"))?,
|
||||
expected_bucket_owner: header_value(headers, "x-amz-expected-bucket-owner")?,
|
||||
fetch_owner: query_bool(query, "fetch-owner")?,
|
||||
max_keys: query_i32(query, "max-keys")?,
|
||||
optional_object_attributes: None,
|
||||
prefix: query_value(query, "prefix"),
|
||||
request_payer: header_value(headers, "x-amz-request-payer")?.map(Into::into),
|
||||
start_after: query_value(query, "start-after"),
|
||||
})
|
||||
}
|
||||
|
||||
/// Whether the query names `key`.
|
||||
pub(crate) fn query_has(query: &str, key: &str) -> bool {
|
||||
form_urlencoded::parse(query.as_bytes()).any(|(name, _)| name == key)
|
||||
}
|
||||
|
||||
/// The first value the query gives `key`.
|
||||
pub(crate) fn query_value(query: &str, key: &str) -> Option<String> {
|
||||
form_urlencoded::parse(query.as_bytes())
|
||||
.find(|(name, _)| name == key)
|
||||
.map(|(_, value)| value.into_owned())
|
||||
}
|
||||
|
||||
fn query_i32(query: &str, key: &str) -> S3Result<Option<i32>> {
|
||||
query_value(query, key)
|
||||
.map(|value| {
|
||||
value
|
||||
.parse::<i32>()
|
||||
.map_err(|_| s3_error!(InvalidArgument, "invalid integer query value"))
|
||||
})
|
||||
.transpose()
|
||||
}
|
||||
|
||||
fn query_bool(query: &str, key: &str) -> S3Result<Option<bool>> {
|
||||
query_value(query, key)
|
||||
.map(|value| {
|
||||
value
|
||||
.parse::<bool>()
|
||||
.map_err(|_| s3_error!(InvalidArgument, "invalid boolean query value"))
|
||||
})
|
||||
.transpose()
|
||||
}
|
||||
|
||||
fn parse_encoding_type(value: Option<String>) -> S3Result<Option<EncodingType>> {
|
||||
value
|
||||
.map(|value| {
|
||||
if value == EncodingType::URL {
|
||||
Ok(EncodingType::from_static(EncodingType::URL))
|
||||
} else {
|
||||
Err(s3_error!(InvalidArgument, "invalid encoding-type"))
|
||||
}
|
||||
})
|
||||
.transpose()
|
||||
}
|
||||
|
||||
fn header_value(headers: &HeaderMap, name: &str) -> S3Result<Option<String>> {
|
||||
headers
|
||||
.get(name)
|
||||
.map(|value| {
|
||||
value
|
||||
.to_str()
|
||||
.map(str::to_owned)
|
||||
.map_err(|_| s3_error!(InvalidArgument, "invalid header value"))
|
||||
})
|
||||
.transpose()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::app::storage_api::s3::dto::EncodingType;
|
||||
|
||||
fn uri(value: &str) -> Uri {
|
||||
value.parse().expect("test URI should parse")
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn metadata_route_list_objects_v2_input_parses_query_headers_and_decodes_bucket() {
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert("x-amz-expected-bucket-owner", "123456789012".parse().expect("valid header"));
|
||||
headers.insert("x-amz-request-payer", "requester".parse().expect("valid header"));
|
||||
|
||||
let input = list_objects_v2_input(
|
||||
"demo bucket".to_string(),
|
||||
&uri("/demo%20bucket?list-type=2&metadata=true&prefix=logs%2F&delimiter=%2F&encoding-type=url&fetch-owner=true&max-keys=25&continuation-token=opaque&start-after=start"),
|
||||
&headers,
|
||||
)
|
||||
.expect("list objects v2 input should parse");
|
||||
|
||||
assert_eq!(input.bucket, "demo bucket");
|
||||
assert_eq!(input.prefix.as_deref(), Some("logs/"));
|
||||
assert_eq!(input.delimiter.as_deref(), Some("/"));
|
||||
assert_eq!(input.encoding_type.as_ref().map(EncodingType::as_str), Some(EncodingType::URL));
|
||||
assert_eq!(input.fetch_owner, Some(true));
|
||||
assert_eq!(input.max_keys, Some(25));
|
||||
assert_eq!(input.continuation_token.as_deref(), Some("opaque"));
|
||||
assert_eq!(input.start_after.as_deref(), Some("start"));
|
||||
assert_eq!(input.expected_bucket_owner.as_deref(), Some("123456789012"));
|
||||
assert_eq!(input.request_payer.as_ref().map(|payer| payer.as_str()), Some("requester"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn metadata_route_list_object_versions_input_parses_markers_and_rejects_invalid_encoding() {
|
||||
let input = list_object_versions_input(
|
||||
"bucket".to_string(),
|
||||
&uri("/bucket?versions&metadata=true&prefix=logs%2F&delimiter=%2F&encoding-type=url&key-marker=start&version-id-marker=v1&max-keys=10"),
|
||||
&HeaderMap::new(),
|
||||
)
|
||||
.expect("list versions input should parse");
|
||||
|
||||
assert_eq!(input.bucket, "bucket");
|
||||
assert_eq!(input.prefix.as_deref(), Some("logs/"));
|
||||
assert_eq!(input.delimiter.as_deref(), Some("/"));
|
||||
assert_eq!(input.encoding_type.as_ref().map(EncodingType::as_str), Some(EncodingType::URL));
|
||||
assert_eq!(input.key_marker.as_deref(), Some("start"));
|
||||
assert_eq!(input.version_id_marker.as_deref(), Some("v1"));
|
||||
assert_eq!(input.max_keys, Some(10));
|
||||
|
||||
let err = list_object_versions_input(
|
||||
"bucket".to_string(),
|
||||
&uri("/bucket?versions&metadata=true&encoding-type=xml"),
|
||||
&HeaderMap::new(),
|
||||
)
|
||||
.expect_err("invalid encoding-type should be rejected");
|
||||
assert_eq!(err.code(), &rustfs_s3_types::S3ErrorCode::InvalidArgument);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn metadata_route_listing_parameters_refuse_malformed_values() {
|
||||
for target in [
|
||||
"/b?list-type=2&metadata=true&max-keys=ten",
|
||||
"/b?list-type=2&metadata=true&max-keys=99999999999",
|
||||
"/b?list-type=2&metadata=true&fetch-owner=yes",
|
||||
"/b?list-type=2&metadata=true&encoding-type=URL",
|
||||
"/b?list-type=2&metadata=true&encoding-type=",
|
||||
] {
|
||||
let err = list_objects_v2_input("b".to_owned(), &uri(target), &HeaderMap::new()).expect_err(target);
|
||||
assert_eq!(err.code(), &rustfs_s3_types::S3ErrorCode::InvalidArgument, "{target}");
|
||||
}
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert(
|
||||
"x-amz-request-payer",
|
||||
http::HeaderValue::from_bytes(b"\xffpayer").expect("an opaque header value"),
|
||||
);
|
||||
let err = list_object_versions_input("b".to_owned(), &uri("/b?versions&metadata=true"), &headers)
|
||||
.expect_err("a non-text header is refused");
|
||||
assert_eq!(err.code(), &rustfs_s3_types::S3ErrorCode::InvalidArgument);
|
||||
}
|
||||
}
|
||||
@@ -12,36 +12,40 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! MinIO-compatible metadata listing extension routes and the website host, on the legacy stack.
|
||||
//!
|
||||
//! Responsible for: [`MetadataRoute`], the legacy custom route that claims a website host's
|
||||
//! requests and the two metadata listings ahead of the admin router, and hands each to the owned
|
||||
//! rules both stacks share (`app/website.rs`, `app/metadata_listing.rs`).
|
||||
//! NOT responsible for: the website or listing rules themselves, or the gateway stack's
|
||||
//! selection of the same requests (`server/stack.rs` and the gateway's RustFS dialect).
|
||||
//! Upstream: `server/http.rs`. Downstream: the website host, the metadata listings, the admin route.
|
||||
//! MinIO-compatible metadata listing extension routes.
|
||||
|
||||
use super::RequestEnvelope;
|
||||
use super::metadata_listing::{self, query_has, query_value};
|
||||
use super::website::WebsiteHost;
|
||||
use super::bucket_usecase::DefaultBucketUsecase;
|
||||
use crate::app::runtime_sources::ServerContextSlot;
|
||||
use crate::auth::{check_key_valid, get_session_token};
|
||||
use crate::storage::access::access_owned::listing_multipart::authorize_bucket_listing;
|
||||
use crate::storage::access::req_info_mut;
|
||||
use crate::storage::req_info::ReqInfo;
|
||||
use crate::storage::request_markers::WebsiteRead;
|
||||
use crate::storage::{get_bucket_website_config_for_store, validate_website_configuration};
|
||||
use async_trait::async_trait;
|
||||
use http::header::{CONTENT_TYPE, HOST};
|
||||
use http::{Extensions, HeaderMap, HeaderValue, Method, Uri};
|
||||
use http::header::CONTENT_TYPE;
|
||||
use http::header::HOST;
|
||||
use http::{Extensions, HeaderMap, HeaderValue, Method, StatusCode, Uri};
|
||||
use rustfs_gateway_types::persistence::{PersistedErrorDocument, PersistedRoutingRule as RoutingRule};
|
||||
use rustfs_policy::policy::action::S3Action;
|
||||
use s3s::access::S3Access;
|
||||
use s3s::dto::{
|
||||
ETagCondition, EncodingType, GetObjectInput, GetObjectOutput, HeadObjectInput, ListObjectVersionsInput, ListObjectsV2Input,
|
||||
Range, Timestamp, TimestampFormat,
|
||||
};
|
||||
use s3s::host::{MultiDomain, S3Host};
|
||||
use s3s::route::S3Route;
|
||||
use s3s::{Body, S3Error, S3Request, S3Response, S3Result, s3_error};
|
||||
use s3s::xml;
|
||||
use s3s::{Body, S3Error, S3ErrorCode, S3Request, S3Response, S3Result, s3_error};
|
||||
use std::sync::Arc;
|
||||
|
||||
use super::metadata_listing::MetadataListing as MetadataOperation;
|
||||
use url::form_urlencoded;
|
||||
|
||||
pub(crate) struct MetadataRoute<A> {
|
||||
admin: A,
|
||||
host: Option<MultiDomain>,
|
||||
website: Option<WebsiteHost>,
|
||||
website_domains: Vec<String>,
|
||||
website_scheme: &'static str,
|
||||
server_ctx: Arc<ServerContextSlot>,
|
||||
}
|
||||
|
||||
pub(crate) fn with_metadata_route<A>(
|
||||
@@ -54,13 +58,9 @@ pub(crate) fn with_metadata_route<A>(
|
||||
MetadataRoute {
|
||||
admin,
|
||||
host,
|
||||
website: WebsiteHost::new(website_domains, website_scheme, server_ctx),
|
||||
}
|
||||
}
|
||||
|
||||
impl<A> MetadataRoute<A> {
|
||||
fn website(&self, uri: &Uri, headers: &HeaderMap) -> Option<&WebsiteHost> {
|
||||
self.website.as_ref().filter(|website| website.claims(uri, headers))
|
||||
website_domains,
|
||||
website_scheme,
|
||||
server_ctx,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -70,15 +70,15 @@ where
|
||||
A: S3Route,
|
||||
{
|
||||
fn is_match(&self, method: &Method, uri: &Uri, headers: &HeaderMap, extensions: &mut Extensions) -> bool {
|
||||
self.website(uri, headers).is_some()
|
||||
website_target(uri, headers, &self.website_domains).is_some()
|
||||
|| metadata_operation(method, uri, headers, self.host.as_ref()).is_some()
|
||||
|| self.admin.is_match(method, uri, headers, extensions)
|
||||
}
|
||||
|
||||
async fn check_access(&self, req: &mut S3Request<Body>) -> S3Result<()> {
|
||||
// A website read is anonymous whatever the request carries; the host decides it.
|
||||
if self.website(&req.uri, &req.headers).is_some() {
|
||||
Ok(())
|
||||
if let Some(target) = website_target(&req.uri, &req.headers, &self.website_domains) {
|
||||
req.extensions.insert(Arc::clone(&self.server_ctx));
|
||||
check_website_access(req, &target).await
|
||||
} else if let Some(target) = metadata_operation(&req.method, &req.uri, &req.headers, self.host.as_ref()) {
|
||||
check_metadata_access(req, target).await
|
||||
} else {
|
||||
@@ -87,25 +87,542 @@ where
|
||||
}
|
||||
|
||||
async fn call(&self, req: S3Request<Body>) -> S3Result<S3Response<Body>> {
|
||||
if let Some(website) = self.website(&req.uri, &req.headers) {
|
||||
let (parts, body) = website
|
||||
.answer(req.method, req.uri, req.headers, req.extensions)
|
||||
if let Some(target) = website_target(&req.uri, &req.headers, &self.website_domains) {
|
||||
let head = req.method == Method::HEAD;
|
||||
return call_website(req, target, Arc::clone(&self.server_ctx), self.website_scheme)
|
||||
.await
|
||||
.into_parts();
|
||||
return Ok(S3Response {
|
||||
output: Body::http_body_unsync(body),
|
||||
status: Some(parts.status),
|
||||
headers: parts.headers,
|
||||
extensions: parts.extensions,
|
||||
});
|
||||
.or_else(|err: S3Error| Ok(website_error(err.status_code().unwrap_or(StatusCode::INTERNAL_SERVER_ERROR), head)));
|
||||
}
|
||||
match metadata_operation(&req.method, &req.uri, &req.headers, self.host.as_ref()) {
|
||||
Some(target) => call_metadata_listing(req, target).await,
|
||||
Some(BucketTarget {
|
||||
operation: MetadataOperation::ListObjectVersions,
|
||||
bucket,
|
||||
}) => call_list_object_versions(req, bucket).await,
|
||||
Some(BucketTarget {
|
||||
operation: MetadataOperation::ListObjectsV2,
|
||||
bucket,
|
||||
}) => call_list_objects_v2(req, bucket).await,
|
||||
None => self.admin.call(req).await,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
struct WebsiteTarget {
|
||||
bucket: String,
|
||||
key: String,
|
||||
trailing_slash: bool,
|
||||
invalid_path: bool,
|
||||
}
|
||||
|
||||
fn website_target(uri: &Uri, headers: &HeaderMap, domains: &[String]) -> Option<WebsiteTarget> {
|
||||
if domains.is_empty() {
|
||||
return None;
|
||||
}
|
||||
let host_header = headers.get(HOST)?.to_str().ok()?;
|
||||
let authority = host_header.parse::<http::uri::Authority>().ok()?;
|
||||
let request_host = authority.host().trim_end_matches('.').to_ascii_lowercase();
|
||||
let bucket = domains.iter().find_map(|domain| {
|
||||
if request_host == *domain {
|
||||
return Some(String::new());
|
||||
}
|
||||
request_host.strip_suffix(&format!(".{domain}")).map(str::to_owned)
|
||||
})?;
|
||||
let raw_path = uri.path();
|
||||
let decoded = urlencoding::decode(raw_path);
|
||||
let invalid_path = decoded.is_err();
|
||||
let decoded = decoded.unwrap_or_default();
|
||||
let path = decoded.trim_start_matches('/');
|
||||
Some(WebsiteTarget {
|
||||
bucket,
|
||||
key: path.to_owned(),
|
||||
trailing_slash: path.is_empty() || path.ends_with('/'),
|
||||
invalid_path,
|
||||
})
|
||||
}
|
||||
|
||||
async fn check_website_access(req: &mut S3Request<Body>, target: &WebsiteTarget) -> S3Result<()> {
|
||||
if req_info_mut(req).is_err() {
|
||||
req.extensions.insert(ReqInfo::default());
|
||||
}
|
||||
let req_info = req_info_mut(req)?;
|
||||
req_info.cred = None;
|
||||
req_info.is_owner = false;
|
||||
req_info.bucket = Some(target.bucket.clone());
|
||||
req_info.object = Some(target.key.clone());
|
||||
req_info.version_id = None;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn call_website(
|
||||
req: S3Request<Body>,
|
||||
target: WebsiteTarget,
|
||||
server_ctx: Arc<ServerContextSlot>,
|
||||
website_scheme: &'static str,
|
||||
) -> S3Result<S3Response<Body>> {
|
||||
if !matches!(req.method, Method::GET | Method::HEAD) {
|
||||
return Ok(website_error(StatusCode::METHOD_NOT_ALLOWED, req.method == Method::HEAD));
|
||||
}
|
||||
if target.invalid_path {
|
||||
return Ok(website_error(StatusCode::BAD_REQUEST, req.method == Method::HEAD));
|
||||
}
|
||||
if target.bucket.is_empty() || target.bucket.starts_with('.') || target.bucket.ends_with('.') || target.bucket.contains("..")
|
||||
{
|
||||
return Ok(website_error(StatusCode::NOT_FOUND, req.method == Method::HEAD));
|
||||
}
|
||||
|
||||
let store = server_ctx
|
||||
.installed_object_store()
|
||||
.ok_or_else(|| S3Error::with_message(S3ErrorCode::InternalError, "website object store unavailable"))?;
|
||||
let config = match get_bucket_website_config_for_store(&store, &target.bucket).await {
|
||||
Ok(config) => config,
|
||||
Err(crate::storage::StorageError::ConfigNotFound) => {
|
||||
return Ok(website_error(StatusCode::NOT_FOUND, req.method == Method::HEAD));
|
||||
}
|
||||
Err(crate::storage::StorageError::BucketNotFound(_)) => {
|
||||
return Ok(website_error(StatusCode::NOT_FOUND, req.method == Method::HEAD));
|
||||
}
|
||||
Err(err) => return Err(crate::error::ApiError::from(err).into()),
|
||||
};
|
||||
if validate_website_configuration(&config).is_err() {
|
||||
return Ok(website_error(StatusCode::INTERNAL_SERVER_ERROR, req.method == Method::HEAD));
|
||||
}
|
||||
|
||||
if let Some(redirect) = config.redirect_all_requests_to.as_ref() {
|
||||
return website_redirect(
|
||||
&req,
|
||||
redirect.host_name.as_str(),
|
||||
redirect.protocol.as_deref(),
|
||||
&target.key,
|
||||
StatusCode::MOVED_PERMANENTLY,
|
||||
website_scheme,
|
||||
);
|
||||
}
|
||||
|
||||
let key = if target.key.is_empty() || target.trailing_slash {
|
||||
let suffix = config
|
||||
.index_document
|
||||
.as_ref()
|
||||
.map(|doc| doc.suffix.as_str())
|
||||
.unwrap_or_default();
|
||||
format!("{}{}", target.key, suffix)
|
||||
} else {
|
||||
target.key.clone()
|
||||
};
|
||||
|
||||
let rules = config.routing_rules.as_deref().unwrap_or_default();
|
||||
// A rule requiring a status cannot be evaluated until the object lookup completes.
|
||||
for rule in rules {
|
||||
if rule
|
||||
.condition
|
||||
.as_ref()
|
||||
.and_then(|condition| condition.http_error_code_returned_equals.as_ref())
|
||||
.is_some()
|
||||
{
|
||||
break;
|
||||
}
|
||||
if website_rule_matches(rule, &target.key, StatusCode::OK) {
|
||||
return website_rule_redirect(&req, rule, &target.key, website_scheme);
|
||||
}
|
||||
}
|
||||
|
||||
let response = website_get_object(&req, &server_ctx, &target.bucket, &key, true).await;
|
||||
if let Err(err) = &response
|
||||
&& matches!(
|
||||
err.code(),
|
||||
S3ErrorCode::NotModified | S3ErrorCode::PreconditionFailed | S3ErrorCode::InvalidRange
|
||||
)
|
||||
&& let Ok(Some(location)) = website_head_redirect(&req, &server_ctx, &target.bucket, &key).await
|
||||
{
|
||||
return website_location(location, StatusCode::MOVED_PERMANENTLY);
|
||||
}
|
||||
let status = match &response {
|
||||
Ok(_) => StatusCode::OK,
|
||||
Err(err) => err.status_code().unwrap_or(StatusCode::INTERNAL_SERVER_ERROR),
|
||||
};
|
||||
if status == StatusCode::NOT_FOUND && !target.trailing_slash && !target.key.is_empty() {
|
||||
let index_key = format!(
|
||||
"{}/{}",
|
||||
target.key,
|
||||
config
|
||||
.index_document
|
||||
.as_ref()
|
||||
.map(|doc| doc.suffix.as_str())
|
||||
.unwrap_or_default()
|
||||
);
|
||||
match website_get_object(&req, &server_ctx, &target.bucket, &index_key, false).await {
|
||||
Ok(_) => return website_relative_redirect(&req, &format!("/{}/", target.key), StatusCode::FOUND),
|
||||
Err(err)
|
||||
if matches!(
|
||||
err.code(),
|
||||
S3ErrorCode::NoSuchKey | S3ErrorCode::NoSuchVersion | S3ErrorCode::AccessDenied
|
||||
) => {}
|
||||
Err(err) => return Err(err),
|
||||
}
|
||||
}
|
||||
if let Some(rule) = rules.iter().find(|rule| website_rule_matches(rule, &target.key, status)) {
|
||||
return website_rule_redirect(&req, rule, &target.key, website_scheme);
|
||||
}
|
||||
match response {
|
||||
Ok(response) => {
|
||||
if let Some(location) = response.output.website_redirect_location.clone() {
|
||||
return website_location(location, StatusCode::MOVED_PERMANENTLY);
|
||||
}
|
||||
response_to_website(response, req.method == Method::HEAD)
|
||||
}
|
||||
Err(err) if matches!(err.code(), S3ErrorCode::NoSuchKey | S3ErrorCode::NoSuchVersion) => {
|
||||
website_error_with_document(&req, &server_ctx, &target.bucket, config.error_document.as_ref(), StatusCode::NOT_FOUND)
|
||||
.await
|
||||
}
|
||||
Err(err) if err.code() == &S3ErrorCode::NotModified => Ok(website_not_modified(&err)),
|
||||
Err(_) if status.is_client_error() => {
|
||||
website_error_with_document(&req, &server_ctx, &target.bucket, config.error_document.as_ref(), status).await
|
||||
}
|
||||
Err(err) => Err(err),
|
||||
}
|
||||
}
|
||||
|
||||
async fn website_get_object(
|
||||
original: &S3Request<Body>,
|
||||
server_ctx: &Arc<ServerContextSlot>,
|
||||
bucket: &str,
|
||||
key: &str,
|
||||
apply_conditions: bool,
|
||||
) -> S3Result<S3Response<GetObjectOutput>> {
|
||||
let header = |name: http::header::HeaderName| {
|
||||
original
|
||||
.headers
|
||||
.get(name)
|
||||
.map(|value| {
|
||||
value
|
||||
.to_str()
|
||||
.map(str::to_owned)
|
||||
.map_err(|_| S3Error::new(S3ErrorCode::InvalidArgument))
|
||||
})
|
||||
.transpose()
|
||||
};
|
||||
let input = if apply_conditions {
|
||||
GetObjectInput {
|
||||
bucket: bucket.to_owned(),
|
||||
key: key.to_owned(),
|
||||
range: header(http::header::RANGE)?
|
||||
.map(|value| Range::parse(&value).map_err(|_| S3Error::new(S3ErrorCode::InvalidRange)))
|
||||
.transpose()?,
|
||||
if_match: original
|
||||
.headers
|
||||
.get(http::header::IF_MATCH)
|
||||
.map(|value| {
|
||||
ETagCondition::parse_http_header(value.as_bytes()).map_err(|_| S3Error::new(S3ErrorCode::InvalidArgument))
|
||||
})
|
||||
.transpose()?,
|
||||
if_none_match: original
|
||||
.headers
|
||||
.get(http::header::IF_NONE_MATCH)
|
||||
.map(|value| {
|
||||
ETagCondition::parse_http_header(value.as_bytes()).map_err(|_| S3Error::new(S3ErrorCode::InvalidArgument))
|
||||
})
|
||||
.transpose()?,
|
||||
if_modified_since: header(http::header::IF_MODIFIED_SINCE)?
|
||||
.map(|value| {
|
||||
Timestamp::parse(TimestampFormat::HttpDate, &value).map_err(|_| S3Error::new(S3ErrorCode::InvalidArgument))
|
||||
})
|
||||
.transpose()?,
|
||||
if_unmodified_since: header(http::header::IF_UNMODIFIED_SINCE)?
|
||||
.map(|value| {
|
||||
Timestamp::parse(TimestampFormat::HttpDate, &value).map_err(|_| S3Error::new(S3ErrorCode::InvalidArgument))
|
||||
})
|
||||
.transpose()?,
|
||||
..Default::default()
|
||||
}
|
||||
} else {
|
||||
GetObjectInput {
|
||||
bucket: bucket.to_owned(),
|
||||
key: key.to_owned(),
|
||||
..Default::default()
|
||||
}
|
||||
};
|
||||
let mut request = S3Request {
|
||||
input,
|
||||
method: Method::GET,
|
||||
uri: original.uri.clone(),
|
||||
headers: if apply_conditions {
|
||||
original.headers.clone()
|
||||
} else {
|
||||
let mut headers = original.headers.clone();
|
||||
for name in [
|
||||
http::header::RANGE,
|
||||
http::header::IF_MATCH,
|
||||
http::header::IF_NONE_MATCH,
|
||||
http::header::IF_MODIFIED_SINCE,
|
||||
http::header::IF_UNMODIFIED_SINCE,
|
||||
] {
|
||||
headers.remove(name);
|
||||
}
|
||||
headers
|
||||
},
|
||||
extensions: original.extensions.clone(),
|
||||
credentials: None,
|
||||
region: None,
|
||||
service: None,
|
||||
trailing_headers: None,
|
||||
};
|
||||
request.extensions.insert(Arc::clone(server_ctx));
|
||||
let req_info = request
|
||||
.extensions
|
||||
.get_mut::<ReqInfo>()
|
||||
.ok_or_else(|| S3Error::with_message(S3ErrorCode::InternalError, "website request info missing"))?;
|
||||
req_info.bucket = Some(bucket.to_owned());
|
||||
req_info.object = Some(key.to_owned());
|
||||
req_info.cred = None;
|
||||
req_info.is_owner = false;
|
||||
request.extensions.insert(WebsiteRead);
|
||||
S3Access::get_object(&crate::storage::ecfs::FS::with_server_ctx(Arc::clone(server_ctx)), &mut request).await?;
|
||||
s3s::S3::get_object(&crate::storage::ecfs::FS::with_server_ctx(Arc::clone(server_ctx)), request).await
|
||||
}
|
||||
|
||||
async fn website_head_redirect(
|
||||
original: &S3Request<Body>,
|
||||
server_ctx: &Arc<ServerContextSlot>,
|
||||
bucket: &str,
|
||||
key: &str,
|
||||
) -> S3Result<Option<String>> {
|
||||
let mut headers = original.headers.clone();
|
||||
for name in [
|
||||
http::header::RANGE,
|
||||
http::header::IF_MATCH,
|
||||
http::header::IF_NONE_MATCH,
|
||||
http::header::IF_MODIFIED_SINCE,
|
||||
http::header::IF_UNMODIFIED_SINCE,
|
||||
] {
|
||||
headers.remove(name);
|
||||
}
|
||||
let mut request = S3Request {
|
||||
input: HeadObjectInput {
|
||||
bucket: bucket.to_owned(),
|
||||
key: key.to_owned(),
|
||||
..Default::default()
|
||||
},
|
||||
method: Method::HEAD,
|
||||
uri: original.uri.clone(),
|
||||
headers,
|
||||
extensions: original.extensions.clone(),
|
||||
credentials: None,
|
||||
region: None,
|
||||
service: None,
|
||||
trailing_headers: None,
|
||||
};
|
||||
request.extensions.insert(Arc::clone(server_ctx));
|
||||
let req_info = request
|
||||
.extensions
|
||||
.get_mut::<ReqInfo>()
|
||||
.ok_or_else(|| S3Error::with_message(S3ErrorCode::InternalError, "website request info missing"))?;
|
||||
req_info.bucket = Some(bucket.to_owned());
|
||||
req_info.object = Some(key.to_owned());
|
||||
req_info.cred = None;
|
||||
req_info.is_owner = false;
|
||||
S3Access::head_object(&crate::storage::ecfs::FS::with_server_ctx(Arc::clone(server_ctx)), &mut request).await?;
|
||||
let response = s3s::S3::head_object(&crate::storage::ecfs::FS::with_server_ctx(Arc::clone(server_ctx)), request).await?;
|
||||
Ok(response.output.website_redirect_location)
|
||||
}
|
||||
|
||||
fn response_to_website(response: S3Response<GetObjectOutput>, head: bool) -> S3Result<S3Response<Body>> {
|
||||
let output = response.output;
|
||||
let mut headers = response.headers;
|
||||
insert_website_header(&mut headers, http::header::CONTENT_TYPE, output.content_type.as_deref())?;
|
||||
insert_website_header(
|
||||
&mut headers,
|
||||
http::header::CONTENT_LENGTH,
|
||||
output.content_length.map(|length| length.to_string()).as_deref(),
|
||||
)?;
|
||||
insert_website_header(&mut headers, http::header::CONTENT_RANGE, output.content_range.as_deref())?;
|
||||
if let Some(tag) = output.e_tag.as_ref() {
|
||||
headers.insert(http::header::ETAG, tag.to_http_header().map_err(S3Error::internal_error)?);
|
||||
}
|
||||
insert_website_header(&mut headers, http::header::CACHE_CONTROL, output.cache_control.as_deref())?;
|
||||
insert_website_header(&mut headers, http::header::CONTENT_ENCODING, output.content_encoding.as_deref())?;
|
||||
insert_website_header(&mut headers, http::header::CONTENT_LANGUAGE, output.content_language.as_deref())?;
|
||||
insert_website_header(&mut headers, http::header::CONTENT_DISPOSITION, output.content_disposition.as_deref())?;
|
||||
if let Some(last_modified) = output.last_modified {
|
||||
let mut formatted = Vec::new();
|
||||
last_modified
|
||||
.format(TimestampFormat::HttpDate, &mut formatted)
|
||||
.map_err(S3Error::internal_error)?;
|
||||
headers.insert(
|
||||
http::header::LAST_MODIFIED,
|
||||
HeaderValue::from_bytes(&formatted).map_err(S3Error::internal_error)?,
|
||||
);
|
||||
}
|
||||
Ok(S3Response {
|
||||
output: if head {
|
||||
Body::empty()
|
||||
} else {
|
||||
output.body.map(Body::from).unwrap_or_default()
|
||||
},
|
||||
status: Some(if output.content_range.is_some() {
|
||||
StatusCode::PARTIAL_CONTENT
|
||||
} else {
|
||||
StatusCode::OK
|
||||
}),
|
||||
headers,
|
||||
extensions: response.extensions,
|
||||
})
|
||||
}
|
||||
|
||||
fn insert_website_header(headers: &mut HeaderMap, name: http::header::HeaderName, value: Option<&str>) -> S3Result<()> {
|
||||
if let Some(value) = value {
|
||||
headers.insert(name, HeaderValue::from_str(value).map_err(S3Error::internal_error)?);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn website_error_with_document(
|
||||
req: &S3Request<Body>,
|
||||
server_ctx: &Arc<ServerContextSlot>,
|
||||
bucket: &str,
|
||||
error_document: Option<&PersistedErrorDocument>,
|
||||
status: http::StatusCode,
|
||||
) -> S3Result<S3Response<Body>> {
|
||||
let Some(error_document) = error_document else {
|
||||
return Ok(website_error(status, req.method == Method::HEAD));
|
||||
};
|
||||
match website_get_object(req, server_ctx, bucket, error_document.key.as_str(), false).await {
|
||||
Ok(response) => {
|
||||
let mut output = response_to_website(response, req.method == Method::HEAD)?;
|
||||
output.status = Some(status);
|
||||
Ok(output)
|
||||
}
|
||||
Err(err)
|
||||
if matches!(
|
||||
err.code(),
|
||||
S3ErrorCode::NoSuchKey | S3ErrorCode::NoSuchVersion | S3ErrorCode::AccessDenied
|
||||
) =>
|
||||
{
|
||||
Ok(website_error(status, req.method == Method::HEAD))
|
||||
}
|
||||
Err(err) => Err(err),
|
||||
}
|
||||
}
|
||||
|
||||
fn website_error(status: StatusCode, head: bool) -> S3Response<Body> {
|
||||
let message = status.canonical_reason().unwrap_or("Error");
|
||||
let body = format!("<html><body>{message}</body></html>");
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert(CONTENT_TYPE, HeaderValue::from_static("text/html; charset=utf-8"));
|
||||
S3Response {
|
||||
output: if head { Body::empty() } else { Body::from(body.into_bytes()) },
|
||||
status: Some(status),
|
||||
headers,
|
||||
extensions: Extensions::new(),
|
||||
}
|
||||
}
|
||||
|
||||
fn website_not_modified(error: &S3Error) -> S3Response<Body> {
|
||||
let mut response = S3Response::new(Body::empty());
|
||||
response.status = Some(StatusCode::NOT_MODIFIED);
|
||||
if let Some(headers) = error.headers() {
|
||||
response.headers = headers.clone();
|
||||
}
|
||||
response.headers.remove(http::header::CONTENT_LENGTH);
|
||||
response.headers.remove(http::header::CONTENT_TYPE);
|
||||
response.headers.remove(http::header::TRANSFER_ENCODING);
|
||||
response
|
||||
}
|
||||
|
||||
fn website_location(location: String, status: StatusCode) -> S3Result<S3Response<Body>> {
|
||||
let mut response = S3Response::new(Body::empty());
|
||||
response.status = Some(status);
|
||||
response.headers.insert(
|
||||
http::header::LOCATION,
|
||||
HeaderValue::try_from(location)
|
||||
.map_err(|_| S3Error::with_message(S3ErrorCode::InternalError, "invalid website redirect location"))?,
|
||||
);
|
||||
Ok(response)
|
||||
}
|
||||
|
||||
fn website_relative_redirect(req: &S3Request<Body>, path: &str, status: StatusCode) -> S3Result<S3Response<Body>> {
|
||||
let mut url = url::Url::parse("http://localhost/").map_err(S3Error::internal_error)?;
|
||||
url.set_path(path);
|
||||
url.set_query(req.uri.query());
|
||||
let location = match url.query() {
|
||||
Some(query) => format!("{}?{query}", url.path()),
|
||||
None => url.path().to_owned(),
|
||||
};
|
||||
website_location(location, status)
|
||||
}
|
||||
|
||||
fn website_redirect(
|
||||
req: &S3Request<Body>,
|
||||
host: &str,
|
||||
protocol: Option<&str>,
|
||||
key: &str,
|
||||
status: StatusCode,
|
||||
website_scheme: &str,
|
||||
) -> S3Result<S3Response<Body>> {
|
||||
let scheme = protocol.unwrap_or(website_scheme);
|
||||
let host = if host.is_empty() {
|
||||
req.headers.get(HOST).and_then(|h| h.to_str().ok()).unwrap_or_default()
|
||||
} else {
|
||||
host
|
||||
};
|
||||
let mut url = url::Url::parse(&format!("{scheme}://{host}/")).map_err(S3Error::internal_error)?;
|
||||
url.set_path(&format!("/{key}"));
|
||||
url.set_query(req.uri.query());
|
||||
website_location(url.into(), status)
|
||||
}
|
||||
|
||||
fn website_rule_matches(rule: &RoutingRule, key: &str, status: StatusCode) -> bool {
|
||||
let Some(condition) = rule.condition.as_ref() else { return true };
|
||||
condition
|
||||
.key_prefix_equals
|
||||
.as_ref()
|
||||
.is_none_or(|prefix| key.starts_with(prefix))
|
||||
&& condition
|
||||
.http_error_code_returned_equals
|
||||
.as_ref()
|
||||
.is_none_or(|code| code == status.as_str())
|
||||
}
|
||||
|
||||
fn website_rule_redirect(
|
||||
req: &S3Request<Body>,
|
||||
rule: &RoutingRule,
|
||||
key: &str,
|
||||
website_scheme: &str,
|
||||
) -> S3Result<S3Response<Body>> {
|
||||
let redirect = &rule.redirect;
|
||||
let key = if let Some(replace) = &redirect.replace_key_with {
|
||||
replace.clone()
|
||||
} else if let Some(replace) = &redirect.replace_key_prefix_with {
|
||||
let prefix = rule
|
||||
.condition
|
||||
.as_ref()
|
||||
.and_then(|condition| condition.key_prefix_equals.as_deref())
|
||||
.unwrap_or_default();
|
||||
format!("{replace}{}", key.strip_prefix(prefix).unwrap_or(key))
|
||||
} else {
|
||||
key.to_owned()
|
||||
};
|
||||
let status = redirect
|
||||
.http_redirect_code
|
||||
.as_deref()
|
||||
.and_then(|code| code.parse::<u16>().ok())
|
||||
.and_then(|code| StatusCode::from_u16(code).ok())
|
||||
.unwrap_or(StatusCode::MOVED_PERMANENTLY);
|
||||
website_redirect(
|
||||
req,
|
||||
redirect.host_name.as_deref().unwrap_or_default(),
|
||||
redirect.protocol.as_deref(),
|
||||
&key,
|
||||
status,
|
||||
website_scheme,
|
||||
)
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
enum MetadataOperation {
|
||||
ListObjectVersions,
|
||||
ListObjectsV2,
|
||||
}
|
||||
|
||||
struct BucketTarget {
|
||||
operation: MetadataOperation,
|
||||
bucket: String,
|
||||
@@ -156,12 +673,39 @@ async fn check_metadata_access(req: &mut S3Request<Body>, target: BucketTarget)
|
||||
req_info.bucket = Some(target.bucket.clone());
|
||||
}
|
||||
|
||||
Ok(metadata_listing::authorize(req, target.operation, &target.bucket).await?)
|
||||
let action = match target.operation {
|
||||
MetadataOperation::ListObjectVersions => S3Action::ListBucketVersionsAction,
|
||||
MetadataOperation::ListObjectsV2 => S3Action::ListBucketAction,
|
||||
};
|
||||
Ok(authorize_bucket_listing(req, &target.bucket, action).await?)
|
||||
}
|
||||
|
||||
async fn call_metadata_listing(req: S3Request<Body>, target: BucketTarget) -> S3Result<S3Response<Body>> {
|
||||
let envelope = RequestEnvelope::from_s3s(&req);
|
||||
let body = metadata_listing::answer(&envelope, target.operation, target.bucket).await?;
|
||||
async fn call_list_object_versions(req: S3Request<Body>, bucket: String) -> S3Result<S3Response<Body>> {
|
||||
let input = list_object_versions_input(bucket, &req.uri, &req.headers)?;
|
||||
let request = req.map_input(|_| input);
|
||||
let output = DefaultBucketUsecase::from_global()
|
||||
.execute_list_object_versions_m(request)
|
||||
.await?;
|
||||
xml_response(&output.output)
|
||||
}
|
||||
|
||||
async fn call_list_objects_v2(req: S3Request<Body>, bucket: String) -> S3Result<S3Response<Body>> {
|
||||
let input = list_objects_v2_input(bucket, &req.uri, &req.headers)?;
|
||||
let request = req.map_input(|_| input);
|
||||
let output = DefaultBucketUsecase::from_global().execute_list_objects_v2m(request).await?;
|
||||
xml_response(&output.output)
|
||||
}
|
||||
|
||||
fn xml_response<T: xml::Serialize>(output: &T) -> S3Result<S3Response<Body>> {
|
||||
let mut body = Vec::with_capacity(1024);
|
||||
{
|
||||
let mut serializer = xml::Serializer::new(&mut body);
|
||||
serializer
|
||||
.decl()
|
||||
.and_then(|()| output.serialize(&mut serializer))
|
||||
.map_err(S3Error::internal_error)?;
|
||||
}
|
||||
|
||||
let mut response = S3Response::new(Body::from(body));
|
||||
response
|
||||
.headers
|
||||
@@ -169,6 +713,41 @@ async fn call_metadata_listing(req: S3Request<Body>, target: BucketTarget) -> S3
|
||||
Ok(response)
|
||||
}
|
||||
|
||||
fn list_object_versions_input(bucket: String, uri: &Uri, headers: &HeaderMap) -> S3Result<ListObjectVersionsInput> {
|
||||
let query = uri.query().unwrap_or_default();
|
||||
|
||||
Ok(ListObjectVersionsInput {
|
||||
bucket,
|
||||
delimiter: query_value(query, "delimiter"),
|
||||
encoding_type: parse_encoding_type(query_value(query, "encoding-type"))?,
|
||||
expected_bucket_owner: header_value(headers, "x-amz-expected-bucket-owner")?,
|
||||
key_marker: query_value(query, "key-marker"),
|
||||
max_keys: query_i32(query, "max-keys")?,
|
||||
optional_object_attributes: None,
|
||||
prefix: query_value(query, "prefix"),
|
||||
request_payer: header_value(headers, "x-amz-request-payer")?.map(Into::into),
|
||||
version_id_marker: query_value(query, "version-id-marker"),
|
||||
})
|
||||
}
|
||||
|
||||
fn list_objects_v2_input(bucket: String, uri: &Uri, headers: &HeaderMap) -> S3Result<ListObjectsV2Input> {
|
||||
let query = uri.query().unwrap_or_default();
|
||||
|
||||
Ok(ListObjectsV2Input {
|
||||
bucket,
|
||||
continuation_token: query_value(query, "continuation-token"),
|
||||
delimiter: query_value(query, "delimiter"),
|
||||
encoding_type: parse_encoding_type(query_value(query, "encoding-type"))?,
|
||||
expected_bucket_owner: header_value(headers, "x-amz-expected-bucket-owner")?,
|
||||
fetch_owner: query_bool(query, "fetch-owner")?,
|
||||
max_keys: query_i32(query, "max-keys")?,
|
||||
optional_object_attributes: None,
|
||||
prefix: query_value(query, "prefix"),
|
||||
request_payer: header_value(headers, "x-amz-request-payer")?.map(Into::into),
|
||||
start_after: query_value(query, "start-after"),
|
||||
})
|
||||
}
|
||||
|
||||
fn bucket_from_request(uri: &Uri, headers: &HeaderMap, host: Option<&MultiDomain>) -> S3Result<String> {
|
||||
if let Some(host) = host
|
||||
&& let Some(host_header) = headers.get(HOST).and_then(|value| value.to_str().ok())
|
||||
@@ -199,6 +778,60 @@ fn path_style_bucket(uri: &Uri) -> S3Result<Option<String>> {
|
||||
.map_err(S3Error::internal_error)
|
||||
}
|
||||
|
||||
fn query_has(query: &str, key: &str) -> bool {
|
||||
form_urlencoded::parse(query.as_bytes()).any(|(name, _)| name == key)
|
||||
}
|
||||
|
||||
fn query_value(query: &str, key: &str) -> Option<String> {
|
||||
form_urlencoded::parse(query.as_bytes())
|
||||
.find(|(name, _)| name == key)
|
||||
.map(|(_, value)| value.into_owned())
|
||||
}
|
||||
|
||||
fn query_i32(query: &str, key: &str) -> S3Result<Option<i32>> {
|
||||
query_value(query, key)
|
||||
.map(|value| {
|
||||
value
|
||||
.parse::<i32>()
|
||||
.map_err(|_| s3_error!(InvalidArgument, "invalid integer query value"))
|
||||
})
|
||||
.transpose()
|
||||
}
|
||||
|
||||
fn query_bool(query: &str, key: &str) -> S3Result<Option<bool>> {
|
||||
query_value(query, key)
|
||||
.map(|value| {
|
||||
value
|
||||
.parse::<bool>()
|
||||
.map_err(|_| s3_error!(InvalidArgument, "invalid boolean query value"))
|
||||
})
|
||||
.transpose()
|
||||
}
|
||||
|
||||
fn parse_encoding_type(value: Option<String>) -> S3Result<Option<EncodingType>> {
|
||||
value
|
||||
.map(|value| {
|
||||
if value == EncodingType::URL {
|
||||
Ok(EncodingType::from_static(EncodingType::URL))
|
||||
} else {
|
||||
Err(s3_error!(InvalidArgument, "invalid encoding-type"))
|
||||
}
|
||||
})
|
||||
.transpose()
|
||||
}
|
||||
|
||||
fn header_value(headers: &HeaderMap, name: &str) -> S3Result<Option<String>> {
|
||||
headers
|
||||
.get(name)
|
||||
.map(|value| {
|
||||
value
|
||||
.to_str()
|
||||
.map(str::to_owned)
|
||||
.map_err(|_| s3_error!(InvalidArgument, "invalid header value"))
|
||||
})
|
||||
.transpose()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
#[tokio::test]
|
||||
@@ -391,9 +1024,49 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
use super::{MetadataOperation, metadata_operation};
|
||||
#[tokio::test]
|
||||
async fn website_304_preserves_cache_headers_without_content() {
|
||||
use http_body_util::BodyExt;
|
||||
let mut error = s3s::S3Error::new(s3s::S3ErrorCode::NotModified);
|
||||
let mut headers = http::HeaderMap::new();
|
||||
for (name, value) in [
|
||||
("etag", "\"abc\""),
|
||||
("cache-control", "max-age=60"),
|
||||
("expires", "Tue, 14 Nov 2023 22:13:20 GMT"),
|
||||
("content-length", "123"),
|
||||
("content-type", "application/xml"),
|
||||
] {
|
||||
headers.insert(name, http::HeaderValue::from_static(value));
|
||||
}
|
||||
error.set_headers(headers);
|
||||
let response = super::website_not_modified(&error);
|
||||
assert_eq!(response.status, Some(http::StatusCode::NOT_MODIFIED));
|
||||
assert_eq!(response.headers["etag"], "\"abc\"");
|
||||
assert_eq!(response.headers["cache-control"], "max-age=60");
|
||||
assert!(response.headers.contains_key("expires"));
|
||||
assert!(!response.headers.contains_key("content-length"));
|
||||
assert!(!response.headers.contains_key("content-type"));
|
||||
assert!(
|
||||
response
|
||||
.output
|
||||
.collect()
|
||||
.await
|
||||
.expect("empty website body")
|
||||
.to_bytes()
|
||||
.is_empty()
|
||||
);
|
||||
}
|
||||
|
||||
use super::{
|
||||
MetadataOperation, list_object_versions_input, list_objects_v2_input, metadata_operation, website_redirect,
|
||||
website_relative_redirect, website_rule_matches, website_target,
|
||||
};
|
||||
use http::header::HOST;
|
||||
use http::{Extensions, HeaderMap, Method, Uri};
|
||||
use http::{Extensions, HeaderMap, Method, StatusCode, Uri};
|
||||
use rustfs_gateway_types::persistence::{
|
||||
PersistedRedirect as Redirect, PersistedRoutingRule as RoutingRule, PersistedRoutingRuleCondition as Condition,
|
||||
};
|
||||
use s3s::dto::EncodingType;
|
||||
use s3s::host::MultiDomain;
|
||||
use s3s::{Body, S3Request};
|
||||
|
||||
@@ -417,6 +1090,93 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn website_host_is_isolated_from_s3_host_and_keeps_directory_suffix() {
|
||||
let domains = vec!["example.com".to_owned()];
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert(HOST, "share.example.com:9000".parse().expect("valid host"));
|
||||
let target = website_target(&uri("/docs/"), &headers, &domains).expect("website host matches");
|
||||
assert_eq!(target.bucket, "share");
|
||||
assert_eq!(target.key, "docs/");
|
||||
assert!(target.trailing_slash);
|
||||
headers.insert(HOST, "s3.other.test".parse().expect("valid host"));
|
||||
assert!(website_target(&uri("/share/docs/"), &headers, &domains).is_none());
|
||||
headers.insert(HOST, "my.bucket.example.com".parse().expect("valid host"));
|
||||
assert_eq!(
|
||||
website_target(&uri("/"), &headers, &domains)
|
||||
.expect("dotted bucket host matches")
|
||||
.bucket,
|
||||
"my.bucket"
|
||||
);
|
||||
headers.insert(HOST, "example.com".parse().expect("valid host"));
|
||||
assert_eq!(
|
||||
website_target(&uri("/share/key"), &headers, &domains)
|
||||
.expect("base domain is reserved")
|
||||
.bucket,
|
||||
""
|
||||
);
|
||||
headers.insert(HOST, "share.example.com".parse().expect("valid host"));
|
||||
assert!(
|
||||
website_target(&uri("/%FF"), &headers, &domains)
|
||||
.expect("invalid path stays in website route")
|
||||
.invalid_path
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn website_redirect_encodes_path_and_preserves_query() {
|
||||
let req = website_request("/old?download=1");
|
||||
let redirect = website_redirect(
|
||||
&req,
|
||||
"new.example.com",
|
||||
Some("https"),
|
||||
"new path/#one",
|
||||
StatusCode::MOVED_PERMANENTLY,
|
||||
"http",
|
||||
)
|
||||
.expect("valid redirect");
|
||||
assert_eq!(
|
||||
redirect.headers[http::header::LOCATION],
|
||||
"https://new.example.com/new%20path/%23one?download=1"
|
||||
);
|
||||
let directory = website_relative_redirect(&req, "/new path/", StatusCode::FOUND).expect("valid directory redirect");
|
||||
assert_eq!(directory.headers[http::header::LOCATION], "/new%20path/?download=1");
|
||||
let secure = website_redirect(&req, "new.example.com", None, "key", StatusCode::MOVED_PERMANENTLY, "https")
|
||||
.expect("TLS listener default scheme");
|
||||
assert_eq!(secure.headers[http::header::LOCATION], "https://new.example.com/key?download=1");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn website_rules_choose_first_match_after_object_status_is_known() {
|
||||
let error_rule = RoutingRule {
|
||||
condition: Some(Condition {
|
||||
http_error_code_returned_equals: Some("404".to_owned()),
|
||||
key_prefix_equals: None,
|
||||
}),
|
||||
redirect: Redirect::default(),
|
||||
};
|
||||
let prefix_rule = RoutingRule {
|
||||
condition: Some(Condition {
|
||||
http_error_code_returned_equals: None,
|
||||
key_prefix_equals: Some("docs/".to_owned()),
|
||||
}),
|
||||
redirect: Redirect::default(),
|
||||
};
|
||||
let rules = [error_rule, prefix_rule];
|
||||
assert_eq!(
|
||||
rules
|
||||
.iter()
|
||||
.position(|rule| website_rule_matches(rule, "docs/missing", StatusCode::NOT_FOUND)),
|
||||
Some(0)
|
||||
);
|
||||
assert_eq!(
|
||||
rules
|
||||
.iter()
|
||||
.position(|rule| website_rule_matches(rule, "docs/existing", StatusCode::OK)),
|
||||
Some(1)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn metadata_operation_matches_path_style_extensions_only() {
|
||||
assert_eq!(
|
||||
@@ -491,4 +1251,55 @@ mod tests {
|
||||
assert_eq!(cname.operation, MetadataOperation::ListObjectsV2);
|
||||
assert_eq!(cname.bucket, "cdn.example.org");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn list_objects_v2_input_parses_query_headers_and_decodes_bucket() {
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert("x-amz-expected-bucket-owner", "123456789012".parse().expect("valid header"));
|
||||
headers.insert("x-amz-request-payer", "requester".parse().expect("valid header"));
|
||||
|
||||
let input = list_objects_v2_input(
|
||||
"demo bucket".to_string(),
|
||||
&uri("/demo%20bucket?list-type=2&metadata=true&prefix=logs%2F&delimiter=%2F&encoding-type=url&fetch-owner=true&max-keys=25&continuation-token=opaque&start-after=start"),
|
||||
&headers,
|
||||
)
|
||||
.expect("list objects v2 input should parse");
|
||||
|
||||
assert_eq!(input.bucket, "demo bucket");
|
||||
assert_eq!(input.prefix.as_deref(), Some("logs/"));
|
||||
assert_eq!(input.delimiter.as_deref(), Some("/"));
|
||||
assert_eq!(input.encoding_type.as_ref().map(EncodingType::as_str), Some(EncodingType::URL));
|
||||
assert_eq!(input.fetch_owner, Some(true));
|
||||
assert_eq!(input.max_keys, Some(25));
|
||||
assert_eq!(input.continuation_token.as_deref(), Some("opaque"));
|
||||
assert_eq!(input.start_after.as_deref(), Some("start"));
|
||||
assert_eq!(input.expected_bucket_owner.as_deref(), Some("123456789012"));
|
||||
assert_eq!(input.request_payer.as_ref().map(|payer| payer.as_str()), Some("requester"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn list_object_versions_input_parses_markers_and_rejects_invalid_encoding() {
|
||||
let input = list_object_versions_input(
|
||||
"bucket".to_string(),
|
||||
&uri("/bucket?versions&metadata=true&prefix=logs%2F&delimiter=%2F&encoding-type=url&key-marker=start&version-id-marker=v1&max-keys=10"),
|
||||
&HeaderMap::new(),
|
||||
)
|
||||
.expect("list versions input should parse");
|
||||
|
||||
assert_eq!(input.bucket, "bucket");
|
||||
assert_eq!(input.prefix.as_deref(), Some("logs/"));
|
||||
assert_eq!(input.delimiter.as_deref(), Some("/"));
|
||||
assert_eq!(input.encoding_type.as_ref().map(EncodingType::as_str), Some(EncodingType::URL));
|
||||
assert_eq!(input.key_marker.as_deref(), Some("start"));
|
||||
assert_eq!(input.version_id_marker.as_deref(), Some("v1"));
|
||||
assert_eq!(input.max_keys, Some(10));
|
||||
|
||||
let err = list_object_versions_input(
|
||||
"bucket".to_string(),
|
||||
&uri("/bucket?versions&metadata=true&encoding-type=xml"),
|
||||
&HeaderMap::new(),
|
||||
)
|
||||
.expect_err("invalid encoding-type should be rejected");
|
||||
assert_eq!(err.code(), &s3s::S3ErrorCode::InvalidArgument);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -26,7 +26,6 @@ pub(crate) mod bucket_list_through;
|
||||
pub mod bucket_usecase;
|
||||
pub mod context;
|
||||
pub(crate) mod dto_values;
|
||||
pub(crate) mod metadata_listing;
|
||||
pub(crate) mod metadata_route;
|
||||
pub mod multipart_usecase;
|
||||
pub mod object;
|
||||
@@ -40,7 +39,6 @@ pub(crate) mod select_request;
|
||||
pub(crate) mod storage_api;
|
||||
pub(crate) mod table_list_isolation;
|
||||
pub(crate) mod trailer_adapter;
|
||||
pub(crate) mod website;
|
||||
|
||||
// Request-scoped facts are not AppContext resolution, so consumers import them
|
||||
// from here rather than through `app::context`.
|
||||
|
||||
@@ -1266,11 +1266,6 @@ impl DefaultObjectUsecase {
|
||||
}
|
||||
}
|
||||
|
||||
#[hotpath::measure(
|
||||
impl_type = "DefaultObjectUsecase",
|
||||
future = true,
|
||||
label = "rustfs::app::object::put::DefaultObjectUsecase::execute_put_object"
|
||||
)]
|
||||
pub(crate) async fn execute_put_object_owned(
|
||||
&self,
|
||||
envelope: RequestEnvelope,
|
||||
|
||||
@@ -1,557 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Static website hosting: which requests a website host claims, and how they are answered.
|
||||
//!
|
||||
//! Responsible for: [`WebsiteHost`], the one copy of the website rules both S3 stacks serve —
|
||||
//! the `Host` that selects a bucket, the index and error documents, the redirect rules and the
|
||||
//! object's own redirect location — read anonymously through the owned GetObject and HeadObject
|
||||
//! authorization and use cases, marking each GetObject with `WebsiteRead` so it reports the
|
||||
//! object's redirect location.
|
||||
//! NOT responsible for: where a stack mounts the host (`app/metadata_route.rs` on the legacy
|
||||
//! stack, `server/stack.rs` on the gateway stack), or any object read itself (the use cases).
|
||||
//! Upstream: the legacy `MetadataRoute` and the gateway stack's website service. Downstream: the
|
||||
//! owned GetObject and HeadObject authorization and use cases, and the bucket's website
|
||||
//! configuration.
|
||||
|
||||
use crate::app::object_usecase::{DefaultObjectUsecase, GetObjectBody, GetObjectFields, GetObjectOutputFacts, HeadObjectFields};
|
||||
use crate::app::runtime_sources::ServerContextSlot;
|
||||
use crate::app::storage_api::s3::dto::{
|
||||
ETagCondition, GetObjectInput, Range, Timestamp as WireTimestamp, TimestampFormat as WireFormat,
|
||||
};
|
||||
use crate::app::{Answered, RequestEnvelope, RequestHead};
|
||||
use crate::storage::access::access_owned::object_read::{authorize_get_object, authorize_head_object};
|
||||
use crate::storage::req_info::ReqInfo;
|
||||
use crate::storage::request_markers::WebsiteRead;
|
||||
use crate::storage::{get_bucket_website_config_for_store, validate_website_configuration};
|
||||
use http::header::{CONTENT_TYPE, HOST, HeaderName};
|
||||
use http::{Extensions, HeaderMap, HeaderValue, Method, Response, StatusCode, Uri};
|
||||
use rustfs_gateway_stream::Body;
|
||||
use rustfs_gateway_types::TimestampFormat;
|
||||
use rustfs_gateway_types::persistence::{PersistedErrorDocument, PersistedRoutingRule as RoutingRule};
|
||||
use rustfs_io_metrics::record_s3_op;
|
||||
use rustfs_s3_ops::S3Operation;
|
||||
use rustfs_s3_types::{S3Error, S3ErrorCode, S3Result};
|
||||
use std::sync::Arc;
|
||||
|
||||
/// An answer of the website host.
|
||||
pub(crate) type WebsiteResponse = Response<Body>;
|
||||
|
||||
/// The headers a website read drops before a lookup that must not be conditional or ranged.
|
||||
const CONDITIONAL_HEADERS: [HeaderName; 5] = [
|
||||
http::header::RANGE,
|
||||
http::header::IF_MATCH,
|
||||
http::header::IF_NONE_MATCH,
|
||||
http::header::IF_MODIFIED_SINCE,
|
||||
http::header::IF_UNMODIFIED_SINCE,
|
||||
];
|
||||
|
||||
/// The website host of one server: the configured website domains, the scheme its redirects
|
||||
/// default to, and the server whose buckets it serves.
|
||||
pub(crate) struct WebsiteHost {
|
||||
domains: Vec<String>,
|
||||
scheme: &'static str,
|
||||
server_ctx: Arc<ServerContextSlot>,
|
||||
}
|
||||
|
||||
impl WebsiteHost {
|
||||
/// The host for `domains`, or `None` when no website domain is configured.
|
||||
pub(crate) fn new(domains: Vec<String>, scheme: &'static str, server_ctx: Arc<ServerContextSlot>) -> Option<Self> {
|
||||
(!domains.is_empty()).then_some(Self {
|
||||
domains,
|
||||
scheme,
|
||||
server_ctx,
|
||||
})
|
||||
}
|
||||
|
||||
/// Whether the request's `Host` is a website domain, so this host answers it.
|
||||
pub(crate) fn claims(&self, uri: &Uri, headers: &HeaderMap) -> bool {
|
||||
website_target(uri, headers, &self.domains).is_some()
|
||||
}
|
||||
|
||||
/// Answers one claimed request anonymously. Every failure is a website error page with the
|
||||
/// failure's status, never an S3 error document.
|
||||
pub(crate) async fn answer(&self, method: Method, uri: Uri, headers: HeaderMap, extensions: Extensions) -> WebsiteResponse {
|
||||
let head = method == Method::HEAD;
|
||||
let Some(target) = website_target(&uri, &headers, &self.domains) else {
|
||||
return website_error(StatusCode::NOT_FOUND, head);
|
||||
};
|
||||
let request = WebsiteRequest {
|
||||
method,
|
||||
uri,
|
||||
headers,
|
||||
extensions,
|
||||
server_ctx: Arc::clone(&self.server_ctx),
|
||||
};
|
||||
// The reads are large futures; boxed, a website request does not grow the caller's stack.
|
||||
match Box::pin(serve(&request, target, self.scheme)).await {
|
||||
Ok(response) => response,
|
||||
Err(err) => website_error(err.status_code().unwrap_or(StatusCode::INTERNAL_SERVER_ERROR), head),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
struct WebsiteTarget {
|
||||
bucket: String,
|
||||
key: String,
|
||||
trailing_slash: bool,
|
||||
invalid_path: bool,
|
||||
}
|
||||
|
||||
fn website_target(uri: &Uri, headers: &HeaderMap, domains: &[String]) -> Option<WebsiteTarget> {
|
||||
if domains.is_empty() {
|
||||
return None;
|
||||
}
|
||||
let host_header = headers.get(HOST)?.to_str().ok()?;
|
||||
let authority = host_header.parse::<http::uri::Authority>().ok()?;
|
||||
let request_host = authority.host().trim_end_matches('.').to_ascii_lowercase();
|
||||
let bucket = domains.iter().find_map(|domain| {
|
||||
if request_host == *domain {
|
||||
return Some(String::new());
|
||||
}
|
||||
request_host.strip_suffix(&format!(".{domain}")).map(str::to_owned)
|
||||
})?;
|
||||
let raw_path = uri.path();
|
||||
let decoded = urlencoding::decode(raw_path);
|
||||
let invalid_path = decoded.is_err();
|
||||
let decoded = decoded.unwrap_or_default();
|
||||
let path = decoded.trim_start_matches('/');
|
||||
Some(WebsiteTarget {
|
||||
bucket,
|
||||
key: path.to_owned(),
|
||||
trailing_slash: path.is_empty() || path.ends_with('/'),
|
||||
invalid_path,
|
||||
})
|
||||
}
|
||||
|
||||
/// One website request as the reads it makes see it.
|
||||
struct WebsiteRequest {
|
||||
method: Method,
|
||||
uri: Uri,
|
||||
headers: HeaderMap,
|
||||
extensions: Extensions,
|
||||
server_ctx: Arc<ServerContextSlot>,
|
||||
}
|
||||
|
||||
impl WebsiteRequest {
|
||||
fn head(&self) -> bool {
|
||||
self.method == Method::HEAD
|
||||
}
|
||||
|
||||
/// An anonymous read of `bucket/key`: no credentials and no owner, whatever the request carried.
|
||||
fn envelope(&self, method: Method, headers: HeaderMap, bucket: &str, key: &str, website_read: bool) -> RequestEnvelope {
|
||||
let mut extensions = self.extensions.clone();
|
||||
extensions.insert(Arc::clone(&self.server_ctx));
|
||||
let mut req_info = extensions.remove::<ReqInfo>().unwrap_or_default();
|
||||
req_info.cred = None;
|
||||
req_info.is_owner = false;
|
||||
req_info.bucket = Some(bucket.to_owned());
|
||||
req_info.object = Some(key.to_owned());
|
||||
req_info.version_id = None;
|
||||
extensions.insert(req_info);
|
||||
if website_read {
|
||||
extensions.insert(WebsiteRead);
|
||||
}
|
||||
let head = RequestHead {
|
||||
method,
|
||||
uri: self.uri.clone(),
|
||||
headers,
|
||||
credentials: None,
|
||||
region: None,
|
||||
service: None,
|
||||
};
|
||||
RequestEnvelope::from_parts(head, extensions)
|
||||
}
|
||||
|
||||
fn usecase(&self) -> DefaultObjectUsecase {
|
||||
DefaultObjectUsecase::with_context(self.server_ctx.app_context())
|
||||
}
|
||||
|
||||
/// GetObject of `bucket/key`, with the request's range and conditions when `apply_conditions`.
|
||||
async fn get_object(&self, bucket: &str, key: &str, apply_conditions: bool) -> S3Result<Answered<GetObjectOutputFacts>> {
|
||||
let (range, headers) = if apply_conditions {
|
||||
(validated_conditions(&self.headers)?, self.headers.clone())
|
||||
} else {
|
||||
(None, without_conditions(&self.headers))
|
||||
};
|
||||
let mut envelope = self.envelope(Method::GET, headers, bucket, key, true);
|
||||
authorize_get_object(&self.server_ctx, &mut envelope, bucket, key, None).await?;
|
||||
record_s3_op(S3Operation::GetObject);
|
||||
let fields = GetObjectFields::from_s3s(GetObjectInput {
|
||||
bucket: bucket.to_owned(),
|
||||
key: key.to_owned(),
|
||||
range,
|
||||
..Default::default()
|
||||
});
|
||||
Box::pin(self.usecase().execute_get_object_fields(envelope, fields)).await
|
||||
}
|
||||
|
||||
/// The redirect location HeadObject reports for `bucket/key`, read unconditionally.
|
||||
async fn head_redirect(&self, bucket: &str, key: &str) -> S3Result<Option<String>> {
|
||||
let mut envelope = self.envelope(Method::HEAD, without_conditions(&self.headers), bucket, key, false);
|
||||
authorize_head_object(&self.server_ctx, &mut envelope, bucket, key, None).await?;
|
||||
record_s3_op(S3Operation::HeadObject);
|
||||
let fields = HeadObjectFields {
|
||||
bucket: bucket.to_owned(),
|
||||
key: key.to_owned(),
|
||||
version_id: None,
|
||||
part_number: None,
|
||||
range: None,
|
||||
sse_customer_key: None,
|
||||
sse_customer_key_md5: None,
|
||||
};
|
||||
let answered = Box::pin(self.usecase().execute_head_object_fields(envelope, fields)).await?;
|
||||
Ok(answered.output.dto.website_redirect_location)
|
||||
}
|
||||
}
|
||||
|
||||
fn header_text(headers: &HeaderMap, name: HeaderName) -> S3Result<Option<String>> {
|
||||
headers
|
||||
.get(name)
|
||||
.map(|value| {
|
||||
value
|
||||
.to_str()
|
||||
.map(str::to_owned)
|
||||
.map_err(|_| S3Error::new(S3ErrorCode::InvalidArgument))
|
||||
})
|
||||
.transpose()
|
||||
}
|
||||
|
||||
/// The range a conditional read asks for, after refusing a malformed range or condition the
|
||||
/// way the S3 GetObject decoder refuses it.
|
||||
fn validated_conditions(headers: &HeaderMap) -> S3Result<Option<Range>> {
|
||||
let range = header_text(headers, http::header::RANGE)?
|
||||
.map(|value| Range::parse(&value).map_err(|_| S3Error::new(S3ErrorCode::InvalidRange)))
|
||||
.transpose()?;
|
||||
for name in [http::header::IF_MATCH, http::header::IF_NONE_MATCH] {
|
||||
if let Some(value) = headers.get(name) {
|
||||
ETagCondition::parse_http_header(value.as_bytes()).map_err(|_| S3Error::new(S3ErrorCode::InvalidArgument))?;
|
||||
}
|
||||
}
|
||||
for name in [http::header::IF_MODIFIED_SINCE, http::header::IF_UNMODIFIED_SINCE] {
|
||||
if let Some(value) = header_text(headers, name)? {
|
||||
WireTimestamp::parse(WireFormat::HttpDate, &value).map_err(|_| S3Error::new(S3ErrorCode::InvalidArgument))?;
|
||||
}
|
||||
}
|
||||
Ok(range)
|
||||
}
|
||||
|
||||
fn without_conditions(headers: &HeaderMap) -> HeaderMap {
|
||||
let mut headers = headers.clone();
|
||||
for name in CONDITIONAL_HEADERS {
|
||||
headers.remove(name);
|
||||
}
|
||||
headers
|
||||
}
|
||||
|
||||
async fn serve(req: &WebsiteRequest, target: WebsiteTarget, website_scheme: &'static str) -> S3Result<WebsiteResponse> {
|
||||
if !matches!(req.method, Method::GET | Method::HEAD) {
|
||||
return Ok(website_error(StatusCode::METHOD_NOT_ALLOWED, req.head()));
|
||||
}
|
||||
if target.invalid_path {
|
||||
return Ok(website_error(StatusCode::BAD_REQUEST, req.head()));
|
||||
}
|
||||
if target.bucket.is_empty() || target.bucket.starts_with('.') || target.bucket.ends_with('.') || target.bucket.contains("..")
|
||||
{
|
||||
return Ok(website_error(StatusCode::NOT_FOUND, req.head()));
|
||||
}
|
||||
|
||||
let store = req
|
||||
.server_ctx
|
||||
.installed_object_store()
|
||||
.ok_or_else(|| S3Error::with_message(S3ErrorCode::InternalError, "website object store unavailable"))?;
|
||||
let config = match get_bucket_website_config_for_store(&store, &target.bucket).await {
|
||||
Ok(config) => config,
|
||||
Err(crate::storage::StorageError::ConfigNotFound | crate::storage::StorageError::BucketNotFound(_)) => {
|
||||
return Ok(website_error(StatusCode::NOT_FOUND, req.head()));
|
||||
}
|
||||
Err(err) => return Err(crate::error::ApiError::from(err).into()),
|
||||
};
|
||||
if validate_website_configuration(&config).is_err() {
|
||||
return Ok(website_error(StatusCode::INTERNAL_SERVER_ERROR, req.head()));
|
||||
}
|
||||
|
||||
if let Some(redirect) = config.redirect_all_requests_to.as_ref() {
|
||||
return website_redirect(
|
||||
&req.uri,
|
||||
&req.headers,
|
||||
redirect.host_name.as_str(),
|
||||
redirect.protocol.as_deref(),
|
||||
&target.key,
|
||||
StatusCode::MOVED_PERMANENTLY,
|
||||
website_scheme,
|
||||
);
|
||||
}
|
||||
|
||||
let index_suffix = config
|
||||
.index_document
|
||||
.as_ref()
|
||||
.map(|doc| doc.suffix.as_str())
|
||||
.unwrap_or_default();
|
||||
let key = if target.key.is_empty() || target.trailing_slash {
|
||||
format!("{}{index_suffix}", target.key)
|
||||
} else {
|
||||
target.key.clone()
|
||||
};
|
||||
|
||||
let rules = config.routing_rules.as_deref().unwrap_or_default();
|
||||
// A rule requiring a status cannot be evaluated until the object lookup completes.
|
||||
for rule in rules {
|
||||
if rule
|
||||
.condition
|
||||
.as_ref()
|
||||
.and_then(|condition| condition.http_error_code_returned_equals.as_ref())
|
||||
.is_some()
|
||||
{
|
||||
break;
|
||||
}
|
||||
if website_rule_matches(rule, &target.key, StatusCode::OK) {
|
||||
return website_rule_redirect(req, rule, &target.key, website_scheme);
|
||||
}
|
||||
}
|
||||
|
||||
let response = req.get_object(&target.bucket, &key, true).await;
|
||||
if let Err(err) = &response
|
||||
&& matches!(
|
||||
err.code(),
|
||||
S3ErrorCode::NotModified | S3ErrorCode::PreconditionFailed | S3ErrorCode::InvalidRange
|
||||
)
|
||||
&& let Ok(Some(location)) = req.head_redirect(&target.bucket, &key).await
|
||||
{
|
||||
return website_location(location, StatusCode::MOVED_PERMANENTLY);
|
||||
}
|
||||
let status = match &response {
|
||||
Ok(_) => StatusCode::OK,
|
||||
Err(err) => err.status_code().unwrap_or(StatusCode::INTERNAL_SERVER_ERROR),
|
||||
};
|
||||
if status == StatusCode::NOT_FOUND && !target.trailing_slash && !target.key.is_empty() {
|
||||
let index_key = format!("{}/{index_suffix}", target.key);
|
||||
match req.get_object(&target.bucket, &index_key, false).await {
|
||||
Ok(_) => return website_relative_redirect(&req.uri, &format!("/{}/", target.key), StatusCode::FOUND),
|
||||
Err(err)
|
||||
if matches!(
|
||||
err.code(),
|
||||
S3ErrorCode::NoSuchKey | S3ErrorCode::NoSuchVersion | S3ErrorCode::AccessDenied
|
||||
) => {}
|
||||
Err(err) => return Err(err),
|
||||
}
|
||||
}
|
||||
if let Some(rule) = rules.iter().find(|rule| website_rule_matches(rule, &target.key, status)) {
|
||||
return website_rule_redirect(req, rule, &target.key, website_scheme);
|
||||
}
|
||||
match response {
|
||||
Ok(response) => {
|
||||
if let Some(location) = response.output.dto.website_redirect_location.clone() {
|
||||
return website_location(location, StatusCode::MOVED_PERMANENTLY);
|
||||
}
|
||||
response_to_website(response, req.head())
|
||||
}
|
||||
Err(err) if matches!(err.code(), S3ErrorCode::NoSuchKey | S3ErrorCode::NoSuchVersion) => {
|
||||
website_error_with_document(req, &target.bucket, config.error_document.as_ref(), StatusCode::NOT_FOUND).await
|
||||
}
|
||||
Err(err) if err.code() == &S3ErrorCode::NotModified => Ok(website_not_modified(&err)),
|
||||
Err(_) if status.is_client_error() => {
|
||||
website_error_with_document(req, &target.bucket, config.error_document.as_ref(), status).await
|
||||
}
|
||||
Err(err) => Err(err),
|
||||
}
|
||||
}
|
||||
|
||||
/// The object's page: its representation headers, and its body unless the request is `HEAD`.
|
||||
fn response_to_website(response: Answered<GetObjectOutputFacts>, head: bool) -> S3Result<WebsiteResponse> {
|
||||
let Answered { output, mut headers, .. } = response;
|
||||
let GetObjectOutputFacts { dto, etag, body } = output;
|
||||
insert_website_header(&mut headers, http::header::CONTENT_TYPE, dto.content_type.as_deref())?;
|
||||
insert_website_header(
|
||||
&mut headers,
|
||||
http::header::CONTENT_LENGTH,
|
||||
dto.content_length.map(|length| length.to_string()).as_deref(),
|
||||
)?;
|
||||
insert_website_header(&mut headers, http::header::CONTENT_RANGE, dto.content_range.as_deref())?;
|
||||
let etag = etag.map(|tag| {
|
||||
if tag.weak {
|
||||
format!("W/\"{}\"", tag.opaque)
|
||||
} else {
|
||||
format!("\"{}\"", tag.opaque)
|
||||
}
|
||||
});
|
||||
insert_website_header(&mut headers, http::header::ETAG, etag.as_deref())?;
|
||||
insert_website_header(&mut headers, http::header::CACHE_CONTROL, dto.cache_control.as_deref())?;
|
||||
insert_website_header(&mut headers, http::header::CONTENT_ENCODING, dto.content_encoding.as_deref())?;
|
||||
insert_website_header(&mut headers, http::header::CONTENT_LANGUAGE, dto.content_language.as_deref())?;
|
||||
insert_website_header(&mut headers, http::header::CONTENT_DISPOSITION, dto.content_disposition.as_deref())?;
|
||||
let last_modified = dto
|
||||
.last_modified
|
||||
.map(|at| at.render(TimestampFormat::HttpDate).map_err(S3Error::internal_error))
|
||||
.transpose()?;
|
||||
insert_website_header(&mut headers, http::header::LAST_MODIFIED, last_modified.as_deref())?;
|
||||
let body = match body {
|
||||
_ if head => Body::empty(),
|
||||
None => Body::empty(),
|
||||
Some(GetObjectBody::Bytes(bytes)) => Body::from_bytes(bytes),
|
||||
Some(GetObjectBody::Stream(stream)) => Body::from_stream(stream).map_err(S3Error::internal_error)?,
|
||||
};
|
||||
let status = if dto.content_range.is_some() {
|
||||
StatusCode::PARTIAL_CONTENT
|
||||
} else {
|
||||
StatusCode::OK
|
||||
};
|
||||
Ok(answer(status, headers, body))
|
||||
}
|
||||
|
||||
fn answer(status: StatusCode, headers: HeaderMap, body: Body) -> WebsiteResponse {
|
||||
let mut response = Response::new(body);
|
||||
*response.status_mut() = status;
|
||||
*response.headers_mut() = headers;
|
||||
response
|
||||
}
|
||||
|
||||
fn insert_website_header(headers: &mut HeaderMap, name: HeaderName, value: Option<&str>) -> S3Result<()> {
|
||||
if let Some(value) = value {
|
||||
headers.insert(name, HeaderValue::from_str(value).map_err(S3Error::internal_error)?);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn website_error_with_document(
|
||||
req: &WebsiteRequest,
|
||||
bucket: &str,
|
||||
error_document: Option<&PersistedErrorDocument>,
|
||||
status: StatusCode,
|
||||
) -> S3Result<WebsiteResponse> {
|
||||
let Some(error_document) = error_document else {
|
||||
return Ok(website_error(status, req.head()));
|
||||
};
|
||||
match req.get_object(bucket, error_document.key.as_str(), false).await {
|
||||
Ok(response) => {
|
||||
let mut output = response_to_website(response, req.head())?;
|
||||
*output.status_mut() = status;
|
||||
Ok(output)
|
||||
}
|
||||
Err(err)
|
||||
if matches!(
|
||||
err.code(),
|
||||
S3ErrorCode::NoSuchKey | S3ErrorCode::NoSuchVersion | S3ErrorCode::AccessDenied
|
||||
) =>
|
||||
{
|
||||
Ok(website_error(status, req.head()))
|
||||
}
|
||||
Err(err) => Err(err),
|
||||
}
|
||||
}
|
||||
|
||||
fn website_error(status: StatusCode, head: bool) -> WebsiteResponse {
|
||||
let message = status.canonical_reason().unwrap_or("Error");
|
||||
let body = format!("<html><body>{message}</body></html>");
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert(CONTENT_TYPE, HeaderValue::from_static("text/html; charset=utf-8"));
|
||||
answer(status, headers, if head { Body::empty() } else { Body::from_bytes(body.into()) })
|
||||
}
|
||||
|
||||
fn website_not_modified(error: &S3Error) -> WebsiteResponse {
|
||||
let mut headers = error.headers().cloned().unwrap_or_default();
|
||||
headers.remove(http::header::CONTENT_LENGTH);
|
||||
headers.remove(http::header::CONTENT_TYPE);
|
||||
headers.remove(http::header::TRANSFER_ENCODING);
|
||||
answer(StatusCode::NOT_MODIFIED, headers, Body::empty())
|
||||
}
|
||||
|
||||
fn website_location(location: String, status: StatusCode) -> S3Result<WebsiteResponse> {
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert(
|
||||
http::header::LOCATION,
|
||||
HeaderValue::try_from(location)
|
||||
.map_err(|_| S3Error::with_message(S3ErrorCode::InternalError, "invalid website redirect location"))?,
|
||||
);
|
||||
Ok(answer(status, headers, Body::empty()))
|
||||
}
|
||||
|
||||
fn website_relative_redirect(uri: &Uri, path: &str, status: StatusCode) -> S3Result<WebsiteResponse> {
|
||||
let mut url = url::Url::parse("http://localhost/").map_err(S3Error::internal_error)?;
|
||||
url.set_path(path);
|
||||
url.set_query(uri.query());
|
||||
let location = match url.query() {
|
||||
Some(query) => format!("{}?{query}", url.path()),
|
||||
None => url.path().to_owned(),
|
||||
};
|
||||
website_location(location, status)
|
||||
}
|
||||
|
||||
fn website_redirect(
|
||||
uri: &Uri,
|
||||
headers: &HeaderMap,
|
||||
host: &str,
|
||||
protocol: Option<&str>,
|
||||
key: &str,
|
||||
status: StatusCode,
|
||||
website_scheme: &str,
|
||||
) -> S3Result<WebsiteResponse> {
|
||||
let scheme = protocol.unwrap_or(website_scheme);
|
||||
let host = if host.is_empty() {
|
||||
headers.get(HOST).and_then(|h| h.to_str().ok()).unwrap_or_default()
|
||||
} else {
|
||||
host
|
||||
};
|
||||
let mut url = url::Url::parse(&format!("{scheme}://{host}/")).map_err(S3Error::internal_error)?;
|
||||
url.set_path(&format!("/{key}"));
|
||||
url.set_query(uri.query());
|
||||
website_location(url.into(), status)
|
||||
}
|
||||
|
||||
fn website_rule_matches(rule: &RoutingRule, key: &str, status: StatusCode) -> bool {
|
||||
let Some(condition) = rule.condition.as_ref() else { return true };
|
||||
condition
|
||||
.key_prefix_equals
|
||||
.as_ref()
|
||||
.is_none_or(|prefix| key.starts_with(prefix))
|
||||
&& condition
|
||||
.http_error_code_returned_equals
|
||||
.as_ref()
|
||||
.is_none_or(|code| code == status.as_str())
|
||||
}
|
||||
|
||||
fn website_rule_redirect(req: &WebsiteRequest, rule: &RoutingRule, key: &str, website_scheme: &str) -> S3Result<WebsiteResponse> {
|
||||
let redirect = &rule.redirect;
|
||||
let key = if let Some(replace) = &redirect.replace_key_with {
|
||||
replace.clone()
|
||||
} else if let Some(replace) = &redirect.replace_key_prefix_with {
|
||||
let prefix = rule
|
||||
.condition
|
||||
.as_ref()
|
||||
.and_then(|condition| condition.key_prefix_equals.as_deref())
|
||||
.unwrap_or_default();
|
||||
format!("{replace}{}", key.strip_prefix(prefix).unwrap_or(key))
|
||||
} else {
|
||||
key.to_owned()
|
||||
};
|
||||
let status = redirect
|
||||
.http_redirect_code
|
||||
.as_deref()
|
||||
.and_then(|code| code.parse::<u16>().ok())
|
||||
.and_then(|code| StatusCode::from_u16(code).ok())
|
||||
.unwrap_or(StatusCode::MOVED_PERMANENTLY);
|
||||
website_redirect(
|
||||
&req.uri,
|
||||
&req.headers,
|
||||
redirect.host_name.as_deref().unwrap_or_default(),
|
||||
redirect.protocol.as_deref(),
|
||||
&key,
|
||||
status,
|
||||
website_scheme,
|
||||
)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
@@ -1,463 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! The website rules, and the host answering real buckets as both stacks serve them.
|
||||
|
||||
use super::*;
|
||||
use crate::app::storage_api::bucket_usecase::contract::bucket::{BucketOperations as _, MakeBucketOptions};
|
||||
use crate::app::storage_api::object_usecase::contract::object::ObjectIO as _;
|
||||
use crate::app::storage_api::object_usecase::{StorageObjectOptions, StoragePutObjReader};
|
||||
use crate::app::storage_api::test::ECStore;
|
||||
use http_body_util::BodyExt;
|
||||
use rustfs_gateway_types::persistence::{
|
||||
PersistedRedirect as Redirect, PersistedRoutingRule as RoutingRule, PersistedRoutingRuleCondition as Condition,
|
||||
};
|
||||
use serial_test::serial;
|
||||
|
||||
fn uri(value: &str) -> Uri {
|
||||
value.parse().expect("test URI should parse")
|
||||
}
|
||||
|
||||
fn host(value: &str) -> HeaderMap {
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert(HOST, value.parse().expect("valid host"));
|
||||
headers
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn website_host_is_isolated_from_s3_host_and_keeps_directory_suffix() {
|
||||
let domains = vec!["example.com".to_owned()];
|
||||
let mut headers = host("share.example.com:9000");
|
||||
let target = website_target(&uri("/docs/"), &headers, &domains).expect("website host matches");
|
||||
assert_eq!(target.bucket, "share");
|
||||
assert_eq!(target.key, "docs/");
|
||||
assert!(target.trailing_slash);
|
||||
headers.insert(HOST, "s3.other.test".parse().expect("valid host"));
|
||||
assert!(website_target(&uri("/share/docs/"), &headers, &domains).is_none());
|
||||
headers.insert(HOST, "my.bucket.example.com".parse().expect("valid host"));
|
||||
assert_eq!(
|
||||
website_target(&uri("/"), &headers, &domains)
|
||||
.expect("dotted bucket host matches")
|
||||
.bucket,
|
||||
"my.bucket"
|
||||
);
|
||||
headers.insert(HOST, "example.com".parse().expect("valid host"));
|
||||
assert_eq!(
|
||||
website_target(&uri("/share/key"), &headers, &domains)
|
||||
.expect("base domain is reserved")
|
||||
.bucket,
|
||||
""
|
||||
);
|
||||
headers.insert(HOST, "share.example.com".parse().expect("valid host"));
|
||||
assert!(
|
||||
website_target(&uri("/%FF"), &headers, &domains)
|
||||
.expect("invalid path stays in website route")
|
||||
.invalid_path
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn website_host_claims_only_its_domains_and_nothing_without_one() {
|
||||
let server_ctx = ServerContextSlot::new();
|
||||
assert!(WebsiteHost::new(Vec::new(), "http", Arc::clone(&server_ctx)).is_none());
|
||||
let website = WebsiteHost::new(vec!["website.test".to_owned()], "http", server_ctx).expect("a configured domain");
|
||||
for claimed in [
|
||||
"website.test",
|
||||
"bucket.website.test",
|
||||
"BUCKET.Website.Test.",
|
||||
"bucket.website.test:9000",
|
||||
] {
|
||||
assert!(website.claims(&uri("/key"), &host(claimed)), "{claimed}");
|
||||
}
|
||||
for unclaimed in [
|
||||
"localhost",
|
||||
"website.test.evil",
|
||||
"bucketwebsite.test",
|
||||
"s3.example.test",
|
||||
"a b",
|
||||
] {
|
||||
assert!(!website.claims(&uri("/key"), &host(unclaimed)), "{unclaimed}");
|
||||
}
|
||||
assert!(!website.claims(&uri("/key"), &HeaderMap::new()), "no Host");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn website_redirect_encodes_path_and_preserves_query() {
|
||||
let target = uri("/old?download=1");
|
||||
let headers = host("site.example.com");
|
||||
let redirect = website_redirect(
|
||||
&target,
|
||||
&headers,
|
||||
"new.example.com",
|
||||
Some("https"),
|
||||
"new path/#one",
|
||||
StatusCode::MOVED_PERMANENTLY,
|
||||
"http",
|
||||
)
|
||||
.expect("valid redirect");
|
||||
assert_eq!(
|
||||
redirect.headers()[http::header::LOCATION],
|
||||
"https://new.example.com/new%20path/%23one?download=1"
|
||||
);
|
||||
let directory = website_relative_redirect(&target, "/new path/", StatusCode::FOUND).expect("valid directory redirect");
|
||||
assert_eq!(directory.headers()[http::header::LOCATION], "/new%20path/?download=1");
|
||||
let secure = website_redirect(&target, &headers, "new.example.com", None, "key", StatusCode::MOVED_PERMANENTLY, "https")
|
||||
.expect("TLS listener default scheme");
|
||||
assert_eq!(secure.headers()[http::header::LOCATION], "https://new.example.com/key?download=1");
|
||||
let same_host = website_redirect(&target, &headers, "", None, "key", StatusCode::MOVED_PERMANENTLY, "http")
|
||||
.expect("an empty host name keeps the request's host");
|
||||
assert_eq!(same_host.headers()[http::header::LOCATION], "http://site.example.com/key?download=1");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn website_rules_choose_first_match_after_object_status_is_known() {
|
||||
let error_rule = RoutingRule {
|
||||
condition: Some(Condition {
|
||||
http_error_code_returned_equals: Some("404".to_owned()),
|
||||
key_prefix_equals: None,
|
||||
}),
|
||||
redirect: Redirect::default(),
|
||||
};
|
||||
let prefix_rule = RoutingRule {
|
||||
condition: Some(Condition {
|
||||
http_error_code_returned_equals: None,
|
||||
key_prefix_equals: Some("docs/".to_owned()),
|
||||
}),
|
||||
redirect: Redirect::default(),
|
||||
};
|
||||
let rules = [error_rule, prefix_rule];
|
||||
assert_eq!(
|
||||
rules
|
||||
.iter()
|
||||
.position(|rule| website_rule_matches(rule, "docs/missing", StatusCode::NOT_FOUND)),
|
||||
Some(0)
|
||||
);
|
||||
assert_eq!(
|
||||
rules
|
||||
.iter()
|
||||
.position(|rule| website_rule_matches(rule, "docs/existing", StatusCode::OK)),
|
||||
Some(1)
|
||||
);
|
||||
assert_eq!(
|
||||
rules
|
||||
.iter()
|
||||
.position(|rule| website_rule_matches(rule, "other", StatusCode::FORBIDDEN)),
|
||||
None
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn website_304_preserves_cache_headers_without_content() {
|
||||
let mut error = S3Error::new(S3ErrorCode::NotModified);
|
||||
let mut headers = HeaderMap::new();
|
||||
for (name, value) in [
|
||||
("etag", "\"abc\""),
|
||||
("cache-control", "max-age=60"),
|
||||
("expires", "Tue, 14 Nov 2023 22:13:20 GMT"),
|
||||
("content-length", "123"),
|
||||
("content-type", "application/xml"),
|
||||
("transfer-encoding", "chunked"),
|
||||
] {
|
||||
headers.insert(name, HeaderValue::from_static(value));
|
||||
}
|
||||
error.set_headers(headers);
|
||||
let response = website_not_modified(&error);
|
||||
assert_eq!(response.status(), StatusCode::NOT_MODIFIED);
|
||||
assert_eq!(response.headers()["etag"], "\"abc\"");
|
||||
assert_eq!(response.headers()["cache-control"], "max-age=60");
|
||||
assert!(response.headers().contains_key("expires"));
|
||||
for dropped in ["content-length", "content-type", "transfer-encoding"] {
|
||||
assert!(!response.headers().contains_key(dropped), "{dropped}");
|
||||
}
|
||||
assert!(
|
||||
response
|
||||
.into_body()
|
||||
.collect()
|
||||
.await
|
||||
.expect("empty website body")
|
||||
.to_bytes()
|
||||
.is_empty()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn website_conditions_are_refused_as_the_s3_decoder_refuses_them() {
|
||||
let refused = |name: &'static str, value: &'static str| {
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert(name, HeaderValue::from_static(value));
|
||||
validated_conditions(&headers).expect_err(value).code().clone()
|
||||
};
|
||||
assert_eq!(refused("range", "bytes=5-1"), S3ErrorCode::InvalidRange);
|
||||
assert_eq!(refused("range", "bytes=0-1,3-4"), S3ErrorCode::InvalidRange);
|
||||
assert_eq!(refused("range", "items=0-1"), S3ErrorCode::InvalidRange);
|
||||
assert_eq!(refused("if-modified-since", "yesterday"), S3ErrorCode::InvalidArgument);
|
||||
assert_eq!(refused("if-unmodified-since", "2024-01-01"), S3ErrorCode::InvalidArgument);
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert("range", HeaderValue::from_static("bytes=2-"));
|
||||
headers.insert("if-none-match", HeaderValue::from_static("*"));
|
||||
headers.insert("if-modified-since", HeaderValue::from_static("Tue, 14 Nov 2023 22:13:20 GMT"));
|
||||
assert!(matches!(
|
||||
validated_conditions(&headers).expect("well-formed conditions"),
|
||||
Some(Range::Int { first: 2, last: None })
|
||||
));
|
||||
let stripped = without_conditions(&headers);
|
||||
assert!(stripped.is_empty(), "{stripped:?}");
|
||||
}
|
||||
|
||||
/// A bucket served as a website, as `website_hosting_test` builds it.
|
||||
struct Site {
|
||||
website: WebsiteHost,
|
||||
bucket: String,
|
||||
}
|
||||
|
||||
const DOMAIN: &str = "website.test";
|
||||
const PUBLIC_READ: &str = "public";
|
||||
|
||||
async fn put(store: &ECStore, bucket: &str, key: &str, body: &str, redirect: Option<&str>) {
|
||||
let mut options = StorageObjectOptions::default();
|
||||
if let Some(location) = redirect {
|
||||
options
|
||||
.user_defined
|
||||
.insert(rustfs_utils::http::AMZ_WEBSITE_REDIRECT_LOCATION.to_owned(), location.to_owned());
|
||||
}
|
||||
store
|
||||
.put_object(bucket, key, &mut StoragePutObjReader::from_vec(body.as_bytes().to_vec()), &options)
|
||||
.await
|
||||
.expect("seed a website object");
|
||||
}
|
||||
|
||||
/// A fresh bucket holding the website pages, with `website` as its website configuration and,
|
||||
/// when `policy` is `PUBLIC_READ`, the public-read policy that denies `private.txt`.
|
||||
async fn site(website: Option<&str>, policy: &str) -> Site {
|
||||
let store = crate::app::gating_test_env::shared_gating_ecstore().await;
|
||||
let server_ctx = ServerContextSlot::new();
|
||||
assert!(server_ctx.install(crate::app::gating_test_env::shared_gating_ambient().await));
|
||||
let bucket = format!("website-{}", uuid::Uuid::new_v4());
|
||||
store
|
||||
.make_bucket(&bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create the website bucket");
|
||||
for (key, body) in [
|
||||
("index.html", "home"),
|
||||
("404.html", "error page"),
|
||||
("docs", "ordinary object"),
|
||||
("section/index.html", "section page"),
|
||||
("private.txt", "private"),
|
||||
] {
|
||||
put(store.as_ref(), &bucket, key, body, None).await;
|
||||
}
|
||||
put(store.as_ref(), &bucket, "move", "ordinary redirect object", Some("/new-page")).await;
|
||||
if let Some(website) = website {
|
||||
store
|
||||
.update_bucket_metadata_config(
|
||||
&bucket,
|
||||
crate::storage::storage_api::ecstore_bucket::metadata::BUCKET_WEBSITE_CONFIG,
|
||||
website.as_bytes().to_vec(),
|
||||
)
|
||||
.await
|
||||
.expect("store the website configuration");
|
||||
}
|
||||
if policy == PUBLIC_READ {
|
||||
let policy = serde_json::json!({
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [
|
||||
{"Effect":"Allow","Principal":"*","Action":"s3:GetObject","Resource":format!("arn:aws:s3:::{bucket}/*")},
|
||||
{"Effect":"Deny","Principal":"*","Action":"s3:GetObject","Resource":format!("arn:aws:s3:::{bucket}/private.txt")}
|
||||
]
|
||||
});
|
||||
store
|
||||
.update_bucket_metadata_config(
|
||||
&bucket,
|
||||
crate::storage::storage_api::ecstore_bucket::metadata::BUCKET_POLICY_CONFIG,
|
||||
serde_json::to_vec(&policy).expect("policy JSON"),
|
||||
)
|
||||
.await
|
||||
.expect("store the public-read policy");
|
||||
}
|
||||
let website = WebsiteHost::new(vec![DOMAIN.to_owned()], "http", server_ctx).expect("a website domain");
|
||||
Site { website, bucket }
|
||||
}
|
||||
|
||||
const INDEX_AND_ERROR: &str = "<WebsiteConfiguration><IndexDocument><Suffix>index.html</Suffix></IndexDocument><ErrorDocument><Key>404.html</Key></ErrorDocument></WebsiteConfiguration>";
|
||||
|
||||
/// Status, `Location`, and body of one website request.
|
||||
struct Page {
|
||||
status: StatusCode,
|
||||
location: Option<String>,
|
||||
body: String,
|
||||
}
|
||||
|
||||
impl Site {
|
||||
async fn ask(&self, method: Method, path: &str, extra: &[(&'static str, &'static str)]) -> Page {
|
||||
self.ask_host(&format!("{}.{DOMAIN}", self.bucket), method, path, extra).await
|
||||
}
|
||||
|
||||
async fn ask_host(&self, host_name: &str, method: Method, path: &str, extra: &[(&'static str, &'static str)]) -> Page {
|
||||
let mut headers = host(host_name);
|
||||
for (name, value) in extra {
|
||||
headers.insert(*name, HeaderValue::from_static(value));
|
||||
}
|
||||
let response = self.website.answer(method, uri(path), headers, Extensions::new()).await;
|
||||
let status = response.status();
|
||||
let location = response
|
||||
.headers()
|
||||
.get(http::header::LOCATION)
|
||||
.map(|value| value.to_str().expect("an ASCII location").to_owned());
|
||||
let body = response.into_body().collect().await.expect("an in-memory body").to_bytes();
|
||||
Page {
|
||||
status,
|
||||
location,
|
||||
body: String::from_utf8_lossy(&body).into_owned(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn website_host_serves_pages_documents_and_redirects_anonymously() {
|
||||
let site = site(Some(INDEX_AND_ERROR), PUBLIC_READ).await;
|
||||
|
||||
let home = site.ask(Method::GET, "/", &[]).await;
|
||||
assert_eq!((home.status, home.body.as_str()), (StatusCode::OK, "home"));
|
||||
let head = site.ask(Method::HEAD, "/", &[]).await;
|
||||
assert_eq!((head.status, head.body.as_str()), (StatusCode::OK, ""));
|
||||
let object = site.ask(Method::GET, "/docs", &[]).await;
|
||||
assert_eq!((object.status, object.body.as_str()), (StatusCode::OK, "ordinary object"));
|
||||
let section = site.ask(Method::GET, "/section/", &[]).await;
|
||||
assert_eq!((section.status, section.body.as_str()), (StatusCode::OK, "section page"));
|
||||
|
||||
let missing = site.ask(Method::GET, "/missing", &[]).await;
|
||||
assert_eq!((missing.status, missing.body.as_str()), (StatusCode::NOT_FOUND, "error page"));
|
||||
let denied = site.ask(Method::GET, "/private.txt", &[]).await;
|
||||
assert_eq!((denied.status, denied.body.as_str()), (StatusCode::FORBIDDEN, "error page"));
|
||||
let head_missing = site.ask(Method::HEAD, "/missing", &[]).await;
|
||||
assert_eq!((head_missing.status, head_missing.body.as_str()), (StatusCode::NOT_FOUND, ""));
|
||||
let directory = site.ask(Method::GET, "/section?x=1", &[]).await;
|
||||
assert_eq!(directory.status, StatusCode::FOUND);
|
||||
assert_eq!(directory.location.as_deref(), Some("/section/?x=1"));
|
||||
let redirect = site.ask(Method::GET, "/move", &[]).await;
|
||||
assert_eq!(redirect.status, StatusCode::MOVED_PERMANENTLY);
|
||||
assert_eq!(redirect.location.as_deref(), Some("/new-page"));
|
||||
for condition in [("if-none-match", "*"), ("range", "bytes=999999-")] {
|
||||
let redirected = site.ask(Method::GET, "/move", &[condition]).await;
|
||||
assert_eq!(redirected.status, StatusCode::MOVED_PERMANENTLY, "{condition:?}");
|
||||
assert_eq!(redirected.location.as_deref(), Some("/new-page"), "{condition:?}");
|
||||
}
|
||||
let unsatisfiable = site.ask(Method::GET, "/index.html", &[("range", "bytes=999999-")]).await;
|
||||
// A client error is answered by the error document, under the error's own status.
|
||||
assert_eq!(
|
||||
(unsatisfiable.status, unsatisfiable.body.as_str()),
|
||||
(StatusCode::RANGE_NOT_SATISFIABLE, "error page")
|
||||
);
|
||||
let ranged = site.ask(Method::GET, "/index.html", &[("range", "bytes=1-2")]).await;
|
||||
assert_eq!((ranged.status, ranged.body.as_str()), (StatusCode::PARTIAL_CONTENT, "om"));
|
||||
let malformed = site
|
||||
.ask(Method::GET, "/index.html", &[("if-modified-since", "yesterday")])
|
||||
.await;
|
||||
assert_eq!(malformed.status, StatusCode::BAD_REQUEST);
|
||||
assert!(!malformed.body.contains("<Error>"), "{}", malformed.body);
|
||||
let post = site.ask(Method::POST, "/", &[]).await;
|
||||
assert_eq!(post.status, StatusCode::METHOD_NOT_ALLOWED);
|
||||
let base_domain = site
|
||||
.ask_host(DOMAIN, Method::GET, &format!("/{}/index.html", site.bucket), &[])
|
||||
.await;
|
||||
assert_eq!(base_domain.status, StatusCode::NOT_FOUND);
|
||||
assert!(!base_domain.body.contains("<Error>"), "{}", base_domain.body);
|
||||
let no_bucket = site
|
||||
.ask_host(&format!("absent-{}.{DOMAIN}", site.bucket), Method::GET, "/", &[])
|
||||
.await;
|
||||
assert_eq!(no_bucket.status, StatusCode::NOT_FOUND);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn website_host_refuses_what_the_bucket_does_not_serve() {
|
||||
// No website configuration: the bucket is not a website, whatever its policy.
|
||||
let unconfigured = site(None, PUBLIC_READ).await;
|
||||
let page = unconfigured.ask(Method::GET, "/", &[]).await;
|
||||
assert_eq!(page.status, StatusCode::NOT_FOUND);
|
||||
assert_eq!(page.body, "<html><body>Not Found</body></html>");
|
||||
|
||||
// A website without a public-read policy is read as the anonymous caller it is: the index and
|
||||
// the error document are both refused, so the page is the plain refusal.
|
||||
let private = site(Some(INDEX_AND_ERROR), "none").await;
|
||||
for path in ["/", "/index.html", "/docs", "/missing"] {
|
||||
let page = private.ask(Method::GET, path, &[]).await;
|
||||
assert_eq!(page.status, StatusCode::FORBIDDEN, "{path}");
|
||||
assert_eq!(page.body, "<html><body>Forbidden</body></html>", "{path}");
|
||||
}
|
||||
// Whatever caller and ownership the request arrived with, a website read is anonymous.
|
||||
let mut owner = Extensions::new();
|
||||
owner.insert(ReqInfo {
|
||||
cred: Some(rustfs_credentials::Credentials {
|
||||
access_key: "website-owner".to_owned(),
|
||||
..Default::default()
|
||||
}),
|
||||
is_owner: true,
|
||||
..ReqInfo::default()
|
||||
});
|
||||
let host_name = format!("{}.{DOMAIN}", private.bucket);
|
||||
let response = private.website.answer(Method::GET, uri("/"), host(&host_name), owner).await;
|
||||
assert_eq!(
|
||||
response.status(),
|
||||
StatusCode::FORBIDDEN,
|
||||
"an owner's request facts must not reach the read"
|
||||
);
|
||||
// A redirect location is only reported to a read the bucket allows.
|
||||
let hidden = private.ask(Method::GET, "/move", &[]).await;
|
||||
assert_eq!((hidden.status, hidden.location), (StatusCode::FORBIDDEN, None));
|
||||
|
||||
// An index document that does not exist is answered by the error document's status.
|
||||
let no_index = site(
|
||||
Some("<WebsiteConfiguration><IndexDocument><Suffix>absent.html</Suffix></IndexDocument></WebsiteConfiguration>"),
|
||||
PUBLIC_READ,
|
||||
)
|
||||
.await;
|
||||
let page = no_index.ask(Method::GET, "/", &[]).await;
|
||||
assert_eq!(page.status, StatusCode::NOT_FOUND);
|
||||
assert_eq!(page.body, "<html><body>Not Found</body></html>");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn website_host_applies_routing_rules_and_redirect_all() {
|
||||
let rules = "<WebsiteConfiguration><IndexDocument><Suffix>index.html</Suffix></IndexDocument><RoutingRules>\
|
||||
<RoutingRule><Condition><HttpErrorCodeReturnedEquals>404</HttpErrorCodeReturnedEquals></Condition><Redirect><ReplaceKeyWith>missing-page</ReplaceKeyWith></Redirect></RoutingRule>\
|
||||
<RoutingRule><Condition><KeyPrefixEquals>docs/</KeyPrefixEquals></Condition><Redirect><ReplaceKeyWith>docs-page</ReplaceKeyWith></Redirect></RoutingRule>\
|
||||
<RoutingRule><Condition><HttpErrorCodeReturnedEquals>416</HttpErrorCodeReturnedEquals></Condition><Redirect><ReplaceKeyWith>range-error</ReplaceKeyWith></Redirect></RoutingRule>\
|
||||
</RoutingRules></WebsiteConfiguration>";
|
||||
let site = site(Some(rules), PUBLIC_READ).await;
|
||||
let host_name = format!("{}.{DOMAIN}", site.bucket);
|
||||
let rule = site.ask(Method::GET, "/docs/missing", &[]).await;
|
||||
assert_eq!(rule.status, StatusCode::MOVED_PERMANENTLY);
|
||||
assert_eq!(rule.location, Some(format!("http://{host_name}/missing-page")));
|
||||
let ranged = site.ask(Method::GET, "/index.html", &[("range", "bytes=999999-")]).await;
|
||||
assert_eq!(ranged.status, StatusCode::MOVED_PERMANENTLY);
|
||||
assert_eq!(ranged.location, Some(format!("http://{host_name}/range-error")));
|
||||
let served = site.ask(Method::GET, "/index.html", &[]).await;
|
||||
assert_eq!((served.status, served.body.as_str()), (StatusCode::OK, "home"));
|
||||
|
||||
let all = site_redirecting_all().await;
|
||||
let page = all.ask(Method::GET, "/a%20b?x=1", &[]).await;
|
||||
assert_eq!(page.status, StatusCode::MOVED_PERMANENTLY);
|
||||
assert_eq!(page.location.as_deref(), Some("http://destination.test/a%20b?x=1"));
|
||||
}
|
||||
|
||||
async fn site_redirecting_all() -> Site {
|
||||
site(
|
||||
Some("<WebsiteConfiguration><RedirectAllRequestsTo><HostName>destination.test</HostName></RedirectAllRequestsTo></WebsiteConfiguration>"),
|
||||
"none",
|
||||
)
|
||||
.await
|
||||
}
|
||||
+4
-5
@@ -180,7 +180,7 @@ enum BucketNotificationLookup {
|
||||
Configured(rustfs_gateway_types::persistence::PersistedNotificationConfiguration),
|
||||
Missing,
|
||||
/// Stored bytes cannot be parsed. Deterministic, so a retry cannot help,
|
||||
/// and registration must not treat it as a readable empty configuration.
|
||||
/// and reading it as missing would clear the bucket's rules.
|
||||
Unreadable,
|
||||
}
|
||||
|
||||
@@ -199,7 +199,7 @@ fn classify_bucket_notification_lookup(
|
||||
subsystem = LOG_SUBSYSTEM_NOTIFICATION,
|
||||
bucket = %bucket,
|
||||
error = %err,
|
||||
"Bucket notification configuration is unreadable; skipping rule registration"
|
||||
"Bucket notification configuration is unreadable; leaving its rules unchanged"
|
||||
);
|
||||
Ok(BucketNotificationLookup::Unreadable)
|
||||
}
|
||||
@@ -210,9 +210,8 @@ fn classify_bucket_notification_lookup(
|
||||
}
|
||||
|
||||
/// Apply one bucket's persisted notification rules. An unreadable config is
|
||||
/// isolated to its bucket and does not abort setup for the other buckets.
|
||||
/// Metadata-bound server dispatch independently suppresses external delivery
|
||||
/// for unreadable snapshots rather than retaining an old subscription.
|
||||
/// isolated to its bucket: it neither aborts setup for the other buckets nor
|
||||
/// clears the bucket's rules as if it had none.
|
||||
async fn apply_bucket_notification_configuration(bucket: &str, region: &str) -> Result<bool, NotificationError> {
|
||||
match classify_bucket_notification_lookup(bucket, get_bucket_notification_config(bucket).await)? {
|
||||
BucketNotificationLookup::Unreadable => Ok(false),
|
||||
|
||||
@@ -79,9 +79,6 @@ fn mark_bucket_notification_rules_reconciled() {
|
||||
}
|
||||
|
||||
fn should_reconcile_bucket_notification_rules(runtime_changed: bool, notify_enabled: bool) -> bool {
|
||||
// Dispatch follows store-owned metadata snapshots, including the periodic
|
||||
// metadata refresh that repairs lost peer signals. Startup validation need
|
||||
// not rescan all buckets on every five-second target-runtime tick.
|
||||
notify_enabled && (runtime_changed || !are_bucket_notification_rules_reconciled())
|
||||
}
|
||||
|
||||
@@ -189,7 +186,6 @@ async fn reconcile_event_notifier_from_store_unlocked(
|
||||
let result = async {
|
||||
validate_notify_module_env().map_err(NotificationError::Initialization)?;
|
||||
let system = ensure_live_events_initialized();
|
||||
system.bind_bucket_metadata_source(&store);
|
||||
let transition_system = system.clone();
|
||||
let transition_store = store.clone();
|
||||
let transition = with_refreshed_notify_module_state_from(store.clone(), move |resolution| async move {
|
||||
@@ -357,8 +353,6 @@ where
|
||||
return Ok(());
|
||||
};
|
||||
|
||||
system.bind_bucket_metadata_source(&store);
|
||||
|
||||
refresh_persisted_module_switches_from(store.clone())
|
||||
.await
|
||||
.map_err(|err| NotificationError::Initialization(format!("failed to refresh notify module switch: {err}")))?;
|
||||
|
||||
@@ -1,129 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! RustFS's two metadata listings on the gateway stack: `GET /{bucket}?list-type=2&metadata=true`
|
||||
//! and `GET /{bucket}?versions&metadata=true` (rustfs/gateway#1588).
|
||||
//!
|
||||
//! Responsible for: [`dialect`], the two rows of the gateway's RustFS dialect that select them
|
||||
//! ahead of the standard listings, and [`register`], their handler: the shared access phase, the
|
||||
//! listing authorization and the listing answer both stacks share (`app/metadata_listing.rs`).
|
||||
//! NOT responsible for: authenticating the caller (the gateway), the listing rules (the shared
|
||||
//! module), or the assembly (`server/gateway_stack.rs`).
|
||||
//! Upstream: the gateway's dispatch. Downstream: `app/metadata_listing.rs`.
|
||||
|
||||
use super::{access, handler_error};
|
||||
use crate::app::metadata_listing::{self, MetadataListing};
|
||||
use crate::runtime_sources::ServerContextSlot;
|
||||
use rustfs_gateway::{Handler, HandlerResult, Req, Resp, ServiceBuilder};
|
||||
use rustfs_gateway_core::dialect::{Dialect, DialectError, DialectOverlay, DialectRoute, OverlayRow};
|
||||
use rustfs_gateway_core::route::ShadowingDecl;
|
||||
use rustfs_gateway_dialect_rustfs_admin::ops::list_object_versions_with_metadata::{
|
||||
self as versions, ListObjectVersionsWithMetadata,
|
||||
};
|
||||
use rustfs_gateway_dialect_rustfs_admin::ops::list_objects_v2_with_metadata::{self as objects, ListObjectsV2WithMetadata};
|
||||
use rustfs_gateway_dialect_rustfs_admin::{AdminResponse, ExtensionOperation};
|
||||
use rustfs_s3_types::{S3Result, s3_error};
|
||||
use std::sync::{Arc, OnceLock};
|
||||
|
||||
/// The two operations, as the gateway names them.
|
||||
pub(crate) const NAMES: [&str; 2] = [versions::NAME, objects::NAME];
|
||||
|
||||
static ROWS: [OverlayRow; 2] = [versions::OVERLAY_ROW, objects::OVERLAY_ROW];
|
||||
|
||||
static RECORD: DialectOverlay = DialectOverlay {
|
||||
name: "rustfs-metadata-listings",
|
||||
vendor: "rustfs",
|
||||
operations: &ROWS,
|
||||
claims: &[],
|
||||
};
|
||||
|
||||
/// The dialect carrying the two listings' rows, as the gateway's RustFS dialect declares them.
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// What the gateway refuses in the record or the declarations.
|
||||
pub(crate) fn dialect() -> Result<Dialect, Vec<DialectError>> {
|
||||
static VERSIONS: OnceLock<Vec<ShadowingDecl>> = OnceLock::new();
|
||||
static OBJECTS: OnceLock<Vec<ShadowingDecl>> = OnceLock::new();
|
||||
Dialect::assemble(&RECORD)
|
||||
.declare::<ListObjectVersionsWithMetadata>(on_the_s3_table(ListObjectVersionsWithMetadata::ROUTE, &VERSIONS))
|
||||
.declare::<ListObjectsV2WithMetadata>(on_the_s3_table(ListObjectsV2WithMetadata::ROUTE, &OBJECTS))
|
||||
.build()
|
||||
}
|
||||
|
||||
/// `route` as the S3 table carries it: the same row, declaring the overlaps with the standard
|
||||
/// operations and the other listing it stands in front of. The admin router's extension rows it
|
||||
/// also shadows in the
|
||||
/// gateway's one-table dialect are served by the control-plane host, not by this table, and a
|
||||
/// declaration naming an operation the table does not hold is refused as stale.
|
||||
fn on_the_s3_table(route: DialectRoute, shadows: &'static OnceLock<Vec<ShadowingDecl>>) -> DialectRoute {
|
||||
let shadows = shadows.get_or_init(|| {
|
||||
route
|
||||
.shadows
|
||||
.iter()
|
||||
.filter(|decl| !decl.shadowed.starts_with("rustfs:") || NAMES.contains(&decl.shadowed))
|
||||
.copied()
|
||||
.collect()
|
||||
});
|
||||
DialectRoute { shadows, ..route }
|
||||
}
|
||||
|
||||
/// Registers the handler of both listings on `builder`.
|
||||
pub(crate) fn register(builder: ServiceBuilder, server_ctx: &Arc<ServerContextSlot>) -> ServiceBuilder {
|
||||
let listings = Arc::new(Listings {
|
||||
server_ctx: Arc::clone(server_ctx),
|
||||
});
|
||||
builder
|
||||
.register::<ListObjectVersionsWithMetadata, _>(Arc::clone(&listings))
|
||||
.register::<ListObjectsV2WithMetadata, _>(listings)
|
||||
}
|
||||
|
||||
/// One metadata listing operation and the listing it is.
|
||||
trait Listing: ExtensionOperation<Input = ()> {
|
||||
const LISTING: MetadataListing;
|
||||
}
|
||||
|
||||
impl Listing for ListObjectVersionsWithMetadata {
|
||||
const LISTING: MetadataListing = MetadataListing::ListObjectVersions;
|
||||
}
|
||||
|
||||
impl Listing for ListObjectsV2WithMetadata {
|
||||
const LISTING: MetadataListing = MetadataListing::ListObjectsV2;
|
||||
}
|
||||
|
||||
struct Listings {
|
||||
server_ctx: Arc<ServerContextSlot>,
|
||||
}
|
||||
|
||||
impl Listings {
|
||||
/// The access phase every gateway operation runs, then the listing check and the listing.
|
||||
async fn serve<O: Listing>(&self, request: Req<O>) -> S3Result<Resp<O>> {
|
||||
let mut envelope = access::admit(&self.server_ctx, request.context()).await?;
|
||||
let bucket = request
|
||||
.context()
|
||||
.bucket()
|
||||
.ok_or_else(|| s3_error!(InternalError, "the listing names no bucket"))?
|
||||
.as_str()
|
||||
.to_owned();
|
||||
metadata_listing::authorize(&mut envelope, O::LISTING, &bucket).await?;
|
||||
let body = metadata_listing::answer(&envelope, O::LISTING, bucket).await?;
|
||||
Ok(Resp::new(AdminResponse::bytes("application/xml", body)))
|
||||
}
|
||||
}
|
||||
|
||||
impl<O: Listing> Handler<O> for Listings {
|
||||
async fn call(&self, request: Req<O>) -> HandlerResult<O> {
|
||||
self.serve(request).await.map_err(handler_error)
|
||||
}
|
||||
}
|
||||
@@ -36,7 +36,6 @@ mod bucket_core;
|
||||
mod bucket_replication_lifecycle;
|
||||
mod lifecycle_input;
|
||||
mod listing_multipart;
|
||||
pub(crate) mod metadata_listing;
|
||||
mod object_commit;
|
||||
mod object_meta;
|
||||
mod object_misc;
|
||||
|
||||
@@ -60,7 +60,7 @@ pub(crate) static CONFIG_HANDLER_CALLS: std::sync::atomic::AtomicUsize = std::sy
|
||||
|
||||
use crate::app::dto_values::owner;
|
||||
use crate::runtime_sources::{ServerContextSlot, current_action_credentials};
|
||||
use crate::server::gateway_handlers::{GatewayBackend, OPERATIONS, OperationTable, metadata_listing, register, serves};
|
||||
use crate::server::gateway_handlers::{GatewayBackend, OPERATIONS, OperationTable, register, serves};
|
||||
use crate::server::ssec_transport::{is_secure_transport, require_tls};
|
||||
use crate::shared_types::{NativeTlsTransport, RemoteAddr};
|
||||
use crate::storage_api::server::http::request_context::RequestContext;
|
||||
@@ -150,8 +150,6 @@ pub(crate) enum GatewayAssemblyError {
|
||||
AdminOverlayMissing,
|
||||
#[error("the native admin dialect refused its assembly: {0:?}")]
|
||||
AdminDialect(Vec<rustfs_gateway_core::dialect::DialectError>),
|
||||
#[error("the metadata listing dialect refused its assembly: {0:?}")]
|
||||
MetadataListingDialect(Vec<rustfs_gateway_core::dialect::DialectError>),
|
||||
#[error("RUSTFS_KEYSTONE_ENABLE configuration is invalid: {0}")]
|
||||
KeystoneConfig(rustfs_keystone::KeystoneError),
|
||||
#[error("RUSTFS_KEYSTONE_ENABLE requires an initialized Keystone provider on RUSTFS_S3_STACK=gateway")]
|
||||
@@ -211,11 +209,11 @@ impl GatewayStack {
|
||||
root: Arc<dyn RootSource>,
|
||||
authorizer: impl Authorizer,
|
||||
) -> Result<Self, GatewayAssemblyError> {
|
||||
let listings = metadata_listing::dialect().map_err(GatewayAssemblyError::MetadataListingDialect)?;
|
||||
let builder = Self::configured_builder(&assembly, iam, root, authorizer, std::convert::identity)?.dialect(&listings);
|
||||
let builder = Self::configured_builder(&assembly, iam, root, authorizer, std::convert::identity)?;
|
||||
let backend = Arc::new(GatewayBackend::new(Arc::clone(&assembly.server_ctx)));
|
||||
let builder = metadata_listing::register(register(OPERATIONS, builder, &backend), &assembly.server_ctx);
|
||||
let service = builder.build().map_err(GatewayAssemblyError::Service)?;
|
||||
let service = register(OPERATIONS, builder, &backend)
|
||||
.build()
|
||||
.map_err(GatewayAssemblyError::Service)?;
|
||||
Ok(Self {
|
||||
service,
|
||||
keystone_auth: assembly.keystone_auth,
|
||||
@@ -325,11 +323,11 @@ impl GatewayStack {
|
||||
}
|
||||
}
|
||||
|
||||
/// The early gate over `table` and the two metadata listings: an operation it serves is allowed on
|
||||
/// to its handler, which decides; every other operation is denied.
|
||||
/// The early gate over `table`: an operation it serves is allowed on to its handler, which decides;
|
||||
/// every other operation is denied.
|
||||
fn early_gate(table: OperationTable) -> impl Authorizer {
|
||||
decide_with(move |request| {
|
||||
if serves(table, request.operation) || metadata_listing::NAMES.contains(&request.operation) {
|
||||
if serves(table, request.operation) {
|
||||
Decision::Allow
|
||||
} else {
|
||||
Decision::Deny
|
||||
|
||||
@@ -149,8 +149,7 @@ fn the_assembled_stack_registers_exactly_the_table() {
|
||||
let stack = GatewayStack::assemble(assembly()).expect("the stack assembles");
|
||||
|
||||
let registered: std::collections::BTreeSet<&str> = stack.service.operations().collect();
|
||||
let table = rows(OPERATIONS).map(|row| row.name);
|
||||
assert_eq!(registered, table.chain(metadata_listing::NAMES).collect());
|
||||
assert_eq!(registered, rows(OPERATIONS).map(|row| row.name).collect());
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
+14
-64
@@ -35,6 +35,7 @@ use crate::server::{
|
||||
IcebergRestErrorCompatLayer, ObjectAttributesEtagFixLayer, S3ErrorMessageCompatLayer, SigV4HeaderGuardLayer,
|
||||
StsQueryApiCompatLayer, VirtualHostStyleHintLayer,
|
||||
},
|
||||
post_object_location::PostObjectLocationFixLayer,
|
||||
rate_limit::{RateLimitLayer, api_rate_limit_layer_from_env, rate_limit_vh_domain_hosts},
|
||||
ssec_transport::SsecTransportLayer,
|
||||
strip_valid_port_suffix,
|
||||
@@ -1538,7 +1539,7 @@ type S3EdgeService = S3Service;
|
||||
/// The S3 service mounted under the middleware: the legacy stack or the gateway stack, selected at
|
||||
/// startup (`server/stack.rs`).
|
||||
#[cfg(feature = "gateway-stack")]
|
||||
type S3EdgeService = S3StackService<S3Service, GatewayEdgeService, NativeConsoleEdgeService, NativeWebsiteEdgeService>;
|
||||
type S3EdgeService = S3StackService<S3Service, GatewayEdgeService, NativeConsoleEdgeService>;
|
||||
|
||||
/// The gateway stack where the legacy service is mounted: the same response and error types, so
|
||||
/// the layers around it do not change with the stack.
|
||||
@@ -1604,56 +1605,12 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
/// The website host on the gateway stack: the owned host the legacy metadata route also serves.
|
||||
#[cfg(feature = "gateway-stack")]
|
||||
#[derive(Clone)]
|
||||
struct NativeWebsiteEdgeService(Arc<crate::app::website::WebsiteHost>);
|
||||
|
||||
#[cfg(feature = "gateway-stack")]
|
||||
impl NativeWebsiteEdgeService {
|
||||
/// The website service and the requests it claims, when website domains are configured
|
||||
/// (`website_host_domains`, which the console turns off on both stacks).
|
||||
fn claiming(website: Option<crate::app::website::WebsiteHost>) -> Option<(crate::server::stack::HostClaim, Self)> {
|
||||
let host = Arc::new(website?);
|
||||
let claims = Arc::clone(&host);
|
||||
let claim: crate::server::stack::HostClaim =
|
||||
Arc::new(move |uri: &http::Uri, headers: &http::HeaderMap| claims.claims(uri, headers));
|
||||
Some((claim, Self(host)))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "gateway-stack")]
|
||||
impl<B> Service<HttpRequest<B>> for NativeWebsiteEdgeService
|
||||
where
|
||||
B: http_body::Body<Data = Bytes> + Send + 'static,
|
||||
B::Error: std::error::Error + Send + Sync + 'static,
|
||||
{
|
||||
type Response = Response<s3s::Body>;
|
||||
type Error = s3s::HttpError;
|
||||
type Future = Pin<Box<dyn Future<Output = std::result::Result<Self::Response, Self::Error>> + Send>>;
|
||||
|
||||
fn poll_ready(&mut self, _cx: &mut Context<'_>) -> Poll<std::result::Result<(), Self::Error>> {
|
||||
Poll::Ready(Ok(()))
|
||||
}
|
||||
|
||||
fn call(&mut self, request: HttpRequest<B>) -> Self::Future {
|
||||
let host = Arc::clone(&self.0);
|
||||
// A website read has no body; it is dropped unread, as the legacy route leaves it.
|
||||
let (parts, _body) = request.into_parts();
|
||||
Box::pin(async move {
|
||||
let answer = host.answer(parts.method, parts.uri, parts.headers, parts.extensions).await;
|
||||
Ok(answer.map(s3s::Body::http_body_unsync))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// Selects the S3 stack of this server, once. A gateway stack that cannot be assembled fails
|
||||
/// startup; the legacy stack is never started in its place.
|
||||
#[cfg(feature = "gateway-stack")]
|
||||
fn select_s3_stack(
|
||||
config: &config::Config,
|
||||
legacy: S3Service,
|
||||
website: Option<crate::app::website::WebsiteHost>,
|
||||
assembly: impl FnOnce() -> Result<GatewayAssembly>,
|
||||
) -> Result<S3EdgeService> {
|
||||
let mut native_admin = None;
|
||||
@@ -1669,7 +1626,6 @@ fn select_s3_stack(
|
||||
Ok(GatewayEdgeService(stack))
|
||||
})?
|
||||
.with_native_console(NativeConsoleEdgeService::for_config(config))
|
||||
.with_native_website(NativeWebsiteEdgeService::claiming(website))
|
||||
.with_native_admin(native_admin)
|
||||
.with_control_plane(control_plane);
|
||||
info!(
|
||||
@@ -2084,7 +2040,7 @@ macro_rules! external_service_stack {
|
||||
let compression_config = $compression_config;
|
||||
let server_ctx = $server_ctx;
|
||||
let server_domains_configured = $server_domains_configured;
|
||||
let vh_domain_hosts = $vh_domain_hosts;
|
||||
let vh_domain_hosts: std::sync::Arc<[String]> = $vh_domain_hosts;
|
||||
ServiceBuilder::new()
|
||||
// NOTE: Both extension types are intentionally inserted to maintain compatibility:
|
||||
// 1. `Option<RemoteAddr>` - Used by existing admin/storage handlers throughout the codebase
|
||||
@@ -2204,6 +2160,7 @@ macro_rules! external_service_stack {
|
||||
.layer(S3ErrorMessageCompatLayer)
|
||||
.layer(IcebergRestErrorCompatLayer)
|
||||
.layer(ObjectAttributesEtagFixLayer)
|
||||
.layer(PostObjectLocationFixLayer::new(vh_domain_hosts.clone(), connection_is_tls))
|
||||
.layer(ConditionalCorsLayer::new().with_vh_domain_hosts(vh_domain_hosts))
|
||||
.option_layer(if is_console { Some(RedirectLayer) } else { None })
|
||||
.layer(BodylessStatusFixLayer)
|
||||
@@ -2544,8 +2501,6 @@ async fn start_http_server_inner(
|
||||
// Same domain set for resolving the bucket of virtual-hosted-style CORS requests.
|
||||
let vh_domain_hosts = rate_limit_vh_domain_hosts(rate_limit_vh_domains.clone());
|
||||
|
||||
let website_domains = website_host_domains(config)?.unwrap_or_default();
|
||||
|
||||
// Setup S3 service
|
||||
// This project uses the S3S library to implement S3 services
|
||||
let s3_service = {
|
||||
@@ -2562,11 +2517,8 @@ async fn start_http_server_inner(
|
||||
.map(MultiDomain::new)
|
||||
.transpose()
|
||||
.map_err(Error::other)?;
|
||||
// The gateway stack's website host is built from the same domains below.
|
||||
#[cfg(feature = "gateway-stack")]
|
||||
let website_route_domains = website_domains.clone();
|
||||
#[cfg(not(feature = "gateway-stack"))]
|
||||
let website_route_domains = website_domains;
|
||||
let website_host_sets = website_host_domains(config)?;
|
||||
let website_route_domains = website_host_sets.unwrap_or_default();
|
||||
|
||||
b.set_auth(IAMAuth::with_server_context(access_key, secret_key, server_ctx.clone()));
|
||||
b.set_access(store);
|
||||
@@ -2714,9 +2666,7 @@ async fn start_http_server_inner(
|
||||
}
|
||||
}
|
||||
#[cfg(feature = "gateway-stack")]
|
||||
let website = crate::app::website::WebsiteHost::new(website_domains, protocol, Arc::clone(&server_ctx));
|
||||
#[cfg(feature = "gateway-stack")]
|
||||
let s3_service = select_s3_stack(config, s3_service, website, || {
|
||||
let s3_service = select_s3_stack(config, s3_service, || {
|
||||
Ok(GatewayAssembly {
|
||||
server_ctx: Arc::clone(&server_ctx),
|
||||
root_access_key: config.access_key.clone(),
|
||||
@@ -3415,6 +3365,7 @@ fn process_connection(
|
||||
// 15. S3ErrorMessageCompatLayer — missing S3 error message compatibility
|
||||
// 16. IcebergRestErrorCompatLayer — Iceberg REST JSON error compatibility
|
||||
// 17. ObjectAttributesEtagFixLayer — ETag fix for GetObjectAttributes
|
||||
// PostObjectLocationFixLayer — absolute, encoded POST Object Location
|
||||
// 18. ConditionalCorsLayer — S3 API CORS
|
||||
// 19. RedirectLayer — console redirect (conditional)
|
||||
// 20. BodylessStatusFixLayer — clears body for 1xx/204/205/304 responses
|
||||
@@ -4563,8 +4514,6 @@ mod tests {
|
||||
|
||||
#[path = "control_plane_tests.rs"]
|
||||
mod control_plane_tests;
|
||||
#[path = "metadata_route_parity_tests.rs"]
|
||||
mod metadata_route_parity_tests;
|
||||
|
||||
struct UnusedS3;
|
||||
#[async_trait::async_trait]
|
||||
@@ -4864,7 +4813,7 @@ mod tests {
|
||||
let mut config = config::Config::new("127.0.0.1:9001", Vec::new());
|
||||
config.s3_stack = config::S3Stack::Gateway;
|
||||
config.console_enable = false;
|
||||
let edge = select_s3_stack(&config, legacy.clone(), None, || Ok(assembly(slot.clone()))).unwrap();
|
||||
let edge = select_s3_stack(&config, legacy.clone(), || Ok(assembly(slot.clone()))).unwrap();
|
||||
let native_service = GatewayEdgeService(GatewayStack::assemble_admin(assembly(slot.clone())).unwrap());
|
||||
let context = slot.installed_app_context().unwrap();
|
||||
let expiration = chrono::Utc::now().timestamp() + 900;
|
||||
@@ -5142,7 +5091,7 @@ mod tests {
|
||||
let mut config = config::Config::new("127.0.0.1:9001", Vec::new());
|
||||
config.s3_stack = config::S3Stack::Gateway;
|
||||
config.console_enable = false;
|
||||
let edge = select_s3_stack(&config, legacy.clone(), None, || Ok(assembly(slot))).unwrap();
|
||||
let edge = select_s3_stack(&config, legacy.clone(), || Ok(assembly(slot))).unwrap();
|
||||
let mut failures = Vec::new();
|
||||
let mut check = |condition: bool, label: String| {
|
||||
if !condition {
|
||||
@@ -5612,7 +5561,7 @@ mod tests {
|
||||
let mut config = config::Config::new("127.0.0.1:9001", Vec::new());
|
||||
config.s3_stack = config::S3Stack::Gateway;
|
||||
config.console_enable = false;
|
||||
let edge = select_s3_stack(&config, legacy.clone(), None, || Ok(assembly(slot))).unwrap();
|
||||
let edge = select_s3_stack(&config, legacy.clone(), || Ok(assembly(slot))).unwrap();
|
||||
let mut failures = Vec::new();
|
||||
let mut check = |condition: bool, label: String| {
|
||||
if !condition {
|
||||
@@ -5969,7 +5918,7 @@ mod tests {
|
||||
let mut config = config::Config::new("127.0.0.1:9001", Vec::new());
|
||||
config.s3_stack = config::S3Stack::Gateway;
|
||||
config.console_enable = false;
|
||||
let edge = select_s3_stack(&config, legacy.clone(), None, || Ok(assembly(slot))).unwrap();
|
||||
let edge = select_s3_stack(&config, legacy.clone(), || Ok(assembly(slot))).unwrap();
|
||||
let selected = answer(
|
||||
edge.clone(),
|
||||
signed(
|
||||
@@ -10180,7 +10129,7 @@ mod tests {
|
||||
/// the s3s patch layers that `server::legacy_compat` groups; T2.12
|
||||
/// (rustfs/backlog#2771) reuses this test to prove which layers the gateway
|
||||
/// stack keeps.
|
||||
const EXTERNAL_STACK_CENSUS: [&str; 28] = [
|
||||
const EXTERNAL_STACK_CENSUS: [&str; 29] = [
|
||||
"AddExtension", // Option<RemoteAddr>
|
||||
"AddExtension", // SocketAddr (optional)
|
||||
"TrustedProxyMiddleware", // TrustedProxyLayer (optional)
|
||||
@@ -10201,6 +10150,7 @@ mod tests {
|
||||
"S3ErrorMessageCompatService", // legacy_compat
|
||||
"IcebergRestErrorCompatService", // legacy_compat
|
||||
"ObjectAttributesEtagFixService", // legacy_compat
|
||||
"PostObjectLocationFixService", // POST Object Location encoding
|
||||
"ConditionalCorsService", //
|
||||
"RedirectService", // optional
|
||||
"BodylessStatusFixService", // legacy_compat
|
||||
|
||||
@@ -1,480 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! The two request classes the legacy metadata route claims — website hosts and `metadata=true`
|
||||
//! listings — sent to the legacy stack and to the gateway stack over the same store, IAM and
|
||||
//! signatures: each answers the same status, body and representation headers on both.
|
||||
|
||||
use super::*;
|
||||
use crate::app::storage_api::bucket_usecase::contract::bucket::{BucketOperations as _, MakeBucketOptions};
|
||||
use crate::app::storage_api::object_usecase::contract::object::ObjectIO as _;
|
||||
use crate::app::storage_api::object_usecase::{StorageObjectOptions, StoragePutObjReader};
|
||||
|
||||
const WEBSITE_DOMAIN: &str = "website.test";
|
||||
const S3_HOST: &str = "localhost:9000";
|
||||
const BUCKET: &str = "parity-site";
|
||||
const EMPTY: &str = "parity-empty";
|
||||
|
||||
/// Both stacks, each over the same store: the legacy one with the metadata route mounted as
|
||||
/// `server/http.rs` mounts it.
|
||||
async fn stacks() -> (rustfs_test_utils::TestECStoreEnv, crate::server::http::S3Service, S3EdgeService) {
|
||||
let env = rustfs_test_utils::TestECStoreEnv::builder()
|
||||
.prefix("metadata_route_parity")
|
||||
.disk_count(1)
|
||||
.init_bucket_metadata(true)
|
||||
.build()
|
||||
.await;
|
||||
ObjectStore::new(env.ecstore.clone())
|
||||
.save_iam_config(serde_json::json!({"version":1}), format!("{}/format.json", *IAM_CONFIG_PREFIX))
|
||||
.await
|
||||
.unwrap();
|
||||
let iam = rustfs_iam::build_iam_sys(env.ecstore.clone()).await.unwrap();
|
||||
for (key, actions) in [
|
||||
("list-allowed", Some(vec!["s3:ListBucket", "s3:ListBucketVersions"])),
|
||||
("list-none", None),
|
||||
] {
|
||||
iam.create_user(
|
||||
key,
|
||||
&AddOrUpdateUserReq {
|
||||
secret_key: USER_SECRET.to_owned(),
|
||||
policy: None,
|
||||
status: AccountStatus::Enabled,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
if let Some(actions) = actions {
|
||||
let document =
|
||||
serde_json::json!({"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":actions,"Resource":"*"}]});
|
||||
iam.set_policy(key, Policy::parse_config(&serde_json::to_vec(&document).unwrap()).unwrap())
|
||||
.await
|
||||
.unwrap();
|
||||
iam.policy_db_set(key, UserType::Reg, false, key).await.unwrap();
|
||||
}
|
||||
}
|
||||
let context = Arc::new(AppContext::with_default_interfaces(
|
||||
env.ecstore.clone(),
|
||||
iam,
|
||||
Arc::new(rustfs_kms::KmsServiceManager::new()),
|
||||
));
|
||||
assert!(context.publish_action_credentials(rustfs_credentials::Credentials {
|
||||
access_key: ROOT.to_owned(),
|
||||
secret_key: ROOT_SECRET.to_owned(),
|
||||
status: "on".to_owned(),
|
||||
..Default::default()
|
||||
}));
|
||||
publish_test_app_context(context.clone());
|
||||
let slot = ServerContextSlot::new();
|
||||
assert!(slot.install(context));
|
||||
|
||||
let store = env.ecstore.clone();
|
||||
for bucket in [BUCKET, EMPTY] {
|
||||
store.make_bucket(bucket, &MakeBucketOptions::default()).await.unwrap();
|
||||
}
|
||||
for (key, body, redirect) in [
|
||||
("index.html", "home", None),
|
||||
("404.html", "error page", None),
|
||||
("docs", "ordinary object", None),
|
||||
("section/index.html", "section page", None),
|
||||
("private.txt", "private", None),
|
||||
("move", "ordinary redirect object", Some("/new-page")),
|
||||
("a b/c", "encoded key", None),
|
||||
] {
|
||||
let mut options = StorageObjectOptions::default();
|
||||
options
|
||||
.user_defined
|
||||
.insert("x-amz-meta-project".to_owned(), "alpha".to_owned());
|
||||
if let Some(location) = redirect {
|
||||
options
|
||||
.user_defined
|
||||
.insert(rustfs_utils::http::AMZ_WEBSITE_REDIRECT_LOCATION.to_owned(), location.to_owned());
|
||||
}
|
||||
store
|
||||
.put_object(BUCKET, key, &mut StoragePutObjReader::from_vec(body.as_bytes().to_vec()), &options)
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
let website = "<WebsiteConfiguration><IndexDocument><Suffix>index.html</Suffix></IndexDocument><ErrorDocument><Key>404.html</Key></ErrorDocument></WebsiteConfiguration>";
|
||||
store
|
||||
.update_bucket_metadata_config(
|
||||
BUCKET,
|
||||
crate::app::storage_api::bucket::metadata::BUCKET_WEBSITE_CONFIG,
|
||||
website.as_bytes().to_vec(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let policy = serde_json::json!({"Version": "2012-10-17", "Statement": [
|
||||
{"Effect":"Allow","Principal":"*","Action":"s3:GetObject","Resource":format!("arn:aws:s3:::{BUCKET}/*")},
|
||||
{"Effect":"Deny","Principal":"*","Action":"s3:GetObject","Resource":format!("arn:aws:s3:::{BUCKET}/private.txt")}
|
||||
]});
|
||||
store
|
||||
.update_bucket_metadata_config(
|
||||
BUCKET,
|
||||
crate::app::storage_api::bucket::metadata::BUCKET_POLICY_CONFIG,
|
||||
serde_json::to_vec(&policy).unwrap(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let website_domains = vec![WEBSITE_DOMAIN.to_owned()];
|
||||
let mut legacy = crate::server::http::S3ServiceBuilder::new(UnusedS3);
|
||||
legacy.set_auth(crate::auth::IAMAuth::with_server_context(ROOT, ROOT_SECRET, slot.clone()));
|
||||
legacy.set_route(crate::app::metadata_route::with_metadata_route(
|
||||
crate::admin::make_admin_route(false, slot.clone()).unwrap(),
|
||||
Some(crate::server::http::MultiDomain::new(["s3.example.test"]).unwrap()),
|
||||
website_domains.clone(),
|
||||
"http",
|
||||
slot.clone(),
|
||||
));
|
||||
legacy.set_host(crate::server::http::MultiDomain::new(["s3.example.test"]).unwrap());
|
||||
let legacy = legacy.build();
|
||||
let mut config = config::Config::new("127.0.0.1:9001", Vec::new());
|
||||
config.s3_stack = config::S3Stack::Gateway;
|
||||
config.console_enable = false;
|
||||
let website = crate::app::website::WebsiteHost::new(website_domains, "http", slot.clone());
|
||||
let edge = select_s3_stack(&config, legacy.clone(), website, || Ok(assembly(slot.clone()))).unwrap();
|
||||
(env, legacy, edge)
|
||||
}
|
||||
|
||||
/// What both stacks must agree on: the status, the body (an S3 error by its code), and the
|
||||
/// representation and redirect headers.
|
||||
///
|
||||
/// An S3 error is compared by its status and code: its framing is the gateway's for every
|
||||
/// operation, which states the error document's length where the legacy stack streams it.
|
||||
fn observed(response: (StatusCode, HeaderMap, Bytes)) -> String {
|
||||
let (status, mut headers, body) = response;
|
||||
let body = if body.starts_with(b"<?xml") && body.windows(7).any(|window| window == b"<Error>") {
|
||||
headers.remove(http::header::CONTENT_LENGTH);
|
||||
format!("error {:?}", refusal(&body).0)
|
||||
} else {
|
||||
String::from_utf8_lossy(&body).into_owned()
|
||||
};
|
||||
let named = [
|
||||
"content-type",
|
||||
"content-length",
|
||||
"content-range",
|
||||
"etag",
|
||||
"last-modified",
|
||||
"location",
|
||||
]
|
||||
.map(|name| format!("{name}={:?}", headers.get(name)));
|
||||
format!("{status} {} {body}", named.join(" "))
|
||||
}
|
||||
|
||||
fn website_request(
|
||||
method: Method,
|
||||
path: &str,
|
||||
host: &str,
|
||||
extra: Option<(&'static str, &'static str)>,
|
||||
) -> HttpRequest<Empty<Bytes>> {
|
||||
let mut builder = HttpRequest::builder()
|
||||
.method(method)
|
||||
.uri(path)
|
||||
.header(http::header::HOST, host)
|
||||
.header("x-request-id", REQUEST_ID);
|
||||
if let Some((name, value)) = extra {
|
||||
builder = builder.header(name, value);
|
||||
}
|
||||
builder.body(Empty::new()).unwrap()
|
||||
}
|
||||
|
||||
/// One website request: method, path, host, an extra header, and the status both stacks answer.
|
||||
type WebsiteCase<'a> = (Method, &'a str, String, Option<(&'static str, &'static str)>, StatusCode);
|
||||
|
||||
/// One listing request: its label, how to build it, and the status both stacks answer.
|
||||
type ListingCase = (&'static str, Box<dyn Fn() -> HttpRequest<Empty<Bytes>>>, StatusCode);
|
||||
|
||||
/// Sends `request()` to both stacks and returns each answer as [`observed`] reads it.
|
||||
async fn both(
|
||||
legacy: &crate::server::http::S3Service,
|
||||
edge: &S3EdgeService,
|
||||
request: impl Fn() -> HttpRequest<Empty<Bytes>>,
|
||||
) -> (String, String) {
|
||||
(
|
||||
observed(answer(legacy.clone(), request()).await),
|
||||
observed(answer(edge.clone(), request()).await),
|
||||
)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn website_and_metadata_route_answers_match_the_legacy_stack() {
|
||||
let (_env, legacy, edge) = stacks().await;
|
||||
let mut failures = Vec::new();
|
||||
let site = format!("{BUCKET}.{WEBSITE_DOMAIN}");
|
||||
let path_style = format!("/{BUCKET}/index.html");
|
||||
let website_cases: Vec<WebsiteCase<'_>> = vec![
|
||||
(Method::GET, "/", site.clone(), None, StatusCode::OK),
|
||||
(Method::HEAD, "/", site.clone(), None, StatusCode::OK),
|
||||
(Method::GET, "/docs", site.clone(), None, StatusCode::OK),
|
||||
(
|
||||
Method::GET,
|
||||
"/index.html",
|
||||
site.clone(),
|
||||
Some(("range", "bytes=1-2")),
|
||||
StatusCode::PARTIAL_CONTENT,
|
||||
),
|
||||
(Method::GET, "/missing", site.clone(), None, StatusCode::NOT_FOUND),
|
||||
(Method::HEAD, "/missing", site.clone(), None, StatusCode::NOT_FOUND),
|
||||
(Method::GET, "/private.txt", site.clone(), None, StatusCode::FORBIDDEN),
|
||||
(Method::GET, "/section?x=1", site.clone(), None, StatusCode::FOUND),
|
||||
(Method::GET, "/move", site.clone(), None, StatusCode::MOVED_PERMANENTLY),
|
||||
(
|
||||
Method::GET,
|
||||
"/move",
|
||||
site.clone(),
|
||||
Some(("if-none-match", "*")),
|
||||
StatusCode::MOVED_PERMANENTLY,
|
||||
),
|
||||
(
|
||||
Method::GET,
|
||||
"/index.html",
|
||||
site.clone(),
|
||||
Some(("range", "bytes=999999-")),
|
||||
StatusCode::RANGE_NOT_SATISFIABLE,
|
||||
),
|
||||
(
|
||||
Method::GET,
|
||||
"/index.html",
|
||||
site.clone(),
|
||||
Some(("if-modified-since", "yesterday")),
|
||||
StatusCode::BAD_REQUEST,
|
||||
),
|
||||
(
|
||||
Method::GET,
|
||||
"/index.html",
|
||||
site.clone(),
|
||||
Some(("if-none-match", "*")),
|
||||
StatusCode::NOT_MODIFIED,
|
||||
),
|
||||
(Method::POST, "/", site.clone(), None, StatusCode::METHOD_NOT_ALLOWED),
|
||||
(Method::GET, "/rustfs/admin/v3/info", site.clone(), None, StatusCode::NOT_FOUND),
|
||||
(Method::GET, "/", format!("{EMPTY}.{WEBSITE_DOMAIN}"), None, StatusCode::NOT_FOUND),
|
||||
(Method::GET, "/", format!("absent.{WEBSITE_DOMAIN}"), None, StatusCode::NOT_FOUND),
|
||||
(Method::GET, path_style.as_str(), WEBSITE_DOMAIN.to_owned(), None, StatusCode::NOT_FOUND),
|
||||
];
|
||||
for (method, path, host, extra, status) in &website_cases {
|
||||
let (old, new) = both(&legacy, &edge, || website_request(method.clone(), path, host, *extra)).await;
|
||||
if old != new || !old.starts_with(status.as_str()) {
|
||||
failures.push(format!("website {method} {host}{path} {extra:?}\n legacy {old}\n gateway {new}"));
|
||||
}
|
||||
}
|
||||
|
||||
let signed_as = |path: String, query: &'static str, key: &'static str, secret: &'static str| {
|
||||
move || signed(&path, query, S3_HOST, key, secret, Signed::Header)
|
||||
};
|
||||
let listing_cases: Vec<ListingCase> = vec![
|
||||
(
|
||||
"root v2",
|
||||
Box::new(signed_as(format!("/{BUCKET}"), "list-type=2&metadata=true", ROOT, ROOT_SECRET)),
|
||||
StatusCode::OK,
|
||||
),
|
||||
(
|
||||
"root versions",
|
||||
Box::new(signed_as(format!("/{BUCKET}"), "metadata=true&versions=", ROOT, ROOT_SECRET)),
|
||||
StatusCode::OK,
|
||||
),
|
||||
(
|
||||
"empty bucket",
|
||||
Box::new(signed_as(format!("/{EMPTY}"), "list-type=2&metadata=true", ROOT, ROOT_SECRET)),
|
||||
StatusCode::OK,
|
||||
),
|
||||
(
|
||||
"first page",
|
||||
Box::new(signed_as(format!("/{BUCKET}"), "list-type=2&max-keys=2&metadata=true", ROOT, ROOT_SECRET)),
|
||||
StatusCode::OK,
|
||||
),
|
||||
(
|
||||
"url encoding",
|
||||
Box::new(signed_as(
|
||||
format!("/{BUCKET}"),
|
||||
"encoding-type=url&list-type=2&metadata=true&prefix=a",
|
||||
ROOT,
|
||||
ROOT_SECRET,
|
||||
)),
|
||||
StatusCode::OK,
|
||||
),
|
||||
(
|
||||
"listing user",
|
||||
Box::new(signed_as(format!("/{BUCKET}"), "list-type=2&metadata=true", "list-allowed", USER_SECRET)),
|
||||
StatusCode::OK,
|
||||
),
|
||||
(
|
||||
"bad encoding",
|
||||
Box::new(signed_as(
|
||||
format!("/{BUCKET}"),
|
||||
"encoding-type=xml&list-type=2&metadata=true",
|
||||
ROOT,
|
||||
ROOT_SECRET,
|
||||
)),
|
||||
StatusCode::BAD_REQUEST,
|
||||
),
|
||||
(
|
||||
"bad token",
|
||||
Box::new(signed_as(
|
||||
format!("/{BUCKET}"),
|
||||
"continuation-token=%25%25&list-type=2&metadata=true",
|
||||
ROOT,
|
||||
ROOT_SECRET,
|
||||
)),
|
||||
StatusCode::BAD_REQUEST,
|
||||
),
|
||||
(
|
||||
"bad max-keys",
|
||||
Box::new(signed_as(
|
||||
format!("/{BUCKET}"),
|
||||
"list-type=2&max-keys=ten&metadata=true",
|
||||
ROOT,
|
||||
ROOT_SECRET,
|
||||
)),
|
||||
StatusCode::BAD_REQUEST,
|
||||
),
|
||||
(
|
||||
"no policy user",
|
||||
Box::new(signed_as(format!("/{BUCKET}"), "list-type=2&metadata=true", "list-none", USER_SECRET)),
|
||||
StatusCode::FORBIDDEN,
|
||||
),
|
||||
(
|
||||
"no policy versions",
|
||||
Box::new(signed_as(format!("/{BUCKET}"), "metadata=true&versions=", "list-none", USER_SECRET)),
|
||||
StatusCode::FORBIDDEN,
|
||||
),
|
||||
(
|
||||
"missing bucket",
|
||||
Box::new(signed_as("/parity-absent".to_owned(), "list-type=2&metadata=true", ROOT, ROOT_SECRET)),
|
||||
StatusCode::NOT_FOUND,
|
||||
),
|
||||
(
|
||||
"anonymous",
|
||||
Box::new(move || website_request(Method::GET, &format!("/{BUCKET}?list-type=2&metadata=true"), S3_HOST, None)),
|
||||
StatusCode::FORBIDDEN,
|
||||
),
|
||||
(
|
||||
"anonymous versions",
|
||||
Box::new(move || website_request(Method::GET, &format!("/{BUCKET}?versions&metadata=true"), S3_HOST, None)),
|
||||
StatusCode::FORBIDDEN,
|
||||
),
|
||||
];
|
||||
for (label, request, status) in &listing_cases {
|
||||
let (old, new) = both(&legacy, &edge, request).await;
|
||||
if old != new || !old.starts_with(status.as_str()) {
|
||||
failures.push(format!("listing {label}\n legacy {old}\n gateway {new}"));
|
||||
}
|
||||
}
|
||||
|
||||
// The listing documents carry what the extension adds, so equal bodies are not two empty ones.
|
||||
let root_v2 = answer(
|
||||
edge.clone(),
|
||||
signed(
|
||||
&format!("/{BUCKET}"),
|
||||
"list-type=2&metadata=true",
|
||||
S3_HOST,
|
||||
ROOT,
|
||||
ROOT_SECRET,
|
||||
Signed::Header,
|
||||
),
|
||||
)
|
||||
.await;
|
||||
let text = String::from_utf8_lossy(&root_v2.2);
|
||||
for part in [
|
||||
"<ListBucketResult",
|
||||
"<UserMetadata>",
|
||||
"<project>alpha</project>",
|
||||
"<Internal>",
|
||||
] {
|
||||
if !text.contains(part) {
|
||||
failures.push(format!("gateway listing lacks {part}: {text}"));
|
||||
}
|
||||
}
|
||||
let user = answer(
|
||||
edge.clone(),
|
||||
signed(
|
||||
&format!("/{BUCKET}"),
|
||||
"list-type=2&metadata=true",
|
||||
S3_HOST,
|
||||
"list-allowed",
|
||||
USER_SECRET,
|
||||
Signed::Header,
|
||||
),
|
||||
)
|
||||
.await;
|
||||
// The bucket policy denies everyone `private.txt`, so its entry carries no metadata, while
|
||||
// the entries the policy lets everyone read do.
|
||||
let listed = String::from_utf8_lossy(&user.2).into_owned();
|
||||
let entry = |key: &str| {
|
||||
listed
|
||||
.split("<Contents>")
|
||||
.find(|entry| entry.contains(&format!("<Key>{key}</Key>")))
|
||||
.unwrap_or_default()
|
||||
.to_owned()
|
||||
};
|
||||
if entry("private.txt").contains("<UserMetadata>") || !entry("docs").contains("<UserMetadata>") {
|
||||
failures.push(format!("the listing's metadata ignores what the caller may read: {listed}"));
|
||||
}
|
||||
let first = String::from_utf8_lossy(
|
||||
&answer(
|
||||
edge.clone(),
|
||||
signed(
|
||||
&format!("/{BUCKET}"),
|
||||
"list-type=2&max-keys=2&metadata=true",
|
||||
S3_HOST,
|
||||
ROOT,
|
||||
ROOT_SECRET,
|
||||
Signed::Header,
|
||||
),
|
||||
)
|
||||
.await
|
||||
.2,
|
||||
)
|
||||
.into_owned();
|
||||
match first
|
||||
.split_once("<NextContinuationToken>")
|
||||
.and_then(|(_, tail)| tail.split_once('<'))
|
||||
{
|
||||
Some((token, _)) => {
|
||||
let query: &'static str = Box::leak(
|
||||
format!("continuation-token={}&list-type=2&metadata=true", urlencoding::encode(token)).into_boxed_str(),
|
||||
);
|
||||
let (old, new) = both(&legacy, &edge, signed_as(format!("/{BUCKET}"), query, ROOT, ROOT_SECRET)).await;
|
||||
if old != new || !old.starts_with("200") || !old.contains("<ContinuationToken>") {
|
||||
failures.push(format!("listing second page\n legacy {old}\n gateway {new}"));
|
||||
}
|
||||
}
|
||||
None => failures.push(format!("the first page has no continuation token: {first}")),
|
||||
}
|
||||
assert!(failures.is_empty(), "{}", failures.join("\n"));
|
||||
}
|
||||
|
||||
/// A malformed path and a SigV4 `Authorization` header sent to a website host.
|
||||
///
|
||||
/// `/%zz` is not a percent-encoding the key decoder refuses, so it names the key `%zz`: both
|
||||
/// stacks answer the website's `404` error document. A bad `Authorization` header is the one
|
||||
/// difference: the legacy stack's S3 service refuses the header before routing (`400
|
||||
/// InvalidRequest`), while the gateway stack's website service never reads it, because a website
|
||||
/// read is anonymous and the header can neither refuse nor widen it (`200`).
|
||||
#[tokio::test]
|
||||
async fn website_host_answers_a_malformed_path_and_ignores_an_authorization_header() {
|
||||
let (_env, legacy, edge) = stacks().await;
|
||||
let site = format!("{BUCKET}.{WEBSITE_DOMAIN}");
|
||||
let bad_auth = Some(("authorization", "AWS4-HMAC-SHA256 Credential=nobody/20200101/us-east-1/s3/aws4_request"));
|
||||
|
||||
let request = || website_request(Method::GET, "/%zz", &site, None);
|
||||
let (old, new) = both(&legacy, &edge, request).await;
|
||||
assert_eq!(old, new);
|
||||
assert!(new.starts_with("404") && new.ends_with("error page"), "{new}");
|
||||
|
||||
let request = || website_request(Method::GET, "/index.html", &site, bad_auth);
|
||||
let (old, new) = both(&legacy, &edge, request).await;
|
||||
assert!(old.starts_with("400") && old.ends_with("error \"InvalidRequest\""), "{old}");
|
||||
assert!(new.starts_with("200") && new.ends_with("home"), "{new}");
|
||||
}
|
||||
@@ -26,6 +26,7 @@ mod hybrid;
|
||||
mod layer;
|
||||
mod legacy_compat;
|
||||
mod module_switch;
|
||||
mod post_object_location;
|
||||
mod prefix;
|
||||
pub mod rate_limit;
|
||||
mod readiness;
|
||||
|
||||
@@ -0,0 +1,397 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! POST Object `PostResponse` Location rewrite for AWS-compatible absolute URIs.
|
||||
//!
|
||||
//! Both protocol stacks currently emit a relative, unencoded path
|
||||
//! (`/{bucket}/{raw-key}`) when `success_action_status=201`. The s3s serializer
|
||||
//! hard-codes that spelling, and the gateway's RustFS-profile POST forms path
|
||||
//! does the same. This layer rebuilds `<Location>` from the response's `<Bucket>`
|
||||
//! and `<Key>` plus the request's host, scheme, and addressing style.
|
||||
//!
|
||||
//! Encoding follows RFC 3986 path rules (spaces as `%20`, `/` left as a
|
||||
//! separator, uppercase hex), matching MinIO's `getObjectLocation` and the
|
||||
//! gateway's non-legacy POST serializer. `<Key>` is left unchanged.
|
||||
|
||||
use crate::server::hybrid::HybridBody;
|
||||
use crate::server::legacy_compat::MAX_XML_COMPAT_BODY_BYTES;
|
||||
use crate::server::rate_limit::virtual_host_bucket;
|
||||
use bytes::Bytes;
|
||||
use http::{HeaderMap, Method, Request as HttpRequest, Response, StatusCode, Uri, header};
|
||||
use http_body::Body;
|
||||
use http_body_util::BodyExt;
|
||||
use rustfs_utils::http::get_source_scheme;
|
||||
use std::future::Future;
|
||||
use std::pin::Pin;
|
||||
use std::sync::Arc;
|
||||
use std::task::{Context, Poll};
|
||||
use tower::{Layer, Service};
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct PostObjectLocationFixLayer {
|
||||
vh_domain_hosts: Arc<[String]>,
|
||||
connection_is_tls: bool,
|
||||
}
|
||||
|
||||
impl PostObjectLocationFixLayer {
|
||||
pub fn new(vh_domain_hosts: Arc<[String]>, connection_is_tls: bool) -> Self {
|
||||
Self {
|
||||
vh_domain_hosts,
|
||||
connection_is_tls,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<S> Layer<S> for PostObjectLocationFixLayer {
|
||||
type Service = PostObjectLocationFixService<S>;
|
||||
|
||||
fn layer(&self, inner: S) -> Self::Service {
|
||||
PostObjectLocationFixService {
|
||||
inner,
|
||||
vh_domain_hosts: self.vh_domain_hosts.clone(),
|
||||
connection_is_tls: self.connection_is_tls,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct PostObjectLocationFixService<S> {
|
||||
inner: S,
|
||||
vh_domain_hosts: Arc<[String]>,
|
||||
connection_is_tls: bool,
|
||||
}
|
||||
|
||||
impl<S, ReqBody, RestBody, GrpcBody> Service<HttpRequest<ReqBody>> for PostObjectLocationFixService<S>
|
||||
where
|
||||
S: Service<HttpRequest<ReqBody>, Response = Response<HybridBody<RestBody, GrpcBody>>> + Clone + Send + 'static,
|
||||
S::Future: Send + 'static,
|
||||
S::Error: Send + 'static,
|
||||
ReqBody: Send + 'static,
|
||||
RestBody: Body<Data = Bytes> + From<Bytes> + Send + 'static,
|
||||
RestBody::Error: Into<S::Error> + Send + 'static,
|
||||
GrpcBody: Send + 'static,
|
||||
{
|
||||
type Response = Response<HybridBody<RestBody, GrpcBody>>;
|
||||
type Error = S::Error;
|
||||
type Future = Pin<Box<dyn Future<Output = Result<Self::Response, Self::Error>> + Send>>;
|
||||
|
||||
fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
|
||||
self.inner.poll_ready(cx)
|
||||
}
|
||||
|
||||
fn call(&mut self, req: HttpRequest<ReqBody>) -> Self::Future {
|
||||
let should_inspect = req.method() == Method::POST;
|
||||
let scheme = request_scheme(req.headers(), req.uri(), self.connection_is_tls);
|
||||
let host = request_host(req.headers(), req.uri()).map(str::to_owned);
|
||||
let virtual_hosted_bucket = virtual_host_bucket(&req, &self.vh_domain_hosts).map(str::to_owned);
|
||||
let mut inner = self.inner.clone();
|
||||
|
||||
Box::pin(async move {
|
||||
let response = inner.call(req).await?;
|
||||
if !should_inspect || response.status() != StatusCode::CREATED || !is_xml_content_type(response.headers()) {
|
||||
return Ok(response);
|
||||
}
|
||||
|
||||
let (parts, body) = response.into_parts();
|
||||
let response = match body {
|
||||
HybridBody::Rest { rest_body } if !is_bounded_xml_body(&rest_body) => {
|
||||
Response::from_parts(parts, HybridBody::Rest { rest_body })
|
||||
}
|
||||
HybridBody::Rest { rest_body } => {
|
||||
let rest_body = rewrite_location_body(rest_body, &scheme, host.as_deref(), virtual_hosted_bucket.as_deref())
|
||||
.await
|
||||
.map_err(Into::into)?;
|
||||
let mut parts = parts;
|
||||
parts.headers.remove(header::CONTENT_LENGTH);
|
||||
Response::from_parts(parts, HybridBody::Rest { rest_body })
|
||||
}
|
||||
HybridBody::Grpc {
|
||||
grpc_body,
|
||||
first_frame_started,
|
||||
body_started,
|
||||
first_frame_recorded,
|
||||
} => Response::from_parts(
|
||||
parts,
|
||||
HybridBody::Grpc {
|
||||
grpc_body,
|
||||
first_frame_started,
|
||||
body_started,
|
||||
first_frame_recorded,
|
||||
},
|
||||
),
|
||||
};
|
||||
Ok(response)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
async fn rewrite_location_body<RestBody>(
|
||||
body: RestBody,
|
||||
scheme: &str,
|
||||
host: Option<&str>,
|
||||
virtual_hosted_bucket: Option<&str>,
|
||||
) -> Result<RestBody, RestBody::Error>
|
||||
where
|
||||
RestBody: Body<Data = Bytes> + From<Bytes>,
|
||||
{
|
||||
let bytes = BodyExt::collect(body).await?.to_bytes();
|
||||
let xml = String::from_utf8(bytes.to_vec()).unwrap_or_else(|_| String::from_utf8_lossy(&bytes).into_owned());
|
||||
match rewrite_post_response_location(&xml, scheme, host, virtual_hosted_bucket) {
|
||||
Some(fixed) => Ok(RestBody::from(Bytes::from(fixed))),
|
||||
None => Ok(RestBody::from(Bytes::from(xml))),
|
||||
}
|
||||
}
|
||||
|
||||
fn is_bounded_xml_body<B: Body>(body: &B) -> bool {
|
||||
body.size_hint()
|
||||
.upper()
|
||||
.is_some_and(|length| length <= MAX_XML_COMPAT_BODY_BYTES)
|
||||
}
|
||||
|
||||
fn is_xml_content_type(headers: &HeaderMap) -> bool {
|
||||
let is_xml = headers
|
||||
.get(header::CONTENT_TYPE)
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.map(|content_type| content_type.to_ascii_lowercase().contains("xml"))
|
||||
.unwrap_or(false);
|
||||
if !is_xml {
|
||||
return false;
|
||||
}
|
||||
match headers.get(header::CONTENT_ENCODING).and_then(|value| value.to_str().ok()) {
|
||||
Some(encoding) => encoding.trim().is_empty() || encoding.eq_ignore_ascii_case("identity"),
|
||||
None => true,
|
||||
}
|
||||
}
|
||||
|
||||
fn request_scheme(headers: &HeaderMap, uri: &Uri, connection_is_tls: bool) -> String {
|
||||
get_source_scheme(headers)
|
||||
.and_then(|value| {
|
||||
value
|
||||
.split(',')
|
||||
.next()
|
||||
.map(str::trim)
|
||||
.filter(|value| !value.is_empty())
|
||||
.map(str::to_owned)
|
||||
})
|
||||
.or_else(|| uri.scheme_str().map(str::to_owned))
|
||||
.or_else(|| connection_is_tls.then(|| "https".to_string()))
|
||||
.unwrap_or_else(|| "http".to_string())
|
||||
.to_ascii_lowercase()
|
||||
}
|
||||
|
||||
fn request_host<'a>(headers: &'a HeaderMap, uri: &'a Uri) -> Option<&'a str> {
|
||||
uri.authority().map(|authority| authority.as_str()).or_else(|| {
|
||||
headers
|
||||
.get(header::HOST)
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.map(str::trim)
|
||||
.filter(|value| !value.is_empty())
|
||||
})
|
||||
}
|
||||
|
||||
/// Absolute object URI for a POST Object `PostResponse` `<Location>`.
|
||||
///
|
||||
/// Path-style requests include the bucket in the path. Virtual-hosted requests
|
||||
/// already carry the bucket in the host, so only the encoded key remains.
|
||||
/// When no host is available the encoded path is returned without a scheme.
|
||||
pub(crate) fn post_object_location(scheme: &str, host: Option<&str>, bucket: &str, key: &str, virtual_hosted: bool) -> String {
|
||||
let mut path = String::new();
|
||||
path.push('/');
|
||||
if !virtual_hosted {
|
||||
path.push_str(bucket);
|
||||
path.push('/');
|
||||
}
|
||||
push_encoded_object_path(&mut path, key.as_bytes());
|
||||
|
||||
match host.map(str::trim).filter(|value| !value.is_empty()) {
|
||||
Some(host) => format!("{scheme}://{host}{path}"),
|
||||
None => path,
|
||||
}
|
||||
}
|
||||
|
||||
fn push_encoded_object_path(out: &mut String, bytes: &[u8]) {
|
||||
for &byte in bytes {
|
||||
if byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_' | b'.' | b'~' | b'/') {
|
||||
out.push(char::from(byte));
|
||||
} else {
|
||||
out.push('%');
|
||||
out.push(char::from(hex_digit(byte >> 4)));
|
||||
out.push(char::from(hex_digit(byte & 0x0f)));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const fn hex_digit(nibble: u8) -> u8 {
|
||||
match nibble {
|
||||
0..=9 => b'0' + nibble,
|
||||
10..=15 => b'A' + nibble - 10,
|
||||
_ => b'?',
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn rewrite_post_response_location(
|
||||
xml: &str,
|
||||
scheme: &str,
|
||||
host: Option<&str>,
|
||||
virtual_hosted_bucket: Option<&str>,
|
||||
) -> Option<String> {
|
||||
if !xml.contains("<PostResponse") {
|
||||
return None;
|
||||
}
|
||||
let bucket = xml_unescape(xml_element(xml, "Bucket")?);
|
||||
let key = xml_unescape(xml_element(xml, "Key")?);
|
||||
let virtual_hosted = virtual_hosted_bucket.is_some_and(|name| name.eq_ignore_ascii_case(&bucket));
|
||||
let location = xml_escape(&post_object_location(scheme, host, &bucket, &key, virtual_hosted));
|
||||
let (start, end) = xml_element_span(xml, "Location")?;
|
||||
if xml[start..end] == location {
|
||||
return None;
|
||||
}
|
||||
let mut fixed = String::with_capacity(xml.len() - (end - start) + location.len());
|
||||
fixed.push_str(&xml[..start]);
|
||||
fixed.push_str(&location);
|
||||
fixed.push_str(&xml[end..]);
|
||||
Some(fixed)
|
||||
}
|
||||
|
||||
fn xml_element<'a>(xml: &'a str, name: &str) -> Option<&'a str> {
|
||||
let (start, end) = xml_element_span(xml, name)?;
|
||||
Some(&xml[start..end])
|
||||
}
|
||||
|
||||
fn xml_element_span(xml: &str, name: &str) -> Option<(usize, usize)> {
|
||||
let open = format!("<{name}>");
|
||||
let close = format!("</{name}>");
|
||||
let start = xml.find(&open)? + open.len();
|
||||
let end = start + xml[start..].find(&close)?;
|
||||
Some((start, end))
|
||||
}
|
||||
|
||||
fn xml_escape(value: &str) -> String {
|
||||
let mut escaped = String::with_capacity(value.len());
|
||||
for ch in value.chars() {
|
||||
match ch {
|
||||
'&' => escaped.push_str("&"),
|
||||
'<' => escaped.push_str("<"),
|
||||
'>' => escaped.push_str(">"),
|
||||
'"' => escaped.push_str("""),
|
||||
'\'' => escaped.push_str("'"),
|
||||
_ => escaped.push(ch),
|
||||
}
|
||||
}
|
||||
escaped
|
||||
}
|
||||
|
||||
fn xml_unescape(value: &str) -> String {
|
||||
value
|
||||
.replace("&", "&")
|
||||
.replace("<", "<")
|
||||
.replace(">", ">")
|
||||
.replace(""", "\"")
|
||||
.replace("'", "'")
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn encodes_reserved_and_unicode_key_characters_in_an_absolute_path_style_url() {
|
||||
let location = post_object_location("http", Some("s3.rustfs.localhost:9000"), "my-bucket", "test/a+b c%d.txt", false);
|
||||
assert_eq!(location, "http://s3.rustfs.localhost:9000/my-bucket/test/a%2Bb%20c%25d.txt");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn virtual_host_omits_the_bucket_from_the_path() {
|
||||
let location =
|
||||
post_object_location("https", Some("my-bucket.s3.rustfs.localhost:9000"), "my-bucket", "test/a+b c%d.txt", true);
|
||||
assert_eq!(location, "https://my-bucket.s3.rustfs.localhost:9000/test/a%2Bb%20c%25d.txt");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn leaves_unreserved_characters_and_slashes_in_the_key() {
|
||||
let location = post_object_location("http", Some("host.example:9000"), "bucket", "a-b_c.d~e/photo.jpg", false);
|
||||
assert_eq!(location, "http://host.example:9000/bucket/a-b_c.d~e/photo.jpg");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn percent_encodes_ampersand_question_hash_and_utf8() {
|
||||
let location = post_object_location("http", Some("host"), "bucket", "naïve/a&b?c#d.txt", false);
|
||||
assert_eq!(location, "http://host/bucket/na%C3%AFve/a%26b%3Fc%23d.txt");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn falls_back_to_an_encoded_path_when_the_host_is_missing() {
|
||||
assert_eq!(
|
||||
post_object_location("http", None, "bucket", "dir/file name.txt", false),
|
||||
"/bucket/dir/file%20name.txt"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn uses_x_forwarded_proto_and_host_header_when_rewriting_xml() {
|
||||
let xml = concat!(
|
||||
r#"<?xml version="1.0" encoding="UTF-8"?>"#,
|
||||
"<PostResponse>",
|
||||
"<Location>/my-bucket/test/a+b c%d.txt</Location>",
|
||||
"<Bucket>my-bucket</Bucket>",
|
||||
"<Key>test/a+b c%d.txt</Key>",
|
||||
r#"<ETag>9dd4e461268c8034f5c8564e155c67a6</ETag>"#,
|
||||
"</PostResponse>"
|
||||
);
|
||||
let rewritten = rewrite_post_response_location(xml, "https", Some("s3.example:9000"), None).unwrap();
|
||||
assert!(
|
||||
rewritten.contains("<Location>https://s3.example:9000/my-bucket/test/a%2Bb%20c%25d.txt</Location>"),
|
||||
"{rewritten}"
|
||||
);
|
||||
assert!(rewritten.contains("<Key>test/a+b c%d.txt</Key>"), "{rewritten}");
|
||||
assert!(rewritten.contains("<Bucket>my-bucket</Bucket>"), "{rewritten}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unescapes_xml_entities_in_the_stored_key_before_encoding() {
|
||||
let xml = concat!(
|
||||
"<PostResponse>",
|
||||
"<Location>/example-bucket/uploads/a b&résumé?.txt</Location>",
|
||||
"<Bucket>example-bucket</Bucket>",
|
||||
"<Key>uploads/a b&résumé?.txt</Key>",
|
||||
"<ETag>storage-etag</ETag>",
|
||||
"</PostResponse>"
|
||||
);
|
||||
let rewritten = rewrite_post_response_location(xml, "http", Some("host.invalid"), None).unwrap();
|
||||
assert!(
|
||||
rewritten.contains("<Location>http://host.invalid/example-bucket/uploads/a%20b%26r%C3%A9sum%C3%A9%3F.txt</Location>"),
|
||||
"{rewritten}"
|
||||
);
|
||||
assert!(rewritten.contains("<Key>uploads/a b&résumé?.txt</Key>"), "{rewritten}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn leaves_non_post_response_xml_unchanged() {
|
||||
let xml = "<CompleteMultipartUploadResult><Location>/bucket/key</Location></CompleteMultipartUploadResult>";
|
||||
assert_eq!(rewrite_post_response_location(xml, "http", Some("host"), None), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn request_scheme_prefers_forwarded_proto() {
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert("x-forwarded-proto", http::HeaderValue::from_static("https,http"));
|
||||
assert_eq!(request_scheme(&headers, &Uri::from_static("http://ignored/bucket"), false), "https");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn request_scheme_falls_back_to_a_tls_connection() {
|
||||
assert_eq!(request_scheme(&HeaderMap::new(), &Uri::from_static("/bucket"), true), "https");
|
||||
assert_eq!(request_scheme(&HeaderMap::new(), &Uri::from_static("/bucket"), false), "http");
|
||||
}
|
||||
}
|
||||
+13
-142
@@ -18,10 +18,9 @@
|
||||
//! be, and [`select_stack`], the only place a stack is chosen. The whole stack is one or the
|
||||
//! other: no request is routed between stacks by its operation, and a request the gateway stack
|
||||
//! refuses is answered by that refusal and never retried on the legacy stack. With the gateway
|
||||
//! selected, a request to a website host is answered by the website service before any other
|
||||
//! routing, as the legacy metadata route claims it first; console static requests and signed
|
||||
//! system-status reads use their native services and every other route the admin router claims
|
||||
//! goes to the control-plane host; the gateway selection holds no legacy service at all.
|
||||
//! selected, console static requests and signed system-status reads use their native
|
||||
//! services and every other route the admin router claims goes to the control-plane host; the
|
||||
//! gateway selection holds no legacy service at all.
|
||||
//! NOT responsible for: assembling either stack (`server/http.rs` builds the legacy service,
|
||||
//! `server/gateway_stack.rs` the gateway one), or the middleware around them.
|
||||
//! Upstream: `server/http.rs`. Downstream: the native console, legacy S3 service, gateway stack
|
||||
@@ -35,45 +34,38 @@ use crate::config::S3Stack;
|
||||
use http::Request;
|
||||
use std::future::Future;
|
||||
use std::pin::Pin;
|
||||
use std::sync::Arc;
|
||||
use std::task::{Context, Poll, ready};
|
||||
use tower::Service;
|
||||
|
||||
/// The S3 service of one server: the legacy stack, or the gateway stack with its control-plane
|
||||
/// services. The gateway selection holds no legacy service, so none of its requests can reach one.
|
||||
#[derive(Clone)]
|
||||
pub(crate) struct S3StackService<L, G, C = G, W = G> {
|
||||
stack: Stack<L, G, C, W>,
|
||||
pub(crate) struct S3StackService<L, G, C = G> {
|
||||
stack: Stack<L, G, C>,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
enum Stack<L, G, C, W> {
|
||||
enum Stack<L, G, C> {
|
||||
Legacy(L),
|
||||
Gateway(GatewayServices<G, C, W>),
|
||||
Gateway(GatewayServices<G, C>),
|
||||
}
|
||||
|
||||
/// Whether a request's target and `Host` select the service it is paired with.
|
||||
pub(crate) type HostClaim = Arc<dyn Fn(&http::Uri, &http::HeaderMap) -> bool + Send + Sync>;
|
||||
|
||||
/// The gateway selection: the S3 service, the website host, and the hosts of the routes the admin
|
||||
/// router claims.
|
||||
/// The gateway selection: the S3 service and the hosts of the routes the admin router claims.
|
||||
#[derive(Clone)]
|
||||
struct GatewayServices<G, C, W> {
|
||||
struct GatewayServices<G, C> {
|
||||
s3: G,
|
||||
native_website: Option<(HostClaim, W)>,
|
||||
native_console: Option<C>,
|
||||
native_admin: Option<G>,
|
||||
control_plane: Option<G>,
|
||||
}
|
||||
|
||||
impl<L, G, C, W> S3StackService<L, G, C, W> {
|
||||
impl<L, G, C> S3StackService<L, G, C> {
|
||||
/// Attaches the console service used only by the gateway selection.
|
||||
pub(crate) fn with_native_console<N>(self, native_console: Option<N>) -> S3StackService<L, G, N, W> {
|
||||
pub(crate) fn with_native_console<N>(self, native_console: Option<N>) -> S3StackService<L, G, N> {
|
||||
let stack = match self.stack {
|
||||
Stack::Legacy(legacy) => Stack::Legacy(legacy),
|
||||
Stack::Gateway(services) => Stack::Gateway(GatewayServices {
|
||||
s3: services.s3,
|
||||
native_website: services.native_website,
|
||||
native_console,
|
||||
native_admin: services.native_admin,
|
||||
control_plane: services.control_plane,
|
||||
@@ -82,22 +74,6 @@ impl<L, G, C, W> S3StackService<L, G, C, W> {
|
||||
S3StackService { stack }
|
||||
}
|
||||
|
||||
/// Attaches the website service used only by the gateway selection: every request `claim`
|
||||
/// selects goes to it before any other routing.
|
||||
pub(crate) fn with_native_website<N>(self, native_website: Option<(HostClaim, N)>) -> S3StackService<L, G, C, N> {
|
||||
let stack = match self.stack {
|
||||
Stack::Legacy(legacy) => Stack::Legacy(legacy),
|
||||
Stack::Gateway(services) => Stack::Gateway(GatewayServices {
|
||||
s3: services.s3,
|
||||
native_website,
|
||||
native_console: services.native_console,
|
||||
native_admin: services.native_admin,
|
||||
control_plane: services.control_plane,
|
||||
}),
|
||||
};
|
||||
S3StackService { stack }
|
||||
}
|
||||
|
||||
/// Attaches the signed admin service used only by the gateway selection.
|
||||
pub(crate) fn with_native_admin(mut self, native_admin: Option<G>) -> Self {
|
||||
if let Stack::Gateway(services) = &mut self.stack {
|
||||
@@ -142,7 +118,6 @@ pub(crate) fn select_stack<L, G, E>(
|
||||
S3Stack::Legacy => Stack::Legacy(legacy),
|
||||
S3Stack::Gateway => Stack::Gateway(GatewayServices {
|
||||
s3: assemble_gateway()?,
|
||||
native_website: None,
|
||||
native_console: None,
|
||||
native_admin: None,
|
||||
control_plane: None,
|
||||
@@ -162,22 +137,15 @@ fn bare<R: Default, E>(status: http::StatusCode) -> Answer<R, E> {
|
||||
})
|
||||
}
|
||||
|
||||
impl<G, C, W> GatewayServices<G, C, W> {
|
||||
impl<G, C> GatewayServices<G, C> {
|
||||
fn call<B, R, E>(&mut self, request: Request<B>) -> Answer<R, E>
|
||||
where
|
||||
G: Service<Request<B>, Response = http::Response<R>, Error = E>,
|
||||
C: Service<Request<B>, Response = http::Response<R>, Error = E>,
|
||||
W: Service<Request<B>, Response = http::Response<R>, Error = E>,
|
||||
G::Future: Send + 'static,
|
||||
C::Future: Send + 'static,
|
||||
W::Future: Send + 'static,
|
||||
R: Default,
|
||||
{
|
||||
if let Some((claim, website)) = &mut self.native_website
|
||||
&& claim(request.uri(), request.headers())
|
||||
{
|
||||
return Box::pin(website.call(request));
|
||||
}
|
||||
let claimed = claims_route(request.method(), request.uri(), request.headers());
|
||||
if claimed && (request.headers().contains_key("x-auth-token") || request.headers().contains_key("x-storage-token")) {
|
||||
return bare(http::StatusCode::FORBIDDEN);
|
||||
@@ -203,16 +171,14 @@ impl<G, C, W> GatewayServices<G, C, W> {
|
||||
}
|
||||
}
|
||||
|
||||
impl<L, G, C, W, B, R, E> Service<Request<B>> for S3StackService<L, G, C, W>
|
||||
impl<L, G, C, B, R, E> Service<Request<B>> for S3StackService<L, G, C>
|
||||
where
|
||||
L: Service<Request<B>, Response = http::Response<R>, Error = E>,
|
||||
G: Service<Request<B>, Response = http::Response<R>, Error = E>,
|
||||
C: Service<Request<B>, Response = http::Response<R>, Error = E>,
|
||||
W: Service<Request<B>, Response = http::Response<R>, Error = E>,
|
||||
L::Future: Send + 'static,
|
||||
G::Future: Send + 'static,
|
||||
C::Future: Send + 'static,
|
||||
W::Future: Send + 'static,
|
||||
R: Default,
|
||||
{
|
||||
type Response = http::Response<R>;
|
||||
@@ -233,9 +199,6 @@ where
|
||||
if let Some(console) = &mut services.native_console {
|
||||
ready!(console.poll_ready(cx))?;
|
||||
}
|
||||
if let Some((_, website)) = &mut services.native_website {
|
||||
ready!(website.poll_ready(cx))?;
|
||||
}
|
||||
Poll::Ready(Ok(()))
|
||||
}
|
||||
}
|
||||
@@ -645,98 +608,6 @@ mod tests {
|
||||
assert_eq!(answered_by(&mut service, sts).await, "control-plane");
|
||||
assert_eq!(answered_by(&mut service, request(Method::POST, "/")).await, "gateway");
|
||||
}
|
||||
/// A gateway stack whose website host claims every request to `*.website.test`.
|
||||
fn gateway_stack_with_website() -> S3StackService<Named, Named, Named, Named> {
|
||||
let claim: HostClaim = Arc::new(|_uri: &http::Uri, headers: &http::HeaderMap| {
|
||||
headers
|
||||
.get(http::header::HOST)
|
||||
.and_then(|host| host.to_str().ok())
|
||||
.is_some_and(|host| host.ends_with(".website.test"))
|
||||
});
|
||||
gateway_stack()
|
||||
.with_native_admin(Some(Named("admin")))
|
||||
.with_native_console(Some(Named("console")))
|
||||
.with_native_website(Some((claim, Named("website"))))
|
||||
}
|
||||
|
||||
fn to_host(method: Method, target: &str, host: &str, extra: Option<(&'static str, &'static str)>) -> Request<()> {
|
||||
let mut builder = Request::builder().method(method).uri(target).header(http::header::HOST, host);
|
||||
if let Some((name, value)) = extra {
|
||||
builder = builder.header(name, value);
|
||||
}
|
||||
builder.body(()).expect("static request")
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_website_host_is_answered_by_the_website_service_before_any_other_route() {
|
||||
let mut service = gateway_stack_with_website();
|
||||
let console = format!("{}/index.html", crate::server::console_prefix());
|
||||
for (method, target, extra) in [
|
||||
(Method::GET, "/", None),
|
||||
(Method::HEAD, "/index.html", None),
|
||||
(Method::POST, "/", Some(("content-type", "application/x-www-form-urlencoded"))),
|
||||
(Method::GET, "/rustfs/admin/v3/info", None),
|
||||
(Method::GET, "/rustfs/admin/v3/info", Some(("x-auth-token", "token"))),
|
||||
(Method::GET, "/rustfs/admin/v3/datausageinfo", None),
|
||||
(Method::GET, "/health", None),
|
||||
(Method::GET, "/profile/cpu", None),
|
||||
(Method::GET, "/bucket?replication-metrics", None),
|
||||
(Method::GET, "/bucket?list-type=2&metadata=true", None),
|
||||
(Method::GET, console.as_str(), None),
|
||||
(Method::PUT, "/bucket/key", None),
|
||||
] {
|
||||
let request = to_host(method.clone(), target, "site.website.test", extra);
|
||||
assert_eq!(answered_by_any(&mut service, request).await, "website", "{method} {target} {extra:?}");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_request_to_any_other_host_never_reaches_the_website_service() {
|
||||
let mut service = gateway_stack_with_website();
|
||||
let console = format!("{}/index.html", crate::server::console_prefix());
|
||||
for (method, target, host, expected) in [
|
||||
(Method::GET, "/", "localhost", "gateway"),
|
||||
(Method::GET, "/bucket/key", "website.test.example", "gateway"),
|
||||
(Method::GET, "/bucket?list-type=2&metadata=true", "localhost", "gateway"),
|
||||
(Method::GET, "/rustfs/admin/v3/info", "localhost", "admin"),
|
||||
(Method::GET, "/health", "localhost", "control-plane"),
|
||||
(Method::GET, "/bucket?replication-metrics", "s3.example.test", "control-plane"),
|
||||
(Method::GET, console.as_str(), "localhost", "console"),
|
||||
] {
|
||||
let request = to_host(method.clone(), target, host, None);
|
||||
assert_eq!(answered_by_any(&mut service, request).await, expected, "{method} {host} {target}");
|
||||
}
|
||||
let refused = service
|
||||
.call(to_host(Method::GET, "/health", "localhost", Some(("x-auth-token", "token"))))
|
||||
.await
|
||||
.expect("infallible");
|
||||
assert_eq!(refused.status(), StatusCode::FORBIDDEN, "a Keystone token on a claimed route");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn the_legacy_stack_never_reaches_a_website_service() {
|
||||
let claim: HostClaim = Arc::new(|_uri: &http::Uri, _headers: &http::HeaderMap| true);
|
||||
let mut service = select_stack(S3Stack::Legacy, Named("legacy"), || Ok::<_, Infallible>(Named("gateway")))
|
||||
.expect("legacy needs no assembly")
|
||||
.with_native_website(Some((claim, Named("website"))));
|
||||
for target in ["/", "/index.html", "/rustfs/admin/v3/info"] {
|
||||
let request = to_host(Method::GET, target, "site.website.test", None);
|
||||
assert_eq!(answered_by_any(&mut service, request).await, "legacy", "{target}");
|
||||
}
|
||||
}
|
||||
|
||||
async fn answered_by_any<C, W>(service: &mut S3StackService<Named, Named, C, W>, request: Request<()>) -> &'static str
|
||||
where
|
||||
C: Service<Request<()>, Response = Response<&'static str>, Error = Infallible>,
|
||||
W: Service<Request<()>, Response = Response<&'static str>, Error = Infallible>,
|
||||
C::Future: Send + 'static,
|
||||
W::Future: Send + 'static,
|
||||
{
|
||||
let response = service.call(request).await.expect("infallible");
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
response.into_body()
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn native_admin_selection_is_exact_and_host_independent() {
|
||||
let mut service = gateway_stack().with_native_admin(Some(Named("admin")));
|
||||
|
||||
@@ -21,10 +21,11 @@ use rustfs_gateway_types::compat::ConversionError;
|
||||
use rustfs_gateway_types::compat::s3s_5761ddfe::generated::ops::list_buckets;
|
||||
use rustfs_io_metrics::record_s3_op;
|
||||
use rustfs_s3_ops::S3Operation;
|
||||
use s3s::{S3, S3Error, S3ErrorCode, S3Request, S3Response, S3Result, TrailingHeaders, dto::*};
|
||||
use s3s::{S3, S3Error, S3ErrorCode, S3Request, S3Response, S3Result, TrailingHeaders, dto::*, s3_error};
|
||||
use std::fmt::Debug;
|
||||
use std::future::Future;
|
||||
use tracing::{error, instrument};
|
||||
use uuid::Uuid;
|
||||
|
||||
const LOG_COMPONENT_STORAGE: &str = "storage";
|
||||
const LOG_SUBSYSTEM_OBJECT: &str = "object";
|
||||
@@ -66,6 +67,12 @@ pub struct FS {
|
||||
server_ctx: std::sync::Arc<runtime_sources::ServerContextSlot>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, serde::Deserialize)]
|
||||
pub(crate) struct ListObjectUnorderedQuery {
|
||||
#[serde(rename = "allow-unordered")]
|
||||
pub(crate) allow_unordered: Option<String>,
|
||||
}
|
||||
|
||||
impl Default for FS {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
@@ -106,6 +113,22 @@ impl FS {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn parse_object_version_id(version_id: Option<String>) -> S3Result<Option<Uuid>> {
|
||||
if let Some(vid) = version_id {
|
||||
if vid == "null" {
|
||||
// A nil UUID selects the stored null version; None selects latest.
|
||||
return Ok(Some(Uuid::nil()));
|
||||
}
|
||||
let uuid = Uuid::parse_str(&vid).map_err(|e| {
|
||||
error!("Invalid version ID: {}", e);
|
||||
s3_error!(InvalidArgument, "Invalid version ID")
|
||||
})?;
|
||||
Ok(Some(uuid))
|
||||
} else {
|
||||
Ok(None)
|
||||
}
|
||||
}
|
||||
|
||||
/// Runs a use case that speaks the request envelope and the gateway's types behind this edge: the
|
||||
/// input and the output cross the gateway's compatibility seam, and nothing else is converted.
|
||||
pub(in crate::storage) async fn through_gateway_seam<I, GatewayInput, GatewayOutput, O, F>(
|
||||
|
||||
@@ -12,14 +12,15 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use super::request_markers::{RequestedObjectLegalHold, RequestedObjectLockRetention};
|
||||
use super::access::{RequestedObjectLegalHold, RequestedObjectLockRetention};
|
||||
use super::{StorageError, add_object_lock_years, get_bucket_cors_config};
|
||||
use crate::config::{RustFSBufferConfig, WorkloadProfile, is_buffer_profile_enabled};
|
||||
use crate::error::ApiError;
|
||||
use crate::server::cors;
|
||||
use crate::storage::ecfs::ListObjectUnorderedQuery;
|
||||
use crate::storage::storage_api::ecfs_extend_consumer::contract::bucket::{BucketOperations, BucketOptions};
|
||||
use crate::storage::storage_api::ecfs_extend_consumer::contract::multipart::MAX_MULTIPART_PART_NUMBER;
|
||||
use crate::storage::storage_api::ecstore_bucket::metadata_sys::{self, ObjectLockConfigState};
|
||||
use crate::storage::storage_api::s3_support_consumer::contract::bucket::{BucketOperations, BucketOptions};
|
||||
use crate::storage::storage_api::s3_support_consumer::contract::multipart::MAX_MULTIPART_PART_NUMBER;
|
||||
use http::header::{IF_MATCH, IF_MODIFIED_SINCE, IF_NONE_MATCH, IF_UNMODIFIED_SINCE};
|
||||
use http::{HeaderMap, HeaderValue, StatusCode};
|
||||
use metrics::counter;
|
||||
@@ -46,38 +47,13 @@ use time::format_description::well_known::Rfc3339;
|
||||
use time::{format_description::FormatItem, macros::format_description};
|
||||
use tracing::{debug, warn};
|
||||
|
||||
use crate::storage::storage_api::ecfs_extend_consumer::StorageObjectInfo as ObjectInfo;
|
||||
use crate::storage::storage_api::runtime_sources_consumer::runtime_sources;
|
||||
use crate::storage::storage_api::s3_support_consumer::StorageObjectInfo as ObjectInfo;
|
||||
|
||||
const LOG_COMPONENT_STORAGE: &str = "storage";
|
||||
const LOG_SUBSYSTEM_OBJECT_LOCK: &str = "object_lock";
|
||||
const LOG_SUBSYSTEM_OBJECT_KEY: &str = "object_key";
|
||||
|
||||
/// Query-string view used to reject `allow-unordered=true` combined with a delimiter.
|
||||
#[derive(Debug, Default, serde::Deserialize)]
|
||||
pub(crate) struct ListObjectUnorderedQuery {
|
||||
#[serde(rename = "allow-unordered")]
|
||||
pub(crate) allow_unordered: Option<String>,
|
||||
}
|
||||
|
||||
/// Parse an optional `versionId` selector: `None` selects the latest version and `"null"` the
|
||||
/// stored null version (a nil UUID).
|
||||
pub(crate) fn parse_object_version_id(version_id: Option<String>) -> S3Result<Option<uuid::Uuid>> {
|
||||
if let Some(vid) = version_id {
|
||||
if vid == "null" {
|
||||
// A nil UUID selects the stored null version; None selects latest.
|
||||
return Ok(Some(uuid::Uuid::nil()));
|
||||
}
|
||||
let uuid = uuid::Uuid::parse_str(&vid).map_err(|e| {
|
||||
tracing::error!("Invalid version ID: {}", e);
|
||||
S3Error::with_message(S3ErrorCode::InvalidArgument, "Invalid version ID".to_string())
|
||||
})?;
|
||||
Ok(Some(uuid))
|
||||
} else {
|
||||
Ok(None)
|
||||
}
|
||||
}
|
||||
|
||||
pub const RFC1123: &[FormatItem<'_>] =
|
||||
format_description!("[weekday repr:short], [day] [month repr:short] [year] [hour]:[minute]:[second] GMT");
|
||||
|
||||
@@ -24,12 +24,12 @@ mod tests {
|
||||
"file__XLDIR__/",
|
||||
"a__XLDIR__b",
|
||||
] {
|
||||
let error = crate::storage::s3_support::validate_object_key(key, "PUT")
|
||||
let error = crate::storage::ecfs_extend::validate_object_key(key, "PUT")
|
||||
.expect_err("client keys must not alias internal directory markers");
|
||||
assert_eq!(error.code(), &s3s::S3ErrorCode::InvalidArgument, "{key}");
|
||||
}
|
||||
for key in ["file", "folder/", "prefix/folder/", "__xldir__", "percent%20name"] {
|
||||
crate::storage::s3_support::validate_object_key(key, "PUT")
|
||||
crate::storage::ecfs_extend::validate_object_key(key, "PUT")
|
||||
.expect("ordinary objects and directory objects remain valid");
|
||||
}
|
||||
}
|
||||
@@ -37,10 +37,9 @@ mod tests {
|
||||
use crate::config::WorkloadProfile;
|
||||
use crate::server::cors;
|
||||
use crate::storage::StorageError;
|
||||
use crate::storage::ecfs::FS;
|
||||
use crate::storage::ecfs::{FS, parse_object_version_id};
|
||||
use crate::storage::ecfs_extend::{apply_bucket_default_lock_retention, map_bucket_object_lock_config_state};
|
||||
use crate::storage::s3_api::common::{rustfs_initiator, rustfs_owner};
|
||||
use crate::storage::s3_support::parse_object_version_id;
|
||||
use crate::storage::s3_support::{apply_bucket_default_lock_retention, map_bucket_object_lock_config_state};
|
||||
use crate::storage::storage_api::ecstore_bucket::metadata_sys::ObjectLockConfigState;
|
||||
use crate::storage::storage_api::test_consumer::{
|
||||
BucketMetadata, DEFAULT_READ_BUFFER_SIZE, StorageObjectInfo as ObjectInfo, apply_cors_headers,
|
||||
|
||||
@@ -24,12 +24,12 @@ use crate::app::RequestEnvelope;
|
||||
use crate::app::storage_api::object_usecase::bucket::restore_request_from_wire;
|
||||
use crate::storage::access::legacy_copy_source;
|
||||
use crate::storage::ecfs::*;
|
||||
use crate::storage::ecfs_extend::validate_new_object_key;
|
||||
use crate::storage::helper::audit_only_operation;
|
||||
use crate::storage::s3_api::{
|
||||
self, CopyObjectWriteInput, CopyObjectWriteOutput, CopySourceCondition, DefaultObjectUsecase, DeleteBatchETag,
|
||||
DeleteBatchEntry, DeleteBatchInput, DeleteBatchOutput,
|
||||
};
|
||||
use crate::storage::s3_support::validate_new_object_key;
|
||||
use crate::storage::storage_api::to_s3s_xml_etag;
|
||||
use http::{HeaderMap, HeaderValue};
|
||||
use rustfs_gateway_types::compat::s3s_5761ddfe::generated::ops::complete_multipart_upload;
|
||||
|
||||
@@ -36,6 +36,7 @@ pub mod tonic_service;
|
||||
mod concurrent_fix_test;
|
||||
#[cfg(test)]
|
||||
mod concurrent_get_object_test;
|
||||
mod ecfs_extend;
|
||||
#[cfg(test)]
|
||||
mod ecfs_test;
|
||||
pub(crate) mod head_prefix;
|
||||
@@ -44,13 +45,12 @@ mod minio_generated_read_test;
|
||||
#[cfg(test)]
|
||||
mod multi_factor_scheduler_integration_test;
|
||||
pub(crate) mod runtime_sources;
|
||||
mod s3_support;
|
||||
pub(crate) mod scanner_dirty_journal;
|
||||
#[cfg(test)]
|
||||
mod sse_test;
|
||||
pub(crate) mod storage_api;
|
||||
|
||||
pub(crate) use s3_support::*;
|
||||
pub(crate) use ecfs_extend::*;
|
||||
#[allow(unused_imports)]
|
||||
pub(crate) use storage_api::{
|
||||
BUCKET_ACCELERATE_CONFIG, BUCKET_REQUEST_PAYMENT_CONFIG, BUCKET_TABLE_CATALOG_META_PREFIX,
|
||||
@@ -67,12 +67,12 @@ pub(crate) use storage_api::{
|
||||
WorkloadAdmissionSnapshotProviderRef, WriteEncryption, WritePlan, access_consumer, add_object_lock_years, all_local_disk,
|
||||
all_local_disk_path, collect_local_metrics, compression_metadata_value, contract, decode_tags, decode_tags_to_map,
|
||||
delete_bucket_metadata_config, delete_bucket_metadata_config_if_incarnation, disk_drive_path, disk_endpoint, ecfs_consumer,
|
||||
ecstore_admin, ecstore_bucket, ecstore_capacity, ecstore_client, ecstore_cluster, ecstore_compression, ecstore_config,
|
||||
ecstore_data_usage, ecstore_disk, ecstore_error, ecstore_event, ecstore_layout, ecstore_metrics, ecstore_notification,
|
||||
ecstore_rebalance, ecstore_rio, ecstore_rpc, ecstore_set_disk, ecstore_storage, ecstore_tier, encode_tags,
|
||||
find_local_disk_by_ref, get_bucket_cors_config, get_bucket_metadata, get_bucket_notification_config, get_bucket_sse_config,
|
||||
get_bucket_website_config_for_store, get_local_server_property, get_lock_acquire_timeout, head_prefix_consumer,
|
||||
helper_consumer, init_background_replication, init_bucket_metadata_sys, init_ecstore_config,
|
||||
ecfs_extend_consumer, ecstore_admin, ecstore_bucket, ecstore_capacity, ecstore_client, ecstore_cluster, ecstore_compression,
|
||||
ecstore_config, ecstore_data_usage, ecstore_disk, ecstore_error, ecstore_event, ecstore_layout, ecstore_metrics,
|
||||
ecstore_notification, ecstore_rebalance, ecstore_rio, ecstore_rpc, ecstore_set_disk, ecstore_storage, ecstore_tier,
|
||||
encode_tags, find_local_disk_by_ref, get_bucket_cors_config, get_bucket_metadata, get_bucket_notification_config,
|
||||
get_bucket_sse_config, get_bucket_website_config_for_store, get_local_server_property, get_lock_acquire_timeout,
|
||||
head_prefix_consumer, helper_consumer, init_background_replication, init_bucket_metadata_sys, init_ecstore_config,
|
||||
init_local_disks_with_instance_ctx, init_lock_clients, is_err_bucket_not_found, is_err_object_not_found,
|
||||
is_err_version_not_found, is_valid_storage_class, options_consumer, prewarm_local_disk_id_map_with_instance_ctx, read_config,
|
||||
record_replication_proxy, rpc_consumer, runtime_sources_consumer, s3_api_consumer, table_catalog_path_hash, to_s3s_etag,
|
||||
|
||||
@@ -134,14 +134,13 @@ pub(crate) type StoragePrepareSelectObjectSnapshotError = ecstore_object::Prepar
|
||||
pub(crate) type StorageSelectObjectSnapshot = ecstore_object::SelectObjectSnapshot;
|
||||
pub(crate) type StorageObjectToDelete = contract::object::ObjectToDelete;
|
||||
pub(crate) type StoragePutObjReader = super::PutObjReader;
|
||||
pub(crate) use super::s3_support::{
|
||||
pub(crate) use super::ecfs_extend::{
|
||||
RFC1123, apply_bucket_default_lock_retention, apply_cors_headers, check_preconditions, evaluate_read_preconditions,
|
||||
get_buffer_size_opt_in, get_validated_store, get_validated_store_owned, has_read_preconditions,
|
||||
load_bucket_object_lock_config_state, parse_object_lock_legal_hold_native, parse_object_lock_retention_native,
|
||||
parse_object_version_id, parse_part_number_i32_to_usize, process_lambda_configurations, process_queue_configurations,
|
||||
process_topic_configurations, remove_object_lock_metadata_for_copy, validate_bucket_exists,
|
||||
validate_bucket_object_lock_enabled_state, validate_list_object_unordered_with_delimiter, validate_new_object_key,
|
||||
validate_object_key,
|
||||
parse_part_number_i32_to_usize, process_lambda_configurations, process_queue_configurations, process_topic_configurations,
|
||||
remove_object_lock_metadata_for_copy, validate_bucket_exists, validate_bucket_object_lock_enabled_state,
|
||||
validate_list_object_unordered_with_delimiter, validate_new_object_key, validate_object_key,
|
||||
};
|
||||
pub(crate) use super::sse::{
|
||||
DecryptionRequest, EncryptionRequest, ObjectDekRewrapOutcome, PrepareEncryptionRequest, SseKmsPrincipal,
|
||||
@@ -250,7 +249,7 @@ pub(crate) mod ecfs_consumer {
|
||||
pub(crate) type StorageObjectOptions = super::StorageObjectOptions;
|
||||
}
|
||||
|
||||
pub(crate) mod s3_support_consumer {
|
||||
pub(crate) mod ecfs_extend_consumer {
|
||||
pub(crate) mod contract {
|
||||
pub(crate) mod bucket {
|
||||
pub(crate) use super::super::super::contract::bucket::{BucketOperations, BucketOptions};
|
||||
@@ -864,6 +863,7 @@ pub(crate) type FileReader = ecstore_disk::FileReader;
|
||||
pub(crate) type FileWriter = ecstore_disk::FileWriter;
|
||||
pub(crate) type SnapshotLeaseToken = ecstore_disk::SnapshotLeaseToken;
|
||||
pub(crate) type FS = super::ecfs::FS;
|
||||
pub(crate) use super::ecfs::parse_object_version_id;
|
||||
pub(crate) type HashReader = ecstore_rio::HashReader;
|
||||
pub(crate) type EtagAlgorithm = ecstore_rio::EtagAlgorithm;
|
||||
pub(crate) type InstanceContext = ecstore_runtime::InstanceContext;
|
||||
|
||||
@@ -147,18 +147,18 @@ async fn attributes_wire(server: RustFSServer, stack: Option<&str>) {
|
||||
.await;
|
||||
assert_eq!((observed.status, observed.count), (200, 1), "{key}");
|
||||
assert!(observed.headers.contains_key(http::header::LAST_MODIFIED), "{key} date");
|
||||
// The legacy writer escapes a double quote in element text; the gateway's writes it as is.
|
||||
let quote = if stack == LEGACY { "a"b" } else { "a\"b" };
|
||||
let expected = ["abc", "abc", "abc", "", "", "a&b", quote];
|
||||
let body = std::str::from_utf8(&observed.body).unwrap();
|
||||
assert_eq!(
|
||||
body,
|
||||
format!(
|
||||
"<?xml version=\"1.0\" encoding=\"UTF-8\"?><GetObjectAttributesResponse xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\"><ETag>{}</ETag><ObjectSize>7</ObjectSize><StorageClass>STANDARD</StorageClass></GetObjectAttributesResponse>",
|
||||
expected[index]
|
||||
),
|
||||
"{key} raw XML"
|
||||
);
|
||||
let expected = ["abc", "abc", "abc", "", "", "a&b", "a"b"];
|
||||
if stack == LEGACY {
|
||||
let body = std::str::from_utf8(&observed.body).unwrap();
|
||||
assert_eq!(
|
||||
body,
|
||||
format!(
|
||||
"<?xml version=\"1.0\" encoding=\"UTF-8\"?><GetObjectAttributesResponse xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\"><ETag>{}</ETag><ObjectSize>7</ObjectSize><StorageClass>STANDARD</StorageClass></GetObjectAttributesResponse>",
|
||||
expected[index]
|
||||
),
|
||||
"{key} raw XML"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
sdk.put_object()
|
||||
@@ -277,13 +277,8 @@ async fn attributes_wire(server: RustFSServer, stack: Option<&str>) {
|
||||
path.push_str(&format!("&versionId={version}"));
|
||||
}
|
||||
let observed = observe(&server, &path, &headers, None).await;
|
||||
{
|
||||
// The gateway refuses an empty selection, and a malformed SSE-C header on a missing
|
||||
// object or a delete marker, before the object is looked up; the legacy stack looks up first.
|
||||
let refused_first =
|
||||
stack == GATEWAY && matches!(label, "empty-selection" | "missing-object" | "current-marker" | "explicit-marker");
|
||||
if stack == LEGACY {
|
||||
let (status, count, code) = match label {
|
||||
_ if refused_first => (400, 0, "InvalidRequest"),
|
||||
"absent-selection" => (400, 0, "InvalidRequest"),
|
||||
"bad-version-before-lookup" | "empty-version" => (400, 1, "InvalidArgument"),
|
||||
"preserved-short-tag-parts" => (400, 1, "InvalidArgument"),
|
||||
@@ -302,7 +297,7 @@ async fn attributes_wire(server: RustFSServer, stack: Option<&str>) {
|
||||
}
|
||||
}
|
||||
match label {
|
||||
"normalized-selection" | "etag-only" => observed.contains("ETag", "abc", label),
|
||||
"normalized-selection" | "etag-only" if stack == LEGACY => observed.contains("ETag", "abc", label),
|
||||
"size-only" => {
|
||||
observed.contains("ObjectSize", "7", label);
|
||||
observed.omits("ObjectParts", label);
|
||||
@@ -536,13 +531,15 @@ async fn attributes_wire(server: RustFSServer, stack: Option<&str>) {
|
||||
observed.contains("ChecksumSHA256", "lU0buD2Au29udGso8N4+xMTtmAz+Z+0jqRWc1GT/M5o=", label);
|
||||
} else {
|
||||
assert_eq!(observed.status, 400, "{label}: {}", observed.xml());
|
||||
observed.contains("Code", "InvalidRequest", label);
|
||||
if label == "ssec-wrong" {
|
||||
observed.contains(
|
||||
"Message",
|
||||
"The provided encryption parameters did not match the ones used originally to encrypt the object.",
|
||||
label,
|
||||
);
|
||||
if stack == LEGACY {
|
||||
observed.contains("Code", "InvalidRequest", label);
|
||||
if label == "ssec-wrong" {
|
||||
observed.contains(
|
||||
"Message",
|
||||
"The provided encryption parameters did not match the ones used originally to encrypt the object.",
|
||||
label,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -189,226 +189,184 @@ fn part(number: Option<i32>, etag: &str) -> CompletedPart {
|
||||
CompletedPart::builder().set_part_number(number).e_tag(etag).build()
|
||||
}
|
||||
|
||||
/// CompleteMultipartUpload answers every request shape as before its use case moved onto the
|
||||
/// gateway's types. Every answer was measured on the legacy stack on `main` before the move, and
|
||||
/// both stacks are held to the same table.
|
||||
async fn completes_every_upload(server: rustfs::embedded::RustFSServer, divergences: &[(&str, &str)]) {
|
||||
let client = s3_client(&server.endpoint(), server.access_key(), server.secret_key());
|
||||
client
|
||||
.create_bucket()
|
||||
.bucket(ORDINARY_BUCKET)
|
||||
.send()
|
||||
.await
|
||||
.expect("CreateBucket");
|
||||
for bucket in RESERVED_SHAPED_BUCKETS {
|
||||
create_bucket_in_the_store(bucket).await;
|
||||
}
|
||||
let mut answered = Vec::new();
|
||||
|
||||
for bucket in [ORDINARY_BUCKET].into_iter().chain(RESERVED_SHAPED_BUCKETS) {
|
||||
let (upload_id, etag) = one_part_upload(&client, bucket, "object").await;
|
||||
let answer = complete(&client, bucket, "object", &upload_id, part(Some(1), &etag), &[]).await;
|
||||
answered.push((format!("bucket {bucket}"), answer));
|
||||
}
|
||||
|
||||
let too_long = "k".repeat(1025);
|
||||
for (label, key) in [("NUL key", "nul\0key"), ("1025-byte key", too_long.as_str())] {
|
||||
let answer = complete(&client, ORDINARY_BUCKET, key, "no-such-upload", part(Some(1), "\"0\""), &[]).await;
|
||||
answered.push((label.to_owned(), answer));
|
||||
}
|
||||
|
||||
type ShapeCase<'a> = (&'a str, Option<i32>, fn(&str) -> String, &'a [(&'a str, &'a str)]);
|
||||
let shapes: [ShapeCase<'_>; 10] = [
|
||||
("empty part ETag", Some(1), |_| String::new(), &[]),
|
||||
("weak spelling of the stored part ETag", Some(1), |etag| format!("W/{etag}"), &[]),
|
||||
("unquoted stored part ETag", Some(1), |etag| etag.trim_matches('"').to_owned(), &[]),
|
||||
(
|
||||
"quoted other part ETag",
|
||||
Some(1),
|
||||
|_| "\"00000000000000000000000000000000\"".to_owned(),
|
||||
&[],
|
||||
),
|
||||
(
|
||||
"unquoted other part ETag",
|
||||
Some(1),
|
||||
|_| "00000000000000000000000000000000".to_owned(),
|
||||
&[],
|
||||
),
|
||||
(
|
||||
"weak other part ETag",
|
||||
Some(1),
|
||||
|_| "W/\"00000000000000000000000000000000\"".to_owned(),
|
||||
&[],
|
||||
),
|
||||
("part without a number", None, str::to_owned, &[]),
|
||||
(
|
||||
"two checksum headers",
|
||||
Some(1),
|
||||
str::to_owned,
|
||||
&[
|
||||
("crc32", "AAAAAA=="),
|
||||
("sha256", "47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU="),
|
||||
],
|
||||
),
|
||||
("checksum of the wrong width", Some(1), str::to_owned, &[("crc32", "AAAA")]),
|
||||
("ordinary", Some(1), str::to_owned, &[("crc32", "AAAAAA==")]),
|
||||
];
|
||||
for (index, (label, number, etag_for, checksums)) in shapes.into_iter().enumerate() {
|
||||
let key = format!("shape-{index}");
|
||||
let (upload_id, etag) = one_part_upload(&client, ORDINARY_BUCKET, &key).await;
|
||||
let answer = complete(&client, ORDINARY_BUCKET, &key, &upload_id, part(number, &etag_for(&etag)), checksums).await;
|
||||
answered.push((label.to_owned(), answer));
|
||||
}
|
||||
|
||||
for (label, upload_id) in [
|
||||
("upload id with a dot-dot segment", "a/../b"),
|
||||
("upload id never minted", "bm90LWFuLXVwbG9hZA"),
|
||||
] {
|
||||
let answer = complete(&client, ORDINARY_BUCKET, "object", upload_id, part(Some(1), "\"0\""), &[]).await;
|
||||
answered.push((label.to_owned(), answer));
|
||||
}
|
||||
|
||||
// No body at all: the legacy decoder refuses it before the operation.
|
||||
let (upload_id, _) = one_part_upload(&client, ORDINARY_BUCKET, "no-body").await;
|
||||
let answer = match client
|
||||
.complete_multipart_upload()
|
||||
.bucket(ORDINARY_BUCKET)
|
||||
.key("no-body")
|
||||
.upload_id(&upload_id)
|
||||
.send()
|
||||
.await
|
||||
{
|
||||
Ok(_) => "200".to_owned(),
|
||||
Err(error) => format!(
|
||||
"{:?} {:?} {:?}",
|
||||
error.raw_response().map(|response| response.status().as_u16()),
|
||||
error.code(),
|
||||
error.message()
|
||||
),
|
||||
};
|
||||
answered.push(("no body".to_owned(), answer));
|
||||
|
||||
let expected = [
|
||||
(
|
||||
"bucket e1-complete",
|
||||
"200 bucket=Some(\"e1-complete\") key_len=Some(6) etag=Some(true) crc32=None sha256=None type=None",
|
||||
),
|
||||
(
|
||||
"bucket sthree-e1",
|
||||
"200 bucket=Some(\"sthree-e1\") key_len=Some(6) etag=Some(true) crc32=None sha256=None type=None",
|
||||
),
|
||||
(
|
||||
"bucket e1-s3alias",
|
||||
"200 bucket=Some(\"e1-s3alias\") key_len=Some(6) etag=Some(true) crc32=None sha256=None type=None",
|
||||
),
|
||||
(
|
||||
"NUL key",
|
||||
"Some(400) Some(\"InvalidArgument\") Some(\"Object key contains invalid control characters: \\\"nul\\\\0key\\\"\")",
|
||||
),
|
||||
("1025-byte key", "Some(400) Some(\"KeyTooLongError\") Some(\"Your key is too long.\")"),
|
||||
(
|
||||
"empty part ETag",
|
||||
"Some(400) Some(\"InvalidPart\") Some(\"One or more of the specified parts could not be found. The part may not have been uploaded, or the specified entity tag may not match the part's entity tag.\")",
|
||||
),
|
||||
(
|
||||
"weak spelling of the stored part ETag",
|
||||
"200 bucket=Some(\"e1-complete\") key_len=Some(7) etag=Some(true) crc32=None sha256=None type=None",
|
||||
),
|
||||
(
|
||||
"unquoted stored part ETag",
|
||||
"200 bucket=Some(\"e1-complete\") key_len=Some(7) etag=Some(true) crc32=None sha256=None type=None",
|
||||
),
|
||||
(
|
||||
"quoted other part ETag",
|
||||
"Some(400) Some(\"InvalidPart\") Some(\"One or more of the specified parts could not be found. The part may not have been uploaded, or the specified entity tag may not match the part's entity tag.\")",
|
||||
),
|
||||
(
|
||||
"unquoted other part ETag",
|
||||
"Some(400) Some(\"InvalidPart\") Some(\"One or more of the specified parts could not be found. The part may not have been uploaded, or the specified entity tag may not match the part's entity tag.\")",
|
||||
),
|
||||
(
|
||||
"weak other part ETag",
|
||||
"Some(400) Some(\"InvalidPart\") Some(\"One or more of the specified parts could not be found. The part may not have been uploaded, or the specified entity tag may not match the part's entity tag.\")",
|
||||
),
|
||||
(
|
||||
"part without a number",
|
||||
"Some(400) Some(\"InvalidPart\") Some(\"Part number 0 must be between 1 and 10000\")",
|
||||
),
|
||||
(
|
||||
"two checksum headers",
|
||||
"200 bucket=Some(\"e1-complete\") key_len=Some(7) etag=Some(true) crc32=Some(\"AAAAAA==\") sha256=Some(\"47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=\") type=None",
|
||||
),
|
||||
(
|
||||
"checksum of the wrong width",
|
||||
"Some(400) Some(\"BadDigest\") Some(\"The Content-Md5 you specified did not match what we received.\")",
|
||||
),
|
||||
(
|
||||
"ordinary",
|
||||
"200 bucket=Some(\"e1-complete\") key_len=Some(7) etag=Some(true) crc32=Some(\"AAAAAA==\") sha256=None type=None",
|
||||
),
|
||||
(
|
||||
"upload id with a dot-dot segment",
|
||||
"Some(404) Some(\"NoSuchUpload\") Some(\"The specified multipart upload does not exist. The upload ID might be invalid, or the multipart upload might have been aborted or completed.\")",
|
||||
),
|
||||
(
|
||||
"upload id never minted",
|
||||
"Some(404) Some(\"NoSuchUpload\") Some(\"The specified multipart upload does not exist. The upload ID may be invalid, or the upload may have been aborted or completed.\")",
|
||||
),
|
||||
(
|
||||
"no body",
|
||||
"Some(400) Some(\"MalformedXML\") Some(\"This happens when the user sends malformed XML (XML that doesn't conform to the published XSD) for the configuration. The error message is, \\\"The XML you provided was not well-formed or did not validate against our published schema.\\\"\")",
|
||||
),
|
||||
];
|
||||
let mut expected: Vec<(String, String)> = expected
|
||||
.into_iter()
|
||||
.map(|(label, answer)| (label.to_owned(), answer.to_owned()))
|
||||
.collect();
|
||||
for (label, answer) in divergences {
|
||||
let row = expected
|
||||
.iter_mut()
|
||||
.find(|(row_label, _)| row_label == label)
|
||||
.unwrap_or_else(|| panic!("no row named {label}"));
|
||||
assert_ne!(row.1, *answer, "{label}: a divergence that equals the table is no divergence");
|
||||
row.1 = (*answer).to_owned();
|
||||
}
|
||||
assert_eq!(answered, expected);
|
||||
server.shutdown().await;
|
||||
}
|
||||
|
||||
/// The rows on which the gateway stack answers differently from the legacy stack, with the
|
||||
/// gateway's answer. Every other row is held to the legacy table above.
|
||||
const GATEWAY_COMPLETION_DIVERGENCES: &[(&str, &str)] = &[
|
||||
("NUL key", "Some(400) Some(\"InvalidArgument\") Some(\"the name contains a NUL byte\")"),
|
||||
(
|
||||
"1025-byte key",
|
||||
"Some(400) Some(\"KeyTooLongError\") Some(\"the name is longer than the maximum length in bytes\")",
|
||||
),
|
||||
(
|
||||
"empty part ETag",
|
||||
"Some(400) Some(\"InvalidArgument\") Some(\"the request body carries a value this gateway cannot carry exactly\")",
|
||||
),
|
||||
(
|
||||
"part without a number",
|
||||
"Some(400) Some(\"MalformedXML\") Some(\"the body omits a member the schema requires\")",
|
||||
),
|
||||
(
|
||||
"two checksum headers",
|
||||
"Some(400) Some(\"InvalidRequest\") Some(\"Expecting a single x-amz-checksum- header\")",
|
||||
),
|
||||
(
|
||||
"no body",
|
||||
"Some(400) Some(\"MalformedXML\") Some(\"the request body is not the XML this operation accepts\")",
|
||||
),
|
||||
];
|
||||
|
||||
/// CompleteMultipartUpload on the legacy stack answers every request shape as before its use case
|
||||
/// moved onto the gateway's types. Every answer was measured on `main` before the move.
|
||||
#[test]
|
||||
fn the_legacy_stack_completes_every_upload_as_before() {
|
||||
with_stack(LEGACY, |server| completes_every_upload(server, &[]));
|
||||
}
|
||||
with_stack(LEGACY, |server| async move {
|
||||
let client = s3_client(&server.endpoint(), server.access_key(), server.secret_key());
|
||||
client
|
||||
.create_bucket()
|
||||
.bucket(ORDINARY_BUCKET)
|
||||
.send()
|
||||
.await
|
||||
.expect("CreateBucket");
|
||||
for bucket in RESERVED_SHAPED_BUCKETS {
|
||||
create_bucket_in_the_store(bucket).await;
|
||||
}
|
||||
let mut answered = Vec::new();
|
||||
|
||||
#[test]
|
||||
fn the_gateway_stack_completes_every_upload() {
|
||||
with_stack(super::GATEWAY, |server| completes_every_upload(server, GATEWAY_COMPLETION_DIVERGENCES));
|
||||
for bucket in [ORDINARY_BUCKET].into_iter().chain(RESERVED_SHAPED_BUCKETS) {
|
||||
let (upload_id, etag) = one_part_upload(&client, bucket, "object").await;
|
||||
let answer = complete(&client, bucket, "object", &upload_id, part(Some(1), &etag), &[]).await;
|
||||
answered.push((format!("bucket {bucket}"), answer));
|
||||
}
|
||||
|
||||
let too_long = "k".repeat(1025);
|
||||
for (label, key) in [("NUL key", "nul\0key"), ("1025-byte key", too_long.as_str())] {
|
||||
let answer = complete(&client, ORDINARY_BUCKET, key, "no-such-upload", part(Some(1), "\"0\""), &[]).await;
|
||||
answered.push((label.to_owned(), answer));
|
||||
}
|
||||
|
||||
type ShapeCase<'a> = (&'a str, Option<i32>, fn(&str) -> String, &'a [(&'a str, &'a str)]);
|
||||
let shapes: [ShapeCase<'_>; 10] = [
|
||||
("empty part ETag", Some(1), |_| String::new(), &[]),
|
||||
("weak spelling of the stored part ETag", Some(1), |etag| format!("W/{etag}"), &[]),
|
||||
("unquoted stored part ETag", Some(1), |etag| etag.trim_matches('"').to_owned(), &[]),
|
||||
(
|
||||
"quoted other part ETag",
|
||||
Some(1),
|
||||
|_| "\"00000000000000000000000000000000\"".to_owned(),
|
||||
&[],
|
||||
),
|
||||
(
|
||||
"unquoted other part ETag",
|
||||
Some(1),
|
||||
|_| "00000000000000000000000000000000".to_owned(),
|
||||
&[],
|
||||
),
|
||||
(
|
||||
"weak other part ETag",
|
||||
Some(1),
|
||||
|_| "W/\"00000000000000000000000000000000\"".to_owned(),
|
||||
&[],
|
||||
),
|
||||
("part without a number", None, str::to_owned, &[]),
|
||||
(
|
||||
"two checksum headers",
|
||||
Some(1),
|
||||
str::to_owned,
|
||||
&[
|
||||
("crc32", "AAAAAA=="),
|
||||
("sha256", "47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU="),
|
||||
],
|
||||
),
|
||||
("checksum of the wrong width", Some(1), str::to_owned, &[("crc32", "AAAA")]),
|
||||
("ordinary", Some(1), str::to_owned, &[("crc32", "AAAAAA==")]),
|
||||
];
|
||||
for (index, (label, number, etag_for, checksums)) in shapes.into_iter().enumerate() {
|
||||
let key = format!("shape-{index}");
|
||||
let (upload_id, etag) = one_part_upload(&client, ORDINARY_BUCKET, &key).await;
|
||||
let answer = complete(&client, ORDINARY_BUCKET, &key, &upload_id, part(number, &etag_for(&etag)), checksums).await;
|
||||
answered.push((label.to_owned(), answer));
|
||||
}
|
||||
|
||||
for (label, upload_id) in [
|
||||
("upload id with a dot-dot segment", "a/../b"),
|
||||
("upload id never minted", "bm90LWFuLXVwbG9hZA"),
|
||||
] {
|
||||
let answer = complete(&client, ORDINARY_BUCKET, "object", upload_id, part(Some(1), "\"0\""), &[]).await;
|
||||
answered.push((label.to_owned(), answer));
|
||||
}
|
||||
|
||||
// No body at all: the legacy decoder refuses it before the operation.
|
||||
let (upload_id, _) = one_part_upload(&client, ORDINARY_BUCKET, "no-body").await;
|
||||
let answer = match client
|
||||
.complete_multipart_upload()
|
||||
.bucket(ORDINARY_BUCKET)
|
||||
.key("no-body")
|
||||
.upload_id(&upload_id)
|
||||
.send()
|
||||
.await
|
||||
{
|
||||
Ok(_) => "200".to_owned(),
|
||||
Err(error) => format!(
|
||||
"{:?} {:?} {:?}",
|
||||
error.raw_response().map(|response| response.status().as_u16()),
|
||||
error.code(),
|
||||
error.message()
|
||||
),
|
||||
};
|
||||
answered.push(("no body".to_owned(), answer));
|
||||
|
||||
let expected = [
|
||||
(
|
||||
"bucket e1-complete",
|
||||
"200 bucket=Some(\"e1-complete\") key_len=Some(6) etag=Some(true) crc32=None sha256=None type=None",
|
||||
),
|
||||
(
|
||||
"bucket sthree-e1",
|
||||
"200 bucket=Some(\"sthree-e1\") key_len=Some(6) etag=Some(true) crc32=None sha256=None type=None",
|
||||
),
|
||||
(
|
||||
"bucket e1-s3alias",
|
||||
"200 bucket=Some(\"e1-s3alias\") key_len=Some(6) etag=Some(true) crc32=None sha256=None type=None",
|
||||
),
|
||||
(
|
||||
"NUL key",
|
||||
"Some(400) Some(\"InvalidArgument\") Some(\"Object key contains invalid control characters: \\\"nul\\\\0key\\\"\")",
|
||||
),
|
||||
("1025-byte key", "Some(400) Some(\"KeyTooLongError\") Some(\"Your key is too long.\")"),
|
||||
(
|
||||
"empty part ETag",
|
||||
"Some(400) Some(\"InvalidPart\") Some(\"One or more of the specified parts could not be found. The part may not have been uploaded, or the specified entity tag may not match the part's entity tag.\")",
|
||||
),
|
||||
(
|
||||
"weak spelling of the stored part ETag",
|
||||
"200 bucket=Some(\"e1-complete\") key_len=Some(7) etag=Some(true) crc32=None sha256=None type=None",
|
||||
),
|
||||
(
|
||||
"unquoted stored part ETag",
|
||||
"200 bucket=Some(\"e1-complete\") key_len=Some(7) etag=Some(true) crc32=None sha256=None type=None",
|
||||
),
|
||||
(
|
||||
"quoted other part ETag",
|
||||
"Some(400) Some(\"InvalidPart\") Some(\"One or more of the specified parts could not be found. The part may not have been uploaded, or the specified entity tag may not match the part's entity tag.\")",
|
||||
),
|
||||
(
|
||||
"unquoted other part ETag",
|
||||
"Some(400) Some(\"InvalidPart\") Some(\"One or more of the specified parts could not be found. The part may not have been uploaded, or the specified entity tag may not match the part's entity tag.\")",
|
||||
),
|
||||
(
|
||||
"weak other part ETag",
|
||||
"Some(400) Some(\"InvalidPart\") Some(\"One or more of the specified parts could not be found. The part may not have been uploaded, or the specified entity tag may not match the part's entity tag.\")",
|
||||
),
|
||||
(
|
||||
"part without a number",
|
||||
"Some(400) Some(\"InvalidPart\") Some(\"Part number 0 must be between 1 and 10000\")",
|
||||
),
|
||||
(
|
||||
"two checksum headers",
|
||||
"200 bucket=Some(\"e1-complete\") key_len=Some(7) etag=Some(true) crc32=Some(\"AAAAAA==\") sha256=Some(\"47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=\") type=None",
|
||||
),
|
||||
(
|
||||
"checksum of the wrong width",
|
||||
"Some(400) Some(\"BadDigest\") Some(\"The Content-Md5 you specified did not match what we received.\")",
|
||||
),
|
||||
(
|
||||
"ordinary",
|
||||
"200 bucket=Some(\"e1-complete\") key_len=Some(7) etag=Some(true) crc32=Some(\"AAAAAA==\") sha256=None type=None",
|
||||
),
|
||||
(
|
||||
"upload id with a dot-dot segment",
|
||||
"Some(404) Some(\"NoSuchUpload\") Some(\"The specified multipart upload does not exist. The upload ID might be invalid, or the multipart upload might have been aborted or completed.\")",
|
||||
),
|
||||
(
|
||||
"upload id never minted",
|
||||
"Some(404) Some(\"NoSuchUpload\") Some(\"The specified multipart upload does not exist. The upload ID may be invalid, or the upload may have been aborted or completed.\")",
|
||||
),
|
||||
(
|
||||
"no body",
|
||||
"Some(400) Some(\"MalformedXML\") Some(\"This happens when the user sends malformed XML (XML that doesn't conform to the published XSD) for the configuration. The error message is, \\\"The XML you provided was not well-formed or did not validate against our published schema.\\\"\")",
|
||||
),
|
||||
];
|
||||
let expected: Vec<(String, String)> = expected
|
||||
.into_iter()
|
||||
.map(|(label, answer)| (label.to_owned(), answer.to_owned()))
|
||||
.collect();
|
||||
assert_eq!(answered, expected);
|
||||
server.shutdown().await;
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -791,13 +791,8 @@ fn the_legacy_stack_swaps_the_tagging_action_for_a_named_version() {
|
||||
with_stack(LEGACY, the_tagging_action_follows_the_named_version);
|
||||
}
|
||||
|
||||
/// What a stack answers for a key outside the floor, independent of the operation and bucket.
|
||||
type UniformAnswers = (Answer, Answer);
|
||||
|
||||
/// Checks boundary keys across bucket configurations. The legacy stack answers per operation and
|
||||
/// bucket from the table below; the gateway stack refuses the key before the bucket is consulted,
|
||||
/// so it answers every row with `uniform`: `(NUL key, 1025-byte key)`.
|
||||
async fn keys_outside_the_floor_are_refused(server: rustfs::embedded::RustFSServer, uniform: Option<UniformAnswers>) {
|
||||
/// Checks boundary keys across bucket configurations.
|
||||
async fn keys_outside_the_floor_are_refused(server: rustfs::embedded::RustFSServer) {
|
||||
const BUCKET: &str = "object-meta-keys";
|
||||
let client = s3_client(&server.endpoint(), server.access_key(), server.secret_key());
|
||||
create_bucket_in_the_store(BUCKET).await;
|
||||
@@ -822,23 +817,15 @@ async fn keys_outside_the_floor_are_refused(server: rustfs::embedded::RustFSServ
|
||||
("GetObjectRetention", _) => ("InternalError", 500, "Failed to load Object Lock configuration"),
|
||||
_ => ("InvalidArgument", 400, "Invalid argument"),
|
||||
};
|
||||
let expected = match &uniform {
|
||||
Some((nul, _)) => nul.clone(),
|
||||
None => (Some(code.to_owned()), Some(status), Some(message.to_owned())),
|
||||
};
|
||||
let expected = (Some(code.to_owned()), Some(status), Some(message.to_owned()));
|
||||
assert_eq!(call(&client, bucket, "a\0b").await, expected, "{operation} bucket {bucket}");
|
||||
}
|
||||
let expected = match &uniform {
|
||||
Some((_, long)) => long.clone(),
|
||||
None => (Some("KeyTooLongError".to_owned()), Some(400), Some("Your key is too long.".to_owned())),
|
||||
};
|
||||
let expected = (Some("KeyTooLongError".to_owned()), Some(400), Some("Your key is too long.".to_owned()));
|
||||
assert_eq!(call(&client, BUCKET, &long_key).await, expected, "{operation} long key");
|
||||
}
|
||||
server.shutdown().await;
|
||||
}
|
||||
|
||||
type Answer = (Option<String>, Option<u16>, Option<String>);
|
||||
|
||||
type KeyCall = for<'a> fn(&'a Client, &'a str, &'a str) -> BoxFuture<'a, (Option<String>, Option<u16>, Option<String>)>;
|
||||
|
||||
const KEY_CALLS: &[(&str, KeyCall)] = &[
|
||||
@@ -878,17 +865,7 @@ fn answer<T, E: ProvideErrorMetadata>(result: Result<T, SdkError<E>>) -> (Option
|
||||
|
||||
#[test]
|
||||
fn the_legacy_stack_refuses_keys_outside_the_floor() {
|
||||
with_stack(LEGACY, |server| keys_outside_the_floor_are_refused(server, None));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_gateway_stack_refuses_keys_outside_the_floor() {
|
||||
let refused = |code: &str, message: &str| (Some(code.to_owned()), Some(400), Some(message.to_owned()));
|
||||
let uniform = (
|
||||
refused("InvalidArgument", "the name contains a NUL byte"),
|
||||
refused("KeyTooLongError", "the name is longer than the maximum length in bytes"),
|
||||
);
|
||||
with_stack(GATEWAY, move |server| keys_outside_the_floor_are_refused(server, Some(uniform)));
|
||||
with_stack(LEGACY, keys_outside_the_floor_are_refused);
|
||||
}
|
||||
|
||||
fn legal_hold(status: &str) -> aws_sdk_s3::types::ObjectLockLegalHold {
|
||||
|
||||
@@ -100,7 +100,8 @@ async fn post_wire_observes_success_controls_and_storage(server: rustfs::embedde
|
||||
match status {
|
||||
200 | 204 => assert!(answer.body.is_empty(), "{key}: {}", answer.body),
|
||||
201 => {
|
||||
assert!(answer.body.contains(&format!("<Location>/{BUCKET}/{key}</Location>")), "{}", answer.body);
|
||||
let location = format!("{endpoint}/{BUCKET}/{key}");
|
||||
assert!(answer.body.contains(&format!("<Location>{location}</Location>")), "{}", answer.body);
|
||||
assert!(answer.body.contains(&format!("<Bucket>{BUCKET}</Bucket>")), "{}", answer.body);
|
||||
assert!(answer.body.contains(&format!("<Key>{key}</Key>")), "{}", answer.body);
|
||||
assert!(answer.body.contains(&format!("<ETag>{ETAG}</ETag>")), "{}", answer.body);
|
||||
@@ -229,6 +230,54 @@ fn gateway_post_wire_preserves_success_controls_and_storage() {
|
||||
stack::with_stack(stack::GATEWAY, |server| post_wire_observes_success_controls_and_storage(server, true));
|
||||
}
|
||||
|
||||
async fn post_response_location_is_absolute_and_percent_encoded(server: rustfs::embedded::RustFSServer) {
|
||||
const BUCKET: &str = "post-location-encoding";
|
||||
stack::create_bucket_in_the_store(BUCKET).await;
|
||||
let endpoint = server.endpoint();
|
||||
let client = WireClient {
|
||||
endpoint: &endpoint,
|
||||
access_key: server.access_key(),
|
||||
secret_key: server.secret_key(),
|
||||
};
|
||||
for (key, encoded_key) in [
|
||||
("test/a+b c%d.txt", "test/a%2Bb%20c%25d.txt"),
|
||||
("naïve/résumé.txt", "na%C3%AFve/r%C3%A9sum%C3%A9.txt"),
|
||||
("a&b?c#d.txt", "a%26b%3Fc%23d.txt"),
|
||||
("a-b_c.d~e/photo.jpg", "a-b_c.d~e/photo.jpg"),
|
||||
] {
|
||||
let answer = client
|
||||
.post_form(BUCKET, &[("key", key), ("success_action_status", "201")], &[], true, true)
|
||||
.await;
|
||||
assert_eq!(answer.status, 201, "{key}: {}", answer.body);
|
||||
let location = format!("{endpoint}/{BUCKET}/{encoded_key}");
|
||||
let xml_key = key.replace("&", "&");
|
||||
assert!(
|
||||
answer.body.contains(&format!("<Location>{location}</Location>")),
|
||||
"{key}: {}",
|
||||
answer.body
|
||||
);
|
||||
assert!(answer.body.contains(&format!("<Key>{xml_key}</Key>")), "{key}: {}", answer.body);
|
||||
assert!(
|
||||
!answer.body.contains(&format!("<Location>/{BUCKET}/{key}</Location>")),
|
||||
"{key}: {}",
|
||||
answer.body
|
||||
);
|
||||
}
|
||||
server.shutdown().await;
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial]
|
||||
fn gateway_post_response_location_is_absolute_and_percent_encoded() {
|
||||
stack::with_stack(stack::GATEWAY, post_response_location_is_absolute_and_percent_encoded);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial]
|
||||
fn legacy_post_response_location_is_absolute_and_percent_encoded() {
|
||||
stack::with_stack(stack::LEGACY, post_response_location_is_absolute_and_percent_encoded);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial]
|
||||
fn gateway_post_filename_substitution_supports_streamed_uploads() {
|
||||
|
||||
@@ -14,7 +14,6 @@
|
||||
|
||||
const GET_OBJECT: &str = include_str!("../src/app/object/get.rs");
|
||||
const PUT_OBJECT: &str = include_str!("../src/storage/legacy_edge/object_write.rs");
|
||||
const PUT_OBJECT_OWNED: &str = include_str!("../src/app/object/put.rs");
|
||||
const SELECT_OBJECT: &str = include_str!("../src/app/object/mod.rs");
|
||||
|
||||
fn assert_async_method_is_measured(source: &str, method: &str) {
|
||||
@@ -48,11 +47,6 @@ fn object_request_profiling_is_attached_to_the_matching_operation() {
|
||||
"the native entry must retain the established profiling label"
|
||||
);
|
||||
assert_async_method_is_measured(PUT_OBJECT, "execute_put_object");
|
||||
assert_async_method_is_measured(PUT_OBJECT_OWNED, "execute_put_object_owned");
|
||||
assert!(
|
||||
PUT_OBJECT_OWNED.contains("label = \"rustfs::app::object::put::DefaultObjectUsecase::execute_put_object\""),
|
||||
"the gateway write path must keep the established PutObject profiling label"
|
||||
);
|
||||
assert_async_method_is_measured(SELECT_OBJECT, "execute_select_object_content_owned");
|
||||
assert!(
|
||||
SELECT_OBJECT.contains("label = \"rustfs::app::object::DefaultObjectUsecase::execute_select_object_content\""),
|
||||
|
||||
@@ -2197,7 +2197,7 @@ fi
|
||||
rg -n --with-filename 'crate::app::context::(?:\{[^}]*\b(?:resolve_object_store_handle|resolve_buffer_config|resolve_internode_metrics|resolve_local_node_name|resolve_action_credentials|resolve_notify_interface|resolve_performance_metrics|resolve_encryption_service|resolve_region|resolve_ready_iam_handle)\b|(?:resolve_object_store_handle|resolve_buffer_config|resolve_internode_metrics|resolve_local_node_name|resolve_action_credentials|resolve_notify_interface|resolve_performance_metrics|resolve_encryption_service|resolve_region|resolve_ready_iam_handle)\b)' \
|
||||
rustfs/src/storage/access.rs \
|
||||
"${ECFS_GUARD_SOURCES[@]}" \
|
||||
rustfs/src/storage/s3_support.rs \
|
||||
rustfs/src/storage/ecfs_extend.rs \
|
||||
rustfs/src/storage/helper.rs \
|
||||
rustfs/src/storage/concurrency/manager.rs \
|
||||
rustfs/src/storage/sse.rs \
|
||||
@@ -4623,7 +4623,7 @@ fi
|
||||
rg -n --with-filename 'crate::storage::core_storage_compat' \
|
||||
rustfs/src/storage/mod.rs \
|
||||
"${ECFS_GUARD_SOURCES[@]}" \
|
||||
rustfs/src/storage/s3_support.rs \
|
||||
rustfs/src/storage/ecfs_extend.rs \
|
||||
rustfs/src/storage/access.rs \
|
||||
rustfs/src/storage/head_prefix.rs \
|
||||
rustfs/src/storage/options.rs \
|
||||
|
||||
@@ -20,6 +20,8 @@ cycle|infra<->interface
|
||||
dep|rustfs/src/app/admin_usecase.rs|app->interface|crate::server::collect_dependency_readiness_report
|
||||
dep|rustfs/src/cluster_snapshot.rs|infra->interface|crate::server::snapshot_dependency_readiness_report
|
||||
dep|rustfs/src/runtime_sources.rs|infra->app|crate::app::context
|
||||
dep|rustfs/src/storage/ecfs_extend.rs|infra->interface|crate::server::cors
|
||||
dep|rustfs/src/storage/ecfs_extend.rs|infra->interface|crate::storage::ecfs::ListObjectUnorderedQuery
|
||||
dep|rustfs/src/storage/helper.rs|infra->interface|crate::server::refresh_audit_module_enabled
|
||||
dep|rustfs/src/storage/helper.rs|infra->interface|crate::server::refresh_notify_module_enabled
|
||||
dep|rustfs/src/storage/rpc/http_service.rs|infra->interface|crate::server::RPC_PREFIX
|
||||
@@ -29,4 +31,3 @@ dep|rustfs/src/storage/rpc/node_service.rs|infra->interface|crate::admin::servic
|
||||
dep|rustfs/src/storage/rpc/node_service.rs|infra->interface|crate::admin::service::config::reload_runtime_config_snapshot
|
||||
dep|rustfs/src/storage/rpc/node_service.rs|infra->interface|crate::admin::service::site_replication::reload_site_replication_runtime_state
|
||||
dep|rustfs/src/storage/rpc/node_service.rs|infra->interface|crate::server::MODULE_SWITCHES_SIGNAL_SUBSYSTEM
|
||||
dep|rustfs/src/storage/s3_support.rs|infra->interface|crate::server::cors
|
||||
|
||||
Reference in New Issue
Block a user