mirror of
https://github.com/rustfs/rustfs.git
synced 2026-11-16 04:52:36 +00:00
feat(gateway): serve SelectObjectContent natively
Share GetObject authorization and the existing Select query producer across both stacks. Carry live events through the typed response with the projected SSE headers. The producer retains its single deadline, snapshot checks, bounded channel and receiver cancellation.
On c5b4a8577, the restored build passed 207 focused tests, both RustFS Clippy feature modes, formatting, script tests and test wiring. Signed socket probes matched all 17 legacy cases and 12 gateway cases. Five gateway refusals omit the legacy default message, tracked in rustfs/backlog#2684.
The fault build passed all 98 controls. Bypassing authorization failed on both stacks. Removing receiver-close handling, dropping the terminal Records permit, skipping the Stats fence, replacing an error with End, and dropping progress or scan range each failed the corresponding test.
Verification used local stores and SSE-C. The full workspace suite and external KMS were outside the local run.
This commit is contained in:
@@ -35,7 +35,7 @@ pub(crate) mod object_traffic_health;
|
||||
pub mod object_usecase;
|
||||
pub(crate) mod runtime_sources;
|
||||
mod select_object;
|
||||
mod select_request;
|
||||
pub(crate) mod select_request;
|
||||
pub(crate) mod storage_api;
|
||||
pub(crate) mod table_list_isolation;
|
||||
pub(crate) mod trailer_adapter;
|
||||
|
||||
@@ -145,6 +145,7 @@ use rustfs_audit::ObjectVersion as AuditObjectVersion;
|
||||
use rustfs_concurrency::GetObjectQueueSnapshot;
|
||||
use rustfs_config::MI_B;
|
||||
use rustfs_filemeta::{NULL_VERSION_ID, RestoreStatusOps, parse_restore_obj_status};
|
||||
use rustfs_gateway_types::dto::SelectObjectContentOutput;
|
||||
use rustfs_gateway_types::persistence::PersistedBucketEncryptionConfiguration as ServerSideEncryptionConfiguration;
|
||||
use rustfs_io_core::{BytesPool, PooledBuffer};
|
||||
use rustfs_io_metrics;
|
||||
@@ -152,6 +153,7 @@ use rustfs_notify::EventArgsBuilder;
|
||||
use rustfs_object_capacity::capacity_manager::get_capacity_manager;
|
||||
use rustfs_policy::policy::action::{Action, S3Action};
|
||||
use rustfs_s3_ops::{S3Operation, delete_event_name_for_marker, put_event_name_for_post_object};
|
||||
use rustfs_s3select_api::request::SelectObjectContentInput;
|
||||
use rustfs_targets::{EventName, get_request_host, get_request_port, get_request_user_agent};
|
||||
use rustfs_utils::CompressionAlgorithm;
|
||||
#[cfg(test)]
|
||||
@@ -178,8 +180,9 @@ use rustfs_utils::http::{
|
||||
use rustfs_utils::path::{encode_dir_object, is_dir_object, path_join_buf};
|
||||
use rustfs_utils::retry::{DEFAULT_RETRY_CAP, DEFAULT_RETRY_UNIT, MAX_JITTER, RetryTimer};
|
||||
use rustfs_zip::{ArchiveLimits, CompressionFormat, ZipError};
|
||||
use s3s::dto::{SelectObjectContentInput, SelectObjectContentOutput};
|
||||
use s3s::{S3Error, S3ErrorCode, S3Request, S3Response, S3Result};
|
||||
use s3s::{S3Error, S3ErrorCode, S3Result};
|
||||
#[cfg(test)]
|
||||
use s3s::{S3Request, S3Response};
|
||||
|
||||
fn object_s3_error(code: S3ErrorCode, message: impl Into<std::borrow::Cow<'static, str>>) -> S3Error {
|
||||
S3Error::with_message(code, message)
|
||||
@@ -382,16 +385,21 @@ impl DefaultObjectUsecase {
|
||||
map_quota_check_outcome(bucket, quota_checker.check_quota(bucket, op, size).await).map(Some)
|
||||
}
|
||||
|
||||
#[hotpath::measure(impl_type = "DefaultObjectUsecase", future = true)]
|
||||
#[instrument(level = "debug", skip(self, req))]
|
||||
pub async fn execute_select_object_content(
|
||||
#[instrument(name = "execute_select_object_content", level = "debug", skip(self, envelope, input))]
|
||||
#[hotpath::measure(
|
||||
impl_type = "DefaultObjectUsecase",
|
||||
future = true,
|
||||
label = "rustfs::app::object::DefaultObjectUsecase::execute_select_object_content"
|
||||
)]
|
||||
pub(crate) async fn execute_select_object_content_owned(
|
||||
&self,
|
||||
req: S3Request<SelectObjectContentInput>,
|
||||
) -> S3Result<S3Response<SelectObjectContentOutput>> {
|
||||
envelope: crate::app::RequestEnvelope,
|
||||
input: SelectObjectContentInput,
|
||||
) -> rustfs_s3_types::S3Result<crate::app::Answered<SelectObjectContentOutput>> {
|
||||
if let Some(context) = &self.context {
|
||||
let _ = context.object_store();
|
||||
}
|
||||
|
||||
crate::app::select_object::execute_select_object_content(req).await
|
||||
crate::app::select_object::execute_select_object_content(envelope, input).await
|
||||
}
|
||||
}
|
||||
|
||||
+223
-204
@@ -4,11 +4,11 @@ use super::storage_api::select_object::options::get_opts;
|
||||
use super::storage_api::select_object::request_context::spawn_traced;
|
||||
use super::storage_api::select_object::sse::{SseKmsPrincipal, authorize_sse_kms_object_read, project_sse_read_response_headers};
|
||||
use super::storage_api::select_object::{
|
||||
StoragePrepareSelectObjectSnapshotError, StorageSelectObjectSnapshot, get_validated_store, validate_sse_headers_for_read,
|
||||
validate_ssec_for_read,
|
||||
StoragePrepareSelectObjectSnapshotError, StorageSelectObjectSnapshot, get_validated_store_owned,
|
||||
validate_sse_headers_for_read, validate_ssec_for_read,
|
||||
};
|
||||
use crate::app::runtime_sources::current_s3select_db;
|
||||
use crate::app::select_request::select_input_from_s3s;
|
||||
use crate::app::{Answered, RequestEnvelope};
|
||||
use crate::error::ApiError;
|
||||
use bytes::{Bytes, BytesMut};
|
||||
use datafusion::arrow::{
|
||||
@@ -27,19 +27,22 @@ use datafusion::common::DataFusionError;
|
||||
use datafusion::physical_plan::SendableRecordBatchStream;
|
||||
use futures::StreamExt;
|
||||
use http::{HeaderMap, StatusCode, header::RANGE};
|
||||
use rustfs_gateway_types::dto::{
|
||||
ContinuationEvent, EndEvent, Progress, ProgressEvent, RecordsEvent,
|
||||
SelectObjectContentEventStream as SelectObjectContentEvent, SelectObjectContentOutput, Stats, StatsEvent,
|
||||
};
|
||||
use rustfs_gateway_types::{SelectEventError, SelectEventStream};
|
||||
use rustfs_s3_types::{S3Error, S3ErrorCode, S3Result};
|
||||
use rustfs_s3select_api::request::{
|
||||
CompressionType, CsvOutput, ExpressionType, FileHeaderInfo, InputSerialization, JsonInput, JsonOutput, JsonType,
|
||||
OutputSerialization, QuoteFields, SelectObjectContentInput, SelectObjectContentRequest,
|
||||
};
|
||||
use rustfs_s3select_api::{
|
||||
QueryError, SelectError, SelectInputMetrics,
|
||||
object_store::{INVALID_SCAN_RANGE_MESSAGE, is_noop_scan_range, validate_scan_range_bounds},
|
||||
query::{Context, Query},
|
||||
};
|
||||
use rustfs_s3select_query::instance::s3_select_query_timeout;
|
||||
use s3s::dto::{
|
||||
CSVOutput, CompressionType, ContinuationEvent, EndEvent, ExpressionType, FileHeaderInfo, InputSerialization, JSONInput,
|
||||
JSONOutput, JSONType, OutputSerialization, Progress, ProgressEvent, QuoteFields, RecordsEvent, SelectObjectContentEvent,
|
||||
SelectObjectContentEventStream, SelectObjectContentInput, SelectObjectContentOutput, SelectObjectContentRequest, Stats,
|
||||
StatsEvent,
|
||||
};
|
||||
use s3s::{S3Error, S3ErrorCode, S3Request, S3Response, S3Result};
|
||||
use std::{
|
||||
fmt,
|
||||
future::poll_fn,
|
||||
@@ -86,8 +89,8 @@ struct SelectValidation {
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
enum SelectOutputFormat {
|
||||
Csv(CSVOutput),
|
||||
Json(JSONOutput),
|
||||
Csv(CsvOutput),
|
||||
Json(JsonOutput),
|
||||
}
|
||||
|
||||
enum SelectProducerOutcome {
|
||||
@@ -116,27 +119,24 @@ impl SelectSnapshotFence for Arc<StorageSelectObjectSnapshot> {
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn execute_select_object_content(
|
||||
req: S3Request<SelectObjectContentInput>,
|
||||
) -> S3Result<S3Response<SelectObjectContentOutput>> {
|
||||
let read_principal = SseKmsPrincipal::from_request(&req);
|
||||
let mut input = req.input;
|
||||
let validation = validate_select_request(&req.headers, &mut input)?;
|
||||
pub(super) async fn execute_select_object_content(
|
||||
envelope: RequestEnvelope,
|
||||
mut input: SelectObjectContentInput,
|
||||
) -> S3Result<Answered<SelectObjectContentOutput>> {
|
||||
let read_principal = SseKmsPrincipal::from_request(&envelope);
|
||||
let validation = validate_select_request(envelope.headers(), &mut input)?;
|
||||
log_select_request_summary(&input, &validation);
|
||||
let query_timeout = s3_select_query_timeout();
|
||||
let query_deadline = Instant::now() + query_timeout;
|
||||
let engine_input = Arc::new(select_input_from_s3s(input.clone()).map_err(|err| {
|
||||
S3Error::with_message(S3ErrorCode::InternalError, format!("convert the SelectObjectContent request: {err}"))
|
||||
})?);
|
||||
let input = Arc::new(input);
|
||||
let db = timeout_at(query_deadline, current_s3select_db((*engine_input).clone(), false))
|
||||
let db = timeout_at(query_deadline, current_s3select_db((*input).clone(), false))
|
||||
.await
|
||||
.map_err(|_| select_query_timeout_error(query_timeout.as_secs()))?
|
||||
.map_err(map_query_error_to_s3)?;
|
||||
let admission = db.try_reserve_query().map_err(map_query_error_to_s3)?;
|
||||
let (snapshot, response_headers) = timeout_at(
|
||||
query_deadline,
|
||||
prepare_select_object_snapshot(&req.headers, &input, read_principal.as_ref()),
|
||||
prepare_select_object_snapshot(envelope.headers(), &input, read_principal.as_ref()),
|
||||
)
|
||||
.await
|
||||
.map_err(|_| select_query_timeout_error(query_timeout.as_secs()))??;
|
||||
@@ -146,8 +146,13 @@ pub async fn execute_select_object_content(
|
||||
return Err(map_select_error_to_s3(&SelectError::TruncatedInput));
|
||||
}
|
||||
let snapshot = Arc::new(snapshot);
|
||||
let query =
|
||||
Query::new_with_snapshot(Context { input: engine_input }, input.request.expression.clone(), Arc::clone(&snapshot));
|
||||
let query = Query::new_with_snapshot(
|
||||
Context {
|
||||
input: Arc::clone(&input),
|
||||
},
|
||||
input.request.expression.clone(),
|
||||
Arc::clone(&snapshot),
|
||||
);
|
||||
let query_handle = timeout_at(query_deadline, db.execute_admitted(&query, admission))
|
||||
.await
|
||||
.map_err(|_| select_query_timeout_error(query_timeout.as_secs()))?
|
||||
@@ -167,10 +172,6 @@ pub async fn execute_select_object_content(
|
||||
.clone()
|
||||
.try_reserve_owned()
|
||||
.map_err(|_| map_select_error_to_s3(&SelectError::InternalError))?;
|
||||
let mut response = S3Response::new(SelectObjectContentOutput {
|
||||
payload: Some(SelectObjectContentEventStream::new(ReceiverStream::new(rx))),
|
||||
});
|
||||
response.headers = response_headers;
|
||||
spawn_traced(async move {
|
||||
send_select_events_until_deadline(
|
||||
output,
|
||||
@@ -188,7 +189,19 @@ pub async fn execute_select_object_content(
|
||||
.await;
|
||||
});
|
||||
|
||||
Ok(response)
|
||||
Ok(Answered {
|
||||
output: select_output(rx),
|
||||
status: None,
|
||||
headers: response_headers,
|
||||
})
|
||||
}
|
||||
|
||||
fn select_output(rx: mpsc::Receiver<S3Result<SelectObjectContentEvent>>) -> SelectObjectContentOutput {
|
||||
let events = ReceiverStream::new(rx)
|
||||
.map(|item| item.map_err(|error| SelectEventError::new(error.code().as_str(), error.message().unwrap_or_default())));
|
||||
SelectObjectContentOutput {
|
||||
payload: Some(SelectEventStream::new(events)),
|
||||
}
|
||||
}
|
||||
|
||||
async fn send_select_events_until_deadline<L: SelectSnapshotFence>(
|
||||
@@ -604,20 +617,20 @@ fn validate_select_request(headers: &http::HeaderMap, input: &mut SelectObjectCo
|
||||
if input.request.expression.len() > MAX_SELECT_EXPRESSION_BYTES {
|
||||
return Err(S3Error::new(S3ErrorCode::ExpressionTooLong));
|
||||
}
|
||||
let Some(expression_type) = canonical_enum(input.request.expression_type.as_str(), &[ExpressionType::SQL]) else {
|
||||
if !input
|
||||
.request
|
||||
.expression_type
|
||||
.as_str()
|
||||
.eq_ignore_ascii_case(ExpressionType::SQL.as_str())
|
||||
{
|
||||
return Err(S3Error::new(S3ErrorCode::InvalidExpressionType));
|
||||
};
|
||||
input.request.expression_type = ExpressionType::from_static(expression_type);
|
||||
}
|
||||
input.request.expression_type = ExpressionType::SQL;
|
||||
|
||||
normalize_input_serialization(&mut input.request.input_serialization)?;
|
||||
let compressed_input = is_compressed_input(&input.request.input_serialization);
|
||||
if compressed_input
|
||||
&& input.request.scan_range.as_ref().is_some_and(|range| {
|
||||
is_noop_scan_range(&rustfs_s3select_api::request::ScanRange {
|
||||
start: range.start,
|
||||
end: range.end,
|
||||
})
|
||||
})
|
||||
&& input.request.scan_range.as_ref().is_some_and(is_noop_scan_range)
|
||||
&& !input
|
||||
.request
|
||||
.input_serialization
|
||||
@@ -656,16 +669,15 @@ fn is_compressed_input(input: &InputSerialization) -> bool {
|
||||
input
|
||||
.compression_type
|
||||
.as_ref()
|
||||
.is_some_and(|compression| compression.as_str() != CompressionType::NONE)
|
||||
.is_some_and(|compression| *compression != CompressionType::NONE)
|
||||
}
|
||||
|
||||
/// Matches an S3 enum value in any ASCII case, as MinIO does, and returns the
|
||||
/// canonical spelling that downstream code compares against.
|
||||
fn canonical_enum(value: &str, allowed: &[&'static str]) -> Option<&'static str> {
|
||||
fn canonical_enum<T>(value: &T, allowed: impl IntoIterator<Item = T>, as_str: impl Fn(&T) -> &str) -> Option<T> {
|
||||
allowed
|
||||
.iter()
|
||||
.copied()
|
||||
.find(|candidate| candidate.eq_ignore_ascii_case(value))
|
||||
.into_iter()
|
||||
.find(|candidate| as_str(candidate).eq_ignore_ascii_case(as_str(value)))
|
||||
}
|
||||
|
||||
fn normalize_input_serialization(input: &mut InputSerialization) -> S3Result<()> {
|
||||
@@ -680,8 +692,9 @@ fn normalize_input_serialization(input: &mut InputSerialization) -> S3Result<()>
|
||||
|
||||
if let Some(compression) = input.compression_type.as_mut() {
|
||||
let Some(canonical) = canonical_enum(
|
||||
compression.as_str(),
|
||||
&[CompressionType::NONE, CompressionType::GZIP, CompressionType::BZIP2],
|
||||
compression,
|
||||
[CompressionType::NONE, CompressionType::GZIP, CompressionType::BZIP2],
|
||||
CompressionType::as_str,
|
||||
) else {
|
||||
return Err(map_select_error_to_s3(&SelectError::InvalidCompressionFormat));
|
||||
};
|
||||
@@ -691,26 +704,23 @@ fn normalize_input_serialization(input: &mut InputSerialization) -> S3Result<()>
|
||||
INVALID_REQUEST_PARAMETER_MESSAGE,
|
||||
));
|
||||
}
|
||||
*compression = CompressionType::from_static(canonical);
|
||||
*compression = canonical;
|
||||
}
|
||||
input
|
||||
.compression_type
|
||||
.get_or_insert_with(|| CompressionType::from_static(CompressionType::NONE));
|
||||
input.compression_type.get_or_insert(CompressionType::NONE);
|
||||
|
||||
if let Some(csv) = input.csv.as_mut() {
|
||||
let file_header_info = csv
|
||||
.file_header_info
|
||||
.get_or_insert_with(|| FileHeaderInfo::from_static(FileHeaderInfo::NONE));
|
||||
let file_header_info = csv.file_header_info.get_or_insert(FileHeaderInfo::NONE);
|
||||
let Some(canonical) = canonical_enum(
|
||||
file_header_info.as_str(),
|
||||
&[FileHeaderInfo::NONE, FileHeaderInfo::USE, FileHeaderInfo::IGNORE],
|
||||
file_header_info,
|
||||
[FileHeaderInfo::NONE, FileHeaderInfo::USE, FileHeaderInfo::IGNORE],
|
||||
FileHeaderInfo::as_str,
|
||||
) else {
|
||||
return Err(S3Error::with_message(
|
||||
S3ErrorCode::InvalidFileHeaderInfo,
|
||||
"The FileHeaderInfo value is not valid. Only NONE, USE, and IGNORE are supported.",
|
||||
));
|
||||
};
|
||||
*file_header_info = FileHeaderInfo::from_static(canonical);
|
||||
*file_header_info = canonical;
|
||||
validate_single_byte(csv.comments.as_deref(), S3ErrorCode::InvalidRequestParameter)?;
|
||||
validate_single_character(csv.quote_character.as_deref())?;
|
||||
validate_single_character(csv.quote_escape_character.as_deref())?;
|
||||
@@ -719,11 +729,11 @@ fn normalize_input_serialization(input: &mut InputSerialization) -> S3Result<()>
|
||||
}
|
||||
|
||||
if let Some(json) = input.json.as_mut() {
|
||||
let json_type = json.type_.get_or_insert_with(|| JSONType::from_static(JSONType::LINES));
|
||||
let Some(canonical) = canonical_enum(json_type.as_str(), &[JSONType::DOCUMENT, JSONType::LINES]) else {
|
||||
let json_type = json.r#type.get_or_insert(JsonType::LINES);
|
||||
let Some(canonical) = canonical_enum(json_type, [JsonType::DOCUMENT, JsonType::LINES], JsonType::as_str) else {
|
||||
return Err(S3Error::new(S3ErrorCode::InvalidJsonType));
|
||||
};
|
||||
*json_type = JSONType::from_static(canonical);
|
||||
*json_type = canonical;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
@@ -744,10 +754,11 @@ fn normalize_output_serialization(output: &mut OutputSerialization) -> S3Result<
|
||||
validate_single_byte(csv.quote_escape_character.as_deref(), S3ErrorCode::InvalidRequestParameter)?;
|
||||
validate_output_record_delimiter(csv.record_delimiter.as_deref())?;
|
||||
if let Some(quote_fields) = csv.quote_fields.as_mut() {
|
||||
let Some(canonical) = canonical_enum(quote_fields.as_str(), &[QuoteFields::ALWAYS, QuoteFields::ASNEEDED]) else {
|
||||
let Some(canonical) = canonical_enum(quote_fields, [QuoteFields::ALWAYS, QuoteFields::ASNEEDED], QuoteFields::as_str)
|
||||
else {
|
||||
return Err(S3Error::new(S3ErrorCode::InvalidQuoteFields));
|
||||
};
|
||||
*quote_fields = QuoteFields::from_static(canonical);
|
||||
*quote_fields = canonical;
|
||||
}
|
||||
return Ok(SelectOutputFormat::Csv(csv.clone()));
|
||||
}
|
||||
@@ -863,7 +874,7 @@ async fn prepare_select_object_snapshot(
|
||||
let opts = get_opts(&input.bucket, &input.key, None, None, headers)
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
let store = get_validated_store(&input.bucket).await?;
|
||||
let store = get_validated_store_owned(&input.bucket).await?;
|
||||
let snapshot = store
|
||||
.prepare_select_object_snapshot(&input.bucket, &input.key, headers, &opts)
|
||||
.await
|
||||
@@ -941,7 +952,7 @@ impl SelectOutputEncoder {
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
fn encode_csv_batch(batch: &RecordBatch, config: &CSVOutput, buffer: &mut BytesMut) -> S3Result<()> {
|
||||
fn encode_csv_batch(batch: &RecordBatch, config: &CsvOutput, buffer: &mut BytesMut) -> S3Result<()> {
|
||||
if encode_csv_batch_limited(batch, 0..batch.num_rows(), config, buffer, usize::MAX)? == batch.num_rows() {
|
||||
Ok(())
|
||||
} else {
|
||||
@@ -952,7 +963,7 @@ fn encode_csv_batch(batch: &RecordBatch, config: &CSVOutput, buffer: &mut BytesM
|
||||
fn encode_csv_batch_limited(
|
||||
batch: &RecordBatch,
|
||||
rows: Range<usize>,
|
||||
config: &CSVOutput,
|
||||
config: &CsvOutput,
|
||||
buffer: &mut BytesMut,
|
||||
max_bytes: usize,
|
||||
) -> S3Result<usize> {
|
||||
@@ -965,7 +976,7 @@ fn encode_csv_batch_limited(
|
||||
let quote_all = config
|
||||
.quote_fields
|
||||
.as_ref()
|
||||
.is_some_and(|quote_fields| quote_fields.as_str() == QuoteFields::ALWAYS);
|
||||
.is_some_and(|quote_fields| *quote_fields == QuoteFields::ALWAYS);
|
||||
let mut output = LimitedBytesWriter::new(buffer, max_bytes);
|
||||
let mut field = BoundedText::default();
|
||||
let mut encoded_rows = 0;
|
||||
@@ -1431,7 +1442,7 @@ fn write_limited(output: &mut LimitedBytesWriter<'_>, bytes: &[u8]) -> bool {
|
||||
output.write_all(bytes).is_ok()
|
||||
}
|
||||
|
||||
fn json_record_delimiter(config: &JSONOutput) -> &[u8] {
|
||||
fn json_record_delimiter(config: &JsonOutput) -> &[u8] {
|
||||
if let Some(delimiter) = config.record_delimiter.as_deref() {
|
||||
delimiter.as_bytes()
|
||||
} else {
|
||||
@@ -1440,7 +1451,7 @@ fn json_record_delimiter(config: &JSONOutput) -> &[u8] {
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
fn encode_json_batch(batch: &RecordBatch, config: &JSONOutput, buffer: &mut BytesMut) -> S3Result<()> {
|
||||
fn encode_json_batch(batch: &RecordBatch, config: &JsonOutput, buffer: &mut BytesMut) -> S3Result<()> {
|
||||
if encode_json_batch_limited(batch, 0..batch.num_rows(), config, buffer, usize::MAX)? == batch.num_rows() {
|
||||
Ok(())
|
||||
} else {
|
||||
@@ -1451,7 +1462,7 @@ fn encode_json_batch(batch: &RecordBatch, config: &JSONOutput, buffer: &mut Byte
|
||||
fn encode_json_batch_limited(
|
||||
batch: &RecordBatch,
|
||||
rows: Range<usize>,
|
||||
config: &JSONOutput,
|
||||
config: &JsonOutput,
|
||||
buffer: &mut BytesMut,
|
||||
max_bytes: usize,
|
||||
) -> S3Result<usize> {
|
||||
@@ -1653,10 +1664,8 @@ fn select_query_timeout_error(seconds: u64) -> S3Error {
|
||||
map_query_error_to_s3(SelectError::QueryTimeout { seconds }.into())
|
||||
}
|
||||
|
||||
fn is_json_document(json: &JSONInput) -> bool {
|
||||
json.type_
|
||||
.as_ref()
|
||||
.is_some_and(|json_type| json_type.as_str() == JSONType::DOCUMENT)
|
||||
fn is_json_document(json: &JsonInput) -> bool {
|
||||
json.r#type.as_ref().is_some_and(|json_type| *json_type == JsonType::DOCUMENT)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -1677,8 +1686,9 @@ mod tests {
|
||||
physical_plan::stream::RecordBatchStreamAdapter,
|
||||
sql::sqlparser::parser::ParserError,
|
||||
};
|
||||
use rustfs_gateway_types::compat::s3s_5761ddfe::select::output_to_s3s;
|
||||
use rustfs_s3select_api::request::{CsvInput, ParquetInput, ScanRange};
|
||||
use rustfs_test_utils::TestECStoreEnv;
|
||||
use s3s::dto::{CSVInput, ParquetInput, ScanRange};
|
||||
|
||||
fn event_stream_headers(mut bytes: &[u8]) -> Vec<Vec<(String, String)>> {
|
||||
let mut messages = Vec::new();
|
||||
@@ -1717,7 +1727,9 @@ mod tests {
|
||||
}
|
||||
|
||||
async fn http_xml_error(error: S3Error) -> (StatusCode, String) {
|
||||
let response = error.to_http_response().expect("S3 error should serialize to HTTP");
|
||||
let response = s3s::S3Error::from(error)
|
||||
.to_http_response()
|
||||
.expect("S3 error should serialize to HTTP");
|
||||
let status = response.status();
|
||||
let body = http_body_util::BodyExt::collect(response.into_body())
|
||||
.await
|
||||
@@ -1817,18 +1829,18 @@ mod tests {
|
||||
let mut input = base_input();
|
||||
input.bucket = bucket.clone();
|
||||
input.key = "input.csv".to_owned();
|
||||
let request = S3Request {
|
||||
input,
|
||||
method: http::Method::POST,
|
||||
uri: format!("/{bucket}/input.csv?select&select-type=2").parse().expect("URI"),
|
||||
headers: HeaderMap::new(),
|
||||
extensions: http::Extensions::new(),
|
||||
credentials: None,
|
||||
region: None,
|
||||
service: None,
|
||||
trailing_headers: None,
|
||||
};
|
||||
let response = execute_select_object_content(request)
|
||||
let envelope = RequestEnvelope::from_parts(
|
||||
crate::app::RequestHead {
|
||||
method: http::Method::POST,
|
||||
uri: format!("/{bucket}/input.csv?select&select-type=2").parse().expect("URI"),
|
||||
headers: HeaderMap::new(),
|
||||
credentials: None,
|
||||
region: None,
|
||||
service: None,
|
||||
},
|
||||
http::Extensions::new(),
|
||||
);
|
||||
let response = execute_select_object_content(envelope, input)
|
||||
.await
|
||||
.expect("start actual SELECT response");
|
||||
// Keep the response alive without polling its event stream. The
|
||||
@@ -1875,15 +1887,15 @@ mod tests {
|
||||
sse_customer_key_md5: None,
|
||||
request: SelectObjectContentRequest {
|
||||
expression: "SELECT * FROM s3object".to_string(),
|
||||
expression_type: ExpressionType::from_static(ExpressionType::SQL),
|
||||
expression_type: ExpressionType::SQL,
|
||||
input_serialization: InputSerialization {
|
||||
csv: Some(CSVInput::default()),
|
||||
csv: Some(CsvInput::default()),
|
||||
compression_type: None,
|
||||
json: None,
|
||||
parquet: None,
|
||||
},
|
||||
output_serialization: OutputSerialization {
|
||||
csv: Some(CSVOutput::default()),
|
||||
csv: Some(CsvOutput::default()),
|
||||
json: None,
|
||||
},
|
||||
request_progress: None,
|
||||
@@ -1894,7 +1906,7 @@ mod tests {
|
||||
|
||||
fn csv_validation() -> SelectValidation {
|
||||
SelectValidation {
|
||||
output_format: SelectOutputFormat::Csv(CSVOutput::default()),
|
||||
output_format: SelectOutputFormat::Csv(CsvOutput::default()),
|
||||
progress_enabled: false,
|
||||
reports_input_metrics: true,
|
||||
}
|
||||
@@ -2062,9 +2074,7 @@ mod tests {
|
||||
StatusCode::BAD_REQUEST,
|
||||
),
|
||||
(
|
||||
SelectError::InvalidCompressionFormatForObject {
|
||||
compression: CompressionType::GZIP,
|
||||
},
|
||||
SelectError::InvalidCompressionFormatForObject { compression: "GZIP" },
|
||||
S3ErrorCode::InvalidCompressionFormat,
|
||||
StatusCode::BAD_REQUEST,
|
||||
),
|
||||
@@ -2279,7 +2289,7 @@ mod tests {
|
||||
let values = builder.finish();
|
||||
let schema = Arc::new(Schema::new(vec![Field::new("value", values.data_type().clone(), false)]));
|
||||
let batch = RecordBatch::try_new(schema, vec![Arc::new(values)]).expect("test record batch should be valid");
|
||||
let mut encoder = SelectOutputEncoder::new(SelectOutputFormat::Csv(CSVOutput::default()));
|
||||
let mut encoder = SelectOutputEncoder::new(SelectOutputFormat::Csv(CsvOutput::default()));
|
||||
let mut buffer = BytesMut::new();
|
||||
|
||||
let encoded_rows =
|
||||
@@ -2298,7 +2308,7 @@ mod tests {
|
||||
let values = DictionaryArray::<Int32Type>::try_new(keys, dictionary).expect("test dictionary should be valid");
|
||||
let schema = Arc::new(Schema::new(vec![Field::new("value", values.data_type().clone(), false)]));
|
||||
let batch = RecordBatch::try_new(schema, vec![Arc::new(values)]).expect("test record batch should be valid");
|
||||
let mut encoder = SelectOutputEncoder::new(SelectOutputFormat::Csv(CSVOutput::default()));
|
||||
let mut encoder = SelectOutputEncoder::new(SelectOutputFormat::Csv(CsvOutput::default()));
|
||||
let mut buffer = BytesMut::new();
|
||||
|
||||
let encoded_rows = encode_batch_turn(&mut encoder, &batch, 0, &mut buffer).expect("referenced empty values should batch");
|
||||
@@ -2318,7 +2328,7 @@ mod tests {
|
||||
DictionaryArray::<Int32Type>::try_new(Int32Array::from(keys), dictionary).expect("test dictionary should be valid");
|
||||
let schema = Arc::new(Schema::new(vec![Field::new("value", values.data_type().clone(), false)]));
|
||||
let batch = RecordBatch::try_new(schema, vec![Arc::new(values)]).expect("test record batch should be valid");
|
||||
let mut encoder = SelectOutputEncoder::new(SelectOutputFormat::Json(JSONOutput::default()));
|
||||
let mut encoder = SelectOutputEncoder::new(SelectOutputFormat::Json(JsonOutput::default()));
|
||||
let mut buffer = BytesMut::new();
|
||||
|
||||
let encoded_rows =
|
||||
@@ -2334,7 +2344,7 @@ mod tests {
|
||||
let values = StringArray::from(vec!["x"; row_count]);
|
||||
let schema = Arc::new(Schema::new(vec![Field::new("value", DataType::Utf8, false)]));
|
||||
let batch = RecordBatch::try_new(schema, vec![Arc::new(values)]).expect("test record batch should be valid");
|
||||
let mut encoder = SelectOutputEncoder::new(SelectOutputFormat::Csv(CSVOutput::default()));
|
||||
let mut encoder = SelectOutputEncoder::new(SelectOutputFormat::Csv(CsvOutput::default()));
|
||||
let mut buffer = BytesMut::new();
|
||||
|
||||
let encoded_rows =
|
||||
@@ -2350,7 +2360,7 @@ mod tests {
|
||||
let values = ListArray::from_iter_primitive::<Int32Type, _, _>([Some([Some(1)]), Some([Some(2)])]);
|
||||
let schema = Arc::new(Schema::new(vec![Field::new("items", values.data_type().clone(), false)]));
|
||||
let batch = RecordBatch::try_new(schema, vec![Arc::new(values)]).expect("test record batch should be valid");
|
||||
let mut encoder = SelectOutputEncoder::new(SelectOutputFormat::Json(JSONOutput::default()));
|
||||
let mut encoder = SelectOutputEncoder::new(SelectOutputFormat::Json(JsonOutput::default()));
|
||||
let mut buffer = BytesMut::new();
|
||||
|
||||
let encoded_rows = encode_batch_turn(&mut encoder, &batch, 0, &mut buffer).expect("small nested JSON rows should encode");
|
||||
@@ -2371,7 +2381,7 @@ mod tests {
|
||||
let values = builder.finish();
|
||||
let schema = Arc::new(Schema::new(vec![Field::new("items", values.data_type().clone(), false)]));
|
||||
let batch = RecordBatch::try_new(schema, vec![Arc::new(values)]).expect("test record batch should be valid");
|
||||
let mut encoder = SelectOutputEncoder::new(SelectOutputFormat::Json(JSONOutput::default()));
|
||||
let mut encoder = SelectOutputEncoder::new(SelectOutputFormat::Json(JsonOutput::default()));
|
||||
let mut buffer = BytesMut::new();
|
||||
|
||||
let encoded_rows = encode_batch_turn(&mut encoder, &batch, 0, &mut buffer).expect("the complete first row should encode");
|
||||
@@ -2393,7 +2403,7 @@ mod tests {
|
||||
let values = StringArray::from(vec!["ok", quoted.as_str()]);
|
||||
let schema = Arc::new(Schema::new(vec![Field::new("value", DataType::Utf8, false)]));
|
||||
let batch = RecordBatch::try_new(schema, vec![Arc::new(values)]).expect("test record batch should be valid");
|
||||
let mut encoder = SelectOutputEncoder::new(SelectOutputFormat::Csv(CSVOutput::default()));
|
||||
let mut encoder = SelectOutputEncoder::new(SelectOutputFormat::Csv(CsvOutput::default()));
|
||||
let mut buffer = BytesMut::new();
|
||||
|
||||
let encoded_rows =
|
||||
@@ -2413,7 +2423,7 @@ mod tests {
|
||||
]));
|
||||
let batch =
|
||||
RecordBatch::try_new(schema, vec![Arc::new(values), Arc::new(nested)]).expect("test record batch should be valid");
|
||||
let mut encoder = SelectOutputEncoder::new(SelectOutputFormat::Csv(CSVOutput::default()));
|
||||
let mut encoder = SelectOutputEncoder::new(SelectOutputFormat::Csv(CsvOutput::default()));
|
||||
let mut buffer = BytesMut::from(b"staged\n".as_slice());
|
||||
|
||||
let error = encode_batch_turn(&mut encoder, &batch, 0, &mut buffer)
|
||||
@@ -2434,7 +2444,7 @@ mod tests {
|
||||
let values = builder.finish();
|
||||
let schema = Arc::new(Schema::new(vec![Field::new("items", values.data_type().clone(), false)]));
|
||||
let batch = RecordBatch::try_new(schema, vec![Arc::new(values)]).expect("test record batch should be valid");
|
||||
let mut encoder = SelectOutputEncoder::new(SelectOutputFormat::Json(JSONOutput::default()));
|
||||
let mut encoder = SelectOutputEncoder::new(SelectOutputFormat::Json(JsonOutput::default()));
|
||||
let mut buffer = BytesMut::new();
|
||||
|
||||
let error = encode_batch_turn(&mut encoder, &batch, 0, &mut buffer)
|
||||
@@ -2472,7 +2482,7 @@ mod tests {
|
||||
]));
|
||||
let batch = RecordBatch::try_new(schema, vec![Arc::new(profile), Arc::new(labels), Arc::new(dictionary), Arc::new(run)])
|
||||
.expect("test record batch should be valid");
|
||||
let mut encoder = SelectOutputEncoder::new(SelectOutputFormat::Json(JSONOutput::default()));
|
||||
let mut encoder = SelectOutputEncoder::new(SelectOutputFormat::Json(JsonOutput::default()));
|
||||
let mut buffer = BytesMut::new();
|
||||
|
||||
assert_eq!(
|
||||
@@ -2490,7 +2500,7 @@ mod tests {
|
||||
let values = NullArray::new(1);
|
||||
let schema = Arc::new(Schema::new(vec![Field::new("value", DataType::Null, true)]));
|
||||
let batch = RecordBatch::try_new(schema, vec![Arc::new(values)]).expect("test record batch should be valid");
|
||||
let mut encoder = SelectOutputEncoder::new(SelectOutputFormat::Json(JSONOutput::default()));
|
||||
let mut encoder = SelectOutputEncoder::new(SelectOutputFormat::Json(JsonOutput::default()));
|
||||
let mut buffer = BytesMut::new();
|
||||
|
||||
assert_eq!(
|
||||
@@ -2507,7 +2517,7 @@ mod tests {
|
||||
let values = DictionaryArray::<Int32Type>::try_new(keys, dictionary).expect("test dictionary should be valid");
|
||||
let schema = Arc::new(Schema::new(vec![Field::new("value", values.data_type().clone(), true)]));
|
||||
let batch = RecordBatch::try_new(schema, vec![Arc::new(values)]).expect("test record batch should be valid");
|
||||
let mut encoder = SelectOutputEncoder::new(SelectOutputFormat::Json(JSONOutput::default()));
|
||||
let mut encoder = SelectOutputEncoder::new(SelectOutputFormat::Json(JsonOutput::default()));
|
||||
let mut buffer = BytesMut::new();
|
||||
|
||||
assert_eq!(
|
||||
@@ -2523,7 +2533,7 @@ mod tests {
|
||||
.expect("test dictionary should be valid");
|
||||
let schema = Arc::new(Schema::new(vec![Field::new("value", values.data_type().clone(), true)]));
|
||||
let batch = RecordBatch::try_new(schema, vec![Arc::new(values)]).expect("test record batch should be valid");
|
||||
let mut encoder = SelectOutputEncoder::new(SelectOutputFormat::Json(JSONOutput::default()));
|
||||
let mut encoder = SelectOutputEncoder::new(SelectOutputFormat::Json(JsonOutput::default()));
|
||||
let mut buffer = BytesMut::new();
|
||||
|
||||
assert_eq!(
|
||||
@@ -2539,7 +2549,7 @@ mod tests {
|
||||
let values = NullArray::new(1);
|
||||
let schema = Arc::new(Schema::new(vec![Field::new(field_name, DataType::Null, true)]));
|
||||
let batch = RecordBatch::try_new(schema, vec![Arc::new(values)]).expect("test record batch should be valid");
|
||||
let mut encoder = SelectOutputEncoder::new(SelectOutputFormat::Json(JSONOutput::default()));
|
||||
let mut encoder = SelectOutputEncoder::new(SelectOutputFormat::Json(JsonOutput::default()));
|
||||
let mut buffer = BytesMut::new();
|
||||
|
||||
let error = encode_batch_turn(&mut encoder, &batch, 0, &mut buffer)
|
||||
@@ -2556,7 +2566,7 @@ mod tests {
|
||||
let values = StructArray::new_null(vec![child].into(), 1);
|
||||
let schema = Arc::new(Schema::new(vec![Field::new("nested", values.data_type().clone(), true)]));
|
||||
let batch = RecordBatch::try_new(schema, vec![Arc::new(values)]).expect("test record batch should be valid");
|
||||
let mut encoder = SelectOutputEncoder::new(SelectOutputFormat::Json(JSONOutput::default()));
|
||||
let mut encoder = SelectOutputEncoder::new(SelectOutputFormat::Json(JsonOutput::default()));
|
||||
let mut buffer = BytesMut::new();
|
||||
|
||||
assert_eq!(
|
||||
@@ -2572,7 +2582,7 @@ mod tests {
|
||||
let values = StringArray::from(vec!["a", "b"]);
|
||||
let schema = Arc::new(Schema::new(vec![Field::new("value", DataType::Utf8, false)]));
|
||||
let batch = RecordBatch::try_new(schema, vec![Arc::new(values)]).expect("test record batch should be valid");
|
||||
let mut encoder = SelectOutputEncoder::new(SelectOutputFormat::Json(JSONOutput {
|
||||
let mut encoder = SelectOutputEncoder::new(SelectOutputFormat::Json(JsonOutput {
|
||||
record_delimiter: Some(delimiter.clone()),
|
||||
}));
|
||||
let mut buffer = BytesMut::new();
|
||||
@@ -2589,7 +2599,7 @@ mod tests {
|
||||
let values = StringArray::from(vec![value.as_str()]);
|
||||
let schema = Arc::new(Schema::new(vec![Field::new("value", DataType::Utf8, false)]));
|
||||
let batch = RecordBatch::try_new(schema, vec![Arc::new(values)]).expect("test record batch should be valid");
|
||||
let mut encoder = SelectOutputEncoder::new(SelectOutputFormat::Csv(CSVOutput::default()));
|
||||
let mut encoder = SelectOutputEncoder::new(SelectOutputFormat::Csv(CsvOutput::default()));
|
||||
let mut buffer = BytesMut::new();
|
||||
|
||||
let error =
|
||||
@@ -2610,7 +2620,7 @@ mod tests {
|
||||
.map(|_| Arc::clone(&values) as datafusion::arrow::array::ArrayRef)
|
||||
.collect::<Vec<_>>();
|
||||
let batch = RecordBatch::try_new(Arc::new(Schema::new(fields)), columns).expect("test record batch should be valid");
|
||||
let mut encoder = SelectOutputEncoder::new(SelectOutputFormat::Json(JSONOutput::default()));
|
||||
let mut encoder = SelectOutputEncoder::new(SelectOutputFormat::Json(JsonOutput::default()));
|
||||
let mut buffer = BytesMut::new();
|
||||
|
||||
let error = encode_batch_turn(&mut encoder, &batch, 0, &mut buffer)
|
||||
@@ -2627,7 +2637,7 @@ mod tests {
|
||||
let values = StringArray::from(vec![value.as_str()]);
|
||||
let schema = Arc::new(Schema::new(vec![Field::new("value", DataType::Utf8, false)]));
|
||||
let batch = RecordBatch::try_new(schema, vec![Arc::new(values)]).expect("test record batch should be valid");
|
||||
let mut encoder = SelectOutputEncoder::new(SelectOutputFormat::Csv(CSVOutput::default()));
|
||||
let mut encoder = SelectOutputEncoder::new(SelectOutputFormat::Csv(CsvOutput::default()));
|
||||
let mut buffer = BytesMut::new();
|
||||
|
||||
if excess == 0 {
|
||||
@@ -2653,7 +2663,7 @@ mod tests {
|
||||
let values = StringArray::from(vec![value.as_str()]);
|
||||
let schema = Arc::new(Schema::new(vec![Field::new("value", DataType::Utf8, false)]));
|
||||
let batch = RecordBatch::try_new(schema, vec![Arc::new(values)]).expect("test record batch should be valid");
|
||||
let mut encoder = SelectOutputEncoder::new(SelectOutputFormat::Json(JSONOutput::default()));
|
||||
let mut encoder = SelectOutputEncoder::new(SelectOutputFormat::Json(JsonOutput::default()));
|
||||
let mut buffer = BytesMut::new();
|
||||
|
||||
if excess == 0 {
|
||||
@@ -2681,7 +2691,7 @@ mod tests {
|
||||
let values = BinaryArray::from(vec![value.as_slice()]);
|
||||
let schema = Arc::new(Schema::new(vec![Field::new("value", DataType::Binary, false)]));
|
||||
let batch = RecordBatch::try_new(schema, vec![Arc::new(values)]).expect("test record batch should be valid");
|
||||
let mut encoder = SelectOutputEncoder::new(SelectOutputFormat::Json(JSONOutput::default()));
|
||||
let mut encoder = SelectOutputEncoder::new(SelectOutputFormat::Json(JsonOutput::default()));
|
||||
let mut buffer = BytesMut::new();
|
||||
|
||||
if excess == 0 {
|
||||
@@ -2715,7 +2725,7 @@ mod tests {
|
||||
let values = StringArray::from(vec![value.as_str()]);
|
||||
let schema = Arc::new(Schema::new(vec![Field::new("value", DataType::Utf8, false)]));
|
||||
let batch = RecordBatch::try_new(schema, vec![Arc::new(values)]).expect("test record batch should be valid");
|
||||
let mut encoder = SelectOutputEncoder::new(SelectOutputFormat::Json(JSONOutput::default()));
|
||||
let mut encoder = SelectOutputEncoder::new(SelectOutputFormat::Json(JsonOutput::default()));
|
||||
let mut buffer = BytesMut::new();
|
||||
|
||||
if excess == 0 {
|
||||
@@ -3108,7 +3118,10 @@ mod tests {
|
||||
let (producer, rx, lease_released) = spawn_test_producer(output, 4);
|
||||
producer.await.expect("producer should finish successfully");
|
||||
|
||||
let mut byte_stream = SelectObjectContentEventStream::new(ReceiverStream::new(rx)).into_byte_stream();
|
||||
let mut byte_stream = output_to_s3s(select_output(rx))
|
||||
.payload
|
||||
.expect("Select stream")
|
||||
.into_byte_stream();
|
||||
let mut encoded = Vec::new();
|
||||
while let Some(chunk) = byte_stream.next().await {
|
||||
encoded.extend_from_slice(&chunk.expect("event-stream message should serialize"));
|
||||
@@ -3234,7 +3247,10 @@ mod tests {
|
||||
let (producer, rx, lease_released) = spawn_test_producer(output, 2);
|
||||
producer.await.expect("producer should emit the terminal Select error");
|
||||
|
||||
let mut byte_stream = SelectObjectContentEventStream::new(ReceiverStream::new(rx)).into_byte_stream();
|
||||
let mut byte_stream = output_to_s3s(select_output(rx))
|
||||
.payload
|
||||
.expect("Select stream")
|
||||
.into_byte_stream();
|
||||
let mut encoded = Vec::new();
|
||||
while let Some(chunk) = byte_stream.next().await {
|
||||
encoded.extend_from_slice(&chunk.expect("event-stream message should serialize"));
|
||||
@@ -3278,7 +3294,7 @@ mod tests {
|
||||
assert!(sql_body.contains("<Message>"));
|
||||
|
||||
let mut input = base_input();
|
||||
input.request.input_serialization.compression_type = Some(CompressionType::from_static("SNAPPY"));
|
||||
input.request.input_serialization.compression_type = Some(CompressionType::custom("SNAPPY"));
|
||||
let compression_error =
|
||||
validate_select_request(&HeaderMap::new(), &mut input).expect_err("unknown compression must fail before streaming");
|
||||
let (compression_status, compression_body) = http_xml_error(compression_error).await;
|
||||
@@ -3314,7 +3330,10 @@ mod tests {
|
||||
.await
|
||||
.expect("producer should emit records followed by the terminal error");
|
||||
|
||||
let mut byte_stream = SelectObjectContentEventStream::new(ReceiverStream::new(rx)).into_byte_stream();
|
||||
let mut byte_stream = output_to_s3s(select_output(rx))
|
||||
.payload
|
||||
.expect("Select stream")
|
||||
.into_byte_stream();
|
||||
let mut encoded = Vec::new();
|
||||
while let Some(chunk) = byte_stream.next().await {
|
||||
encoded.extend_from_slice(&chunk.expect("event-stream message should serialize"));
|
||||
@@ -3378,39 +3397,45 @@ mod tests {
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn producer_drops_query_stream_and_snapshot_lease_when_receiver_closes() {
|
||||
let (stream_dropped_tx, stream_dropped_rx) = tokio::sync::oneshot::channel::<()>();
|
||||
let output = Box::pin(RecordBatchStreamAdapter::new(
|
||||
Arc::new(Schema::empty()),
|
||||
futures::stream::once(async move {
|
||||
let _stream_dropped = stream_dropped_tx;
|
||||
futures::future::pending::<Result<RecordBatch, DataFusionError>>().await
|
||||
}),
|
||||
));
|
||||
let (event_channel, rx) = test_event_channel(2);
|
||||
let (lease, lease_released) = lease_drop_signal();
|
||||
let producer = send_select_events_until_deadline(
|
||||
output,
|
||||
event_channel,
|
||||
csv_validation(),
|
||||
Arc::new(SelectInputMetrics::default()),
|
||||
Instant::now() + Duration::from_secs(300),
|
||||
300,
|
||||
lease,
|
||||
);
|
||||
tokio::pin!(producer);
|
||||
for legacy in [false, true] {
|
||||
let (stream_dropped_tx, stream_dropped_rx) = tokio::sync::oneshot::channel::<()>();
|
||||
let output = Box::pin(RecordBatchStreamAdapter::new(
|
||||
Arc::new(Schema::empty()),
|
||||
futures::stream::once(async move {
|
||||
let _stream_dropped = stream_dropped_tx;
|
||||
futures::future::pending::<Result<RecordBatch, DataFusionError>>().await
|
||||
}),
|
||||
));
|
||||
let (event_channel, rx) = test_event_channel(2);
|
||||
let (lease, lease_released) = lease_drop_signal();
|
||||
let producer = send_select_events_until_deadline(
|
||||
output,
|
||||
event_channel,
|
||||
csv_validation(),
|
||||
Arc::new(SelectInputMetrics::default()),
|
||||
Instant::now() + Duration::from_secs(300),
|
||||
300,
|
||||
lease,
|
||||
);
|
||||
tokio::pin!(producer);
|
||||
|
||||
assert!(futures::poll!(producer.as_mut()).is_pending());
|
||||
drop(rx);
|
||||
assert!(futures::poll!(producer.as_mut()).is_pending());
|
||||
if legacy {
|
||||
drop(output_to_s3s(select_output(rx)));
|
||||
} else {
|
||||
drop(select_output(rx));
|
||||
}
|
||||
|
||||
assert!(
|
||||
futures::poll!(producer.as_mut()).is_ready(),
|
||||
"producer should observe the closed receiver"
|
||||
);
|
||||
assert!(
|
||||
stream_dropped_rx.await.is_err(),
|
||||
"query stream should be dropped when the receiver closes"
|
||||
);
|
||||
assert!(lease_released.await.is_ok(), "receiver close should release the snapshot lease");
|
||||
assert!(
|
||||
futures::poll!(producer.as_mut()).is_ready(),
|
||||
"producer should observe the closed receiver"
|
||||
);
|
||||
assert!(
|
||||
stream_dropped_rx.await.is_err(),
|
||||
"query stream should be dropped when the receiver closes"
|
||||
);
|
||||
assert!(lease_released.await.is_ok(), "receiver close should release the snapshot lease");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
@@ -3535,26 +3560,17 @@ mod tests {
|
||||
.input_serialization
|
||||
.csv
|
||||
.as_ref()
|
||||
.and_then(|csv| csv.file_header_info.as_ref())
|
||||
.map(|value| value.as_str()),
|
||||
Some(FileHeaderInfo::NONE)
|
||||
);
|
||||
assert_eq!(
|
||||
input
|
||||
.request
|
||||
.input_serialization
|
||||
.compression_type
|
||||
.as_ref()
|
||||
.map(|value| value.as_str()),
|
||||
Some(CompressionType::NONE)
|
||||
.and_then(|csv| csv.file_header_info.as_ref()),
|
||||
Some(&FileHeaderInfo::NONE)
|
||||
);
|
||||
assert_eq!(input.request.input_serialization.compression_type.as_ref(), Some(&CompressionType::NONE));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_preserves_supported_compression_for_csv_and_json_lines() {
|
||||
for compression in [CompressionType::GZIP, CompressionType::BZIP2] {
|
||||
let mut csv_input = base_input();
|
||||
csv_input.request.input_serialization.compression_type = Some(CompressionType::from_static(compression));
|
||||
csv_input.request.input_serialization.compression_type = Some(compression.clone());
|
||||
validate_select_request(&HeaderMap::new(), &mut csv_input).expect("compressed CSV should be accepted");
|
||||
assert_eq!(
|
||||
csv_input
|
||||
@@ -3563,15 +3579,15 @@ mod tests {
|
||||
.compression_type
|
||||
.as_ref()
|
||||
.map(|value| value.as_str()),
|
||||
Some(compression)
|
||||
Some(compression.as_str())
|
||||
);
|
||||
|
||||
let mut json_input = base_input();
|
||||
json_input.request.input_serialization.csv = None;
|
||||
json_input.request.input_serialization.json = Some(JSONInput {
|
||||
type_: Some(JSONType::from_static(JSONType::LINES)),
|
||||
json_input.request.input_serialization.json = Some(JsonInput {
|
||||
r#type: Some(JsonType::LINES),
|
||||
});
|
||||
json_input.request.input_serialization.compression_type = Some(CompressionType::from_static(compression));
|
||||
json_input.request.input_serialization.compression_type = Some(compression.clone());
|
||||
validate_select_request(&HeaderMap::new(), &mut json_input).expect("compressed JSON LINES should be accepted");
|
||||
assert_eq!(
|
||||
json_input
|
||||
@@ -3580,7 +3596,7 @@ mod tests {
|
||||
.compression_type
|
||||
.as_ref()
|
||||
.map(|value| value.as_str()),
|
||||
Some(compression)
|
||||
Some(compression.as_str())
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -3590,7 +3606,7 @@ mod tests {
|
||||
let mut input = base_input();
|
||||
input.request.input_serialization.csv = None;
|
||||
input.request.input_serialization.parquet = Some(ParquetInput {});
|
||||
input.request.input_serialization.compression_type = Some(CompressionType::from_static(CompressionType::GZIP));
|
||||
input.request.input_serialization.compression_type = Some(CompressionType::GZIP);
|
||||
|
||||
let error = validate_select_request(&HeaderMap::new(), &mut input).expect_err("compressed Parquet must fail");
|
||||
|
||||
@@ -3601,7 +3617,7 @@ mod tests {
|
||||
#[test]
|
||||
fn validate_normalizes_noop_compressed_scan_range_and_rejects_real_ranges() {
|
||||
let mut noop = base_input();
|
||||
noop.request.input_serialization.compression_type = Some(CompressionType::from_static(CompressionType::GZIP));
|
||||
noop.request.input_serialization.compression_type = Some(CompressionType::GZIP);
|
||||
noop.request.scan_range = Some(ScanRange {
|
||||
start: Some(0),
|
||||
end: None,
|
||||
@@ -3610,7 +3626,7 @@ mod tests {
|
||||
assert!(noop.request.scan_range.is_none());
|
||||
|
||||
let mut ranged = base_input();
|
||||
ranged.request.input_serialization.compression_type = Some(CompressionType::from_static(CompressionType::GZIP));
|
||||
ranged.request.input_serialization.compression_type = Some(CompressionType::GZIP);
|
||||
ranged.request.scan_range = Some(ScanRange {
|
||||
start: Some(1),
|
||||
end: None,
|
||||
@@ -3622,7 +3638,7 @@ mod tests {
|
||||
assert_eq!(error.message(), Some("Scan range queries are not supported on this type of object."));
|
||||
|
||||
let mut malformed = base_input();
|
||||
malformed.request.input_serialization.compression_type = Some(CompressionType::from_static(CompressionType::GZIP));
|
||||
malformed.request.input_serialization.compression_type = Some(CompressionType::GZIP);
|
||||
malformed.request.scan_range = Some(ScanRange {
|
||||
start: Some(10),
|
||||
end: Some(1),
|
||||
@@ -3683,7 +3699,7 @@ mod tests {
|
||||
.as_mut()
|
||||
.expect("CSV input")
|
||||
.allow_quoted_record_delimiter = Some(true);
|
||||
input.request.input_serialization.compression_type = Some(CompressionType::from_static(compression));
|
||||
input.request.input_serialization.compression_type = Some(compression.clone());
|
||||
input.request.scan_range = Some(range);
|
||||
let error = validate_select_request(&HeaderMap::new(), &mut input)
|
||||
.expect_err("quoted record delimiters cannot be combined with ScanRange");
|
||||
@@ -3725,7 +3741,7 @@ mod tests {
|
||||
.csv
|
||||
.as_mut()
|
||||
.expect("base input should use CSV")
|
||||
.file_header_info = Some(FileHeaderInfo::from_static("INVALID"));
|
||||
.file_header_info = Some(FileHeaderInfo::custom("INVALID"));
|
||||
|
||||
let error = validate_select_request(&HeaderMap::new(), &mut input).expect_err("unknown header mode must fail");
|
||||
|
||||
@@ -3740,8 +3756,8 @@ mod tests {
|
||||
fn json_input(json_type: &str) -> SelectObjectContentInput {
|
||||
let mut input = base_input();
|
||||
input.request.input_serialization.csv = None;
|
||||
input.request.input_serialization.json = Some(JSONInput {
|
||||
type_: Some(JSONType::from(json_type.to_string())),
|
||||
input.request.input_serialization.json = Some(JsonInput {
|
||||
r#type: Some(JsonType::custom(json_type.to_string())),
|
||||
});
|
||||
input
|
||||
}
|
||||
@@ -3754,7 +3770,7 @@ mod tests {
|
||||
("none", CompressionType::NONE),
|
||||
] {
|
||||
let mut input = base_input();
|
||||
input.request.input_serialization.compression_type = Some(CompressionType::from(value.to_string()));
|
||||
input.request.input_serialization.compression_type = Some(CompressionType::custom(value.to_string()));
|
||||
validate_select_request(&HeaderMap::new(), &mut input).expect("CompressionType should match any case");
|
||||
assert_eq!(
|
||||
input
|
||||
@@ -3763,11 +3779,11 @@ mod tests {
|
||||
.compression_type
|
||||
.as_ref()
|
||||
.map(|v| v.as_str()),
|
||||
Some(expected)
|
||||
Some(expected.as_str())
|
||||
);
|
||||
}
|
||||
|
||||
for (value, expected) in [("lines", JSONType::LINES), ("Document", JSONType::DOCUMENT)] {
|
||||
for (value, expected) in [("lines", JsonType::LINES), ("Document", JsonType::DOCUMENT)] {
|
||||
let mut input = json_input(value);
|
||||
validate_select_request(&HeaderMap::new(), &mut input).expect("JSON Type should match any case");
|
||||
let json_type = input
|
||||
@@ -3775,8 +3791,8 @@ mod tests {
|
||||
.input_serialization
|
||||
.json
|
||||
.as_ref()
|
||||
.and_then(|json| json.type_.as_ref());
|
||||
assert_eq!(json_type.map(|v| v.as_str()), Some(expected));
|
||||
.and_then(|json| json.r#type.as_ref());
|
||||
assert_eq!(json_type.map(|v| v.as_str()), Some(expected.as_str()));
|
||||
}
|
||||
let mut input = json_input("document");
|
||||
validate_select_request(&HeaderMap::new(), &mut input).expect("lowercase DOCUMENT should be accepted");
|
||||
@@ -3794,10 +3810,10 @@ mod tests {
|
||||
.csv
|
||||
.as_mut()
|
||||
.expect("base input should use CSV")
|
||||
.file_header_info = Some(FileHeaderInfo::from(value.to_string()));
|
||||
.file_header_info = Some(FileHeaderInfo::custom(value.to_string()));
|
||||
validate_select_request(&HeaderMap::new(), &mut input).expect("FileHeaderInfo should match any case");
|
||||
let csv = input.request.input_serialization.csv.as_ref().expect("CSV input");
|
||||
assert_eq!(csv.file_header_info.as_ref().map(|v| v.as_str()), Some(expected));
|
||||
assert_eq!(csv.file_header_info.as_ref().map(|v| v.as_str()), Some(expected.as_str()));
|
||||
}
|
||||
|
||||
for (value, expected) in [("always", QuoteFields::ALWAYS), ("AsNeeded", QuoteFields::ASNEEDED)] {
|
||||
@@ -3808,25 +3824,25 @@ mod tests {
|
||||
.csv
|
||||
.as_mut()
|
||||
.expect("base output should use CSV")
|
||||
.quote_fields = Some(QuoteFields::from(value.to_string()));
|
||||
.quote_fields = Some(QuoteFields::custom(value.to_string()));
|
||||
let validation = validate_select_request(&HeaderMap::new(), &mut input).expect("QuoteFields should match any case");
|
||||
let SelectOutputFormat::Csv(csv) = validation.output_format else {
|
||||
panic!("CSV output expected");
|
||||
};
|
||||
assert_eq!(csv.quote_fields.as_ref().map(|v| v.as_str()), Some(expected));
|
||||
assert_eq!(csv.quote_fields.as_ref().map(|v| v.as_str()), Some(expected.as_str()));
|
||||
}
|
||||
|
||||
let mut input = base_input();
|
||||
input.request.expression_type = ExpressionType::from("sql".to_string());
|
||||
input.request.expression_type = ExpressionType::custom("sql".to_string());
|
||||
validate_select_request(&HeaderMap::new(), &mut input).expect("ExpressionType should match any case");
|
||||
assert_eq!(input.request.expression_type.as_str(), ExpressionType::SQL);
|
||||
assert_eq!(input.request.expression_type, ExpressionType::SQL);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_still_rejects_unknown_enum_values_in_any_case() {
|
||||
for value in ["gzipx", "snappy", " GZIP"] {
|
||||
let mut input = base_input();
|
||||
input.request.input_serialization.compression_type = Some(CompressionType::from(value.to_string()));
|
||||
input.request.input_serialization.compression_type = Some(CompressionType::custom(value.to_string()));
|
||||
let error = validate_select_request(&HeaderMap::new(), &mut input).expect_err("unknown compression must fail");
|
||||
assert_eq!(error.code(), &S3ErrorCode::InvalidCompressionFormat, "{value}");
|
||||
}
|
||||
@@ -3842,7 +3858,7 @@ mod tests {
|
||||
.csv
|
||||
.as_mut()
|
||||
.expect("base input should use CSV")
|
||||
.file_header_info = Some(FileHeaderInfo::from("used".to_string()));
|
||||
.file_header_info = Some(FileHeaderInfo::custom("used".to_string()));
|
||||
let error = validate_select_request(&HeaderMap::new(), &mut input).expect_err("unknown header mode must fail");
|
||||
assert_eq!(error.code(), &S3ErrorCode::InvalidFileHeaderInfo);
|
||||
|
||||
@@ -3853,12 +3869,12 @@ mod tests {
|
||||
.csv
|
||||
.as_mut()
|
||||
.expect("base output should use CSV")
|
||||
.quote_fields = Some(QuoteFields::from("never".to_string()));
|
||||
.quote_fields = Some(QuoteFields::custom("never".to_string()));
|
||||
let error = validate_select_request(&HeaderMap::new(), &mut input).expect_err("unknown QuoteFields must fail");
|
||||
assert_eq!(error.code(), &S3ErrorCode::InvalidQuoteFields);
|
||||
|
||||
let mut input = base_input();
|
||||
input.request.expression_type = ExpressionType::from("sqlite".to_string());
|
||||
input.request.expression_type = ExpressionType::custom("sqlite".to_string());
|
||||
let error = validate_select_request(&HeaderMap::new(), &mut input).expect_err("unknown ExpressionType must fail");
|
||||
assert_eq!(error.code(), &S3ErrorCode::InvalidExpressionType);
|
||||
}
|
||||
@@ -3989,7 +4005,7 @@ mod tests {
|
||||
writer.finish().expect("Arrow JSON reference should finish");
|
||||
}
|
||||
let mut actual = BytesMut::new();
|
||||
encode_json_batch(&batch, &JSONOutput::default(), &mut actual).expect("S3 Select JSON should encode");
|
||||
encode_json_batch(&batch, &JsonOutput::default(), &mut actual).expect("S3 Select JSON should encode");
|
||||
assert_eq!(actual.as_ref(), expected.as_slice(), "type: {}", batch.column(0).data_type());
|
||||
}
|
||||
|
||||
@@ -4010,7 +4026,7 @@ mod tests {
|
||||
.unwrap();
|
||||
|
||||
let mut bytes = BytesMut::new();
|
||||
encode_json_batch(&batch, &JSONOutput::default(), &mut bytes).unwrap();
|
||||
encode_json_batch(&batch, &JsonOutput::default(), &mut bytes).unwrap();
|
||||
assert_eq!(bytes.as_ref(), b"{\"name\":\"a\"}\n{\"name\":\"b\"}\n");
|
||||
}
|
||||
|
||||
@@ -4033,7 +4049,7 @@ mod tests {
|
||||
let mut bytes = BytesMut::new();
|
||||
encode_json_batch(
|
||||
&batch,
|
||||
&JSONOutput {
|
||||
&JsonOutput {
|
||||
record_delimiter: Some("|".to_string()),
|
||||
},
|
||||
&mut bytes,
|
||||
@@ -4088,7 +4104,7 @@ mod tests {
|
||||
let mut bytes = BytesMut::new();
|
||||
encode_csv_batch(
|
||||
&batch,
|
||||
&CSVOutput {
|
||||
&CsvOutput {
|
||||
field_delimiter: Some("|".to_string()),
|
||||
record_delimiter: Some("\r\n".to_string()),
|
||||
..Default::default()
|
||||
@@ -4108,9 +4124,9 @@ mod tests {
|
||||
|
||||
encode_csv_batch(
|
||||
&batch,
|
||||
&CSVOutput {
|
||||
&CsvOutput {
|
||||
quote_escape_character: Some("\\".to_string()),
|
||||
quote_fields: Some(QuoteFields::from_static(QuoteFields::ASNEEDED)),
|
||||
quote_fields: Some(QuoteFields::ASNEEDED),
|
||||
record_delimiter: Some("|".to_string()),
|
||||
..Default::default()
|
||||
},
|
||||
@@ -4140,10 +4156,10 @@ mod tests {
|
||||
|
||||
encode_csv_batch(
|
||||
&batch,
|
||||
&CSVOutput {
|
||||
&CsvOutput {
|
||||
quote_character: Some("'".to_string()),
|
||||
quote_escape_character: Some("\\".to_string()),
|
||||
quote_fields: Some(QuoteFields::from_static(QuoteFields::ALWAYS)),
|
||||
quote_fields: Some(QuoteFields::ALWAYS),
|
||||
record_delimiter: Some("|".to_string()),
|
||||
..Default::default()
|
||||
},
|
||||
@@ -4214,7 +4230,10 @@ mod tests {
|
||||
.expect("test channel should accept Records");
|
||||
drop(tx);
|
||||
|
||||
let mut byte_stream = SelectObjectContentEventStream::new(ReceiverStream::new(rx)).into_byte_stream();
|
||||
let mut byte_stream = output_to_s3s(select_output(rx))
|
||||
.payload
|
||||
.expect("Select stream")
|
||||
.into_byte_stream();
|
||||
let mut encoded = Vec::new();
|
||||
while let Some(chunk) = byte_stream.next().await {
|
||||
encoded.extend_from_slice(&chunk.expect("Records event should serialize"));
|
||||
@@ -4235,8 +4254,8 @@ mod tests {
|
||||
let mut input = base_input();
|
||||
input.request.input_serialization = InputSerialization {
|
||||
csv: None,
|
||||
json: Some(JSONInput {
|
||||
type_: Some(JSONType::from_static(JSONType::DOCUMENT)),
|
||||
json: Some(JsonInput {
|
||||
r#type: Some(JsonType::DOCUMENT),
|
||||
}),
|
||||
parquet: None,
|
||||
compression_type: None,
|
||||
@@ -4256,8 +4275,8 @@ mod tests {
|
||||
let mut input = base_input();
|
||||
input.request.input_serialization = InputSerialization {
|
||||
csv: None,
|
||||
json: Some(JSONInput {
|
||||
type_: Some(JSONType::from_static(JSONType::LINES)),
|
||||
json: Some(JsonInput {
|
||||
r#type: Some(JsonType::LINES),
|
||||
}),
|
||||
parquet: None,
|
||||
compression_type: None,
|
||||
|
||||
@@ -12,13 +12,11 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! SelectObjectContent requests across the legacy S3 edge.
|
||||
//! SelectObjectContent requests from either S3 edge.
|
||||
//!
|
||||
//! Responsible for: converting the s3s request the edge decoded into the
|
||||
//! request the query engine runs (rustfs/backlog#2746), through the gateway's
|
||||
//! s3s seam for the serialization shapes. Not responsible for: validating the
|
||||
//! request; the select use case does that before the engine is built.
|
||||
//! DELETE BY T4.1 (rustfs/backlog#2783) with the rest of the legacy edge.
|
||||
//! Both S3 edges pass the request in the shape the query engine reads.
|
||||
//! The select use case validates it before building the engine.
|
||||
//! The s3s conversion is deleted with the legacy edge (rustfs/backlog#2783).
|
||||
|
||||
use rustfs_gateway_types::compat::ConversionError;
|
||||
use rustfs_gateway_types::compat::s3s_5761ddfe::generated::shapes::input_serialization::input_serialization_from_s3s;
|
||||
@@ -69,6 +67,43 @@ pub(crate) fn select_input_from_s3s(
|
||||
})
|
||||
}
|
||||
|
||||
/// Moves the gateway's flat request into the shape the query engine reads.
|
||||
#[cfg(any(feature = "gateway-stack", test))]
|
||||
pub(crate) fn select_input_from_gateway(
|
||||
input: rustfs_gateway_types::dto::SelectObjectContentInput,
|
||||
) -> engine::SelectObjectContentInput {
|
||||
let rustfs_gateway_types::dto::SelectObjectContentInput {
|
||||
bucket,
|
||||
key,
|
||||
expected_bucket_owner,
|
||||
sse_customer_algorithm,
|
||||
sse_customer_key,
|
||||
sse_customer_key_md5,
|
||||
expression,
|
||||
expression_type,
|
||||
input_serialization,
|
||||
output_serialization,
|
||||
request_progress,
|
||||
scan_range,
|
||||
} = input;
|
||||
engine::SelectObjectContentInput {
|
||||
bucket: bucket.as_str().to_owned(),
|
||||
key: key.as_str().to_owned(),
|
||||
expected_bucket_owner,
|
||||
sse_customer_algorithm,
|
||||
sse_customer_key: sse_customer_key.map(|key| key.expose_secret().to_owned()),
|
||||
sse_customer_key_md5,
|
||||
request: engine::SelectObjectContentRequest {
|
||||
expression,
|
||||
expression_type,
|
||||
input_serialization,
|
||||
output_serialization,
|
||||
request_progress,
|
||||
scan_range,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::select_input_from_s3s;
|
||||
@@ -159,6 +194,31 @@ mod tests {
|
||||
assert_eq!((range.start, range.end), (Some(3), Some(9)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn gateway_projection_preserves_the_query_and_read_parameters() {
|
||||
let source = input(InputSerialization {
|
||||
csv: Some(CSVInput::default()),
|
||||
..Default::default()
|
||||
});
|
||||
let owned = rustfs_gateway_types::compat::s3s_5761ddfe::select::input_from_s3s(source, &Default::default())
|
||||
.expect("materialize the test request");
|
||||
let converted = super::select_input_from_gateway(owned);
|
||||
assert_eq!((converted.bucket.as_str(), converted.key.as_str()), ("bucket", "key"));
|
||||
assert_eq!(converted.expected_bucket_owner.as_deref(), Some("owner"));
|
||||
assert_eq!(converted.sse_customer_algorithm.as_deref(), Some("AES256"));
|
||||
assert_eq!(converted.sse_customer_key.as_deref(), Some("key-text"));
|
||||
assert_eq!(converted.sse_customer_key_md5.as_deref(), Some("client-md5"));
|
||||
let request = converted.request;
|
||||
assert_eq!(request.expression, "SELECT * FROM S3Object");
|
||||
assert_eq!(request.expression_type, super::engine::ExpressionType::SQL);
|
||||
assert!(request.input_serialization.csv.is_some());
|
||||
let csv = request.output_serialization.csv.expect("CSV output");
|
||||
assert_eq!(csv.field_delimiter.as_deref(), Some(";"));
|
||||
assert_eq!(request.request_progress.and_then(|progress| progress.enabled), Some(true));
|
||||
let range = request.scan_range.expect("scan range");
|
||||
assert_eq!((range.start, range.end), (Some(3), Some(9)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn keeps_the_json_type_and_an_unknown_expression_type() {
|
||||
let mut source = input(InputSerialization {
|
||||
|
||||
@@ -1902,8 +1902,8 @@ pub(crate) mod select_object {
|
||||
#[cfg(test)]
|
||||
pub(crate) use crate::storage::storage_api::StorageError;
|
||||
pub(crate) use crate::storage::storage_api::{
|
||||
StoragePrepareSelectObjectSnapshotError, StorageSelectObjectSnapshot, get_validated_store, validate_sse_headers_for_read,
|
||||
validate_ssec_for_read,
|
||||
StoragePrepareSelectObjectSnapshotError, StorageSelectObjectSnapshot, get_validated_store_owned,
|
||||
validate_sse_headers_for_read, validate_ssec_for_read,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -13,23 +13,29 @@
|
||||
// limitations under the License.
|
||||
|
||||
//! The gateway operations of migration batch E2 (rustfs/backlog#2765): DeleteObject, DeleteObjects, RestoreObject and SelectObjectContent.
|
||||
//! DeleteObject and RestoreObject are served here.
|
||||
//! DeleteObject, RestoreObject and SelectObjectContent are served here.
|
||||
|
||||
use super::{OpRow, Served, row};
|
||||
use super::{OpRow, Served, resp, row};
|
||||
use crate::app::RequestEnvelope;
|
||||
use crate::app::object_usecase::DefaultObjectUsecase;
|
||||
use crate::app::storage_api::object_usecase::bucket::{
|
||||
RestoreOutputLocation, RestoreRequest, RestoreS3Location, RestoreTag, RestoreTagging,
|
||||
};
|
||||
use crate::runtime_sources::AppContext;
|
||||
use crate::storage_api::server::http::access::access_owned::object_misc::{authorize_delete_object, authorize_restore_object};
|
||||
use crate::storage_api::server::http::access::access_owned::object_misc::{
|
||||
authorize_delete_object, authorize_restore_object, authorize_select_object_content,
|
||||
};
|
||||
use http::StatusCode;
|
||||
use rustfs_gateway::{Req, Resp, dto};
|
||||
use rustfs_s3_ops::S3Operation;
|
||||
use rustfs_s3_types::{S3Result, s3_error};
|
||||
use std::sync::Arc;
|
||||
|
||||
pub(super) const ROWS: &[OpRow] = &[row::<dto::DeleteObject>(), row::<dto::RestoreObject>()];
|
||||
pub(super) const ROWS: &[OpRow] = &[
|
||||
row::<dto::DeleteObject>(),
|
||||
row::<dto::RestoreObject>(),
|
||||
row::<dto::SelectObjectContent>(),
|
||||
];
|
||||
|
||||
impl Served for dto::DeleteObject {
|
||||
const METRIC: S3Operation = S3Operation::DeleteObject;
|
||||
@@ -119,3 +125,20 @@ fn restore_request(request: dto::RestoreRequest) -> RestoreRequest {
|
||||
type_: request.r#type.map(|value| value.as_str().to_owned()),
|
||||
}
|
||||
}
|
||||
|
||||
impl Served for dto::SelectObjectContent {
|
||||
const METRIC: S3Operation = S3Operation::SelectObjectContent;
|
||||
|
||||
async fn authorize(envelope: &mut RequestEnvelope, request: &Req<Self>) -> S3Result<()> {
|
||||
let input = request.input();
|
||||
Ok(authorize_select_object_content(envelope, input.bucket.as_str(), input.key.as_str()).await?)
|
||||
}
|
||||
|
||||
async fn execute(context: Option<Arc<AppContext>>, envelope: RequestEnvelope, request: Req<Self>) -> S3Result<Resp<Self>> {
|
||||
let input = crate::app::select_request::select_input_from_gateway(request.into_input());
|
||||
let answered = DefaultObjectUsecase::with_context(context)
|
||||
.execute_select_object_content_owned(envelope, input)
|
||||
.await?;
|
||||
Ok(resp::<Self>(answered))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4279,11 +4279,9 @@ impl S3Access for FS {
|
||||
///
|
||||
/// This method returns `Ok(())` by default.
|
||||
async fn select_object_content(&self, req: &mut S3Request<SelectObjectContentInput>) -> S3Result<()> {
|
||||
let req_info = ext_req_info_mut(&mut req.extensions)?;
|
||||
req_info.bucket = Some(req.input.bucket.clone());
|
||||
req_info.object = Some(req.input.key.clone());
|
||||
|
||||
authorize_request(req, Action::S3Action(S3Action::GetObjectAction)).await
|
||||
let bucket = req.input.bucket.clone();
|
||||
let key = req.input.key.clone();
|
||||
access_owned::object_misc::authorize_select_object_content(req, &bucket, &key).await
|
||||
}
|
||||
|
||||
/// Checks whether the UploadPart request has accesses to the resources.
|
||||
|
||||
@@ -144,6 +144,18 @@ async fn decide_delete_object<R: AuthorizationRequest + ?Sized>(
|
||||
Ok(PendingDeleteBucketGenerationGuard::from_result(bucket_generation))
|
||||
}
|
||||
|
||||
/// Select reads the current object and needs the same GetObject permission on either edge.
|
||||
pub(crate) async fn authorize_select_object_content<R: AuthorizationRequest + ?Sized>(
|
||||
req: &mut R,
|
||||
bucket: &str,
|
||||
key: &str,
|
||||
) -> S3Result<()> {
|
||||
let req_info = req_info_mut(req)?;
|
||||
req_info.bucket = Some(bucket.to_owned());
|
||||
req_info.object = Some(key.to_owned());
|
||||
authorize_request(req, Action::S3Action(S3Action::GetObjectAction)).await
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[path = "object_misc_test.rs"]
|
||||
mod tests;
|
||||
|
||||
@@ -894,11 +894,15 @@ pub fn invalidate_bucket_validation_cache(bucket: &str) {
|
||||
/// Uses adaptive cache with 5s TTL to avoid repeated stat_volume() calls.
|
||||
/// Returns store directly on cache hit without calling get_bucket_info().
|
||||
pub(crate) async fn get_validated_store(bucket: &str) -> S3Result<Arc<super::ECStore>> {
|
||||
Ok(get_validated_store_owned(bucket).await?)
|
||||
}
|
||||
|
||||
pub(crate) async fn get_validated_store_owned(bucket: &str) -> rustfs_s3_types::S3Result<Arc<super::ECStore>> {
|
||||
let Some(store) = runtime_sources::current_object_store_handle() else {
|
||||
return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()));
|
||||
return Err(rustfs_s3_types::s3_error!(InternalError, "Not init"));
|
||||
};
|
||||
|
||||
validate_bucket_exists(&store, bucket).await?;
|
||||
validate_bucket_exists_owned(&store, bucket).await?;
|
||||
|
||||
Ok(store)
|
||||
}
|
||||
@@ -908,6 +912,10 @@ pub(crate) async fn get_validated_store(bucket: &str) -> S3Result<Arc<super::ECS
|
||||
/// store through the request-bound server context (backlog#1052 S6) use this
|
||||
/// to keep bucket validation on that store instead of the process-global one.
|
||||
pub(crate) async fn validate_bucket_exists(store: &super::ECStore, bucket: &str) -> S3Result<()> {
|
||||
Ok(validate_bucket_exists_owned(store, bucket).await?)
|
||||
}
|
||||
|
||||
async fn validate_bucket_exists_owned(store: &super::ECStore, bucket: &str) -> rustfs_s3_types::S3Result<()> {
|
||||
// Check cache — TTL is checked manually.
|
||||
if let Some(inserted_at) = cache_get(bucket)
|
||||
&& inserted_at.elapsed() < BUCKET_VALIDATION_TTL
|
||||
|
||||
@@ -224,6 +224,23 @@ impl FS {
|
||||
}
|
||||
|
||||
impl DefaultObjectUsecase {
|
||||
/// Projects a legacy SelectObjectContent request onto the shared Select query path.
|
||||
pub async fn execute_select_object_content(
|
||||
&self,
|
||||
req: S3Request<SelectObjectContentInput>,
|
||||
) -> S3Result<S3Response<SelectObjectContentOutput>> {
|
||||
let envelope = RequestEnvelope::from_s3s(&req);
|
||||
let input = crate::app::select_request::select_input_from_s3s(req.input).map_err(|err| {
|
||||
S3Error::with_message(S3ErrorCode::InternalError, format!("convert the SelectObjectContent request: {err}"))
|
||||
})?;
|
||||
let answer = self.execute_select_object_content_owned(envelope, input).await?;
|
||||
let output = rustfs_gateway_types::compat::s3s_5761ddfe::select::output_to_s3s(answer.output);
|
||||
let mut response = S3Response::new(output);
|
||||
response.status = answer.status;
|
||||
response.headers = answer.headers;
|
||||
Ok(response)
|
||||
}
|
||||
|
||||
/// Projects a legacy RestoreObject request onto the shared restore path.
|
||||
pub async fn execute_restore_object(&self, req: S3Request<RestoreObjectInput>) -> S3Result<S3Response<RestoreObjectOutput>> {
|
||||
let envelope = RequestEnvelope::from_s3s(&req);
|
||||
|
||||
@@ -136,9 +136,9 @@ pub(crate) type StorageObjectToDelete = contract::object::ObjectToDelete;
|
||||
pub(crate) type StoragePutObjReader = super::PutObjReader;
|
||||
pub(crate) use super::ecfs_extend::{
|
||||
RFC1123, apply_bucket_default_lock_retention, apply_cors_headers, check_preconditions, evaluate_read_preconditions,
|
||||
get_buffer_size_opt_in, get_validated_store, has_read_preconditions, load_bucket_object_lock_config_state,
|
||||
parse_object_lock_legal_hold_native, parse_object_lock_retention_native, parse_part_number_i32_to_usize,
|
||||
process_lambda_configurations, process_queue_configurations, process_topic_configurations,
|
||||
get_buffer_size_opt_in, get_validated_store, get_validated_store_owned, has_read_preconditions,
|
||||
load_bucket_object_lock_config_state, parse_object_lock_legal_hold_native, parse_object_lock_retention_native,
|
||||
parse_part_number_i32_to_usize, process_lambda_configurations, process_queue_configurations, process_topic_configurations,
|
||||
remove_object_lock_metadata_for_copy, validate_bucket_exists, validate_bucket_object_lock_enabled_state,
|
||||
validate_list_object_unordered_with_delimiter, validate_new_object_key, validate_object_key,
|
||||
};
|
||||
|
||||
@@ -361,6 +361,7 @@ async fn every_served_operation_serves_root_and_refuses_the_callers_it_cannot_ad
|
||||
create_bucket_in_the_store(object_meta::MATRIX_BUCKET).await;
|
||||
create_bucket_in_the_store(object_misc::MATRIX_BUCKET).await;
|
||||
object_misc::seed_restore_object(object_misc::MATRIX_BUCKET, false).await;
|
||||
object_misc::seed_select_matrix().await;
|
||||
create_bucket_in_the_store(object_read::MATRIX_BUCKET).await;
|
||||
create_bucket_in_the_store(object_commit::COPY_MATRIX_BUCKET).await;
|
||||
put_object_in_the_store(object_commit::COPY_MATRIX_BUCKET, object_commit::COPY_MATRIX_SOURCE).await;
|
||||
@@ -410,6 +411,7 @@ async fn every_served_operation_serves_root_and_refuses_the_callers_it_cannot_ad
|
||||
| S3Operation::PutBucketTagging
|
||||
| S3Operation::DeleteBucketEncryption
|
||||
| S3Operation::DeleteBucketTagging
|
||||
| S3Operation::SelectObjectContent
|
||||
| S3Operation::RestoreObject
|
||||
) {
|
||||
"unknown"
|
||||
|
||||
@@ -22,8 +22,14 @@ use super::{
|
||||
use aws_sdk_s3::Client;
|
||||
use aws_sdk_s3::error::ProvideErrorMetadata as _;
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::{BucketVersioningStatus, RestoreRequest, VersioningConfiguration};
|
||||
use aws_sdk_s3::types::{
|
||||
BucketVersioningStatus, CsvInput, CsvOutput, ExpressionType, FileHeaderInfo, InputSerialization, OutputSerialization,
|
||||
RestoreRequest, VersioningConfiguration,
|
||||
};
|
||||
use bytes::Bytes;
|
||||
use futures::future::BoxFuture;
|
||||
use http_body_util::{BodyExt as _, Full};
|
||||
use hyper_util::rt::TokioIo;
|
||||
use rustfs::app::context::get_global_app_context;
|
||||
use rustfs::embedded::RustFSServer;
|
||||
use rustfs_ecstore::api::error::is_err_object_not_found;
|
||||
@@ -46,6 +52,12 @@ pub(crate) const MATRIX: &[MatrixRow] = &[
|
||||
has_error_body: true,
|
||||
call: restore_object,
|
||||
},
|
||||
MatrixRow {
|
||||
operation: "SelectObjectContent",
|
||||
metric: S3Operation::SelectObjectContent,
|
||||
has_error_body: true,
|
||||
call: select_object,
|
||||
},
|
||||
];
|
||||
|
||||
pub(super) const MATRIX_BUCKET: &str = "e2-delete-matrix";
|
||||
@@ -466,3 +478,217 @@ fn the_legacy_stack_refuses_then_deletes_an_object() {
|
||||
delete_object_refuses_then_deletes(server, Some("Object key contains invalid control characters: \"bad\\0key\""))
|
||||
});
|
||||
}
|
||||
|
||||
const SELECT_BUCKET: &str = "e2-select-object";
|
||||
const SELECT_FIXTURE_KEY: &str = "input.csv";
|
||||
const SELECT_FIXTURE: &[u8] = b"value\nold\n";
|
||||
const SELECT_REQUEST: &str = concat!(
|
||||
"<SelectObjectContentRequest><Expression>SELECT * FROM S3Object</Expression><ExpressionType>SQL</ExpressionType>",
|
||||
"<InputSerialization><CSV><FileHeaderInfo>USE</FileHeaderInfo></CSV></InputSerialization>",
|
||||
"<OutputSerialization><CSV/></OutputSerialization></SelectObjectContentRequest>"
|
||||
);
|
||||
const SELECT_SSE_ALGORITHM: &str = "x-amz-server-side-encryption-customer-algorithm";
|
||||
const SELECT_SSE_KEY: &str = "x-amz-server-side-encryption-customer-key";
|
||||
const SELECT_SSE_MD5: &str = "x-amz-server-side-encryption-customer-key-md5";
|
||||
|
||||
pub(super) async fn seed_select_matrix() {
|
||||
let store = get_global_app_context().expect("embedded server context").object_store();
|
||||
store
|
||||
.put_object(
|
||||
MATRIX_BUCKET,
|
||||
SELECT_FIXTURE_KEY,
|
||||
&mut PutObjReader::from_vec(SELECT_FIXTURE.to_vec()),
|
||||
&ObjectOptions::default(),
|
||||
)
|
||||
.await
|
||||
.expect("store the matrix CSV");
|
||||
}
|
||||
|
||||
fn select_object<'a>(client: &'a Client, _bucket: &'a str) -> BoxFuture<'a, Outcome> {
|
||||
Box::pin(async move {
|
||||
outcome(
|
||||
client
|
||||
.select_object_content()
|
||||
.bucket(MATRIX_BUCKET)
|
||||
.key(SELECT_FIXTURE_KEY)
|
||||
.expression("SELECT * FROM S3Object")
|
||||
.expression_type(ExpressionType::Sql)
|
||||
.input_serialization(
|
||||
InputSerialization::builder()
|
||||
.csv(CsvInput::builder().file_header_info(FileHeaderInfo::Use).build())
|
||||
.build(),
|
||||
)
|
||||
.output_serialization(OutputSerialization::builder().csv(CsvOutput::builder().build()).build())
|
||||
.send()
|
||||
.await,
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
async fn select_wire(
|
||||
server: &RustFSServer,
|
||||
key: &str,
|
||||
xml: &str,
|
||||
headers: &[(&str, &str)],
|
||||
caller: (&str, &str),
|
||||
) -> (http::response::Parts, Bytes) {
|
||||
let path = format!("/{SELECT_BUCKET}/{key}?select&select-type=2");
|
||||
let uri: http::Uri = format!("{}{path}", server.endpoint()).parse().expect("Select URI");
|
||||
let mut request = http::Request::post(&uri)
|
||||
.header(http::header::HOST, uri.authority().expect("server authority").as_str())
|
||||
.header(http::header::CONTENT_TYPE, "application/xml")
|
||||
.header("x-amz-content-sha256", "UNSIGNED-PAYLOAD");
|
||||
for (name, value) in headers {
|
||||
request = request.header(*name, *value);
|
||||
}
|
||||
let request = request
|
||||
.body(Full::new(Bytes::copy_from_slice(xml.as_bytes())))
|
||||
.expect("Select request");
|
||||
let mut request =
|
||||
rustfs_signer::try_sign_v4(request, xml.len() as i64, caller.0, caller.1, "", "us-east-1").expect("sign Select request");
|
||||
*request.uri_mut() = path.parse().expect("Select request path");
|
||||
tokio::time::timeout(std::time::Duration::from_secs(20), async {
|
||||
let stream = tokio::net::TcpStream::connect(server.address()).await.expect("connect");
|
||||
let (mut sender, connection) = hyper::client::conn::http1::handshake(TokioIo::new(stream))
|
||||
.await
|
||||
.expect("HTTP connection");
|
||||
let transport = tokio::spawn(connection);
|
||||
let (parts, body) = sender.send_request(request).await.expect("Select response").into_parts();
|
||||
let bytes = body.collect().await.expect("Select response reaches EOF").to_bytes();
|
||||
drop(sender);
|
||||
transport.abort();
|
||||
let _ = transport.await;
|
||||
(parts, bytes)
|
||||
})
|
||||
.await
|
||||
.expect("Select response deadline")
|
||||
}
|
||||
|
||||
fn assert_select_event<'a>(mut body: &'a [u8], kind: &str, expected_payload: &[u8]) -> &'a [u8] {
|
||||
let len = u32::from_be_bytes(body[..4].try_into().expect("frame length")) as usize;
|
||||
let headers_len = u32::from_be_bytes(body[4..8].try_into().expect("header length")) as usize;
|
||||
assert!(16 <= len && len <= body.len() && headers_len <= len - 16);
|
||||
let (frame, rest) = body.split_at(len);
|
||||
body = &frame[12..12 + headers_len];
|
||||
let mut headers = Vec::new();
|
||||
while !body.is_empty() {
|
||||
let key_len = usize::from(body[0]);
|
||||
assert_eq!(body[1 + key_len], 7, "string header");
|
||||
let value_len = u16::from_be_bytes(body[2 + key_len..4 + key_len].try_into().expect("value length")) as usize;
|
||||
let end = 4 + key_len + value_len;
|
||||
headers.push((&body[1..1 + key_len], &body[4 + key_len..end]));
|
||||
body = &body[end..];
|
||||
}
|
||||
let header = |name: &[u8]| headers.iter().find_map(|(key, value)| (*key == name).then_some(*value));
|
||||
if kind == "OverMaxRecordSize" {
|
||||
assert_eq!(header(b":message-type"), Some(b"error".as_slice()));
|
||||
assert_eq!(header(b":error-code"), Some(kind.as_bytes()));
|
||||
assert!(header(b":event-type").is_none());
|
||||
assert!(header(b":error-message").is_some_and(|message| !message.is_empty()));
|
||||
} else {
|
||||
assert_eq!(header(b":message-type"), Some(b"event".as_slice()));
|
||||
assert_eq!(header(b":event-type"), Some(kind.as_bytes()));
|
||||
}
|
||||
assert_eq!(&frame[12 + headers_len..len - 4], expected_payload);
|
||||
rest
|
||||
}
|
||||
|
||||
async fn select_parity(server: RustFSServer) {
|
||||
create_bucket_in_the_store(SELECT_BUCKET).await;
|
||||
create_user_without_a_policy(NO_POLICY_USER).await;
|
||||
let client = s3_client(&server.endpoint(), server.access_key(), server.secret_key());
|
||||
let key = base64_simd::STANDARD.encode_to_string([0_u8; 32]);
|
||||
let md5 = "cLyPS3KoaSFGi/joRB3OUQ==";
|
||||
let mut oversized = b"value\n".to_vec();
|
||||
oversized.extend(std::iter::repeat_n(b'a', (1 << 20) + 1));
|
||||
oversized.push(b'\n');
|
||||
for (name, body) in [
|
||||
(SELECT_FIXTURE_KEY, ByteStream::from_static(SELECT_FIXTURE)),
|
||||
("large.csv", ByteStream::from(oversized)),
|
||||
] {
|
||||
client
|
||||
.put_object()
|
||||
.bucket(SELECT_BUCKET)
|
||||
.key(name)
|
||||
.body(body)
|
||||
.send()
|
||||
.await
|
||||
.expect("put CSV");
|
||||
}
|
||||
client
|
||||
.put_object()
|
||||
.bucket(SELECT_BUCKET)
|
||||
.key("encrypted.csv")
|
||||
.body(ByteStream::from_static(SELECT_FIXTURE))
|
||||
.sse_customer_algorithm("AES256")
|
||||
.sse_customer_key(&key)
|
||||
.sse_customer_key_md5(md5)
|
||||
.send()
|
||||
.await
|
||||
.expect("put encrypted CSV");
|
||||
|
||||
let root = (server.access_key(), server.secret_key());
|
||||
let request_headers = [
|
||||
(SELECT_SSE_ALGORITHM, "AES256"),
|
||||
(SELECT_SSE_KEY, key.as_str()),
|
||||
(SELECT_SSE_MD5, md5),
|
||||
];
|
||||
for (name, sse) in [(SELECT_FIXTURE_KEY, false), ("encrypted.csv", true)] {
|
||||
let headers = if sse { request_headers.as_slice() } else { &[] };
|
||||
let (parts, body) = select_wire(&server, name, SELECT_REQUEST, headers, root).await;
|
||||
assert_eq!(parts.status, 200);
|
||||
assert_eq!(
|
||||
parts
|
||||
.headers
|
||||
.get(SELECT_SSE_ALGORITHM)
|
||||
.and_then(|header| header.to_str().ok()),
|
||||
sse.then_some("AES256")
|
||||
);
|
||||
assert_eq!(
|
||||
parts.headers.get(SELECT_SSE_MD5).and_then(|header| header.to_str().ok()),
|
||||
sse.then_some(md5)
|
||||
);
|
||||
assert!(!parts.headers.contains_key(SELECT_SSE_KEY));
|
||||
let rest = assert_select_event(&body, "Records", b"old\n");
|
||||
let rest = assert_select_event(
|
||||
rest,
|
||||
"Stats",
|
||||
b"<?xml version=\"1.0\" encoding=\"UTF-8\"?><Stats><BytesProcessed>10</BytesProcessed><BytesReturned>4</BytesReturned><BytesScanned>10</BytesScanned></Stats>",
|
||||
);
|
||||
assert!(assert_select_event(rest, "End", b"").is_empty());
|
||||
}
|
||||
|
||||
let (parts, body) = select_wire(&server, "large.csv", SELECT_REQUEST, &[], root).await;
|
||||
assert_eq!(parts.status, 200);
|
||||
assert!(assert_select_event(&body, "OverMaxRecordSize", b"").is_empty());
|
||||
|
||||
let empty_expression = SELECT_REQUEST.replace("SELECT * FROM S3Object", "");
|
||||
let whitespace_expression = SELECT_REQUEST.replace("SELECT * FROM S3Object", " ");
|
||||
let invalid_range = SELECT_REQUEST.replace(
|
||||
"</SelectObjectContentRequest>",
|
||||
"<ScanRange><Start>999999</Start></ScanRange></SelectObjectContentRequest>",
|
||||
);
|
||||
for (name, xml, caller, status, code) in [
|
||||
("encrypted.csv", SELECT_REQUEST, root, 400, "InvalidRequest"),
|
||||
(SELECT_FIXTURE_KEY, empty_expression.as_str(), root, 400, "ParseSelectFailure"),
|
||||
(SELECT_FIXTURE_KEY, whitespace_expression.as_str(), root, 400, "ParseSelectFailure"),
|
||||
(SELECT_FIXTURE_KEY, invalid_range.as_str(), root, 400, "InvalidRequestParameter"),
|
||||
(SELECT_FIXTURE_KEY, SELECT_REQUEST, (NO_POLICY_USER, USER_SECRET_KEY), 403, "AccessDenied"),
|
||||
] {
|
||||
let (parts, body) = select_wire(&server, name, xml, &[], caller).await;
|
||||
assert_eq!(parts.status, status, "{code}: {}", String::from_utf8_lossy(&body));
|
||||
let body = std::str::from_utf8(&body).expect("Select error XML");
|
||||
assert!(body.contains(&format!("<Code>{code}</Code>")), "{body}");
|
||||
}
|
||||
server.shutdown().await;
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn gateway_select_preserves_the_event_sequence_and_denies_unauthorized_reads() {
|
||||
with_stack(GATEWAY, select_parity);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn legacy_select_preserves_the_event_sequence_and_denies_unauthorized_reads() {
|
||||
with_stack(LEGACY, select_parity);
|
||||
}
|
||||
|
||||
@@ -47,5 +47,9 @@ fn object_request_profiling_is_attached_to_the_matching_operation() {
|
||||
"the native entry must retain the established profiling label"
|
||||
);
|
||||
assert_async_method_is_measured(PUT_OBJECT, "execute_put_object");
|
||||
assert_async_method_is_measured(SELECT_OBJECT, "execute_select_object_content");
|
||||
assert_async_method_is_measured(SELECT_OBJECT, "execute_select_object_content_owned");
|
||||
assert!(
|
||||
SELECT_OBJECT.contains("label = \"rustfs::app::object::DefaultObjectUsecase::execute_select_object_content\""),
|
||||
"Select must retain the established profiling label"
|
||||
);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user