refactor: Restructure project layout and clean up dependencies (#30)

This commit introduces a significant reorganization of the project structure to improve maintainability and clarity.

Key changes include:
- Adjusted the directory layout for a more logical module organization.
- Removed unused crate dependencies, reducing the overall project size and potentially speeding up build times.
- Updated import paths and configuration files to reflect the structural changes.

crates/s3select-query/src/sql/analyzer.rs

@@ -0,0 +1,47 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use datafusion::logical_expr::LogicalPlan;
+use datafusion::optimizer::analyzer::Analyzer as DFAnalyzer;
+use rustfs_s3select_api::QueryResult;
+use rustfs_s3select_api::query::analyzer::Analyzer;
+use rustfs_s3select_api::query::session::SessionCtx;
+
+pub struct DefaultAnalyzer {
+    inner: DFAnalyzer,
+}
+
+impl DefaultAnalyzer {
+    pub fn new() -> Self {
+        let analyzer = DFAnalyzer::default();
+        // we can add analyzer rule at here
+
+        Self { inner: analyzer }
+    }
+}
+
+impl Default for DefaultAnalyzer {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl Analyzer for DefaultAnalyzer {
+    fn analyze(&self, plan: &LogicalPlan, session: &SessionCtx) -> QueryResult<LogicalPlan> {
+        let plan = self
+            .inner
+            .execute_and_check(plan.to_owned(), session.inner().config_options(), |_, _| {})?;
+        Ok(plan)
+    }
+}

crates/s3select-query/src/sql/dialect.rs

@@ -0,0 +1,345 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use datafusion::sql::sqlparser::dialect::Dialect;
+
+#[derive(Debug, Default)]
+pub struct RustFsDialect;
+
+impl Dialect for RustFsDialect {
+    fn is_identifier_start(&self, ch: char) -> bool {
+        ch.is_alphabetic() || ch == '_' || ch == '#' || ch == '@'
+    }
+
+    fn is_identifier_part(&self, ch: char) -> bool {
+        ch.is_alphabetic() || ch.is_ascii_digit() || ch == '@' || ch == '$' || ch == '#' || ch == '_'
+    }
+
+    fn supports_group_by_expr(&self) -> bool {
+        true
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_rustfs_dialect_creation() {
+        let _dialect = RustFsDialect;
+
+        // Test that dialect can be created successfully
+        assert!(std::mem::size_of::<RustFsDialect>() == 0, "Dialect should be zero-sized");
+    }
+
+    #[test]
+    fn test_rustfs_dialect_debug() {
+        let dialect = RustFsDialect;
+
+        let debug_str = format!("{dialect:?}");
+        assert!(!debug_str.is_empty(), "Debug output should not be empty");
+        assert!(debug_str.contains("RustFsDialect"), "Debug output should contain dialect name");
+    }
+
+    #[test]
+    fn test_is_identifier_start_alphabetic() {
+        let dialect = RustFsDialect;
+
+        // Test alphabetic characters
+        assert!(dialect.is_identifier_start('a'), "Lowercase letter should be valid identifier start");
+        assert!(dialect.is_identifier_start('A'), "Uppercase letter should be valid identifier start");
+        assert!(dialect.is_identifier_start('z'), "Last lowercase letter should be valid identifier start");
+        assert!(dialect.is_identifier_start('Z'), "Last uppercase letter should be valid identifier start");
+
+        // Test Unicode alphabetic characters
+        assert!(dialect.is_identifier_start('α'), "Greek letter should be valid identifier start");
+        assert!(dialect.is_identifier_start('中'), "Chinese character should be valid identifier start");
+        assert!(dialect.is_identifier_start('ñ'), "Accented letter should be valid identifier start");
+    }
+
+    #[test]
+    fn test_is_identifier_start_special_chars() {
+        let dialect = RustFsDialect;
+
+        // Test special characters that are allowed
+        assert!(dialect.is_identifier_start('_'), "Underscore should be valid identifier start");
+        assert!(dialect.is_identifier_start('#'), "Hash should be valid identifier start");
+        assert!(dialect.is_identifier_start('@'), "At symbol should be valid identifier start");
+    }
+
+    #[test]
+    fn test_is_identifier_start_invalid_chars() {
+        let dialect = RustFsDialect;
+
+        // Test characters that should not be valid identifier starts
+        assert!(!dialect.is_identifier_start('0'), "Digit should not be valid identifier start");
+        assert!(!dialect.is_identifier_start('9'), "Digit should not be valid identifier start");
+        assert!(!dialect.is_identifier_start('$'), "Dollar sign should not be valid identifier start");
+        assert!(!dialect.is_identifier_start(' '), "Space should not be valid identifier start");
+        assert!(!dialect.is_identifier_start('\t'), "Tab should not be valid identifier start");
+        assert!(!dialect.is_identifier_start('\n'), "Newline should not be valid identifier start");
+        assert!(!dialect.is_identifier_start('.'), "Dot should not be valid identifier start");
+        assert!(!dialect.is_identifier_start(','), "Comma should not be valid identifier start");
+        assert!(!dialect.is_identifier_start(';'), "Semicolon should not be valid identifier start");
+        assert!(!dialect.is_identifier_start('('), "Left paren should not be valid identifier start");
+        assert!(!dialect.is_identifier_start(')'), "Right paren should not be valid identifier start");
+        assert!(!dialect.is_identifier_start('['), "Left bracket should not be valid identifier start");
+        assert!(!dialect.is_identifier_start(']'), "Right bracket should not be valid identifier start");
+        assert!(!dialect.is_identifier_start('{'), "Left brace should not be valid identifier start");
+        assert!(!dialect.is_identifier_start('}'), "Right brace should not be valid identifier start");
+        assert!(!dialect.is_identifier_start('='), "Equals should not be valid identifier start");
+        assert!(!dialect.is_identifier_start('+'), "Plus should not be valid identifier start");
+        assert!(!dialect.is_identifier_start('-'), "Minus should not be valid identifier start");
+        assert!(!dialect.is_identifier_start('*'), "Asterisk should not be valid identifier start");
+        assert!(!dialect.is_identifier_start('/'), "Slash should not be valid identifier start");
+        assert!(!dialect.is_identifier_start('%'), "Percent should not be valid identifier start");
+        assert!(!dialect.is_identifier_start('<'), "Less than should not be valid identifier start");
+        assert!(!dialect.is_identifier_start('>'), "Greater than should not be valid identifier start");
+        assert!(!dialect.is_identifier_start('!'), "Exclamation should not be valid identifier start");
+        assert!(!dialect.is_identifier_start('?'), "Question mark should not be valid identifier start");
+        assert!(!dialect.is_identifier_start('&'), "Ampersand should not be valid identifier start");
+        assert!(!dialect.is_identifier_start('|'), "Pipe should not be valid identifier start");
+        assert!(!dialect.is_identifier_start('^'), "Caret should not be valid identifier start");
+        assert!(!dialect.is_identifier_start('~'), "Tilde should not be valid identifier start");
+        assert!(!dialect.is_identifier_start('`'), "Backtick should not be valid identifier start");
+        assert!(!dialect.is_identifier_start('"'), "Double quote should not be valid identifier start");
+        assert!(!dialect.is_identifier_start('\''), "Single quote should not be valid identifier start");
+    }
+
+    #[test]
+    fn test_is_identifier_part_alphabetic() {
+        let dialect = RustFsDialect;
+
+        // Test alphabetic characters
+        assert!(dialect.is_identifier_part('a'), "Lowercase letter should be valid identifier part");
+        assert!(dialect.is_identifier_part('A'), "Uppercase letter should be valid identifier part");
+        assert!(dialect.is_identifier_part('z'), "Last lowercase letter should be valid identifier part");
+        assert!(dialect.is_identifier_part('Z'), "Last uppercase letter should be valid identifier part");
+
+        // Test Unicode alphabetic characters
+        assert!(dialect.is_identifier_part('α'), "Greek letter should be valid identifier part");
+        assert!(dialect.is_identifier_part('中'), "Chinese character should be valid identifier part");
+        assert!(dialect.is_identifier_part('ñ'), "Accented letter should be valid identifier part");
+    }
+
+    #[test]
+    fn test_is_identifier_part_digits() {
+        let dialect = RustFsDialect;
+
+        // Test ASCII digits
+        assert!(dialect.is_identifier_part('0'), "Digit 0 should be valid identifier part");
+        assert!(dialect.is_identifier_part('1'), "Digit 1 should be valid identifier part");
+        assert!(dialect.is_identifier_part('5'), "Digit 5 should be valid identifier part");
+        assert!(dialect.is_identifier_part('9'), "Digit 9 should be valid identifier part");
+    }
+
+    #[test]
+    fn test_is_identifier_part_special_chars() {
+        let dialect = RustFsDialect;
+
+        // Test special characters that are allowed
+        assert!(dialect.is_identifier_part('_'), "Underscore should be valid identifier part");
+        assert!(dialect.is_identifier_part('#'), "Hash should be valid identifier part");
+        assert!(dialect.is_identifier_part('@'), "At symbol should be valid identifier part");
+        assert!(dialect.is_identifier_part('$'), "Dollar sign should be valid identifier part");
+    }
+
+    #[test]
+    fn test_is_identifier_part_invalid_chars() {
+        let dialect = RustFsDialect;
+
+        // Test characters that should not be valid identifier parts
+        assert!(!dialect.is_identifier_part(' '), "Space should not be valid identifier part");
+        assert!(!dialect.is_identifier_part('\t'), "Tab should not be valid identifier part");
+        assert!(!dialect.is_identifier_part('\n'), "Newline should not be valid identifier part");
+        assert!(!dialect.is_identifier_part('.'), "Dot should not be valid identifier part");
+        assert!(!dialect.is_identifier_part(','), "Comma should not be valid identifier part");
+        assert!(!dialect.is_identifier_part(';'), "Semicolon should not be valid identifier part");
+        assert!(!dialect.is_identifier_part('('), "Left paren should not be valid identifier part");
+        assert!(!dialect.is_identifier_part(')'), "Right paren should not be valid identifier part");
+        assert!(!dialect.is_identifier_part('['), "Left bracket should not be valid identifier part");
+        assert!(!dialect.is_identifier_part(']'), "Right bracket should not be valid identifier part");
+        assert!(!dialect.is_identifier_part('{'), "Left brace should not be valid identifier part");
+        assert!(!dialect.is_identifier_part('}'), "Right brace should not be valid identifier part");
+        assert!(!dialect.is_identifier_part('='), "Equals should not be valid identifier part");
+        assert!(!dialect.is_identifier_part('+'), "Plus should not be valid identifier part");
+        assert!(!dialect.is_identifier_part('-'), "Minus should not be valid identifier part");
+        assert!(!dialect.is_identifier_part('*'), "Asterisk should not be valid identifier part");
+        assert!(!dialect.is_identifier_part('/'), "Slash should not be valid identifier part");
+        assert!(!dialect.is_identifier_part('%'), "Percent should not be valid identifier part");
+        assert!(!dialect.is_identifier_part('<'), "Less than should not be valid identifier part");
+        assert!(!dialect.is_identifier_part('>'), "Greater than should not be valid identifier part");
+        assert!(!dialect.is_identifier_part('!'), "Exclamation should not be valid identifier part");
+        assert!(!dialect.is_identifier_part('?'), "Question mark should not be valid identifier part");
+        assert!(!dialect.is_identifier_part('&'), "Ampersand should not be valid identifier part");
+        assert!(!dialect.is_identifier_part('|'), "Pipe should not be valid identifier part");
+        assert!(!dialect.is_identifier_part('^'), "Caret should not be valid identifier part");
+        assert!(!dialect.is_identifier_part('~'), "Tilde should not be valid identifier part");
+        assert!(!dialect.is_identifier_part('`'), "Backtick should not be valid identifier part");
+        assert!(!dialect.is_identifier_part('"'), "Double quote should not be valid identifier part");
+        assert!(!dialect.is_identifier_part('\''), "Single quote should not be valid identifier part");
+    }
+
+    #[test]
+    fn test_supports_group_by_expr() {
+        let dialect = RustFsDialect;
+
+        assert!(dialect.supports_group_by_expr(), "RustFsDialect should support GROUP BY expressions");
+    }
+
+    #[test]
+    fn test_identifier_validation_comprehensive() {
+        let dialect = RustFsDialect;
+
+        // Test valid identifier patterns
+        let valid_starts = ['a', 'A', 'z', 'Z', '_', '#', '@', 'α', '中'];
+        let valid_parts = ['a', 'A', '0', '9', '_', '#', '@', '$', 'α', '中'];
+
+        for start_char in valid_starts {
+            assert!(
+                dialect.is_identifier_start(start_char),
+                "Character '{start_char}' should be valid identifier start"
+            );
+
+            for part_char in valid_parts {
+                assert!(
+                    dialect.is_identifier_part(part_char),
+                    "Character '{part_char}' should be valid identifier part"
+                );
+            }
+        }
+    }
+
+    #[test]
+    fn test_identifier_edge_cases() {
+        let dialect = RustFsDialect;
+
+        // Test edge cases with control characters
+        assert!(!dialect.is_identifier_start('\0'), "Null character should not be valid identifier start");
+        assert!(!dialect.is_identifier_part('\0'), "Null character should not be valid identifier part");
+
+        assert!(
+            !dialect.is_identifier_start('\x01'),
+            "Control character should not be valid identifier start"
+        );
+        assert!(
+            !dialect.is_identifier_part('\x01'),
+            "Control character should not be valid identifier part"
+        );
+
+        assert!(!dialect.is_identifier_start('\x7F'), "DEL character should not be valid identifier start");
+        assert!(!dialect.is_identifier_part('\x7F'), "DEL character should not be valid identifier part");
+    }
+
+    #[test]
+    fn test_identifier_unicode_support() {
+        let dialect = RustFsDialect;
+
+        // Test various Unicode categories
+        let unicode_letters = ['α', 'β', 'γ', 'Α', 'Β', 'Γ', '中', '文', '日', '本', 'ñ', 'ü', 'ç'];
+
+        for ch in unicode_letters {
+            assert!(dialect.is_identifier_start(ch), "Unicode letter '{ch}' should be valid identifier start");
+            assert!(dialect.is_identifier_part(ch), "Unicode letter '{ch}' should be valid identifier part");
+        }
+    }
+
+    #[test]
+    fn test_identifier_ascii_digits() {
+        let dialect = RustFsDialect;
+
+        // Test all ASCII digits
+        for digit in '0'..='9' {
+            assert!(
+                !dialect.is_identifier_start(digit),
+                "ASCII digit '{digit}' should not be valid identifier start"
+            );
+            assert!(dialect.is_identifier_part(digit), "ASCII digit '{digit}' should be valid identifier part");
+        }
+    }
+
+    #[test]
+    fn test_dialect_consistency() {
+        let dialect = RustFsDialect;
+
+        // Test that all valid identifier starts are also valid identifier parts
+        let test_chars = [
+            'a', 'A', 'z', 'Z', '_', '#', '@', 'α', '中', 'ñ', '0', '9', '$', ' ', '.', ',', ';', '(', ')', '=', '+', '-',
+        ];
+
+        for ch in test_chars {
+            if dialect.is_identifier_start(ch) {
+                assert!(
+                    dialect.is_identifier_part(ch),
+                    "Character '{ch}' that is valid identifier start should also be valid identifier part"
+                );
+            }
+        }
+    }
+
+    #[test]
+    fn test_dialect_memory_efficiency() {
+        let dialect = RustFsDialect;
+
+        // Test that dialect doesn't use excessive memory
+        let dialect_size = std::mem::size_of_val(&dialect);
+        assert!(dialect_size < 100, "Dialect should not use excessive memory");
+    }
+
+    #[test]
+    fn test_dialect_trait_implementation() {
+        let dialect = RustFsDialect;
+
+        // Test that dialect properly implements the Dialect trait
+        let dialect_ref: &dyn Dialect = &dialect;
+
+        // Test basic functionality through trait
+        assert!(dialect_ref.is_identifier_start('a'), "Trait method should work for valid start");
+        assert!(!dialect_ref.is_identifier_start('0'), "Trait method should work for invalid start");
+        assert!(dialect_ref.is_identifier_part('a'), "Trait method should work for valid part");
+        assert!(dialect_ref.is_identifier_part('0'), "Trait method should work for digit part");
+        assert!(
+            dialect_ref.supports_group_by_expr(),
+            "Trait method should return true for GROUP BY support"
+        );
+    }
+
+    #[test]
+    fn test_dialect_clone_and_default() {
+        let dialect1 = RustFsDialect;
+        let dialect2 = RustFsDialect;
+
+        // Test that multiple instances behave the same
+        let test_chars = ['a', 'A', '0', '_', '#', '@', '$', ' ', '.'];
+
+        for ch in test_chars {
+            assert_eq!(
+                dialect1.is_identifier_start(ch),
+                dialect2.is_identifier_start(ch),
+                "Different instances should behave the same for is_identifier_start"
+            );
+            assert_eq!(
+                dialect1.is_identifier_part(ch),
+                dialect2.is_identifier_part(ch),
+                "Different instances should behave the same for is_identifier_part"
+            );
+        }
+
+        assert_eq!(
+            dialect1.supports_group_by_expr(),
+            dialect2.supports_group_by_expr(),
+            "Different instances should behave the same for supports_group_by_expr"
+        );
+    }
+}

crates/s3select-query/src/sql/logical/mod.rs

@@ -0,0 +1,16 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+pub mod optimizer;
+pub mod planner;

crates/s3select-query/src/sql/logical/optimizer.rs

@@ -0,0 +1,125 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::sync::Arc;
+
+use datafusion::{
+    execution::SessionStateBuilder,
+    logical_expr::LogicalPlan,
+    optimizer::{
+        OptimizerRule, common_subexpr_eliminate::CommonSubexprEliminate,
+        decorrelate_predicate_subquery::DecorrelatePredicateSubquery, eliminate_cross_join::EliminateCrossJoin,
+        eliminate_duplicated_expr::EliminateDuplicatedExpr, eliminate_filter::EliminateFilter, eliminate_join::EliminateJoin,
+        eliminate_limit::EliminateLimit, eliminate_outer_join::EliminateOuterJoin,
+        extract_equijoin_predicate::ExtractEquijoinPredicate, filter_null_join_keys::FilterNullJoinKeys,
+        propagate_empty_relation::PropagateEmptyRelation, push_down_filter::PushDownFilter, push_down_limit::PushDownLimit,
+        replace_distinct_aggregate::ReplaceDistinctWithAggregate, scalar_subquery_to_join::ScalarSubqueryToJoin,
+        simplify_expressions::SimplifyExpressions, single_distinct_to_groupby::SingleDistinctToGroupBy,
+        unwrap_cast_in_comparison::UnwrapCastInComparison,
+    },
+};
+use rustfs_s3select_api::{
+    QueryResult,
+    query::{analyzer::AnalyzerRef, logical_planner::QueryPlan, session::SessionCtx},
+};
+use tracing::debug;
+
+use crate::sql::analyzer::DefaultAnalyzer;
+
+pub trait LogicalOptimizer: Send + Sync {
+    fn optimize(&self, plan: &QueryPlan, session: &SessionCtx) -> QueryResult<LogicalPlan>;
+
+    fn inject_optimizer_rule(&mut self, optimizer_rule: Arc<dyn OptimizerRule + Send + Sync>);
+}
+
+pub struct DefaultLogicalOptimizer {
+    // fit datafusion
+    // TODO refactor
+    analyzer: AnalyzerRef,
+    rules: Vec<Arc<dyn OptimizerRule + Send + Sync>>,
+}
+
+impl DefaultLogicalOptimizer {
+    #[allow(dead_code)]
+    fn with_optimizer_rules(mut self, rules: Vec<Arc<dyn OptimizerRule + Send + Sync>>) -> Self {
+        self.rules = rules;
+        self
+    }
+}
+
+impl Default for DefaultLogicalOptimizer {
+    fn default() -> Self {
+        let analyzer = Arc::new(DefaultAnalyzer::default());
+
+        // additional optimizer rule
+        let rules: Vec<Arc<dyn OptimizerRule + Send + Sync>> = vec![
+            // df default rules start
+            Arc::new(SimplifyExpressions::new()),
+            Arc::new(UnwrapCastInComparison::new()),
+            Arc::new(ReplaceDistinctWithAggregate::new()),
+            Arc::new(EliminateJoin::new()),
+            Arc::new(DecorrelatePredicateSubquery::new()),
+            Arc::new(ScalarSubqueryToJoin::new()),
+            Arc::new(ExtractEquijoinPredicate::new()),
+            // simplify expressions does not simplify expressions in subqueries, so we
+            // run it again after running the optimizations that potentially converted
+            // subqueries to joins
+            Arc::new(SimplifyExpressions::new()),
+            Arc::new(EliminateDuplicatedExpr::new()),
+            Arc::new(EliminateFilter::new()),
+            Arc::new(EliminateCrossJoin::new()),
+            Arc::new(CommonSubexprEliminate::new()),
+            Arc::new(EliminateLimit::new()),
+            Arc::new(PropagateEmptyRelation::new()),
+            Arc::new(FilterNullJoinKeys::default()),
+            Arc::new(EliminateOuterJoin::new()),
+            // Filters can't be pushed down past Limits, we should do PushDownFilter after PushDownLimit
+            Arc::new(PushDownLimit::new()),
+            Arc::new(PushDownFilter::new()),
+            Arc::new(SingleDistinctToGroupBy::new()),
+            // The previous optimizations added expressions and projections,
+            // that might benefit from the following rules
+            Arc::new(SimplifyExpressions::new()),
+            Arc::new(UnwrapCastInComparison::new()),
+            Arc::new(CommonSubexprEliminate::new()),
+            // PushDownProjection can pushdown Projections through Limits, do PushDownLimit again.
+            Arc::new(PushDownLimit::new()),
+            // df default rules end
+            // custom rules can add at here
+        ];
+
+        Self { analyzer, rules }
+    }
+}
+
+impl LogicalOptimizer for DefaultLogicalOptimizer {
+    fn optimize(&self, plan: &QueryPlan, session: &SessionCtx) -> QueryResult<LogicalPlan> {
+        let analyzed_plan = { self.analyzer.analyze(&plan.df_plan, session)? };
+
+        debug!("Analyzed logical plan:\n{}\n", plan.df_plan.display_indent_schema(),);
+
+        let optimizeed_plan = {
+            SessionStateBuilder::new_from_existing(session.inner().clone())
+                .with_optimizer_rules(self.rules.clone())
+                .build()
+                .optimize(&analyzed_plan)?
+        };
+
+        Ok(optimizeed_plan)
+    }
+
+    fn inject_optimizer_rule(&mut self, optimizer_rule: Arc<dyn OptimizerRule + Send + Sync>) {
+        self.rules.push(optimizer_rule);
+    }
+}

crates/s3select-query/src/sql/logical/planner.rs

@@ -0,0 +1,17 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::sql::planner::SqlPlanner;
+
+pub type DefaultLogicalPlanner<'a, S> = SqlPlanner<'a, S>;

crates/s3select-query/src/sql/mod.rs

@@ -0,0 +1,21 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+pub mod analyzer;
+pub mod dialect;
+pub mod logical;
+pub mod optimizer;
+pub mod parser;
+pub mod physical;
+pub mod planner;

crates/s3select-query/src/sql/optimizer.rs

@@ -0,0 +1,199 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::sync::Arc;
+
+use async_trait::async_trait;
+use datafusion::physical_plan::{ExecutionPlan, displayable};
+use rustfs_s3select_api::{
+    QueryResult,
+    query::{logical_planner::QueryPlan, optimizer::Optimizer, physical_planner::PhysicalPlanner, session::SessionCtx},
+};
+use tracing::debug;
+
+use super::{
+    logical::optimizer::{DefaultLogicalOptimizer, LogicalOptimizer},
+    physical::{optimizer::PhysicalOptimizer, planner::DefaultPhysicalPlanner},
+};
+
+pub struct CascadeOptimizer {
+    logical_optimizer: Arc<dyn LogicalOptimizer + Send + Sync>,
+    physical_planner: Arc<dyn PhysicalPlanner + Send + Sync>,
+    physical_optimizer: Arc<dyn PhysicalOptimizer + Send + Sync>,
+}
+
+#[async_trait]
+impl Optimizer for CascadeOptimizer {
+    async fn optimize(&self, plan: &QueryPlan, session: &SessionCtx) -> QueryResult<Arc<dyn ExecutionPlan>> {
+        debug!("Original logical plan:\n{}\n", plan.df_plan.display_indent_schema(),);
+
+        let optimized_logical_plan = self.logical_optimizer.optimize(plan, session)?;
+
+        debug!("Final logical plan:\n{}\n", optimized_logical_plan.display_indent_schema(),);
+
+        let physical_plan = {
+            self.physical_planner
+                .create_physical_plan(&optimized_logical_plan, session)
+                .await?
+        };
+
+        debug!("Original physical plan:\n{}\n", displayable(physical_plan.as_ref()).indent(false));
+
+        let optimized_physical_plan = { self.physical_optimizer.optimize(physical_plan, session)? };
+
+        Ok(optimized_physical_plan)
+    }
+}
+
+#[derive(Default)]
+pub struct CascadeOptimizerBuilder {
+    logical_optimizer: Option<Arc<dyn LogicalOptimizer + Send + Sync>>,
+    physical_planner: Option<Arc<dyn PhysicalPlanner + Send + Sync>>,
+    physical_optimizer: Option<Arc<dyn PhysicalOptimizer + Send + Sync>>,
+}
+
+impl CascadeOptimizerBuilder {
+    pub fn with_logical_optimizer(mut self, logical_optimizer: Arc<dyn LogicalOptimizer + Send + Sync>) -> Self {
+        self.logical_optimizer = Some(logical_optimizer);
+        self
+    }
+
+    pub fn with_physical_planner(mut self, physical_planner: Arc<dyn PhysicalPlanner + Send + Sync>) -> Self {
+        self.physical_planner = Some(physical_planner);
+        self
+    }
+
+    pub fn with_physical_optimizer(mut self, physical_optimizer: Arc<dyn PhysicalOptimizer + Send + Sync>) -> Self {
+        self.physical_optimizer = Some(physical_optimizer);
+        self
+    }
+
+    pub fn build(self) -> CascadeOptimizer {
+        let default_logical_optimizer = Arc::new(DefaultLogicalOptimizer::default());
+        let default_physical_planner = Arc::new(DefaultPhysicalPlanner::default());
+
+        let logical_optimizer = self.logical_optimizer.unwrap_or(default_logical_optimizer);
+        let physical_planner = self.physical_planner.unwrap_or_else(|| default_physical_planner.clone());
+        let physical_optimizer = self.physical_optimizer.unwrap_or(default_physical_planner);
+
+        CascadeOptimizer {
+            logical_optimizer,
+            physical_planner,
+            physical_optimizer,
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_cascade_optimizer_builder_default() {
+        let _builder = CascadeOptimizerBuilder::default();
+
+        // Test that builder can be created successfully
+        assert!(
+            std::mem::size_of::<CascadeOptimizerBuilder>() > 0,
+            "Builder should be created successfully"
+        );
+    }
+
+    #[test]
+    fn test_cascade_optimizer_builder_build_with_defaults() {
+        let _builder = CascadeOptimizerBuilder::default();
+        let optimizer = _builder.build();
+
+        // Test that optimizer can be built with default components
+        assert!(std::mem::size_of_val(&optimizer) > 0, "Optimizer should be built successfully");
+    }
+
+    #[test]
+    fn test_cascade_optimizer_builder_basic_functionality() {
+        // Test that builder methods can be called and return self
+        let _builder = CascadeOptimizerBuilder::default();
+
+        // Test that we can call builder methods (even if we don't have mock implementations)
+        // This tests the builder pattern itself
+        assert!(
+            std::mem::size_of::<CascadeOptimizerBuilder>() > 0,
+            "Builder should be created successfully"
+        );
+    }
+
+    #[test]
+    fn test_cascade_optimizer_builder_memory_efficiency() {
+        let _builder = CascadeOptimizerBuilder::default();
+
+        // Test that builder doesn't use excessive memory
+        let builder_size = std::mem::size_of_val(&_builder);
+        assert!(builder_size < 1000, "Builder should not use excessive memory");
+
+        let optimizer = _builder.build();
+        let optimizer_size = std::mem::size_of_val(&optimizer);
+        assert!(optimizer_size < 1000, "Optimizer should not use excessive memory");
+    }
+
+    #[test]
+    fn test_cascade_optimizer_builder_multiple_builds() {
+        let _builder = CascadeOptimizerBuilder::default();
+
+        // Test that we can build multiple optimizers from the same configuration
+        let optimizer1 = _builder.build();
+        assert!(std::mem::size_of_val(&optimizer1) > 0, "First optimizer should be built successfully");
+
+        // Note: builder is consumed by build(), so we can't build again from the same instance
+        // This is the expected behavior
+    }
+
+    #[test]
+    fn test_cascade_optimizer_builder_default_fallbacks() {
+        let _builder = CascadeOptimizerBuilder::default();
+        let optimizer = _builder.build();
+
+        // Test that default components are used when none are specified
+        // We can't directly access the internal components, but we can verify the optimizer was built
+        assert!(std::mem::size_of_val(&optimizer) > 0, "Optimizer should use default components");
+    }
+
+    #[test]
+    fn test_cascade_optimizer_component_types() {
+        let optimizer = CascadeOptimizerBuilder::default().build();
+
+        // Test that optimizer contains the expected component types
+        // We can't directly access the components, but we can verify the optimizer structure
+        assert!(std::mem::size_of_val(&optimizer) > 0, "Optimizer should contain components");
+
+        // The optimizer should have three Arc fields for the components
+        // This is a basic structural test
+    }
+
+    #[test]
+    fn test_cascade_optimizer_builder_consistency() {
+        // Test that multiple builders with the same configuration produce equivalent optimizers
+        let optimizer1 = CascadeOptimizerBuilder::default().build();
+        let optimizer2 = CascadeOptimizerBuilder::default().build();
+
+        // Both optimizers should be built successfully
+        assert!(std::mem::size_of_val(&optimizer1) > 0, "First optimizer should be built");
+        assert!(std::mem::size_of_val(&optimizer2) > 0, "Second optimizer should be built");
+
+        // They should have the same memory footprint (same structure)
+        assert_eq!(
+            std::mem::size_of_val(&optimizer1),
+            std::mem::size_of_val(&optimizer2),
+            "Optimizers with same configuration should have same size"
+        );
+    }
+}

crates/s3select-query/src/sql/parser.rs

@@ -0,0 +1,449 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::{collections::VecDeque, fmt::Display};
+
+use datafusion::sql::sqlparser::{
+    dialect::Dialect,
+    parser::{Parser, ParserError},
+    tokenizer::{Token, Tokenizer},
+};
+use rustfs_s3select_api::{
+    ParserSnafu,
+    query::{ast::ExtStatement, parser::Parser as RustFsParser},
+};
+use snafu::ResultExt;
+
+use super::dialect::RustFsDialect;
+
+pub type Result<T, E = ParserError> = std::result::Result<T, E>;
+
+// Use `Parser::expected` instead, if possible
+macro_rules! parser_err {
+    ($MSG:expr) => {
+        Err(ParserError::ParserError($MSG.to_string()))
+    };
+}
+
+#[derive(Default)]
+pub struct DefaultParser {}
+
+impl RustFsParser for DefaultParser {
+    fn parse(&self, sql: &str) -> rustfs_s3select_api::QueryResult<VecDeque<ExtStatement>> {
+        ExtParser::parse_sql(sql).context(ParserSnafu)
+    }
+}
+
+/// SQL Parser
+pub struct ExtParser<'a> {
+    parser: Parser<'a>,
+}
+
+impl<'a> ExtParser<'a> {
+    /// Parse the specified tokens with dialect
+    fn new_with_dialect(sql: &str, dialect: &'a dyn Dialect) -> Result<Self> {
+        let mut tokenizer = Tokenizer::new(dialect, sql);
+        let tokens = tokenizer.tokenize()?;
+        Ok(ExtParser {
+            parser: Parser::new(dialect).with_tokens(tokens),
+        })
+    }
+
+    /// Parse a SQL statement and produce a set of statements
+    pub fn parse_sql(sql: &str) -> Result<VecDeque<ExtStatement>> {
+        let dialect = &RustFsDialect {};
+        ExtParser::parse_sql_with_dialect(sql, dialect)
+    }
+
+    /// Parse a SQL statement and produce a set of statements
+    pub fn parse_sql_with_dialect(sql: &str, dialect: &dyn Dialect) -> Result<VecDeque<ExtStatement>> {
+        let mut parser = ExtParser::new_with_dialect(sql, dialect)?;
+        let mut stmts = VecDeque::new();
+        let mut expecting_statement_delimiter = false;
+        loop {
+            // ignore empty statements (between successive statement delimiters)
+            while parser.parser.consume_token(&Token::SemiColon) {
+                expecting_statement_delimiter = false;
+            }
+
+            if parser.parser.peek_token() == Token::EOF {
+                break;
+            }
+            if expecting_statement_delimiter {
+                return parser.expected("end of statement", parser.parser.peek_token());
+            }
+
+            let statement = parser.parse_statement()?;
+            stmts.push_back(statement);
+            expecting_statement_delimiter = true;
+        }
+
+        // debug!("Parser sql: {}, stmts: {:#?}", sql, stmts);
+
+        Ok(stmts)
+    }
+
+    /// Parse a new expression
+    fn parse_statement(&mut self) -> Result<ExtStatement> {
+        Ok(ExtStatement::SqlStatement(Box::new(self.parser.parse_statement()?)))
+    }
+
+    // Report unexpected token
+    fn expected<T>(&self, expected: &str, found: impl Display) -> Result<T> {
+        parser_err!(format!("Expected {}, found: {}", expected, found))
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use rustfs_s3select_api::query::ast::ExtStatement;
+
+    #[test]
+    fn test_default_parser_creation() {
+        let _parser = DefaultParser::default();
+
+        // Test that parser can be created successfully
+        assert!(std::mem::size_of::<DefaultParser>() == 0, "Parser should be zero-sized");
+    }
+
+    #[test]
+    fn test_default_parser_simple_select() {
+        let parser = DefaultParser::default();
+        let sql = "SELECT * FROM S3Object";
+
+        let result = parser.parse(sql);
+        assert!(result.is_ok(), "Simple SELECT should parse successfully");
+
+        let statements = result.unwrap();
+        assert_eq!(statements.len(), 1, "Should have exactly one statement");
+
+        // Just verify we get a SQL statement without diving into AST details
+        match &statements[0] {
+            ExtStatement::SqlStatement(_) => {
+                // Successfully parsed as SQL statement
+            }
+        }
+    }
+
+    #[test]
+    fn test_default_parser_select_with_columns() {
+        let parser = DefaultParser::default();
+        let sql = "SELECT id, name, age FROM S3Object";
+
+        let result = parser.parse(sql);
+        assert!(result.is_ok(), "SELECT with columns should parse successfully");
+
+        let statements = result.unwrap();
+        assert_eq!(statements.len(), 1, "Should have exactly one statement");
+
+        match &statements[0] {
+            ExtStatement::SqlStatement(_) => {
+                // Successfully parsed as SQL statement
+            }
+        }
+    }
+
+    #[test]
+    fn test_default_parser_select_with_where() {
+        let parser = DefaultParser::default();
+        let sql = "SELECT * FROM S3Object WHERE age > 25";
+
+        let result = parser.parse(sql);
+        assert!(result.is_ok(), "SELECT with WHERE should parse successfully");
+
+        let statements = result.unwrap();
+        assert_eq!(statements.len(), 1, "Should have exactly one statement");
+
+        match &statements[0] {
+            ExtStatement::SqlStatement(_) => {
+                // Successfully parsed as SQL statement
+            }
+        }
+    }
+
+    #[test]
+    fn test_default_parser_multiple_statements() {
+        let parser = DefaultParser::default();
+        let sql = "SELECT * FROM S3Object; SELECT id FROM S3Object;";
+
+        let result = parser.parse(sql);
+        assert!(result.is_ok(), "Multiple statements should parse successfully");
+
+        let statements = result.unwrap();
+        assert_eq!(statements.len(), 2, "Should have exactly two statements");
+    }
+
+    #[test]
+    fn test_default_parser_empty_statements() {
+        let parser = DefaultParser::default();
+        let sql = ";;; SELECT * FROM S3Object; ;;;";
+
+        let result = parser.parse(sql);
+        assert!(result.is_ok(), "Empty statements should be ignored");
+
+        let statements = result.unwrap();
+        assert_eq!(statements.len(), 1, "Should have exactly one non-empty statement");
+    }
+
+    #[test]
+    fn test_default_parser_invalid_sql() {
+        let parser = DefaultParser::default();
+        let sql = "INVALID SQL SYNTAX";
+
+        let result = parser.parse(sql);
+        assert!(result.is_err(), "Invalid SQL should return error");
+    }
+
+    #[test]
+    fn test_default_parser_empty_sql() {
+        let parser = DefaultParser::default();
+        let sql = "";
+
+        let result = parser.parse(sql);
+        assert!(result.is_ok(), "Empty SQL should parse successfully");
+
+        let statements = result.unwrap();
+        assert!(statements.is_empty(), "Should have no statements");
+    }
+
+    #[test]
+    fn test_default_parser_whitespace_only() {
+        let parser = DefaultParser::default();
+        let sql = "   \n\t  ";
+
+        let result = parser.parse(sql);
+        assert!(result.is_ok(), "Whitespace-only SQL should parse successfully");
+
+        let statements = result.unwrap();
+        assert!(statements.is_empty(), "Should have no statements");
+    }
+
+    #[test]
+    fn test_ext_parser_parse_sql() {
+        let sql = "SELECT * FROM S3Object";
+
+        let result = ExtParser::parse_sql(sql);
+        assert!(result.is_ok(), "ExtParser::parse_sql should work");
+
+        let statements = result.unwrap();
+        assert_eq!(statements.len(), 1, "Should have exactly one statement");
+    }
+
+    #[test]
+    fn test_ext_parser_parse_sql_with_dialect() {
+        let sql = "SELECT * FROM S3Object";
+        let dialect = &RustFsDialect;
+
+        let result = ExtParser::parse_sql_with_dialect(sql, dialect);
+        assert!(result.is_ok(), "ExtParser::parse_sql_with_dialect should work");
+
+        let statements = result.unwrap();
+        assert_eq!(statements.len(), 1, "Should have exactly one statement");
+    }
+
+    #[test]
+    fn test_ext_parser_new_with_dialect() {
+        let sql = "SELECT * FROM S3Object";
+        let dialect = &RustFsDialect;
+
+        let result = ExtParser::new_with_dialect(sql, dialect);
+        assert!(result.is_ok(), "ExtParser::new_with_dialect should work");
+    }
+
+    #[test]
+    fn test_ext_parser_complex_query() {
+        let sql = "SELECT id, name, age FROM S3Object WHERE age > 25 AND department = 'IT' ORDER BY age DESC LIMIT 10";
+
+        let result = ExtParser::parse_sql(sql);
+        assert!(result.is_ok(), "Complex query should parse successfully");
+
+        let statements = result.unwrap();
+        assert_eq!(statements.len(), 1, "Should have exactly one statement");
+
+        match &statements[0] {
+            ExtStatement::SqlStatement(_) => {
+                // Successfully parsed as SQL statement
+            }
+        }
+    }
+
+    #[test]
+    fn test_ext_parser_aggregate_functions() {
+        let sql = "SELECT COUNT(*), AVG(age), MAX(salary) FROM S3Object GROUP BY department";
+
+        let result = ExtParser::parse_sql(sql);
+        assert!(result.is_ok(), "Aggregate functions should parse successfully");
+
+        let statements = result.unwrap();
+        assert_eq!(statements.len(), 1, "Should have exactly one statement");
+
+        match &statements[0] {
+            ExtStatement::SqlStatement(_) => {
+                // Successfully parsed as SQL statement
+            }
+        }
+    }
+
+    #[test]
+    fn test_ext_parser_join_query() {
+        let sql = "SELECT s1.id, s2.name FROM S3Object s1 JOIN S3Object s2 ON s1.id = s2.id";
+
+        let result = ExtParser::parse_sql(sql);
+        assert!(result.is_ok(), "JOIN query should parse successfully");
+
+        let statements = result.unwrap();
+        assert_eq!(statements.len(), 1, "Should have exactly one statement");
+    }
+
+    #[test]
+    fn test_ext_parser_subquery() {
+        let sql = "SELECT * FROM S3Object WHERE id IN (SELECT id FROM S3Object WHERE age > 30)";
+
+        let result = ExtParser::parse_sql(sql);
+        assert!(result.is_ok(), "Subquery should parse successfully");
+
+        let statements = result.unwrap();
+        assert_eq!(statements.len(), 1, "Should have exactly one statement");
+    }
+
+    #[test]
+    fn test_ext_parser_case_insensitive() {
+        let sql = "select * from s3object where age > 25";
+
+        let result = ExtParser::parse_sql(sql);
+        assert!(result.is_ok(), "Case insensitive SQL should parse successfully");
+
+        let statements = result.unwrap();
+        assert_eq!(statements.len(), 1, "Should have exactly one statement");
+    }
+
+    #[test]
+    fn test_ext_parser_quoted_identifiers() {
+        let sql = r#"SELECT "id", "name" FROM "S3Object" WHERE "age" > 25"#;
+
+        let result = ExtParser::parse_sql(sql);
+        assert!(result.is_ok(), "Quoted identifiers should parse successfully");
+
+        let statements = result.unwrap();
+        assert_eq!(statements.len(), 1, "Should have exactly one statement");
+    }
+
+    #[test]
+    fn test_ext_parser_string_literals() {
+        let sql = "SELECT * FROM S3Object WHERE name = 'John Doe' AND department = 'IT'";
+
+        let result = ExtParser::parse_sql(sql);
+        assert!(result.is_ok(), "String literals should parse successfully");
+
+        let statements = result.unwrap();
+        assert_eq!(statements.len(), 1, "Should have exactly one statement");
+    }
+
+    #[test]
+    fn test_ext_parser_numeric_literals() {
+        let sql = "SELECT * FROM S3Object WHERE age = 25 AND salary = 50000.50";
+
+        let result = ExtParser::parse_sql(sql);
+        assert!(result.is_ok(), "Numeric literals should parse successfully");
+
+        let statements = result.unwrap();
+        assert_eq!(statements.len(), 1, "Should have exactly one statement");
+    }
+
+    #[test]
+    fn test_ext_parser_error_handling() {
+        let invalid_sqls = vec![
+            "SELECT FROM",                  // Missing column list
+            "SELECT * FROM",                // Missing table name
+            "SELECT * FROM S3Object WHERE", // Incomplete WHERE clause
+            "SELECT * FROM S3Object GROUP", // Incomplete GROUP BY
+            "SELECT * FROM S3Object ORDER", // Incomplete ORDER BY
+        ];
+
+        for sql in invalid_sqls {
+            let result = ExtParser::parse_sql(sql);
+            assert!(result.is_err(), "Invalid SQL '{sql}' should return error");
+        }
+    }
+
+    #[test]
+    fn test_ext_parser_memory_efficiency() {
+        let sql = "SELECT * FROM S3Object";
+
+        // Test that parser doesn't use excessive memory
+        let result = ExtParser::parse_sql(sql);
+        assert!(result.is_ok(), "Parser should work efficiently");
+
+        let statements = result.unwrap();
+        let memory_size = std::mem::size_of_val(&statements);
+        assert!(memory_size < 10000, "Parsed statements should not use excessive memory");
+    }
+
+    #[test]
+    fn test_ext_parser_large_query() {
+        // Test with a reasonably large query
+        let mut sql = String::from("SELECT ");
+        for i in 0..100 {
+            if i > 0 {
+                sql.push_str(", ");
+            }
+            sql.push_str(&format!("col{i}"));
+        }
+        sql.push_str(" FROM S3Object WHERE ");
+        for i in 0..50 {
+            if i > 0 {
+                sql.push_str(" AND ");
+            }
+            sql.push_str(&format!("col{i} > {i}"));
+        }
+
+        let result = ExtParser::parse_sql(&sql);
+        assert!(result.is_ok(), "Large query should parse successfully");
+
+        let statements = result.unwrap();
+        assert_eq!(statements.len(), 1, "Should have exactly one statement");
+    }
+
+    #[test]
+    fn test_parser_err_macro() {
+        let error: Result<()> = parser_err!("Test error message");
+        assert!(error.is_err(), "parser_err! macro should create error");
+
+        match error {
+            Err(ParserError::ParserError(msg)) => {
+                assert_eq!(msg, "Test error message", "Error message should match");
+            }
+            _ => panic!("Expected ParserError::ParserError"),
+        }
+    }
+
+    #[test]
+    fn test_ext_parser_expected_method() {
+        let sql = "SELECT * FROM S3Object";
+        let dialect = &RustFsDialect;
+        let parser = ExtParser::new_with_dialect(sql, dialect).unwrap();
+
+        let result: Result<()> = parser.expected("test token", "found token");
+        assert!(result.is_err(), "expected method should return error");
+
+        match result {
+            Err(ParserError::ParserError(msg)) => {
+                assert!(msg.contains("Expected test token"), "Error should contain expected message");
+                assert!(msg.contains("found: found token"), "Error should contain found message");
+            }
+            _ => panic!("Expected ParserError::ParserError"),
+        }
+    }
+}

crates/s3select-query/src/sql/physical/mod.rs

@@ -0,0 +1,16 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+pub mod optimizer;
+pub mod planner;

crates/s3select-query/src/sql/physical/optimizer.rs

@@ -0,0 +1,26 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::sync::Arc;
+
+use datafusion::physical_optimizer::PhysicalOptimizerRule;
+use datafusion::physical_plan::ExecutionPlan;
+use rustfs_s3select_api::QueryResult;
+use rustfs_s3select_api::query::session::SessionCtx;
+
+pub trait PhysicalOptimizer {
+    fn optimize(&self, plan: Arc<dyn ExecutionPlan>, session: &SessionCtx) -> QueryResult<Arc<dyn ExecutionPlan>>;
+
+    fn inject_optimizer_rule(&mut self, optimizer_rule: Arc<dyn PhysicalOptimizerRule + Send + Sync>);
+}

crates/s3select-query/src/sql/physical/planner.rs

@@ -0,0 +1,118 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::sync::Arc;
+
+use async_trait::async_trait;
+use datafusion::execution::SessionStateBuilder;
+use datafusion::logical_expr::LogicalPlan;
+use datafusion::physical_optimizer::PhysicalOptimizerRule;
+use datafusion::physical_optimizer::aggregate_statistics::AggregateStatistics;
+use datafusion::physical_optimizer::coalesce_batches::CoalesceBatches;
+use datafusion::physical_optimizer::join_selection::JoinSelection;
+use datafusion::physical_plan::ExecutionPlan;
+use datafusion::physical_planner::{
+    DefaultPhysicalPlanner as DFDefaultPhysicalPlanner, ExtensionPlanner, PhysicalPlanner as DFPhysicalPlanner,
+};
+use rustfs_s3select_api::QueryResult;
+use rustfs_s3select_api::query::physical_planner::PhysicalPlanner;
+use rustfs_s3select_api::query::session::SessionCtx;
+
+use super::optimizer::PhysicalOptimizer;
+
+pub struct DefaultPhysicalPlanner {
+    ext_physical_transform_rules: Vec<Arc<dyn ExtensionPlanner + Send + Sync>>,
+    /// Responsible for optimizing a physical execution plan
+    ext_physical_optimizer_rules: Vec<Arc<dyn PhysicalOptimizerRule + Send + Sync>>,
+}
+
+impl DefaultPhysicalPlanner {
+    #[allow(dead_code)]
+    fn with_physical_transform_rules(mut self, rules: Vec<Arc<dyn ExtensionPlanner + Send + Sync>>) -> Self {
+        self.ext_physical_transform_rules = rules;
+        self
+    }
+}
+
+impl DefaultPhysicalPlanner {
+    #[allow(dead_code)]
+    fn with_optimizer_rules(mut self, rules: Vec<Arc<dyn PhysicalOptimizerRule + Send + Sync>>) -> Self {
+        self.ext_physical_optimizer_rules = rules;
+        self
+    }
+}
+
+impl Default for DefaultPhysicalPlanner {
+    fn default() -> Self {
+        let ext_physical_transform_rules: Vec<Arc<dyn ExtensionPlanner + Send + Sync>> = vec![
+            // can add rules at here
+        ];
+
+        // We need to take care of the rule ordering. They may influence each other.
+        let ext_physical_optimizer_rules: Vec<Arc<dyn PhysicalOptimizerRule + Sync + Send>> = vec![
+            Arc::new(AggregateStatistics::new()),
+            // Statistics-based join selection will change the Auto mode to a real join implementation,
+            // like collect left, or hash join, or future sort merge join, which will influence the
+            // EnforceDistribution and EnforceSorting rules as they decide whether to add additional
+            // repartitioning and local sorting steps to meet distribution and ordering requirements.
+            // Therefore, it should run before EnforceDistribution and EnforceSorting.
+            Arc::new(JoinSelection::new()),
+            // The CoalesceBatches rule will not influence the distribution and ordering of the
+            // whole plan tree. Therefore, to avoid influencing other rules, it should run last.
+            Arc::new(CoalesceBatches::new()),
+        ];
+
+        Self {
+            ext_physical_transform_rules,
+            ext_physical_optimizer_rules,
+        }
+    }
+}
+
+#[async_trait]
+impl PhysicalPlanner for DefaultPhysicalPlanner {
+    async fn create_physical_plan(
+        &self,
+        logical_plan: &LogicalPlan,
+        session: &SessionCtx,
+    ) -> QueryResult<Arc<dyn ExecutionPlan>> {
+        // 将扩展的物理计划优化规则注入 df 的 session state
+        let new_state = SessionStateBuilder::new_from_existing(session.inner().clone())
+            .with_physical_optimizer_rules(self.ext_physical_optimizer_rules.clone())
+            .build();
+
+        // 通过扩展的物理计划转换规则构造 df 的 Physical Planner
+        let planner = DFDefaultPhysicalPlanner::with_extension_planners(self.ext_physical_transform_rules.clone());
+
+        // 执行 df 的物理计划规划及优化
+        planner
+            .create_physical_plan(logical_plan, &new_state)
+            .await
+            .map_err(|e| e.into())
+    }
+
+    fn inject_physical_transform_rule(&mut self, rule: Arc<dyn ExtensionPlanner + Send + Sync>) {
+        self.ext_physical_transform_rules.push(rule)
+    }
+}
+
+impl PhysicalOptimizer for DefaultPhysicalPlanner {
+    fn optimize(&self, plan: Arc<dyn ExecutionPlan>, _session: &SessionCtx) -> QueryResult<Arc<dyn ExecutionPlan>> {
+        Ok(plan)
+    }
+
+    fn inject_optimizer_rule(&mut self, optimizer_rule: Arc<dyn PhysicalOptimizerRule + Send + Sync>) {
+        self.ext_physical_optimizer_rules.push(optimizer_rule);
+    }
+}

crates/s3select-query/src/sql/planner.rs

@@ -0,0 +1,74 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use async_recursion::async_recursion;
+use async_trait::async_trait;
+use datafusion::sql::{planner::SqlToRel, sqlparser::ast::Statement};
+use rustfs_s3select_api::{
+    QueryError, QueryResult,
+    query::{
+        ast::ExtStatement,
+        logical_planner::{LogicalPlanner, Plan, QueryPlan},
+        session::SessionCtx,
+    },
+};
+
+use crate::metadata::ContextProviderExtension;
+
+pub struct SqlPlanner<'a, S: ContextProviderExtension> {
+    _schema_provider: &'a S,
+    df_planner: SqlToRel<'a, S>,
+}
+
+#[async_trait]
+impl<S: ContextProviderExtension + Send + Sync> LogicalPlanner for SqlPlanner<'_, S> {
+    async fn create_logical_plan(&self, statement: ExtStatement, session: &SessionCtx) -> QueryResult<Plan> {
+        let plan = { self.statement_to_plan(statement, session).await? };
+
+        Ok(plan)
+    }
+}
+
+impl<'a, S: ContextProviderExtension + Send + Sync + 'a> SqlPlanner<'a, S> {
+    /// Create a new query planner
+    pub fn new(schema_provider: &'a S) -> Self {
+        SqlPlanner {
+            _schema_provider: schema_provider,
+            df_planner: SqlToRel::new(schema_provider),
+        }
+    }
+
+    /// Generate a logical plan from an  Extent SQL statement
+    #[async_recursion]
+    pub(crate) async fn statement_to_plan(&self, statement: ExtStatement, session: &SessionCtx) -> QueryResult<Plan> {
+        match statement {
+            ExtStatement::SqlStatement(stmt) => self.df_sql_to_plan(*stmt, session).await,
+        }
+    }
+
+    async fn df_sql_to_plan(&self, stmt: Statement, _session: &SessionCtx) -> QueryResult<Plan> {
+        match stmt {
+            Statement::Query(_) => {
+                let df_plan = self.df_planner.sql_statement_to_plan(stmt)?;
+                let plan = Plan::Query(QueryPlan {
+                    df_plan,
+                    is_tag_scan: false,
+                });
+
+                Ok(plan)
+            }
+            _ => Err(QueryError::NotImplemented { err: stmt.to_string() }),
+        }
+    }
+}