Parse binary operations with correct operator precedence

Author: Glyphack

parser/src/parser/parser.rs

@@ -929,7 +929,7 @@ impl<'a> Parser<'a> {
     // https://docs.python.org/3/reference/compound_stmts.html#literal-patterns
     fn parse_literal_pattern(&mut self) -> Result<MatchPattern, ParsingError> {
         let node = self.start_node();
-        let value = self.parse_binary_arithmetic_operation()?;
+        let value = self.parse_binary_arithmetic_operation(0)?;
         Ok(MatchPattern::MatchValue(MatchValue {
             node: self.finish_node(node),
             value,
@@ -2189,15 +2189,15 @@ impl<'a> Parser<'a> {
     // https://docs.python.org/3/reference/expressions.html#shifting-operations
     fn parse_shift_expr(&mut self) -> Result<Expression, ParsingError> {
         let node = self.start_node();
-        let mut arith_expr = self.parse_binary_arithmetic_operation()?;
+        let mut arith_expr = self.parse_binary_arithmetic_operation(0)?;
         if self.at(Kind::LeftShift) || self.at(Kind::RightShift) {
             let op = if self.eat(Kind::LeftShift) {
                 BinaryOperator::LShift
             } else {
                 self.bump(Kind::RightShift);
                 BinaryOperator::RShift
             };
-            let lhs = self.parse_binary_arithmetic_operation()?;
+            let lhs = self.parse_binary_arithmetic_operation(0)?;
             arith_expr = Expression::BinOp(Box::new(BinOp {
                 node: self.finish_node(node),
                 op,
@@ -2209,12 +2209,24 @@ impl<'a> Parser<'a> {
     }
 
     // https://docs.python.org/3/reference/expressions.html#binary-arithmetic-operations
-    fn parse_binary_arithmetic_operation(&mut self) -> Result<Expression, ParsingError> {
+    //
+    fn parse_binary_arithmetic_operation(
+        &mut self,
+        min_precedence: u8,
+    ) -> Result<Expression, ParsingError> {
         let node = self.start_node();
         let mut lhs = self.parse_unary_arithmetic_operation()?;
-        while self.cur_kind().is_bin_arithmetic_op() {
-            let op = self.parse_bin_arithmetic_op()?;
-            let rhs = self.parse_unary_arithmetic_operation()?;
+        while let Some((op, precedence, associativity)) = self.cur_kind().bin_op_precedence() {
+            if precedence < min_precedence {
+                break;
+            }
+            self.bump_any();
+            let next_precedence = match associativity {
+                0 => precedence + 1,
+                1 => precedence,
+                _ => unreachable!(),
+            };
+            let rhs = self.parse_binary_arithmetic_operation(next_precedence)?;
             lhs = Expression::BinOp(Box::new(BinOp {
                 node: self.finish_node(node),
                 op,

parser/src/token.rs

@@ -1,6 +1,8 @@
 use core::panic;
 use std::fmt::Display;
 
+use crate::ast::BinaryOperator;
+
 #[derive(Debug, Clone, PartialEq)]
 pub struct Token {
     pub kind: Kind,
@@ -177,18 +179,18 @@ impl Kind {
         matches!(self, Kind::Not | Kind::BitNot | Kind::Minus | Kind::Plus)
     }
 
-    pub fn is_bin_arithmetic_op(&self) -> bool {
-        matches!(
-            self,
-            Kind::Plus
-                | Kind::Minus
-                | Kind::Mul
-                | Kind::MatrixMul
-                | Kind::Div
-                | Kind::Mod
-                | Kind::Pow
-                | Kind::IntDiv
-        )
+    pub fn bin_op_precedence(&self) -> Option<(BinaryOperator, u8, u8)> {
+        match self {
+            Kind::Plus => Some((BinaryOperator::Add, 9, 0)),
+            Kind::Minus => Some((BinaryOperator::Sub, 9, 0)),
+            Kind::Mul => Some((BinaryOperator::Mult, 10, 0)),
+            Kind::MatrixMul => Some((BinaryOperator::MatMult, 10, 0)),
+            Kind::Div => Some((BinaryOperator::Div, 10, 0)),
+            Kind::Mod => Some((BinaryOperator::Mod, 10, 0)),
+            Kind::Pow => Some((BinaryOperator::Pow, 10, 0)),
+            Kind::IntDiv => Some((BinaryOperator::FloorDiv, 10, 0)),
+            _ => None,
+        }
     }
 
     pub fn is_comparison_operator(&self) -> bool {

parser/test_data/inputs/binary_op.py

@@ -10,7 +10,7 @@
 
 1 % 2
 
-1 ** 2
+1**2
 
 1 << 2
 
@@ -25,3 +25,11 @@
 1 | 2 | 3
 
 1 @ 2
+
+1 + 2 * 3
+
+1 * 2 + 3
+
+1 ^ 2 + 3
+
+3 + (1 + 2) * 3

parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@binary_op.py.snap

@@ -1,6 +1,6 @@
 ---
 source: parser/src/lexer/mod.rs
-description: "1 + 2\n\n1 - 2\n\n1 * 2\n\n1 / 2\n\n1 // 2\n\n1 % 2\n\n1 ** 2\n\n1 << 2\n\n1 >> 2\n\n1 & 2\n\n1 ^ 2\n\n1 | 2\n\n1 | 2 | 3\n\n1 @ 2\n"
+description: "1 + 2\n\n1 - 2\n\n1 * 2\n\n1 / 2\n\n1 // 2\n\n1 % 2\n\n1**2\n\n1 << 2\n\n1 >> 2\n\n1 & 2\n\n1 ^ 2\n\n1 | 2\n\n1 | 2 | 3\n\n1 @ 2\n\n1 + 2 * 3\n\n1 * 2 + 3\n\n1 ^ 2 + 3\n\n3 + (1 + 2) * 3\n"
 input_file: parser/test_data/inputs/binary_op.py
 ---
 0,1: Integer   (Number("1"))
@@ -34,43 +34,75 @@ input_file: parser/test_data/inputs/binary_op.py
 41,42: NewLine   (None)
 42,43: NL   (None)
 43,44: Integer   (Number("1"))
-45,47: **   (None)
-48,49: Integer   (Number("2"))
-49,50: NewLine   (None)
-50,51: NL   (None)
-51,52: Integer   (Number("1"))
-53,55: <<   (None)
-56,57: Integer   (Number("2"))
-57,58: NewLine   (None)
-58,59: NL   (None)
-59,60: Integer   (Number("1"))
-61,63: >>   (None)
-64,65: Integer   (Number("2"))
-65,66: NewLine   (None)
-66,67: NL   (None)
-67,68: Integer   (Number("1"))
-69,70: &   (None)
-71,72: Integer   (Number("2"))
-72,73: NewLine   (None)
-73,74: NL   (None)
-74,75: Integer   (Number("1"))
-76,77: ^   (None)
-78,79: Integer   (Number("2"))
-79,80: NewLine   (None)
-80,81: NL   (None)
-81,82: Integer   (Number("1"))
-83,84: |   (None)
-85,86: Integer   (Number("2"))
-86,87: NewLine   (None)
-87,88: NL   (None)
-88,89: Integer   (Number("1"))
-90,91: |   (None)
-92,93: Integer   (Number("2"))
-94,95: |   (None)
-96,97: Integer   (Number("3"))
-97,98: NewLine   (None)
-98,99: NL   (None)
-99,100: Integer   (Number("1"))
-101,102: @   (None)
-103,104: Integer   (Number("2"))
-104,105: NewLine   (None)
+44,46: **   (None)
+46,47: Integer   (Number("2"))
+47,48: NewLine   (None)
+48,49: NL   (None)
+49,50: Integer   (Number("1"))
+51,53: <<   (None)
+54,55: Integer   (Number("2"))
+55,56: NewLine   (None)
+56,57: NL   (None)
+57,58: Integer   (Number("1"))
+59,61: >>   (None)
+62,63: Integer   (Number("2"))
+63,64: NewLine   (None)
+64,65: NL   (None)
+65,66: Integer   (Number("1"))
+67,68: &   (None)
+69,70: Integer   (Number("2"))
+70,71: NewLine   (None)
+71,72: NL   (None)
+72,73: Integer   (Number("1"))
+74,75: ^   (None)
+76,77: Integer   (Number("2"))
+77,78: NewLine   (None)
+78,79: NL   (None)
+79,80: Integer   (Number("1"))
+81,82: |   (None)
+83,84: Integer   (Number("2"))
+84,85: NewLine   (None)
+85,86: NL   (None)
+86,87: Integer   (Number("1"))
+88,89: |   (None)
+90,91: Integer   (Number("2"))
+92,93: |   (None)
+94,95: Integer   (Number("3"))
+95,96: NewLine   (None)
+96,97: NL   (None)
+97,98: Integer   (Number("1"))
+99,100: @   (None)
+101,102: Integer   (Number("2"))
+102,103: NewLine   (None)
+103,104: NL   (None)
+104,105: Integer   (Number("1"))
+106,107: +   (None)
+108,109: Integer   (Number("2"))
+110,111: *   (None)
+112,113: Integer   (Number("3"))
+113,114: NewLine   (None)
+114,115: NL   (None)
+115,116: Integer   (Number("1"))
+117,118: *   (None)
+119,120: Integer   (Number("2"))
+121,122: +   (None)
+123,124: Integer   (Number("3"))
+124,125: NewLine   (None)
+125,126: NL   (None)
+126,127: Integer   (Number("1"))
+128,129: ^   (None)
+130,131: Integer   (Number("2"))
+132,133: +   (None)
+134,135: Integer   (Number("3"))
+135,136: NewLine   (None)
+136,137: NL   (None)
+137,138: Integer   (Number("3"))
+139,140: +   (None)
+141,142: (   (None)
+142,143: Integer   (Number("1"))
+144,145: +   (None)
+146,147: Integer   (Number("2"))
+147,148: )   (None)
+149,150: *   (None)
+151,152: Integer   (Number("3"))
+152,153: NewLine   (None)