Method Declarations & Trailing Exprs & Constructors (#2337)

Method declarations and a bunch of other AST / HIR / THIR / Codegen
fixes.

<!-- ELLIPSIS_HIDDEN -->


----

> [!IMPORTANT]
> Enhance BAML compiler and VM with method declarations, trailing
expression handling, and improved constructor logic, including
comprehensive test updates.
> 
>   - **Behavior**:
> - Add support for method declarations in classes, including `self`
parameter handling in `parse_named_args_list.rs` and
`parse_type_expression_block.rs`.
> - Implement trailing expression handling in blocks, replacing
`produces_final_value()` with `trailing_expr` in `hir.rs` and `thir.rs`.
> - Update `compile_thir_to_bytecode()` in `codegen.rs` to handle method
bytecode generation.
>     - Modify `Vm::len()` in `native.rs` to use `std.Array.len`.
>   - **Parsing**:
> - Update `parse_expr.rs` and `parse_named_args_list.rs` to support new
syntax for method declarations and assignments.
> - Adjust `parse_type_expression_block.rs` to parse methods within
classes.
>   - **Validation**:
> - Enhance `validate_expr_fns()` in `expr_fns.rs` to check for method
name uniqueness and argument validity.
>   - **Testing**:
> - Add and update tests in `bytecode_tests.rs` and `vm.rs` to cover new
method declaration and execution scenarios.
> - Update bytecode and HIR test files to reflect changes in method
handling and trailing expressions.
> 
> <sup>This description was created by </sup>[<img alt="Ellipsis"
src="https://img.shields.io/badge/Ellipsis-blue?color=175173">](https://www.ellipsis.dev?ref=BoundaryML%2Fbaml&utm_source=github&utm_medium=referral)<sup>
for 7b80c60. You can
[customize](https://app.ellipsis.dev/BoundaryML/settings/summaries) this
summary. It will automatically update as commits are pushed.</sup>


<!-- ELLIPSIS_HIDDEN -->

---------

Co-authored-by: Luke Ramsden <hello@lukeramsden.com>
Co-authored-by: aaronvg <aaron@boundaryml.com>
Co-authored-by: hellovai <vbv@boundaryml.com>
Co-authored-by: Cursor Agent <cursoragent@cursor.com>
Co-authored-by: ellipsis-dev[bot] <65095814+ellipsis-dev[bot]@users.noreply.github.com>
Co-authored-by: Jesús Lapastora <jesus@boundaryml.com>
Co-authored-by: Samuel Lijin <sam@boundaryml.com>
Co-authored-by: Ayush Goyal <ayushg1214@gmail.com>
Co-authored-by: Chris Watts <chris@boundaryml.com>

Author: 10 people

engine/baml-compiler/src/builtin.rs

@@ -17,6 +17,7 @@ pub mod enums {
 pub fn builtin_classes() -> Vec<Class> {
     vec![Class {
         name: String::from(classes::REQUEST),
+        methods: vec![],
         fields: vec![
             Field {
                 name: String::from("base_url"),

engine/baml-compiler/src/codegen.rs

@@ -123,18 +123,20 @@ impl Statement {
                 .append(RcDoc::space())
                 .append(RcDoc::text(name.clone()))
                 .append(RcDoc::text(";")),
-            Statement::Assign { name, value, .. } => RcDoc::text(name.clone())
+            Statement::Assign { left, value, .. } => left
+                .to_doc()
                 .append(RcDoc::space())
                 .append(RcDoc::text("="))
                 .append(RcDoc::space())
                 .append(value.to_doc())
                 .append(RcDoc::text(";")),
             Statement::AssignOp {
-                name,
+                left,
                 value,
                 assign_op,
                 ..
-            } => RcDoc::text(name.clone())
+            } => left
+                .to_doc()
                 .append(RcDoc::space())
                 .append(assign_op.to_doc())
                 .append(RcDoc::space())
@@ -157,7 +159,7 @@ impl Statement {
                 .append(condition.to_doc())
                 .append(RcDoc::text(";")),
             Statement::Expression { expr, .. } => expr.to_doc(),
-            Statement::SemicolonExpression { expr, .. } => expr.to_doc(),
+            Statement::Semicolon { expr, .. } => expr.to_doc(),
             Statement::While {
                 condition, block, ..
             } => RcDoc::text("while")
@@ -268,7 +270,8 @@ impl LlmFunction {
 
 impl ExprFunction {
     pub fn to_doc(&self) -> RcDoc<'static, ()> {
-        let body_doc = if self.body.statements.is_empty() {
+        // TODO: Why nesting doesn't work if calling self.body.to_doc().nest(2)?
+        let mut body_doc = if self.body.statements.is_empty() {
             RcDoc::nil()
         } else {
             // The key is to apply nest() to the entire content that includes line breaks
@@ -284,6 +287,15 @@ impl ExprFunction {
                 .append(RcDoc::hardline())
                 .nest(2)
         };
+
+        if let Some(expr) = &self.body.trailing_expr {
+            body_doc = body_doc.append(
+                RcDoc::hardline()
+                    .append(expr.to_doc().append(RcDoc::hardline()))
+                    .nest(2),
+            );
+        }
+
         RcDoc::text("function")
             .append(RcDoc::space())
             .append(RcDoc::text(self.name.clone()))
@@ -308,7 +320,7 @@ impl ExprFunction {
 
 impl Block {
     pub fn to_doc(&self) -> RcDoc<'static, ()> {
-        if self.statements.is_empty() {
+        let doc = if self.statements.is_empty() {
             RcDoc::nil()
         } else {
             RcDoc::intersperse(
@@ -318,6 +330,12 @@ impl Block {
                     .collect::<Vec<_>>(),
                 RcDoc::hardline(),
             )
+        };
+
+        if let Some(expr) = &self.trailing_expr {
+            doc.append(RcDoc::hardline()).append(expr.to_doc())
+        } else {
+            doc
         }
     }
 }
@@ -448,7 +466,7 @@ impl Expression {
                         .append(RcDoc::space())
                 })
                 .append(RcDoc::text("}")),
-            Expression::ExpressionBlock(block, _) => RcDoc::text("{")
+            Expression::Block(block, _) => RcDoc::text("{")
                 .append(RcDoc::hardline())
                 .append(block.to_doc().nest(2))
                 .append(RcDoc::hardline())

engine/baml-compiler/src/hir/dump.rs

@@ -267,65 +267,35 @@ impl ExprFunction {
             name: function.name.to_string(),
             parameters: lower_fn_args(&function.args),
             return_type: TypeM::from_ast_optional(function.return_type.as_ref()),
-            body: Block::from_function_body(&function.body),
+            body: Block::from_expr_block(&function.body),
             span: function.span.clone(),
         }
     }
 }
 
 impl Block {
-    /// Lower an expression block into HIR for function bodies (ends with Statement::Return).
-    pub fn from_function_body(block: &ast::ExpressionBlock) -> Self {
-        Self::from_ast_with_context(block, true)
-    }
-
-    /// Lower an expression block into HIR for expression blocks (ends with Statement::Expression).
-    pub fn from_expression_block(block: &ast::ExpressionBlock) -> Self {
-        Self::from_ast_with_context(block, false)
-    }
-
-    /// Lower an expression block into HIR with specified context.
-    /// If is_function_body is true, the final expression becomes Statement::Return.
-    /// If is_function_body is false, the final expression becomes Statement::Expression.
-    fn from_ast_with_context(block: &ast::ExpressionBlock, is_function_body: bool) -> Self {
-        let mut statements = vec![];
-
-        // Process statements, checking for if expressions in let bindings
-        for stmt in &block.stmts {
-            let hir_stmt = lower_stmt(stmt);
-            statements.push(hir_stmt);
+    /// Lower an expression block into HIR for expression blocks.
+    pub fn from_expr_block(block: &ast::ExpressionBlock) -> Self {
+        Block {
+            statements: block.stmts.iter().map(lower_stmt).collect(),
+            trailing_expr: block
+                .expr
+                .as_deref()
+                .map(Expression::from_ast)
+                .map(Box::new),
         }
-
-        if let Some(block_final_expr) = block.expr.as_ref() {
-            let final_expr = Expression::from_ast(block_final_expr);
-
-            // Then add the final statement
-            statements.push(if is_function_body {
-                Statement::Return {
-                    expr: final_expr,
-                    span: block_final_expr.span().clone(),
-                }
-            } else {
-                Statement::Expression {
-                    expr: final_expr,
-                    span: block_final_expr.span().clone(),
-                }
-            });
-        }
-
-        Block { statements }
     }
 }
 
 fn lower_stmt(stmt: &ast::Stmt) -> Statement {
-    let hir_stmt = match stmt {
+    match stmt {
         ast::Stmt::CForLoop(stmt) => {
             // we'll add  a block if we an init statement, otherwise we'll just
             // use the current context to push the while statement.
 
             let condition = stmt.condition.as_ref().map(Expression::from_ast);
             let init = stmt.init_stmt.as_ref().map(|b| lower_stmt(b));
-            let block = Block::from_expression_block(&stmt.body);
+            let block = Block::from_expr_block(&stmt.body);
             let after = stmt
                 .after_stmt
                 .as_ref()
@@ -349,9 +319,10 @@ fn lower_stmt(stmt: &ast::Stmt) -> Statement {
                 Some(init) => {
                     // use a block
                     Statement::Expression {
-                        expr: Expression::ExpressionBlock(
+                        expr: Expression::Block(
                             Block {
                                 statements: vec![init, inner_loop],
+                                trailing_expr: None,
                             },
                             stmt.span.clone(),
                         ),
@@ -373,30 +344,26 @@ fn lower_stmt(stmt: &ast::Stmt) -> Statement {
 
             let condition = Expression::from_ast(condition);
 
-            let body = Block::from_expression_block(body);
+            let body = Block::from_expr_block(body);
 
             Statement::While {
                 condition,
                 block: body,
                 span: span.clone(),
             }
         }
-        ast::Stmt::Assign(ast::AssignStmt {
-            identifier,
-            expr,
-            span,
-        }) => Statement::Assign {
-            name: identifier.to_string(),
+        ast::Stmt::Assign(ast::AssignStmt { left, expr, span }) => Statement::Assign {
+            left: Expression::from_ast(left),
             value: Expression::from_ast(expr),
             span: span.clone(),
         },
         ast::Stmt::AssignOp(ast::AssignOpStmt {
-            identifier,
+            left,
             assign_op,
             expr,
             span,
         }) => Statement::AssignOp {
-            name: identifier.to_string(),
+            left: Expression::from_ast(left),
             assign_op: match assign_op {
                 ast::AssignOp::AddAssign => hir::AssignOp::AddAssign,
                 ast::AssignOp::SubAssign => hir::AssignOp::SubAssign,
@@ -447,32 +414,18 @@ fn lower_stmt(stmt: &ast::Stmt) -> Statement {
             Statement::ForLoop {
                 identifier: identifier.name().to_string(),
                 iterator: Box::new(lifted_iterator),
-                block: Block::from_expression_block(body),
+                block: Block::from_expr_block(body),
                 span: span.clone(),
             }
         }
-        ast::Stmt::Expression(expr) => {
-            let hir_expr = Expression::from_ast(expr);
-
-            // Expressions that contain blocks themselves will deal with
-            // return expressions recursively. But expressions that have
-            // no blocks (like function calls or 2 + 2) must drop the
-            // returned value, so we insert semicolon expressions.
-            if matches!(
-                expr,
-                ast::Expression::If(..) | ast::Expression::ExprBlock(..)
-            ) {
-                Statement::Expression {
-                    expr: hir_expr,
-                    span: expr.span().clone(),
-                }
-            } else {
-                Statement::SemicolonExpression {
-                    expr: hir_expr,
-                    span: expr.span().clone(),
-                }
-            }
-        }
+        ast::Stmt::Expression(expr) => Statement::Expression {
+            expr: Expression::from_ast(expr),
+            span: expr.span().clone(),
+        },
+        ast::Stmt::Semicolon(expr) => Statement::Semicolon {
+            expr: Expression::from_ast(expr),
+            span: expr.span().clone(),
+        },
         ast::Stmt::Return(ReturnStmt { value, span }) => Statement::Return {
             expr: Expression::from_ast(value),
             span: span.clone(),
@@ -481,8 +434,7 @@ fn lower_stmt(stmt: &ast::Stmt) -> Statement {
             condition: Expression::from_ast(value),
             span: span.clone(),
         },
-    };
-    hir_stmt
+    }
 }
 
 impl Expression {
@@ -577,7 +529,7 @@ impl Expression {
                 // Expression blocks are lowered to HIR preserving their structure
                 // This maintains proper scoping - variables defined inside the block
                 // are only visible within that block
-                Expression::ExpressionBlock(Block::from_expression_block(block), span.clone())
+                Expression::Block(Block::from_expr_block(block), span.clone())
             }
             ast::Expression::Lambda(_, _, _) => {
                 todo!("lambdas are not yet implemented")
@@ -688,6 +640,7 @@ impl Class {
                     span: field.span().clone(),
                 })
                 .collect(),
+            methods: class.methods.iter().map(ExprFunction::from_ast).collect(),
             span: class.span().clone(),
         }
     }

engine/baml-compiler/src/hir/lowering.rs

@@ -279,6 +279,8 @@ pub struct LlmFunction {
 pub struct Class {
     pub name: String,
     pub fields: Vec<Field>,
+    // TODO: Allow LLM functions here.
+    pub methods: Vec<ExprFunction>,
     pub span: Span,
 }
 
@@ -312,7 +314,11 @@ pub struct Parameter {
 
 #[derive(Clone, Debug)]
 pub struct Block {
+    /// List of statements.
     pub statements: Vec<Statement>,
+
+    /// Final expression in the block without semicolon (used as return).
+    pub trailing_expr: Option<Box<Expression>>,
 }
 
 /// A single unit of execution within a block.
@@ -333,12 +339,12 @@ pub enum Statement {
     },
     /// Assign a mutable variable.
     Assign {
-        name: String,
+        left: Expression,
         value: Expression,
         span: Span,
     },
     AssignOp {
-        name: String,
+        left: Expression,
         assign_op: AssignOp,
         value: Expression,
         span: Span,
@@ -359,7 +365,7 @@ pub enum Statement {
         expr: Expression,
         span: Span,
     },
-    SemicolonExpression {
+    Semicolon {
         expr: Expression,
         span: Span,
     },
@@ -456,7 +462,7 @@ pub enum Expression {
     // MethodCall(Box<Expression>, String, Vec<Expression>), // TODO.
     ClassConstructor(ClassConstructor, Span),
     /// Expression block - has its own scope with statements and evaluates to a value
-    ExpressionBlock(Block, Span),
+    Block(Block, Span),
     BinaryOperation {
         left: Box<Expression>,
         operator: BinaryOperator,
@@ -545,7 +551,7 @@ impl Expression {
             Expression::JinjaExpressionValue(_, span) => span.clone(),
             Expression::Call { span, .. } => span.clone(),
             Expression::ClassConstructor(_, span) => span.clone(),
-            Expression::ExpressionBlock(_, span) => span.clone(),
+            Expression::Block(_, span) => span.clone(),
             Expression::BinaryOperation { span, .. } => span.clone(),
             Expression::UnaryOperation { span, .. } => span.clone(),
             Expression::Paren(_, span) => span.clone(),