Translate MySQL CONVERT() expressions to SQLite (#356)

JanJakes · web-flow · commit b9af09ce0d25 · 2026-04-10T10:59:02.000+02:00
## Summary Fixes #344. MySQL's `CONVERT()` function was passed through to SQLite unchanged, causing queries like `SELECT CONVERT('Customer' USING utf8mb4) COLLATE utf8mb4_bin` to fail with a syntax error. This PR has two parts: 1. **Extract `simpleExprBody` as a named grammar rule.** The `%simpleExpr_factored` fragment is promoted to a real `simpleExprBody` rule so it creates its own AST node. This separates the core expression (CONVERT, CAST, literals, etc.) from trailing modifiers (COLLATE, CONCAT_PIPES) that remain in the parent `simpleExpr` node, making individual expression handlers simpler. 2. **Translate `CONVERT()` expressions.** Adds explicit handling for both forms of `CONVERT()` in the AST-based driver: - **`CONVERT(expr, type)`** is translated to `CAST(expr AS type)`, reusing the existing `castType` translation. - **`CONVERT(expr USING charset)`** is reduced to just the expression, as SQLite stores all text as UTF-8 and charset conversions are not needed. ## Test plan - [x] Added `testConvert` translation test — verifies SQL-to-SQL translation for both CONVERT forms and COLLATE. - [x] Added `testConvertExpression` — verifies CONVERT with type casting (BINARY, CHAR, SIGNED, UNSIGNED, DECIMAL, DATE). - [x] Added `testConvertUsingExpression` — verifies CONVERT with charset conversion (utf8mb4, utf8, latin1). - [x] Added `testConvertUsingWithCollate` — verifies the exact query from #344. - [x] Added `testConvertWithColumnReferences` — verifies CONVERT with column references in SELECT, WHERE, and ORDER BY. - [x] Full test suite passes (640 tests, 0 failures). - [x] PHPCS passes.
diff --git a/.github/workflows/wp-tests-phpunit-run.js b/.github/workflows/wp-tests-phpunit-run.js
@@ -58,9 +58,6 @@ const expectedFailures = [
 	'Tests_DB_Charset::test_strip_invalid_text with data set #32',
 	'Tests_DB_Charset::test_strip_invalid_text with data set #33',
 	'Tests_DB_Charset::test_strip_invalid_text with data set #34',
-	'Tests_DB_Charset::test_strip_invalid_text with data set #35',
-	'Tests_DB_Charset::test_strip_invalid_text with data set #36',
-	'Tests_DB_Charset::test_strip_invalid_text with data set #37',
 	'Tests_DB_Charset::test_strip_invalid_text with data set #39',
 	'Tests_DB_Charset::test_strip_invalid_text with data set #40',
 	'Tests_DB_Charset::test_strip_invalid_text with data set #41',
diff --git a/grammar-tools/MySQLParser.g4 b/grammar-tools/MySQLParser.g4
@@ -2961,13 +2961,13 @@ bitExpr: simpleExpr %bitExpr_rr*;
 
 /*
  * @CHANGED:
- * Factored left recursion.
+ * Factored left recursion and introduced "simpleExprBody" for easier processing.
  */
 simpleExpr: %simpleExpr_collate (CONCAT_PIPES_SYMBOL %simpleExpr_collate)*;
 
-%simpleExpr_collate: %simpleExpr_factored (COLLATE_SYMBOL textOrIdentifier)?;
+%simpleExpr_collate: simpleExprBody (COLLATE_SYMBOL textOrIdentifier)?;
 
-%simpleExpr_factored:
+simpleExprBody:
     literal                                                                                              # simpleExprLiteral
     | sumExpr                                                                                            # simpleExprSum
     | variable (equal expr)?                                                                             # simpleExprVariable
diff --git a/packages/mysql-on-sqlite/src/mysql/mysql-grammar.php b/packages/mysql-on-sqlite/src/mysql/mysql-grammar.php
diff --git a/packages/mysql-on-sqlite/src/sqlite/class-wp-pdo-mysql-on-sqlite.php b/packages/mysql-on-sqlite/src/sqlite/class-wp-pdo-mysql-on-sqlite.php
@@ -3691,8 +3691,8 @@ private function translate( $node ): ?string {
 					return null;
 				}
 				return $this->translate_sequence( $node->get_children() );
-			case 'simpleExpr':
-				return $this->translate_simple_expr( $node );
+			case 'simpleExprBody':
+				return $this->translate_simple_expr_body( $node );
 			case 'predicateOperations':
 				$token = $node->get_first_child_token();
 				if ( WP_MySQL_Lexer::LIKE_SYMBOL === $token->id ) {
@@ -3790,6 +3790,8 @@ private function translate( $node ): ?string {
 						return 'TEXT';
 					case WP_MySQL_Lexer::SIGNED_SYMBOL:
 					case WP_MySQL_Lexer::UNSIGNED_SYMBOL:
+						// @TODO: Emulate UNSIGNED semantics. MySQL wraps negative
+						//        values, but SQLite has no unsigned integer type.
 						return 'INTEGER';
 					case WP_MySQL_Lexer::DECIMAL_SYMBOL:
 					case WP_MySQL_Lexer::FLOAT_SYMBOL:
@@ -4204,13 +4206,13 @@ private function translate_query_specification( WP_Parser_Node $node ): string {
 	}
 
 	/**
-	 * Translate a MySQL simple expression to SQLite.
+	 * Translate a MySQL simple expression body to SQLite.
 	 *
-	 * @param WP_Parser_Node $node        The "simpleExpr" AST node.
+	 * @param WP_Parser_Node $node        The "simpleExprBody" AST node.
 	 * @return string                     The translated value.
 	 * @throws WP_SQLite_Driver_Exception When the translation fails.
 	 */
-	private function translate_simple_expr( WP_Parser_Node $node ): string {
+	private function translate_simple_expr_body( WP_Parser_Node $node ): string {
 		$token = $node->get_first_child_token();
 
 		// Translate "VALUES(col)" to "excluded.col" in ON DUPLICATE KEY UPDATE.
@@ -4221,6 +4223,28 @@ private function translate_simple_expr( WP_Parser_Node $node ): string {
 			);
 		}
 
+		/**
+		 * Translate MySQL CONVERT() expression.
+		 *
+		 * MySQL supports two forms of CONVERT():
+		 *   1. CONVERT(expr, type):         Equivalent to CAST(expr AS type).
+		 *   2. CONVERT(expr USING charset): Converts the character set.
+		 */
+		if ( null !== $token && WP_MySQL_Lexer::CONVERT_SYMBOL === $token->id ) {
+			$expr      = $this->translate( $node->get_first_child_node( 'expr' ) );
+			$cast_type = $node->get_first_child_node( 'castType' );
+
+			if ( null !== $cast_type ) {
+				// CONVERT(expr, type): Translate to cast expression.
+				// TODO: Emulate UNSIGNED cast. SQLite has no unsigned integer type.
+				return sprintf( 'CAST(%s AS %s)', $expr, $this->translate( $cast_type ) );
+			} else {
+				// CONVERT(expr USING charset): Keep "expr" as is (no SQLite support).
+				// TODO: Consider rejecting UTF-8-incompatible charasets.
+				return $expr;
+			}
+		}
+
 		return $this->translate_sequence( $node->get_children() );
 	}
 
@@ -5350,12 +5374,12 @@ private function store_last_column_meta_from_statement( PDOStatement $stmt ): vo
 	private function unnest_parenthesized_expression( WP_Parser_Node $node ): WP_Parser_Node {
 		$children = $node->get_children();
 
-		// Descend the "expr -> boolPri -> predicate -> bitExpr -> simpleExpr" tree,
-		// when on each level we have only a single child node (expression nesting).
+		// Descend the "expr -> boolPri -> predicate -> bitExpr -> simpleExpr" -> "simpleExprBody"
+		// tree, when on each level we have only a single child node (expression nesting).
 		if (
 			1 === count( $children )
 			&& $children[0] instanceof WP_Parser_Node
-			&& in_array( $children[0]->rule_name, array( 'expr', 'boolPri', 'predicate', 'bitExpr', 'simpleExpr' ), true )
+			&& in_array( $children[0]->rule_name, array( 'expr', 'boolPri', 'predicate', 'bitExpr', 'simpleExpr', 'simpleExprBody' ), true )
 		) {
 			$unnested = $this->unnest_parenthesized_expression( $children[0] );
 			return $unnested === $children[0] ? $node : $unnested;
diff --git a/packages/mysql-on-sqlite/tests/WP_SQLite_Driver_Tests.php b/packages/mysql-on-sqlite/tests/WP_SQLite_Driver_Tests.php
@@ -9971,7 +9971,8 @@ public function testCastExpression(): void {
 					'expr_5'  => 'abc',
 					'expr_6'  => 'abc',                 // 'ab' In MySQL
 					'expr_7'  => '-10',
-					'expr_8'  => '-10',                 // 18446744073709551606 in MySQL
+					// @TODO: Emulate UNSIGNED cast. MySQL returns 18446744073709551606 (2^64 - 10).
+					'expr_8'  => '-10',
 					'expr_9'  => '2025-10-05 14:05:28', // 2025-10-05 in MySQL
 					'expr_10' => '2025-10-05 14:05:28', // 14:05:28 in MySQL
 					'expr_11' => '2025-10-05 14:05:28',
@@ -9986,6 +9987,90 @@ public function testCastExpression(): void {
 		);
 	}
 
+	public function testConvertExpression(): void {
+		// CONVERT(expr, type) should behave like CAST(expr AS type).
+		$result = $this->assertQuery(
+			"SELECT
+				CONVERT('abc', BINARY) AS expr_1,
+				CONVERT('abc', CHAR) AS expr_2,
+				CONVERT('-10', SIGNED) AS expr_3,
+				CONVERT('-10', UNSIGNED) AS expr_4,
+				CONVERT('123.456', DECIMAL) AS expr_5,
+				CONVERT('2025-10-05', DATE) AS expr_6
+			"
+		);
+
+		$this->assertEquals(
+			array(
+				(object) array(
+					'expr_1' => 'abc',
+					'expr_2' => 'abc',
+					'expr_3' => '-10',
+					// @TODO: Emulate UNSIGNED cast. MySQL returns 18446744073709551606 (2^64 - 10).
+					'expr_4' => '-10',
+					'expr_5' => '123.456',
+					'expr_6' => '2025-10-05',
+				),
+			),
+			$result
+		);
+	}
+
+	public function testConvertUsingExpression(): void {
+		// CONVERT(expr USING charset) converts character set.
+		// In SQLite, all text is UTF-8 — the conversion is a no-op.
+		$result = $this->assertQuery(
+			"SELECT
+				CONVERT('Customer' USING utf8mb4) AS expr_1,
+				CONVERT('test' USING utf8) AS expr_2,
+				CONVERT('data' USING latin1) AS expr_3
+			"
+		);
+
+		$this->assertEquals(
+			array(
+				(object) array(
+					'expr_1' => 'Customer',
+					'expr_2' => 'test',
+					'expr_3' => 'data',
+				),
+			),
+			$result
+		);
+	}
+
+	public function testConvertUsingWithCollate(): void {
+		$result = $this->assertQuery(
+			"SELECT CONVERT('Customer' USING utf8mb4) COLLATE utf8mb4_bin AS val"
+		);
+
+		$this->assertEquals(
+			array(
+				(object) array( 'val' => 'Customer' ),
+			),
+			$result
+		);
+	}
+
+	public function testConvertWithColumnReferences(): void {
+		$this->assertQuery( 'CREATE TABLE t (val VARCHAR(255), num VARCHAR(255))' );
+		$this->assertQuery( "INSERT INTO t (val, num) VALUES ('hello', '-42')" );
+
+		$result = $this->assertQuery(
+			'SELECT CONVERT(val, BINARY) AS v1, CONVERT(val USING utf8mb4) AS v2
+			FROM t WHERE CONVERT(num, SIGNED) < 0 ORDER BY CONVERT(val USING utf8mb4)'
+		);
+		$this->assertEquals(
+			array(
+				(object) array(
+					'v1' => 'hello',
+					'v2' => 'hello',
+				),
+			),
+			$result
+		);
+	}
+
 	public function testInsertWithoutInto(): void {
 		$this->assertQuery( 'CREATE TABLE t (id INT PRIMARY KEY, name VARCHAR(255))' );
 
diff --git a/packages/mysql-on-sqlite/tests/WP_SQLite_Driver_Translation_Tests.php b/packages/mysql-on-sqlite/tests/WP_SQLite_Driver_Translation_Tests.php
@@ -101,6 +101,41 @@ public function testSelect(): void {
 		);
 	}
 
+	public function testConvert(): void {
+		// CONVERT(expr, type) → CAST(expr AS type)
+		$this->assertQuery(
+			"SELECT CAST('abc' AS BLOB) AS `CONVERT('abc', BINARY)`",
+			"SELECT CONVERT('abc', BINARY)"
+		);
+
+		$this->assertQuery(
+			"SELECT CAST('abc' AS TEXT) AS `CONVERT('abc', CHAR)`",
+			"SELECT CONVERT('abc', CHAR)"
+		);
+
+		$this->assertQuery(
+			"SELECT CAST('-10' AS INTEGER) AS `CONVERT('-10', SIGNED)`",
+			"SELECT CONVERT('-10', SIGNED)"
+		);
+
+		// CONVERT(expr USING charset) → expr
+		$this->assertQuery(
+			"SELECT 'Customer' AS `Customer`",
+			"SELECT CONVERT('Customer' USING utf8mb4)"
+		);
+
+		$this->assertQuery(
+			"SELECT 'test' AS `test`",
+			"SELECT CONVERT('test' USING utf8)"
+		);
+
+		// CONVERT(expr USING charset) COLLATE collation → expr COLLATE collation
+		$this->assertQuery(
+			"SELECT 'Customer' COLLATE `utf8mb4_bin` AS `CONVERT('Customer' USING utf8mb4) COLLATE utf8mb4_bin`",
+			"SELECT CONVERT('Customer' USING utf8mb4) COLLATE utf8mb4_bin"
+		);
+	}
+
 	public function testInsert(): void {
 		$this->driver->query( 'CREATE TABLE t (c INT, c1 INT, c2 INT)' );
 		$this->driver->query( 'CREATE TABLE t1 (c1 INT, c2 INT)' );
