Skip to content

Translate MySQL CONVERT() expressions to SQLite #356

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
JanJakes merged 3 commits into from
Apr 10, 2026
Merged

Translate MySQL CONVERT() expressions to SQLite #356

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
da8a89e
Extract simpleExprBody as a named grammar rule
JanJakes Apr 7, 2026
79c53fa
Translate MySQL CONVERT() expressions to SQLite
JanJakes Apr 4, 2026
edee595
Add TODO for UNSIGNED cast emulation
JanJakes Apr 10, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 0 additions & 3 deletions .github/workflows/wp-tests-phpunit-run.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,9 +58,6 @@ const expectedFailures = [
'Tests_DB_Charset::test_strip_invalid_text with data set #32',
'Tests_DB_Charset::test_strip_invalid_text with data set #33',
'Tests_DB_Charset::test_strip_invalid_text with data set #34',
'Tests_DB_Charset::test_strip_invalid_text with data set #35',
'Tests_DB_Charset::test_strip_invalid_text with data set #36',
'Tests_DB_Charset::test_strip_invalid_text with data set #37',
'Tests_DB_Charset::test_strip_invalid_text with data set #39',
'Tests_DB_Charset::test_strip_invalid_text with data set #40',
'Tests_DB_Charset::test_strip_invalid_text with data set #41',
Expand Down
6 changes: 3 additions & 3 deletions grammar-tools/MySQLParser.g4
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2961,13 +2961,13 @@ bitExpr: simpleExpr %bitExpr_rr*;

/*
* @CHANGED:
* Factored left recursion.
* Factored left recursion and introduced "simpleExprBody" for easier processing.
*/
simpleExpr: %simpleExpr_collate (CONCAT_PIPES_SYMBOL %simpleExpr_collate)*;

%simpleExpr_collate: %simpleExpr_factored (COLLATE_SYMBOL textOrIdentifier)?;
%simpleExpr_collate: simpleExprBody (COLLATE_SYMBOL textOrIdentifier)?;

%simpleExpr_factored:
simpleExprBody:
literal # simpleExprLiteral
| sumExpr # simpleExprSum
| variable (equal expr)? # simpleExprVariable
Expand Down
2 changes: 1 addition & 1 deletion packages/mysql-on-sqlite/src/mysql/mysql-grammar.php
View file
Open in desktop

Large diffs are not rendered by default.

40 changes: 32 additions & 8 deletions packages/mysql-on-sqlite/src/sqlite/class-wp-pdo-mysql-on-sqlite.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3691,8 +3691,8 @@ private function translate( $node ): ?string {
return null;
}
return $this->translate_sequence( $node->get_children() );
case 'simpleExpr':
return $this->translate_simple_expr( $node );
case 'simpleExprBody':
return $this->translate_simple_expr_body( $node );
case 'predicateOperations':
$token = $node->get_first_child_token();
if ( WP_MySQL_Lexer::LIKE_SYMBOL === $token->id ) {
Expand Down Expand Up @@ -3790,6 +3790,8 @@ private function translate( $node ): ?string {
return 'TEXT';
case WP_MySQL_Lexer::SIGNED_SYMBOL:
case WP_MySQL_Lexer::UNSIGNED_SYMBOL:
// @TODO: Emulate UNSIGNED semantics. MySQL wraps negative
// values, but SQLite has no unsigned integer type.
return 'INTEGER';
case WP_MySQL_Lexer::DECIMAL_SYMBOL:
case WP_MySQL_Lexer::FLOAT_SYMBOL:
Expand Down Expand Up @@ -4204,13 +4206,13 @@ private function translate_query_specification( WP_Parser_Node $node ): string {
}

/**
* Translate a MySQL simple expression to SQLite.
* Translate a MySQL simple expression body to SQLite.
*
* @param WP_Parser_Node $node The "simpleExpr" AST node.
* @param WP_Parser_Node $node The "simpleExprBody" AST node.
* @return string The translated value.
* @throws WP_SQLite_Driver_Exception When the translation fails.
*/
private function translate_simple_expr( WP_Parser_Node $node ): string {
private function translate_simple_expr_body( WP_Parser_Node $node ): string {
$token = $node->get_first_child_token();

// Translate "VALUES(col)" to "excluded.col" in ON DUPLICATE KEY UPDATE.
Expand All @@ -4221,6 +4223,28 @@ private function translate_simple_expr( WP_Parser_Node $node ): string {
);
}

/**
* Translate MySQL CONVERT() expression.
*
* MySQL supports two forms of CONVERT():
* 1. CONVERT(expr, type): Equivalent to CAST(expr AS type).
* 2. CONVERT(expr USING charset): Converts the character set.
*/
if ( null !== $token && WP_MySQL_Lexer::CONVERT_SYMBOL === $token->id ) {
$expr = $this->translate( $node->get_first_child_node( 'expr' ) );
$cast_type = $node->get_first_child_node( 'castType' );

if ( null !== $cast_type ) {
// CONVERT(expr, type): Translate to cast expression.
// TODO: Emulate UNSIGNED cast. SQLite has no unsigned integer type.
return sprintf( 'CAST(%s AS %s)', $expr, $this->translate( $cast_type ) );
} else {
// CONVERT(expr USING charset): Keep "expr" as is (no SQLite support).
// TODO: Consider rejecting UTF-8-incompatible charasets.
return $expr;
}
}

return $this->translate_sequence( $node->get_children() );
}

Expand Down Expand Up @@ -5350,12 +5374,12 @@ private function store_last_column_meta_from_statement( PDOStatement $stmt ): vo
private function unnest_parenthesized_expression( WP_Parser_Node $node ): WP_Parser_Node {
$children = $node->get_children();

// Descend the "expr -> boolPri -> predicate -> bitExpr -> simpleExpr" tree,
// when on each level we have only a single child node (expression nesting).
// Descend the "expr -> boolPri -> predicate -> bitExpr -> simpleExpr" -> "simpleExprBody"
// tree, when on each level we have only a single child node (expression nesting).
if (
1 === count( $children )
&& $children[0] instanceof WP_Parser_Node
&& in_array( $children[0]->rule_name, array( 'expr', 'boolPri', 'predicate', 'bitExpr', 'simpleExpr' ), true )
&& in_array( $children[0]->rule_name, array( 'expr', 'boolPri', 'predicate', 'bitExpr', 'simpleExpr', 'simpleExprBody' ), true )
) {
$unnested = $this->unnest_parenthesized_expression( $children[0] );
return $unnested === $children[0] ? $node : $unnested;
Expand Down
87 changes: 86 additions & 1 deletion packages/mysql-on-sqlite/tests/WP_SQLite_Driver_Tests.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9971,7 +9971,8 @@ public function testCastExpression(): void {
'expr_5' => 'abc',
'expr_6' => 'abc', // 'ab' In MySQL
'expr_7' => '-10',
'expr_8' => '-10', // 18446744073709551606 in MySQL
// @TODO: Emulate UNSIGNED cast. MySQL returns 18446744073709551606 (2^64 - 10).
'expr_8' => '-10',
'expr_9' => '2025-10-05 14:05:28', // 2025-10-05 in MySQL
'expr_10' => '2025-10-05 14:05:28', // 14:05:28 in MySQL
'expr_11' => '2025-10-05 14:05:28',
Expand All @@ -9986,6 +9987,90 @@ public function testCastExpression(): void {
);
}

public function testConvertExpression(): void {
// CONVERT(expr, type) should behave like CAST(expr AS type).
$result = $this->assertQuery(
"SELECT
CONVERT('abc', BINARY) AS expr_1,
CONVERT('abc', CHAR) AS expr_2,
CONVERT('-10', SIGNED) AS expr_3,
CONVERT('-10', UNSIGNED) AS expr_4,
Copy link
Copy Markdown
Collaborator

@bgrgicak bgrgicak Apr 10, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I have one question, in MySQL SELECT CONVERT('-10', UNSIGNED); will return 18446744073709551606.
Why is '-10' expected here?

Copy link
Copy Markdown
Member Author

@JanJakes JanJakes Apr 10, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@bgrgicak I missed this, thanks for catching that! It seems that the main problem is that SQLite doesn't support UNSIGNED integers, so it cannot store values as large as 18446744073709551606.

That looks like a bigger problem, and I'm not sure whether it's reasonably solvable 😬 I guess for now, we should add a TODO comment explaining the issue.

Copy link
Copy Markdown
Collaborator

@bgrgicak bgrgicak Apr 10, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

A TODO sounds good if we can't resolve it in this PR.

In this case, is it better to throw an error or silently return the incorrect value?

Copy link
Copy Markdown
Member Author

@JanJakes JanJakes Apr 10, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@bgrgicak There are likely more issues like this where we translate a MySQL construct to a similar one in SQLite, but they don't behave 100% the same. I'm not sure if UNSIGNED is the only incompatibility here, but it will affect not only CONVERT but especially CAST and potentially some arithmetic, etc.

Therefore, I think the TODO is fine (and I can also create a ticket), and we don't need to try to detect and fail the invalid cases at the moment. Fortunately, casting a signed negative integer to an unsigned one seems to be a very strange use case that is hopefully not encountered in reality.

Copy link
Copy Markdown
Member Author

@JanJakes JanJakes Apr 10, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I added the TODOs.

Copy link
Copy Markdown
Member Author

@JanJakes JanJakes Apr 10, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Issue: #359

CONVERT('123.456', DECIMAL) AS expr_5,
CONVERT('2025-10-05', DATE) AS expr_6
"
);

$this->assertEquals(
array(
(object) array(
'expr_1' => 'abc',
'expr_2' => 'abc',
'expr_3' => '-10',
// @TODO: Emulate UNSIGNED cast. MySQL returns 18446744073709551606 (2^64 - 10).
'expr_4' => '-10',
'expr_5' => '123.456',
'expr_6' => '2025-10-05',
),
),
$result
);
}

public function testConvertUsingExpression(): void {
// CONVERT(expr USING charset) converts character set.
// In SQLite, all text is UTF-8 — the conversion is a no-op.
$result = $this->assertQuery(
"SELECT
CONVERT('Customer' USING utf8mb4) AS expr_1,
CONVERT('test' USING utf8) AS expr_2,
CONVERT('data' USING latin1) AS expr_3
"
);

$this->assertEquals(
array(
(object) array(
'expr_1' => 'Customer',
'expr_2' => 'test',
'expr_3' => 'data',
),
),
$result
);
}

public function testConvertUsingWithCollate(): void {
$result = $this->assertQuery(
"SELECT CONVERT('Customer' USING utf8mb4) COLLATE utf8mb4_bin AS val"
);

$this->assertEquals(
array(
(object) array( 'val' => 'Customer' ),
),
$result
);
}

public function testConvertWithColumnReferences(): void {
$this->assertQuery( 'CREATE TABLE t (val VARCHAR(255), num VARCHAR(255))' );
$this->assertQuery( "INSERT INTO t (val, num) VALUES ('hello', '-42')" );

$result = $this->assertQuery(
'SELECT CONVERT(val, BINARY) AS v1, CONVERT(val USING utf8mb4) AS v2
FROM t WHERE CONVERT(num, SIGNED) < 0 ORDER BY CONVERT(val USING utf8mb4)'
);
$this->assertEquals(
array(
(object) array(
'v1' => 'hello',
'v2' => 'hello',
),
),
$result
);
}

public function testInsertWithoutInto(): void {
$this->assertQuery( 'CREATE TABLE t (id INT PRIMARY KEY, name VARCHAR(255))' );

Expand Down
35 changes: 35 additions & 0 deletions packages/mysql-on-sqlite/tests/WP_SQLite_Driver_Translation_Tests.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,41 @@ public function testSelect(): void {
);
}

public function testConvert(): void {
// CONVERT(expr, type) → CAST(expr AS type)
$this->assertQuery(
"SELECT CAST('abc' AS BLOB) AS `CONVERT('abc', BINARY)`",
"SELECT CONVERT('abc', BINARY)"
);

$this->assertQuery(
"SELECT CAST('abc' AS TEXT) AS `CONVERT('abc', CHAR)`",
"SELECT CONVERT('abc', CHAR)"
);

$this->assertQuery(
"SELECT CAST('-10' AS INTEGER) AS `CONVERT('-10', SIGNED)`",
"SELECT CONVERT('-10', SIGNED)"
);

// CONVERT(expr USING charset) → expr
$this->assertQuery(
"SELECT 'Customer' AS `Customer`",
"SELECT CONVERT('Customer' USING utf8mb4)"
);

$this->assertQuery(
"SELECT 'test' AS `test`",
"SELECT CONVERT('test' USING utf8)"
);

// CONVERT(expr USING charset) COLLATE collation → expr COLLATE collation
$this->assertQuery(
"SELECT 'Customer' COLLATE `utf8mb4_bin` AS `CONVERT('Customer' USING utf8mb4) COLLATE utf8mb4_bin`",
"SELECT CONVERT('Customer' USING utf8mb4) COLLATE utf8mb4_bin"
);
}

public function testInsert(): void {
$this->driver->query( 'CREATE TABLE t (c INT, c1 INT, c2 INT)' );
$this->driver->query( 'CREATE TABLE t1 (c1 INT, c2 INT)' );
Expand Down
Loading