diff --git a/lib/liquid/lexer.rb b/lib/liquid/lexer.rb index 9c392fad5..5e8e1023a 100644 --- a/lib/liquid/lexer.rb +++ b/lib/liquid/lexer.rb @@ -90,7 +90,12 @@ class Lexer2 SINGLE_STRING_LITERAL = /'[^\']*'/ WHITESPACE_OR_NOTHING = /\s*/ - COMPARISON_JUMP_TABLE = [].tap do |table| + SINGLE_COMPARISON_TOKENS = [].tap do |table| + table["<".ord] = COMPARISON_LESS_THAN + table[">".ord] = COMPARISON_GREATER_THAN + end + + TWO_CHARS_COMPARISON_JUMP_TABLE = [].tap do |table| table["=".ord] = [].tap do |sub_table| sub_table["=".ord] = COMPARISON_EQUAL sub_table.freeze @@ -99,6 +104,9 @@ class Lexer2 sub_table["=".ord] = COMPARISION_NOT_EQUAL sub_table.freeze end + end + + COMPARISON_JUMP_TABLE = [].tap do |table| table["<".ord] = [].tap do |sub_table| sub_table["=".ord] = COMPARISON_LESS_THAN_OR_EQUAL sub_table[">".ord] = COMPARISON_NOT_EQUAL_ALT @@ -182,7 +190,7 @@ def tokenize else @output << special end - elsif (sub_table = COMPARISON_JUMP_TABLE[peeked]) + elsif (sub_table = TWO_CHARS_COMPARISON_JUMP_TABLE[peeked]) @ss.scan_byte if (found = sub_table[@ss.peek_byte]) @output << found @@ -190,6 +198,14 @@ def tokenize else raise SyntaxError, "Unexpected character #{peeked.chr}" end + elsif (sub_table = COMPARISON_JUMP_TABLE[peeked]) + @ss.scan_byte + if (found = sub_table[@ss.peek_byte]) + @output << found + @ss.scan_byte + else + @output << SINGLE_COMPARISON_TOKENS[peeked] + end else type, pattern = NEXT_MATCHER_JUMP_TABLE[peeked] diff --git a/test/unit/lexer_unit_test.rb b/test/unit/lexer_unit_test.rb index 363740b42..0676033e6 100644 --- a/test/unit/lexer_unit_test.rb +++ b/test/unit/lexer_unit_test.rb @@ -25,6 +25,30 @@ def test_comparison assert_equal([[:comparison, '=='], [:comparison, '<>'], [:comparison, 'contains'], [:end_of_string]], tokens) end + def test_comparison_without_whitespace + tokens = Lexer.new('1>0').tokenize + assert_equal([[:number, '1'], [:comparison, '>'], [:number, '0'], [:end_of_string]], tokens) + end + + def test_comparison_with_negative_number + tokens = Lexer.new('1>-1').tokenize + assert_equal([[:number, '1'], [:comparison, '>'], [:number, '-1'], [:end_of_string]], tokens) + end + + def test_raise_for_invalid_comparison + assert_raises(SyntaxError) do + Lexer.new('1>!1').tokenize + end + + assert_raises(SyntaxError) do + Lexer.new('1=<1').tokenize + end + + assert_raises(SyntaxError) do + Lexer.new('1!!1').tokenize + end + end + def test_specials tokens = Lexer.new('| .:').tokenize assert_equal([[:pipe, '|'], [:dot, '.'], [:colon, ':'], [:end_of_string]], tokens)