Skip to content

Commit 7becc4e

Browse files
committed
Stop the lexer reading past the end of a byte_range
`rbs_next_char` ends the input when `byte_pos == end_pos`. `rbs_skip` advances by a whole character, so a multibyte character starting before `end_pos` and ending after it steps over the boundary and equality never holds again — the lexer then reads to the end of the string. Parser.parse_type('"Ωx" | Integer', byte_range: 0...2) #=> Types::Union spanning 0...14, the whole input Two bytes were requested and fourteen were read, and `require_eof: true` does not catch it because the lexer really is at EOF by then. It needs a character to straddle the boundary, so ASCII-only input never hits it. Compare with `>=` so stepping over the boundary still ends the input.
1 parent aa48bf2 commit 7becc4e

2 files changed

Lines changed: 25 additions & 2 deletions

File tree

src/lexstate.c

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -119,7 +119,9 @@ unsigned int rbs_peek(rbs_lexer_t *lexer) {
119119
}
120120

121121
bool rbs_next_char(rbs_lexer_t *lexer, unsigned int *codepoint, size_t *byte_len) {
122-
if (RBS_UNLIKELY(lexer->current.byte_pos == lexer->end_pos)) {
122+
// `>=`, not `==`: a character straddling `end_pos` moves `byte_pos` past it
123+
// in one step, and equality would never hold again.
124+
if (RBS_UNLIKELY(lexer->current.byte_pos >= lexer->end_pos)) {
123125
return false;
124126
}
125127

@@ -132,7 +134,8 @@ bool rbs_next_char(rbs_lexer_t *lexer, unsigned int *codepoint, size_t *byte_len
132134
return true;
133135
}
134136

135-
*byte_len = lexer->encoding->char_width((const uint8_t *) start, (ptrdiff_t) (lexer->string.end - start));
137+
ptrdiff_t remaining = (ptrdiff_t) (lexer->string.end - start);
138+
*byte_len = lexer->encoding->char_width((const uint8_t *) start, remaining);
136139

137140
if (*byte_len == 0) {
138141
// Invalid byte under the active encoding. Map it to a sentinel code

test/rbs/type_parsing_test.rb

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1029,4 +1029,24 @@ def test_parse__byte_range_incorrect
10291029

10301030
assert_equal "a.rbs:1:2...1:3: Syntax error: unexpected token for simple type, token=`🐈` (ErrorToken)", exn.message
10311031
end
1032+
1033+
def test_parse__byte_range_ending_mid_character
1034+
# `Ω` is two bytes, so this range ends inside it. Advancing by a whole
1035+
# character stepped over `end_pos`, after which the EOF test never held
1036+
# again and the rest of the string was read.
1037+
assert_raises RBS::ParsingError do
1038+
Parser.parse_type('"Ωx" | Integer', byte_range: 0...2)
1039+
end
1040+
1041+
# A range ending on a character boundary is unaffected.
1042+
Parser.parse_type('"Ωx" | Integer', byte_range: 0...5, require_eof: true).tap do |type|
1043+
assert_instance_of Types::Literal, type
1044+
assert_equal "Ωx", type.literal
1045+
end
1046+
1047+
# `end_pos` past the end of the buffer stays in bounds.
1048+
Parser.parse_type("Integer", byte_range: 0...9999).tap do |type|
1049+
assert_instance_of Types::ClassInstance, type
1050+
end
1051+
end
10321052
end

0 commit comments

Comments
 (0)