Skip to content

Commit d3a3a14

Browse files
authored
Fix CRLF handling in line continuation escapes (#1564)
1 parent 39c01c2 commit d3a3a14

2 files changed

Lines changed: 58 additions & 2 deletions

File tree

pkl-parser/src/main/java/org/pkl/parser/Lexer.java

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -461,11 +461,17 @@ private Token lexEscape() {
461461
}
462462
case 'u' -> lexUnicodeEscape();
463463
case '\n' -> Token.STRING_ESCAPE_CONTINUATION;
464+
case '\r' -> {
465+
if (lookahead == '\n') {
466+
nextChar();
467+
}
468+
yield Token.STRING_ESCAPE_CONTINUATION;
469+
}
464470
case ' ', '\t' -> {
465471
var c = cursor;
466472
var next = nextChar();
467473
while (next == ' ' || next == '\t') next = nextChar();
468-
if (next == '\n')
474+
if (next == '\n' || next == '\r')
469475
throw lexError(
470476
ErrorMessages.create("invalidLineContinuationEscapeSequenceWhitespace"),
471477
c - 2,

pkl-parser/src/test/kotlin/org/pkl/parser/LexerTest.kt

Lines changed: 51 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
/*
2-
* Copyright © 2025 Apple Inc. and the Pkl project authors. All rights reserved.
2+
* Copyright © 2025-2026 Apple Inc. and the Pkl project authors. All rights reserved.
33
*
44
* Licensed under the Apache License, Version 2.0 (the "License");
55
* you may not use this file except in compliance with the License.
@@ -63,6 +63,56 @@ class LexerTest {
6363
assertThat(thrown).hasMessageContaining("Invalid identifier")
6464
}
6565

66+
@Test
67+
fun lineContinuationWithCRLF() {
68+
// \r\n line endings must be handled the same as \n for line continuations
69+
val input = "x = \"\"\"\n hello \\\r\n world\r\n \"\"\""
70+
val lexer = Lexer(input)
71+
assertThat(lexer.next()).isEqualTo(Token.IDENTIFIER) // x
72+
assertThat(lexer.next()).isEqualTo(Token.ASSIGN)
73+
assertThat(lexer.next()).isEqualTo(Token.STRING_MULTI_START) // """
74+
assertThat(lexer.next()).isEqualTo(Token.STRING_NEWLINE)
75+
assertThat(lexer.next()).isEqualTo(Token.STRING_PART) // " hello "
76+
assertThat(lexer.next()).isEqualTo(Token.STRING_ESCAPE_CONTINUATION) // \<CRLF> consumed
77+
assertThat(lexer.next()).isEqualTo(Token.STRING_PART) // " world"
78+
assertThat(lexer.next()).isEqualTo(Token.STRING_NEWLINE)
79+
assertThat(lexer.next()).isEqualTo(Token.STRING_PART) // " "
80+
assertThat(lexer.next()).isEqualTo(Token.STRING_END) // """
81+
assertThat(lexer.next()).isEqualTo(Token.EOF)
82+
}
83+
84+
@Test
85+
fun lineContinuationWithCR() {
86+
// bare \r should also work as a line continuation
87+
val input = "x = \"\"\"\n hello \\\r world\n \"\"\""
88+
val lexer = Lexer(input)
89+
assertThat(lexer.next()).isEqualTo(Token.IDENTIFIER)
90+
assertThat(lexer.next()).isEqualTo(Token.ASSIGN)
91+
assertThat(lexer.next()).isEqualTo(Token.STRING_MULTI_START)
92+
assertThat(lexer.next()).isEqualTo(Token.STRING_NEWLINE)
93+
assertThat(lexer.next()).isEqualTo(Token.STRING_PART)
94+
assertThat(lexer.next()).isEqualTo(Token.STRING_ESCAPE_CONTINUATION) // \<CR> consumed
95+
assertThat(lexer.next()).isEqualTo(Token.STRING_PART)
96+
assertThat(lexer.next()).isEqualTo(Token.STRING_NEWLINE)
97+
assertThat(lexer.next()).isEqualTo(Token.STRING_PART)
98+
assertThat(lexer.next()).isEqualTo(Token.STRING_END)
99+
assertThat(lexer.next()).isEqualTo(Token.EOF)
100+
}
101+
102+
@Test
103+
fun lineContinuationWhitespaceErrorWithCRLF() {
104+
// whitespace between \ and \r\n should give the same error as \ and \n
105+
val input = "x = \"\"\"\n hello \\ \r\n world\n \"\"\""
106+
val thrown =
107+
assertThrows<ParserError> {
108+
val lexer = Lexer(input)
109+
while (lexer.next() != Token.EOF) {
110+
/* consume all tokens */
111+
}
112+
}
113+
assertThat(thrown.message).contains("Whitespace")
114+
}
115+
66116
@Test
67117
fun acceptsAllUnicodeCodepointsInComments() {
68118
// Test valid Unicode codepoints can appear literally

0 commit comments

Comments
 (0)