|
1 | 1 | /* |
2 | | - * Copyright © 2025 Apple Inc. and the Pkl project authors. All rights reserved. |
| 2 | + * Copyright © 2025-2026 Apple Inc. and the Pkl project authors. All rights reserved. |
3 | 3 | * |
4 | 4 | * Licensed under the Apache License, Version 2.0 (the "License"); |
5 | 5 | * you may not use this file except in compliance with the License. |
@@ -63,6 +63,56 @@ class LexerTest { |
63 | 63 | assertThat(thrown).hasMessageContaining("Invalid identifier") |
64 | 64 | } |
65 | 65 |
|
| 66 | + @Test |
| 67 | + fun lineContinuationWithCRLF() { |
| 68 | + // \r\n line endings must be handled the same as \n for line continuations |
| 69 | + val input = "x = \"\"\"\n hello \\\r\n world\r\n \"\"\"" |
| 70 | + val lexer = Lexer(input) |
| 71 | + assertThat(lexer.next()).isEqualTo(Token.IDENTIFIER) // x |
| 72 | + assertThat(lexer.next()).isEqualTo(Token.ASSIGN) |
| 73 | + assertThat(lexer.next()).isEqualTo(Token.STRING_MULTI_START) // """ |
| 74 | + assertThat(lexer.next()).isEqualTo(Token.STRING_NEWLINE) |
| 75 | + assertThat(lexer.next()).isEqualTo(Token.STRING_PART) // " hello " |
| 76 | + assertThat(lexer.next()).isEqualTo(Token.STRING_ESCAPE_CONTINUATION) // \<CRLF> consumed |
| 77 | + assertThat(lexer.next()).isEqualTo(Token.STRING_PART) // " world" |
| 78 | + assertThat(lexer.next()).isEqualTo(Token.STRING_NEWLINE) |
| 79 | + assertThat(lexer.next()).isEqualTo(Token.STRING_PART) // " " |
| 80 | + assertThat(lexer.next()).isEqualTo(Token.STRING_END) // """ |
| 81 | + assertThat(lexer.next()).isEqualTo(Token.EOF) |
| 82 | + } |
| 83 | + |
| 84 | + @Test |
| 85 | + fun lineContinuationWithCR() { |
| 86 | + // bare \r should also work as a line continuation |
| 87 | + val input = "x = \"\"\"\n hello \\\r world\n \"\"\"" |
| 88 | + val lexer = Lexer(input) |
| 89 | + assertThat(lexer.next()).isEqualTo(Token.IDENTIFIER) |
| 90 | + assertThat(lexer.next()).isEqualTo(Token.ASSIGN) |
| 91 | + assertThat(lexer.next()).isEqualTo(Token.STRING_MULTI_START) |
| 92 | + assertThat(lexer.next()).isEqualTo(Token.STRING_NEWLINE) |
| 93 | + assertThat(lexer.next()).isEqualTo(Token.STRING_PART) |
| 94 | + assertThat(lexer.next()).isEqualTo(Token.STRING_ESCAPE_CONTINUATION) // \<CR> consumed |
| 95 | + assertThat(lexer.next()).isEqualTo(Token.STRING_PART) |
| 96 | + assertThat(lexer.next()).isEqualTo(Token.STRING_NEWLINE) |
| 97 | + assertThat(lexer.next()).isEqualTo(Token.STRING_PART) |
| 98 | + assertThat(lexer.next()).isEqualTo(Token.STRING_END) |
| 99 | + assertThat(lexer.next()).isEqualTo(Token.EOF) |
| 100 | + } |
| 101 | + |
| 102 | + @Test |
| 103 | + fun lineContinuationWhitespaceErrorWithCRLF() { |
| 104 | + // whitespace between \ and \r\n should give the same error as \ and \n |
| 105 | + val input = "x = \"\"\"\n hello \\ \r\n world\n \"\"\"" |
| 106 | + val thrown = |
| 107 | + assertThrows<ParserError> { |
| 108 | + val lexer = Lexer(input) |
| 109 | + while (lexer.next() != Token.EOF) { |
| 110 | + /* consume all tokens */ |
| 111 | + } |
| 112 | + } |
| 113 | + assertThat(thrown.message).contains("Whitespace") |
| 114 | + } |
| 115 | + |
66 | 116 | @Test |
67 | 117 | fun acceptsAllUnicodeCodepointsInComments() { |
68 | 118 | // Test valid Unicode codepoints can appear literally |
|
0 commit comments