feat: multi-line string literals (#208)

Co-authored-by: Iván Ovejero <ivov.src@gmail.com>

Author: stormpat

crates/emit/src/expressions/literals.rs

@@ -137,15 +137,16 @@ impl Emitter<'_> {
 }
 
 pub(crate) fn emit_raw_string(value: &str) -> String {
-    // Go discards `\r` from backtick raw strings, so we must fall back to
-    // double-quoted form when the content contains CR.
+    // Go backtick raw strings cannot contain backticks, and Go discards `\r`
+    // from them, so fall back to double-quoted form in either case.
     if !value.contains('`') && !value.contains('\r') {
         format!("`{}`", value)
     } else {
         let escaped = value
             .replace('\\', "\\\\")
             .replace('"', "\\\"")
-            .replace('\r', "\\r");
+            .replace('\r', "\\r")
+            .replace('\n', "\\n");
         format!("\"{}\"", escaped)
     }
 }
@@ -188,6 +189,10 @@ pub(crate) fn convert_escape_sequences(s: &str) -> String {
             } else {
                 result.push(c);
             }
+        } else if c == '\n' {
+            result.push_str("\\n");
+        } else if c == '\r' {
+            result.push_str("\\r");
         } else {
             result.push(c);
         }

crates/format/src/formatter.rs

@@ -534,7 +534,13 @@ impl<'a> Formatter<'a> {
                 }
             }
             Literal::Boolean(b) => Document::str(if *b { "true" } else { "false" }),
+            Literal::String { value, raw: true } if value.contains('\n') => {
+                Document::verbatim(format!("r\"{value}\""))
+            }
             Literal::String { value, raw: true } => Document::string(format!("r\"{value}\"")),
+            Literal::String { value, raw: false } if value.contains('\n') => {
+                Document::verbatim(format!("\"{value}\""))
+            }
             Literal::String { value, raw: false } => Document::string(format!("\"{value}\"")),
             Literal::Char(c) => Document::string(format!("'{c}'")),
             Literal::Slice(elements) => self.slice(elements),
@@ -564,6 +570,9 @@ impl<'a> Formatter<'a> {
 
         for part in parts {
             match part {
+                FormatStringPart::Text(s) if s.contains('\n') => {
+                    docs.push(Document::verbatim(s.clone()))
+                }
                 FormatStringPart::Text(s) => docs.push(Document::string(s.clone())),
                 FormatStringPart::Expression(e) => {
                     docs.push(Document::str("{"));

crates/format/src/lindig.rs

@@ -49,6 +49,13 @@ fn fits(
                 current_width += s.graphemes(true).count() as isize;
             }
 
+            Document::VerbatimText(s) => {
+                if s.contains('\n') {
+                    return false;
+                }
+                current_width += s.graphemes(true).count() as isize;
+            }
+
             Document::StrictBreak { unbroken, .. } | Document::FlexBreak { unbroken, .. } => {
                 match mode {
                     Mode::Broken | Mode::ForcedBroken => return true,
@@ -149,6 +156,23 @@ fn format(
                 output.push_str(s);
             }
 
+            Document::VerbatimText(s) => {
+                if pending_indent >= 0 {
+                    write_indent(output, pending_indent);
+                    pending_indent = -1;
+                }
+                let mut segments = s.split('\n');
+                if let Some(first) = segments.next() {
+                    output.push_str(first);
+                    width += first.graphemes(true).count() as isize;
+                }
+                for segment in segments {
+                    output.push('\n');
+                    output.push_str(segment);
+                    width = segment.graphemes(true).count() as isize;
+                }
+            }
+
             Document::Sequence(vec) => {
                 for doc in vec.iter().rev() {
                     docs.push((indent, mode, doc));
@@ -195,6 +219,7 @@ pub enum Document<'a> {
     NestIfBroken(isize, Box<Self>),
     Group(Box<Self>),
     Text(Cow<'a, str>),
+    VerbatimText(Cow<'a, str>),
 }
 
 impl<'a> Document<'a> {
@@ -206,6 +231,10 @@ impl<'a> Document<'a> {
         Document::Text(Cow::Owned(string))
     }
 
+    pub fn verbatim(string: String) -> Self {
+        Document::VerbatimText(Cow::Owned(string))
+    }
+
     pub fn group(self) -> Self {
         Self::Group(Box::new(self))
     }

crates/syntax/src/lex/mod.rs

@@ -873,8 +873,6 @@ impl<'source> Lexer<'source> {
                 terminated = true;
                 self.next();
                 break;
-            } else if byte == b'\n' {
-                break; // unterminated string literal across newline, handled below
             }
 
             self.next();
@@ -888,7 +886,7 @@ impl<'source> Lexer<'source> {
         }
 
         if !terminated {
-            self.error_unterminated_string(start_offset, length);
+            self.error_unterminated_string(start_offset, 1);
         }
 
         Token {
@@ -911,8 +909,6 @@ impl<'source> Lexer<'source> {
                 terminated = true;
                 self.next();
                 break;
-            } else if byte == b'\n' {
-                break;
             } else if byte == 0 {
                 self.error_disallowed_byte_in_raw_string(self.current_offset, byte);
                 self.next();
@@ -925,7 +921,7 @@ impl<'source> Lexer<'source> {
         let length = end_offset - start_offset;
 
         if !terminated {
-            self.error_unterminated_raw_string(start_offset, length);
+            self.error_unterminated_raw_string(start_offset, 2);
         }
 
         Token {
@@ -1163,14 +1159,49 @@ impl<'source> Lexer<'source> {
         None
     }
 
+    // Caller has just consumed `{` of the broken interpolation, so we start
+    // inside it (depth=1). Newlines are not a recovery boundary now that
+    // f-string text spans them, so we balance braces and skip past quoted
+    // strings to avoid stopping at the first inner `"`.
     fn skip_to_format_string_end(&mut self) {
+        let mut depth = 1;
         while !self.at_eof() {
             match self.current_byte() {
-                b'"' => {
+                b'\\' => {
+                    self.next();
+                    if !self.at_eof() {
+                        self.next();
+                    }
+                }
+                b'"' if depth == 0 => {
                     self.next();
                     return;
                 }
-                b'\n' => return,
+                b'"' => {
+                    self.next();
+                    while !self.at_eof() && self.current_byte() != b'"' {
+                        if self.current_byte() == b'\\' {
+                            self.next();
+                            if self.at_eof() {
+                                break;
+                            }
+                        }
+                        self.next();
+                    }
+                    if !self.at_eof() {
+                        self.next();
+                    }
+                }
+                b'{' => {
+                    depth += 1;
+                    self.next();
+                }
+                b'}' => {
+                    if depth > 0 {
+                        depth -= 1;
+                    }
+                    self.next();
+                }
                 _ => self.next(),
             }
         }
@@ -1234,12 +1265,6 @@ impl<'source> Lexer<'source> {
                     return tokens;
                 }
 
-                b'\n' => {
-                    let length = self.current_offset.saturating_sub(start_offset);
-                    self.error_unterminated_format_string(start_offset, length);
-                    return tokens;
-                }
-
                 b'{' => {
                     self.push_format_string_text_if_needed(&mut tokens, text_segment_start);
 
@@ -1258,8 +1283,7 @@ impl<'source> Lexer<'source> {
             }
         }
 
-        let length = self.current_offset.saturating_sub(start_offset);
-        self.error_unterminated_format_string(start_offset, length);
+        self.error_unterminated_format_string(start_offset, 2);
         tokens
     }
 

crates/syntax/src/parse/expressions.rs

@@ -1,5 +1,6 @@
 use ecow::EcoString;
 
+use super::strings::cook_string_contents;
 use super::{MAX_TUPLE_ARITY, ParseError, Parser};
 use crate::ast::{
     Annotation, Attribute, BinaryOperator, Binding, Expression, FormatStringPart, ImportAlias,
@@ -182,7 +183,7 @@ impl<'source> Parser<'source> {
                     s
                 };
                 Literal::String {
-                    value: s_stripped.to_string(),
+                    value: cook_string_contents(s_stripped),
                     raw: false,
                 }
             }
@@ -203,7 +204,7 @@ impl<'source> Parser<'source> {
                     s
                 };
                 Literal::String {
-                    value: s_stripped.to_string(),
+                    value: cook_string_contents(s_stripped),
                     raw: true,
                 }
             }
@@ -1067,7 +1068,7 @@ impl<'source> Parser<'source> {
                 FormatStringText => {
                     let text = self.current_token().text;
                     self.next();
-                    parts.push(FormatStringPart::Text(text.to_string()));
+                    parts.push(FormatStringPart::Text(cook_string_contents(text)));
                 }
                 FormatStringInterpolationStart => {
                     self.ensure(FormatStringInterpolationStart);

crates/syntax/src/parse/mod.rs

@@ -19,6 +19,7 @@ mod expressions;
 mod identifiers;
 mod patterns;
 mod pratt;
+mod strings;
 
 pub use error::ParseError;
 

crates/syntax/src/parse/patterns.rs

@@ -1,5 +1,6 @@
 use ecow::EcoString;
 
+use super::strings::cook_string_contents;
 use super::{MAX_TUPLE_ARITY, ParseError, Parser};
 use crate::ast::{Annotation, Binding, Literal, Pattern, RestPattern, Span, StructFieldPattern};
 use crate::lex::Token;
@@ -243,20 +244,20 @@ impl<'source> Parser<'source> {
         self.next();
         let (value, raw) = if kind == crate::lex::TokenKind::RawString {
             let stripped = if s.len() >= 3 && s.starts_with("r\"") && s.ends_with('"') {
-                s[2..s.len() - 1].to_string()
+                &s[2..s.len() - 1]
             } else if s.len() >= 2 && s.starts_with("r\"") {
-                s[2..].to_string()
+                &s[2..]
             } else {
-                s.to_string()
+                s
             };
-            (stripped, true)
+            (cook_string_contents(stripped), true)
         } else {
             let stripped = if s.len() >= 2 && s.starts_with('"') && s.ends_with('"') {
-                s[1..s.len() - 1].to_string()
+                &s[1..s.len() - 1]
             } else {
-                s.to_string()
+                s
             };
-            (stripped, false)
+            (cook_string_contents(stripped), false)
         };
 
         Pattern::Literal {

crates/syntax/src/parse/strings.rs

@@ -0,0 +1,65 @@
+/// Normalise CRLF to LF; pass other bytes through unchanged.
+pub(crate) fn cook_string_contents(content: &str) -> String {
+    if !content.contains('\r') {
+        return content.to_string();
+    }
+
+    let bytes = content.as_bytes();
+    let mut out: Vec<u8> = Vec::with_capacity(content.len());
+    let mut i = 0;
+    let mut copy_start = 0;
+
+    while i < bytes.len() {
+        if bytes[i] == b'\r' && i + 1 < bytes.len() && bytes[i + 1] == b'\n' {
+            out.extend_from_slice(&bytes[copy_start..i]);
+            out.push(b'\n');
+            i += 2;
+            copy_start = i;
+            continue;
+        }
+        i += 1;
+    }
+
+    out.extend_from_slice(&bytes[copy_start..]);
+
+    // SAFETY: input was valid UTF-8 and we only inserted ASCII LF or skipped
+    // ASCII bytes, so the remaining content is still valid UTF-8.
+    unsafe { String::from_utf8_unchecked(out) }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn preserves_single_line() {
+        assert_eq!(cook_string_contents("hello"), "hello");
+    }
+
+    #[test]
+    fn preserves_embedded_newline() {
+        assert_eq!(cook_string_contents("a\nb"), "a\nb");
+    }
+
+    #[test]
+    fn normalises_crlf_to_lf() {
+        assert_eq!(cook_string_contents("a\r\nb"), "a\nb");
+    }
+
+    #[test]
+    fn preserves_lone_cr() {
+        assert_eq!(cook_string_contents("a\rb"), "a\rb");
+    }
+
+    #[test]
+    fn passes_other_escapes_through() {
+        assert_eq!(cook_string_contents("a\\nb"), "a\\nb");
+        assert_eq!(cook_string_contents("a\\\\b"), "a\\\\b");
+    }
+
+    #[test]
+    fn preserves_multibyte_utf8() {
+        assert_eq!(cook_string_contents("héllo"), "héllo");
+        assert_eq!(cook_string_contents("a\r\né"), "a\né");
+    }
+}