Fix a bug in the formatter that was removing newlines escapes

Fixes smithy-lang#2524

Author: sugmanue

smithy-model/src/main/java/software/amazon/smithy/model/loader/DefaultTokenizer.java

@@ -20,6 +20,7 @@ class DefaultTokenizer implements IdlTokenizer {
     private int currentTokenColumn = -1;
     private Number currentTokenNumber;
     private CharSequence currentTokenStringSlice;
+    private CharSequence currentTextBlockContents;
     private String currentTokenError;
 
     DefaultTokenizer(String filename, CharSequence model) {
@@ -96,6 +97,17 @@ public final CharSequence getCurrentTokenStringSlice() {
         }
     }
 
+    @Override
+    public final CharSequence getCurrentTextBlockContents() {
+        getCurrentToken();
+        if (currentTextBlockContents != null) {
+            return currentTextBlockContents;
+        } else {
+            throw syntax("The current token must be text block: "
+                    + currentTokenType.getDebug(getCurrentTokenLexeme()), getCurrentTokenLocation());
+        }
+    }
+
     @Override
     public final Number getCurrentTokenNumberValue() {
         getCurrentToken();
@@ -124,6 +136,7 @@ public final boolean hasNext() {
     @Override
     public IdlToken next() {
         currentTokenStringSlice = null;
+        currentTextBlockContents = null;
         currentTokenNumber = null;
         currentTokenColumn = parser.column();
         currentTokenLine = parser.line();
@@ -362,7 +375,7 @@ private IdlToken parseString() {
 
         try {
             // Parse the contents of a quoted string.
-            currentTokenStringSlice = parseQuotedTextAndTextBlock(false);
+            currentTokenStringSlice = parseQuotedTextAndTextBlock(null);
             currentTokenEnd = parser.position();
             return currentTokenType = IdlToken.STRING;
         } catch (RuntimeException e) {
@@ -374,7 +387,9 @@ private IdlToken parseString() {
 
     private IdlToken parseTextBlock() {
         try {
-            currentTokenStringSlice = parseQuotedTextAndTextBlock(true);
+            StringBuilder builder = new StringBuilder();
+            currentTextBlockContents = builder;
+            currentTokenStringSlice = parseQuotedTextAndTextBlock(builder);
             currentTokenEnd = parser.position();
             return currentTokenType = IdlToken.TEXT_BLOCK;
         } catch (RuntimeException e) {
@@ -385,8 +400,9 @@ private IdlToken parseTextBlock() {
     }
 
     // Parses both quoted_text and text_block
-    private CharSequence parseQuotedTextAndTextBlock(boolean triple) {
+    private CharSequence parseQuotedTextAndTextBlock(StringBuilder textBlockContents) {
         int start = parser.position();
+        boolean triple = textBlockContents != null;
 
         while (!parser.eof()) {
             char next = parser.peek();
@@ -409,6 +425,6 @@ private CharSequence parseQuotedTextAndTextBlock(boolean triple) {
             parser.expect('"');
         }
 
-        return IdlStringLexer.scanStringContents(result, triple);
+        return IdlStringLexer.scanStringContents(result, textBlockContents);
     }
 }

smithy-model/src/main/java/software/amazon/smithy/model/loader/IdlStringLexer.java

@@ -46,12 +46,16 @@ CharSequence getResult() {
         }
     }
 
-    static CharSequence scanStringContents(CharSequence lexeme, boolean scanningTextBlock) {
+    static CharSequence scanStringContents(
+            CharSequence lexeme,
+            StringBuilder textBlockContents
+    ) {
+        boolean scanningTextBlock = textBlockContents != null;
         lexeme = normalizeLineEndings(lexeme);
 
         // Format the text block and remove incidental whitespace.
         if (scanningTextBlock) {
-            lexeme = formatTextBlock(lexeme);
+            lexeme = formatTextBlock(lexeme, textBlockContents);
         }
 
         //StringBuilder result = new StringBuilder(lexeme.length());
@@ -176,14 +180,13 @@ private static boolean containsCarriageReturn(CharSequence lexeme) {
         return false;
     }
 
-    private static CharSequence formatTextBlock(CharSequence lexeme) {
+    private static CharSequence formatTextBlock(CharSequence lexeme, StringBuilder buffer) {
         if (lexeme.length() == 0) {
             throw new RuntimeException("Text block is empty");
         } else if (lexeme.charAt(0) != '\n') {
             throw new RuntimeException("Text block must start with a new line");
         }
 
-        StringBuilder buffer = new StringBuilder();
         int longestPadding = Integer.MAX_VALUE;
         List<CharSequence> lines = lines(lexeme);
 

smithy-model/src/main/java/software/amazon/smithy/model/loader/IdlTokenizer.java

@@ -151,6 +151,17 @@ default CharSequence getCurrentTokenLexeme() {
      */
     CharSequence getCurrentTokenStringSlice();
 
+    /**
+     * If the current token is a text block, get the formatted content as a CharSequence.
+     * with any incidental leading whitespace already removed.
+     *
+     * @return Returns the parsed string content associated with the current token.
+     * @throws ModelSyntaxException if the current token is not a text block.
+     */
+    default CharSequence getCurrentTextBlockContents() {
+        throw new UnsupportedOperationException();
+    }
+
     /**
      * If the current token is a number, get the associated parsed number.
      *

smithy-syntax/src/main/java/software/amazon/smithy/syntax/CapturedToken.java

@@ -33,6 +33,7 @@ public final class CapturedToken implements FromSourceLocation, ToSmithyBuilder<
     private final int endColumn;
     private final CharSequence lexeme;
     private final String stringContents;
+    private final String textBlockContents;
     private final String errorMessage;
     private final Number numberValue;
 
@@ -46,6 +47,7 @@ private CapturedToken(
             int endColumn,
             CharSequence lexeme,
             String stringContents,
+            String formattedTextBlockContents,
             Number numberValue,
             String errorMessage
     ) {
@@ -64,6 +66,7 @@ private CapturedToken(
         } else {
             this.stringContents = stringContents;
         }
+        this.textBlockContents = formattedTextBlockContents;
 
         if (errorMessage == null && token == IdlToken.ERROR) {
             this.errorMessage = "";
@@ -92,6 +95,7 @@ public static final class Builder implements SmithyBuilder<CapturedToken> {
         private int endColumn;
         private CharSequence lexeme;
         private String stringContents;
+        private String formattedTextBlockContents;
         private String errorMessage;
         private Number numberValue;
 
@@ -109,6 +113,7 @@ public CapturedToken build() {
                     endColumn,
                     lexeme,
                     stringContents,
+                    formattedTextBlockContents,
                     numberValue,
                     errorMessage);
         }
@@ -158,6 +163,11 @@ public Builder stringContents(String stringContents) {
             return this;
         }
 
+        public Builder textBlockContents(String formattedTextBlockContents) {
+            this.formattedTextBlockContents = formattedTextBlockContents;
+            return this;
+        }
+
         public Builder errorMessage(String errorMessage) {
             this.errorMessage = errorMessage;
             return this;
@@ -200,6 +210,9 @@ public static CapturedToken from(IdlTokenizer tokenizer, Function<CharSequence,
                 .stringContents(tok == IdlToken.STRING || tok == IdlToken.TEXT_BLOCK || tok == IdlToken.IDENTIFIER
                         ? stringTable.apply(tokenizer.getCurrentTokenStringSlice())
                         : null)
+                .textBlockContents(tok == IdlToken.TEXT_BLOCK
+                        ? stringTable.apply(tokenizer.getCurrentTextBlockContents())
+                        : null)
                 .numberValue(tok == IdlToken.NUMBER ? tokenizer.getCurrentTokenNumberValue() : null)
                 .errorMessage(tok == IdlToken.ERROR ? tokenizer.getCurrentTokenError() : null)
                 .build();
@@ -218,7 +231,8 @@ public Builder toBuilder() {
                 .lexeme(lexeme)
                 .errorMessage(errorMessage)
                 .numberValue(numberValue)
-                .stringContents(stringContents);
+                .stringContents(stringContents)
+                .textBlockContents(textBlockContents);
     }
 
     /**
@@ -277,10 +291,21 @@ public CharSequence getLexeme() {
      *
      * @return Returns the string contents of the lexeme, or null if not a string|text block|identifier.
      */
+    // Can we capture the formatted block as well?
     public String getStringContents() {
         return stringContents;
     }
 
+    /**
+     * Get the String contents of a TEXT_BLOCK token already with any incidental leading whitespace
+     * already removed.
+     *
+     * @return Returns the string contents of a TEXT_BLOCK lexeme, or null if not a text block.
+     */
+    public String getTextBlockContents() {
+        return textBlockContents;
+    }
+
     /**
      * Gets the associated error message with the token if it's an error.
      *

smithy-syntax/src/main/java/software/amazon/smithy/syntax/CapturingTokenizer.java

@@ -96,6 +96,11 @@ public int getCurrentTokenEnd() {
         return getPosition();
     }
 
+    @Override
+    public CharSequence getCurrentTextBlockContents() {
+        return getToken().getTextBlockContents();
+    }
+
     @Override
     public CharSequence getCurrentTokenStringSlice() {
         return getToken().getStringContents();

smithy-syntax/src/main/java/software/amazon/smithy/syntax/FormatVisitor.java

@@ -462,7 +462,7 @@ Doc visit(TreeCursor cursor) {
                         .tokens()
                         .findFirst()
                         .orElseThrow(() -> new RuntimeException("TEXT_BLOCK cursor does not have an IDL token"))
-                        .getStringContents();
+                        .getTextBlockContents();
 
                 // If the last character is a newline, then the closing triple quote must be on the next line.
                 boolean endQuoteOnNextLine = stringValue.endsWith("\n") || stringValue.endsWith("\r");

smithy-syntax/src/test/resources/software/amazon/smithy/syntax/formatter/trait-textblock.formatted.smithy

@@ -86,3 +86,16 @@ string ExtraTrailingNewlines
     """
 )
 string EmptyTextBlock
+
+// Ensure that the escaped new lines are preserved and the contents are aligned with the opening quote.
+@pattern(
+    """
+    ^[a-z\
+    A-Z]+\
+    [a-z\
+    A-Z\
+    0-9]+\
+    $
+    """
+)
+string identifier

smithy-syntax/src/test/resources/software/amazon/smithy/syntax/formatter/trait-textblock.smithy

@@ -76,3 +76,16 @@ string ExtraTrailingNewlines
 @documentation("""
     """)
 string EmptyTextBlock
+
+// Ensure that the escaped new lines are preserved and the contents are aligned with the opening quote.
+@pattern(
+    """
+^[a-z\
+A-Z]+\
+[a-z\
+A-Z\
+0-9]+\
+$
+    """
+)
+string identifier