Add heredocs.

lib/core/print.toit

@@ -7,7 +7,6 @@ import system.api.print show PrintService PrintServiceClient
 /**
 Prints the $message.
 
-/* TODO(florian): the following sentence will probably get stale soon. */
 The resulting message is stringified using $Object.stringify.
 */
 print message/any:

src/compiler/scanner.cc

@@ -852,20 +852,75 @@ void Scanner::capture_multi_line_comment(int peek) {
   int begin = index_ - 2;
 
   bool is_toitdoc = peek == '*' && look_ahead(1) != '/';
+  if (is_toitdoc) {
+    // Skip the '*'.
+    peek = advance();
+  }
+  if (peek == ' ') {
+    // Skip the space after the '*'.
+    peek = advance();
+  }
+
+  bool is_heredoc = peek == '<' && look_ahead(1) == '<';
+  int heredoc_begin = -1;
+  int heredoc_length = -1;
+  if (is_heredoc) {
+    // Skip the '<<'.
+    advance();
+    peek = advance();
+    while (peek == ' ') {
+      peek = advance();
+    }
+    if (is_newline(peek)) {
+      diagnostics_->report_warning(source_->range(begin, index_),
+                                   "Heredoc comment without a delimiter");
+      is_heredoc = false;
+    } else {
+      heredoc_begin = index_;
+      while (!at_eos() && !is_newline(peek)) {
+        peek = advance();
+      }
+      heredoc_length = index_ - heredoc_begin;
+    }
+  }
 
   int nesting_count = 1;
   while (!at_eos()) {
     if (peek == '*') {
       peek = advance();
       if (peek == '/') {
         peek = advance();
-        nesting_count--;
-        if (nesting_count == 0) break;
+        if (is_heredoc) {
+          // Check whether the heredoc delimiter is present.
+          int offset = 2;
+          if (input_[index_ - 3] == ' ') {
+            // Allow a space before the delimiter.
+            offset = 3;
+          }
+          bool has_match = true;
+          for (int i = 0; i < heredoc_length; i++) {
+            if (input_[heredoc_begin + i] != input_[index_ - offset - heredoc_length + i]) {
+              has_match = false;
+              break;
+            }
+          }
+          if (has_match) {
+            nesting_count--;
+            break;
+          }
+        } else {
+          nesting_count--;
+          if (nesting_count == 0) {
+            break;
+          }
+        }
       }
-    } else if (peek == '/') {
+    } else if (!is_heredoc && peek == '/') {
       peek = advance();
       if (peek == '*') {
         peek = advance();
+        diagnostics_->report_warning(source_->range(index_ - 2, index_),
+                                     "Nested multi-line comments are deprecated. Use heredocs instead.");
         nesting_count++;
       }
     } else if (peek == '\\') {

src/compiler/toitdoc_parser.cc

@@ -199,7 +199,6 @@ class ToitdocParser {
                          bool keep_delimiters_and_escapes,
                          const char* error_message);
   toitdoc::Ref* parse_ref();
-  void skip_comment(bool should_report_error = true);
 
  private:  // Scanning related.
   enum Construct {
@@ -210,7 +209,6 @@ class ToitdocParser {
     ITEM,
     PARAGRAPH,
     CODE_SECTION,
-    COMMENT,
   };
 
   class ConstructScope {
@@ -607,10 +605,6 @@ toitdoc::Paragraph* ToitdocParser::parse_paragraph(int indentation_override) {
         is_special_char = true;
         break;
 
-      case '/':
-        is_special_char = look_ahead(1) == '*';
-        break;
-
       case '\\':
         // Ignore the escape if it is at the end of a line.
         if (is_eol(look_ahead(1))) break;
@@ -661,12 +655,6 @@ toitdoc::Paragraph* ToitdocParser::parse_paragraph(int indentation_override) {
       case '`': expressions.add(parse_code()); break;
       case '"': expressions.add(parse_string()); break;
       case '$': expressions.add(parse_ref()); break;
-      case '/':
-        // We know that '/' is a special char, and therefore that the next character must be
-        // a '*'.
-        ASSERT(look_ahead(1) == '*');
-        skip_comment();
-        break;
       default: UNREACHABLE();
     }
 
@@ -789,34 +777,6 @@ toitdoc::Ref* ToitdocParser::parse_ref() {
   return _new toitdoc::Ref(id, make_symbol(begin, end));
 }
 
-void ToitdocParser::skip_comment(bool should_report_error) {
-  ConstructScope scope(this, COMMENT);
-  ASSERT(look_ahead(0) == '/' && look_ahead(1) == '*');
-  int begin = index_;
-  advance(2);
-  int c;
-  do {
-    c = peek();
-    if (c == '\0') {
-      break;
-    } else if (c == '\\') {
-      if (look_ahead(1) != '\0') {
-        advance(2);
-      } else {
-        advance();
-      }
-    } else if (c == '*' && look_ahead(1) == '/') {
-      advance(2);
-      return;
-    } else {
-      advance();
-    }
-  } while (true);
-  if (should_report_error) {
-    report_error(begin, index_, "Unterminated comment");
-  }
-}
-
 void ToitdocParser::push_construct(Construct construct, int indentation) {
   indentation_stack_.push_back(indentation);
   construct_stack_.push_back(construct);
@@ -848,7 +808,6 @@ std::string ToitdocParser::make_string(int from, int to) {
       replace_newlines_with_space = false;
       break;
 
-    case COMMENT:
     case ITEMIZED:
     case ITEM_START:
     case ITEM:
@@ -947,9 +906,6 @@ int ToitdocParser::peek() {
       allows_empty_line = false;
       must_be_indented = true;
       break;
-
-    case COMMENT:
-      return toitdoc_source_->text()[index_];
   }
 
   if (is_at_dedent_) return '\0';

tests/health/gold/sdk/tests__multi-line-comments-test.toit.gold

@@ -0,0 +1,24 @@
+tests/multi-line-comments-test.toit:26:1: warning: Nested multi-line comments are deprecated. Use heredocs instead.
+/*x/ */
+^~
+tests/multi-line-comments-test.toit:31:1: warning: Nested multi-line comments are deprecated. Use heredocs instead.
+/**/*
+^~
+tests/multi-line-comments-test.toit:37:1: warning: Nested multi-line comments are deprecated. Use heredocs instead.
+/*\*/foo*/
+^~
+tests/multi-line-comments-test.toit:38:1: warning: Nested multi-line comments are deprecated. Use heredocs instead.
+/**\/bar*/
+^~
+tests/multi-line-comments-test.toit:39:1: warning: Nested multi-line comments are deprecated. Use heredocs instead.
+/*\\*/
+^~
+tests/multi-line-comments-test.toit:52:5: warning: Nested multi-line comments are deprecated. Use heredocs instead.
+    /* deeper
+    ^~
+tests/multi-line-comments-test.toit:54:7: warning: Nested multi-line comments are deprecated. Use heredocs instead.
+      /* and deeper */  ::::  // Some bad code that would trigger syntax errors if parsed.
+      ^~
+tests/multi-line-comments-test.toit:56:7: warning: Nested multi-line comments are deprecated. Use heredocs instead.
+      /* deep again, closing with trailing * */*
+      ^~

tests/health/gold/sdk/tests__syntax-test.toit.gold

@@ -1,3 +1,12 @@
+tests/syntax-test.toit:47:1: warning: Nested multi-line comments are deprecated. Use heredocs instead.
+/* nested multiline */
+^~
+tests/syntax-test.toit:63:1: warning: Nested multi-line comments are deprecated. Use heredocs instead.
+/* with nested comment */
+^~
+tests/syntax-test.toit:69:1: warning: Nested multi-line comments are deprecated. Use heredocs instead.
+/**
+^~
 <sdk>/bytes.toit:254:7: warning: Class 'BufferConsumer' is deprecated
 class BufferSizeCounter extends BufferConsumer:
       ^~~~~~~~~~~~~~~~~

tests/lsp/toitdoc2-compiler-test.toit

@@ -418,7 +418,6 @@ test client/LspClient:
         - 1
           - nest1
           - nest2
-         /* weird indentation follows. also a test with comments. */
          ```
          code section
          ```
@@ -579,30 +578,3 @@ test client/LspClient:
             ],
           ],
       ]
-
-  test-toitdoc
-      client
-      """
-        /**
-        foo/*:  '\$', '"', '`'.*/bar
-        /*
-        dollar needs to be followed by id: \$5.4 \$, 5\$
-        strings, too: "'", "\$", "`", "```"
-        */
-        /*"`'*/
-        /*\\*/still in comment*/
-        /*\\\\*/
-        done
-        */
-      """
-      Contents [
-        Section null
-          [
-            Paragraph [
-              Text "foobar"
-            ],
-            Paragraph [
-              Text "done"
-            ],
-          ],
-      ]

tests/multi-line-comments-heredoc-test.toit

@@ -0,0 +1,39 @@
+// Copyright (C) 2024 Toitware ApS.
+// Use of this source code is governed by a Zero-Clause BSD license that can
+// be found in the tests/LICENSE file.
+
+import expect show *
+
+// Tests whether the parser correctly handles heredoc multiline comments.
+
+main:
+  expect test1 == "correct"
+  expect test2 == "correct"
+  expect test3 == "correct"
+
+/** << with spaces
+*/ closing ignored.
+with spaces */
+test1:
+  /* << heredoc
+  */ closing ignored.
+  heredoc */
+  return "correct"
+
+/**<<with spaces
+*/ closing ignored.
+with spaces*/
+test2:
+  /*<<heredoc
+  */ closing ignored.
+  heredoc*/
+  return "correct"
+
+/** << with spaces
+*/ closing ignored.
+with spaces*/
+test3:
+  /* << heredoc
+  /* nested ignored
+  heredoc*/
+  return "correct"

tests/negative/gold/heredoc-test.gold

@@ -0,0 +1,7 @@
+tests/negative/heredoc-test.toit:6:3: warning: Heredoc comment without a delimiter
+  /*<<
+  ^~~~
+tests/negative/heredoc-test.toit:8:3: error: Unresolved identifier: 'unresolved'
+  unresolved
+  ^~~~~~~~~~
+Compilation failed

tests/negative/gold/incomplete-comments2-test.gold

@@ -1,3 +1,6 @@
+tests/negative/incomplete-comments2-test.toit:6:5: warning: Nested multi-line comments are deprecated. Use heredocs instead.
+  /*/*
+    ^~
 tests/negative/incomplete-comments2-test.toit:6:3: error: Unterminated multi-line comment
   /*/*
   ^~~~

tests/negative/heredoc-test.toit

@@ -0,0 +1,8 @@
+// Copyright (C) 2024 Toitware ApS.
+// Use of this source code is governed by a Zero-Clause BSD license that can
+// be found in the tests/LICENSE file.
+
+main:
+  /*<<
+  */
+  unresolved

tools/firmware.toit

@@ -1557,24 +1557,24 @@ class ImageHeader:
 The image format is as follows:
 
   typedef struct {
-    uint8_t magic;              /*!< Magic word ESP_IMAGE_HEADER_MAGIC */
-    uint8_t segment_count;      /*!< Count of memory segments */
-    uint8_t spi_mode;           /*!< flash read mode (esp_image_spi_mode_t as uint8_t) */
-    uint8_t spi_speed: 4;       /*!< flash frequency (esp_image_spi_freq_t as uint8_t) */
-    uint8_t spi_size: 4;        /*!< flash chip size (esp_image_flash_size_t as uint8_t) */
-    uint32_t entry_addr;        /*!< Entry address */
-    uint8_t wp_pin;             /*!< WP pin when SPI pins set via efuse (read by ROM bootloader,
-                                * the IDF bootloader uses software to configure the WP
-                                * pin and sets this field to 0xEE=disabled) */
-    uint8_t spi_pin_drv[3];     /*!< Drive settings for the SPI flash pins (read by ROM bootloader) */
-    esp_chip_id_t chip_id;      /*!< Chip identification number */
-    uint8_t min_chip_rev;       /*!< Minimum chip revision supported by image */
-    uint8_t reserved[8];        /*!< Reserved bytes in additional header space, currently unused */
-    uint8_t hash_appended;      /*!< If 1, a SHA256 digest "simple hash" (of the entire image) is appended after the checksum.
-                                * Included in image length. This digest
-                                * is separate to secure boot and only used for detecting corruption.
-                                * For secure boot signed images, the signature
-                                * is appended after this (and the simple hash is included in the signed data). */
+    uint8_t magic;              // !< Magic word ESP_IMAGE_HEADER_MAGIC
+    uint8_t segment_count;      // !< Count of memory segments
+    uint8_t spi_mode;           // !< flash read mode (esp_image_spi_mode_t as uint8_t)
+    uint8_t spi_speed: 4;       // !< flash frequency (esp_image_spi_freq_t as uint8_t)
+    uint8_t spi_size: 4;        // !< flash chip size (esp_image_flash_size_t as uint8_t)
+    uint32_t entry_addr;        // !< Entry address
+    uint8_t wp_pin;             // !< WP pin when SPI pins set via efuse (read by ROM bootloader,
+                                //  the IDF bootloader uses software to configure the WP
+                                //  pin and sets this field to 0xEE=disabled)
+    uint8_t spi_pin_drv[3];     // !< Drive settings for the SPI flash pins (read by ROM bootloader)
+    esp_chip_id_t chip_id;      // !< Chip identification number
+    uint8_t min_chip_rev;       // !< Minimum chip revision supported by image
+    uint8_t reserved[8];        // !< Reserved bytes in additional header space, currently unused
+    uint8_t hash_appended;      // !< If 1, a SHA256 digest "simple hash" (of the entire image) is appended after the checksum
+                                //  Included in image length. This digest
+                                //  is separate to secure boot and only used for detecting corruption.
+                                //  For secure boot signed images, the signature
+                                //  is appended after this (and the simple hash is included in the signed data).
   } __attribute__((packed)) esp_image_header_t;
 
 See https://docs.espressif.com/projects/esp-idf/en/latest/api-reference/system/app_image_format.html