Support skipping "// ..." and "/* ... */" comments

Summary: Implement JSON5 comment parsing. Now `skipWhitespace()` will invoke both `skipWhitespaceOnly()` and `skipComment` to skip both.

Reviewed By: yfeldblum

Differential Revision: D93452598

fbshipit-source-id: e212d62dd0f1055c492a627c6d85743bf7daaf53

Author: Mizuchi

third-party/folly/src/folly/json/json.cpp

@@ -353,7 +353,7 @@ struct Input {
     });
   }
 
-  void skipWhitespace() {
+  bool skipWhitespaceOnly() {
     unsigned index = 0;
     while (true) {
       while (index < range_.size() && range_[index] == ' ') {
@@ -374,6 +374,48 @@ struct Input {
     }
     range_.advance(index);
     storeCurrent();
+    return index;
+  }
+
+  bool skipComment() {
+    if (!opts_.allow_json5_experimental) {
+      return false;
+    }
+    if (consume("//")) {
+      // Single-line comment: skip until CR or LF.
+      char prev = 0;
+      skipWhile([&prev](char curr) {
+        if (prev == '\r' || prev == '\n') {
+          return false;
+        }
+        prev = curr;
+        return true;
+      });
+      return true;
+    }
+    if (consume("/*")) {
+      // Block comment: skip until closing "*/".
+      char prev = 0;
+      skipWhile([&prev](char curr) {
+        if (prev == '*' && curr == '/') {
+          return false;
+        }
+        prev = curr;
+        return true;
+      });
+      if (consume("/")) {
+        return true;
+      }
+      error("unterminated block comment");
+    }
+    return false;
+  }
+
+  void skipWhitespace() {
+    do {
+      skipWhitespaceOnly();
+      // Loop to handle adjacent comments, e.g. `/*a*//*b*/`.
+    } while (skipComment());
   }
 
   void expect(char c) {

third-party/folly/src/folly/json/test/Json5Test.cpp

@@ -67,14 +67,25 @@ constexpr auto kPassedTests = {
     "src/arrays/regular-array.json",
     "src/arrays/trailing-comma-array.json5",
     // Comments
+    "src/comments/block-comment-following-array-element.json5",
+    "src/comments/block-comment-following-top-level-value.json5",
     "src/comments/block-comment-in-string.json",
+    "src/comments/block-comment-preceding-top-level-value.json5",
+    "src/comments/block-comment-with-asterisks.json5",
+    "src/comments/inline-comment-following-array-element.json5",
+    "src/comments/inline-comment-following-top-level-value.json5",
     "src/comments/inline-comment-in-string.json",
+    "src/comments/inline-comment-preceding-top-level-value.json5",
     "src/comments/top-level-block-comment.txt",
     "src/comments/top-level-inline-comment.txt",
     "src/comments/unterminated-block-comment.txt",
     // Misc
     "src/misc/empty.txt",
     "src/misc/npm-package.json",
+    // New lines
+    "src/new-lines/comment-cr.json5",
+    "src/new-lines/comment-crlf.json5",
+    "src/new-lines/comment-lf.json5",
     // Numbers
     "src/numbers/float.json",
     "src/numbers/float-leading-zero.json",
@@ -125,22 +136,11 @@ constexpr auto kPassedTests = {
 };
 
 constexpr auto kFailedTests = {
-    // Comments
-    "src/comments/block-comment-following-array-element.json5",
-    "src/comments/block-comment-following-top-level-value.json5",
-    "src/comments/block-comment-preceding-top-level-value.json5",
-    "src/comments/block-comment-with-asterisks.json5",
-    "src/comments/inline-comment-following-array-element.json5",
-    "src/comments/inline-comment-following-top-level-value.json5",
-    "src/comments/inline-comment-preceding-top-level-value.json5",
     // Misc
     "src/misc/npm-package.json5",
     "src/misc/readme-example.json5",
     "src/misc/valid-whitespace.json5",
     // New lines
-    "src/new-lines/comment-cr.json5",
-    "src/new-lines/comment-crlf.json5",
-    "src/new-lines/comment-lf.json5",
     "src/new-lines/escaped-cr.json5",
     "src/new-lines/escaped-crlf.json5",
     "src/new-lines/escaped-lf.json5",

third-party/folly/src/folly/json/test/JsonOtherTest.cpp

@@ -17,6 +17,12 @@
 #include <folly/json/json.h>
 #include <folly/portability/GTest.h>
 
+// Create a wrapper to suppress lint warnings
+folly::dynamic fromJson5(folly::StringPiece s) {
+  // @lint-ignore CLANGTIDY clang-diagnostic-deprecated-declarations
+  return folly::parseJson5(s);
+}
+
 TEST(Json, StripComments) {
   auto testStr = folly::stripLeftMargin(R"JSON(
     {
@@ -29,8 +35,8 @@ TEST(Json, StripComments) {
       "test4": "foo /* bar", /* comment */
       "te//": "foo",
       "te/*": "bar",
-      "\\\"": "\\" /* comment */
-    }
+      "\\\"": "\\" /* comment *//* adjacent comment */
+    } // more comments
   )JSON");
   auto expectedStr = folly::stripLeftMargin(R"JSON(
     {
@@ -44,8 +50,18 @@ TEST(Json, StripComments) {
       "te//": "foo",
       "te/*": "bar",
       "\\\"": "\\" 
-    }
+    } 
   )JSON");
 
   EXPECT_EQ(expectedStr, folly::json::stripComments(testStr));
+  EXPECT_EQ(folly::parseJson(expectedStr), fromJson5(testStr));
+
+  // Without json5, comments should fail
+  EXPECT_THROW(folly::parseJson(testStr), std::exception);
+
+  // Unterminated block comment should fail
+  EXPECT_THROW(fromJson5(testStr + " /* unterminated"), std::exception);
+
+  EXPECT_EQ(fromJson5("42 /**/"), 42);
+  EXPECT_THROW(fromJson5("42 /*/"), std::exception);
 }