Parser: Friendly token names in Unexpected Token error messages (#194)

This pull request changes the parser's error messages from technical
token names to user-friendly format.

**Before:**
```
Unexpected Token. Expected: `TOKEN_IDENTIFIER, TOKEN_AT, TOKEN_ERB_START,TOKEN_WHITESPACE, or TOKEN_NEWLINE`, found: `TOKEN_COLON`.
```

**After:**
```
Unexpected Token. Expected: an identifier, `@`, `<%`, whitespace, or a newline, found: `:`.
```

### Format conventions

- **Literal tokens** (punctuation/delimiters) are backtick-quoted: ``
`<` ``, `` `<%` ``, `` `@` ``, `` `:` ``
- **Abstract tokens** use natural English with articles: `an
identifier`, `a quote`, `whitespace`, `end of file`
- **Lists** use Oxford comma: `an identifier, @, or <%`

Author: marcoroth

config.yml

@@ -12,7 +12,7 @@ errors:
   types:
     - name: UnexpectedError
       message:
-        template: "%s. Expected: `%s`, found: `%s`."
+        template: "%s. Expected: %s, found: %s."
         arguments:
           - description
           - expected
@@ -30,10 +30,10 @@ errors:
 
     - name: UnexpectedTokenError
       message:
-        template: "Found `%s` when expecting `%s` at (%u:%u)."
+        template: "Found %s when expecting %s at (%u:%u)."
         arguments:
-          - token_type_to_string(found->type)
-          - token_type_to_string(expected_type)
+          - token_type_to_friendly_string(found->type)
+          - token_type_to_friendly_string(expected_type)
           - found->location.start.line
           - found->location.start.column
 

javascript/packages/linter/test/__snapshots__/cli.test.ts.snap

@@ -13,11 +13,21 @@ void parser_push_open_tag(const parser_T* parser, token_T* tag_name);
 bool parser_check_matching_tag(const parser_T* parser, hb_string_T tag_name);
 token_T* parser_pop_open_tag(const parser_T* parser);
 
-void parser_append_unexpected_error(
+void parser_append_unexpected_error_impl(
   parser_T* parser,
+  hb_array_T* errors,
   const char* description,
-  const char* expected,
-  hb_array_T* errors
+  token_type_T first_token,
+  ...
+);
+#define parser_append_unexpected_error(parser, errors, description, ...)                                               \
+  parser_append_unexpected_error_impl(parser, errors, description, __VA_ARGS__, TOKEN_SENTINEL)
+
+void parser_append_unexpected_error_string(
+  parser_T* parser,
+  hb_array_T* errors,
+  const char* description,
+  const char* expected
 );
 void parser_append_unexpected_token_error(parser_T* parser, token_type_T expected_type, hb_array_T* errors);
 

src/include/parser_helpers.h

@@ -6,9 +6,16 @@
 #include "token_struct.h"
 #include "util/hb_string.h"
 
+#include <stdarg.h>
+
 token_T* token_init(hb_string_T value, token_type_T type, lexer_T* lexer);
 hb_string_T token_to_string(const token_T* token);
 const char* token_type_to_string(token_type_T type);
+const char* token_type_to_friendly_string(token_type_T type);
+char* token_types_to_friendly_string_va(token_type_T first_token, ...);
+char* token_types_to_friendly_string_valist(token_type_T first_token, va_list args);
+
+#define token_types_to_friendly_string(...) token_types_to_friendly_string_va(__VA_ARGS__, TOKEN_SENTINEL)
 
 token_T* token_copy(token_T* token);
 

src/include/token.h

@@ -49,6 +49,9 @@ typedef enum {
   TOKEN_EOF,
 } token_type_T;
 
+// Sentinel value for variadic functions
+#define TOKEN_SENTINEL 99999999
+
 typedef struct TOKEN_STRUCT {
   char* value;
   range_T range;

src/include/token_struct.h

@@ -260,17 +260,7 @@ static AST_HTML_TEXT_NODE_T* parser_parse_text_content(parser_T* parser, hb_arra
     if (token_is(parser, TOKEN_ERROR)) {
       free(content.value);
 
-      token_T* token = parser_consume_expected(parser, TOKEN_ERROR, document_errors);
-      append_unexpected_error(
-        "Token Error",
-        "not TOKEN_ERROR",
-        token->value,
-        token->location.start,
-        token->location.end,
-        document_errors
-      );
-
-      token_free(token);
+      parser_append_unexpected_error_string(parser, document_errors, "Token Error", "not an error token");
 
       return NULL;
     }
@@ -641,7 +631,7 @@ static AST_HTML_ATTRIBUTE_VALUE_NODE_T* parser_parse_html_attribute_value(parser
     append_unexpected_error(
       "Invalid quote character for HTML attribute",
       "single quote (') or double quote (\")",
-      "backtick (`)",
+      "a backtick",
       start,
       end,
       errors
@@ -655,15 +645,19 @@ static AST_HTML_ATTRIBUTE_VALUE_NODE_T* parser_parse_html_attribute_value(parser
     return value;
   }
 
+  char* expected = token_types_to_friendly_string(TOKEN_IDENTIFIER, TOKEN_QUOTE, TOKEN_ERB_START);
+
   append_unexpected_error(
     "Unexpected Token",
-    "TOKEN_IDENTIFIER, TOKEN_QUOTE, TOKEN_ERB_START",
-    token_type_to_string(parser->current_token->type),
+    expected,
+    token_type_to_friendly_string(parser->current_token->type),
     parser->current_token->location.start,
     parser->current_token->location.end,
     errors
   );
 
+  free(expected);
+
   AST_HTML_ATTRIBUTE_VALUE_NODE_T* value = ast_html_attribute_value_node_init(
     NULL,
     children,
@@ -1057,9 +1051,13 @@ static AST_HTML_OPEN_TAG_NODE_T* parser_parse_html_open_tag(parser_T* parser) {
 
     parser_append_unexpected_error(
       parser,
+      errors,
       "Unexpected Token",
-      "TOKEN_IDENTIFIER, TOKEN_AT, TOKEN_ERB_START,TOKEN_WHITESPACE, or TOKEN_NEWLINE",
-      errors
+      TOKEN_IDENTIFIER,
+      TOKEN_AT,
+      TOKEN_ERB_START,
+      TOKEN_WHITESPACE,
+      TOKEN_NEWLINE
     );
   }
 
@@ -1474,10 +1472,17 @@ static void parser_parse_in_data_state(parser_T* parser, hb_array_T* children, h
 
     parser_append_unexpected_error(
       parser,
+      errors,
       "Unexpected token",
-      "TOKEN_ERB_START, TOKEN_HTML_DOCTYPE, TOKEN_HTML_COMMENT_START, TOKEN_IDENTIFIER, TOKEN_WHITESPACE, "
-      "TOKEN_NBSP, TOKEN_AT, TOKEN_BACKSLASH, or TOKEN_NEWLINE",
-      errors
+      TOKEN_ERB_START,
+      TOKEN_HTML_DOCTYPE,
+      TOKEN_HTML_COMMENT_START,
+      TOKEN_IDENTIFIER,
+      TOKEN_WHITESPACE,
+      TOKEN_NBSP,
+      TOKEN_AT,
+      TOKEN_BACKSLASH,
+      TOKEN_NEWLINE
     );
 
     parser_synchronize(parser, errors);

src/parser.c

@@ -8,6 +8,7 @@
 #include "include/util/hb_buffer.h"
 #include "include/util/hb_string.h"
 
+#include <stdarg.h>
 #include <stdio.h>
 
 void parser_push_open_tag(const parser_T* parser, token_T* tag_name) {
@@ -90,18 +91,45 @@ void parser_exit_foreign_content(parser_T* parser) {
   parser->foreign_content_type = FOREIGN_CONTENT_UNKNOWN;
 }
 
-void parser_append_unexpected_error(
+void parser_append_unexpected_error_impl(
   parser_T* parser,
+  hb_array_T* errors,
   const char* description,
-  const char* expected,
-  hb_array_T* errors
+  token_type_T first_token,
+  ...
+) {
+  token_T* token = parser_advance(parser);
+
+  va_list args;
+  va_start(args, first_token);
+  char* expected = token_types_to_friendly_string_valist(first_token, args);
+  va_end(args);
+
+  append_unexpected_error(
+    description,
+    expected,
+    token_type_to_friendly_string(token->type),
+    token->location.start,
+    token->location.end,
+    errors
+  );
+
+  free(expected);
+  token_free(token);
+}
+
+void parser_append_unexpected_error_string(
+  parser_T* parser,
+  hb_array_T* errors,
+  const char* description,
+  const char* expected
 ) {
   token_T* token = parser_advance(parser);
 
   append_unexpected_error(
     description,
     expected,
-    token_type_to_string(token->type),
+    token_type_to_friendly_string(token->type),
     token->location.start,
     token->location.end,
     errors

src/parser_helpers.c

@@ -3,7 +3,9 @@
 #include "include/range.h"
 #include "include/token_struct.h"
 #include "include/util.h"
+#include "include/util/hb_buffer.h"
 
+#include <stdarg.h>
 #include <stdbool.h>
 #include <stdio.h>
 #include <stdlib.h>
@@ -78,8 +80,82 @@ const char* token_type_to_string(const token_type_T type) {
     case TOKEN_ERROR: return "TOKEN_ERROR";
     case TOKEN_EOF: return "TOKEN_EOF";
   }
+}
+
+const char* token_type_to_friendly_string(const token_type_T type) {
+  switch (type) {
+    case TOKEN_WHITESPACE: return "whitespace";
+    case TOKEN_NBSP: return "non-breaking space";
+    case TOKEN_NEWLINE: return "a newline";
+    case TOKEN_IDENTIFIER: return "an identifier";
+    case TOKEN_HTML_DOCTYPE: return "`<!DOCTYPE`";
+    case TOKEN_XML_DECLARATION: return "`<?xml`";
+    case TOKEN_XML_DECLARATION_END: return "`?>`";
+    case TOKEN_CDATA_START: return "`<![CDATA[`";
+    case TOKEN_CDATA_END: return "`]]>`";
+    case TOKEN_HTML_TAG_START: return "`<`";
+    case TOKEN_HTML_TAG_END: return "`>`";
+    case TOKEN_HTML_TAG_START_CLOSE: return "`</`";
+    case TOKEN_HTML_TAG_SELF_CLOSE: return "`/>`";
+    case TOKEN_HTML_COMMENT_START: return "`<!--`";
+    case TOKEN_HTML_COMMENT_END: return "`-->`";
+    case TOKEN_HTML_COMMENT_INVALID_END: return "`--!>`";
+    case TOKEN_EQUALS: return "`=`";
+    case TOKEN_QUOTE: return "a quote";
+    case TOKEN_BACKTICK: return "a backtick";
+    case TOKEN_BACKSLASH: return "`\\`";
+    case TOKEN_DASH: return "`-`";
+    case TOKEN_UNDERSCORE: return "`_`";
+    case TOKEN_EXCLAMATION: return "`!`";
+    case TOKEN_SLASH: return "`/`";
+    case TOKEN_SEMICOLON: return "`;`";
+    case TOKEN_COLON: return "`:`";
+    case TOKEN_AT: return "`@`";
+    case TOKEN_LT: return "`<`";
+    case TOKEN_PERCENT: return "`%`";
+    case TOKEN_AMPERSAND: return "`&`";
+    case TOKEN_ERB_START: return "`<%`";
+    case TOKEN_ERB_CONTENT: return "ERB content";
+    case TOKEN_ERB_END: return "`%>`";
+    case TOKEN_CHARACTER: return "a character";
+    case TOKEN_ERROR: return "an error token";
+    case TOKEN_EOF: return "end of file";
+  }
+}
+
+char* token_types_to_friendly_string_valist(token_type_T first_token, va_list args) {
+  if ((int) first_token == TOKEN_SENTINEL) { return herb_strdup(""); }
+
+  size_t count = 0;
+  const char* names[32];
+  token_type_T current = first_token;
+
+  while ((int) current != TOKEN_SENTINEL && count < 32) {
+    names[count++] = token_type_to_friendly_string(current);
+    current = va_arg(args, token_type_T);
+  }
+
+  hb_buffer_T buffer;
+  hb_buffer_init(&buffer, 128);
+
+  for (size_t i = 0; i < count; i++) {
+    hb_buffer_append(&buffer, names[i]);
+
+    if (i < count - 1) {
+      if (count > 2) { hb_buffer_append(&buffer, ", "); }
+      if (i == count - 2) { hb_buffer_append(&buffer, count == 2 ? " or " : "or "); }
+    }
+  }
+
+  return hb_buffer_value(&buffer);
+}
 
-  return "Unknown token_type_T";
+char* token_types_to_friendly_string_va(token_type_T first_token, ...) {
+  va_list args;
+  va_start(args, first_token);
+  char* result = token_types_to_friendly_string_valist(first_token, args);
+  va_end(args);
+  return result;
 }
 
 hb_string_T token_to_string(const token_T* token) {

src/token.c

@@ -7,6 +7,48 @@ TEST(test_token)
   ck_assert_str_eq(token_type_to_string(TOKEN_IDENTIFIER), "TOKEN_IDENTIFIER");
 END
 
+TEST(test_token_type_to_friendly_string)
+  ck_assert_str_eq(token_type_to_friendly_string(TOKEN_IDENTIFIER), "an identifier");
+  ck_assert_str_eq(token_type_to_friendly_string(TOKEN_WHITESPACE), "whitespace");
+  ck_assert_str_eq(token_type_to_friendly_string(TOKEN_NEWLINE), "a newline");
+  ck_assert_str_eq(token_type_to_friendly_string(TOKEN_QUOTE), "a quote");
+  ck_assert_str_eq(token_type_to_friendly_string(TOKEN_CHARACTER), "a character");
+  ck_assert_str_eq(token_type_to_friendly_string(TOKEN_EOF), "end of file");
+  ck_assert_str_eq(token_type_to_friendly_string(TOKEN_HTML_TAG_START), "`<`");
+  ck_assert_str_eq(token_type_to_friendly_string(TOKEN_HTML_TAG_END), "`>`");
+  ck_assert_str_eq(token_type_to_friendly_string(TOKEN_HTML_TAG_SELF_CLOSE), "`/>`");
+  ck_assert_str_eq(token_type_to_friendly_string(TOKEN_HTML_TAG_START_CLOSE), "`</`");
+  ck_assert_str_eq(token_type_to_friendly_string(TOKEN_HTML_COMMENT_START), "`<!--`");
+  ck_assert_str_eq(token_type_to_friendly_string(TOKEN_HTML_COMMENT_END), "`-->`");
+  ck_assert_str_eq(token_type_to_friendly_string(TOKEN_EQUALS), "`=`");
+  ck_assert_str_eq(token_type_to_friendly_string(TOKEN_SLASH), "`/`");
+  ck_assert_str_eq(token_type_to_friendly_string(TOKEN_COLON), "`:`");
+  ck_assert_str_eq(token_type_to_friendly_string(TOKEN_ERB_START), "`<%`");
+  ck_assert_str_eq(token_type_to_friendly_string(TOKEN_ERB_END), "`%>`");
+END
+
+TEST(test_token_types_to_friendly_string)
+  char* result1 = token_types_to_friendly_string(TOKEN_IDENTIFIER);
+  ck_assert_str_eq(result1, "an identifier");
+  free(result1);
+
+  char* result2 = token_types_to_friendly_string(TOKEN_IDENTIFIER, TOKEN_QUOTE);
+  ck_assert_str_eq(result2, "an identifier or a quote");
+  free(result2);
+
+  char* result3 = token_types_to_friendly_string(TOKEN_IDENTIFIER, TOKEN_QUOTE, TOKEN_ERB_START);
+  ck_assert_str_eq(result3, "an identifier, a quote, or `<%`");
+  free(result3);
+
+  char* result4 = token_types_to_friendly_string(TOKEN_IDENTIFIER, TOKEN_ERB_START, TOKEN_WHITESPACE, TOKEN_NEWLINE);
+  ck_assert_str_eq(result4, "an identifier, `<%`, whitespace, or a newline");
+  free(result4);
+
+  char* result5 = token_types_to_friendly_string(TOKEN_HTML_TAG_START, TOKEN_HTML_TAG_END, TOKEN_EQUALS);
+  ck_assert_str_eq(result5, "`<`, `>`, or `=`");
+  free(result5);
+END
+
 TEST(test_token_to_string)
   hb_buffer_T output;
   hb_buffer_init(&output, 1024);
@@ -25,6 +67,8 @@ TCase *token_tests(void) {
   TCase *token = tcase_create("Token");
 
   tcase_add_test(token, test_token);
+  tcase_add_test(token, test_token_type_to_friendly_string);
+  tcase_add_test(token, test_token_types_to_friendly_string);
   tcase_add_test(token, test_token_to_string);
 
   return token;