C lexer usable in Bun

croqaz · croqaz · commit 4fde77fb0ea3 · 2025-07-24T22:38:07.000+01:00
diff --git a/cimp/lexer.c b/cimp/lexer.c
@@ -0,0 +1,23 @@
+#include "lexer.h"
+
+//
+// ▰ ▰ ▰ ▰ ▰ ▰ ▰
+// Export to JS
+// ▰ ▰ ▰ ▰ ▰ ▰ ▰
+//
+
+char* lex(uint32_t* text, size_t text_len) {
+    printf("[lex] Lexing %zu characters\n", text_len);
+
+    Lexer lexer;
+    lexer_init(&lexer);
+    lexer_parse_chunk(&lexer, text, text_len);
+    lexer_finish(&lexer);
+
+    char out[1024 * 1024];
+    lexer_to_js(&lexer, out, sizeof(out));
+    printf("[lex] Processed %zu tokens\n", lexer.processed_len);
+    lexer_free(&lexer);
+    // Return a copy of the output string
+    return strdup(out);
+}
diff --git a/cimp/lexer.h b/cimp/lexer.h
@@ -121,8 +121,61 @@ void lexer_reset(Lexer *lexer) {
     lexer->processed_len = 0;
 }
 
-// void lexer_to_js(const Lexer *lexer, ...)
-// TODO
+/*
+ * Convert the lexer state to a JavaScript array of tokens.
+ */
+static inline void lexer_to_js(const Lexer *lexer, char *out, size_t out_size) {
+    if (!lexer || !out || out_size == 0) {
+        if (out && out_size > 0) {
+            out[0] = '\0';
+        }
+        return;
+    }
+
+    // Initialize an empty string in case there are no tokens
+    out[0] = '\0';
+    size_t current_len = 0;
+    size_t remaining_size = out_size;
+
+    // Start the JavaScript array
+    int written = snprintf(out, remaining_size, "[\n");
+    if (written < 0 || (size_t)written >= remaining_size) {
+        // Not enough space, out is now truncated but null-terminated.
+        return;
+    }
+    current_len += written;
+    remaining_size -= written;
+
+    // Iterate over each processed token
+    for (size_t i = 0; i < lexer->processed_len; ++i) {
+        // Convert the current token to its JS representation
+        // We pass the pointer to the current end of the output string
+        token_to_js(&lexer->processed[i], out + current_len, remaining_size);
+
+        // Update current length and remaining size
+        size_t token_len = strlen(out + current_len);
+        current_len += token_len;
+
+        if (current_len >= out_size - 1) {
+            // Buffer is full, we can't add a comma or the closing bracket.
+            return;
+        }
+        remaining_size = out_size - current_len;
+
+        // Add a comma if this is not the last token
+        if (i < lexer->processed_len - 1) {
+            written = snprintf(out + current_len, remaining_size, ",\n");
+            if (written < 0 || (size_t)written >= remaining_size) {
+                return;  // Not enough space for comma
+            }
+            current_len += written;
+            remaining_size -= written;
+        }
+    }
+
+    // Close the JavaScript array
+    snprintf(out + current_len, remaining_size, "\n]");
+}
 
 /*
  * Transition to a new lexer state.
@@ -166,8 +219,8 @@ static void lexer__commit(Lexer *lexer) {
                last_pos - token->pos_start, token->pos_start, last_pos);
     } else {
         const char *name = token_name_utf8(token);
-        printf("[Lexer__commit] Commit token: name=%s, type: %d, pos: %zu-%zu\n",
-               name, token->type, token->pos_start, last_pos);
+        printf("[Lexer__commit] Commit token type: %d: name=%s, params: %zu, pos: %zu-%zu\n",
+               token->type, name, token->param_len, token->pos_start, last_pos);
     }
 
     // Check if we need to reallocate the processed tokens array
@@ -181,27 +234,25 @@ static void lexer__commit(Lexer *lexer) {
         }
     }
 
-    // Add the pending token to the processed tokens
-    // This creates a copy, so the pending token must
-    // be freed or reset later
+    // Shallow copy the pending token to the processed tokens
     lexer->processed[lexer->processed_len++] = *token;
-    // Re-create the pending token
-    token_reset(&lexer->pendNode);
-    // lexer->pendNode = *token_create();
+    // Create a new pending token for the next cycle
+    // TODO :: check if this causes memory leaks !!
+    lexer->pendNode = *token_create();
     lexer->pendNode.pos_start = last_pos;
     lexer->pendNode.pos_end = last_pos;
 }
 
 static inline void lexer__commit_param(Lexer *lexer) {
     token_param_append(&lexer->pendNode, &lexer->pendParam);
     // Re-create the pending parameter
-    // Maybe is should be reset instead?
+    // TODO :: check if this causes memory leaks !!
     lexer->pendParam = *param_create();
 }
 
 static inline void lexer__parse_one(Lexer *lexer, uint32_t curr, uint32_t prev) {
-    // printf("i=%ld - STATE :: %u ;; new CHAR :: (%d) ;; prev CHAR :: (%d)\n",
-    //        lexer->index, lexer->state, (int)curr, (int)prev);
+    printf("i=%ld - STATE :: %u ;; new CHAR :: (%d) ;; prev CHAR :: (%d)\n",
+           lexer->index, lexer->state, (int)curr, (int)prev);
 
     if (lexer->state == STATE_RAW_TEXT) {
         // Could this be the beginning of a new tag?
@@ -389,6 +440,7 @@ static inline void lexer__parse_one(Lexer *lexer, uint32_t curr, uint32_t prev)
         }
         // Is this a valid closing quote?
         else if (curr == value_0 && is_quote(curr) && value_z != '\\') {
+            param_val_append(&lexer->pendParam, curr);
             lexer__commit_param(lexer);
             lexer__transition(lexer, STATE_INSIDE_TAG);
         }
diff --git a/cimp/lexer2.ts b/cimp/lexer2.ts
@@ -0,0 +1,29 @@
+import { dlopen, FFIType, ptr } from "bun:ffi";
+const path = "liblexer.so";
+const {
+  symbols: { lex },
+} = dlopen(path, {
+  lex: {
+    args: ["ptr", "usize"],
+    returns: FFIType.cstring,
+  },
+});
+
+// Method 2: Using Bun's FFI with a C source file
+// import { cc, FFIType, ptr } from "bun:ffi";
+// import source from "./lexer.c" with { type: "file" };
+// const {
+//   symbols: { lex },
+// } = cc({
+//   source,
+//   symbols: {
+//     lex: {
+//       args: ["ptr", "usize"],
+//       returns: FFIType.cstring,
+//     },
+//   },
+// });
+
+const buff = new Uint32Array(Buffer.from("hello\0", "utf8"));
+const result = lex(ptr(buff), buff.length);
+console.log(eval(result.toString()));
diff --git a/cimp/main.c b/cimp/main.c
@@ -0,0 +1,25 @@
+#include "lexer.h"
+
+int main(int argc, char *argv[]) {
+    // The program name is argv[0], first arg is argv[1].
+    // The app expects exactly one argument: the filename.
+    if (argc != 2) {
+        // Print usage instructions to standard error if the argument is missing.
+        fprintf(stderr, "Usage: %s <filename>\n", argv[0]);
+        return 1;
+    }
+
+    Lexer lexer;
+    lexer_init(&lexer);
+
+    const char *filename = argv[1];
+    printf("Processing file: %s\n", filename);
+    lexer_parse_file(&lexer, filename);
+
+    char out[1024 * 1024];
+    lexer_to_js(&lexer, out, sizeof(out));
+    printf("Lexer tokens: %s\n", out);
+
+    lexer_free(&lexer);
+    return 0;
+}
diff --git a/cimp/test/test_lex.c b/cimp/test/test_lex.c
@@ -40,6 +40,17 @@ void test_simple_single(void) {
     TEST_ASSERT_EQUAL(5, lexer.processed[1].pos_end);
     TEST_ASSERT_EQUAL_STRING("b", token_name_utf8(&lexer.processed[1]));
     lexer_reset(&lexer);
+
+    lexer_parse_chunk(&lexer, U"<div id=\"main\"/>", 0);
+    TEST_ASSERT_EQUAL(16, lexer.index);
+    TEST_ASSERT_EQUAL(1, lexer.processed_len);
+    TEST_ASSERT_EQUAL(TYPE_SINGLE_TAG, lexer.processed[0].type);
+    TEST_ASSERT_EQUAL(0, lexer.processed[0].pos_start);
+    TEST_ASSERT_EQUAL(16, lexer.processed[0].pos_end);
+
+    char out[96];
+    lexer_to_js(&lexer, out, sizeof(out));
+    printf("Lexer output: %s\n", out);
 }
 
 void test_simple_double(void) {
@@ -71,7 +82,6 @@ void test_simple_double(void) {
 void test_unicode_chars(void) {
     lexer_init(&lexer);
     lexer_parse_text(&lexer, (const uint32_t *)U"French <àéìòùÀÉÌÒÙ/> German <äöüßÄÖÜ/>\0");
-    // lexer_display(&lexer);
     TEST_ASSERT_EQUAL(38, lexer.index);
     TEST_ASSERT_EQUAL(4, lexer.processed_len);
     TEST_ASSERT_EQUAL(TYPE_RAW_TEXT, lexer.processed[0].type);
@@ -100,10 +110,13 @@ void test_unicode_chars(void) {
 void test_parse_file(void) {
     lexer_init(&lexer);
     lexer_parse_file(&lexer, "fixtures/menu.xml");
-    // lexer_display(&lexer);
     TEST_ASSERT_EQUAL(250, lexer.index);
     TEST_ASSERT_EQUAL(27, lexer.processed_len);
 
+    // char out[1200];
+    // lexer_to_js(&lexer, out, sizeof(out));
+    // printf("Lexer output: %s\n", out);
+
     TEST_ASSERT_EQUAL(TYPE_RAW_TEXT, lexer.processed[0].type);
     TEST_ASSERT_EQUAL(TYPE_DOUBLE_TAG, lexer.processed[1].type);
     TEST_ASSERT_EQUAL_STRING(U"breakfast_menu", lexer.processed[1].name);
@@ -122,8 +135,8 @@ void test_parse_file(void) {
 
 //
 // ▰ ▰ ▰ ▰ ▰ ▰ ▰
-// Serious testss
-// ▰ ▰ ▰ ▰ ▰ ▰ ▰ ▰
+// Serious testz
+// ▰ ▰ ▰ ▰ ▰ ▰ ▰
 //
 
 typedef struct {
@@ -279,6 +292,6 @@ int main(void) {
     RUN_TEST(test_simple_double);
     RUN_TEST(test_unicode_chars);
     RUN_TEST(test_parse_file);
-    // RUN_TEST(test_input_output);
+    RUN_TEST(test_input_output);
     return UNITY_END();
 }
diff --git a/cimp/test/test_tok.c b/cimp/test/test_tok.c
@@ -131,7 +131,7 @@ void test_token_param_append(void) {
 
     char out[64];
     token_to_js(token, out, sizeof(out));
-    TEST_ASSERT_EQUAL_STRING("{type:0,pos_start:0,pos_end:0}", out);
+    TEST_ASSERT_EQUAL_STRING("{}", out);
     token->type = TYPE_SINGLE_TAG;  // Set type for testing
     token->pos_end = 10;
     token_to_js(token, out, sizeof(out));
diff --git a/cimp/token.h b/cimp/token.h
@@ -72,7 +72,7 @@ static inline bool param_val_append(LexParam *param, uint32_t codepoint) {
 
 static inline void param_to_js(const LexParam *param, char *out, size_t out_size) {
     if (!param || !out || out_size == 0) return;
-    if (param->key_len == 0 && param->val.len == 0) {
+    if (param->key_len == 0 || param->val.len == 0) {
         snprintf(out, out_size, "{}");
         return;  // Empty param
     }
@@ -206,7 +206,7 @@ static inline bool token_grow_params(LexToken *tok) {
  */
 static inline const char *token_name_utf8(const LexToken *tok) {
     if (!tok || tok->name_len == 0) return NULL;  // No name
-    static char utf8_name[MAX_NAME_LEN * 4];  // Enough for UTF-8 encoding
+    static char utf8_name[MAX_NAME_LEN * 4];      // Enough for UTF-8 encoding
     size_t pos = 0;
     for (size_t i = 0; i < tok->name_len && pos < sizeof(utf8_name) - 1; i++) {
         char temp[4];
@@ -238,8 +238,6 @@ static inline bool token_param_append(LexToken *tok, LexParam *p) {
     if (tok->param_len >= tok->param_cap) {
         if (!token_grow_params(tok)) return false;  // Allocation failed
     }
-    // printf("[Token_param_append] Appending param: key_len=%zu, val_len=%zu\n",
-    //        p->key_len, p->val.len);
     tok->params[tok->param_len++] = *p;  // Copy the Param
     return true;
 }
@@ -253,7 +251,7 @@ static inline void token_to_js(const LexToken *tok, char *out, size_t out_size)
     // Example double tag:
     //   {type: 2, pos_start: 0, pos_end: 10, name: 'name2', params: [{param_key: 'param_value'}]}
     if (!tok || !out || out_size == 0) return;
-    if (tok->name_len == 0 && tok->param_len == 0) {
+    if (tok->pos_end <= tok->pos_start) {
         snprintf(out, out_size, "{}");
         return;  // Empty token
     }
@@ -273,7 +271,8 @@ static inline void token_to_js(const LexToken *tok, char *out, size_t out_size)
         if (tok->param_len > 0) {
             pos += snprintf(out + pos, out_size - pos, ",params:[{");
             for (size_t i = 0; i < tok->param_len; ++i) {
-                char param_js[256];  // Buffer for a single parameter's JS representation
+                // Limited buffer for a single parameter's JS representation
+                char param_js[1024];
                 param_to_js(&tok->params[i], param_js, sizeof(param_js));
                 pos += snprintf(out + pos, out_size - pos, "%s", param_js);
                 if (i < tok->param_len - 1) {
