From 50c8653759ba00598ae841ce6ba7ba0624965c96 Mon Sep 17 00:00:00 2001 From: Amaan Qureshi Date: Thu, 22 Feb 2024 22:55:50 -0500 Subject: [PATCH 1/5] feat!: use the array api in the scanner --- src/grammar.json | 23 +--- src/parser.c | 2 + src/scanner.c | 213 +++++++++--------------------- src/tree_sitter/alloc.h | 60 +++++++++ src/tree_sitter/array.h | 272 +++++++++++++++++++++++++++++++++++++++ src/tree_sitter/parser.h | 10 +- 6 files changed, 402 insertions(+), 178 deletions(-) create mode 100644 src/tree_sitter/alloc.h create mode 100644 src/tree_sitter/array.h diff --git a/src/grammar.json b/src/grammar.json index 482bfc68..c8278dae 100644 --- a/src/grammar.json +++ b/src/grammar.json @@ -6281,26 +6281,5 @@ "primary_expression", "pattern", "parameter" - ], - "PREC": { - "lambda": -2, - "typed_parameter": -1, - "conditional": -1, - "parenthesized_expression": 1, - "parenthesized_list_splat": 1, - "or": 10, - "and": 11, - "not": 12, - "compare": 13, - "bitwise_or": 14, - "bitwise_and": 15, - "xor": 16, - "shift": 17, - "plus": 18, - "times": 19, - "unary": 20, - "power": 21, - "call": 22 - } + ] } - diff --git a/src/parser.c b/src/parser.c index 7775f9d6..43c8c604 100644 --- a/src/parser.c +++ b/src/parser.c @@ -8219,6 +8219,7 @@ static inline bool sym_identifier_character_set_3(int32_t c) { static bool ts_lex(TSLexer *lexer, TSStateId state) { START_LEXER(); + eof = lexer->eof(lexer); switch (state) { case 0: if (eof) ADVANCE(57); @@ -9403,6 +9404,7 @@ static bool ts_lex(TSLexer *lexer, TSStateId state) { static bool ts_lex_keywords(TSLexer *lexer, TSStateId state) { START_LEXER(); + eof = lexer->eof(lexer); switch (state) { case 0: if (('\t' <= lookahead && lookahead <= '\f') || diff --git a/src/scanner.c b/src/scanner.c index 44058d9e..8c15505f 100644 --- a/src/scanner.c +++ b/src/scanner.c @@ -1,3 +1,4 @@ +#include "tree_sitter/array.h" #include "tree_sitter/parser.h" #include @@ -5,40 +6,6 @@ #include #include -#define MAX(a, b) ((a) > (b) ? (a) : (b)) - -#define VEC_RESIZE(vec, _cap) \ - void *tmp = realloc((vec).data, (_cap) * sizeof((vec).data[0])); \ - assert(tmp != NULL); \ - (vec).data = tmp; \ - (vec).cap = (_cap); - -#define VEC_GROW(vec, _cap) \ - if ((vec).cap < (_cap)) { \ - VEC_RESIZE((vec), (_cap)); \ - } - -#define VEC_PUSH(vec, el) \ - if ((vec).cap == (vec).len) { \ - VEC_RESIZE((vec), MAX(16, (vec).len * 2)); \ - } \ - (vec).data[(vec).len++] = (el); - -#define VEC_POP(vec) (vec).len--; - -#define VEC_NEW \ - { .len = 0, .cap = 0, .data = NULL } - -#define VEC_BACK(vec) ((vec).data[(vec).len - 1]) - -#define VEC_FREE(vec) \ - { \ - if ((vec).data != NULL) \ - free((vec).data); \ - } - -#define VEC_CLEAR(vec) (vec).len = 0; - enum TokenType { NEWLINE, INDENT, @@ -70,21 +37,13 @@ typedef struct { static inline Delimiter new_delimiter() { return (Delimiter){0}; } -static inline bool is_format(Delimiter *delimiter) { - return delimiter->flags & Format; -} +static inline bool is_format(Delimiter *delimiter) { return delimiter->flags & Format; } -static inline bool is_raw(Delimiter *delimiter) { - return delimiter->flags & Raw; -} +static inline bool is_raw(Delimiter *delimiter) { return delimiter->flags & Raw; } -static inline bool is_triple(Delimiter *delimiter) { - return delimiter->flags & Triple; -} +static inline bool is_triple(Delimiter *delimiter) { return delimiter->flags & Triple; } -static inline bool is_bytes(Delimiter *delimiter) { - return delimiter->flags & Bytes; -} +static inline bool is_bytes(Delimiter *delimiter) { return delimiter->flags & Bytes; } static inline int32_t end_character(Delimiter *delimiter) { if (delimiter->flags & SingleQuote) { @@ -99,19 +58,13 @@ static inline int32_t end_character(Delimiter *delimiter) { return 0; } -static inline void set_format(Delimiter *delimiter) { - delimiter->flags |= Format; -} +static inline void set_format(Delimiter *delimiter) { delimiter->flags |= Format; } static inline void set_raw(Delimiter *delimiter) { delimiter->flags |= Raw; } -static inline void set_triple(Delimiter *delimiter) { - delimiter->flags |= Triple; -} +static inline void set_triple(Delimiter *delimiter) { delimiter->flags |= Triple; } -static inline void set_bytes(Delimiter *delimiter) { - delimiter->flags |= Bytes; -} +static inline void set_bytes(Delimiter *delimiter) { delimiter->flags |= Bytes; } static inline void set_end_character(Delimiter *delimiter, int32_t character) { switch (character) { @@ -130,34 +83,8 @@ static inline void set_end_character(Delimiter *delimiter, int32_t character) { } typedef struct { - uint32_t len; - uint32_t cap; - uint16_t *data; -} indent_vec; - -static indent_vec indent_vec_new() { - indent_vec vec = VEC_NEW; - vec.data = calloc(1, sizeof(uint16_t)); - vec.cap = 1; - return vec; -} - -typedef struct { - uint32_t len; - uint32_t cap; - Delimiter *data; -} delimiter_vec; - -static delimiter_vec delimiter_vec_new() { - delimiter_vec vec = VEC_NEW; - vec.data = calloc(1, sizeof(Delimiter)); - vec.cap = 1; - return vec; -} - -typedef struct { - indent_vec indents; - delimiter_vec delimiters; + Array(uint16_t) indents; + Array(Delimiter) delimiters; bool inside_f_string; } Scanner; @@ -165,28 +92,22 @@ static inline void advance(TSLexer *lexer) { lexer->advance(lexer, false); } static inline void skip(TSLexer *lexer) { lexer->advance(lexer, true); } -bool tree_sitter_python_external_scanner_scan(void *payload, TSLexer *lexer, - const bool *valid_symbols) { +bool tree_sitter_python_external_scanner_scan(void *payload, TSLexer *lexer, const bool *valid_symbols) { Scanner *scanner = (Scanner *)payload; - bool error_recovery_mode = - valid_symbols[STRING_CONTENT] && valid_symbols[INDENT]; - bool within_brackets = valid_symbols[CLOSE_BRACE] || - valid_symbols[CLOSE_PAREN] || - valid_symbols[CLOSE_BRACKET]; + bool error_recovery_mode = valid_symbols[STRING_CONTENT] && valid_symbols[INDENT]; + bool within_brackets = valid_symbols[CLOSE_BRACE] || valid_symbols[CLOSE_PAREN] || valid_symbols[CLOSE_BRACKET]; bool advanced_once = false; - if (valid_symbols[ESCAPE_INTERPOLATION] && scanner->delimiters.len > 0 && - (lexer->lookahead == '{' || lexer->lookahead == '}') && - !error_recovery_mode) { - Delimiter delimiter = VEC_BACK(scanner->delimiters); - if (is_format(&delimiter)) { + if (valid_symbols[ESCAPE_INTERPOLATION] && scanner->delimiters.size > 0 && + (lexer->lookahead == '{' || lexer->lookahead == '}') && !error_recovery_mode) { + Delimiter *delimiter = array_back(&scanner->delimiters); + if (is_format(delimiter)) { lexer->mark_end(lexer); bool is_left_brace = lexer->lookahead == '{'; advance(lexer); advanced_once = true; - if ((lexer->lookahead == '{' && is_left_brace) || - (lexer->lookahead == '}' && !is_left_brace)) { + if ((lexer->lookahead == '{' && is_left_brace) || (lexer->lookahead == '}' && !is_left_brace)) { advance(lexer); lexer->mark_end(lexer); lexer->result_symbol = ESCAPE_INTERPOLATION; @@ -196,26 +117,22 @@ bool tree_sitter_python_external_scanner_scan(void *payload, TSLexer *lexer, } } - if (valid_symbols[STRING_CONTENT] && scanner->delimiters.len > 0 && - !error_recovery_mode) { - Delimiter delimiter = VEC_BACK(scanner->delimiters); - int32_t end_char = end_character(&delimiter); + if (valid_symbols[STRING_CONTENT] && scanner->delimiters.size > 0 && !error_recovery_mode) { + Delimiter *delimiter = array_back(&scanner->delimiters); + int32_t end_char = end_character(delimiter); bool has_content = advanced_once; while (lexer->lookahead) { - if ((advanced_once || lexer->lookahead == '{' || - lexer->lookahead == '}') && - is_format(&delimiter)) { + if ((advanced_once || lexer->lookahead == '{' || lexer->lookahead == '}') && is_format(delimiter)) { lexer->mark_end(lexer); lexer->result_symbol = STRING_CONTENT; return has_content; } if (lexer->lookahead == '\\') { - if (is_raw(&delimiter)) { + if (is_raw(delimiter)) { // Step over the backslash. advance(lexer); // Step over any escaped quotes. - if (lexer->lookahead == end_character(&delimiter) || - lexer->lookahead == '\\') { + if (lexer->lookahead == end_character(delimiter) || lexer->lookahead == '\\') { advance(lexer); } // Step over newlines @@ -229,11 +146,10 @@ bool tree_sitter_python_external_scanner_scan(void *payload, TSLexer *lexer, } continue; } - if (is_bytes(&delimiter)) { + if (is_bytes(delimiter)) { lexer->mark_end(lexer); advance(lexer); - if (lexer->lookahead == 'N' || lexer->lookahead == 'u' || - lexer->lookahead == 'U') { + if (lexer->lookahead == 'N' || lexer->lookahead == 'u' || lexer->lookahead == 'U') { // In bytes string, \N{...}, \uXXXX and \UXXXXXXXX are // not escape sequences // https://docs.python.org/3/reference/lexical_analysis.html#string-and-bytes-literals @@ -248,7 +164,7 @@ bool tree_sitter_python_external_scanner_scan(void *payload, TSLexer *lexer, return has_content; } } else if (lexer->lookahead == end_char) { - if (is_triple(&delimiter)) { + if (is_triple(delimiter)) { lexer->mark_end(lexer); advance(lexer); if (lexer->lookahead == end_char) { @@ -259,7 +175,7 @@ bool tree_sitter_python_external_scanner_scan(void *payload, TSLexer *lexer, } else { advance(lexer); lexer->mark_end(lexer); - VEC_POP(scanner->delimiters); + array_pop(&scanner->delimiters); lexer->result_symbol = STRING_END; scanner->inside_f_string = false; } @@ -277,15 +193,14 @@ bool tree_sitter_python_external_scanner_scan(void *payload, TSLexer *lexer, lexer->result_symbol = STRING_CONTENT; } else { advance(lexer); - VEC_POP(scanner->delimiters); + array_pop(&scanner->delimiters); lexer->result_symbol = STRING_END; scanner->inside_f_string = false; } lexer->mark_end(lexer); return true; - } else if (lexer->lookahead == '\n' && has_content && - !is_triple(&delimiter)) { + } else if (lexer->lookahead == '\n' && has_content && !is_triple(delimiter)) { return false; } advance(lexer); @@ -312,9 +227,8 @@ bool tree_sitter_python_external_scanner_scan(void *payload, TSLexer *lexer, } else if (lexer->lookahead == '\t') { indent_length += 8; skip(lexer); - } else if (lexer->lookahead == '#' && - (valid_symbols[INDENT] || valid_symbols[DEDENT] || - valid_symbols[NEWLINE] || valid_symbols[EXCEPT])) { + } else if (lexer->lookahead == '#' && (valid_symbols[INDENT] || valid_symbols[DEDENT] || + valid_symbols[NEWLINE] || valid_symbols[EXCEPT])) { // If we haven't found an EOL yet, // then this is a comment after an expression: // foo = bar # comment @@ -351,32 +265,28 @@ bool tree_sitter_python_external_scanner_scan(void *payload, TSLexer *lexer, } if (found_end_of_line) { - if (scanner->indents.len > 0) { - uint16_t current_indent_length = VEC_BACK(scanner->indents); + if (scanner->indents.size > 0) { + uint16_t current_indent_length = *array_back(&scanner->indents); - if (valid_symbols[INDENT] && - indent_length > current_indent_length) { - VEC_PUSH(scanner->indents, indent_length); + if (valid_symbols[INDENT] && indent_length > current_indent_length) { + array_push(&scanner->indents, indent_length); lexer->result_symbol = INDENT; return true; } - bool next_tok_is_string_start = lexer->lookahead == '\"' || - lexer->lookahead == '\'' || - lexer->lookahead == '`'; + bool next_tok_is_string_start = + lexer->lookahead == '\"' || lexer->lookahead == '\'' || lexer->lookahead == '`'; if ((valid_symbols[DEDENT] || - (!valid_symbols[NEWLINE] && - !(valid_symbols[STRING_START] && next_tok_is_string_start) && + (!valid_symbols[NEWLINE] && !(valid_symbols[STRING_START] && next_tok_is_string_start) && !within_brackets)) && - indent_length < current_indent_length && - !scanner->inside_f_string && + indent_length < current_indent_length && !scanner->inside_f_string && // Wait to create a dedent token until we've consumed any // comments // whose indentation matches the current block. first_comment_indent_length < (int32_t)current_indent_length) { - VEC_POP(scanner->indents); + array_pop(&scanner->indents); lexer->result_symbol = DEDENT; return true; } @@ -437,7 +347,7 @@ bool tree_sitter_python_external_scanner_scan(void *payload, TSLexer *lexer, } if (end_character(&delimiter)) { - VEC_PUSH(scanner->delimiters, delimiter); + array_push(&scanner->delimiters, delimiter); lexer->result_symbol = STRING_START; scanner->inside_f_string = is_format(&delimiter); return true; @@ -450,43 +360,38 @@ bool tree_sitter_python_external_scanner_scan(void *payload, TSLexer *lexer, return false; } -unsigned tree_sitter_python_external_scanner_serialize(void *payload, - char *buffer) { +unsigned tree_sitter_python_external_scanner_serialize(void *payload, char *buffer) { Scanner *scanner = (Scanner *)payload; size_t size = 0; buffer[size++] = (char)scanner->inside_f_string; - size_t delimiter_count = scanner->delimiters.len; + size_t delimiter_count = scanner->delimiters.size; if (delimiter_count > UINT8_MAX) { delimiter_count = UINT8_MAX; } buffer[size++] = (char)delimiter_count; if (delimiter_count > 0) { - memcpy(&buffer[size], scanner->delimiters.data, delimiter_count); + memcpy(&buffer[size], scanner->delimiters.contents, delimiter_count); } size += delimiter_count; int iter = 1; - for (; iter < scanner->indents.len && - size < TREE_SITTER_SERIALIZATION_BUFFER_SIZE; - ++iter) { - buffer[size++] = (char)scanner->indents.data[iter]; + for (; iter < scanner->indents.size && size < TREE_SITTER_SERIALIZATION_BUFFER_SIZE; ++iter) { + buffer[size++] = (char)*array_get(&scanner->indents, iter); } return size; } -void tree_sitter_python_external_scanner_deserialize(void *payload, - const char *buffer, - unsigned length) { +void tree_sitter_python_external_scanner_deserialize(void *payload, const char *buffer, unsigned length) { Scanner *scanner = (Scanner *)payload; - VEC_CLEAR(scanner->delimiters); - VEC_CLEAR(scanner->indents); - VEC_PUSH(scanner->indents, 0); + array_delete(&scanner->delimiters); + array_delete(&scanner->indents); + array_push(&scanner->indents, 0); if (length > 0) { size_t size = 0; @@ -495,14 +400,14 @@ void tree_sitter_python_external_scanner_deserialize(void *payload, size_t delimiter_count = (uint8_t)buffer[size++]; if (delimiter_count > 0) { - VEC_GROW(scanner->delimiters, delimiter_count); - scanner->delimiters.len = delimiter_count; - memcpy(scanner->delimiters.data, &buffer[size], delimiter_count); + array_reserve(&scanner->delimiters, delimiter_count); + scanner->delimiters.size = delimiter_count; + memcpy(scanner->delimiters.contents, &buffer[size], delimiter_count); size += delimiter_count; } for (; size < length; size++) { - VEC_PUSH(scanner->indents, (unsigned char)buffer[size]); + array_push(&scanner->indents, (unsigned char)buffer[size]); } } } @@ -514,15 +419,15 @@ void *tree_sitter_python_external_scanner_create() { assert(sizeof(Delimiter) == sizeof(char)); #endif Scanner *scanner = calloc(1, sizeof(Scanner)); - scanner->indents = indent_vec_new(); - scanner->delimiters = delimiter_vec_new(); + array_init(&scanner->indents); + array_init(&scanner->delimiters); tree_sitter_python_external_scanner_deserialize(scanner, NULL, 0); return scanner; } void tree_sitter_python_external_scanner_destroy(void *payload) { Scanner *scanner = (Scanner *)payload; - VEC_FREE(scanner->indents); - VEC_FREE(scanner->delimiters); + array_delete(&scanner->indents); + array_delete(&scanner->delimiters); free(scanner); } diff --git a/src/tree_sitter/alloc.h b/src/tree_sitter/alloc.h new file mode 100644 index 00000000..1f7610a3 --- /dev/null +++ b/src/tree_sitter/alloc.h @@ -0,0 +1,60 @@ +#ifndef TREE_SITTER_ALLOC_H_ +#define TREE_SITTER_ALLOC_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include + +#ifdef _WIN32 +#define TS_PUBLIC __declspec(dllexport) +#else +#define TS_PUBLIC __attribute__((visibility("default"))) +#endif + +TS_PUBLIC extern void *(*ts_current_malloc)(size_t); +TS_PUBLIC extern void *(*ts_current_calloc)(size_t, size_t); +TS_PUBLIC extern void *(*ts_current_realloc)(void *, size_t); +TS_PUBLIC extern void (*ts_current_free)(void *); + +// Allow clients to override allocation functions +#ifdef TS_REUSE_ALLOCATOR + +#ifndef ts_malloc +#define ts_malloc ts_current_malloc +#endif +#ifndef ts_calloc +#define ts_calloc ts_current_calloc +#endif +#ifndef ts_realloc +#define ts_realloc ts_current_realloc +#endif +#ifndef ts_free +#define ts_free ts_current_free +#endif + +#else + +#ifndef ts_malloc +#define ts_malloc malloc +#endif +#ifndef ts_calloc +#define ts_calloc calloc +#endif +#ifndef ts_realloc +#define ts_realloc realloc +#endif +#ifndef ts_free +#define ts_free free +#endif + +#endif + +#ifdef __cplusplus +} +#endif + +#endif // TREE_SITTER_ALLOC_H_ diff --git a/src/tree_sitter/array.h b/src/tree_sitter/array.h new file mode 100644 index 00000000..43358fcc --- /dev/null +++ b/src/tree_sitter/array.h @@ -0,0 +1,272 @@ +#ifndef TREE_SITTER_ARRAY_H_ +#define TREE_SITTER_ARRAY_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include "./alloc.h" + +#include +#include +#include +#include +#include + +#define Array(T) \ + struct { \ + T *contents; \ + uint32_t size; \ + uint32_t capacity; \ + } + +/// Initialize an array. +#define array_init(self) \ + ((self)->size = 0, (self)->capacity = 0, (self)->contents = NULL) + +/// Create an empty array. +#define array_new() \ + { NULL, 0, 0 } + +/// Get a pointer to the element at a given `index` in the array. +#define array_get(self, _index) \ + (assert((uint32_t)(_index) < (self)->size), &(self)->contents[_index]) + +/// Get a pointer to the first element in the array. +#define array_front(self) array_get(self, 0) + +/// Get a pointer to the last element in the array. +#define array_back(self) array_get(self, (self)->size - 1) + +/// Clear the array, setting its size to zero. Note that this does not free any +/// memory allocated for the array's contents. +#define array_clear(self) ((self)->size = 0) + +/// Reserve `new_capacity` elements of space in the array. If `new_capacity` is +/// less than the array's current capacity, this function has no effect. +#define array_reserve(self, new_capacity) \ + _array__reserve((Array *)(self), array_elem_size(self), new_capacity) + +/// Free any memory allocated for this array. Note that this does not free any +/// memory allocated for the array's contents. +#define array_delete(self) _array__delete((Array *)(self)) + +/// Push a new `element` onto the end of the array. +#define array_push(self, element) \ + (_array__grow((Array *)(self), 1, array_elem_size(self)), \ + (self)->contents[(self)->size++] = (element)) + +/// Increase the array's size by `count` elements. +/// New elements are zero-initialized. +#define array_grow_by(self, count) \ + (_array__grow((Array *)(self), count, array_elem_size(self)), \ + memset((self)->contents + (self)->size, 0, (count) * array_elem_size(self)), \ + (self)->size += (count)) + +/// Append all elements from one array to the end of another. +#define array_push_all(self, other) \ + array_extend((self), (other)->size, (other)->contents) + +/// Append `count` elements to the end of the array, reading their values from the +/// `contents` pointer. +#define array_extend(self, count, contents) \ + _array__splice( \ + (Array *)(self), array_elem_size(self), (self)->size, \ + 0, count, contents \ + ) + +/// Remove `old_count` elements from the array starting at the given `index`. At +/// the same index, insert `new_count` new elements, reading their values from the +/// `new_contents` pointer. +#define array_splice(self, _index, old_count, new_count, new_contents) \ + _array__splice( \ + (Array *)(self), array_elem_size(self), _index, \ + old_count, new_count, new_contents \ + ) + +/// Insert one `element` into the array at the given `index`. +#define array_insert(self, _index, element) \ + _array__splice((Array *)(self), array_elem_size(self), _index, 0, 1, &(element)) + +/// Remove one element from the array at the given `index`. +#define array_erase(self, _index) \ + _array__erase((Array *)(self), array_elem_size(self), _index) + +/// Pop the last element off the array, returning the element by value. +#define array_pop(self) ((self)->contents[--(self)->size]) + +/// Assign the contents of one array to another, reallocating if necessary. +#define array_assign(self, other) \ + _array__assign((Array *)(self), (const Array *)(other), array_elem_size(self)) + +#define array_swap(self, other) \ + _array__swap((Array *)(self), (Array *)(other)) + +/// Search a sorted array for a given `needle` value, using the given `compare` +/// callback to determine the order. +/// +/// If an existing element is found to be equal to `needle`, then the `index` +/// out-parameter is set to the existing value's index, and the `exists` +/// out-parameter is set to true. Otherwise, `index` is set to an index where +/// `needle` should be inserted in order to preserve the sorting, and `exists` +/// is set to false. +#define array_search_sorted_with(self, compare, needle, _index, _exists) \ + _array__search_sorted(self, 0, compare, , needle, _index, _exists) + +/// Search a sorted array for a given `needle` value, using integer comparisons +/// of a given struct field (specified with a leading dot) to determine the order. +/// +/// See also `array_search_sorted_with`. +#define array_search_sorted_by(self, field, needle, _index, _exists) \ + _array__search_sorted(self, 0, compare_int, field, needle, _index, _exists) + +/// Insert a given `value` into a sorted array, using the given `compare` +/// callback to determine the order. +#define array_insert_sorted_with(self, compare, value) \ + do { \ + unsigned _index, _exists; \ + array_search_sorted_with(self, compare, &(value), &_index, &_exists); \ + if (!_exists) array_insert(self, _index, value); \ + } while (0) + +/// Insert a given `value` into a sorted array, using integer comparisons of +/// a given struct field (specified with a leading dot) to determine the order. +/// +/// See also `array_search_sorted_by`. +#define array_insert_sorted_by(self, field, value) \ + do { \ + unsigned _index, _exists; \ + array_search_sorted_by(self, field, (value) field, &_index, &_exists); \ + if (!_exists) array_insert(self, _index, value); \ + } while (0) + +// Private + +typedef Array(void) Array; + +#define array_elem_size(self) sizeof(*(self)->contents) + +/// This is not what you're looking for, see `array_delete`. +static inline void _array__delete(Array *self) { + if (self->contents) { + ts_free(self->contents); + self->contents = NULL; + self->size = 0; + self->capacity = 0; + } +} + +/// This is not what you're looking for, see `array_erase`. +static inline void _array__erase(Array *self, size_t element_size, + uint32_t index) { + assert(index < self->size); + char *contents = (char *)self->contents; + memmove(contents + index * element_size, contents + (index + 1) * element_size, + (self->size - index - 1) * element_size); + self->size--; +} + +/// This is not what you're looking for, see `array_reserve`. +static inline void _array__reserve(Array *self, size_t element_size, uint32_t new_capacity) { + if (new_capacity > self->capacity) { + if (self->contents) { + self->contents = ts_realloc(self->contents, new_capacity * element_size); + } else { + self->contents = ts_malloc(new_capacity * element_size); + } + self->capacity = new_capacity; + } +} + +/// This is not what you're looking for, see `array_assign`. +static inline void _array__assign(Array *self, const Array *other, size_t element_size) { + _array__reserve(self, element_size, other->size); + self->size = other->size; + memcpy(self->contents, other->contents, self->size * element_size); +} + +/// This is not what you're looking for, see `array_swap`. +static inline void _array__swap(Array *self, Array *other) { + Array swap = *other; + *other = *self; + *self = swap; +} + +/// This is not what you're looking for, see `array_push` or `array_grow_by`. +static inline void _array__grow(Array *self, uint32_t count, size_t element_size) { + uint32_t new_size = self->size + count; + if (new_size > self->capacity) { + uint32_t new_capacity = self->capacity * 2; + if (new_capacity < 8) new_capacity = 8; + if (new_capacity < new_size) new_capacity = new_size; + _array__reserve(self, element_size, new_capacity); + } +} + +/// This is not what you're looking for, see `array_splice`. +static inline void _array__splice(Array *self, size_t element_size, + uint32_t index, uint32_t old_count, + uint32_t new_count, const void *elements) { + uint32_t new_size = self->size + new_count - old_count; + uint32_t old_end = index + old_count; + uint32_t new_end = index + new_count; + assert(old_end <= self->size); + + _array__reserve(self, element_size, new_size); + + char *contents = (char *)self->contents; + if (self->size > old_end) { + memmove( + contents + new_end * element_size, + contents + old_end * element_size, + (self->size - old_end) * element_size + ); + } + if (new_count > 0) { + if (elements) { + memcpy( + (contents + index * element_size), + elements, + new_count * element_size + ); + } else { + memset( + (contents + index * element_size), + 0, + new_count * element_size + ); + } + } + self->size += new_count - old_count; +} + +/// A binary search routine, based on Rust's `std::slice::binary_search_by`. +/// This is not what you're looking for, see `array_search_sorted_with` or `array_search_sorted_by`. +#define _array__search_sorted(self, start, compare, suffix, needle, _index, _exists) \ + do { \ + *(_index) = start; \ + *(_exists) = false; \ + uint32_t size = (self)->size - *(_index); \ + if (size == 0) break; \ + int comparison; \ + while (size > 1) { \ + uint32_t half_size = size / 2; \ + uint32_t mid_index = *(_index) + half_size; \ + comparison = compare(&((self)->contents[mid_index] suffix), (needle)); \ + if (comparison <= 0) *(_index) = mid_index; \ + size -= half_size; \ + } \ + comparison = compare(&((self)->contents[*(_index)] suffix), (needle)); \ + if (comparison == 0) *(_exists) = true; \ + else if (comparison < 0) *(_index) += 1; \ + } while (0) + +/// Helper macro for the `_sorted_by` routines below. This takes the left (existing) +/// parameter by reference in order to work with the generic sorting function above. +#define compare_int(a, b) ((int)*(a) - (int)(b)) + +#ifdef __cplusplus +} +#endif + +#endif // TREE_SITTER_ARRAY_H_ diff --git a/src/tree_sitter/parser.h b/src/tree_sitter/parser.h index d2103259..17b4fde9 100644 --- a/src/tree_sitter/parser.h +++ b/src/tree_sitter/parser.h @@ -129,9 +129,16 @@ struct TSLanguage { * Lexer Macros */ +#ifdef _MSC_VER +#define UNUSED __pragma(warning(suppress : 4101)) +#else +#define UNUSED __attribute__((unused)) +#endif + #define START_LEXER() \ bool result = false; \ bool skip = false; \ + UNUSED \ bool eof = false; \ int32_t lookahead; \ goto start; \ @@ -139,8 +146,7 @@ struct TSLanguage { lexer->advance(lexer, skip); \ start: \ skip = false; \ - lookahead = lexer->lookahead; \ - eof = lexer->eof(lexer); + lookahead = lexer->lookahead; #define ADVANCE(state_value) \ { \ From c4eb35263b0ea5aa11b0034417cb0fea40ce49f7 Mon Sep 17 00:00:00 2001 From: Amaan Qureshi Date: Mon, 26 Feb 2024 12:20:51 -0500 Subject: [PATCH 2/5] fix: regenerate with operator precedence fix --- src/parser.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/parser.c b/src/parser.c index 43c8c604..28ed64b9 100644 --- a/src/parser.c +++ b/src/parser.c @@ -8357,8 +8357,8 @@ static bool ts_lex(TSLexer *lexer, TSStateId state) { if (lookahead != 0) ADVANCE(25); END_STATE(); case 26: - if (!eof && lookahead == 0 || - lookahead == '\n') ADVANCE(168); + if (!eof && (lookahead == 0 || + lookahead == '\n')) ADVANCE(168); if (lookahead == '\r') ADVANCE(1); END_STATE(); case 27: @@ -9142,7 +9142,7 @@ static bool ts_lex(TSLexer *lexer, TSStateId state) { END_STATE(); case 133: ACCEPT_TOKEN(sym__not_escape_sequence); - if (!eof && lookahead == 0) ADVANCE(168); + if (!eof && (lookahead == 0)) ADVANCE(168); if (lookahead == '\n') ADVANCE(132); if (lookahead == '\r') ADVANCE(2); if (lookahead == 'N') ADVANCE(24); @@ -9162,7 +9162,7 @@ static bool ts_lex(TSLexer *lexer, TSStateId state) { END_STATE(); case 134: ACCEPT_TOKEN(aux_sym_format_specifier_token1); - if (!eof && lookahead == 0) ADVANCE(136); + if (!eof && (lookahead == 0)) ADVANCE(136); if (lookahead == '\r') ADVANCE(136); if (lookahead != 0 && lookahead != '\n' && From 94b477e28b578d19e9f8455a3ebc8b645e2435d5 Mon Sep 17 00:00:00 2001 From: Amaan Qureshi Date: Mon, 26 Feb 2024 12:21:07 -0500 Subject: [PATCH 3/5] fix(queries): use last-precedence for highlights --- queries/highlights.scm | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/queries/highlights.scm b/queries/highlights.scm index 8ccf82b8..4b821e4b 100644 --- a/queries/highlights.scm +++ b/queries/highlights.scm @@ -1,19 +1,13 @@ ; Identifier naming conventions +(identifier) @variable + ((identifier) @constructor (#match? @constructor "^[A-Z]")) ((identifier) @constant (#match? @constant "^[A-Z][A-Z_]*$")) -; Builtin functions - -((call - function: (identifier) @function.builtin) - (#match? - @function.builtin - "^(abs|all|any|ascii|bin|bool|breakpoint|bytearray|bytes|callable|chr|classmethod|compile|complex|delattr|dict|dir|divmod|enumerate|eval|exec|filter|float|format|frozenset|getattr|globals|hasattr|hash|help|hex|id|input|int|isinstance|issubclass|iter|len|list|locals|map|max|memoryview|min|next|object|oct|open|ord|pow|print|property|range|repr|reversed|round|set|setattr|slice|sorted|staticmethod|str|sum|super|tuple|type|vars|zip|__import__)$")) - ; Function calls (decorator) @function @@ -23,12 +17,19 @@ (call function: (identifier) @function) +; Builtin functions + +((call + function: (identifier) @function.builtin) + (#match? + @function.builtin + "^(abs|all|any|ascii|bin|bool|breakpoint|bytearray|bytes|callable|chr|classmethod|compile|complex|delattr|dict|dir|divmod|enumerate|eval|exec|filter|float|format|frozenset|getattr|globals|hasattr|hash|help|hex|id|input|int|isinstance|issubclass|iter|len|list|locals|map|max|memoryview|min|next|object|oct|open|ord|pow|print|property|range|repr|reversed|round|set|setattr|slice|sorted|staticmethod|str|sum|super|tuple|type|vars|zip|__import__)$")) + ; Function definitions (function_definition name: (identifier) @function) -(identifier) @variable (attribute attribute: (identifier) @property) (type (identifier) @type) From b07533e0b2cca7899f7c1d294fb70a6d19a0a5ed Mon Sep 17 00:00:00 2001 From: Amaan Qureshi Date: Mon, 26 Feb 2024 12:24:39 -0500 Subject: [PATCH 4/5] feat: improve bindings --- .editorconfig | 39 ++ .gitattributes | 14 +- .npmignore | 21 +- Cargo.toml | 8 +- Makefile | 94 +++++ Package.swift | 27 +- binding.gyp | 9 +- bindings/c/tree-sitter-python.h | 16 + bindings/c/tree-sitter-python.pc.in | 11 + bindings/go/binding.go | 13 + bindings/go/binding_test.go | 15 + bindings/go/go.mod | 5 + bindings/node/binding.cc | 27 +- bindings/node/index.js | 4 +- .../build/lib/tree_sitter_python/__init__.py | 5 + .../python/tree_sitter_python/__init__.py | 3 + .../python/tree_sitter_python/__init__.pyi | 1 + bindings/python/tree_sitter_python/binding.c | 27 ++ bindings/python/tree_sitter_python/py.typed | 0 bindings/rust/build.rs | 7 +- bindings/rust/lib.rs | 35 +- bindings/swift/TreeSitterPython/python.h | 2 +- pyproject.toml | 26 ++ setup.py | 57 +++ src/scanner.c | 2 +- src/tree_sitter/array.h | 18 + types/dsl.d.ts | 379 ++++++++++++++++++ 27 files changed, 794 insertions(+), 71 deletions(-) create mode 100644 .editorconfig create mode 100644 Makefile create mode 100644 bindings/c/tree-sitter-python.h create mode 100644 bindings/c/tree-sitter-python.pc.in create mode 100644 bindings/go/binding.go create mode 100644 bindings/go/binding_test.go create mode 100644 bindings/go/go.mod create mode 100644 bindings/python/build/lib/tree_sitter_python/__init__.py create mode 100644 bindings/python/tree_sitter_python/__init__.py create mode 100644 bindings/python/tree_sitter_python/__init__.pyi create mode 100644 bindings/python/tree_sitter_python/binding.c create mode 100644 bindings/python/tree_sitter_python/py.typed create mode 100644 pyproject.toml create mode 100644 setup.py create mode 100644 types/dsl.d.ts diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 00000000..d3a8b5b6 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,39 @@ +root = true + +[*] +charset = utf-8 +end_of_line = lf +insert_final_newline = true +trim_trailing_whitespace = true + +[*.{json,toml,yml,gyp}] +indent_style = space +indent_size = 2 + +[*.js] +indent_style = space +indent_size = 2 + +[*.rs] +indent_style = space +indent_size = 4 + +[*.{c,cc,h}] +indent_style = space +indent_size = 4 + +[*.{py,pyi}] +indent_style = space +indent_size = 4 + +[*.swift] +indent_style = space +indent_size = 4 + +[*.go] +indent_style = tab +indent_size = 8 + +[Makefile] +indent_style = tab +indent_size = 8 diff --git a/.gitattributes b/.gitattributes index 1491f7e1..9f71c8f1 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,10 +1,10 @@ -/src/** linguist-vendored -/examples/* linguist-vendored +* text eol=lf -src/grammar.json linguist-generated -src/node-types.json linguist-generated +src/*.json linguist-generated src/parser.c linguist-generated +src/tree_sitter/* linguist-generated -src/grammar.json -diff -src/node-types.json -diff -src/parser.c -diff +bindings/** linguist-generated +binding.gyp linguist-generated +setup.py linguist-generated +Makefile linguist-generated diff --git a/.npmignore b/.npmignore index 0f438b55..ac723165 100644 --- a/.npmignore +++ b/.npmignore @@ -1,6 +1,17 @@ -/test -/examples -/build -/script -/target +bindings/c +bindings/go +bindings/python bindings/rust +bindings/swift +Cargo.toml +Makefile +examples +pyproject.toml +setup.py +test +.editorconfig +.github +.gitignore +.gitattributes +.gitmodules +.npmignore diff --git a/Cargo.toml b/Cargo.toml index b43ffb0a..374d5bc9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,8 +3,8 @@ name = "tree-sitter-python" description = "Python grammar for tree-sitter" version = "0.20.4" authors = [ - "Max Brunsfeld ", - "Douglas Creager ", + "Max Brunsfeld ", + "Douglas Creager ", ] license = "MIT" readme = "bindings/rust/README.md" @@ -21,7 +21,7 @@ include = ["bindings/rust/*", "grammar.js", "queries/*", "src/*"] path = "bindings/rust/lib.rs" [dependencies] -tree-sitter = "~0.20.10" +tree-sitter = "0.21.0" [build-dependencies] -cc = "~1.0" +cc = "1.0.88" diff --git a/Makefile b/Makefile new file mode 100644 index 00000000..d54bd5a7 --- /dev/null +++ b/Makefile @@ -0,0 +1,94 @@ +VERSION := 0.0.1 + +LANGUAGE_NAME := tree-sitter-python + +# repository +SRC_DIR := src + +PARSER_REPO_URL := $(shell git -C $(SRC_DIR) remote get-url origin 2>/dev/null) + +ifeq ($(PARSER_URL),) + PARSER_URL := $(subst .git,,$(PARSER_REPO_URL)) +ifeq ($(shell echo $(PARSER_URL) | grep '^[a-z][-+.0-9a-z]*://'),) + PARSER_URL := $(subst :,/,$(PARSER_URL)) + PARSER_URL := $(subst git@,https://,$(PARSER_URL)) +endif +endif + +# ABI versioning +SONAME_MAJOR := $(word 1,$(subst ., ,$(VERSION))) +SONAME_MINOR := $(word 2,$(subst ., ,$(VERSION))) + +# install directory layout +PREFIX ?= /usr/local +INCLUDEDIR ?= $(PREFIX)/include +LIBDIR ?= $(PREFIX)/lib +PCLIBDIR ?= $(LIBDIR)/pkgconfig + +# object files +OBJS := $(patsubst %.c,%.o,$(wildcard $(SRC_DIR)/*.c)) + +# flags +ARFLAGS := rcs +override CFLAGS += -I$(SRC_DIR) -std=c11 + +# OS-specific bits +ifeq ($(shell uname),Darwin) + SOEXT = dylib + SOEXTVER_MAJOR = $(SONAME_MAJOR).dylib + SOEXTVER = $(SONAME_MAJOR).$(SONAME_MINOR).dylib + LINKSHARED := $(LINKSHARED)-dynamiclib -Wl, + ifneq ($(ADDITIONAL_LIBS),) + LINKSHARED := $(LINKSHARED)$(ADDITIONAL_LIBS), + endif + LINKSHARED := $(LINKSHARED)-install_name,$(LIBDIR)/lib$(LANGUAGE_NAME).$(SONAME_MAJOR).dylib,-rpath,@executable_path/../Frameworks +else ifneq ($(filter $(shell uname),Linux FreeBSD NetBSD DragonFly),) + SOEXT = so + SOEXTVER_MAJOR = so.$(SONAME_MAJOR) + SOEXTVER = so.$(SONAME_MAJOR).$(SONAME_MINOR) + LINKSHARED := $(LINKSHARED)-shared -Wl, + ifneq ($(ADDITIONAL_LIBS),) + LINKSHARED := $(LINKSHARED)$(ADDITIONAL_LIBS) + endif + LINKSHARED := $(LINKSHARED)-soname,lib$(LANGUAGE_NAME).so.$(SONAME_MAJOR) +else ifeq ($(OS),Windows_NT) + $(error "Windows is not supported") +endif +ifneq ($(filter $(shell uname),FreeBSD NetBSD DragonFly),) + PCLIBDIR := $(PREFIX)/libdata/pkgconfig +endif + +all: lib$(LANGUAGE_NAME).a lib$(LANGUAGE_NAME).$(SOEXT) $(LANGUAGE_NAME).pc + +$(SRC_DIR)/%.o: $(SRC_DIR)/%.c + $(CC) -c $^ -o $@ + +lib$(LANGUAGE_NAME).a: $(OBJS) + $(AR) $(ARFLAGS) $@ $^ + +lib$(LANGUAGE_NAME).$(SOEXT): $(OBJS) + $(CC) -fPIC $(LDFLAGS) $(LINKSHARED) $^ $(LDLIBS) -o $@ + +$(LANGUAGE_NAME).pc: + sed > $@ bindings/c/$(LANGUAGE_NAME).pc.in \ + -e 's|@URL@|$(PARSER_URL)|' \ + -e 's|@VERSION@|$(VERSION)|' \ + -e 's|@LIBDIR@|$(LIBDIR)|;' \ + -e 's|@INCLUDEDIR@|$(INCLUDEDIR)|;' \ + -e 's|=$(PREFIX)|=$${prefix}|' \ + -e 's|@PREFIX@|$(PREFIX)|' \ + -e 's|@REQUIRES@|$(REQUIRES)|' \ + -e 's|@ADDITIONAL_LIBS@|$(ADDITIONAL_LIBS)|' + +install: all + install -Dm644 bindings/c/$(LANGUAGE_NAME).h '$(DESTDIR)$(INCLUDEDIR)'/tree_sitter/$(LANGUAGE_NAME).h + install -Dm644 $(LANGUAGE_NAME).pc '$(DESTDIR)$(PCLIBDIR)'/$(LANGUAGE_NAME).pc + install -Dm755 lib$(LANGUAGE_NAME).a '$(DESTDIR)$(LIBDIR)'/lib$(LANGUAGE_NAME).a + install -Dm755 lib$(LANGUAGE_NAME).$(SOEXT) '$(DESTDIR)$(LIBDIR)'/lib$(LANGUAGE_NAME).$(SOEXTVER) + ln -sf lib$(LANGUAGE_NAME).$(SOEXTVER) '$(DESTDIR)$(LIBDIR)'/lib$(LANGUAGE_NAME).$(SOEXTVER_MAJOR) + ln -sf lib$(LANGUAGE_NAME).$(SOEXTVER_MAJOR) '$(DESTDIR)$(LIBDIR)'/lib$(LANGUAGE_NAME).$(SOEXT) + +clean: + $(RM) $(OBJS) $(LANGUAGE_NAME).pc lib$(LANGUAGE_NAME).a lib$(LANGUAGE_NAME).$(SOEXT) + +.PHONY: all install clean diff --git a/Package.swift b/Package.swift index a5e4524d..495eca79 100644 --- a/Package.swift +++ b/Package.swift @@ -3,6 +3,7 @@ import PackageDescription let package = Package( name: "TreeSitterPython", + platforms: [.macOS(.v10_13), .iOS(.v11)], products: [ .library(name: "TreeSitterPython", targets: ["TreeSitterPython"]), ], @@ -11,18 +12,32 @@ let package = Package( .target(name: "TreeSitterPython", path: ".", exclude: [ - "binding.gyp", - "bindings", "Cargo.toml", - "corpus", + "Makefile", + "binding.gyp", + "bindings/c", + "bindings/go", + "bindings/node", + "bindings/python", + "bindings/rust", + "examples", "grammar.js", - "LICENSE", "package.json", - "README.md", + "package-lock.json", + "pyproject.toml", + "setup.py", + "test", + "types", + ".editorconfig", + ".github", + ".gitignore", + ".gitattributes", + ".gitmodules", + ".npmignore", ], sources: [ "src/parser.c", - "src/scanner.c", + // NOTE: if your language has an external scanner, add it here. ], resources: [ .copy("queries") diff --git a/binding.gyp b/binding.gyp index d76793ab..a2d04f2d 100644 --- a/binding.gyp +++ b/binding.gyp @@ -4,15 +4,18 @@ "target_name": "tree_sitter_python_binding", "include_dirs": [ " #include "nan.h" +#include using namespace v8; -extern "C" TSLanguage * tree_sitter_python(); +typedef struct TSLanguage TSLanguage; + +extern "C" const TSLanguage *tree_sitter_python(void); namespace { NAN_METHOD(New) {} void Init(Local exports, Local module) { - Local tpl = Nan::New(New); - tpl->SetClassName(Nan::New("Language").ToLocalChecked()); - tpl->InstanceTemplate()->SetInternalFieldCount(1); + Local tpl = Nan::New(New); + tpl->SetClassName(Nan::New("Language").ToLocalChecked()); + tpl->InstanceTemplate()->SetInternalFieldCount(1); - Local constructor = Nan::GetFunction(tpl).ToLocalChecked(); - Local instance = constructor->NewInstance(Nan::GetCurrentContext()).ToLocalChecked(); - Nan::SetInternalFieldPointer(instance, 0, tree_sitter_python()); + Local constructor = Nan::GetFunction(tpl).ToLocalChecked(); + Local instance = constructor->NewInstance(Nan::GetCurrentContext()).ToLocalChecked(); + Nan::SetInternalFieldPointer(instance, 0, (void *)tree_sitter_python()); - Nan::Set(instance, Nan::New("name").ToLocalChecked(), Nan::New("python").ToLocalChecked()); - Nan::Set(module, Nan::New("exports").ToLocalChecked(), instance); + Nan::Set(instance, Nan::New("name").ToLocalChecked(), Nan::New("python").ToLocalChecked()); + Nan::Set(module, Nan::New("exports").ToLocalChecked(), instance); } -NODE_MODULE(tree_sitter_python_binding, Init) +NODE_MODULE_CONTEXT_AWARE(tree_sitter_python_binding, Init) -} // namespace +} // namespace diff --git a/bindings/node/index.js b/bindings/node/index.js index e0f77d80..2fd841d9 100644 --- a/bindings/node/index.js +++ b/bindings/node/index.js @@ -1,13 +1,13 @@ try { module.exports = require("../../build/Release/tree_sitter_python_binding"); } catch (error1) { - if (error1.code !== 'MODULE_NOT_FOUND') { + if (error1.code !== "MODULE_NOT_FOUND") { throw error1; } try { module.exports = require("../../build/Debug/tree_sitter_python_binding"); } catch (error2) { - if (error2.code !== 'MODULE_NOT_FOUND') { + if (error2.code !== "MODULE_NOT_FOUND") { throw error2; } throw error1 diff --git a/bindings/python/build/lib/tree_sitter_python/__init__.py b/bindings/python/build/lib/tree_sitter_python/__init__.py new file mode 100644 index 00000000..7d4a8665 --- /dev/null +++ b/bindings/python/build/lib/tree_sitter_python/__init__.py @@ -0,0 +1,5 @@ +from ._tree_sitter_python import lib as _lib, ffi as _ffi + +def language(): + """Get the tree-sitter language for this grammar.""" + return int(_ffi.cast("uintptr_t", _lib.tree_sitter_python())) diff --git a/bindings/python/tree_sitter_python/__init__.py b/bindings/python/tree_sitter_python/__init__.py new file mode 100644 index 00000000..3c892c31 --- /dev/null +++ b/bindings/python/tree_sitter_python/__init__.py @@ -0,0 +1,3 @@ +"Python grammar for tree-sitter" + +from ._binding import language diff --git a/bindings/python/tree_sitter_python/__init__.pyi b/bindings/python/tree_sitter_python/__init__.pyi new file mode 100644 index 00000000..5416666f --- /dev/null +++ b/bindings/python/tree_sitter_python/__init__.pyi @@ -0,0 +1 @@ +def language() -> int: ... diff --git a/bindings/python/tree_sitter_python/binding.c b/bindings/python/tree_sitter_python/binding.c new file mode 100644 index 00000000..b3fd7bbc --- /dev/null +++ b/bindings/python/tree_sitter_python/binding.c @@ -0,0 +1,27 @@ +#include + +typedef struct TSLanguage TSLanguage; + +extern const TSLanguage *tree_sitter_python(void); + +static PyObject* _binding_language(PyObject *self, PyObject *args) { + return PyLong_FromVoidPtr((void *)tree_sitter_python()); +} + +static PyMethodDef methods[] = { + {"language", _binding_language, METH_NOARGS, + "Get the tree-sitter language for this grammar."}, + {NULL, NULL, 0, NULL} +}; + +static struct PyModuleDef module = { + .m_base = PyModuleDef_HEAD_INIT, + .m_name = "_binding", + .m_doc = NULL, + .m_size = -1, + .m_methods = methods +}; + +PyMODINIT_FUNC PyInit__binding(void) { + return PyModule_Create(&module); +} diff --git a/bindings/python/tree_sitter_python/py.typed b/bindings/python/tree_sitter_python/py.typed new file mode 100644 index 00000000..e69de29b diff --git a/bindings/rust/build.rs b/bindings/rust/build.rs index 8851fed1..a92e388f 100644 --- a/bindings/rust/build.rs +++ b/bindings/rust/build.rs @@ -2,11 +2,8 @@ fn main() { let src_dir = std::path::Path::new("src"); let mut c_config = cc::Build::new(); + c_config.flag_if_supported("-Wno-unused-value"); c_config.include(src_dir); - c_config - .flag_if_supported("-Wno-unused-parameter") - .flag_if_supported("-Wno-unused-but-set-variable") - .flag_if_supported("-Wno-trigraphs"); let parser_path = src_dir.join("parser.c"); c_config.file(&parser_path); @@ -14,6 +11,6 @@ fn main() { c_config.file(&scanner_path); println!("cargo:rerun-if-changed={}", scanner_path.to_str().unwrap()); - c_config.compile("parser"); + c_config.compile("tree-sitter-python"); println!("cargo:rerun-if-changed={}", parser_path.to_str().unwrap()); } diff --git a/bindings/rust/lib.rs b/bindings/rust/lib.rs index 71bc80bd..123b697d 100644 --- a/bindings/rust/lib.rs +++ b/bindings/rust/lib.rs @@ -1,27 +1,17 @@ -// -*- coding: utf-8 -*- -// ------------------------------------------------------------------------------------------------ -// Copyright © 2020, tree-sitter-python authors. -// See the LICENSE file in this repo for license details. -// ------------------------------------------------------------------------------------------------ - -//! This crate provides a Python grammar for the [tree-sitter][] parsing library. +//! This crate provides Python language support for the [tree-sitter][] parsing library. //! -//! Typically, you will use the [language][language func] function to add this grammar to a +//! Typically, you will use the [language][language func] function to add this language to a //! tree-sitter [Parser][], and then use the parser to parse some code: //! //! ``` -//! use tree_sitter::Parser; -//! //! let code = r#" //! def double(x): //! return x * 2 //! "#; -//! let mut parser = Parser::new(); -//! parser.set_language(tree_sitter_python::language()).expect("Error loading Python grammar"); -//! let parsed = parser.parse(code, None); -//! # let parsed = parsed.unwrap(); -//! # let root = parsed.root_node(); -//! # assert!(!root.has_error()); +//! let mut parser = tree_sitter::Parser::new(); +//! parser.set_language(&tree_sitter_python::language()).expect("Error loading Python grammar"); +//! let tree = parser.parse(code, None).unwrap(); +//! assert!(!tree.root_node().has_error()); //! ``` //! //! [Language]: https://docs.rs/tree-sitter/*/tree_sitter/struct.Language.html @@ -35,7 +25,7 @@ extern "C" { fn tree_sitter_python() -> Language; } -/// Returns the tree-sitter [Language][] for this grammar. +/// Get the tree-sitter [Language][] for this grammar. /// /// [Language]: https://docs.rs/tree-sitter/*/tree_sitter/struct.Language.html pub fn language() -> Language { @@ -45,24 +35,21 @@ pub fn language() -> Language { /// The source of the Python tree-sitter grammar description. pub const GRAMMAR: &str = include_str!("../../grammar.js"); -/// The syntax highlighting query for this language. -pub const HIGHLIGHT_QUERY: &str = include_str!("../../queries/highlights.scm"); - /// The content of the [`node-types.json`][] file for this grammar. /// /// [`node-types.json`]: https://tree-sitter.github.io/tree-sitter/using-parsers#static-node-types pub const NODE_TYPES: &str = include_str!("../../src/node-types.json"); -/// The symbol tagging query for this language. -pub const TAGGING_QUERY: &str = include_str!("../../queries/tags.scm"); +pub const HIGHLIGHTS_QUERY: &str = include_str!("../../queries/highlights.scm"); +pub const TAGS_QUERY: &str = include_str!("../../queries/tags.scm"); #[cfg(test)] mod tests { #[test] - fn can_load_grammar() { + fn test_can_load_grammar() { let mut parser = tree_sitter::Parser::new(); parser - .set_language(super::language()) + .set_language(&super::language()) .expect("Error loading Python grammar"); } } diff --git a/bindings/swift/TreeSitterPython/python.h b/bindings/swift/TreeSitterPython/python.h index 6f2548a2..747317a1 100644 --- a/bindings/swift/TreeSitterPython/python.h +++ b/bindings/swift/TreeSitterPython/python.h @@ -7,7 +7,7 @@ typedef struct TSLanguage TSLanguage; extern "C" { #endif -extern TSLanguage *tree_sitter_python(); +extern const TSLanguage *tree_sitter_python(void); #ifdef __cplusplus } diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..a8eca355 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,26 @@ +[build-system] +requires = ["setuptools>=42", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "tree-sitter-python" +description = "Python grammar for tree-sitter" +version = "0.0.1" +keywords = ["parsing", "incremental", "python"] +classifiers = [ + "Development Status :: 4 - Beta", + "Intended Audience :: Developers", + "License :: OSI Approved :: MIT License", + "Topic :: Software Development :: Compilers", + "Topic :: Text Processing :: Linguistic", +] +requires-python = ">=3.8" +license.file = "LICENSE" +readme = "README.md" + +[project.optional-dependencies] +core = ["tree-sitter~=0.21"] + +[tool.cibuildwheel] +build = "cp38-*" +build-frontend = "build" diff --git a/setup.py b/setup.py new file mode 100644 index 00000000..65b70b47 --- /dev/null +++ b/setup.py @@ -0,0 +1,57 @@ +from os.path import isdir, join +from platform import system + +from setuptools import Extension, find_packages, setup +from setuptools.command.build import build +from wheel.bdist_wheel import bdist_wheel + + +class Build(build): + def run(self): + if isdir("queries"): + dest = join(self.build_lib, "tree_sitter_python", "queries") + self.copy_tree("queries", dest) + super().run() + + +class BdistWheel(bdist_wheel): + def get_tag(self): + python, abi, platform = super().get_tag() + if python.startswith("cp"): + python, abi = "cp38", "abi3" + return python, abi, platform + + +setup( + packages=find_packages("bindings/python"), + package_dir={"": "bindings/python"}, + package_data={ + "tree_sitter_python": ["*.pyi", "py.typed"], + "tree_sitter_python.queries": ["*.scm"], + }, + ext_package="tree_sitter_python", + ext_modules=[ + Extension( + name="_binding", + sources=[ + "bindings/python/tree_sitter_python/binding.c", + "src/parser.c", + "src/scanner.c", + ], + extra_compile_args=( + ["-std=c11"] if system() != 'Windows' else [] + ), + define_macros=[ + ("Py_LIMITED_API", "0x03080000"), + ("PY_SSIZE_T_CLEAN", None) + ], + include_dirs=["src"], + py_limited_api=True, + ) + ], + cmdclass={ + "build": Build, + "bdist_wheel": BdistWheel + }, + zip_safe=False, +) diff --git a/src/scanner.c b/src/scanner.c index 8c15505f..4022fccd 100644 --- a/src/scanner.c +++ b/src/scanner.c @@ -378,7 +378,7 @@ unsigned tree_sitter_python_external_scanner_serialize(void *payload, char *buff } size += delimiter_count; - int iter = 1; + uint32_t iter = 1; for (; iter < scanner->indents.size && size < TREE_SITTER_SERIALIZATION_BUFFER_SIZE; ++iter) { buffer[size++] = (char)*array_get(&scanner->indents, iter); } diff --git a/src/tree_sitter/array.h b/src/tree_sitter/array.h index 43358fcc..45969242 100644 --- a/src/tree_sitter/array.h +++ b/src/tree_sitter/array.h @@ -13,6 +13,16 @@ extern "C" { #include #include +#ifdef _MSC_VER +#pragma warning(disable : 4101) +#elif defined(__clang__) +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wunused-variable" +#elif defined(__GNUC__) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-variable" +#endif + #define Array(T) \ struct { \ T *contents; \ @@ -265,6 +275,14 @@ static inline void _array__splice(Array *self, size_t element_size, /// parameter by reference in order to work with the generic sorting function above. #define compare_int(a, b) ((int)*(a) - (int)(b)) +#ifdef _MSC_VER +#pragma warning(default : 4101) +#elif defined(__clang__) +#pragma clang diagnostic pop +#elif defined(__GNUC__) +#pragma GCC diagnostic pop +#endif + #ifdef __cplusplus } #endif diff --git a/types/dsl.d.ts b/types/dsl.d.ts new file mode 100644 index 00000000..63f9ed49 --- /dev/null +++ b/types/dsl.d.ts @@ -0,0 +1,379 @@ +type AliasRule = { type: 'ALIAS'; named: boolean; content: Rule; value: string }; +type BlankRule = { type: 'BLANK' }; +type ChoiceRule = { type: 'CHOICE'; members: Rule[] }; +type FieldRule = { type: 'FIELD'; name: string; content: Rule }; +type ImmediateTokenRule = { type: 'IMMEDIATE_TOKEN'; content: Rule }; +type PatternRule = { type: 'PATTERN'; value: string }; +type PrecDynamicRule = { type: 'PREC_DYNAMIC'; content: Rule; value: number }; +type PrecLeftRule = { type: 'PREC_LEFT'; content: Rule; value: number }; +type PrecRightRule = { type: 'PREC_RIGHT'; content: Rule; value: number }; +type PrecRule = { type: 'PREC'; content: Rule; value: number }; +type Repeat1Rule = { type: 'REPEAT1'; content: Rule }; +type RepeatRule = { type: 'REPEAT'; content: Rule }; +type SeqRule = { type: 'SEQ'; members: Rule[] }; +type StringRule = { type: 'STRING'; value: string }; +type SymbolRule = { type: 'SYMBOL'; name: Name }; +type TokenRule = { type: 'TOKEN'; content: Rule }; + +type Rule = + | AliasRule + | BlankRule + | ChoiceRule + | FieldRule + | ImmediateTokenRule + | PatternRule + | PrecDynamicRule + | PrecLeftRule + | PrecRightRule + | PrecRule + | Repeat1Rule + | RepeatRule + | SeqRule + | StringRule + | SymbolRule + | TokenRule; + +type RuleOrLiteral = Rule | RegExp | string; + +type GrammarSymbols = { + [name in RuleName]: SymbolRule; +} & + Record>; + +type RuleBuilder = ( + $: GrammarSymbols, + previous: Rule, +) => RuleOrLiteral; + +type RuleBuilders< + RuleName extends string, + BaseGrammarRuleName extends string +> = { + [name in RuleName]: RuleBuilder; + }; + +interface Grammar< + RuleName extends string, + BaseGrammarRuleName extends string = never, + Rules extends RuleBuilders = RuleBuilders< + RuleName, + BaseGrammarRuleName + > +> { + /** + * Name of the grammar language. + */ + name: string; + + /** Mapping of grammar rule names to rule builder functions. */ + rules: Rules; + + /** + * An array of arrays of precedence names or rules. Each inner array represents + * a *descending* ordering. Names/rules listed earlier in one of these arrays + * have higher precedence than any names/rules listed later in the same array. + * + * Using rules is just a shorthand way for using a name then calling prec() + * with that name. It is just a convenience. + */ + precedences?: ( + $: GrammarSymbols, + previous: Rule[][], + ) => RuleOrLiteral[][], + + /** + * An array of arrays of rule names. Each inner array represents a set of + * rules that's involved in an _LR(1) conflict_ that is _intended to exist_ + * in the grammar. When these conflicts occur at runtime, Tree-sitter will + * use the GLR algorithm to explore all of the possible interpretations. If + * _multiple_ parses end up succeeding, Tree-sitter will pick the subtree + * whose corresponding rule has the highest total _dynamic precedence_. + * + * @param $ grammar rules + */ + conflicts?: ( + $: GrammarSymbols, + previous: Rule[][], + ) => RuleOrLiteral[][]; + + /** + * An array of token names which can be returned by an _external scanner_. + * External scanners allow you to write custom C code which runs during the + * lexing process in order to handle lexical rules (e.g. Python's indentation + * tokens) that cannot be described by regular expressions. + * + * @param $ grammar rules + * @param previous array of externals from the base schema, if any + * + * @see https://tree-sitter.github.io/tree-sitter/creating-parsers#external-scanners + */ + externals?: ( + $: Record>, + previous: Rule[], + ) => RuleOrLiteral[]; + + /** + * An array of tokens that may appear anywhere in the language. This + * is often used for whitespace and comments. The default value of + * extras is to accept whitespace. To control whitespace explicitly, + * specify extras: `$ => []` in your grammar. + * + * @param $ grammar rules + */ + extras?: ( + $: GrammarSymbols, + ) => RuleOrLiteral[]; + + /** + * An array of rules that should be automatically removed from the + * grammar by replacing all of their usages with a copy of their definition. + * This is useful for rules that are used in multiple places but for which + * you don't want to create syntax tree nodes at runtime. + * + * @param $ grammar rules + */ + inline?: ( + $: GrammarSymbols, + previous: Rule[], + ) => RuleOrLiteral[]; + + /** + * A list of hidden rule names that should be considered supertypes in the + * generated node types file. + * + * @param $ grammar rules + * + * @see https://tree-sitter.github.io/tree-sitter/using-parsers#static-node-types + */ + supertypes?: ( + $: GrammarSymbols, + previous: Rule[], + ) => RuleOrLiteral[]; + + /** + * The name of a token that will match keywords for the purpose of the + * keyword extraction optimization. + * + * @param $ grammar rules + * + * @see https://tree-sitter.github.io/tree-sitter/creating-parsers#keyword-extraction + */ + word?: ($: GrammarSymbols) => RuleOrLiteral; +} + +type GrammarSchema = { + [K in keyof Grammar]: K extends 'rules' + ? Record + : Grammar[K]; +}; + +/** + * Causes the given rule to appear with an alternative name in the syntax tree. + * For instance with `alias($.foo, 'bar')`, the aliased rule will appear as an + * anonymous node, as if the rule had been written as the simple string. + * + * @param rule rule that will be aliased + * @param name target name for the alias + */ +declare function alias(rule: RuleOrLiteral, name: string): AliasRule; + +/** + * Causes the given rule to appear as an alternative named node, for instance + * with `alias($.foo, $.bar)`, the aliased rule `foo` will appear as a named + * node called `bar`. + * + * @param rule rule that will be aliased + * @param symbol target symbol for the alias + */ +declare function alias( + rule: RuleOrLiteral, + symbol: SymbolRule, +): AliasRule; + +/** + * Creates a blank rule, matching nothing. + */ +declare function blank(): BlankRule; + +/** + * Assigns a field name to the child node(s) matched by the given rule. + * In the resulting syntax tree, you can then use that field name to + * access specific children. + * + * @param name name of the field + * @param rule rule the field should match + */ +declare function field(name: string, rule: RuleOrLiteral): FieldRule; + +/** + * Creates a rule that matches one of a set of possible rules. The order + * of the arguments does not matter. This is analogous to the `|` (pipe) + * operator in EBNF notation. + * + * @param options possible rule choices + */ +declare function choice(...options: RuleOrLiteral[]): ChoiceRule; + +/** + * Creates a rule that matches zero or one occurrence of a given rule. + * It is analogous to the `[x]` (square bracket) syntax in EBNF notation. + * + * @param value rule to be made optional + */ +declare function optional(rule: RuleOrLiteral): ChoiceRule; + +/** + * Marks the given rule with a precedence which will be used to resolve LR(1) + * conflicts at parser-generation time. When two rules overlap in a way that + * represents either a true ambiguity or a _local_ ambiguity given one token + * of lookahead, Tree-sitter will try to resolve the conflict by matching the + * rule with the higher precedence. + * + * Precedence values can either be strings or numbers. When comparing rules + * with numerical precedence, higher numbers indicate higher precedences. To + * compare rules with string precedence, Tree-sitter uses the grammar's `precedences` + * field. + * + * rules is zero. This works similarly to the precedence directives in Yacc grammars. + * + * @param value precedence weight + * @param rule rule being weighted + * + * @see https://en.wikipedia.org/wiki/LR_parser#Conflicts_in_the_constructed_tables + * @see https://docs.oracle.com/cd/E19504-01/802-5880/6i9k05dh3/index.html + */ +declare const prec: { + (value: String | number, rule: RuleOrLiteral): PrecRule; + + /** + * Marks the given rule as left-associative (and optionally applies a + * numerical precedence). When an LR(1) conflict arises in which all of the + * rules have the same numerical precedence, Tree-sitter will consult the + * rules' associativity. If there is a left-associative rule, Tree-sitter + * will prefer matching a rule that ends _earlier_. This works similarly to + * associativity directives in Yacc grammars. + * + * @param value (optional) precedence weight + * @param rule rule to mark as left-associative + * + * @see https://docs.oracle.com/cd/E19504-01/802-5880/6i9k05dh3/index.html + */ + left(rule: RuleOrLiteral): PrecLeftRule; + left(value: String | number, rule: RuleOrLiteral): PrecLeftRule; + + /** + * Marks the given rule as right-associative (and optionally applies a + * numerical precedence). When an LR(1) conflict arises in which all of the + * rules have the same numerical precedence, Tree-sitter will consult the + * rules' associativity. If there is a right-associative rule, Tree-sitter + * will prefer matching a rule that ends _later_. This works similarly to + * associativity directives in Yacc grammars. + * + * @param value (optional) precedence weight + * @param rule rule to mark as right-associative + * + * @see https://docs.oracle.com/cd/E19504-01/802-5880/6i9k05dh3/index.html + */ + right(rule: RuleOrLiteral): PrecRightRule; + right(value: String | number, rule: RuleOrLiteral): PrecRightRule; + + /** + * Marks the given rule with a numerical precedence which will be used to + * resolve LR(1) conflicts at _runtime_ instead of parser-generation time. + * This is only necessary when handling a conflict dynamically using the + * `conflicts` field in the grammar, and when there is a genuine _ambiguity_: + * multiple rules correctly match a given piece of code. In that event, + * Tree-sitter compares the total dynamic precedence associated with each + * rule, and selects the one with the highest total. This is similar to + * dynamic precedence directives in Bison grammars. + * + * @param value precedence weight + * @param rule rule being weighted + * + * @see https://www.gnu.org/software/bison/manual/html_node/Generalized-LR-Parsing.html + */ + dynamic(value: String | number, rule: RuleOrLiteral): PrecDynamicRule; +}; + +/** + * Creates a rule that matches _zero-or-more_ occurrences of a given rule. + * It is analogous to the `{x}` (curly brace) syntax in EBNF notation. This + * rule is implemented in terms of `repeat1` but is included because it + * is very commonly used. + * + * @param rule rule to repeat, zero or more times + */ +declare function repeat(rule: RuleOrLiteral): RepeatRule; + +/** + * Creates a rule that matches one-or-more occurrences of a given rule. + * + * @param rule rule to repeat, one or more times + */ +declare function repeat1(rule: RuleOrLiteral): Repeat1Rule; + +/** + * Creates a rule that matches any number of other rules, one after another. + * It is analogous to simply writing multiple symbols next to each other + * in EBNF notation. + * + * @param rules ordered rules that comprise the sequence + */ +declare function seq(...rules: RuleOrLiteral[]): SeqRule; + +/** + * Creates a symbol rule, representing another rule in the grammar by name. + * + * @param name name of the target rule + */ +declare function sym(name: Name): SymbolRule; + +/** + * Marks the given rule as producing only a single token. Tree-sitter's + * default is to treat each String or RegExp literal in the grammar as a + * separate token. Each token is matched separately by the lexer and + * returned as its own leaf node in the tree. The token function allows + * you to express a complex rule using the DSL functions (rather + * than as a single regular expression) but still have Tree-sitter treat + * it as a single token. + * + * @param rule rule to represent as a single token + */ +declare const token: { + (rule: RuleOrLiteral): TokenRule; + + /** + * Marks the given rule as producing an immediate token. This allows + * the parser to produce a different token based on whether or not + * there are `extras` preceding the token's main content. When there + * are _no_ leading `extras`, an immediate token is preferred over a + * normal token which would otherwise match. + * + * @param rule rule to represent as an immediate token + */ + immediate(rule: RuleOrLiteral): ImmediateTokenRule; +}; + +/** + * Creates a new language grammar with the provided schema. + * + * @param options grammar options + */ +declare function grammar( + options: Grammar, +): GrammarSchema; + +/** + * Extends an existing language grammar with the provided options, + * creating a new language. + * + * @param baseGrammar base grammar schema to extend from + * @param options grammar options for the new extended language + */ +declare function grammar< + BaseGrammarRuleName extends string, + RuleName extends string +>( + baseGrammar: GrammarSchema, + options: Grammar, +): GrammarSchema; From 6911908d5b59e6dfa8a01ee578276828027ff0b8 Mon Sep 17 00:00:00 2001 From: Amaan Qureshi Date: Mon, 26 Feb 2024 12:44:31 -0500 Subject: [PATCH 5/5] ci: use new upstream workflows --- .github/workflows/ci.yml | 86 ++- .github/workflows/fuzz.yml | 22 - .github/workflows/lint.yml | 19 - .github/workflows/release.yml | 114 +--- .gitignore | 41 +- package-lock.json | 1151 +++++++++++++++++++++++++++++++++ package.json | 7 +- script/known_failures.txt | 4 - script/parse-examples | 47 -- 9 files changed, 1270 insertions(+), 221 deletions(-) delete mode 100644 .github/workflows/fuzz.yml delete mode 100644 .github/workflows/lint.yml create mode 100644 package-lock.json delete mode 100644 script/known_failures.txt delete mode 100755 script/parse-examples diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 84dc9bad..4635f6c3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,32 +1,78 @@ name: CI on: - pull_request: - branches: - - "**" push: - branches: - - "master" + branches: ["*"] + paths: + - grammar.js + - src/** + - test/** + - bindings/** + - binding.gyp + pull_request: + paths: + - grammar.js + - src/** + - test/** + - bindings/** + - binding.gyp + +concurrency: + group: ${{github.workflow}}-${{github.ref}} + cancel-in-progress: true + jobs: test: - runs-on: ${{ matrix.os }} + name: Test parser + runs-on: ${{matrix.os}} strategy: - fail-fast: true + fail-fast: false matrix: - os: [macos-latest, ubuntu-latest] + os: [ubuntu-latest, windows-latest, macos-14] + include: + - examples: + - { repo: 'numpy/numpy', path: 'examples/numpy' } + - { repo: 'django/django', path: 'examples/django' } + - { repo: 'pallets/flask', path: 'examples/flask' } + - { repo: 'python/cpython', path: 'examples/python' } steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 + - name: Set up the repo + uses: tree-sitter/parser-setup-action@v1.1 + with: + node-version: ${{vars.NODE_VERSION}} + - name: Set up examples + uses: actions/checkout@v4 + with: + repository: ${{matrix.examples.repo}} + path: ${{matrix.examples.path}} + clean: false + - name: Run tests + uses: tree-sitter/parser-test-action@v1.1 with: - node-version: 18 - - run: npm install - - run: npm test - test_windows: - runs-on: windows-latest + lint: true + test-library: ${{runner.os == 'Linux'}} + examples: | + examples/examples/**/*.py + !examples/cpython/Lib/test/badsyntax_3131.py + !examples/cpython/Lib/test/badsyntax_future8.py + !examples/cpython/Lib/test/test_compile.py + !examples/cpython/Tools/build/generate_re_casefix.py + fuzz: + name: Fuzz parser + runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 + - name: Checkout repository + uses: actions/checkout@v4 with: - node-version: 18 - - run: npm install - - run: npm run-script test-windows + fetch-depth: 2 + - name: Check for scanner changes + id: scanner-changes + run: |- + if git diff --quiet HEAD^ -- src/scanner.c; then + printf 'changed=false\n' >> "$GITHUB_OUTPUT" + else + printf 'changed=true\n' >> "$GITHUB_OUTPUT" + fi + - name: Fuzz parser + uses: tree-sitter/fuzz-action@v4 + if: steps.scanner-changes.outputs.changed == 'true' diff --git a/.github/workflows/fuzz.yml b/.github/workflows/fuzz.yml deleted file mode 100644 index 8718ea03..00000000 --- a/.github/workflows/fuzz.yml +++ /dev/null @@ -1,22 +0,0 @@ -name: Fuzz Parser - -on: - push: - paths: - - src/scanner.c - pull_request: - paths: - - src/scanner.c - workflow_dispatch: - -jobs: - test: - name: Parser fuzzing - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: vigoux/tree-sitter-fuzz-action@v1 - with: - language: python - external-scanner: src/scanner.c - time: 60 diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml deleted file mode 100644 index 103e92ae..00000000 --- a/.github/workflows/lint.yml +++ /dev/null @@ -1,19 +0,0 @@ -name: Lint - -on: - push: - branches: - - master - pull_request: - branches: - - "**" - -jobs: - lint: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Install modules - run: npm install - - name: Run ESLint - run: npm run lint diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 870eb84b..5f9525a1 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,103 +1,19 @@ -name: Release +name: Publish package on: - workflow_run: - workflows: ["CI"] - branches: - - master - types: - - completed + push: + tags: ["*"] -jobs: - release: - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Get previous commit SHA - id: get_previous_commit - run: | - LATEST_TAG=$(git describe --tags --abbrev=0) - if [[ -z "$LATEST_TAG" ]]; then - echo "No tag found. Failing..." - exit 1 - fi - echo "latest_tag=${LATEST_TAG#v}" >> "$GITHUB_ENV" # Remove 'v' prefix from the tag - - - name: Check if version changed and is greater than the previous - id: version_check - run: | - # Compare the current version with the version from the previous commit - PREVIOUS_NPM_VERSION=${{ env.latest_tag }} - CURRENT_NPM_VERSION=$(jq -r '.version' package.json) - CURRENT_CARGO_VERSION=$(awk -F '"' '/^version/ {print $2}' Cargo.toml) - if [[ "$CURRENT_NPM_VERSION" != "$CURRENT_CARGO_VERSION" ]]; then # Cargo.toml and package.json versions must match - echo "Mismatch: NPM version ($CURRENT_NPM_VERSION) and Cargo.toml version ($CURRENT_CARGO_VERSION)" - echo "version_changed=false" >> "$GITHUB_ENV" - else - if [[ "$PREVIOUS_NPM_VERSION" == "$CURRENT_NPM_VERSION" ]]; then - echo "version_changed=" >> "$GITHUB_ENV" - else - IFS='.' read -ra PREVIOUS_VERSION_PARTS <<< "$PREVIOUS_NPM_VERSION" - IFS='.' read -ra CURRENT_VERSION_PARTS <<< "$CURRENT_NPM_VERSION" - VERSION_CHANGED=false - for i in "${!PREVIOUS_VERSION_PARTS[@]}"; do - if [[ ${CURRENT_VERSION_PARTS[i]} -gt ${PREVIOUS_VERSION_PARTS[i]} ]]; then - VERSION_CHANGED=true - break - elif [[ ${CURRENT_VERSION_PARTS[i]} -lt ${PREVIOUS_VERSION_PARTS[i]} ]]; then - break - fi - done - - echo "version_changed=$VERSION_CHANGED" >> "$GITHUB_ENV" - echo "current_version=${CURRENT_NPM_VERSION}" >> "$GITHUB_ENV" - fi - fi - - - name: Display result - run: | - echo "Version bump detected: ${{ env.version_changed }}" +concurrency: + group: ${{github.workflow}}-${{github.ref}} + cancel-in-progress: true - - name: Fail if version is lower - if: env.version_changed == 'false' - run: exit 1 - - - name: Setup Node - if: env.version_changed == 'true' - uses: actions/setup-node@v4 - with: - node-version: 18 - registry-url: "https://registry.npmjs.org" - - name: Publish to NPM - if: env.version_changed == 'true' - env: - NODE_AUTH_TOKEN: ${{secrets.NPM_TOKEN}} - run: npm publish - - - name: Setup Rust - if: env.version_changed == 'true' - uses: actions-rs/toolchain@v1 - with: - profile: minimal - toolchain: stable - override: true - - name: Publish to Crates.io - if: env.version_changed == 'true' - uses: katyo/publish-crates@v2 - with: - registry-token: ${{ secrets.CARGO_REGISTRY_TOKEN }} - - - name: Tag versions - if: env.version_changed == 'true' - run: | - git checkout master - git config user.name github-actions[bot] - git config user.email github-actions[bot]@users.noreply.github.com - git tag -d "v${{ env.current_version }}" || true - git push origin --delete "v${{ env.current_version }}" || true - git tag -a "v${{ env.current_version }}" -m "Version ${{ env.current_version }}" - git push origin "v${{ env.current_version }}" +jobs: + npm: + uses: tree-sitter/workflows/.github/workflows/package-npm.yml@main + secrets: + NODE_AUTH_TOKEN: ${{secrets.NPM_TOKEN}} + crates: + uses: tree-sitter/workflows/.github/workflows/package-crates.yml@main + secrets: + CARGO_REGISTRY_TOKEN: ${{secrets.CARGO_TOKEN}} diff --git a/.gitignore b/.gitignore index c9971383..c1ab2b25 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,36 @@ -Cargo.lock -package-lock.json -node_modules -build -*.log -/examples/*/ +# Rust artifacts +/Cargo.lock /target/ + +# Node artifacts +/build/ +/node_modules/ + +# Swift artifacts +/.build/ + +# Python artifacts +/dist/ +*.egg-info +*.whl + +# Zig artifacts +/zig-cache/ +/zig-out/ + +# C artifacts +*.a +*.so +*.so.* +*.dylib +*.dll +*.pc + +# Example dirs +/examples/*/ + +# Grammar volatiles +dsl.d.ts +*.wasm +*.obj +*.o diff --git a/package-lock.json b/package-lock.json new file mode 100644 index 00000000..d2ec89db --- /dev/null +++ b/package-lock.json @@ -0,0 +1,1151 @@ +{ + "name": "tree-sitter-python", + "version": "0.20.4", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "tree-sitter-python", + "version": "0.20.4", + "license": "MIT", + "dependencies": { + "nan": "^2.18.0" + }, + "devDependencies": { + "eslint": "^8.57.0", + "eslint-config-google": "^0.14.0", + "tree-sitter-cli": "^0.21.0" + } + }, + "node_modules/@aashutoshrathi/word-wrap": { + "version": "1.2.6", + "resolved": "https://registry.npmjs.org/@aashutoshrathi/word-wrap/-/word-wrap-1.2.6.tgz", + "integrity": "sha512-1Yjs2SvM8TflER/OD3cOjhWWOZb58A2t7wpE2S9XfBYTiIl+XFhQG2bjy4Pu1I+EAlCNUzRDYDdFwFYUKvXcIA==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/@eslint-community/eslint-utils": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/@eslint-community/eslint-utils/-/eslint-utils-4.4.0.tgz", + "integrity": "sha512-1/sA4dwrzBAyeUoQ6oxahHKmrZvsnLCg4RfxW3ZFGGmQkSNQPFNLV9CUEFQP1x9EYXHTo5p6xdhZM1Ne9p/AfA==", + "dev": true, + "dependencies": { + "eslint-visitor-keys": "^3.3.0" + }, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + }, + "peerDependencies": { + "eslint": "^6.0.0 || ^7.0.0 || >=8.0.0" + } + }, + "node_modules/@eslint-community/regexpp": { + "version": "4.10.0", + "resolved": "https://registry.npmjs.org/@eslint-community/regexpp/-/regexpp-4.10.0.tgz", + "integrity": "sha512-Cu96Sd2By9mCNTx2iyKOmq10v22jUVQv0lQnlGNy16oE9589yE+QADPbrMGCkA51cKZSg3Pu/aTJVTGfL/qjUA==", + "dev": true, + "engines": { + "node": "^12.0.0 || ^14.0.0 || >=16.0.0" + } + }, + "node_modules/@eslint/eslintrc": { + "version": "2.1.4", + "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-2.1.4.tgz", + "integrity": "sha512-269Z39MS6wVJtsoUl10L60WdkhJVdPG24Q4eZTH3nnF6lpvSShEK3wQjDX9JRWAUPvPh7COouPpU9IrqaZFvtQ==", + "dev": true, + "dependencies": { + "ajv": "^6.12.4", + "debug": "^4.3.2", + "espree": "^9.6.0", + "globals": "^13.19.0", + "ignore": "^5.2.0", + "import-fresh": "^3.2.1", + "js-yaml": "^4.1.0", + "minimatch": "^3.1.2", + "strip-json-comments": "^3.1.1" + }, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/@eslint/js": { + "version": "8.57.0", + "resolved": "https://registry.npmjs.org/@eslint/js/-/js-8.57.0.tgz", + "integrity": "sha512-Ys+3g2TaW7gADOJzPt83SJtCDhMjndcDMFVQ/Tj9iA1BfJzFKD9mAUXT3OenpuPHbI6P/myECxRJrofUsDx/5g==", + "dev": true, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + } + }, + "node_modules/@humanwhocodes/config-array": { + "version": "0.11.14", + "resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.11.14.tgz", + "integrity": "sha512-3T8LkOmg45BV5FICb15QQMsyUSWrQ8AygVfC7ZG32zOalnqrilm018ZVCw0eapXux8FtA33q8PSRSstjee3jSg==", + "dev": true, + "dependencies": { + "@humanwhocodes/object-schema": "^2.0.2", + "debug": "^4.3.1", + "minimatch": "^3.0.5" + }, + "engines": { + "node": ">=10.10.0" + } + }, + "node_modules/@humanwhocodes/module-importer": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@humanwhocodes/module-importer/-/module-importer-1.0.1.tgz", + "integrity": "sha512-bxveV4V8v5Yb4ncFTT3rPSgZBOpCkjfK0y4oVVVJwIuDVBRMDXrPyXRL988i5ap9m9bnyEEjWfm5WkBmtffLfA==", + "dev": true, + "engines": { + "node": ">=12.22" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/nzakas" + } + }, + "node_modules/@humanwhocodes/object-schema": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/@humanwhocodes/object-schema/-/object-schema-2.0.2.tgz", + "integrity": "sha512-6EwiSjwWYP7pTckG6I5eyFANjPhmPjUX9JRLUSfNPC7FX7zK9gyZAfUEaECL6ALTpGX5AjnBq3C9XmVWPitNpw==", + "dev": true + }, + "node_modules/@nodelib/fs.scandir": { + "version": "2.1.5", + "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", + "integrity": "sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==", + "dev": true, + "dependencies": { + "@nodelib/fs.stat": "2.0.5", + "run-parallel": "^1.1.9" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/@nodelib/fs.stat": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/@nodelib/fs.stat/-/fs.stat-2.0.5.tgz", + "integrity": "sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A==", + "dev": true, + "engines": { + "node": ">= 8" + } + }, + "node_modules/@nodelib/fs.walk": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/@nodelib/fs.walk/-/fs.walk-1.2.8.tgz", + "integrity": "sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==", + "dev": true, + "dependencies": { + "@nodelib/fs.scandir": "2.1.5", + "fastq": "^1.6.0" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/@ungap/structured-clone": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@ungap/structured-clone/-/structured-clone-1.2.0.tgz", + "integrity": "sha512-zuVdFrMJiuCDQUMCzQaD6KL28MjnqqN8XnAqiEq9PNm/hCPTSGfrXCOfwj1ow4LFb/tNymJPwsNbVePc1xFqrQ==", + "dev": true + }, + "node_modules/acorn": { + "version": "8.11.3", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.11.3.tgz", + "integrity": "sha512-Y9rRfJG5jcKOE0CLisYbojUjIrIEE7AGMzA/Sm4BslANhbS+cDMpgBdcPT91oJ7OuJ9hYJBx59RjbhxVnrF8Xg==", + "dev": true, + "bin": { + "acorn": "bin/acorn" + }, + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/acorn-jsx": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/acorn-jsx/-/acorn-jsx-5.3.2.tgz", + "integrity": "sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ==", + "dev": true, + "peerDependencies": { + "acorn": "^6.0.0 || ^7.0.0 || ^8.0.0" + } + }, + "node_modules/ajv": { + "version": "6.12.6", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", + "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", + "dev": true, + "dependencies": { + "fast-deep-equal": "^3.1.1", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.4.1", + "uri-js": "^4.2.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dev": true, + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/argparse": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", + "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", + "dev": true + }, + "node_modules/balanced-match": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", + "dev": true + }, + "node_modules/brace-expansion": { + "version": "1.1.11", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", + "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "dev": true, + "dependencies": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "node_modules/callsites": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz", + "integrity": "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==", + "dev": true, + "engines": { + "node": ">=6" + } + }, + "node_modules/chalk": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", + "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", + "dev": true, + "dependencies": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/chalk?sponsor=1" + } + }, + "node_modules/color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dev": true, + "dependencies": { + "color-name": "~1.1.4" + }, + "engines": { + "node": ">=7.0.0" + } + }, + "node_modules/color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "dev": true + }, + "node_modules/concat-map": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", + "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==", + "dev": true + }, + "node_modules/cross-spawn": { + "version": "7.0.3", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", + "integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==", + "dev": true, + "dependencies": { + "path-key": "^3.1.0", + "shebang-command": "^2.0.0", + "which": "^2.0.1" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/debug": { + "version": "4.3.4", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", + "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==", + "dev": true, + "dependencies": { + "ms": "2.1.2" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/deep-is": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.4.tgz", + "integrity": "sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ==", + "dev": true + }, + "node_modules/doctrine": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/doctrine/-/doctrine-3.0.0.tgz", + "integrity": "sha512-yS+Q5i3hBf7GBkd4KG8a7eBNNWNGLTaEwwYWUijIYM7zrlYDM0BFXHjjPWlWZ1Rg7UaddZeIDmi9jF3HmqiQ2w==", + "dev": true, + "dependencies": { + "esutils": "^2.0.2" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/escape-string-regexp": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz", + "integrity": "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==", + "dev": true, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/eslint": { + "version": "8.57.0", + "resolved": "https://registry.npmjs.org/eslint/-/eslint-8.57.0.tgz", + "integrity": "sha512-dZ6+mexnaTIbSBZWgou51U6OmzIhYM2VcNdtiTtI7qPNZm35Akpr0f6vtw3w1Kmn5PYo+tZVfh13WrhpS6oLqQ==", + "dev": true, + "dependencies": { + "@eslint-community/eslint-utils": "^4.2.0", + "@eslint-community/regexpp": "^4.6.1", + "@eslint/eslintrc": "^2.1.4", + "@eslint/js": "8.57.0", + "@humanwhocodes/config-array": "^0.11.14", + "@humanwhocodes/module-importer": "^1.0.1", + "@nodelib/fs.walk": "^1.2.8", + "@ungap/structured-clone": "^1.2.0", + "ajv": "^6.12.4", + "chalk": "^4.0.0", + "cross-spawn": "^7.0.2", + "debug": "^4.3.2", + "doctrine": "^3.0.0", + "escape-string-regexp": "^4.0.0", + "eslint-scope": "^7.2.2", + "eslint-visitor-keys": "^3.4.3", + "espree": "^9.6.1", + "esquery": "^1.4.2", + "esutils": "^2.0.2", + "fast-deep-equal": "^3.1.3", + "file-entry-cache": "^6.0.1", + "find-up": "^5.0.0", + "glob-parent": "^6.0.2", + "globals": "^13.19.0", + "graphemer": "^1.4.0", + "ignore": "^5.2.0", + "imurmurhash": "^0.1.4", + "is-glob": "^4.0.0", + "is-path-inside": "^3.0.3", + "js-yaml": "^4.1.0", + "json-stable-stringify-without-jsonify": "^1.0.1", + "levn": "^0.4.1", + "lodash.merge": "^4.6.2", + "minimatch": "^3.1.2", + "natural-compare": "^1.4.0", + "optionator": "^0.9.3", + "strip-ansi": "^6.0.1", + "text-table": "^0.2.0" + }, + "bin": { + "eslint": "bin/eslint.js" + }, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/eslint-config-google": { + "version": "0.14.0", + "resolved": "https://registry.npmjs.org/eslint-config-google/-/eslint-config-google-0.14.0.tgz", + "integrity": "sha512-WsbX4WbjuMvTdeVL6+J3rK1RGhCTqjsFjX7UMSMgZiyxxaNLkoJENbrGExzERFeoTpGw3F3FypTiWAP9ZXzkEw==", + "dev": true, + "engines": { + "node": ">=0.10.0" + }, + "peerDependencies": { + "eslint": ">=5.16.0" + } + }, + "node_modules/eslint-scope": { + "version": "7.2.2", + "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-7.2.2.tgz", + "integrity": "sha512-dOt21O7lTMhDM+X9mB4GX+DZrZtCUJPL/wlcTqxyrx5IvO0IYtILdtrQGQp+8n5S0gwSVmOf9NQrjMOgfQZlIg==", + "dev": true, + "dependencies": { + "esrecurse": "^4.3.0", + "estraverse": "^5.2.0" + }, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/eslint-visitor-keys": { + "version": "3.4.3", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-3.4.3.tgz", + "integrity": "sha512-wpc+LXeiyiisxPlEkUzU6svyS1frIO3Mgxj1fdy7Pm8Ygzguax2N3Fa/D/ag1WqbOprdI+uY6wMUl8/a2G+iag==", + "dev": true, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/espree": { + "version": "9.6.1", + "resolved": "https://registry.npmjs.org/espree/-/espree-9.6.1.tgz", + "integrity": "sha512-oruZaFkjorTpF32kDSI5/75ViwGeZginGGy2NoOSg3Q9bnwlnmDm4HLnkl0RE3n+njDXR037aY1+x58Z/zFdwQ==", + "dev": true, + "dependencies": { + "acorn": "^8.9.0", + "acorn-jsx": "^5.3.2", + "eslint-visitor-keys": "^3.4.1" + }, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/esquery": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/esquery/-/esquery-1.5.0.tgz", + "integrity": "sha512-YQLXUplAwJgCydQ78IMJywZCceoqk1oH01OERdSAJc/7U2AylwjhSCLDEtqwg811idIS/9fIU5GjG73IgjKMVg==", + "dev": true, + "dependencies": { + "estraverse": "^5.1.0" + }, + "engines": { + "node": ">=0.10" + } + }, + "node_modules/esrecurse": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/esrecurse/-/esrecurse-4.3.0.tgz", + "integrity": "sha512-KmfKL3b6G+RXvP8N1vr3Tq1kL/oCFgn2NYXEtqP8/L3pKapUA4G8cFVaoF3SU323CD4XypR/ffioHmkti6/Tag==", + "dev": true, + "dependencies": { + "estraverse": "^5.2.0" + }, + "engines": { + "node": ">=4.0" + } + }, + "node_modules/estraverse": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.3.0.tgz", + "integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==", + "dev": true, + "engines": { + "node": ">=4.0" + } + }, + "node_modules/esutils": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz", + "integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/fast-deep-equal": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", + "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", + "dev": true + }, + "node_modules/fast-json-stable-stringify": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz", + "integrity": "sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==", + "dev": true + }, + "node_modules/fast-levenshtein": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz", + "integrity": "sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==", + "dev": true + }, + "node_modules/fastq": { + "version": "1.15.0", + "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.15.0.tgz", + "integrity": "sha512-wBrocU2LCXXa+lWBt8RoIRD89Fi8OdABODa/kEnyeyjS5aZO5/GNvI5sEINADqP/h8M29UHTHUb53sUu5Ihqdw==", + "dev": true, + "dependencies": { + "reusify": "^1.0.4" + } + }, + "node_modules/file-entry-cache": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-6.0.1.tgz", + "integrity": "sha512-7Gps/XWymbLk2QLYK4NzpMOrYjMhdIxXuIvy2QBsLE6ljuodKvdkWs/cpyJJ3CVIVpH0Oi1Hvg1ovbMzLdFBBg==", + "dev": true, + "dependencies": { + "flat-cache": "^3.0.4" + }, + "engines": { + "node": "^10.12.0 || >=12.0.0" + } + }, + "node_modules/find-up": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-5.0.0.tgz", + "integrity": "sha512-78/PXT1wlLLDgTzDs7sjq9hzz0vXD+zn+7wypEe4fXQxCmdmqfGsEPQxmiCSQI3ajFV91bVSsvNtrJRiW6nGng==", + "dev": true, + "dependencies": { + "locate-path": "^6.0.0", + "path-exists": "^4.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/flat-cache": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/flat-cache/-/flat-cache-3.0.4.tgz", + "integrity": "sha512-dm9s5Pw7Jc0GvMYbshN6zchCA9RgQlzzEZX3vylR9IqFfS8XciblUXOKfW6SiuJ0e13eDYZoZV5wdrev7P3Nwg==", + "dev": true, + "dependencies": { + "flatted": "^3.1.0", + "rimraf": "^3.0.2" + }, + "engines": { + "node": "^10.12.0 || >=12.0.0" + } + }, + "node_modules/flatted": { + "version": "3.2.7", + "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.2.7.tgz", + "integrity": "sha512-5nqDSxl8nn5BSNxyR3n4I6eDmbolI6WT+QqR547RwxQapgjQBmtktdP+HTBb/a/zLsbzERTONyUB5pefh5TtjQ==", + "dev": true + }, + "node_modules/fs.realpath": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", + "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==", + "dev": true + }, + "node_modules/glob": { + "version": "7.2.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", + "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", + "dev": true, + "dependencies": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.1.1", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + }, + "engines": { + "node": "*" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/glob-parent": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-6.0.2.tgz", + "integrity": "sha512-XxwI8EOhVQgWp6iDL+3b0r86f4d6AX6zSU55HfB4ydCEuXLXc5FcYeOu+nnGftS4TEju/11rt4KJPTMgbfmv4A==", + "dev": true, + "dependencies": { + "is-glob": "^4.0.3" + }, + "engines": { + "node": ">=10.13.0" + } + }, + "node_modules/globals": { + "version": "13.24.0", + "resolved": "https://registry.npmjs.org/globals/-/globals-13.24.0.tgz", + "integrity": "sha512-AhO5QUcj8llrbG09iWhPU2B204J1xnPeL8kQmVorSsy+Sjj1sk8gIyh6cUocGmH4L0UuhAJy+hJMRA4mgA4mFQ==", + "dev": true, + "dependencies": { + "type-fest": "^0.20.2" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/graphemer": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/graphemer/-/graphemer-1.4.0.tgz", + "integrity": "sha512-EtKwoO6kxCL9WO5xipiHTZlSzBm7WLT627TqC/uVRd0HKmq8NXyebnNYxDoBi7wt8eTWrUrKXCOVaFq9x1kgag==", + "dev": true + }, + "node_modules/has-flag": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", + "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/ignore": { + "version": "5.3.1", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.1.tgz", + "integrity": "sha512-5Fytz/IraMjqpwfd34ke28PTVMjZjJG2MPn5t7OE4eUCUNf8BAa7b5WUS9/Qvr6mwOQS7Mk6vdsMno5he+T8Xw==", + "dev": true, + "engines": { + "node": ">= 4" + } + }, + "node_modules/import-fresh": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.3.0.tgz", + "integrity": "sha512-veYYhQa+D1QBKznvhUHxb8faxlrwUnxseDAbAp457E0wLNio2bOSKnjYDhMj+YiAq61xrMGhQk9iXVk5FzgQMw==", + "dev": true, + "dependencies": { + "parent-module": "^1.0.0", + "resolve-from": "^4.0.0" + }, + "engines": { + "node": ">=6" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/imurmurhash": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/imurmurhash/-/imurmurhash-0.1.4.tgz", + "integrity": "sha512-JmXMZ6wuvDmLiHEml9ykzqO6lwFbof0GG4IkcGaENdCRDDmMVnny7s5HsIgHCbaq0w2MyPhDqkhTUgS2LU2PHA==", + "dev": true, + "engines": { + "node": ">=0.8.19" + } + }, + "node_modules/inflight": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", + "integrity": "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==", + "dev": true, + "dependencies": { + "once": "^1.3.0", + "wrappy": "1" + } + }, + "node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", + "dev": true + }, + "node_modules/is-extglob": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", + "integrity": "sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-glob": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz", + "integrity": "sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==", + "dev": true, + "dependencies": { + "is-extglob": "^2.1.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-path-inside": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/is-path-inside/-/is-path-inside-3.0.3.tgz", + "integrity": "sha512-Fd4gABb+ycGAmKou8eMftCupSir5lRxqf4aD/vd0cD2qc4HL07OjCeuHMr8Ro4CoMaeCKDB0/ECBOVWjTwUvPQ==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/isexe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", + "dev": true + }, + "node_modules/js-yaml": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz", + "integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==", + "dev": true, + "dependencies": { + "argparse": "^2.0.1" + }, + "bin": { + "js-yaml": "bin/js-yaml.js" + } + }, + "node_modules/json-schema-traverse": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", + "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", + "dev": true + }, + "node_modules/json-stable-stringify-without-jsonify": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/json-stable-stringify-without-jsonify/-/json-stable-stringify-without-jsonify-1.0.1.tgz", + "integrity": "sha512-Bdboy+l7tA3OGW6FjyFHWkP5LuByj1Tk33Ljyq0axyzdk9//JSi2u3fP1QSmd1KNwq6VOKYGlAu87CisVir6Pw==", + "dev": true + }, + "node_modules/levn": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/levn/-/levn-0.4.1.tgz", + "integrity": "sha512-+bT2uH4E5LGE7h/n3evcS/sQlJXCpIp6ym8OWJ5eV6+67Dsql/LaaT7qJBAt2rzfoa/5QBGBhxDix1dMt2kQKQ==", + "dev": true, + "dependencies": { + "prelude-ls": "^1.2.1", + "type-check": "~0.4.0" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/locate-path": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-6.0.0.tgz", + "integrity": "sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw==", + "dev": true, + "dependencies": { + "p-locate": "^5.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/lodash.merge": { + "version": "4.6.2", + "resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz", + "integrity": "sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==", + "dev": true + }, + "node_modules/minimatch": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "dev": true, + "dependencies": { + "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" + } + }, + "node_modules/ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", + "dev": true + }, + "node_modules/nan": { + "version": "2.18.0", + "resolved": "https://registry.npmjs.org/nan/-/nan-2.18.0.tgz", + "integrity": "sha512-W7tfG7vMOGtD30sHoZSSc/JVYiyDPEyQVso/Zz+/uQd0B0L46gtC+pHha5FFMRpil6fm/AoEcRWyOVi4+E/f8w==" + }, + "node_modules/natural-compare": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", + "integrity": "sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==", + "dev": true + }, + "node_modules/once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "dev": true, + "dependencies": { + "wrappy": "1" + } + }, + "node_modules/optionator": { + "version": "0.9.3", + "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.3.tgz", + "integrity": "sha512-JjCoypp+jKn1ttEFExxhetCKeJt9zhAgAve5FXHixTvFDW/5aEktX9bufBKLRRMdU7bNtpLfcGu94B3cdEJgjg==", + "dev": true, + "dependencies": { + "@aashutoshrathi/word-wrap": "^1.2.3", + "deep-is": "^0.1.3", + "fast-levenshtein": "^2.0.6", + "levn": "^0.4.1", + "prelude-ls": "^1.2.1", + "type-check": "^0.4.0" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/p-limit": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", + "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==", + "dev": true, + "dependencies": { + "yocto-queue": "^0.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-locate": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-5.0.0.tgz", + "integrity": "sha512-LaNjtRWUBY++zB5nE/NwcaoMylSPk+S+ZHNB1TzdbMJMny6dynpAGt7X/tl/QYq3TIeE6nxHppbo2LGymrG5Pw==", + "dev": true, + "dependencies": { + "p-limit": "^3.0.2" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/parent-module": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz", + "integrity": "sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g==", + "dev": true, + "dependencies": { + "callsites": "^3.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/path-exists": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", + "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/path-is-absolute": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", + "integrity": "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/path-key": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/prelude-ls": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.2.1.tgz", + "integrity": "sha512-vkcDPrRZo1QZLbn5RLGPpg/WmIQ65qoWWhcGKf/b5eplkkarX0m9z8ppCat4mlOqUsWpyNuYgO3VRyrYHSzX5g==", + "dev": true, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/punycode": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz", + "integrity": "sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==", + "dev": true, + "engines": { + "node": ">=6" + } + }, + "node_modules/queue-microtask": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz", + "integrity": "sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ] + }, + "node_modules/resolve-from": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz", + "integrity": "sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==", + "dev": true, + "engines": { + "node": ">=4" + } + }, + "node_modules/reusify": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/reusify/-/reusify-1.0.4.tgz", + "integrity": "sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw==", + "dev": true, + "engines": { + "iojs": ">=1.0.0", + "node": ">=0.10.0" + } + }, + "node_modules/rimraf": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz", + "integrity": "sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==", + "dev": true, + "dependencies": { + "glob": "^7.1.3" + }, + "bin": { + "rimraf": "bin.js" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/run-parallel": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/run-parallel/-/run-parallel-1.2.0.tgz", + "integrity": "sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "dependencies": { + "queue-microtask": "^1.2.2" + } + }, + "node_modules/shebang-command": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", + "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "dev": true, + "dependencies": { + "shebang-regex": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/shebang-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", + "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dev": true, + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-json-comments": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.1.1.tgz", + "integrity": "sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==", + "dev": true, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/supports-color": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", + "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", + "dev": true, + "dependencies": { + "has-flag": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/text-table": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/text-table/-/text-table-0.2.0.tgz", + "integrity": "sha512-N+8UisAXDGk8PFXP4HAzVR9nbfmVJ3zYLAWiTIoqC5v5isinhr+r5uaO8+7r3BMfuNIufIsA7RdpVgacC2cSpw==", + "dev": true + }, + "node_modules/tree-sitter-cli": { + "version": "0.21.0", + "resolved": "https://registry.npmjs.org/tree-sitter-cli/-/tree-sitter-cli-0.21.0.tgz", + "integrity": "sha512-wA7wT5724fNQW82XDH6zT6ZcYonjrAKLCHHuhLsPcAKULrhp3rNuMvlgBdB5FUBvmjHNhtTZF/qpHenMoRJPBw==", + "dev": true, + "hasInstallScript": true, + "bin": { + "tree-sitter": "cli.js" + } + }, + "node_modules/type-check": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz", + "integrity": "sha512-XleUoc9uwGXqjWwXaUTZAmzMcFZ5858QA2vvx1Ur5xIcixXIP+8LnFDgRplU30us6teqdlskFfu+ae4K79Ooew==", + "dev": true, + "dependencies": { + "prelude-ls": "^1.2.1" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/type-fest": { + "version": "0.20.2", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.20.2.tgz", + "integrity": "sha512-Ne+eE4r0/iWnpAxD852z3A+N0Bt5RN//NjJwRd2VFHEmrywxf5vsZlh4R6lixl6B+wz/8d+maTSAkN1FIkI3LQ==", + "dev": true, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/uri-js": { + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.4.1.tgz", + "integrity": "sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==", + "dev": true, + "dependencies": { + "punycode": "^2.1.0" + } + }, + "node_modules/which": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "dev": true, + "dependencies": { + "isexe": "^2.0.0" + }, + "bin": { + "node-which": "bin/node-which" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", + "dev": true + }, + "node_modules/yocto-queue": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", + "integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==", + "dev": true, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + } + } +} diff --git a/package.json b/package.json index 17c81f21..cb627bd7 100644 --- a/package.json +++ b/package.json @@ -14,16 +14,15 @@ "nan": "^2.18.0" }, "devDependencies": { - "eslint": "^8.53.0", + "eslint": "^8.57.0", "eslint-config-google": "^0.14.0", - "tree-sitter-cli": "^0.20.8" + "tree-sitter-cli": "^0.21.0" }, "scripts": { "build": "tree-sitter generate && node-gyp build", "lint": "eslint grammar.js", "parse": "tree-sitter parse", - "test": "tree-sitter test && script/parse-examples", - "test-windows": "tree-sitter test" + "test": "tree-sitter test" }, "repository": "https://github.com/tree-sitter/tree-sitter-python", "tree-sitter": [ diff --git a/script/known_failures.txt b/script/known_failures.txt deleted file mode 100644 index 5dbaaf7c..00000000 --- a/script/known_failures.txt +++ /dev/null @@ -1,4 +0,0 @@ -examples/cpython/Lib/test/badsyntax_3131.py -examples/cpython/Lib/test/badsyntax_future8.py -examples/cpython/Lib/test/test_compile.py -examples/cpython/Tools/build/generate_re_casefix.py diff --git a/script/parse-examples b/script/parse-examples deleted file mode 100755 index 7c62ebd7..00000000 --- a/script/parse-examples +++ /dev/null @@ -1,47 +0,0 @@ -#!/usr/bin/env bash - -set -eu - -cd "$(dirname "$0")/.." - -function clone_repo { - owner=$1 - name=$2 - sha=$3 - - path=examples/$name - if [ ! -d "$path" ]; then - echo "Cloning $owner/$name" - git clone "https://github.com/$owner/$name" "$path" - fi - - pushd "$path" >/dev/null - actual_sha=$(git rev-parse HEAD) - if [ "$actual_sha" != "$sha" ]; then - echo "Updating $owner/$name to $sha" - git fetch - git reset --hard "$sha" - fi - popd >/dev/null -} - -clone_repo numpy numpy 058851c5cfc98f50f11237b1c13d77cfd1f40475 -clone_repo django django 01974d7f7549b2dca2a729c3c1a1ea7d4585eb3a -clone_repo pallets flask de464c03e134127140e5622e230790806a133ff9 -clone_repo python cpython bb456a08a3db851e6feaefc3328f39096919ec8d - -known_failures="$(cat script/known_failures.txt)" - -# shellcheck disable=2046 -tree-sitter parse -q \ - 'examples/**/*.py' \ - $(for file in $known_failures; do echo "!${file}"; done) - -example_count=$(find examples -name '*.py' | wc -l) -failure_count=$(wc -w <<<"$known_failures") -success_count=$((example_count - failure_count)) -success_percent=$(bc -l <<<"100*${success_count}/${example_count}") - -printf \ - "Successfully parsed %d of %d example files (%.1f%%)\n" \ - "$success_count" "$example_count" "$success_percent"