diff --git a/.gitattributes b/.gitattributes index cee6011..5c345d8 100644 --- a/.gitattributes +++ b/.gitattributes @@ -8,7 +8,7 @@ */src/tree_sitter/* linguist-generated # C bindings -*/bindings/c/* linguist-generated +*/bindings/c/** linguist-generated # Rust bindings bindings/rust/build.rs linguist-generated @@ -35,3 +35,7 @@ go.sum linguist-generated */bindings/swift/** linguist-generated Package.swift linguist-generated Package.resolved linguist-generated + +# Zig bindings +build.zig linguist-generated +build.zig.zon linguist-generated diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..e806045 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,16 @@ +version: 2 +updates: + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" + cooldown: + default-days: 3 + commit-message: + prefix: "ci" + labels: + - "CI" + groups: + actions: + patterns: ["*"] + diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4cfc0d3..22ab869 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,15 +21,13 @@ jobs: os: [ubuntu-latest, windows-latest, macos-latest] steps: - name: Checkout repository - uses: actions/checkout@v5 + uses: actions/checkout@v6 - name: Set up tree-sitter - uses: tree-sitter/setup-action@v2 - with: - install-lib: false + uses: tree-sitter/setup-action/cli@v2 - name: Set up Node.js - uses: actions/setup-node@v4 + uses: actions/setup-node@v6 with: node-version: ${{vars.NODE_VERSION}} @@ -58,7 +56,7 @@ jobs: parser: [tree-sitter-markdown, tree-sitter-markdown-inline] steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@v6 with: fetch-depth: 2 - name: Check for scanner changes @@ -74,3 +72,26 @@ jobs: if: steps.scanner-changes.outputs.changed == 'true' with: directory: ${{matrix.parser}} + + query: + name: Validate queries + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + parser: [markdown, markdown-inline] + steps: + - name: Set up repository + uses: actions/checkout@v6 + + - name: Set up tree-sitter + uses: tree-sitter/setup-action/cli@v2 + + - name: Build parser + run: tree-sitter build tree-sitter-${{matrix.parser}} + + - name: Set up ts_query_ls + run: curl -fL https://github.com/ribru17/ts_query_ls/releases/latest/download/ts_query_ls-x86_64-unknown-linux-gnu.tar.gz | tar -xz + + - name: Check queries + run: ./ts_query_ls check -f tree-sitter-${{matrix.parser}}/queries/ diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b677185..5a39296 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -17,5 +17,3 @@ permissions: jobs: release: uses: tree-sitter/workflows/.github/workflows/release.yml@main - with: - attestations: true diff --git a/.gitignore b/.gitignore index 623310c..7c0cb7f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,18 +1,15 @@ # Rust artifacts -Cargo.lock target/ # Node artifacts build/ prebuilds/ node_modules/ -*.tgz # Swift artifacts .build/ # Go artifacts -go.sum _obj/ # Python artifacts @@ -28,6 +25,13 @@ dist/ *.dylib *.dll *.pc +*.exp +*.lib + +# Zig artifacts +.zig-cache/ +zig-cache/ +zig-out/ # Example dirs /examples/*/ @@ -37,7 +41,8 @@ dist/ *.obj *.o -# Moved bindings -bindings/c/ -bindings/swift/TreeSitterMarkdown/ -bindings/swift/TreeSitterMarkdownInline/ +# Archives +*.tar.gz +*.tgz +*.zip +*.jar diff --git a/.tsqueryrc.json b/.tsqueryrc.json new file mode 100644 index 0000000..a7a343a --- /dev/null +++ b/.tsqueryrc.json @@ -0,0 +1,5 @@ +{ + "$schema": "https://raw.githubusercontent.com/ribru17/ts_query_ls/refs/heads/master/schemas/config.json", + "parser_install_directories": ["."], + "language_retrieval_patterns": ["tree-sitter-([^/]+)/queries/[^/]+\\.scm$"] +} diff --git a/Cargo.toml b/Cargo.toml index bd03c73..c24f360 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -32,13 +32,13 @@ path = "bindings/rust/lib.rs" [dependencies] tree-sitter-language = "0.1" -tree-sitter = { version = "0.24", optional = true } +tree-sitter = { version = "0.26", optional = true } [dev-dependencies] -tree-sitter = "0.24.3" +tree-sitter = "0.26.3" [build-dependencies] -cc = "1.1.22" +cc = "1.2" [[bin]] name = "benchmark" diff --git a/Package.swift b/Package.swift index 375750c..4f6d2f1 100644 --- a/Package.swift +++ b/Package.swift @@ -8,7 +8,7 @@ let package = Package( .library(name: "TreeSitterMarkdown", targets: ["TreeSitterMarkdown", "TreeSitterMarkdownInline"]), ], dependencies: [ - .package(url: "https://github.com/ChimeHQ/SwiftTreeSitter", from: "0.8.0"), + .package(name: "SwiftTreeSitter", url: "https://github.com/tree-sitter/swift-tree-sitter", from: "0.8.0"), ], targets: [ .target( diff --git a/bindings/go/binding_test.go b/bindings/go/binding_test.go deleted file mode 100644 index b2381f6..0000000 --- a/bindings/go/binding_test.go +++ /dev/null @@ -1,22 +0,0 @@ -package tree_sitter_markdown_test - -import ( - "testing" - - tree_sitter "github.com/tree-sitter/go-tree-sitter" - tree_sitter_markdown "github.com/tree-sitter-grammars/tree-sitter-markdown/bindings/go" -) - -func TestCanLoadBlockGrammar(t *testing.T) { - language := tree_sitter.NewLanguage(tree_sitter_markdown.Language()) - if language == nil { - t.Errorf("Error loading Markdown block grammar") - } -} - -func TestCanLoadInlineGrammar(t *testing.T) { - language := tree_sitter.NewLanguage(tree_sitter_markdown.InlineLanguage()) - if language == nil { - t.Errorf("Error loading Markdown inline grammar") - } -} diff --git a/bindings/go/markdown.go b/bindings/go/markdown.go deleted file mode 100644 index f38fda6..0000000 --- a/bindings/go/markdown.go +++ /dev/null @@ -1,14 +0,0 @@ -package tree_sitter_markdown - -// #cgo CPPFLAGS: -I../../tree-sitter-markdown -// #cgo CFLAGS: -std=c11 -fPIC -// #include "../../tree-sitter-markdown/src/parser.c" -// #include "../../tree-sitter-markdown/src/scanner.c" -import "C" - -import "unsafe" - -// Get the tree-sitter Language for the block grammar. -func Language() unsafe.Pointer { - return unsafe.Pointer(C.tree_sitter_markdown()) -} diff --git a/bindings/go/markdown_inline.go b/bindings/go/markdown_inline.go deleted file mode 100644 index f779990..0000000 --- a/bindings/go/markdown_inline.go +++ /dev/null @@ -1,14 +0,0 @@ -package tree_sitter_markdown - -// #cgo CPPFLAGS: -I../../tree-sitter-markdown-inline -// #cgo CFLAGS: -std=c11 -fPIC -// #include "../../tree-sitter-markdown-inline/src/parser.c" -// #include "../../tree-sitter-markdown-inline/src/scanner.c" -import "C" - -import "unsafe" - -// Get the tree-sitter Language for the inline grammar. -func InlineLanguage() unsafe.Pointer { - return unsafe.Pointer(C.tree_sitter_markdown_inline()) -} diff --git a/bindings/node/binding_test.js b/bindings/node/binding_test.js index cbba7a6..7a91a84 100644 --- a/bindings/node/binding_test.js +++ b/bindings/node/binding_test.js @@ -1,16 +1,11 @@ -/// +import assert from "node:assert"; +import { test } from "node:test"; +import Parser from "tree-sitter"; -const assert = require("node:assert"); -const { test } = require("node:test"); - -const Parser = require("tree-sitter"); - -test("can load block grammar", () => { - const parser = new Parser(); - assert.doesNotThrow(() => parser.setLanguage(require("."))); -}); - -test("can load inline grammar", () => { +test("can load grammar", () => { const parser = new Parser(); - assert.doesNotThrow(() => parser.setLanguage(require(".").inline)); + assert.doesNotReject(async () => { + const { default: language } = await import("./index.js"); + parser.setLanguage(language); + }); }); diff --git a/bindings/node/index.d.ts b/bindings/node/index.d.ts index 299635b..d75fe74 100644 --- a/bindings/node/index.d.ts +++ b/bindings/node/index.d.ts @@ -18,13 +18,43 @@ type NodeInfo = children: ChildNode[]; }); -type Language = { - name: string; +/** + * The tree-sitter language object for this grammar. + * + * @see {@linkcode https://tree-sitter.github.io/node-tree-sitter/interfaces/Parser.Language.html Parser.Language} + * + * @example + * import Parser from "tree-sitter"; + * import Markdown from "tree-sitter-markdown"; + * + * const parser = new Parser(); + * parser.setLanguage(Markdown); + */ +declare const binding: { + /** + * The inner language object. + * @private + */ language: unknown; + + /** + * The content of the `node-types.json` file for this grammar. + * + * @see {@linkplain https://tree-sitter.github.io/tree-sitter/using-parsers/6-static-node-types Static Node Types} + */ nodeTypeInfo: NodeInfo[]; -}; -declare const _exports: Language & { - inline: Language + /** The syntax highlighting query for this grammar. */ + HIGHLIGHTS_QUERY?: string; + + /** The language injection query for this grammar. */ + INJECTIONS_QUERY?: string; + + /** The local variable query for this grammar. */ + LOCALS_QUERY?: string; + + /** The symbol tagging query for this grammar. */ + TAGS_QUERY?: string; }; -export = _exports; + +export default binding; diff --git a/bindings/node/index.js b/bindings/node/index.js index 6452853..e4e8e21 100644 --- a/bindings/node/index.js +++ b/bindings/node/index.js @@ -1,8 +1,37 @@ -const root = require("path").join(__dirname, "..", ".."); +import { readFileSync } from "node:fs"; +import { fileURLToPath } from "node:url"; -module.exports = require("node-gyp-build")(root); +const root = fileURLToPath(new URL("../..", import.meta.url)); + +const binding = typeof process.versions.bun === "string" + // Support `bun build --compile` by being statically analyzable enough to find the .node file at build-time + ? await import(`${root}/prebuilds/${process.platform}-${process.arch}/tree-sitter-markdown.node`) + : (await import("node-gyp-build")).default(root); try { - module.exports.nodeTypeInfo = require("../../tree-sitter-markdown/src/node-types.json"); - module.exports.inline.nodeTypeInfo = require("../../tree-sitter-markdown-inline/src/node-types.json"); -} catch (_) { } + const nodeTypes = await import(`${root}/src/node-types.json`, { with: { type: "json" } }); + binding.nodeTypeInfo = nodeTypes.default; +} catch { } + +const queries = [ + ["HIGHLIGHTS_QUERY", `${root}/tree-sitter-markdown/queries/highlights.scm`], + ["INJECTIONS_QUERY", `${root}/tree-sitter-markdown/queries/injections.scm`], + ["LOCALS_QUERY", `${root}/queries/locals.scm`], + ["TAGS_QUERY", `${root}/queries/tags.scm`], +]; + +for (const [prop, path] of queries) { + Object.defineProperty(binding, prop, { + configurable: true, + enumerable: true, + get() { + delete binding[prop]; + try { + binding[prop] = readFileSync(path, "utf8"); + } catch { } + return binding[prop]; + } + }); +} + +export default binding; diff --git a/bindings/python/tests/test_binding.py b/bindings/python/tests/test_binding.py index 856599a..a8c3b03 100644 --- a/bindings/python/tests/test_binding.py +++ b/bindings/python/tests/test_binding.py @@ -6,12 +6,12 @@ class TestLanguage(TestCase): def test_can_load_block_grammar(self): try: - tree_sitter.Language(tree_sitter_markdown.language()) + Parser(Language(tree_sitter_markdown.language())) except Exception: self.fail("Error loading Markdown block grammar") def test_can_load_block_grammar(self): try: - tree_sitter.Language(tree_sitter_markdown.inline_language()) + Parser(Language(tree_sitter_markdown.inline_language()) except Exception: self.fail("Error loading Markdown inline grammar") diff --git a/bindings/python/tree_sitter_markdown/__init__.py b/bindings/python/tree_sitter_markdown/__init__.py index 08a73e9..d64fec5 100644 --- a/bindings/python/tree_sitter_markdown/__init__.py +++ b/bindings/python/tree_sitter_markdown/__init__.py @@ -1,5 +1,43 @@ -"Markdown grammar for tree-sitter" +"""Markdown grammar for tree-sitter""" -from ._binding import language, inline_language +from importlib.resources import files as _files -__all__ = ["language", "inline_language"] +from ._binding import language + + +def _get_query(name, file): + try: + query = _files(f"{__package__}") / file + globals()[name] = query.read_text() + except FileNotFoundError: + globals()[name] = None + return globals()[name] + + +def __getattr__(name): + if name == "HIGHLIGHTS_QUERY": + return _get_query("HIGHLIGHTS_QUERY", "tree-sitter-markdown/queries/highlights.scm") + if name == "INJECTIONS_QUERY": + return _get_query("INJECTIONS_QUERY", "tree-sitter-markdown/queries/injections.scm") + if name == "LOCALS_QUERY": + return _get_query("LOCALS_QUERY", "queries/locals.scm") + if name == "TAGS_QUERY": + return _get_query("TAGS_QUERY", "queries/tags.scm") + + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + + +__all__ = [ + "language", + "HIGHLIGHTS_QUERY", + "INJECTIONS_QUERY", + "LOCALS_QUERY", + "TAGS_QUERY", +] + + +def __dir__(): + return sorted(__all__ + [ + "__all__", "__builtins__", "__cached__", "__doc__", "__file__", + "__loader__", "__name__", "__package__", "__path__", "__spec__", + ]) diff --git a/bindings/python/tree_sitter_markdown/__init__.pyi b/bindings/python/tree_sitter_markdown/__init__.pyi index 0acdc0a..a99f688 100644 --- a/bindings/python/tree_sitter_markdown/__init__.pyi +++ b/bindings/python/tree_sitter_markdown/__init__.pyi @@ -1,3 +1,3 @@ -def language() -> object: ... +def language() -> CapsuleType: ... -def inline_language() -> object: ... +def inline_language() -> CapsuleType: ... diff --git a/bindings/python/tree_sitter_markdown/binding.c b/bindings/python/tree_sitter_markdown/binding.c index f9973f7..2a9d86b 100644 --- a/bindings/python/tree_sitter_markdown/binding.c +++ b/bindings/python/tree_sitter_markdown/binding.c @@ -14,6 +14,13 @@ static PyObject* _binding_inline_language(PyObject *Py_UNUSED(self), PyObject *P return PyCapsule_New(tree_sitter_markdown_inline(), "tree_sitter.Language", NULL); } +static struct PyModuleDef_Slot slots[] = { +#ifdef Py_GIL_DISABLED + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, +#endif + {0, NULL} +}; + static PyMethodDef methods[] = { {"language", _binding_language, METH_NOARGS, "Get the tree-sitter language for the block grammar."}, @@ -26,10 +33,11 @@ static struct PyModuleDef module = { .m_base = PyModuleDef_HEAD_INIT, .m_name = "_binding", .m_doc = NULL, - .m_size = -1, - .m_methods = methods + .m_size = 0, + .m_methods = methods, + .m_slots = slots, }; PyMODINIT_FUNC PyInit__binding(void) { - return PyModule_Create(&module); + return PyModuleDef_Init(&module); } diff --git a/bindings/rust/build.rs b/bindings/rust/build.rs index c9f5943..58939aa 100644 --- a/bindings/rust/build.rs +++ b/bindings/rust/build.rs @@ -8,6 +8,24 @@ fn main() { #[cfg(target_env = "msvc")] c_config.flag("-utf-8"); + if std::env::var("TARGET").unwrap() == "wasm32-unknown-unknown" { + let Ok(wasm_headers) = std::env::var("DEP_TREE_SITTER_LANGUAGE_WASM_HEADERS") else { + panic!("Environment variable DEP_TREE_SITTER_LANGUAGE_WASM_HEADERS must be set by the language crate"); + }; + let Ok(wasm_src) = + std::env::var("DEP_TREE_SITTER_LANGUAGE_WASM_SRC").map(std::path::PathBuf::from) + else { + panic!("Environment variable DEP_TREE_SITTER_LANGUAGE_WASM_SRC must be set by the language crate"); + }; + + c_config.include(&wasm_headers); + c_config.files([ + wasm_src.join("stdio.c"), + wasm_src.join("stdlib.c"), + wasm_src.join("string.c"), + ]); + } + for path in &[ block_dir.join("parser.c"), block_dir.join("scanner.c"), @@ -19,4 +37,25 @@ fn main() { } c_config.compile("tree-sitter-markdown"); + + println!("cargo:rustc-check-cfg=cfg(with_highlights_query)"); + if !"tree-sitter-markdown/queries/highlights.scm".is_empty() + && std::path::Path::new("tree-sitter-markdown/queries/highlights.scm").exists() + { + println!("cargo:rustc-cfg=with_highlights_query"); + } + println!("cargo:rustc-check-cfg=cfg(with_injections_query)"); + if !"tree-sitter-markdown/queries/injections.scm".is_empty() + && std::path::Path::new("tree-sitter-markdown/queries/injections.scm").exists() + { + println!("cargo:rustc-cfg=with_injections_query"); + } + println!("cargo:rustc-check-cfg=cfg(with_locals_query)"); + if !"queries/locals.scm".is_empty() && std::path::Path::new("queries/locals.scm").exists() { + println!("cargo:rustc-cfg=with_locals_query"); + } + println!("cargo:rustc-check-cfg=cfg(with_tags_query)"); + if !"queries/tags.scm".is_empty() && std::path::Path::new("queries/tags.scm").exists() { + println!("cargo:rustc-cfg=with_tags_query"); + } } diff --git a/bindings/rust/parser.rs b/bindings/rust/parser.rs index 2fd8d5e..4a17618 100644 --- a/bindings/rust/parser.rs +++ b/bindings/rust/parser.rs @@ -1,7 +1,9 @@ use std::collections::HashMap; use std::num::NonZeroU16; -use tree_sitter::{InputEdit, Language, Node, Parser, Point, Range, Tree, TreeCursor}; +use tree_sitter::{ + InputEdit, Language, Node, ParseOptions, Parser, Point, Range, Tree, TreeCursor, +}; use crate::{INLINE_LANGUAGE, LANGUAGE}; @@ -16,7 +18,7 @@ pub struct MarkdownParser { /// A stateful object for walking a [`MarkdownTree`] efficiently. /// -/// This exposes the same methdos as [`TreeCursor`], but abstracts away the +/// This exposes the same methods as [`TreeCursor`], but abstracts away the /// double block / inline structure of [`MarkdownTree`]. pub struct MarkdownCursor<'a> { markdown_tree: &'a MarkdownTree, @@ -33,7 +35,7 @@ impl<'a> MarkdownCursor<'a> { } } - /// Returns `true` if the current node is from the (inline language)[INLINE_LANGUAGE] + /// Returns `true` if the current node is from the [INLINE_LANGUAGE](inline language) /// /// This information is needed to handle "tree-sitter internal" data like /// [`field_id`](Self::field_id) correctly. @@ -221,7 +223,7 @@ impl MarkdownTree { } /// Create a new [`MarkdownCursor`] starting from the root of the tree. - pub fn walk(&self) -> MarkdownCursor { + pub fn walk(&self) -> MarkdownCursor<'_> { MarkdownCursor { markdown_tree: self, block_cursor: self.block_tree.walk(), @@ -230,6 +232,28 @@ impl MarkdownTree { } } +/// The options used while parsing a [`MarkdownTree`]. +/// +/// This abstracts away the double block / inline structure of [`MarkdownParser`]. +#[derive(Default)] +pub struct MarkdownParseOptions<'a> { + block_options: Option>, + inline_options: Option>, +} + +impl<'a> MarkdownParseOptions<'a> { + /// Creates a new [MarkdownParseOptions] instance. + pub fn new( + block_options: Option>, + inline_options: Option>, + ) -> Self { + MarkdownParseOptions { + block_options, + inline_options, + } + } +} + impl Default for MarkdownParser { fn default() -> Self { let block_language = LANGUAGE.into(); @@ -252,14 +276,17 @@ impl MarkdownParser { /// If the text of the document has changed since `old_tree` was /// created, then you must edit `old_tree` to match the new text using /// [MarkdownTree::edit]. + /// * `options` The [options][MarkdownParseOptions] used for parsing. + /// Use `MarkdownParseOptions::default()` if you don't need to pass any options. /// /// Returns a [MarkdownTree] if parsing succeeded, or `None` if: /// * The timeout set with [tree_sitter::Parser::set_timeout_micros] expired /// * The cancellation flag set with [tree_sitter::Parser::set_cancellation_flag] was flipped - pub fn parse_with, F: FnMut(usize, Point) -> T>( + pub fn parse_with_options, F: FnMut(usize, Point) -> T>( &mut self, callback: &mut F, old_tree: Option<&MarkdownTree>, + mut options: MarkdownParseOptions<'_>, ) -> Option { let MarkdownParser { parser, @@ -272,7 +299,11 @@ impl MarkdownParser { parser .set_language(block_language) .expect("Could not load block grammar"); - let block_tree = parser.parse_with(callback, old_tree.map(|tree| &tree.block_tree))?; + let block_tree = parser.parse_with_options( + callback, + old_tree.map(|tree| &tree.block_tree), + options.block_options.as_mut().map(|b_opt| b_opt.reborrow()), + )?; let (mut inline_trees, mut inline_indices) = if let Some(old_tree) = old_tree { let len = old_tree.inline_trees.len(); (Vec::with_capacity(len), HashMap::with_capacity(len)) @@ -322,9 +353,13 @@ impl MarkdownParser { } ranges.push(range); parser.set_included_ranges(&ranges).ok()?; - let inline_tree = parser.parse_with( + let inline_tree = parser.parse_with_options( callback, old_tree.and_then(|old_tree| old_tree.inline_trees.get(i)), + options + .inline_options + .as_mut() + .map(|b_opt| b_opt.reborrow()), )?; inline_trees.push(inline_tree); inline_indices.insert(node.id(), i); @@ -353,7 +388,11 @@ impl MarkdownParser { /// * The timeout set with [tree_sitter::Parser::set_timeout_micros] expired /// * The cancellation flag set with [tree_sitter::Parser::set_cancellation_flag] was flipped pub fn parse(&mut self, text: &[u8], old_tree: Option<&MarkdownTree>) -> Option { - self.parse_with(&mut |byte, _| &text[byte..], old_tree) + self.parse_with_options( + &mut |byte, _| &text[byte..], + old_tree, + MarkdownParseOptions::default(), + ) } } diff --git a/common/common.mak b/common/common.mak index dc78d9d..5f96de0 100644 --- a/common/common.mak +++ b/common/common.mak @@ -1,7 +1,3 @@ -ifeq ($(OS),Windows_NT) -$(error Windows is not supported) -endif - HOMEPAGE_URL := https://github.com/tree-sitter-grammars/tree-sitter-markdown VERSION := 0.5.1 @@ -12,8 +8,10 @@ TS ?= tree-sitter # install directory layout PREFIX ?= /usr/local +DATADIR ?= $(PREFIX)/share INCLUDEDIR ?= $(PREFIX)/include LIBDIR ?= $(PREFIX)/lib +BINDIR ?= $(PREFIX)/bin PCLIBDIR ?= $(LIBDIR)/pkgconfig # source/object files @@ -30,21 +28,25 @@ SONAME_MAJOR = $(shell sed -n 's/\#define LANGUAGE_VERSION //p' $(PARSER)) SONAME_MINOR = $(word 1,$(subst ., ,$(VERSION))) # OS-specific bits -ifeq ($(shell uname),Darwin) +MACHINE := $(shell $(CC) -dumpmachine) + +ifneq ($(findstring darwin,$(MACHINE)),) SOEXT = dylib SOEXTVER_MAJOR = $(SONAME_MAJOR).$(SOEXT) SOEXTVER = $(SONAME_MAJOR).$(SONAME_MINOR).$(SOEXT) LINKSHARED = -dynamiclib -Wl,-install_name,$(LIBDIR)/lib$(LANGUAGE_NAME).$(SOEXTVER),-rpath,@executable_path/../Frameworks +else ifneq ($(findstring mingw32,$(MACHINE)),) + SOEXT = dll + LINKSHARED += -s -shared -Wl,--out-implib,lib$(LANGUAGE_NAME).dll.a else SOEXT = so SOEXTVER_MAJOR = $(SOEXT).$(SONAME_MAJOR) SOEXTVER = $(SOEXT).$(SONAME_MAJOR).$(SONAME_MINOR) LINKSHARED = -shared -Wl,-soname,lib$(LANGUAGE_NAME).$(SOEXTVER) -endif ifneq ($(filter $(shell uname),FreeBSD NetBSD DragonFly),) PCLIBDIR := $(PREFIX)/libdata/pkgconfig endif - +endif all: lib$(LANGUAGE_NAME).a lib$(LANGUAGE_NAME).$(SOEXT) $(LANGUAGE_NAME).pc @@ -57,25 +59,42 @@ ifneq ($(STRIP),) $(STRIP) $@ endif +ifneq ($(findstring mingw32,$(MACHINE)),) +lib$(LANGUAGE_NAME).dll.a: lib$(LANGUAGE_NAME).$(SOEXT) +endif + $(LANGUAGE_NAME).pc: bindings/c/$(LANGUAGE_NAME).pc.in - sed -e 's|@CMAKE_PROJECT_VERSION@|$(VERSION)|' \ + sed -e 's|@PROJECT_VERSION@|$(VERSION)|' \ -e 's|@CMAKE_INSTALL_LIBDIR@|$(LIBDIR:$(PREFIX)/%=%)|' \ -e 's|@CMAKE_INSTALL_INCLUDEDIR@|$(INCLUDEDIR:$(PREFIX)/%=%)|' \ -e 's|@PROJECT_DESCRIPTION@|$(DESCRIPTION)|' \ - -e 's|@CMAKE_PROJECT_HOMEPAGE_URL@|$(HOMEPAGE_URL)|' \ - -e 's|@CMAKE_INSTALL_PREFIX@|$(PREFIX)|' \ - -e 's|@TS_REQUIRES@|$(REQUIRES)|' $< > $@ + -e 's|@PROJECT_HOMEPAGE_URL@|$(HOMEPAGE_URL)|' \ + -e 's|@CMAKE_INSTALL_PREFIX@|$(PREFIX)|' $< > $@ + +$(SRC_DIR)/grammar.json: grammar.js + $(TS) generate --no-parser $^ -$(PARSER): $(SRC_DIR)/grammar.js +$(PARSER): $(SRC_DIR)/grammar.json $(TS) generate $^ install: all - install -Dm644 bindings/c/$(LANGUAGE_NAME).h '$(DESTDIR)$(INCLUDEDIR)'/tree_sitter/$(LANGUAGE_NAME).h - install -Dm644 $(LANGUAGE_NAME).pc '$(DESTDIR)$(PCLIBDIR)'/$(LANGUAGE_NAME).pc - install -Dm755 lib$(LANGUAGE_NAME).a '$(DESTDIR)$(LIBDIR)'/lib$(LANGUAGE_NAME).a + install -d '$(DESTDIR)$(DATADIR)'/tree-sitter/queries/$(LANGUAGE_NAME) '$(DESTDIR)$(INCLUDEDIR)'/tree_sitter '$(DESTDIR)$(PCLIBDIR)' '$(DESTDIR)$(LIBDIR)' + install -m644 bindings/c/tree_sitter/$(LANGUAGE_NAME).h '$(DESTDIR)$(INCLUDEDIR)'/tree_sitter/$(LANGUAGE_NAME).h + install -m644 $(LANGUAGE_NAME).pc '$(DESTDIR)$(PCLIBDIR)'/$(LANGUAGE_NAME).pc + install -m644 lib$(LANGUAGE_NAME).a '$(DESTDIR)$(LIBDIR)'/lib$(LANGUAGE_NAME).a + install -m755 lib$(LANGUAGE_NAME).$(SOEXT) '$(DESTDIR)$(LIBDIR)'/lib$(LANGUAGE_NAME).$(SOEXTVER) +ifneq ($(findstring mingw32,$(MACHINE)),) + install -d '$(DESTDIR)$(BINDIR)' + install -m755 lib$(LANGUAGE_NAME).dll '$(DESTDIR)$(BINDIR)'/lib$(LANGUAGE_NAME).dll + install -m755 lib$(LANGUAGE_NAME).dll.a '$(DESTDIR)$(LIBDIR)'/lib$(LANGUAGE_NAME).dll.a +else install -m755 lib$(LANGUAGE_NAME).$(SOEXT) '$(DESTDIR)$(LIBDIR)'/lib$(LANGUAGE_NAME).$(SOEXTVER) - ln -sf lib$(LANGUAGE_NAME).$(SOEXTVER) '$(DESTDIR)$(LIBDIR)'/lib$(LANGUAGE_NAME).$(SOEXTVER_MAJOR) - ln -sf lib$(LANGUAGE_NAME).$(SOEXTVER_MAJOR) '$(DESTDIR)$(LIBDIR)'/lib$(LANGUAGE_NAME).$(SOEXT) + cd '$(DESTDIR)$(LIBDIR)' && ln -sf lib$(LANGUAGE_NAME).$(SOEXTVER) lib$(LANGUAGE_NAME).$(SOEXTVER_MAJOR) + cd '$(DESTDIR)$(LIBDIR)' && ln -sf lib$(LANGUAGE_NAME).$(SOEXTVER_MAJOR) lib$(LANGUAGE_NAME).$(SOEXT) +endif +ifneq ($(wildcard queries/*.scm),) + install -m644 queries/*.scm '$(DESTDIR)$(DATADIR)'/tree-sitter/queries/$(LANGUAGE_NAME) +endif uninstall: $(RM) '$(DESTDIR)$(LIBDIR)'/lib$(LANGUAGE_NAME).a \ @@ -84,9 +103,10 @@ uninstall: '$(DESTDIR)$(LIBDIR)'/lib$(LANGUAGE_NAME).$(SOEXT) \ '$(DESTDIR)$(INCLUDEDIR)'/tree_sitter/$(LANGUAGE_NAME).h \ '$(DESTDIR)$(PCLIBDIR)'/$(LANGUAGE_NAME).pc + $(RM) -r '$(DESTDIR)$(DATADIR)'/tree-sitter/queries/$(LANGUAGE_NAME) clean: - $(RM) $(OBJS) $(LANGUAGE_NAME).pc lib$(LANGUAGE_NAME).a lib$(LANGUAGE_NAME).$(SOEXT) + $(RM) $(OBJS) $(LANGUAGE_NAME).pc lib$(LANGUAGE_NAME).a lib$(LANGUAGE_NAME).$(SOEXT) lib$(LANGUAGE_NAME).dll.a test: $(TS) test diff --git a/go.mod b/go.mod deleted file mode 100644 index 9ff2f6b..0000000 --- a/go.mod +++ /dev/null @@ -1,9 +0,0 @@ -module github.com/tree-sitter-grammars/tree-sitter-markdown - -go 1.23 - -toolchain go1.23.0 - -require github.com/tree-sitter/go-tree-sitter v0.23.1 - -require github.com/mattn/go-pointer v0.0.1 // indirect diff --git a/pyproject.toml b/pyproject.toml index 081d451..57abad9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,7 +17,7 @@ classifiers = [ authors = [ {name = "MDeiml"} ] -requires-python = ">=3.9" +requires-python = ">=3.10" license.text = "MIT" readme = "README.md" @@ -28,5 +28,5 @@ Homepage = "https://github.com/tree-sitter-grammars/tree-sitter-markdown" core = ["tree-sitter~=0.23"] [tool.cibuildwheel] -build = "cp39-*" +build = "cp310-*" build-frontend = "build" diff --git a/setup.py b/setup.py index aac54d7..c791ca4 100644 --- a/setup.py +++ b/setup.py @@ -1,30 +1,49 @@ -from pathlib import Path -from platform import system +from os import path +from sysconfig import get_config_var from setuptools import Extension, find_packages, setup from setuptools.command.build import build +from setuptools.command.build_ext import build_ext +from setuptools.command.egg_info import egg_info from wheel.bdist_wheel import bdist_wheel class Build(build): def run(self): - if (block_queries := Path("tree-sitter-markdown", "queries")).is_dir(): - dest = Path(self.build_lib, "tree_sitter_markdown", "queries", "markdown") - self.copy_tree(str(block_queries), str(dest)) - if (inline_queries := Path("tree-sitter-markdown-inline", "queries")).is_dir(): - dest = Path(self.build_lib, "tree_sitter_markdown", "queries", "markdown_inline") - self.copy_tree(str(inline_queries), str(dest)) + if path.isdir("queries"): + dest = path.join(self.build_lib, "tree_sitter_markdown", "queries") + self.copy_tree("queries", dest) super().run() +class BuildExt(build_ext): + def build_extension(self, ext: Extension): + if self.compiler.compiler_type != "msvc": + ext.extra_compile_args = ["-std=c11", "-fvisibility=hidden"] + else: + ext.extra_compile_args = ["/std:c11", "/utf-8"] + if path.exists("src/scanner.c"): + ext.sources.append("src/scanner.c") + if ext.py_limited_api: + ext.define_macros.append(("Py_LIMITED_API", "0x030A0000")) + super().build_extension(ext) + + class BdistWheel(bdist_wheel): def get_tag(self): python, abi, platform = super().get_tag() if python.startswith("cp"): - python, abi = "cp39", "abi3" + python, abi = "cp310", "abi3" return python, abi, platform +class EggInfo(egg_info): + def find_sources(self): + super().find_sources() + self.filelist.recursive_include("queries", "*.scm") + self.filelist.include("src/tree_sitter/*.h") + + setup( packages=find_packages("bindings/python"), package_dir={"": "bindings/python"}, @@ -38,25 +57,21 @@ def get_tag(self): name="_binding", sources=[ "bindings/python/tree_sitter_markdown/binding.c", - "tree-sitter-markdown/src/parser.c", - "tree-sitter-markdown/src/scanner.c", - "tree-sitter-markdown-inline/src/parser.c", - "tree-sitter-markdown-inline/src/scanner.c", + "src/parser.c", ], - extra_compile_args=( - ["-std=c11"] if system() != "Windows" else [] - ), define_macros=[ - ("Py_LIMITED_API", "0x03090000"), - ("PY_SSIZE_T_CLEAN", None) + ("PY_SSIZE_T_CLEAN", None), + ("TREE_SITTER_HIDE_SYMBOLS", None), ], - include_dirs=["tree-sitter-markdown/src"], - py_limited_api=True, + include_dirs=["src"], + py_limited_api=not get_config_var("Py_GIL_DISABLED"), ) ], cmdclass={ "build": Build, - "bdist_wheel": BdistWheel + "build_ext": BuildExt, + "bdist_wheel": BdistWheel, + "egg_info": EggInfo, }, zip_safe=False ) diff --git a/tree-sitter-markdown-inline/bindings/c/tree-sitter-markdown-inline.h b/tree-sitter-markdown-inline/bindings/c/tree-sitter/tree-sitter-markdown-inline.h similarity index 100% rename from tree-sitter-markdown-inline/bindings/c/tree-sitter-markdown-inline.h rename to tree-sitter-markdown-inline/bindings/c/tree-sitter/tree-sitter-markdown-inline.h diff --git a/tree-sitter-markdown-inline/queries/highlights.scm b/tree-sitter-markdown-inline/queries/highlights.scm index 8c87c78..ee71396 100644 --- a/tree-sitter-markdown-inline/queries/highlights.scm +++ b/tree-sitter-markdown-inline/queries/highlights.scm @@ -1,4 +1,4 @@ -;; From nvim-treesitter/nvim-treesitter +; From nvim-treesitter/nvim-treesitter [ (code_span) (link_title) @@ -29,9 +29,28 @@ (hard_line_break) ] @string.escape -(image ["!" "[" "]" "(" ")"] @punctuation.delimiter) -(inline_link ["[" "]" "(" ")"] @punctuation.delimiter) -(shortcut_link ["[" "]"] @punctuation.delimiter) +(image + [ + "!" + "[" + "]" + "(" + ")" + ] @punctuation.delimiter) + +(inline_link + [ + "[" + "]" + "(" + ")" + ] @punctuation.delimiter) + +(shortcut_link + [ + "[" + "]" + ] @punctuation.delimiter) ; NOTE: extension not enabled by default ; (wiki_link ["[" "|" "]"] @punctuation.delimiter) diff --git a/tree-sitter-markdown-inline/queries/injections.scm b/tree-sitter-markdown-inline/queries/injections.scm index 9a0b8cd..0f29635 100644 --- a/tree-sitter-markdown-inline/queries/injections.scm +++ b/tree-sitter-markdown-inline/queries/injections.scm @@ -1,2 +1,5 @@ -((html_tag) @injection.content (#set! injection.language "html")) -((latex_block) @injection.content (#set! injection.language "latex")) +((html_tag) @injection.content + (#set! injection.language "html")) + +((latex_block) @injection.content + (#set! injection.language "latex")) diff --git a/tree-sitter-markdown-inline/src/parser.c b/tree-sitter-markdown-inline/src/parser.c index 5cce477..ee22b48 100644 --- a/tree-sitter-markdown-inline/src/parser.c +++ b/tree-sitter-markdown-inline/src/parser.c @@ -1,4 +1,4 @@ -/* Automatically @generated by tree-sitter v0.25.9 */ +/* Automatically @generated by tree-sitter */ #include "tree_sitter/parser.h" diff --git a/tree-sitter-markdown/bindings/c/tree-sitter-markdown.h b/tree-sitter-markdown/bindings/c/tree-sitter/tree-sitter-markdown.h similarity index 100% rename from tree-sitter-markdown/bindings/c/tree-sitter-markdown.h rename to tree-sitter-markdown/bindings/c/tree-sitter/tree-sitter-markdown.h diff --git a/tree-sitter-markdown/queries/highlights.scm b/tree-sitter-markdown/queries/highlights.scm index 3c2a896..21962fd 100644 --- a/tree-sitter-markdown/queries/highlights.scm +++ b/tree-sitter-markdown/queries/highlights.scm @@ -1,6 +1,9 @@ ;From nvim-treesitter/nvim-treesitter -(atx_heading (inline) @text.title) -(setext_heading (paragraph) @text.title) +(atx_heading + (inline) @text.title) + +(setext_heading + (paragraph) @text.title) [ (atx_h1_marker) @@ -19,19 +22,13 @@ (fenced_code_block) ] @text.literal -[ - (fenced_code_block_delimiter) -] @punctuation.delimiter +(fenced_code_block_delimiter) @punctuation.delimiter (code_fence_content) @none -[ - (link_destination) -] @text.uri +(link_destination) @text.uri -[ - (link_label) -] @text.reference +(link_label) @text.reference [ (list_marker_plus) @@ -47,6 +44,4 @@ (block_quote_marker) ] @punctuation.special -[ - (backslash_escape) -] @string.escape +(backslash_escape) @string.escape diff --git a/tree-sitter-markdown/queries/injections.scm b/tree-sitter-markdown/queries/injections.scm index 809b618..7a8c34a 100644 --- a/tree-sitter-markdown/queries/injections.scm +++ b/tree-sitter-markdown/queries/injections.scm @@ -3,12 +3,23 @@ (language) @injection.language) (code_fence_content) @injection.content) -((html_block) @injection.content (#set! injection.language "html")) +((html_block) @injection.content + (#set! injection.language "html")) -(document . (section . (thematic_break) (_) @injection.content (thematic_break)) (#set! injection.language "yaml")) +(document + . + (section + . + (thematic_break) + (_) @injection.content + (thematic_break)) + (#set! injection.language "yaml")) -((minus_metadata) @injection.content (#set! injection.language "yaml")) +((minus_metadata) @injection.content + (#set! injection.language "yaml")) -((plus_metadata) @injection.content (#set! injection.language "toml")) +((plus_metadata) @injection.content + (#set! injection.language "toml")) -((inline) @injection.content (#set! injection.language "markdown_inline")) +((inline) @injection.content + (#set! injection.language "markdown_inline")) diff --git a/tree-sitter-markdown/src/parser.c b/tree-sitter-markdown/src/parser.c index b5f214c..113bdd1 100644 --- a/tree-sitter-markdown/src/parser.c +++ b/tree-sitter-markdown/src/parser.c @@ -1,4 +1,4 @@ -/* Automatically @generated by tree-sitter v0.25.9 */ +/* Automatically @generated by tree-sitter */ #include "tree_sitter/parser.h" diff --git a/tree-sitter.json b/tree-sitter.json index 3149560..8aa4202 100644 --- a/tree-sitter.json +++ b/tree-sitter.json @@ -38,5 +38,15 @@ "common/common.js" ] } - ] + ], + "bindings": { + "rust": true, + "node": true, + "c": true, + "python": true, + "swift": true, + "go": false, + "java": false, + "zig": false + } }