diff --git a/.gitattributes b/.gitattributes
index cee6011..5c345d8 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -8,7 +8,7 @@
*/src/tree_sitter/* linguist-generated
# C bindings
-*/bindings/c/* linguist-generated
+*/bindings/c/** linguist-generated
# Rust bindings
bindings/rust/build.rs linguist-generated
@@ -35,3 +35,7 @@ go.sum linguist-generated
*/bindings/swift/** linguist-generated
Package.swift linguist-generated
Package.resolved linguist-generated
+
+# Zig bindings
+build.zig linguist-generated
+build.zig.zon linguist-generated
diff --git a/.github/dependabot.yml b/.github/dependabot.yml
new file mode 100644
index 0000000..e806045
--- /dev/null
+++ b/.github/dependabot.yml
@@ -0,0 +1,16 @@
+version: 2
+updates:
+ - package-ecosystem: "github-actions"
+ directory: "/"
+ schedule:
+ interval: "weekly"
+ cooldown:
+ default-days: 3
+ commit-message:
+ prefix: "ci"
+ labels:
+ - "CI"
+ groups:
+ actions:
+ patterns: ["*"]
+
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 4cfc0d3..22ab869 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -21,15 +21,13 @@ jobs:
os: [ubuntu-latest, windows-latest, macos-latest]
steps:
- name: Checkout repository
- uses: actions/checkout@v5
+ uses: actions/checkout@v6
- name: Set up tree-sitter
- uses: tree-sitter/setup-action@v2
- with:
- install-lib: false
+ uses: tree-sitter/setup-action/cli@v2
- name: Set up Node.js
- uses: actions/setup-node@v4
+ uses: actions/setup-node@v6
with:
node-version: ${{vars.NODE_VERSION}}
@@ -58,7 +56,7 @@ jobs:
parser: [tree-sitter-markdown, tree-sitter-markdown-inline]
steps:
- name: Checkout repository
- uses: actions/checkout@v4
+ uses: actions/checkout@v6
with:
fetch-depth: 2
- name: Check for scanner changes
@@ -74,3 +72,26 @@ jobs:
if: steps.scanner-changes.outputs.changed == 'true'
with:
directory: ${{matrix.parser}}
+
+ query:
+ name: Validate queries
+ runs-on: ubuntu-latest
+ strategy:
+ fail-fast: false
+ matrix:
+ parser: [markdown, markdown-inline]
+ steps:
+ - name: Set up repository
+ uses: actions/checkout@v6
+
+ - name: Set up tree-sitter
+ uses: tree-sitter/setup-action/cli@v2
+
+ - name: Build parser
+ run: tree-sitter build tree-sitter-${{matrix.parser}}
+
+ - name: Set up ts_query_ls
+ run: curl -fL https://github.com/ribru17/ts_query_ls/releases/latest/download/ts_query_ls-x86_64-unknown-linux-gnu.tar.gz | tar -xz
+
+ - name: Check queries
+ run: ./ts_query_ls check -f tree-sitter-${{matrix.parser}}/queries/
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index b677185..5a39296 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -17,5 +17,3 @@ permissions:
jobs:
release:
uses: tree-sitter/workflows/.github/workflows/release.yml@main
- with:
- attestations: true
diff --git a/.gitignore b/.gitignore
index 623310c..7c0cb7f 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,18 +1,15 @@
# Rust artifacts
-Cargo.lock
target/
# Node artifacts
build/
prebuilds/
node_modules/
-*.tgz
# Swift artifacts
.build/
# Go artifacts
-go.sum
_obj/
# Python artifacts
@@ -28,6 +25,13 @@ dist/
*.dylib
*.dll
*.pc
+*.exp
+*.lib
+
+# Zig artifacts
+.zig-cache/
+zig-cache/
+zig-out/
# Example dirs
/examples/*/
@@ -37,7 +41,8 @@ dist/
*.obj
*.o
-# Moved bindings
-bindings/c/
-bindings/swift/TreeSitterMarkdown/
-bindings/swift/TreeSitterMarkdownInline/
+# Archives
+*.tar.gz
+*.tgz
+*.zip
+*.jar
diff --git a/.tsqueryrc.json b/.tsqueryrc.json
new file mode 100644
index 0000000..a7a343a
--- /dev/null
+++ b/.tsqueryrc.json
@@ -0,0 +1,5 @@
+{
+ "$schema": "https://raw.githubusercontent.com/ribru17/ts_query_ls/refs/heads/master/schemas/config.json",
+ "parser_install_directories": ["."],
+ "language_retrieval_patterns": ["tree-sitter-([^/]+)/queries/[^/]+\\.scm$"]
+}
diff --git a/Cargo.toml b/Cargo.toml
index bd03c73..c24f360 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -32,13 +32,13 @@ path = "bindings/rust/lib.rs"
[dependencies]
tree-sitter-language = "0.1"
-tree-sitter = { version = "0.24", optional = true }
+tree-sitter = { version = "0.26", optional = true }
[dev-dependencies]
-tree-sitter = "0.24.3"
+tree-sitter = "0.26.3"
[build-dependencies]
-cc = "1.1.22"
+cc = "1.2"
[[bin]]
name = "benchmark"
diff --git a/Package.swift b/Package.swift
index 375750c..4f6d2f1 100644
--- a/Package.swift
+++ b/Package.swift
@@ -8,7 +8,7 @@ let package = Package(
.library(name: "TreeSitterMarkdown", targets: ["TreeSitterMarkdown", "TreeSitterMarkdownInline"]),
],
dependencies: [
- .package(url: "https://github.com/ChimeHQ/SwiftTreeSitter", from: "0.8.0"),
+ .package(name: "SwiftTreeSitter", url: "https://github.com/tree-sitter/swift-tree-sitter", from: "0.8.0"),
],
targets: [
.target(
diff --git a/bindings/go/binding_test.go b/bindings/go/binding_test.go
deleted file mode 100644
index b2381f6..0000000
--- a/bindings/go/binding_test.go
+++ /dev/null
@@ -1,22 +0,0 @@
-package tree_sitter_markdown_test
-
-import (
- "testing"
-
- tree_sitter "github.com/tree-sitter/go-tree-sitter"
- tree_sitter_markdown "github.com/tree-sitter-grammars/tree-sitter-markdown/bindings/go"
-)
-
-func TestCanLoadBlockGrammar(t *testing.T) {
- language := tree_sitter.NewLanguage(tree_sitter_markdown.Language())
- if language == nil {
- t.Errorf("Error loading Markdown block grammar")
- }
-}
-
-func TestCanLoadInlineGrammar(t *testing.T) {
- language := tree_sitter.NewLanguage(tree_sitter_markdown.InlineLanguage())
- if language == nil {
- t.Errorf("Error loading Markdown inline grammar")
- }
-}
diff --git a/bindings/go/markdown.go b/bindings/go/markdown.go
deleted file mode 100644
index f38fda6..0000000
--- a/bindings/go/markdown.go
+++ /dev/null
@@ -1,14 +0,0 @@
-package tree_sitter_markdown
-
-// #cgo CPPFLAGS: -I../../tree-sitter-markdown
-// #cgo CFLAGS: -std=c11 -fPIC
-// #include "../../tree-sitter-markdown/src/parser.c"
-// #include "../../tree-sitter-markdown/src/scanner.c"
-import "C"
-
-import "unsafe"
-
-// Get the tree-sitter Language for the block grammar.
-func Language() unsafe.Pointer {
- return unsafe.Pointer(C.tree_sitter_markdown())
-}
diff --git a/bindings/go/markdown_inline.go b/bindings/go/markdown_inline.go
deleted file mode 100644
index f779990..0000000
--- a/bindings/go/markdown_inline.go
+++ /dev/null
@@ -1,14 +0,0 @@
-package tree_sitter_markdown
-
-// #cgo CPPFLAGS: -I../../tree-sitter-markdown-inline
-// #cgo CFLAGS: -std=c11 -fPIC
-// #include "../../tree-sitter-markdown-inline/src/parser.c"
-// #include "../../tree-sitter-markdown-inline/src/scanner.c"
-import "C"
-
-import "unsafe"
-
-// Get the tree-sitter Language for the inline grammar.
-func InlineLanguage() unsafe.Pointer {
- return unsafe.Pointer(C.tree_sitter_markdown_inline())
-}
diff --git a/bindings/node/binding_test.js b/bindings/node/binding_test.js
index cbba7a6..7a91a84 100644
--- a/bindings/node/binding_test.js
+++ b/bindings/node/binding_test.js
@@ -1,16 +1,11 @@
-///
+import assert from "node:assert";
+import { test } from "node:test";
+import Parser from "tree-sitter";
-const assert = require("node:assert");
-const { test } = require("node:test");
-
-const Parser = require("tree-sitter");
-
-test("can load block grammar", () => {
- const parser = new Parser();
- assert.doesNotThrow(() => parser.setLanguage(require(".")));
-});
-
-test("can load inline grammar", () => {
+test("can load grammar", () => {
const parser = new Parser();
- assert.doesNotThrow(() => parser.setLanguage(require(".").inline));
+ assert.doesNotReject(async () => {
+ const { default: language } = await import("./index.js");
+ parser.setLanguage(language);
+ });
});
diff --git a/bindings/node/index.d.ts b/bindings/node/index.d.ts
index 299635b..d75fe74 100644
--- a/bindings/node/index.d.ts
+++ b/bindings/node/index.d.ts
@@ -18,13 +18,43 @@ type NodeInfo =
children: ChildNode[];
});
-type Language = {
- name: string;
+/**
+ * The tree-sitter language object for this grammar.
+ *
+ * @see {@linkcode https://tree-sitter.github.io/node-tree-sitter/interfaces/Parser.Language.html Parser.Language}
+ *
+ * @example
+ * import Parser from "tree-sitter";
+ * import Markdown from "tree-sitter-markdown";
+ *
+ * const parser = new Parser();
+ * parser.setLanguage(Markdown);
+ */
+declare const binding: {
+ /**
+ * The inner language object.
+ * @private
+ */
language: unknown;
+
+ /**
+ * The content of the `node-types.json` file for this grammar.
+ *
+ * @see {@linkplain https://tree-sitter.github.io/tree-sitter/using-parsers/6-static-node-types Static Node Types}
+ */
nodeTypeInfo: NodeInfo[];
-};
-declare const _exports: Language & {
- inline: Language
+ /** The syntax highlighting query for this grammar. */
+ HIGHLIGHTS_QUERY?: string;
+
+ /** The language injection query for this grammar. */
+ INJECTIONS_QUERY?: string;
+
+ /** The local variable query for this grammar. */
+ LOCALS_QUERY?: string;
+
+ /** The symbol tagging query for this grammar. */
+ TAGS_QUERY?: string;
};
-export = _exports;
+
+export default binding;
diff --git a/bindings/node/index.js b/bindings/node/index.js
index 6452853..e4e8e21 100644
--- a/bindings/node/index.js
+++ b/bindings/node/index.js
@@ -1,8 +1,37 @@
-const root = require("path").join(__dirname, "..", "..");
+import { readFileSync } from "node:fs";
+import { fileURLToPath } from "node:url";
-module.exports = require("node-gyp-build")(root);
+const root = fileURLToPath(new URL("../..", import.meta.url));
+
+const binding = typeof process.versions.bun === "string"
+ // Support `bun build --compile` by being statically analyzable enough to find the .node file at build-time
+ ? await import(`${root}/prebuilds/${process.platform}-${process.arch}/tree-sitter-markdown.node`)
+ : (await import("node-gyp-build")).default(root);
try {
- module.exports.nodeTypeInfo = require("../../tree-sitter-markdown/src/node-types.json");
- module.exports.inline.nodeTypeInfo = require("../../tree-sitter-markdown-inline/src/node-types.json");
-} catch (_) { }
+ const nodeTypes = await import(`${root}/src/node-types.json`, { with: { type: "json" } });
+ binding.nodeTypeInfo = nodeTypes.default;
+} catch { }
+
+const queries = [
+ ["HIGHLIGHTS_QUERY", `${root}/tree-sitter-markdown/queries/highlights.scm`],
+ ["INJECTIONS_QUERY", `${root}/tree-sitter-markdown/queries/injections.scm`],
+ ["LOCALS_QUERY", `${root}/queries/locals.scm`],
+ ["TAGS_QUERY", `${root}/queries/tags.scm`],
+];
+
+for (const [prop, path] of queries) {
+ Object.defineProperty(binding, prop, {
+ configurable: true,
+ enumerable: true,
+ get() {
+ delete binding[prop];
+ try {
+ binding[prop] = readFileSync(path, "utf8");
+ } catch { }
+ return binding[prop];
+ }
+ });
+}
+
+export default binding;
diff --git a/bindings/python/tests/test_binding.py b/bindings/python/tests/test_binding.py
index 856599a..a8c3b03 100644
--- a/bindings/python/tests/test_binding.py
+++ b/bindings/python/tests/test_binding.py
@@ -6,12 +6,12 @@
class TestLanguage(TestCase):
def test_can_load_block_grammar(self):
try:
- tree_sitter.Language(tree_sitter_markdown.language())
+ Parser(Language(tree_sitter_markdown.language()))
except Exception:
self.fail("Error loading Markdown block grammar")
def test_can_load_block_grammar(self):
try:
- tree_sitter.Language(tree_sitter_markdown.inline_language())
+ Parser(Language(tree_sitter_markdown.inline_language())
except Exception:
self.fail("Error loading Markdown inline grammar")
diff --git a/bindings/python/tree_sitter_markdown/__init__.py b/bindings/python/tree_sitter_markdown/__init__.py
index 08a73e9..d64fec5 100644
--- a/bindings/python/tree_sitter_markdown/__init__.py
+++ b/bindings/python/tree_sitter_markdown/__init__.py
@@ -1,5 +1,43 @@
-"Markdown grammar for tree-sitter"
+"""Markdown grammar for tree-sitter"""
-from ._binding import language, inline_language
+from importlib.resources import files as _files
-__all__ = ["language", "inline_language"]
+from ._binding import language
+
+
+def _get_query(name, file):
+ try:
+ query = _files(f"{__package__}") / file
+ globals()[name] = query.read_text()
+ except FileNotFoundError:
+ globals()[name] = None
+ return globals()[name]
+
+
+def __getattr__(name):
+ if name == "HIGHLIGHTS_QUERY":
+ return _get_query("HIGHLIGHTS_QUERY", "tree-sitter-markdown/queries/highlights.scm")
+ if name == "INJECTIONS_QUERY":
+ return _get_query("INJECTIONS_QUERY", "tree-sitter-markdown/queries/injections.scm")
+ if name == "LOCALS_QUERY":
+ return _get_query("LOCALS_QUERY", "queries/locals.scm")
+ if name == "TAGS_QUERY":
+ return _get_query("TAGS_QUERY", "queries/tags.scm")
+
+ raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
+
+
+__all__ = [
+ "language",
+ "HIGHLIGHTS_QUERY",
+ "INJECTIONS_QUERY",
+ "LOCALS_QUERY",
+ "TAGS_QUERY",
+]
+
+
+def __dir__():
+ return sorted(__all__ + [
+ "__all__", "__builtins__", "__cached__", "__doc__", "__file__",
+ "__loader__", "__name__", "__package__", "__path__", "__spec__",
+ ])
diff --git a/bindings/python/tree_sitter_markdown/__init__.pyi b/bindings/python/tree_sitter_markdown/__init__.pyi
index 0acdc0a..a99f688 100644
--- a/bindings/python/tree_sitter_markdown/__init__.pyi
+++ b/bindings/python/tree_sitter_markdown/__init__.pyi
@@ -1,3 +1,3 @@
-def language() -> object: ...
+def language() -> CapsuleType: ...
-def inline_language() -> object: ...
+def inline_language() -> CapsuleType: ...
diff --git a/bindings/python/tree_sitter_markdown/binding.c b/bindings/python/tree_sitter_markdown/binding.c
index f9973f7..2a9d86b 100644
--- a/bindings/python/tree_sitter_markdown/binding.c
+++ b/bindings/python/tree_sitter_markdown/binding.c
@@ -14,6 +14,13 @@ static PyObject* _binding_inline_language(PyObject *Py_UNUSED(self), PyObject *P
return PyCapsule_New(tree_sitter_markdown_inline(), "tree_sitter.Language", NULL);
}
+static struct PyModuleDef_Slot slots[] = {
+#ifdef Py_GIL_DISABLED
+ {Py_mod_gil, Py_MOD_GIL_NOT_USED},
+#endif
+ {0, NULL}
+};
+
static PyMethodDef methods[] = {
{"language", _binding_language, METH_NOARGS,
"Get the tree-sitter language for the block grammar."},
@@ -26,10 +33,11 @@ static struct PyModuleDef module = {
.m_base = PyModuleDef_HEAD_INIT,
.m_name = "_binding",
.m_doc = NULL,
- .m_size = -1,
- .m_methods = methods
+ .m_size = 0,
+ .m_methods = methods,
+ .m_slots = slots,
};
PyMODINIT_FUNC PyInit__binding(void) {
- return PyModule_Create(&module);
+ return PyModuleDef_Init(&module);
}
diff --git a/bindings/rust/build.rs b/bindings/rust/build.rs
index c9f5943..58939aa 100644
--- a/bindings/rust/build.rs
+++ b/bindings/rust/build.rs
@@ -8,6 +8,24 @@ fn main() {
#[cfg(target_env = "msvc")]
c_config.flag("-utf-8");
+ if std::env::var("TARGET").unwrap() == "wasm32-unknown-unknown" {
+ let Ok(wasm_headers) = std::env::var("DEP_TREE_SITTER_LANGUAGE_WASM_HEADERS") else {
+ panic!("Environment variable DEP_TREE_SITTER_LANGUAGE_WASM_HEADERS must be set by the language crate");
+ };
+ let Ok(wasm_src) =
+ std::env::var("DEP_TREE_SITTER_LANGUAGE_WASM_SRC").map(std::path::PathBuf::from)
+ else {
+ panic!("Environment variable DEP_TREE_SITTER_LANGUAGE_WASM_SRC must be set by the language crate");
+ };
+
+ c_config.include(&wasm_headers);
+ c_config.files([
+ wasm_src.join("stdio.c"),
+ wasm_src.join("stdlib.c"),
+ wasm_src.join("string.c"),
+ ]);
+ }
+
for path in &[
block_dir.join("parser.c"),
block_dir.join("scanner.c"),
@@ -19,4 +37,25 @@ fn main() {
}
c_config.compile("tree-sitter-markdown");
+
+ println!("cargo:rustc-check-cfg=cfg(with_highlights_query)");
+ if !"tree-sitter-markdown/queries/highlights.scm".is_empty()
+ && std::path::Path::new("tree-sitter-markdown/queries/highlights.scm").exists()
+ {
+ println!("cargo:rustc-cfg=with_highlights_query");
+ }
+ println!("cargo:rustc-check-cfg=cfg(with_injections_query)");
+ if !"tree-sitter-markdown/queries/injections.scm".is_empty()
+ && std::path::Path::new("tree-sitter-markdown/queries/injections.scm").exists()
+ {
+ println!("cargo:rustc-cfg=with_injections_query");
+ }
+ println!("cargo:rustc-check-cfg=cfg(with_locals_query)");
+ if !"queries/locals.scm".is_empty() && std::path::Path::new("queries/locals.scm").exists() {
+ println!("cargo:rustc-cfg=with_locals_query");
+ }
+ println!("cargo:rustc-check-cfg=cfg(with_tags_query)");
+ if !"queries/tags.scm".is_empty() && std::path::Path::new("queries/tags.scm").exists() {
+ println!("cargo:rustc-cfg=with_tags_query");
+ }
}
diff --git a/bindings/rust/parser.rs b/bindings/rust/parser.rs
index 2fd8d5e..4a17618 100644
--- a/bindings/rust/parser.rs
+++ b/bindings/rust/parser.rs
@@ -1,7 +1,9 @@
use std::collections::HashMap;
use std::num::NonZeroU16;
-use tree_sitter::{InputEdit, Language, Node, Parser, Point, Range, Tree, TreeCursor};
+use tree_sitter::{
+ InputEdit, Language, Node, ParseOptions, Parser, Point, Range, Tree, TreeCursor,
+};
use crate::{INLINE_LANGUAGE, LANGUAGE};
@@ -16,7 +18,7 @@ pub struct MarkdownParser {
/// A stateful object for walking a [`MarkdownTree`] efficiently.
///
-/// This exposes the same methdos as [`TreeCursor`], but abstracts away the
+/// This exposes the same methods as [`TreeCursor`], but abstracts away the
/// double block / inline structure of [`MarkdownTree`].
pub struct MarkdownCursor<'a> {
markdown_tree: &'a MarkdownTree,
@@ -33,7 +35,7 @@ impl<'a> MarkdownCursor<'a> {
}
}
- /// Returns `true` if the current node is from the (inline language)[INLINE_LANGUAGE]
+ /// Returns `true` if the current node is from the [INLINE_LANGUAGE](inline language)
///
/// This information is needed to handle "tree-sitter internal" data like
/// [`field_id`](Self::field_id) correctly.
@@ -221,7 +223,7 @@ impl MarkdownTree {
}
/// Create a new [`MarkdownCursor`] starting from the root of the tree.
- pub fn walk(&self) -> MarkdownCursor {
+ pub fn walk(&self) -> MarkdownCursor<'_> {
MarkdownCursor {
markdown_tree: self,
block_cursor: self.block_tree.walk(),
@@ -230,6 +232,28 @@ impl MarkdownTree {
}
}
+/// The options used while parsing a [`MarkdownTree`].
+///
+/// This abstracts away the double block / inline structure of [`MarkdownParser`].
+#[derive(Default)]
+pub struct MarkdownParseOptions<'a> {
+ block_options: Option>,
+ inline_options: Option>,
+}
+
+impl<'a> MarkdownParseOptions<'a> {
+ /// Creates a new [MarkdownParseOptions] instance.
+ pub fn new(
+ block_options: Option>,
+ inline_options: Option>,
+ ) -> Self {
+ MarkdownParseOptions {
+ block_options,
+ inline_options,
+ }
+ }
+}
+
impl Default for MarkdownParser {
fn default() -> Self {
let block_language = LANGUAGE.into();
@@ -252,14 +276,17 @@ impl MarkdownParser {
/// If the text of the document has changed since `old_tree` was
/// created, then you must edit `old_tree` to match the new text using
/// [MarkdownTree::edit].
+ /// * `options` The [options][MarkdownParseOptions] used for parsing.
+ /// Use `MarkdownParseOptions::default()` if you don't need to pass any options.
///
/// Returns a [MarkdownTree] if parsing succeeded, or `None` if:
/// * The timeout set with [tree_sitter::Parser::set_timeout_micros] expired
/// * The cancellation flag set with [tree_sitter::Parser::set_cancellation_flag] was flipped
- pub fn parse_with, F: FnMut(usize, Point) -> T>(
+ pub fn parse_with_options, F: FnMut(usize, Point) -> T>(
&mut self,
callback: &mut F,
old_tree: Option<&MarkdownTree>,
+ mut options: MarkdownParseOptions<'_>,
) -> Option {
let MarkdownParser {
parser,
@@ -272,7 +299,11 @@ impl MarkdownParser {
parser
.set_language(block_language)
.expect("Could not load block grammar");
- let block_tree = parser.parse_with(callback, old_tree.map(|tree| &tree.block_tree))?;
+ let block_tree = parser.parse_with_options(
+ callback,
+ old_tree.map(|tree| &tree.block_tree),
+ options.block_options.as_mut().map(|b_opt| b_opt.reborrow()),
+ )?;
let (mut inline_trees, mut inline_indices) = if let Some(old_tree) = old_tree {
let len = old_tree.inline_trees.len();
(Vec::with_capacity(len), HashMap::with_capacity(len))
@@ -322,9 +353,13 @@ impl MarkdownParser {
}
ranges.push(range);
parser.set_included_ranges(&ranges).ok()?;
- let inline_tree = parser.parse_with(
+ let inline_tree = parser.parse_with_options(
callback,
old_tree.and_then(|old_tree| old_tree.inline_trees.get(i)),
+ options
+ .inline_options
+ .as_mut()
+ .map(|b_opt| b_opt.reborrow()),
)?;
inline_trees.push(inline_tree);
inline_indices.insert(node.id(), i);
@@ -353,7 +388,11 @@ impl MarkdownParser {
/// * The timeout set with [tree_sitter::Parser::set_timeout_micros] expired
/// * The cancellation flag set with [tree_sitter::Parser::set_cancellation_flag] was flipped
pub fn parse(&mut self, text: &[u8], old_tree: Option<&MarkdownTree>) -> Option {
- self.parse_with(&mut |byte, _| &text[byte..], old_tree)
+ self.parse_with_options(
+ &mut |byte, _| &text[byte..],
+ old_tree,
+ MarkdownParseOptions::default(),
+ )
}
}
diff --git a/common/common.mak b/common/common.mak
index dc78d9d..5f96de0 100644
--- a/common/common.mak
+++ b/common/common.mak
@@ -1,7 +1,3 @@
-ifeq ($(OS),Windows_NT)
-$(error Windows is not supported)
-endif
-
HOMEPAGE_URL := https://github.com/tree-sitter-grammars/tree-sitter-markdown
VERSION := 0.5.1
@@ -12,8 +8,10 @@ TS ?= tree-sitter
# install directory layout
PREFIX ?= /usr/local
+DATADIR ?= $(PREFIX)/share
INCLUDEDIR ?= $(PREFIX)/include
LIBDIR ?= $(PREFIX)/lib
+BINDIR ?= $(PREFIX)/bin
PCLIBDIR ?= $(LIBDIR)/pkgconfig
# source/object files
@@ -30,21 +28,25 @@ SONAME_MAJOR = $(shell sed -n 's/\#define LANGUAGE_VERSION //p' $(PARSER))
SONAME_MINOR = $(word 1,$(subst ., ,$(VERSION)))
# OS-specific bits
-ifeq ($(shell uname),Darwin)
+MACHINE := $(shell $(CC) -dumpmachine)
+
+ifneq ($(findstring darwin,$(MACHINE)),)
SOEXT = dylib
SOEXTVER_MAJOR = $(SONAME_MAJOR).$(SOEXT)
SOEXTVER = $(SONAME_MAJOR).$(SONAME_MINOR).$(SOEXT)
LINKSHARED = -dynamiclib -Wl,-install_name,$(LIBDIR)/lib$(LANGUAGE_NAME).$(SOEXTVER),-rpath,@executable_path/../Frameworks
+else ifneq ($(findstring mingw32,$(MACHINE)),)
+ SOEXT = dll
+ LINKSHARED += -s -shared -Wl,--out-implib,lib$(LANGUAGE_NAME).dll.a
else
SOEXT = so
SOEXTVER_MAJOR = $(SOEXT).$(SONAME_MAJOR)
SOEXTVER = $(SOEXT).$(SONAME_MAJOR).$(SONAME_MINOR)
LINKSHARED = -shared -Wl,-soname,lib$(LANGUAGE_NAME).$(SOEXTVER)
-endif
ifneq ($(filter $(shell uname),FreeBSD NetBSD DragonFly),)
PCLIBDIR := $(PREFIX)/libdata/pkgconfig
endif
-
+endif
all: lib$(LANGUAGE_NAME).a lib$(LANGUAGE_NAME).$(SOEXT) $(LANGUAGE_NAME).pc
@@ -57,25 +59,42 @@ ifneq ($(STRIP),)
$(STRIP) $@
endif
+ifneq ($(findstring mingw32,$(MACHINE)),)
+lib$(LANGUAGE_NAME).dll.a: lib$(LANGUAGE_NAME).$(SOEXT)
+endif
+
$(LANGUAGE_NAME).pc: bindings/c/$(LANGUAGE_NAME).pc.in
- sed -e 's|@CMAKE_PROJECT_VERSION@|$(VERSION)|' \
+ sed -e 's|@PROJECT_VERSION@|$(VERSION)|' \
-e 's|@CMAKE_INSTALL_LIBDIR@|$(LIBDIR:$(PREFIX)/%=%)|' \
-e 's|@CMAKE_INSTALL_INCLUDEDIR@|$(INCLUDEDIR:$(PREFIX)/%=%)|' \
-e 's|@PROJECT_DESCRIPTION@|$(DESCRIPTION)|' \
- -e 's|@CMAKE_PROJECT_HOMEPAGE_URL@|$(HOMEPAGE_URL)|' \
- -e 's|@CMAKE_INSTALL_PREFIX@|$(PREFIX)|' \
- -e 's|@TS_REQUIRES@|$(REQUIRES)|' $< > $@
+ -e 's|@PROJECT_HOMEPAGE_URL@|$(HOMEPAGE_URL)|' \
+ -e 's|@CMAKE_INSTALL_PREFIX@|$(PREFIX)|' $< > $@
+
+$(SRC_DIR)/grammar.json: grammar.js
+ $(TS) generate --no-parser $^
-$(PARSER): $(SRC_DIR)/grammar.js
+$(PARSER): $(SRC_DIR)/grammar.json
$(TS) generate $^
install: all
- install -Dm644 bindings/c/$(LANGUAGE_NAME).h '$(DESTDIR)$(INCLUDEDIR)'/tree_sitter/$(LANGUAGE_NAME).h
- install -Dm644 $(LANGUAGE_NAME).pc '$(DESTDIR)$(PCLIBDIR)'/$(LANGUAGE_NAME).pc
- install -Dm755 lib$(LANGUAGE_NAME).a '$(DESTDIR)$(LIBDIR)'/lib$(LANGUAGE_NAME).a
+ install -d '$(DESTDIR)$(DATADIR)'/tree-sitter/queries/$(LANGUAGE_NAME) '$(DESTDIR)$(INCLUDEDIR)'/tree_sitter '$(DESTDIR)$(PCLIBDIR)' '$(DESTDIR)$(LIBDIR)'
+ install -m644 bindings/c/tree_sitter/$(LANGUAGE_NAME).h '$(DESTDIR)$(INCLUDEDIR)'/tree_sitter/$(LANGUAGE_NAME).h
+ install -m644 $(LANGUAGE_NAME).pc '$(DESTDIR)$(PCLIBDIR)'/$(LANGUAGE_NAME).pc
+ install -m644 lib$(LANGUAGE_NAME).a '$(DESTDIR)$(LIBDIR)'/lib$(LANGUAGE_NAME).a
+ install -m755 lib$(LANGUAGE_NAME).$(SOEXT) '$(DESTDIR)$(LIBDIR)'/lib$(LANGUAGE_NAME).$(SOEXTVER)
+ifneq ($(findstring mingw32,$(MACHINE)),)
+ install -d '$(DESTDIR)$(BINDIR)'
+ install -m755 lib$(LANGUAGE_NAME).dll '$(DESTDIR)$(BINDIR)'/lib$(LANGUAGE_NAME).dll
+ install -m755 lib$(LANGUAGE_NAME).dll.a '$(DESTDIR)$(LIBDIR)'/lib$(LANGUAGE_NAME).dll.a
+else
install -m755 lib$(LANGUAGE_NAME).$(SOEXT) '$(DESTDIR)$(LIBDIR)'/lib$(LANGUAGE_NAME).$(SOEXTVER)
- ln -sf lib$(LANGUAGE_NAME).$(SOEXTVER) '$(DESTDIR)$(LIBDIR)'/lib$(LANGUAGE_NAME).$(SOEXTVER_MAJOR)
- ln -sf lib$(LANGUAGE_NAME).$(SOEXTVER_MAJOR) '$(DESTDIR)$(LIBDIR)'/lib$(LANGUAGE_NAME).$(SOEXT)
+ cd '$(DESTDIR)$(LIBDIR)' && ln -sf lib$(LANGUAGE_NAME).$(SOEXTVER) lib$(LANGUAGE_NAME).$(SOEXTVER_MAJOR)
+ cd '$(DESTDIR)$(LIBDIR)' && ln -sf lib$(LANGUAGE_NAME).$(SOEXTVER_MAJOR) lib$(LANGUAGE_NAME).$(SOEXT)
+endif
+ifneq ($(wildcard queries/*.scm),)
+ install -m644 queries/*.scm '$(DESTDIR)$(DATADIR)'/tree-sitter/queries/$(LANGUAGE_NAME)
+endif
uninstall:
$(RM) '$(DESTDIR)$(LIBDIR)'/lib$(LANGUAGE_NAME).a \
@@ -84,9 +103,10 @@ uninstall:
'$(DESTDIR)$(LIBDIR)'/lib$(LANGUAGE_NAME).$(SOEXT) \
'$(DESTDIR)$(INCLUDEDIR)'/tree_sitter/$(LANGUAGE_NAME).h \
'$(DESTDIR)$(PCLIBDIR)'/$(LANGUAGE_NAME).pc
+ $(RM) -r '$(DESTDIR)$(DATADIR)'/tree-sitter/queries/$(LANGUAGE_NAME)
clean:
- $(RM) $(OBJS) $(LANGUAGE_NAME).pc lib$(LANGUAGE_NAME).a lib$(LANGUAGE_NAME).$(SOEXT)
+ $(RM) $(OBJS) $(LANGUAGE_NAME).pc lib$(LANGUAGE_NAME).a lib$(LANGUAGE_NAME).$(SOEXT) lib$(LANGUAGE_NAME).dll.a
test:
$(TS) test
diff --git a/go.mod b/go.mod
deleted file mode 100644
index 9ff2f6b..0000000
--- a/go.mod
+++ /dev/null
@@ -1,9 +0,0 @@
-module github.com/tree-sitter-grammars/tree-sitter-markdown
-
-go 1.23
-
-toolchain go1.23.0
-
-require github.com/tree-sitter/go-tree-sitter v0.23.1
-
-require github.com/mattn/go-pointer v0.0.1 // indirect
diff --git a/pyproject.toml b/pyproject.toml
index 081d451..57abad9 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -17,7 +17,7 @@ classifiers = [
authors = [
{name = "MDeiml"}
]
-requires-python = ">=3.9"
+requires-python = ">=3.10"
license.text = "MIT"
readme = "README.md"
@@ -28,5 +28,5 @@ Homepage = "https://github.com/tree-sitter-grammars/tree-sitter-markdown"
core = ["tree-sitter~=0.23"]
[tool.cibuildwheel]
-build = "cp39-*"
+build = "cp310-*"
build-frontend = "build"
diff --git a/setup.py b/setup.py
index aac54d7..c791ca4 100644
--- a/setup.py
+++ b/setup.py
@@ -1,30 +1,49 @@
-from pathlib import Path
-from platform import system
+from os import path
+from sysconfig import get_config_var
from setuptools import Extension, find_packages, setup
from setuptools.command.build import build
+from setuptools.command.build_ext import build_ext
+from setuptools.command.egg_info import egg_info
from wheel.bdist_wheel import bdist_wheel
class Build(build):
def run(self):
- if (block_queries := Path("tree-sitter-markdown", "queries")).is_dir():
- dest = Path(self.build_lib, "tree_sitter_markdown", "queries", "markdown")
- self.copy_tree(str(block_queries), str(dest))
- if (inline_queries := Path("tree-sitter-markdown-inline", "queries")).is_dir():
- dest = Path(self.build_lib, "tree_sitter_markdown", "queries", "markdown_inline")
- self.copy_tree(str(inline_queries), str(dest))
+ if path.isdir("queries"):
+ dest = path.join(self.build_lib, "tree_sitter_markdown", "queries")
+ self.copy_tree("queries", dest)
super().run()
+class BuildExt(build_ext):
+ def build_extension(self, ext: Extension):
+ if self.compiler.compiler_type != "msvc":
+ ext.extra_compile_args = ["-std=c11", "-fvisibility=hidden"]
+ else:
+ ext.extra_compile_args = ["/std:c11", "/utf-8"]
+ if path.exists("src/scanner.c"):
+ ext.sources.append("src/scanner.c")
+ if ext.py_limited_api:
+ ext.define_macros.append(("Py_LIMITED_API", "0x030A0000"))
+ super().build_extension(ext)
+
+
class BdistWheel(bdist_wheel):
def get_tag(self):
python, abi, platform = super().get_tag()
if python.startswith("cp"):
- python, abi = "cp39", "abi3"
+ python, abi = "cp310", "abi3"
return python, abi, platform
+class EggInfo(egg_info):
+ def find_sources(self):
+ super().find_sources()
+ self.filelist.recursive_include("queries", "*.scm")
+ self.filelist.include("src/tree_sitter/*.h")
+
+
setup(
packages=find_packages("bindings/python"),
package_dir={"": "bindings/python"},
@@ -38,25 +57,21 @@ def get_tag(self):
name="_binding",
sources=[
"bindings/python/tree_sitter_markdown/binding.c",
- "tree-sitter-markdown/src/parser.c",
- "tree-sitter-markdown/src/scanner.c",
- "tree-sitter-markdown-inline/src/parser.c",
- "tree-sitter-markdown-inline/src/scanner.c",
+ "src/parser.c",
],
- extra_compile_args=(
- ["-std=c11"] if system() != "Windows" else []
- ),
define_macros=[
- ("Py_LIMITED_API", "0x03090000"),
- ("PY_SSIZE_T_CLEAN", None)
+ ("PY_SSIZE_T_CLEAN", None),
+ ("TREE_SITTER_HIDE_SYMBOLS", None),
],
- include_dirs=["tree-sitter-markdown/src"],
- py_limited_api=True,
+ include_dirs=["src"],
+ py_limited_api=not get_config_var("Py_GIL_DISABLED"),
)
],
cmdclass={
"build": Build,
- "bdist_wheel": BdistWheel
+ "build_ext": BuildExt,
+ "bdist_wheel": BdistWheel,
+ "egg_info": EggInfo,
},
zip_safe=False
)
diff --git a/tree-sitter-markdown-inline/bindings/c/tree-sitter-markdown-inline.h b/tree-sitter-markdown-inline/bindings/c/tree-sitter/tree-sitter-markdown-inline.h
similarity index 100%
rename from tree-sitter-markdown-inline/bindings/c/tree-sitter-markdown-inline.h
rename to tree-sitter-markdown-inline/bindings/c/tree-sitter/tree-sitter-markdown-inline.h
diff --git a/tree-sitter-markdown-inline/queries/highlights.scm b/tree-sitter-markdown-inline/queries/highlights.scm
index 8c87c78..ee71396 100644
--- a/tree-sitter-markdown-inline/queries/highlights.scm
+++ b/tree-sitter-markdown-inline/queries/highlights.scm
@@ -1,4 +1,4 @@
-;; From nvim-treesitter/nvim-treesitter
+; From nvim-treesitter/nvim-treesitter
[
(code_span)
(link_title)
@@ -29,9 +29,28 @@
(hard_line_break)
] @string.escape
-(image ["!" "[" "]" "(" ")"] @punctuation.delimiter)
-(inline_link ["[" "]" "(" ")"] @punctuation.delimiter)
-(shortcut_link ["[" "]"] @punctuation.delimiter)
+(image
+ [
+ "!"
+ "["
+ "]"
+ "("
+ ")"
+ ] @punctuation.delimiter)
+
+(inline_link
+ [
+ "["
+ "]"
+ "("
+ ")"
+ ] @punctuation.delimiter)
+
+(shortcut_link
+ [
+ "["
+ "]"
+ ] @punctuation.delimiter)
; NOTE: extension not enabled by default
; (wiki_link ["[" "|" "]"] @punctuation.delimiter)
diff --git a/tree-sitter-markdown-inline/queries/injections.scm b/tree-sitter-markdown-inline/queries/injections.scm
index 9a0b8cd..0f29635 100644
--- a/tree-sitter-markdown-inline/queries/injections.scm
+++ b/tree-sitter-markdown-inline/queries/injections.scm
@@ -1,2 +1,5 @@
-((html_tag) @injection.content (#set! injection.language "html"))
-((latex_block) @injection.content (#set! injection.language "latex"))
+((html_tag) @injection.content
+ (#set! injection.language "html"))
+
+((latex_block) @injection.content
+ (#set! injection.language "latex"))
diff --git a/tree-sitter-markdown-inline/src/parser.c b/tree-sitter-markdown-inline/src/parser.c
index 5cce477..ee22b48 100644
--- a/tree-sitter-markdown-inline/src/parser.c
+++ b/tree-sitter-markdown-inline/src/parser.c
@@ -1,4 +1,4 @@
-/* Automatically @generated by tree-sitter v0.25.9 */
+/* Automatically @generated by tree-sitter */
#include "tree_sitter/parser.h"
diff --git a/tree-sitter-markdown/bindings/c/tree-sitter-markdown.h b/tree-sitter-markdown/bindings/c/tree-sitter/tree-sitter-markdown.h
similarity index 100%
rename from tree-sitter-markdown/bindings/c/tree-sitter-markdown.h
rename to tree-sitter-markdown/bindings/c/tree-sitter/tree-sitter-markdown.h
diff --git a/tree-sitter-markdown/queries/highlights.scm b/tree-sitter-markdown/queries/highlights.scm
index 3c2a896..21962fd 100644
--- a/tree-sitter-markdown/queries/highlights.scm
+++ b/tree-sitter-markdown/queries/highlights.scm
@@ -1,6 +1,9 @@
;From nvim-treesitter/nvim-treesitter
-(atx_heading (inline) @text.title)
-(setext_heading (paragraph) @text.title)
+(atx_heading
+ (inline) @text.title)
+
+(setext_heading
+ (paragraph) @text.title)
[
(atx_h1_marker)
@@ -19,19 +22,13 @@
(fenced_code_block)
] @text.literal
-[
- (fenced_code_block_delimiter)
-] @punctuation.delimiter
+(fenced_code_block_delimiter) @punctuation.delimiter
(code_fence_content) @none
-[
- (link_destination)
-] @text.uri
+(link_destination) @text.uri
-[
- (link_label)
-] @text.reference
+(link_label) @text.reference
[
(list_marker_plus)
@@ -47,6 +44,4 @@
(block_quote_marker)
] @punctuation.special
-[
- (backslash_escape)
-] @string.escape
+(backslash_escape) @string.escape
diff --git a/tree-sitter-markdown/queries/injections.scm b/tree-sitter-markdown/queries/injections.scm
index 809b618..7a8c34a 100644
--- a/tree-sitter-markdown/queries/injections.scm
+++ b/tree-sitter-markdown/queries/injections.scm
@@ -3,12 +3,23 @@
(language) @injection.language)
(code_fence_content) @injection.content)
-((html_block) @injection.content (#set! injection.language "html"))
+((html_block) @injection.content
+ (#set! injection.language "html"))
-(document . (section . (thematic_break) (_) @injection.content (thematic_break)) (#set! injection.language "yaml"))
+(document
+ .
+ (section
+ .
+ (thematic_break)
+ (_) @injection.content
+ (thematic_break))
+ (#set! injection.language "yaml"))
-((minus_metadata) @injection.content (#set! injection.language "yaml"))
+((minus_metadata) @injection.content
+ (#set! injection.language "yaml"))
-((plus_metadata) @injection.content (#set! injection.language "toml"))
+((plus_metadata) @injection.content
+ (#set! injection.language "toml"))
-((inline) @injection.content (#set! injection.language "markdown_inline"))
+((inline) @injection.content
+ (#set! injection.language "markdown_inline"))
diff --git a/tree-sitter-markdown/src/parser.c b/tree-sitter-markdown/src/parser.c
index b5f214c..113bdd1 100644
--- a/tree-sitter-markdown/src/parser.c
+++ b/tree-sitter-markdown/src/parser.c
@@ -1,4 +1,4 @@
-/* Automatically @generated by tree-sitter v0.25.9 */
+/* Automatically @generated by tree-sitter */
#include "tree_sitter/parser.h"
diff --git a/tree-sitter.json b/tree-sitter.json
index 3149560..8aa4202 100644
--- a/tree-sitter.json
+++ b/tree-sitter.json
@@ -38,5 +38,15 @@
"common/common.js"
]
}
- ]
+ ],
+ "bindings": {
+ "rust": true,
+ "node": true,
+ "c": true,
+ "python": true,
+ "swift": true,
+ "go": false,
+ "java": false,
+ "zig": false
+ }
}