diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c561bd429..a0baa0917 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -55,7 +55,7 @@ jobs: - name: Run lsp-positions tests without tree-sitter run: cargo test -p lsp-positions --no-default-features - name: Run test suite with all features enabled - run: cargo test --all-features + run: cargo test --all-features --features=mlua/lua54,mlua/vendored - name: Run test suite with all optimizations run: cargo test --release # Do the new project test last, because it adds the crate in the current source diff --git a/.github/workflows/publish-lsp-positions.yml b/.github/workflows/publish-lsp-positions.yml index 210662a00..6fd1b213a 100644 --- a/.github/workflows/publish-lsp-positions.yml +++ b/.github/workflows/publish-lsp-positions.yml @@ -19,7 +19,7 @@ jobs: uses: actions/checkout@v3 # TODO Verify the crate version matches the tag - name: Test crate - run: cargo test --all-features + run: cargo test --all-features --features=mlua/lua54,mlua/vendored working-directory: ${{ env.CRATE_DIR }} - name: Verify publish crate run: cargo publish --dry-run diff --git a/.github/workflows/publish-stack-graphs.yml b/.github/workflows/publish-stack-graphs.yml index 641a93488..a2689033d 100644 --- a/.github/workflows/publish-stack-graphs.yml +++ b/.github/workflows/publish-stack-graphs.yml @@ -19,7 +19,7 @@ jobs: uses: actions/checkout@v3 # TODO Verify the crate version matches the tag - name: Test crate - run: cargo test --all-features + run: cargo test --all-features --features=mlua/lua54,mlua/vendored working-directory: ${{ env.CRATE_DIR }} - name: Verify publish crate run: cargo publish --dry-run diff --git a/.github/workflows/publish-tree-sitter-stack-graphs-java.yml b/.github/workflows/publish-tree-sitter-stack-graphs-java.yml index 6d7afb769..c1d904942 100644 --- a/.github/workflows/publish-tree-sitter-stack-graphs-java.yml +++ b/.github/workflows/publish-tree-sitter-stack-graphs-java.yml @@ -19,7 +19,7 @@ jobs: uses: actions/checkout@v3 # TODO Verify the crate version matches the tag - name: Test crate - run: cargo test --all-features + run: cargo test --all-features --features=mlua/lua54,mlua/vendored working-directory: ${{ env.CRATE_DIR }} - name: Verify publish crate run: cargo publish --dry-run diff --git a/.github/workflows/publish-tree-sitter-stack-graphs-javascript.yml b/.github/workflows/publish-tree-sitter-stack-graphs-javascript.yml index ff6826770..a9ab7b448 100644 --- a/.github/workflows/publish-tree-sitter-stack-graphs-javascript.yml +++ b/.github/workflows/publish-tree-sitter-stack-graphs-javascript.yml @@ -19,7 +19,7 @@ jobs: uses: actions/checkout@v3 # TODO Verify the crate version matches the tag - name: Test crate - run: cargo test --all-features + run: cargo test --all-features --features=mlua/lua54,mlua/vendored working-directory: ${{ env.CRATE_DIR }} - name: Verify publish crate run: cargo publish --dry-run diff --git a/.github/workflows/publish-tree-sitter-stack-graphs-typescript.yml b/.github/workflows/publish-tree-sitter-stack-graphs-typescript.yml index a3409f397..5446e2c5a 100644 --- a/.github/workflows/publish-tree-sitter-stack-graphs-typescript.yml +++ b/.github/workflows/publish-tree-sitter-stack-graphs-typescript.yml @@ -19,7 +19,7 @@ jobs: uses: actions/checkout@v3 # TODO Verify the crate version matches the tag - name: Test crate - run: cargo test --all-features + run: cargo test --all-features --features=mlua/lua54,mlua/vendored working-directory: ${{ env.CRATE_DIR }} - name: Verify publish crate run: cargo publish --dry-run diff --git a/.github/workflows/publish-tree-sitter-stack-graphs.yml b/.github/workflows/publish-tree-sitter-stack-graphs.yml index e9395ab1f..7b7db7e0f 100644 --- a/.github/workflows/publish-tree-sitter-stack-graphs.yml +++ b/.github/workflows/publish-tree-sitter-stack-graphs.yml @@ -19,7 +19,7 @@ jobs: uses: actions/checkout@v3 # TODO Verify the crate version matches the tag - name: Test crate - run: cargo test --all-features + run: cargo test --all-features --features=mlua/lua54,mlua/vendored working-directory: ${{ env.CRATE_DIR }} - name: Verify publish crate run: cargo publish --dry-run diff --git a/Cargo.toml b/Cargo.toml index 587e70d61..e44137d10 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,6 +3,7 @@ resolver = "1" members = [ # library projects "lsp-positions", + "lua-helpers", "stack-graphs", "tree-sitter-stack-graphs", "languages/*", @@ -12,3 +13,12 @@ default-members = [ "stack-graphs", "tree-sitter-stack-graphs", ] + +[patch.crates-io] +# TODO: Revert to regular versioned dependencies once tree-sitter#2773 has been +# merged. +tree-sitter = { git="https://github.com/dcreager/tree-sitter", branch="rust-linking" } +tree-sitter-graph = { git="https://github.com/tree-sitter/tree-sitter-graph", branch="ts-bump" } +tree-sitter-highlight = { git="https://github.com/dcreager/tree-sitter", branch="rust-linking" } +tree-sitter-loader = { git="https://github.com/dcreager/tree-sitter", branch="rust-linking" } +tree-sitter-tags = { git="https://github.com/dcreager/tree-sitter", branch="rust-linking" } diff --git a/lsp-positions/Cargo.toml b/lsp-positions/Cargo.toml index 242a894ff..db874de99 100644 --- a/lsp-positions/Cargo.toml +++ b/lsp-positions/Cargo.toml @@ -18,11 +18,19 @@ test = false [features] bincode = ["dep:bincode"] +lua = ["tree-sitter", "dep:mlua", "dep:mlua-tree-sitter"] tree-sitter = ["dep:tree-sitter"] [dependencies] memchr = "2.4" +mlua = { version = "0.9", optional = true } +mlua-tree-sitter = { version = "0.1", git="https://github.com/dcreager/mlua-tree-sitter", optional = true } tree-sitter = { version=">= 0.19", optional=true } unicode-segmentation = { version="1.8" } serde = { version="1", optional=true, features=["derive"] } bincode = { version="2.0.0-rc.3", optional=true } + +[dev-dependencies] +anyhow = { version = "1.0" } +lua-helpers = { path = "../lua-helpers" } +tree-sitter-python = { version = "0.19.1" } diff --git a/lsp-positions/src/lib.rs b/lsp-positions/src/lib.rs index 628cec2a7..d9fa74199 100644 --- a/lsp-positions/src/lib.rs +++ b/lsp-positions/src/lib.rs @@ -33,6 +33,9 @@ use memchr::memchr; use unicode_segmentation::UnicodeSegmentation as _; +#[cfg(feature = "lua")] +pub mod lua; + fn grapheme_len(string: &str) -> usize { string.graphemes(true).count() } diff --git a/lsp-positions/src/lua.rs b/lsp-positions/src/lua.rs new file mode 100644 index 000000000..f71e7a3ad --- /dev/null +++ b/lsp-positions/src/lua.rs @@ -0,0 +1,217 @@ +// -*- coding: utf-8 -*- +// ------------------------------------------------------------------------------------------------ +// Copyright © 2023, stack-graphs authors. +// Licensed under either of Apache License, Version 2.0, or MIT license, at your option. +// Please see the LICENSE-APACHE or LICENSE-MIT files in this distribution for license details. +// ------------------------------------------------------------------------------------------------ + +use std::ops::Range; + +use mlua::Error; +use mlua::FromLua; +use mlua::IntoLua; +use mlua::Lua; +use mlua::UserData; +use mlua::UserDataMethods; +use mlua::Value; +use mlua_tree_sitter::TSNode; +use mlua_tree_sitter::TreeWithSource; + +use crate::Offset; +use crate::Position; +use crate::Span; +use crate::SpanCalculator; + +/// An extension trait that lets you load the `lsp_positions` module into a Lua environment. +pub trait Module { + /// Loads the `lsp_positions` module into a Lua environment. + fn open_lsp_positions(&self) -> Result<(), mlua::Error>; +} + +impl Module for Lua { + fn open_lsp_positions(&self) -> Result<(), mlua::Error> { + let exports = self.create_table()?; + let sc_type = self.create_table()?; + + let function = self.create_function(|lua, source_value: mlua::String| { + // We are going to add the Lua string as a user value of the SpanCalculator's Lua + // wrapper. That will ensure that the string is not garbage collected before the + // SpanCalculator, which makes it safe to transmute into a 'static reference. + let source = source_value.to_str()?; + let source: &'static str = unsafe { std::mem::transmute(source) }; + let sc = SpanCalculator::new(source); + let sc = lua.create_userdata(sc)?; + sc.set_user_value(source_value)?; + Ok(sc) + })?; + sc_type.set("new", function)?; + + #[cfg(feature = "tree-sitter")] + { + let function = self.create_function(|lua, tws_value: Value| { + // We are going to add the tree-sitter treee as a user value of the + // SpanCalculator's Lua wrapper. That will ensure that the tree is not garbage + // collected before the SpanCalculator, which makes it safe to transmute into a + // 'static reference. + let tws = TreeWithSource::from_lua(tws_value.clone(), lua)?; + let source: &'static str = unsafe { std::mem::transmute(tws.src) }; + let sc = SpanCalculator::new(source); + let sc = lua.create_userdata(sc)?; + sc.set_user_value(tws_value)?; + Ok(sc) + })?; + sc_type.set("new_from_tree", function)?; + } + + exports.set("SpanCalculator", sc_type)?; + self.globals().set("lsp_positions", exports)?; + Ok(()) + } +} + +impl<'lua> FromLua<'lua> for Offset { + fn from_lua(value: Value<'lua>, _: &'lua Lua) -> Result { + let table = match value { + Value::Table(table) => table, + Value::Nil => return Ok(Offset::default()), + _ => { + return Err(mlua::Error::FromLuaConversionError { + from: value.type_name(), + to: "Offset", + message: None, + }) + } + }; + let utf8_offset = table.get::<_, Option<_>>("utf8_offset")?.unwrap_or(0); + let utf16_offset = table.get::<_, Option<_>>("utf16_offset")?.unwrap_or(0); + let grapheme_offset = table.get::<_, Option<_>>("grapheme_offset")?.unwrap_or(0); + Ok(Offset { + utf8_offset, + utf16_offset, + grapheme_offset, + }) + } +} + +impl<'lua> IntoLua<'lua> for Offset { + fn into_lua(self, l: &'lua Lua) -> Result, Error> { + let result = l.create_table()?; + result.set("utf8_offset", self.utf8_offset)?; + result.set("utf16_offset", self.utf16_offset)?; + result.set("grapheme_offset", self.grapheme_offset)?; + Ok(Value::Table(result)) + } +} + +fn range_from_lua<'lua>(value: Value<'lua>) -> Result, Error> { + let table = match value { + Value::Table(table) => table, + Value::Nil => return Ok(0..0), + _ => { + return Err(mlua::Error::FromLuaConversionError { + from: value.type_name(), + to: "Range", + message: None, + }) + } + }; + let start = table.get("start")?; + let end = table.get("end")?; + Ok(start..end) +} + +fn range_into_lua<'lua>(range: Range, l: &'lua Lua) -> Result, Error> { + let result = l.create_table()?; + result.set("start", range.start)?; + result.set("end", range.end)?; + Ok(Value::Table(result)) +} + +impl<'lua> FromLua<'lua> for Position { + fn from_lua(value: Value<'lua>, _: &'lua Lua) -> Result { + let table = match value { + Value::Table(table) => table, + Value::Nil => return Ok(Position::default()), + _ => { + return Err(mlua::Error::FromLuaConversionError { + from: value.type_name(), + to: "Position", + message: None, + }) + } + }; + let line = table.get("line")?; + let column = table.get("column")?; + let containing_line = range_from_lua(table.get("containing_line")?)?; + let trimmed_line = range_from_lua(table.get("trimmed_line")?)?; + Ok(Position { + line, + column, + containing_line, + trimmed_line, + }) + } +} + +impl<'lua> IntoLua<'lua> for Position { + fn into_lua(self, l: &'lua Lua) -> Result, Error> { + let result = l.create_table()?; + result.set("line", self.line)?; + result.set("column", self.column)?; + result.set("containing_line", range_into_lua(self.containing_line, l)?)?; + result.set("trimmed_line", range_into_lua(self.trimmed_line, l)?)?; + Ok(Value::Table(result)) + } +} + +impl<'lua> FromLua<'lua> for Span { + fn from_lua(value: Value<'lua>, _: &'lua Lua) -> Result { + let table = match value { + Value::Table(table) => table, + Value::Nil => return Ok(Span::default()), + _ => { + return Err(mlua::Error::FromLuaConversionError { + from: value.type_name(), + to: "Span", + message: None, + }) + } + }; + let start = table.get("start")?; + let end = table.get("end")?; + Ok(Span { start, end }) + } +} + +impl<'lua> IntoLua<'lua> for Span { + fn into_lua(self, l: &'lua Lua) -> Result, Error> { + if self == Span::default() { + return Ok(Value::Nil); + } + let result = l.create_table()?; + result.set("start", self.start)?; + result.set("end", self.end)?; + Ok(Value::Table(result)) + } +} + +impl UserData for SpanCalculator<'static> { + fn add_methods<'lua, M: UserDataMethods<'lua, Self>>(methods: &mut M) { + methods.add_method_mut( + "for_line_and_column", + |_, sc, (line, line_utf8_offset, column_utf8_offset)| { + Ok(sc.for_line_and_column(line, line_utf8_offset, column_utf8_offset)) + }, + ); + + methods.add_method_mut( + "for_line_and_grapheme", + |_, sc, (line, line_utf8_offset, column_grapheme_offset)| { + Ok(sc.for_line_and_grapheme(line, line_utf8_offset, column_grapheme_offset)) + }, + ); + + #[cfg(feature = "tree-sitter")] + methods.add_method_mut("for_node", |_, sc, node: TSNode| Ok(sc.for_node(&node))); + } +} diff --git a/lsp-positions/tests/it/lua.rs b/lsp-positions/tests/it/lua.rs new file mode 100644 index 000000000..42b7663d5 --- /dev/null +++ b/lsp-positions/tests/it/lua.rs @@ -0,0 +1,88 @@ +// -*- coding: utf-8 -*- +// ------------------------------------------------------------------------------------------------ +// Copyright © 2023, stack-graphs authors. +// Licensed under either of Apache License, Version 2.0, or MIT license, at your option. +// Please see the LICENSE-APACHE or LICENSE-MIT files in this distribution for license details. +// ------------------------------------------------------------------------------------------------ + +use lsp_positions::lua::Module; +use lua_helpers::new_lua; +use lua_helpers::CheckLua; + +#[test] +fn can_calculate_positions_from_lua() -> Result<(), mlua::Error> { + let l = new_lua()?; + l.open_lsp_positions()?; + l.check( + r#" + local source = " from a import * " + local sc = lsp_positions.SpanCalculator.new(source) + local position = sc:for_line_and_column(0, 0, 9) + local expected = { + line=0, + column={ + utf8_offset=9, + utf16_offset=9, + grapheme_offset=9, + }, + containing_line={start=0, ["end"]=21}, + trimmed_line={start=3, ["end"]=18}, + } + assert_deepeq("position", expected, position) + "#, + )?; + Ok(()) +} + +#[cfg(feature = "tree-sitter")] +#[test] +fn can_calculate_tree_sitter_spans_from_lua() -> Result<(), anyhow::Error> { + let code = br#" + def double(x): + return x * 2 + "#; + let mut parser = tree_sitter::Parser::new(); + parser.set_language(tree_sitter_python::language()).unwrap(); + let parsed = parser.parse(code, None).unwrap(); + + use mlua_tree_sitter::Module; + use mlua_tree_sitter::WithSource; + let l = new_lua()?; + l.open_lsp_positions()?; + l.open_ltreesitter()?; + l.globals().set("parsed", parsed.with_source(code))?; + + l.check( + r#" + local module = parsed:root() + local double = module:child(0) + local name = double:child_by_field_name("name") + local sc = lsp_positions.SpanCalculator.new_from_tree(parsed) + local position = sc:for_node(name) + local expected = { + start={ + line=1, + column={ + utf8_offset=10, + utf16_offset=10, + grapheme_offset=10, + }, + containing_line={start=1, ["end"]=21}, + trimmed_line={start=7, ["end"]=21}, + }, + ["end"]={ + line=1, + column={ + utf8_offset=16, + utf16_offset=16, + grapheme_offset=16, + }, + containing_line={start=1, ["end"]=21}, + trimmed_line={start=7, ["end"]=21}, + }, + } + assert_deepeq("position", expected, position) + "#, + )?; + Ok(()) +} diff --git a/lsp-positions/tests/it/main.rs b/lsp-positions/tests/it/main.rs index 44dd4230f..8a3e340e7 100644 --- a/lsp-positions/tests/it/main.rs +++ b/lsp-positions/tests/it/main.rs @@ -9,6 +9,9 @@ use unicode_segmentation::UnicodeSegmentation as _; use lsp_positions::Offset; +#[cfg(feature = "lua")] +mod lua; + fn check_offsets(line: &str) { let offsets = Offset::all_chars(line).collect::>(); assert!(!offsets.is_empty()); diff --git a/lua-helpers/Cargo.toml b/lua-helpers/Cargo.toml new file mode 100644 index 000000000..cca90bf07 --- /dev/null +++ b/lua-helpers/Cargo.toml @@ -0,0 +1,7 @@ +[package] +name = "lua-helpers" +version = "0.0.0" +description = "Unpublished helper methods for our Lua test cases" + +[dependencies] +mlua = { version = "0.9" } diff --git a/lua-helpers/src/lib.rs b/lua-helpers/src/lib.rs new file mode 100644 index 000000000..92a9b34d5 --- /dev/null +++ b/lua-helpers/src/lib.rs @@ -0,0 +1,119 @@ +// -*- coding: utf-8 -*- +// ------------------------------------------------------------------------------------------------ +// Copyright © 2023, stack-graphs authors. +// Licensed under either of Apache License, Version 2.0, or MIT license, at your option. +// Please see the LICENSE-APACHE or LICENSE-MIT files in this distribution for license details. +// ------------------------------------------------------------------------------------------------ + +const TEST_PRELUDE: &str = r#" + function assert_eq(thing, expected, actual) + if expected ~= actual then + error("Expected "..thing.." "..expected..", got "..actual) + end + end + + function deepeq(t1, t2, prefix) + prefix = prefix or "" + local ty1 = type(t1) + local ty2 = type(t2) + if ty1 ~= ty2 then + local msg = "different types for lhs"..prefix.." ("..ty1..") and rhs"..prefix.." ("..ty2..")" + return false, {msg} + end + + -- non-table types can be directly compared + if ty1 ~= 'table' and ty2 ~= 'table' then + if t1 ~= t2 then + local msg = "different values for lhs"..prefix.." ("..t1..") and rhs"..prefix.." ("..t2..")" + return false, {msg} + end + return true, {} + end + + local equal = true + local diffs = {} + for k2, v2 in pairs(t2) do + local v1 = t1[k2] + if v1 == nil then + equal = false + diffs[#diffs+1] = "missing lhs"..prefix.."."..k2 + else + local e, d = deepeq(v1, v2, prefix.."."..k2) + equal = equal and e + table.move(d, 1, #d, #diffs+1, diffs) + end + end + for k1, v1 in pairs(t1) do + local v2 = t2[k1] + if v2 == nil then + equal = false + diffs[#diffs+1] = "missing rhs"..prefix.."."..k1 + end + end + return equal, diffs + end + + function assert_deepeq(thing, expected, actual) + local eq, diffs = deepeq(expected, actual) + if not eq then + error("Unexpected "..thing..": "..table.concat(diffs, ", ")) + end + end + + function values(t) + local i = 0 + return function() i = i + 1; return t[i] end + end + + function iter_tostring(...) + local result = {} + for element in ... do + table.insert(result, tostring(element)) + end + return result + end +"#; + +pub fn new_lua() -> Result { + let l = mlua::Lua::new(); + l.load(TEST_PRELUDE).set_name("test prelude").exec()?; + Ok(l) +} + +pub trait CheckLua { + fn check(&self, chunk: &str) -> Result<(), mlua::Error>; +} + +impl CheckLua for mlua::Lua { + fn check(&self, chunk: &str) -> Result<(), mlua::Error> { + self.load(chunk).set_name("test chunk").exec() + } +} + +#[test] +fn can_deepeq_from_lua() -> Result<(), mlua::Error> { + let l = new_lua()?; + l.check( + r#" + function check_deepeq(lhs, rhs, expected, expected_diffs) + local actual, actual_diffs = deepeq(lhs, rhs) + actual_diffs = table.concat(actual_diffs, ", ") + assert_eq("deepeq", expected, actual) + assert_eq("differences", expected_diffs, actual_diffs) + end + + check_deepeq(0, 0, true, "") + check_deepeq(0, 1, false, "different values for lhs (0) and rhs (1)") + + check_deepeq({"a", "b", "c"}, {"a", "b", "c"}, true, "") + check_deepeq({"a", "b", "c"}, {"a", "b"}, false, "missing rhs.3") + check_deepeq({"a", "b", "c"}, {"a", "b", "d"}, false, "different values for lhs.3 (c) and rhs.3 (d)") + + check_deepeq({a=1, b=2, c=3}, {a=1, b=2, c=3}, true, "") + check_deepeq({a=1, b=2, c=3}, {a=1, b=2}, false, "missing rhs.c") + check_deepeq({a=1, b=2, c=3}, {a=1, b=2, c=4}, false, "different values for lhs.c (3) and rhs.c (4)") + check_deepeq({a=1, b=2, c=3}, {a=1, b=2, d=3}, false, "missing lhs.d, missing rhs.c") + "#, + )?; + Ok(()) +} diff --git a/stack-graphs/CHANGELOG.md b/stack-graphs/CHANGELOG.md index c61221e8e..0c15532a2 100644 --- a/stack-graphs/CHANGELOG.md +++ b/stack-graphs/CHANGELOG.md @@ -5,6 +5,12 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## v0.13.0 -- Unreleased + +### Added + +- Added Lua bindings for constructing stack graphs. These bindings are optional, and will only be built when the `lua` feature flag is enabled. + ## v0.12.0 -- 2023-07-27 ### Added diff --git a/stack-graphs/Cargo.toml b/stack-graphs/Cargo.toml index 2e55e1768..31c89ae09 100644 --- a/stack-graphs/Cargo.toml +++ b/stack-graphs/Cargo.toml @@ -15,6 +15,7 @@ edition = "2018" [features] bincode = ["dep:bincode", "lsp-positions/bincode"] copious-debugging = [] +lua = ["dep:mlua", "lsp-positions/lua"] serde = ["dep:serde", "serde_with", "lsp-positions/serde"] storage = ["bincode", "rusqlite"] visualization = ["serde", "serde_json"] @@ -33,6 +34,7 @@ fxhash = "0.2" itertools = "0.10" libc = "0.2" lsp-positions = { version = "0.3", path = "../lsp-positions" } +mlua = { version = "0.9", optional = true } rusqlite = { version = "0.28", optional = true, features = ["bundled", "functions"] } serde = { version = "1.0", optional = true, features = ["derive"] } serde_json = { version = "1.0", optional = true } @@ -41,11 +43,15 @@ smallvec = { version = "1.6", features = ["union"] } thiserror = { version = "1.0" } [dev-dependencies] +anyhow = "1.0" assert-json-diff = "2" itertools = "0.10" +lua-helpers = { path = "../lua-helpers" } maplit = "1.0" pretty_assertions = "0.7" serde_json = { version = "1.0" } [package.metadata.docs.rs] all-features = true +features = ["mlua/lua54", "mlua/vendored"] +rustdoc-args = ["--cfg", "docsrs"] diff --git a/stack-graphs/README.md b/stack-graphs/README.md index 7805d19ef..9fed588ba 100644 --- a/stack-graphs/README.md +++ b/stack-graphs/README.md @@ -17,6 +17,31 @@ how to use this library. Notable changes for each version are documented in the [release notes](https://github.com/github/stack-graphs/blob/main/stack-graphs/CHANGELOG.md). +## Lua bindings + +This crate includes optional Lua bindings, allowing you to construct stack +graphs using Lua code. Lua support is only enabled if you compile with the `lua` +feature. This feature is not enough on its own, because the `mlua` crate +supports multiple Lua versions, and can either link against a system-installed +copy of Lua, or build its own copy from vendored Lua source. These choices are +all controlled via additional features on the `mlua` crate. + +When building and testing this crate, make sure to provide all necessary +features on the command line: + +``` console +$ cargo test --features lua,mlua/lua54,mlua/vendored +``` + +When building a crate that depends on this crate, add a dependency on `mlua` so +that you can set its feature flags: + +``` toml +[dependencies] +stack-graphs = { version="0.13", features=["lua"] } +mlua = { version="0.9", features=["lua54", "vendored"] } +``` + ## Credits Stack graphs are heavily based on the [_scope graphs_][scope graphs] framework diff --git a/stack-graphs/src/lib.rs b/stack-graphs/src/lib.rs index 009076af9..edcae832c 100644 --- a/stack-graphs/src/lib.rs +++ b/stack-graphs/src/lib.rs @@ -66,6 +66,8 @@ pub mod cycles; #[macro_use] mod debugging; pub mod graph; +#[cfg(feature = "lua")] +pub mod lua; pub mod partial; pub mod paths; pub mod serde; diff --git a/stack-graphs/src/lua.rs b/stack-graphs/src/lua.rs new file mode 100644 index 000000000..0e9cce12b --- /dev/null +++ b/stack-graphs/src/lua.rs @@ -0,0 +1,1046 @@ +// -*- coding: utf-8 -*- +// ------------------------------------------------------------------------------------------------ +// Copyright © 2023, stack-graphs authors. +// Licensed under either of Apache License, Version 2.0, or MIT license, at your option. +// Please see the LICENSE-APACHE or LICENSE-MIT files in this distribution for license details. +// ------------------------------------------------------------------------------------------------ + +#![cfg_attr(docsrs, doc(cfg(feature = "lua")))] +//! Provides access to `StackGraph` instances from Lua. +//! +//! With the `lua` feature enabled, you can add [`StackGraph`] instances to a [`Lua`][mlua::Lua] +//! interpreter. You might typically use this to _create_ stack graphs from Lua, by calling a Lua +//! function with an empty stack graph as a parameter. Note that you'll almost certainly need to +//! use `mlua`'s [scoped values](mlua::Lua::scope) mechanism so that you can still use the +//! [`StackGraph`] on the Rust side once the Lua function has finished. +//! +//! ``` +//! # use mlua::Lua; +//! # use stack_graphs::graph::StackGraph; +//! # fn main() -> Result<(), mlua::Error> { +//! let lua = Lua::new(); +//! let chunk = r#" +//! function process_graph(graph) +//! local file = graph:file("test.py") +//! local def = file:definition_node("foo") +//! def:add_edge_from(graph:root_node()) +//! end +//! "#; +//! lua.load(chunk).set_name("stack graph chunk").exec()?; +//! let process_graph: mlua::Function = lua.globals().get("process_graph")?; +//! +//! let mut graph = StackGraph::new(); +//! lua.scope(|scope| { +//! let graph = graph.lua_ref_mut(&scope)?; +//! process_graph.call(graph) +//! })?; +//! assert_eq!(graph.iter_nodes().count(), 3); +//! # Ok(()) +//! # } +//! ``` +//! +//! ## Building +//! +//! Lua support is only enabled if you compile with the `lua` feature. This feature is not enough +//! on its own, because the `mlua` crate supports multiple Lua versions, and can either link +//! against a system-installed copy of Lua, or build its own copy from vendored Lua source. These +//! choices are all controlled via additional features on the `mlua` crate. +//! +//! When building and testing this crate, make sure to provide all necessary features on the +//! command line: +//! +//! ``` console +//! $ cargo test --features lua,mlua/lua54,mlua/vendored +//! ``` +//! +//! When building a crate that depends on this crate, add a dependency on `mlua` so that you can +//! set its feature flags: +//! +//! ``` toml +//! [dependencies] +//! stack-graphs = { version="0.13", features=["lua"] } +//! mlua = { version="0.9", features=["lua54", "vendored"] } +//! ``` +//! +//! ## Lua API +//! +//! ### Stack graphs +//! +//! The following Lua methods are available on a stack graph instance: +//! +//! #### `edges` +//! +//! ``` lua +//! let edges = graph:edges() +//! ``` +//! +//! Returns an array containing all of the edges in the graph. +//! +//! #### `file` +//! +//! ``` lua +//! local file = graph:file(name) +//! ``` +//! +//! Returns the file in the stack graph with the given name, creating it if necessary. +//! +//! #### `jump_to_node` +//! +//! ``` lua +//! local node = graph:jump_to_node() +//! ``` +//! +//! Returns the graph's jump-to node. +//! +//! #### `nodes` +//! +//! ``` lua +//! for node in graph:nodes() do +//! -- whatever +//! end +//! ``` +//! +//! Returns an iterator of every node in the stack graph. +//! +//! #### `root_node` +//! +//! ``` lua +//! local node = graph:root_node() +//! ``` +//! +//! Returns the graph's root node. +//! +//! ### Files +//! +//! The following Lua methods are available on a file instance: +//! +//! #### `definition_node` +//! +//! ``` lua +//! local node = file:definition_node(symbol) +//! ``` +//! +//! Adds a new definition node to this file. `symbol` must be a string, or an instance that can be +//! converted to a string via its `tostring` method. +//! +//! #### `drop_scopes_node` +//! +//! ``` lua +//! local node = file:drop_scopes_node() +//! ``` +//! +//! Adds a new drop scopes node to this file. +//! +//! #### `edges` +//! +//! ``` lua +//! let edges = file:edges() +//! ``` +//! +//! Returns an array containing all of the edges starting from or leaving a node in this file. +//! +//! #### `exported_scope_node` +//! +//! ``` lua +//! local node = file:exported_scope_node() +//! ``` +//! +//! Adds a new exported scope node to this file. +//! +//! #### `internal_scope_node` +//! +//! ``` lua +//! local node = file:internal_scope_node() +//! ``` +//! +//! Adds a new internal scope node to this file. +//! +//! #### `jump_to_node` +//! +//! ``` lua +//! local node = file:jump_to_node() +//! ``` +//! +//! Returns the root node of the graph containing this file. +//! +//! #### `pop_scoped_symbol_node` +//! +//! ``` lua +//! local node = file:pop_scoped_symbol_node(symbol) +//! ``` +//! +//! Adds a new pop scoped symbol node to this file. `symbol` must be a string, or an instance that +//! can be converted to a string via its `tostring` method. +//! +//! #### `pop_symbol_node` +//! +//! ``` lua +//! local node = file:pop_symbol_node(symbol) +//! ``` +//! +//! Adds a new pop symbol node to this file. `symbol` must be a string, or an instance that can be +//! converted to a string via its `tostring` method. +//! +//! #### `push_scoped_symbol_node` +//! +//! ``` lua +//! local node = file:push_scoped_symbol_node(symbol, scope) +//! ``` +//! +//! Adds a new push scoped symbol node to this file. `symbol` must be a string, or an instance +//! that can be converted to a string via its `tostring` method. `scope` must be an exported scope +//! node. +//! +//! #### `push_symbol_node` +//! +//! ``` lua +//! local node = file:push_symbol_node(symbol) +//! ``` +//! +//! Adds a new push symbol node to this file. `symbol` must be a string, or an instance that can +//! be converted to a string via its `tostring` method. +//! +//! #### `reference_node` +//! +//! ``` lua +//! local node = file:reference_node(symbol) +//! ``` +//! +//! Adds a new definition node to this file. `symbol` must be a string, or an instance that can be +//! converted to a string via its `tostring` method. +//! +//! #### `root_node` +//! +//! ``` lua +//! local node = file:root_node() +//! ``` +//! +//! Returns the root node of the graph containing this file. +//! +//! #### `scoped_definition_node` +//! +//! ``` lua +//! local node = file:scoped_definition_node(symbol) +//! ``` +//! +//! Adds a new scoped definition node to this file. `symbol` must be a string, or an instance that +//! can be converted to a string via its `tostring` method. +//! +//! #### `scoped_reference_node` +//! +//! ``` lua +//! local node = file:scoped_reference_node(symbol) +//! ``` +//! +//! Adds a new scoped reference node to this file. `symbol` must be a string, or an instance that +//! can be converted to a string via its `tostring` method. +//! +//! ### Nodes +//! +//! The following Lua methods are available on a node instance: +//! +//! #### `add_edge_from` +//! +//! ``` lua +//! node:add_edge_from(other, precedence) +//! ``` +//! +//! Adds an edge from another node to this node. `precedence` is optional; it defaults to 0 if not +//! given. +//! +//! #### `add_edge_to` +//! +//! ``` lua +//! node:add_edge_to(other, precedence) +//! ``` +//! +//! Adds an edge from this node to another node. `precedence` is optional; it defaults to 0 if not +//! given. +//! +//! #### `debug_info` +//! +//! ``` lua +//! let info = node:debug_info() +//! ``` +//! +//! Returns a Lua table containing all of the debug info added to this node. +//! +//! #### `definiens_span` +//! +//! ``` lua +//! let span = node:definiens_span() +//! ``` +//! +//! Returns the definiens span of this node. (See [`set_definiens_span`](#set_definiens_span) for +//! the structure of a span.) +//! +//! #### `local_id` +//! +//! ``` lua +//! let local_id = node:local_id() +//! ``` +//! +//! Returns the local ID of this node within its file. +//! +//! #### `outgoing_edges` +//! +//! ``` lua +//! let edges = node:outgoing_edges() +//! ``` +//! +//! Returns an array containing all of the edges leaving this node. +//! +//! #### `set_debug_info` +//! +//! ``` lua +//! node:add_debug_info(key, value) +//! ``` +//! +//! Adds a new debug info to this node. `key` and `value` must each be a string, or an instance +//! that can be converted to a string via its `tostring` method. +//! +//! #### `set_definiens_span` +//! +//! ``` lua +//! node:set_definiens_span { +//! start = { +//! line = 1, +//! column = { utf8_offset = 1, utf16_offset = 1, grapheme_offset = 1 }, +//! -- UTF-8 offsets within the source file of the line containing the span +//! containing_line = { start = 1, end = 14 }, +//! -- UTF-8 offsets within the source file of the line containing the span, with leading and +//! -- trailing whitespace removed +//! trimmed_line = { start = 2, end = 12 }, +//! }, +//! end = { +//! line = 2, +//! column = { utf8_offset = 12, utf16_offset = 10, grapheme_offset = 8 }, +//! containing_line = { start = 1, end = 14 }, +//! trimmed_line = { start = 1, end = 14 }, +//! }, +//! } +//! ``` +//! +//! Sets the definiens span of this node. All entries in the table are optional, and default to 0 +//! if not provided. +//! +//! #### `set_span` +//! +//! ``` lua +//! node:set_span { +//! start = { +//! line = 1, +//! column = { utf8_offset = 1, utf16_offset = 1, grapheme_offset = 1 }, +//! -- UTF-8 offsets within the source file of the line containing the span +//! containing_line = { start = 1, end = 14 }, +//! -- UTF-8 offsets within the source file of the line containing the span, with leading and +//! -- trailing whitespace removed +//! trimmed_line = { start = 2, end = 12 }, +//! }, +//! end = { +//! line = 2, +//! column = { utf8_offset = 12, utf16_offset = 10, grapheme_offset = 8 }, +//! containing_line = { start = 1, end = 14 }, +//! trimmed_line = { start = 1, end = 14 }, +//! }, +//! } +//! ``` +//! +//! Sets the span of this node. All entries in the table are optional, and default to 0 if not +//! provided. +//! +//! #### `set_syntax_type` +//! +//! ``` lua +//! node:set_syntax_type(syntax_type) +//! ``` +//! +//! Sets the syntax type of this node. `syntax_type` must be a string, or an instance that can be +//! converted to a string via its `tostring` method. +//! +//! #### `span` +//! +//! ``` lua +//! let span = node:span() +//! ``` +//! +//! Returns the span of this node. (See [`set_span`](#set_span) for the structure of a span.) +//! +//! #### `syntax_type` +//! +//! ``` lua +//! let syntax_type = node:syntax_type() +//! ``` +//! +//! Returns the syntax type of this node. + +// Implementation notes: Stack graphs, files, and nodes can live inside the Lua interpreter as +// objects. They are each wrapped in a userdata, with a metatable defining the methods that are +// available. With mlua, the UserData trait is the way to define these metatables and methods. +// +// Complicating matters is that files and nodes need to be represented by a _pair_ of Lua values: +// the handle of the file or node, and a reference to the StackGraph that the file or node lives +// in. We need both because some of the methods need to dereference the handle to get e.g. the +// `Node` instance. It's not safe to dereference the handle when we create the userdata, because +// the resulting pointer is not guaranteed to be stable. (If you add another node, the arena's +// storage might get resized, moving the node instances around in memory.) +// +// To handle this, we leverage Lua's ability to associate “user values” with each userdata. For +// files and nodes, we store the graph's userdata (i.e. its Lua representation) as the user value +// of each file and node userdata. +// +// That, in turn, means that we must use `add_function` to define each metatable method, since that +// gives us an `mlua::AnyUserData`, which lets us access the userdata's underlying Rust value _and_ +// its user value. (Typically, you would use the more ergonomic `add_method` or `add_method_mut`, +// which take care of unwrapping the userdata and giving you a &ref or &mut ref to the underlying +// Rust type. But then you don't have access to the userdata's user value.) + +use std::fmt::Write; +use std::num::NonZeroU32; + +use controlled_option::ControlledOption; +use lsp_positions::Span; +use mlua::AnyUserData; +use mlua::Lua; +use mlua::Scope; +use mlua::UserData; +use mlua::UserDataMethods; + +use crate::arena::Handle; +use crate::graph::Edge; +use crate::graph::File; +use crate::graph::Node; +use crate::graph::StackGraph; + +impl StackGraph { + // Returns a Lua wrapper for this stack graph. Takes ownership of the stack graph. If you + // want to access the stack graph after your Lua code is done with it, use [`lua_ref_mut`] + // instead. + pub fn lua_value<'lua>(self, lua: &'lua Lua) -> Result, mlua::Error> { + lua.create_userdata(self) + } + + // Returns a scoped Lua wrapper for this stack graph. + pub fn lua_ref_mut<'lua, 'scope>( + &'scope mut self, + scope: &Scope<'lua, 'scope>, + ) -> Result, mlua::Error> { + scope.create_userdata_ref_mut(self) + } + + // Returns a scoped Lua wrapper for a file in this stack graph. + pub fn file_lua_ref_mut<'lua, 'scope>( + &'scope mut self, + file: Handle, + scope: &Scope<'lua, 'scope>, + ) -> Result, mlua::Error> { + let graph_ud = self.lua_ref_mut(scope)?; + let file_ud = scope.create_userdata(file)?; + file_ud.set_user_value(graph_ud)?; + Ok(file_ud) + } +} + +impl UserData for StackGraph { + fn add_methods<'lua, M: UserDataMethods<'lua, Self>>(methods: &mut M) { + methods.add_function("edges", |l, graph_ud: AnyUserData| { + let graph = graph_ud.borrow::()?; + let mut edges = Vec::new(); + for node in graph.iter_nodes() { + for edge in graph.outgoing_edges(node) { + let edge_ud = l.create_userdata(edge)?; + edge_ud.set_user_value(graph_ud.clone())?; + edges.push(edge_ud); + } + } + Ok(edges) + }); + + methods.add_function("file", |l, (graph_ud, name): (AnyUserData, String)| { + let file = { + let mut graph = graph_ud.borrow_mut::()?; + graph.get_or_create_file(&name) + }; + let file_ud = l.create_userdata(file)?; + file_ud.set_user_value(graph_ud)?; + Ok(file_ud) + }); + + methods.add_function("jump_to_node", |l, graph_ud: AnyUserData| { + let node = StackGraph::jump_to_node(); + let node_ud = l.create_userdata(node)?; + node_ud.set_user_value(graph_ud)?; + Ok(node_ud) + }); + + methods.add_function("nodes", |l, graph_ud: AnyUserData| { + let iter = l.create_function( + |l, (graph_ud, prev_node_ud): (AnyUserData, Option)| { + let prev_index = match prev_node_ud { + Some(prev_node_ud) => { + let prev_node = prev_node_ud.borrow::>()?; + prev_node.as_u32() + } + None => 0, + }; + let node_index = { + let graph = graph_ud.borrow::()?; + let node_count = graph.nodes.len() as u32; + if prev_index == node_count - 1 { + return Ok(None); + } + unsafe { NonZeroU32::new_unchecked(prev_index + 1) } + }; + let node = Handle::new(node_index); + let node_ud = l.create_userdata::>(node)?; + node_ud.set_user_value(graph_ud)?; + Ok(Some(node_ud)) + }, + )?; + Ok((iter, graph_ud, None::)) + }); + + methods.add_function("root_node", |l, graph_ud: AnyUserData| { + let node = StackGraph::root_node(); + let node_ud = l.create_userdata(node)?; + node_ud.set_user_value(graph_ud)?; + Ok(node_ud) + }); + } +} + +impl UserData for Handle { + fn add_methods<'lua, M: UserDataMethods<'lua, Self>>(methods: &mut M) { + methods.add_function( + "definition_node", + |l, (file_ud, symbol): (AnyUserData, String)| { + let file = *file_ud.borrow::>()?; + let graph_ud = file_ud.user_value::()?; + let node = { + let mut graph = graph_ud.borrow_mut::()?; + let symbol = graph.add_symbol(&symbol); + let node_id = graph.new_node_id(file); + graph + .add_pop_symbol_node(node_id, symbol, true) + .expect("Node ID collision") + }; + let node_ud = l.create_userdata(node)?; + node_ud.set_user_value(graph_ud)?; + Ok(node_ud) + }, + ); + + methods.add_function("drop_scopes_node", |l, file_ud: AnyUserData| { + let file = *file_ud.borrow::>()?; + let graph_ud = file_ud.user_value::()?; + let node = { + let mut graph = graph_ud.borrow_mut::()?; + let node_id = graph.new_node_id(file); + graph + .add_drop_scopes_node(node_id) + .expect("Node ID collision") + }; + let node_ud = l.create_userdata(node)?; + node_ud.set_user_value(graph_ud)?; + Ok(node_ud) + }); + + methods.add_function("edges", |l, file_ud: AnyUserData| { + let file = *file_ud.borrow::>()?; + let graph_ud = file_ud.user_value::()?; + let graph = graph_ud.borrow::()?; + let mut edges = Vec::new(); + // First find any edges from the singleton nodes _to_ a node in this file. + for edge in graph.outgoing_edges(StackGraph::root_node()) { + if !graph[edge.sink].file().map(|f| f == file).unwrap_or(false) { + continue; + } + let edge_ud = l.create_userdata(edge)?; + edge_ud.set_user_value(graph_ud.clone())?; + edges.push(edge_ud); + } + for edge in graph.outgoing_edges(StackGraph::jump_to_node()) { + if !graph[edge.sink].file().map(|f| f == file).unwrap_or(false) { + continue; + } + let edge_ud = l.create_userdata(edge)?; + edge_ud.set_user_value(graph_ud.clone())?; + edges.push(edge_ud); + } + // Then find any edges _starting_ from a node in this file. + for node in graph.nodes_for_file(file) { + for edge in graph.outgoing_edges(node) { + let edge_ud = l.create_userdata(edge)?; + edge_ud.set_user_value(graph_ud.clone())?; + edges.push(edge_ud); + } + } + Ok(edges) + }); + + methods.add_function("exported_scope_node", |l, file_ud: AnyUserData| { + let file = *file_ud.borrow::>()?; + let graph_ud = file_ud.user_value::()?; + let node = { + let mut graph = graph_ud.borrow_mut::()?; + let node_id = graph.new_node_id(file); + graph + .add_scope_node(node_id, true) + .expect("Node ID collision") + }; + let node_ud = l.create_userdata(node)?; + node_ud.set_user_value(graph_ud)?; + Ok(node_ud) + }); + + methods.add_function("internal_scope_node", |l, file_ud: AnyUserData| { + let file = *file_ud.borrow::>()?; + let graph_ud = file_ud.user_value::()?; + let node = { + let mut graph = graph_ud.borrow_mut::()?; + let node_id = graph.new_node_id(file); + graph + .add_scope_node(node_id, false) + .expect("Node ID collision") + }; + let node_ud = l.create_userdata(node)?; + node_ud.set_user_value(graph_ud)?; + Ok(node_ud) + }); + + methods.add_function("jump_to_node", |l, file_ud: AnyUserData| { + let graph_ud = file_ud.user_value::()?; + let node = StackGraph::jump_to_node(); + let node_ud = l.create_userdata(node)?; + node_ud.set_user_value(graph_ud)?; + Ok(node_ud) + }); + + methods.add_function("nodes", |l, file_ud: AnyUserData| { + let iter = l.create_function( + |l, (file_ud, prev_node_ud): (AnyUserData, Option)| { + // Pull out the node handle from the previous iteration. + let prev_index = match prev_node_ud { + Some(prev_node_ud) => { + let prev_node = prev_node_ud.borrow::>()?; + prev_node.as_u32() + } + None => 0, + }; + + // Loop through the next node handles until we find one belonging to the file. + let graph_ud = file_ud.user_value::()?; + let node = { + let file = *file_ud.borrow::>()?; + let graph = graph_ud.borrow::()?; + let node_count = graph.nodes.len() as u32; + let mut node_index = unsafe { NonZeroU32::new_unchecked(prev_index + 1) }; + loop { + let handle = Handle::::new(node_index); + + // If we reach the end without finding a matching node, return nil + // to terminate the iterator. + if node_index.get() == node_count { + return Ok(None); + } + + // If the node belongs to the file, break out of the loop to use this + // node as the next result of the iterator. + if graph[handle].file().map(|f| f == file).unwrap_or(false) { + break handle; + } + + // Otherwise try the next node. + node_index = node_index.checked_add(1).unwrap(); + } + }; + + // Wrap up the node handle that we just found. + let node_ud = l.create_userdata::>(node)?; + node_ud.set_user_value(graph_ud)?; + Ok(Some(node_ud)) + }, + )?; + Ok((iter, file_ud, None::)) + }); + + methods.add_function( + "pop_scoped_symbol_node", + |l, (file_ud, symbol): (AnyUserData, String)| { + let file = *file_ud.borrow::>()?; + let graph_ud = file_ud.user_value::()?; + let node = { + let mut graph = graph_ud.borrow_mut::()?; + let symbol = graph.add_symbol(&symbol); + let node_id = graph.new_node_id(file); + graph + .add_pop_scoped_symbol_node(node_id, symbol, false) + .expect("Node ID collision") + }; + let node_ud = l.create_userdata(node)?; + node_ud.set_user_value(graph_ud)?; + Ok(node_ud) + }, + ); + + methods.add_function( + "pop_symbol_node", + |l, (file_ud, symbol): (AnyUserData, String)| { + let file = *file_ud.borrow::>()?; + let graph_ud = file_ud.user_value::()?; + let node = { + let mut graph = graph_ud.borrow_mut::()?; + let symbol = graph.add_symbol(&symbol); + let node_id = graph.new_node_id(file); + graph + .add_pop_symbol_node(node_id, symbol, false) + .expect("Node ID collision") + }; + let node_ud = l.create_userdata(node)?; + node_ud.set_user_value(graph_ud)?; + Ok(node_ud) + }, + ); + + methods.add_function( + "push_scoped_symbol_node", + |l, (file_ud, symbol, scope_ud): (AnyUserData, String, AnyUserData)| { + let file = *file_ud.borrow::>()?; + let graph_ud = file_ud.user_value::()?; + let scope = *scope_ud.borrow::>()?; + let node = { + let mut graph = graph_ud.borrow_mut::()?; + let scope_id = { + let scope = &graph[scope]; + if !scope.is_exported_scope() { + return Err(mlua::Error::RuntimeError( + "Can only push exported scope nodes".to_string(), + )); + } + scope.id() + }; + let symbol = graph.add_symbol(&symbol); + let node_id = graph.new_node_id(file); + graph + .add_push_scoped_symbol_node(node_id, symbol, scope_id, false) + .expect("Node ID collision") + }; + let node_ud = l.create_userdata(node)?; + node_ud.set_user_value(graph_ud)?; + Ok(node_ud) + }, + ); + + methods.add_function( + "push_symbol_node", + |l, (file_ud, symbol): (AnyUserData, String)| { + let file = *file_ud.borrow::>()?; + let graph_ud = file_ud.user_value::()?; + let node = { + let mut graph = graph_ud.borrow_mut::()?; + let symbol = graph.add_symbol(&symbol); + let node_id = graph.new_node_id(file); + graph + .add_push_symbol_node(node_id, symbol, false) + .expect("Node ID collision") + }; + let node_ud = l.create_userdata(node)?; + node_ud.set_user_value(graph_ud)?; + Ok(node_ud) + }, + ); + + methods.add_function( + "reference_node", + |l, (file_ud, symbol): (AnyUserData, String)| { + let file = *file_ud.borrow::>()?; + let graph_ud = file_ud.user_value::()?; + let node = { + let mut graph = graph_ud.borrow_mut::()?; + let symbol = graph.add_symbol(&symbol); + let node_id = graph.new_node_id(file); + graph + .add_push_symbol_node(node_id, symbol, true) + .expect("Node ID collision") + }; + let node_ud = l.create_userdata(node)?; + node_ud.set_user_value(graph_ud)?; + Ok(node_ud) + }, + ); + + methods.add_function("root_node", |l, file_ud: AnyUserData| { + let graph_ud = file_ud.user_value::()?; + let node = StackGraph::root_node(); + let node_ud = l.create_userdata(node)?; + node_ud.set_user_value(graph_ud)?; + Ok(node_ud) + }); + + methods.add_function( + "scoped_definition_node", + |l, (file_ud, symbol): (AnyUserData, String)| { + let file = *file_ud.borrow::>()?; + let graph_ud = file_ud.user_value::()?; + let node = { + let mut graph = graph_ud.borrow_mut::()?; + let symbol = graph.add_symbol(&symbol); + let node_id = graph.new_node_id(file); + graph + .add_pop_scoped_symbol_node(node_id, symbol, true) + .expect("Node ID collision") + }; + let node_ud = l.create_userdata(node)?; + node_ud.set_user_value(graph_ud)?; + Ok(node_ud) + }, + ); + + methods.add_function( + "scoped_reference_node", + |l, (file_ud, symbol, scope_ud): (AnyUserData, String, AnyUserData)| { + let file = *file_ud.borrow::>()?; + let graph_ud = file_ud.user_value::()?; + let scope = *scope_ud.borrow::>()?; + let node = { + let mut graph = graph_ud.borrow_mut::()?; + let scope_id = { + let scope = &graph[scope]; + if !scope.is_exported_scope() { + return Err(mlua::Error::RuntimeError( + "Can only push exported scope nodes".to_string(), + )); + } + scope.id() + }; + let symbol = graph.add_symbol(&symbol); + let node_id = graph.new_node_id(file); + graph + .add_push_scoped_symbol_node(node_id, symbol, scope_id, true) + .expect("Node ID collision") + }; + let node_ud = l.create_userdata(node)?; + node_ud.set_user_value(graph_ud)?; + Ok(node_ud) + }, + ); + } +} + +impl UserData for Handle { + fn add_methods<'lua, M: UserDataMethods<'lua, Self>>(methods: &mut M) { + methods.add_function( + "add_edge_from", + |l, (this_ud, from_ud, precedence): (AnyUserData, AnyUserData, Option)| { + let this = *this_ud.borrow::>()?; + let from = *from_ud.borrow::>()?; + let graph_ud = this_ud.user_value::()?; + let precedence = precedence.unwrap_or(0); + { + let mut graph = graph_ud.borrow_mut::()?; + graph.add_edge(from, this, precedence); + } + let edge = Edge { + source: from, + sink: this, + precedence, + }; + let edge_ud = l.create_userdata(edge)?; + edge_ud.set_user_value(graph_ud)?; + Ok(edge_ud) + }, + ); + + methods.add_function( + "add_edge_to", + |l, (this_ud, to_ud, precedence): (AnyUserData, AnyUserData, Option)| { + let this = *this_ud.borrow::>()?; + let to = *to_ud.borrow::>()?; + let graph_ud = this_ud.user_value::()?; + let precedence = precedence.unwrap_or(0); + { + let mut graph = graph_ud.borrow_mut::()?; + graph.add_edge(this, to, precedence); + } + let edge = Edge { + source: this, + sink: to, + precedence, + }; + let edge_ud = l.create_userdata(edge)?; + edge_ud.set_user_value(graph_ud)?; + Ok(edge_ud) + }, + ); + + methods.add_function("debug_info", |l, node_ud: AnyUserData| { + let node = *node_ud.borrow::>()?; + let graph_ud = node_ud.user_value::()?; + let graph = graph_ud.borrow::()?; + let debug_info = match graph.node_debug_info(node) { + Some(debug_info) => debug_info, + None => return Ok(None), + }; + let result = l.create_table()?; + for entry in debug_info.iter() { + result.set(&graph[entry.key], &graph[entry.value])?; + } + Ok(Some(result)) + }); + + methods.add_function("definiens_span", |_, node_ud: AnyUserData| { + let node = *node_ud.borrow::>()?; + let graph_ud = node_ud.user_value::()?; + let graph = graph_ud.borrow::()?; + let source_info = match graph.source_info(node) { + Some(source_info) => source_info, + None => return Ok(None), + }; + Ok(Some(source_info.definiens_span.clone())) + }); + + methods.add_function("local_id", |_, node_ud: AnyUserData| { + let node = *node_ud.borrow::>()?; + let graph_ud = node_ud.user_value::()?; + let graph = graph_ud.borrow::()?; + Ok(graph[node].id().local_id()) + }); + + methods.add_function("outgoing_edges", |l, node_ud: AnyUserData| { + let node = *node_ud.borrow::>()?; + let graph_ud = node_ud.user_value::()?; + let graph = graph_ud.borrow::()?; + let mut edges = Vec::new(); + for edge in graph.outgoing_edges(node) { + let edge_ud = l.create_userdata(edge)?; + edge_ud.set_user_value(graph_ud.clone())?; + edges.push(edge_ud); + } + Ok(edges) + }); + + methods.add_function( + "set_debug_info", + |_, (node_ud, k, v): (AnyUserData, String, String)| { + let node = *node_ud.borrow::>()?; + let graph_ud = node_ud.user_value::()?; + let mut graph = graph_ud.borrow_mut::()?; + let k = graph.add_string(&k); + let v = graph.add_string(&v); + graph.node_debug_info_mut(node).add(k, v); + Ok(()) + }, + ); + + methods.add_function( + "set_definiens_span", + |_, (node_ud, definiens_span): (AnyUserData, Span)| { + let node = *node_ud.borrow::>()?; + let graph_ud = node_ud.user_value::()?; + let mut graph = graph_ud.borrow_mut::()?; + graph.source_info_mut(node).definiens_span = definiens_span; + Ok(()) + }, + ); + + methods.add_function("set_span", |_, (node_ud, span): (AnyUserData, Span)| { + let node = *node_ud.borrow::>()?; + let graph_ud = node_ud.user_value::()?; + let mut graph = graph_ud.borrow_mut::()?; + graph.source_info_mut(node).span = span; + Ok(()) + }); + + methods.add_function( + "set_syntax_type", + |_, (node_ud, syntax_type): (AnyUserData, String)| { + let node = *node_ud.borrow::>()?; + let graph_ud = node_ud.user_value::()?; + let mut graph = graph_ud.borrow_mut::()?; + let syntax_type = graph.add_string(&syntax_type); + graph.source_info_mut(node).syntax_type = ControlledOption::some(syntax_type); + Ok(()) + }, + ); + + methods.add_function("span", |_, node_ud: AnyUserData| { + let node = *node_ud.borrow::>()?; + let graph_ud = node_ud.user_value::()?; + let graph = graph_ud.borrow::()?; + let source_info = match graph.source_info(node) { + Some(source_info) => source_info, + None => return Ok(None), + }; + Ok(Some(source_info.span.clone())) + }); + + methods.add_function("syntax_type", |_, node_ud: AnyUserData| { + let node = *node_ud.borrow::>()?; + let graph_ud = node_ud.user_value::()?; + let graph = graph_ud.borrow::()?; + let source_info = match graph.source_info(node) { + Some(source_info) => source_info, + None => return Ok(None), + }; + let syntax_type = match source_info.syntax_type.into_option() { + Some(syntax_type) => syntax_type, + None => return Ok(None), + }; + Ok(Some(graph[syntax_type].to_string())) + }); + + methods.add_meta_function(mlua::MetaMethod::ToString, |_, node_ud: AnyUserData| { + let node = *node_ud.borrow::>()?; + let graph_ud = node_ud.user_value::()?; + let graph = graph_ud.borrow::()?; + let mut display = graph[node].display(&graph).to_string(); + if let Some(source_info) = graph.source_info(node) { + display.pop(); // remove the trailing ] + if let Some(syntax_type) = source_info.syntax_type.into_option() { + write!(&mut display, " ({})", syntax_type.display(&graph)).unwrap(); + } + if source_info.span != Span::default() { + write!( + &mut display, + " at {}:{}-{}:{}", + source_info.span.start.line, + source_info.span.start.column.utf8_offset, + source_info.span.end.line, + source_info.span.end.column.utf8_offset, + ) + .unwrap(); + } + if source_info.definiens_span != Span::default() { + write!( + &mut display, + " def {}:{}-{}:{}", + source_info.definiens_span.start.line, + source_info.definiens_span.start.column.utf8_offset, + source_info.definiens_span.end.line, + source_info.definiens_span.end.column.utf8_offset, + ) + .unwrap(); + } + display.push(']'); + } + Ok(display) + }); + } +} + +impl UserData for Edge { + fn add_methods<'lua, M: UserDataMethods<'lua, Self>>(methods: &mut M) { + methods.add_meta_function(mlua::MetaMethod::ToString, |_, edge_ud: AnyUserData| { + let edge = *edge_ud.borrow::()?; + let graph_ud = edge_ud.user_value::()?; + let graph = graph_ud.borrow::()?; + let display = format!( + "{} -{}-> {}", + edge.source.display(&graph), + edge.precedence, + edge.sink.display(&graph), + ); + Ok(display) + }); + } +} diff --git a/stack-graphs/tests/it/lua.rs b/stack-graphs/tests/it/lua.rs new file mode 100644 index 000000000..fe4e96d7e --- /dev/null +++ b/stack-graphs/tests/it/lua.rs @@ -0,0 +1,238 @@ +// -*- coding: utf-8 -*- +// ------------------------------------------------------------------------------------------------ +// Copyright © 2023, stack-graphs authors. +// Licensed under either of Apache License, Version 2.0, or MIT license, at your option. +// Please see the LICENSE-APACHE or LICENSE-MIT files in this distribution for license details. +// ------------------------------------------------------------------------------------------------ + +use lua_helpers::new_lua; +use stack_graphs::graph::NodeID; +use stack_graphs::graph::StackGraph; + +trait CheckLua { + fn check(&self, graph: &mut StackGraph, chunk: &str) -> Result<(), mlua::Error>; +} + +impl CheckLua for mlua::Lua { + fn check(&self, graph: &mut StackGraph, chunk: &str) -> Result<(), mlua::Error> { + self.scope(|scope| { + let graph = graph.lua_ref_mut(&scope)?; + self.load(chunk).set_name("test chunk").call(graph) + }) + } +} + +#[test] +fn can_create_nodes_from_lua() -> Result<(), anyhow::Error> { + let l = new_lua()?; + let mut graph = StackGraph::new(); + l.check( + &mut graph, + r#" + local graph = ... + local file = graph:file("test.py") + local n0 = file:internal_scope_node() + local n1 = file:internal_scope_node() + assert_eq("local ID", 0, n0:local_id()) + assert_eq("local ID", 1, n1:local_id()) + "#, + )?; + + let node_count = graph.iter_nodes().count(); + assert_eq!(node_count, 4); // Include the predefined ROOT and JUMP TO nodes in the count + + let file = graph.get_file("test.py").expect("Cannot find file"); + let n0 = graph.node_for_id(NodeID::new_in_file(file, 0)); + assert!(n0.is_some(), "Cannot find node 0"); + let n1 = graph.node_for_id(NodeID::new_in_file(file, 1)); + assert!(n1.is_some(), "Cannot find node 1"); + + Ok(()) +} + +#[test] +fn can_set_source_info_from_lua() -> Result<(), anyhow::Error> { + let l = new_lua()?; + let mut graph = StackGraph::new(); + l.check( + &mut graph, + r#" + local graph = ... + local file = graph:file("test.py") + local n0 = file:internal_scope_node() + + n0:set_syntax_type("function") + assert_eq("syntax type", "function", n0:syntax_type()) + + n0:set_span { + start={line=1, column={utf8_offset=1}}, + ["end"]={line=1, column={utf8_offset=19}}, + } + assert_eq("start line", 1, n0:span().start.line) + assert_eq("start column", 1, n0:span().start.column.utf8_offset) + assert_eq("end line", 1, n0:span()["end"].line) + assert_eq("end column", 19, n0:span()["end"].column.utf8_offset) + + n0:set_definiens_span { + start={line=2, column={utf8_offset=1}}, + ["end"]={line=78, column={utf8_offset=24}}, + } + assert_eq("start line", 2, n0:definiens_span().start.line) + assert_eq("start column", 1, n0:definiens_span().start.column.utf8_offset) + assert_eq("end line", 78, n0:definiens_span()["end"].line) + assert_eq("end column", 24, n0:definiens_span()["end"].column.utf8_offset) + + assert_eq("node", "[test.py(0) scope (function) at 1:1-1:19 def 2:1-78:24]", tostring(n0)) + "#, + )?; + Ok(()) +} + +#[test] +fn can_set_debug_info_from_lua() -> Result<(), anyhow::Error> { + let l = new_lua()?; + let mut graph = StackGraph::new(); + l.check( + &mut graph, + r#" + local graph = ... + local file = graph:file("test.py") + local n0 = file:internal_scope_node() + n0:set_debug_info("k1", "v1") + n0:set_debug_info("k2", "v2") + local expected = { k1="v1", k2="v2" } + assert_deepeq("debug info", expected, n0:debug_info()) + "#, + )?; + Ok(()) +} + +#[test] +fn can_create_edges_from_lua() -> Result<(), anyhow::Error> { + let l = new_lua()?; + let mut graph = StackGraph::new(); + l.check( + &mut graph, + r#" + local graph = ... + local root = graph:root_node() + local file = graph:file("test.py") + local n0 = file:internal_scope_node() + local n1 = file:internal_scope_node() + local e0 = n0:add_edge_to(n1) + local e1 = n0:add_edge_from(n1, 10) + local e2 = n0:add_edge_to(root) + local e3 = n0:add_edge_from(root) + assert_eq("edge", "[test.py(0) scope] -0-> [test.py(1) scope]", tostring(e0)) + assert_eq("edge", "[test.py(1) scope] -10-> [test.py(0) scope]", tostring(e1)) + + assert_deepeq("node edges", { + "[test.py(0) scope] -0-> [root]", + "[test.py(0) scope] -0-> [test.py(1) scope]", + }, iter_tostring(values(n0:outgoing_edges()))) + assert_deepeq("node edges", { + "[test.py(1) scope] -10-> [test.py(0) scope]", + }, iter_tostring(values(n1:outgoing_edges()))) + assert_deepeq("node edges", { + "[root] -0-> [test.py(0) scope]", + }, iter_tostring(values(root:outgoing_edges()))) + + assert_deepeq("file edges", { + "[root] -0-> [test.py(0) scope]", + "[test.py(0) scope] -0-> [root]", + "[test.py(0) scope] -0-> [test.py(1) scope]", + "[test.py(1) scope] -10-> [test.py(0) scope]", + }, iter_tostring(values(file:edges()))) + + assert_deepeq("graph edges", { + "[root] -0-> [test.py(0) scope]", + "[test.py(0) scope] -0-> [root]", + "[test.py(0) scope] -0-> [test.py(1) scope]", + "[test.py(1) scope] -10-> [test.py(0) scope]", + }, iter_tostring(values(graph:edges()))) + "#, + )?; + Ok(()) +} + +#[test] +fn can_create_all_node_types_from_lua() -> Result<(), anyhow::Error> { + let l = new_lua()?; + let mut graph = StackGraph::new(); + l.check( + &mut graph, + r#" + local graph = ... + local root = graph:root_node() + local jump_to = graph:jump_to_node() + local file = graph:file("test.py") + local file_root = file:root_node() + local file_jump_to = file:jump_to_node() + local drop_scopes = file:drop_scopes_node() + local exported = file:exported_scope_node() + local internal = file:internal_scope_node() + local pop_scoped_symbol = file:pop_scoped_symbol_node("foo") + local scoped_definition = file:scoped_definition_node("bar") + local pop_symbol = file:pop_symbol_node("foo") + local definition = file:definition_node("bar") + local push_scoped_symbol = file:push_scoped_symbol_node("foo", exported) + local scoped_reference = file:scoped_reference_node("bar", exported) + local push_symbol = file:push_symbol_node("foo") + local reference = file:reference_node("bar") + + assert_deepeq("nodes", { + "[root]", + "[jump to scope]", + "[test.py(0) drop scopes]", + "[test.py(1) exported scope]", + "[test.py(2) scope]", + "[test.py(3) pop scoped foo]", + "[test.py(4) scoped definition bar]", + "[test.py(5) pop foo]", + "[test.py(6) definition bar]", + "[test.py(7) push scoped foo test.py(1)]", + "[test.py(8) scoped reference bar test.py(1)]", + "[test.py(9) push foo]", + "[test.py(10) reference bar]", + }, iter_tostring(graph:nodes())) + "#, + )?; + Ok(()) +} + +#[test] +fn can_iterate_nodes_in_file() -> Result<(), anyhow::Error> { + let l = new_lua()?; + let mut graph = StackGraph::new(); + l.check( + &mut graph, + r#" + local graph = ... + local file1 = graph:file("test1.py") + local file2 = graph:file("test2.py") + file1:internal_scope_node() + file2:internal_scope_node() + file1:internal_scope_node() + file2:internal_scope_node() + file2:internal_scope_node() + file1:internal_scope_node() + file2:internal_scope_node() + file1:internal_scope_node() + + assert_deepeq("nodes", { + "[test1.py(0) scope]", + "[test1.py(1) scope]", + "[test1.py(2) scope]", + "[test1.py(3) scope]", + }, iter_tostring(file1:nodes())) + + assert_deepeq("nodes", { + "[test2.py(0) scope]", + "[test2.py(1) scope]", + "[test2.py(2) scope]", + "[test2.py(3) scope]", + }, iter_tostring(file2:nodes())) + "#, + )?; + Ok(()) +} diff --git a/stack-graphs/tests/it/main.rs b/stack-graphs/tests/it/main.rs index bea8a6030..f5d12306c 100644 --- a/stack-graphs/tests/it/main.rs +++ b/stack-graphs/tests/it/main.rs @@ -21,6 +21,8 @@ mod can_jump_to_definition; mod can_jump_to_definition_with_forward_partial_path_stitching; mod cycles; mod graph; +#[cfg(feature = "lua")] +mod lua; mod partial; #[cfg(feature = "serde")] mod serde; diff --git a/tree-sitter-stack-graphs/Cargo.toml b/tree-sitter-stack-graphs/Cargo.toml index 2de26a86f..d6e3b911c 100644 --- a/tree-sitter-stack-graphs/Cargo.toml +++ b/tree-sitter-stack-graphs/Cargo.toml @@ -47,6 +47,13 @@ lsp = [ "tokio", "tower-lsp", ] +lua = [ + "dep:mlua", + "dep:mlua-tree-sitter", + "lsp-positions/lua", + "lsp-positions/tree-sitter", + "stack-graphs/lua", +] [dependencies] anyhow = "1.0" @@ -63,6 +70,8 @@ indoc = { version = "1.0", optional = true } itertools = "0.10" log = "0.4" lsp-positions = { version="0.3", path="../lsp-positions", features=["tree-sitter"] } +mlua = { version = "0.9", optional = true } +mlua-tree-sitter = { version = "0.1", git="https://github.com/dcreager/mlua-tree-sitter", optional = true } once_cell = "1" pathdiff = { version = "0.2.1", optional = true } regex = "1" @@ -81,5 +90,6 @@ tree-sitter-loader = "0.20" walkdir = { version = "2.3", optional = true } [dev-dependencies] +lua-helpers = { path = "../lua-helpers" } pretty_assertions = "0.7" tree-sitter-python = "0.19.1" diff --git a/tree-sitter-stack-graphs/src/lib.rs b/tree-sitter-stack-graphs/src/lib.rs index 86651b6be..99ebf660a 100644 --- a/tree-sitter-stack-graphs/src/lib.rs +++ b/tree-sitter-stack-graphs/src/lib.rs @@ -357,6 +357,7 @@ use std::time::Duration; use std::time::Instant; use thiserror::Error; use tree_sitter::Parser; +use tree_sitter::Tree; use tree_sitter_graph::functions::Functions; use tree_sitter_graph::graph::Edge; use tree_sitter_graph::graph::Graph; @@ -375,6 +376,8 @@ pub mod ci; pub mod cli; pub mod functions; pub mod loader; +#[cfg(feature = "lua")] +pub mod lua; pub mod test; mod util; @@ -578,6 +581,29 @@ impl StackGraphLanguage { } } +pub(crate) fn parse_file( + language: tree_sitter::Language, + source: &str, + cancellation_flag: &dyn CancellationFlag, +) -> Result { + let tree = { + let mut parser = Parser::new(); + parser.set_language(language)?; + let ts_cancellation_flag = TreeSitterCancellationFlag::from(cancellation_flag); + // The parser.set_cancellation_flag` is unsafe, because it does not tie the + // lifetime of the parser to the lifetime of the cancellation flag in any way. + // To make it more obvious that the parser does not outlive the cancellation flag, + // it is put into its own block here, instead of extending to the end of the method. + unsafe { parser.set_cancellation_flag(Some(ts_cancellation_flag.as_ref())) }; + parser.parse(source, None).ok_or(BuildError::ParseError)? + }; + let parse_errors = ParseError::into_all(tree); + if parse_errors.errors().len() > 0 { + return Err(BuildError::ParseErrors(parse_errors)); + } + Ok(parse_errors.into_tree()) +} + pub struct Builder<'a> { sgl: &'a StackGraphLanguage, stack_graph: &'a mut StackGraph, @@ -615,24 +641,7 @@ impl<'a> Builder<'a> { globals: &'a Variables<'a>, cancellation_flag: &dyn CancellationFlag, ) -> Result<(), BuildError> { - let tree = { - let mut parser = Parser::new(); - parser.set_language(self.sgl.language)?; - let ts_cancellation_flag = TreeSitterCancellationFlag::from(cancellation_flag); - // The parser.set_cancellation_flag` is unsafe, because it does not tie the - // lifetime of the parser to the lifetime of the cancellation flag in any way. - // To make it more obvious that the parser does not outlive the cancellation flag, - // it is put into its own block here, instead of extending to the end of the method. - unsafe { parser.set_cancellation_flag(Some(ts_cancellation_flag.as_ref())) }; - parser - .parse(self.source, None) - .ok_or(BuildError::ParseError)? - }; - let parse_errors = ParseError::into_all(tree); - if parse_errors.errors().len() > 0 { - return Err(BuildError::ParseErrors(parse_errors)); - } - let tree = parse_errors.into_tree(); + let tree = parse_file(self.sgl.language, self.source, cancellation_flag)?; let mut globals = Variables::nested(globals); if globals.get(&ROOT_NODE_VAR.into()).is_none() { @@ -826,6 +835,9 @@ pub enum BuildError { LanguageError(#[from] tree_sitter::LanguageError), #[error("Expected exported symbol scope in {0}, got {1}")] SymbolScopeError(String, String), + #[cfg(feature = "lua")] + #[error(transparent)] + LuaError(#[from] mlua::Error), } impl From for BuildError { diff --git a/tree-sitter-stack-graphs/src/loader.rs b/tree-sitter-stack-graphs/src/loader.rs index 0feb6a28c..2744bcc16 100644 --- a/tree-sitter-stack-graphs/src/loader.rs +++ b/tree-sitter-stack-graphs/src/loader.rs @@ -743,7 +743,7 @@ impl SupplementedTsLoader { .map_err(LoadError::TreeSitter)?; let configurations = self .0 - .find_language_configurations_at_path(&path) + .find_language_configurations_at_path(&path, false) .map_err(LoadError::TreeSitter)?; let languages = languages .into_iter() diff --git a/tree-sitter-stack-graphs/src/lua.rs b/tree-sitter-stack-graphs/src/lua.rs new file mode 100644 index 000000000..7b0390bf3 --- /dev/null +++ b/tree-sitter-stack-graphs/src/lua.rs @@ -0,0 +1,105 @@ +// -*- coding: utf-8 -*- +// ------------------------------------------------------------------------------------------------ +// Copyright © 2023, stack-graphs authors. +// Licensed under either of Apache License, Version 2.0, or MIT license, at your option. +// Please see the LICENSE-APACHE or LICENSE-MIT files in this distribution for license details. +// ------------------------------------------------------------------------------------------------ + +//! Construct stack graphs using a Lua script that consumes a tree-sitter parse tree + +use std::borrow::Cow; + +use lsp_positions::lua::Module as _; +use mlua::Lua; +use mlua_tree_sitter::Module as _; +use mlua_tree_sitter::WithSource; +use stack_graphs::arena::Handle; +use stack_graphs::graph::File; +use stack_graphs::graph::StackGraph; + +use crate::parse_file; +use crate::BuildError; +use crate::CancellationFlag; + +/// Holds information about how to construct stack graphs for a particular language. +pub struct StackGraphLanguageLua { + language: tree_sitter::Language, + lua_source: Cow<'static, [u8]>, + lua_source_name: String, +} + +impl StackGraphLanguageLua { + /// Creates a new stack graph language for the given language, loading the Lua stack graph + /// construction rules from a static string. + pub fn from_static_str( + language: tree_sitter::Language, + lua_source: &'static [u8], + lua_source_name: &str, + ) -> StackGraphLanguageLua { + StackGraphLanguageLua { + language, + lua_source: Cow::from(lua_source), + lua_source_name: lua_source_name.to_string(), + } + } + + /// Creates a new stack graph language for the given language, loading the Lua stack graph + /// construction rules from a string. + pub fn from_str( + language: tree_sitter::Language, + lua_source: &[u8], + lua_source_name: &str, + ) -> StackGraphLanguageLua { + StackGraphLanguageLua { + language, + lua_source: Cow::from(lua_source.to_vec()), + lua_source_name: lua_source_name.to_string(), + } + } + + pub fn language(&self) -> tree_sitter::Language { + self.language + } + + pub fn lua_source_name(&self) -> &str { + &self.lua_source_name + } + + pub fn lua_source(&self) -> &Cow<'static, [u8]> { + &self.lua_source + } + + /// Executes the graph construction rules for this language against a source file, creating new + /// nodes and edges in `stack_graph`. Any new nodes that we create will belong to `file`. + /// (The source file must be implemented in this language, otherwise you'll probably get a + /// parse error.) + pub fn build_stack_graph_into<'a>( + &'a self, + stack_graph: &'a mut StackGraph, + file: Handle, + source: &'a str, + cancellation_flag: &'a dyn CancellationFlag, + ) -> Result<(), BuildError> { + // Create a Lua environment and load the language's stack graph rules. + // TODO: Sandbox the Lua environment + let lua = Lua::new(); + lua.open_lsp_positions()?; + lua.open_ltreesitter()?; + lua.load(self.lua_source.as_ref()) + .set_name(&self.lua_source_name) + .exec()?; + let process: mlua::Function = lua.globals().get("process")?; + + // Parse the source using the requested grammar. + let tree = parse_file(self.language, source, cancellation_flag)?; + let tree = tree.with_source(source.as_bytes()); + + // Invoke the Lua `process` function with the parsed tree and the stack graph file. + // TODO: Add a debug hook that checks the cancellation flag during execution + lua.scope(|scope| { + let file = stack_graph.file_lua_ref_mut(file, scope)?; + process.call((tree, file)) + })?; + Ok(()) + } +} diff --git a/tree-sitter-stack-graphs/tests/it/lua.rs b/tree-sitter-stack-graphs/tests/it/lua.rs new file mode 100644 index 000000000..a4184a042 --- /dev/null +++ b/tree-sitter-stack-graphs/tests/it/lua.rs @@ -0,0 +1,66 @@ +// -*- coding: utf-8 -*- +// ------------------------------------------------------------------------------------------------ +// Copyright © 2023, stack-graphs authors. +// Licensed under either of Apache License, Version 2.0, or MIT license, at your option. +// Please see the LICENSE-APACHE or LICENSE-MIT files in this distribution for license details. +// ------------------------------------------------------------------------------------------------ + +use lua_helpers::new_lua; +use stack_graphs::graph::StackGraph; +use tree_sitter_stack_graphs::lua::StackGraphLanguageLua; +use tree_sitter_stack_graphs::NoCancellation; + +trait CheckLua { + fn check(&self, graph: &mut StackGraph, chunk: &str) -> Result<(), mlua::Error>; +} + +impl CheckLua for mlua::Lua { + fn check(&self, graph: &mut StackGraph, chunk: &str) -> Result<(), mlua::Error> { + self.scope(|scope| { + let graph = graph.lua_ref_mut(&scope)?; + self.load(chunk).set_name("test chunk").call(graph) + }) + } +} + +// This doesn't build a very _interesting_ stack graph, but it does test that the end-to-end +// spackle all works correctly. +#[test] +fn can_build_stack_graph_from_lua() -> Result<(), anyhow::Error> { + const LUA: &[u8] = br#" + function process(parsed, file) + local sc = lsp_positions.SpanCalculator.new_from_tree(parsed) + local module_ast = parsed:root() + local module = file:internal_scope_node() + module:set_definiens_span(sc:for_node(module_ast)) + module:add_edge_from(file:root_node()) + end + "#; + + let code = r#" + def double(x): + return x * 2 + "#; + let mut graph = StackGraph::new(); + let file = graph.get_or_create_file("test.py"); + let language = + StackGraphLanguageLua::from_static_str(tree_sitter_python::language(), LUA, "test"); + language.build_stack_graph_into(&mut graph, file, code, &NoCancellation)?; + + let l = new_lua()?; + l.check( + &mut graph, + r#" + local graph = ... + local file = graph:file("test.py") + assert_deepeq("nodes", { + "[test.py(0) scope def 1:6-3:4]", + }, iter_tostring(file:nodes())) + assert_deepeq("edges", { + "[root] -0-> [test.py(0) scope]", + }, iter_tostring(values(file:edges()))) + "#, + )?; + + Ok(()) +} diff --git a/tree-sitter-stack-graphs/tests/it/main.rs b/tree-sitter-stack-graphs/tests/it/main.rs index c1e37a40e..c4e5d4bc9 100644 --- a/tree-sitter-stack-graphs/tests/it/main.rs +++ b/tree-sitter-stack-graphs/tests/it/main.rs @@ -19,6 +19,9 @@ mod loader; mod nodes; mod test; +#[cfg(feature = "lua")] +mod lua; + pub(self) fn build_stack_graph( python_source: &str, tsg_source: &str,