feat: Implement indexing using AST (#14)

Author: felix-andreas

Cargo.lock

@@ -4,28 +4,27 @@ version = "0.0.3"
 edition = "2024"
 
 [dependencies]
-clap = { version = "4", features = ["derive"] }
-console = "0.15.10"
+clap = { version = "4.5.38", features = ["derive"] }
+console = "0.15.11"
 dashmap = "6.1.0"
 ignore = "0.4.23"
-indoc = "2.0.5"
+indoc = "2.0.6"
 itertools = "0.14.0"
-regex = "1.11.1"
-ropey = "1"
-serde = "1.0.218"
+ropey = "1.6.1"
+serde = "1.0.219"
 similar = { version = "2.7.0", features = ["inline"] }
-thiserror = "2.0.11"
+thiserror = "2.0.12"
 tokio = { version = "1", features = [
 	"rt-multi-thread",
 	"macros",
 	"io-std",
 	"time",
 ] }
-toml = "0.8.20"
+toml = "0.8.22"
 tracing = "0.1.41"
 tracing-subscriber = { version = "0.3.19", features = ["env-filter"] }
 tracing-tree = "0.4.0"
-tree-sitter = "0.25.1"
+tree-sitter = "0.25.4"
 tree-sitter-r = "1.1.0"
 # tree-sitter-r = { path = "../tree-sitter-r/" }
 
@@ -38,10 +37,10 @@ tower = { version = "0.5.2", optional = true }
 futures = { version = "0.3.31", optional = true }
 
 [target.'cfg(not(unix))'.dependencies]
-tokio-util = { version = "0.7.14", features = ["compat"] }
+tokio-util = { version = "0.7.15", features = ["compat"] }
 
 [dev-dependencies]
-insta = "1"
+insta = "1.43.1"
 
 [features]
 default = ["async-lsp"]

Cargo.toml

@@ -111,6 +111,13 @@ if cursor.goto_first_child() {
 out
 ```
 
+## Tree-sitter: How to handle required fields
+
+* For formatting we do an initial check if there are no errors, so we can safely assume that all required fields are present
+* For type-checking we should do the same
+* For checks/diagnostics that run while typing (syntax & fast), we cannot make any assumption
+* Same is true for index. It should still be possible to index a file, while there are parse errors
+
 ## tree-sitter-r vs R parser
 
 ### No default for parameter

docs/content/development.md

@@ -71,6 +71,7 @@
                   ];
                 }
               ))
+              cargo-edit
               cargo-insta
               nodejs
               bun

flake.nix

@@ -5,15 +5,9 @@
 default:
   @just --list
 
-run *args:
+roughly *args:
 	@cargo run -q -- {{args}}
 
-fmt *args:
-	@cargo run -q -- fmt {{args}}
-
-lint *args:
-	@cargo run -q -- lint {{args}}
-
 test *args:
 	cargo test -- --nocapture {{args}}
 

justfile

@@ -337,7 +337,11 @@ pub fn server(experimental: bool) {
 #[derive(Debug)]
 pub struct DebugError;
 
-pub fn index(paths: Option<&[PathBuf]>, print_items: bool) -> Result<(), DebugError> {
+pub fn index(
+    paths: Option<&[PathBuf]>,
+    recursive: bool,
+    print_items: bool,
+) -> Result<(), DebugError> {
     let root: Vec<PathBuf> = vec![".".into()];
     let paths = paths.unwrap_or(&root);
 
@@ -364,28 +368,30 @@ pub fn index(paths: Option<&[PathBuf]>, print_items: bool) -> Result<(), DebugEr
             })
             .collect::<Result<Vec<_>, _>>()?
         {
-            let text = std::fs::read_to_string(&path).map_err(|err| {
+            let rope = utils::read_to_rope(&path).map_err(|err| {
                 error(&format!("failed to index: {}", path.display()));
                 eprintln!("{err}");
                 DebugError
             })?;
 
             // Only time the indexing operation, not the I/O
             let start = std::time::Instant::now();
-            let symbols = index::index(&text);
+            let tree = tree::parse_rope(&rope, None);
+            let symbols = index::index(tree.root_node(), &rope, recursive);
             let elapsed = start.elapsed();
 
-            total_bytes += text.len();
+            let bytes = rope.len_bytes();
+            total_bytes += bytes;
             total_files += 1;
             total_symbols += symbols.len();
             total_time += elapsed;
 
             eprintln!(
                 "{} ({}, {} ms, {}/s)",
                 style(path.display().to_string()).bold().blue(),
-                utils::human_bytes(text.len() as f64),
+                utils::human_bytes(bytes as f64),
                 elapsed.as_millis(),
-                utils::human_bytes(text.len() as f64 / elapsed.as_secs_f64()),
+                utils::human_bytes(bytes as f64 / elapsed.as_secs_f64()),
             );
 
             if print_items {

src/cli.rs

@@ -34,19 +34,19 @@ pub fn get(
         tree.root_node()
             .descendant_for_point_range(point, point)
             .map(|node| {
+                // TODO: just use document children (requires to think about where to use nested and where not)
                 std::iter::successors(Some(node), |node| node.parent())
                     .filter(|node| node.kind() == "function_definition")
-                    .flat_map(|func_node| {
+                    .flat_map(|node| {
                         let mut items = Vec::new();
 
-                        if let Some(parameters) = func_node.child_by_field_name("parameters") {
+                        if let Some(parameters) = node.child_by_field_name("parameters") {
                             items.extend(
                                 parameters
                                     .children_by_field_name("parameter", &mut parameters.walk())
                                     .filter_map(|parameter| {
                                         parameter.child_by_field_name("name").map(|name| {
-                                            rope.byte_slice(name.start_byte()..name.end_byte())
-                                                .to_string()
+                                            rope.byte_slice(name.byte_range()).to_string()
                                         })
                                     })
                                     .filter(|name| utils::starts_with_lowercase(name, &query))
@@ -58,9 +58,10 @@ pub fn get(
                             );
                         }
 
-                        if let Some(body) = func_node.child_by_field_name("body") {
+                        if let Some(body) = node.child_by_field_name("body") {
                             items.extend(
-                                index::symbols_for_block(body, rope)
+                                // todo: use enable nested and use children
+                                index::index(body, rope, false)
                                     .into_iter()
                                     .filter(|symbol| {
                                         utils::starts_with_lowercase(&symbol.name, &query)

src/completions.rs

@@ -3,6 +3,7 @@ use {
         diagnostics::{self, DiagnosticsError, field},
         lsp_types::{Diagnostic, DiagnosticSeverity},
     },
+    async_lsp::lsp_types::DiagnosticTag,
     ropey::Rope,
     std::collections::HashMap,
     tree_sitter::{Node, TreeCursor},
@@ -173,7 +174,7 @@ pub fn analyze(node: Node, rope: &Rope) -> Result<Vec<Diagnostic>, DiagnosticsEr
             code_description: None,
             source: None,
             related_information: None,
-            tags: None,
+            tags: Some(vec![DiagnosticTag::UNNECESSARY]),
             data: None,
         })
         .collect())