Use parse_lex instead of parse for Ruby and ERB documents

lib/ruby_indexer/lib/ruby_indexer/declaration_listener.rb

@@ -9,7 +9,7 @@ class DeclarationListener
     #: Array[String]
     attr_reader :indexing_errors
 
-    #: (Index index, Prism::Dispatcher dispatcher, Prism::ParseResult parse_result, URI::Generic uri, ?collect_comments: bool) -> void
+    #: (Index index, Prism::Dispatcher dispatcher, Prism::ParseLexResult | Prism::ParseResult parse_result, URI::Generic uri, ?collect_comments: bool) -> void
     def initialize(index, dispatcher, parse_result, uri, collect_comments: false)
       @index = index
       @uri = uri

lib/ruby_lsp/erb_document.rb

@@ -2,7 +2,7 @@
 # frozen_string_literal: true
 
 module RubyLsp
-  #: [ParseResultType = Prism::ParseResult]
+  #: [ParseResultType = Prism::ParseLexResult]
   class ERBDocument < Document
     #: String
     attr_reader :host_language_source
@@ -31,11 +31,16 @@ def parse!
       @host_language_source = scanner.host_language
       # Use partial script to avoid syntax errors in ERB files where keywords may be used without the full context in
       # which they will be evaluated
-      @parse_result = Prism.parse(scanner.ruby, partial_script: true)
+      @parse_result = Prism.parse_lex(scanner.ruby, partial_script: true)
       @code_units_cache = @parse_result.code_units_cache(@encoding)
       true
     end
 
+    #: -> Prism::ProgramNode
+    def ast
+      @parse_result.value.first
+    end
+
     # @override
     #: -> bool
     def syntax_error?
@@ -53,7 +58,7 @@ def locate_node(position, node_types: [])
       char_position, _ = find_index_by_position(position)
 
       RubyDocument.locate(
-        @parse_result.value,
+        ast,
         char_position,
         code_units_cache: @code_units_cache,
         node_types: node_types,

lib/ruby_lsp/requests/code_action_resolve.rb

@@ -100,7 +100,7 @@ def refactor_variable
 
         # Find the closest statements node, so that we place the refactor in a valid position
         node_context = RubyDocument
-          .locate(@document.parse_result.value,
+          .locate(@document.ast,
             start_index,
             node_types: [
               Prism::StatementsNode,
@@ -207,7 +207,7 @@ def refactor_method
 
         # Find the closest method declaration node, so that we place the refactor in a valid position
         node_context = RubyDocument.locate(
-          @document.parse_result.value,
+          @document.ast,
           start_index,
           node_types: [Prism::DefNode],
           code_units_cache: @document.code_units_cache,

lib/ruby_lsp/requests/completion.rb

@@ -33,7 +33,7 @@ def initialize(document, global_state, params, sorbet_level, dispatcher)
         delegate_request_if_needed!(global_state, document, char_position)
 
         node_context = RubyDocument.locate(
-          document.parse_result.value,
+          document.ast,
           char_position,
           node_types: [
             Prism::CallNode,

lib/ruby_lsp/requests/definition.rb

@@ -20,7 +20,7 @@ def initialize(document, global_state, position, dispatcher, sorbet_level)
         delegate_request_if_needed!(global_state, document, char_position)
 
         node_context = RubyDocument.locate(
-          document.parse_result.value,
+          document.ast,
           char_position,
           node_types: [
             Prism::CallNode,

lib/ruby_lsp/requests/discover_tests.rb

@@ -43,7 +43,7 @@ def perform
             addon.create_discover_tests_listener(@response_builder, @dispatcher, @document.uri)
           end
 
-          @dispatcher.visit(@document.parse_result.value)
+          @dispatcher.visit(@document.ast)
         else
           @global_state.synchronize do
             RubyIndexer::DeclarationListener.new(
@@ -64,7 +64,7 @@ def perform
             # Dispatch the events both for indexing the test file and discovering the tests. The order here is
             # important because we need the index to be aware of the existing classes/modules/methods before the test
             # listeners can do their work
-            @dispatcher.visit(@document.parse_result.value)
+            @dispatcher.visit(@document.ast)
           end
         end
 

lib/ruby_lsp/requests/document_highlight.rb

@@ -20,7 +20,7 @@ def initialize(global_state, document, position, dispatcher)
         delegate_request_if_needed!(global_state, document, char_position)
 
         node_context = RubyDocument.locate(
-          document.parse_result.value,
+          document.ast,
           char_position,
           code_units_cache: document.code_units_cache,
         )

lib/ruby_lsp/requests/hover.rb

@@ -24,7 +24,7 @@ def initialize(document, global_state, position, dispatcher, sorbet_level)
         delegate_request_if_needed!(global_state, document, char_position)
 
         node_context = RubyDocument.locate(
-          document.parse_result.value,
+          document.ast,
           char_position,
           node_types: Listeners::Hover::ALLOWED_TARGETS,
           code_units_cache: document.code_units_cache,

lib/ruby_lsp/requests/prepare_rename.rb

@@ -22,7 +22,7 @@ def perform
         char_position, _ = @document.find_index_by_position(@position)
 
         node_context = RubyDocument.locate(
-          @document.parse_result.value,
+          @document.ast,
           char_position,
           node_types: [Prism::ConstantReadNode, Prism::ConstantPathNode, Prism::ConstantPathTargetNode],
           code_units_cache: @document.code_units_cache,

lib/ruby_lsp/requests/references.rb

@@ -26,7 +26,7 @@ def perform
         char_position, _ = @document.find_index_by_position(position)
 
         node_context = RubyDocument.locate(
-          @document.parse_result.value,
+          @document.ast,
           char_position,
           node_types: [
             Prism::ConstantReadNode,

lib/ruby_lsp/requests/rename.rb

@@ -34,7 +34,7 @@ def perform
         char_position, _ = @document.find_index_by_position(@position)
 
         node_context = RubyDocument.locate(
-          @document.parse_result.value,
+          @document.ast,
           char_position,
           node_types: [Prism::ConstantReadNode, Prism::ConstantPathNode, Prism::ConstantPathTargetNode],
           code_units_cache: @document.code_units_cache,
@@ -136,25 +136,27 @@ def collect_text_edits(target, name)
           next if @store.key?(uri)
 
           parse_result = Prism.parse_file(path)
-          edits = collect_changes(target, parse_result, name, uri)
+          edits = collect_changes(target, parse_result.value, name, uri)
           changes[uri.to_s] = edits unless edits.empty?
         rescue Errno::EISDIR, Errno::ENOENT
           # If `path` is a directory, just ignore it and continue. If the file doesn't exist, then we also ignore it.
         end
 
         @store.each do |uri, document|
-          edits = collect_changes(target, document.parse_result, name, document.uri)
+          next unless document.is_a?(RubyDocument) || document.is_a?(ERBDocument)
+
+          edits = collect_changes(target, document.ast, name, document.uri)
           changes[uri] = edits unless edits.empty?
         end
 
         changes
       end
 
-      #: (RubyIndexer::ReferenceFinder::Target target, Prism::ParseResult parse_result, String name, URI::Generic uri) -> Array[Interface::TextEdit]
-      def collect_changes(target, parse_result, name, uri)
+      #: (RubyIndexer::ReferenceFinder::Target target, Prism::Node ast, String name, URI::Generic uri) -> Array[Interface::TextEdit]
+      def collect_changes(target, ast, name, uri)
         dispatcher = Prism::Dispatcher.new
         finder = RubyIndexer::ReferenceFinder.new(target, @global_state.index, dispatcher, uri)
-        dispatcher.visit(parse_result.value)
+        dispatcher.visit(ast)
 
         finder.references.map do |reference|
           adjust_reference_for_edit(name, reference)

lib/ruby_lsp/requests/selection_ranges.rb

@@ -25,7 +25,7 @@ def initialize(document)
       #: -> (Array[Support::SelectionRange] & Object)
       def perform
         # [node, parent]
-        queue = [[@document.parse_result.value, nil]]
+        queue = [[@document.ast, nil]]
 
         until queue.empty?
           node, parent = queue.shift

lib/ruby_lsp/requests/show_syntax_tree.rb

@@ -12,7 +12,7 @@ def initialize(document, range)
         super()
         @document = document
         @range = range
-        @tree = document.parse_result.value #: Prism::ProgramNode
+        @tree = document.ast #: Prism::ProgramNode
       end
 
       # @override

lib/ruby_lsp/requests/signature_help.rb

@@ -27,7 +27,7 @@ def initialize(document, global_state, position, context, dispatcher, sorbet_lev
         delegate_request_if_needed!(global_state, document, char_position)
 
         node_context = RubyDocument.locate(
-          document.parse_result.value,
+          document.ast,
           char_position,
           node_types: [Prism::CallNode],
           code_units_cache: document.code_units_cache,

lib/ruby_lsp/ruby_document.rb

@@ -2,7 +2,7 @@
 # frozen_string_literal: true
 
 module RubyLsp
-  #: [ParseResultType = Prism::ParseResult]
+  #: [ParseResultType = Prism::ParseLexResult]
   class RubyDocument < Document
     METHODS_THAT_CHANGE_DECLARATIONS = [
       :private_constant,
@@ -129,11 +129,16 @@ def parse!
       return false unless @needs_parsing
 
       @needs_parsing = false
-      @parse_result = Prism.parse(@source)
+      @parse_result = Prism.parse_lex(@source)
       @code_units_cache = @parse_result.code_units_cache(@encoding)
       true
     end
 
+    #: -> Prism::ProgramNode
+    def ast
+      @parse_result.value.first
+    end
+
     # @override
     #: -> bool
     def syntax_error?
@@ -151,7 +156,7 @@ def locate_first_within_range(range, node_types: [])
       start_position, end_position = find_index_by_position(range[:start], range[:end])
 
       desired_range = (start_position...end_position)
-      queue = @parse_result.value.child_nodes.compact #: Array[Prism::Node?]
+      queue = ast.child_nodes.compact #: Array[Prism::Node?]
 
       until queue.empty?
         candidate = queue.shift
@@ -179,7 +184,7 @@ def locate_node(position, node_types: [])
       char_position, _ = find_index_by_position(position)
 
       RubyDocument.locate(
-        @parse_result.value,
+        ast,
         char_position,
         code_units_cache: @code_units_cache,
         node_types: node_types,

lib/ruby_lsp/server.rb

@@ -513,12 +513,12 @@ def run_combined_requests(message)
             index.delete(uri, skip_require_paths_tree: true)
             RubyIndexer::DeclarationListener.new(index, dispatcher, parse_result, uri, collect_comments: true)
             code_lens = Requests::CodeLens.new(@global_state, document, dispatcher)
-            dispatcher.dispatch(parse_result.value)
+            dispatcher.dispatch(document.ast)
           end
         end
       else
         code_lens = Requests::CodeLens.new(@global_state, document, dispatcher)
-        dispatcher.dispatch(parse_result.value)
+        dispatcher.dispatch(document.ast)
       end
 
       # Store all responses retrieve in this round of visits in the cache and then return the response for the request
@@ -557,7 +557,7 @@ def text_document_semantic_tokens_full(message)
 
       dispatcher = Prism::Dispatcher.new
       semantic_highlighting = Requests::SemanticHighlighting.new(@global_state, dispatcher, document, nil)
-      dispatcher.visit(document.parse_result.value)
+      dispatcher.visit(document.ast)
 
       send_message(Result.new(id: message[:id], response: semantic_highlighting.perform))
     end
@@ -583,7 +583,7 @@ def text_document_semantic_tokens_delta(message)
         document,
         message.dig(:params, :previousResultId),
       )
-      dispatcher.visit(document.parse_result.value)
+      dispatcher.visit(document.ast)
       send_message(Result.new(id: message[:id], response: request.perform))
     end
 
@@ -612,7 +612,7 @@ def text_document_semantic_tokens_range(message)
         nil,
         range: range.dig(:start, :line)..range.dig(:end, :line),
       )
-      dispatcher.visit(document.parse_result.value)
+      dispatcher.visit(document.ast)
       send_message(Result.new(id: message[:id], response: request.perform))
     end
 
@@ -704,7 +704,7 @@ def text_document_document_highlight(message)
       end
 
       request = Requests::DocumentHighlight.new(@global_state, document, params[:position], dispatcher)
-      dispatcher.dispatch(document.parse_result.value)
+      dispatcher.dispatch(document.ast)
       send_message(Result.new(id: message[:id], response: request.perform))
     end
 
@@ -851,7 +851,7 @@ def text_document_inlay_hint(message)
       end
 
       request = Requests::InlayHints.new(document, hints_configurations, dispatcher)
-      dispatcher.visit(document.parse_result.value)
+      dispatcher.visit(document.ast)
       result = request.perform
       document.cache_set("textDocument/inlayHint", result)
 

test/requests/code_lens_expectations_test.rb

@@ -14,7 +14,7 @@ def run_expectations(source)
     dispatcher = Prism::Dispatcher.new
     stub_test_library("minitest")
     listener = RubyLsp::Requests::CodeLens.new(@global_state, document, dispatcher)
-    dispatcher.dispatch(document.parse_result.value)
+    dispatcher.dispatch(document.ast)
     listener.perform
   end
 
@@ -31,7 +31,7 @@ def test_bar; end
 
     dispatcher = Prism::Dispatcher.new
     listener = RubyLsp::Requests::CodeLens.new(@global_state, document, dispatcher)
-    dispatcher.dispatch(document.parse_result.value)
+    dispatcher.dispatch(document.ast)
     response = listener.perform
 
     assert_equal(6, response.size)
@@ -65,7 +65,7 @@ class FooTest < MiniTest::Spec
 
     dispatcher = Prism::Dispatcher.new
     listener = RubyLsp::Requests::CodeLens.new(@global_state, document, dispatcher)
-    dispatcher.dispatch(document.parse_result.value)
+    dispatcher.dispatch(document.ast)
     response = listener.perform
 
     assert_equal(9, response.size)
@@ -103,7 +103,7 @@ def test_command_generation_for_minitest_spec_handles_specify_alias_for_it
 
     dispatcher = Prism::Dispatcher.new
     listener = RubyLsp::Requests::CodeLens.new(@global_state, document, dispatcher)
-    dispatcher.dispatch(document.parse_result.value)
+    dispatcher.dispatch(document.ast)
     response = listener.perform
 
     # 3 for the describe, 3 for the specify
@@ -123,7 +123,7 @@ def test_bar; end
 
     dispatcher = Prism::Dispatcher.new
     listener = RubyLsp::Requests::CodeLens.new(@global_state, document, dispatcher)
-    dispatcher.dispatch(document.parse_result.value)
+    dispatcher.dispatch(document.ast)
     response = listener.perform
 
     assert_equal(6, response.size)
@@ -155,7 +155,7 @@ def test_bar; end
     dispatcher = Prism::Dispatcher.new
     stub_test_library("unknown")
     listener = RubyLsp::Requests::CodeLens.new(@global_state, document, dispatcher)
-    dispatcher.dispatch(document.parse_result.value)
+    dispatcher.dispatch(document.ast)
     response = listener.perform
 
     assert_empty(response)
@@ -174,7 +174,7 @@ def test_bar; end
     dispatcher = Prism::Dispatcher.new
     stub_test_library("rspec")
     listener = RubyLsp::Requests::CodeLens.new(@global_state, document, dispatcher)
-    dispatcher.dispatch(document.parse_result.value)
+    dispatcher.dispatch(document.ast)
     response = listener.perform
 
     assert_empty(response)
@@ -193,7 +193,7 @@ def test_bar; end
     dispatcher = Prism::Dispatcher.new
     stub_test_library("minitest")
     listener = RubyLsp::Requests::CodeLens.new(@global_state, document, dispatcher)
-    dispatcher.dispatch(document.parse_result.value)
+    dispatcher.dispatch(document.ast)
     response = listener.perform
 
     assert_empty(response)
@@ -212,7 +212,7 @@ def test_no_code_lens_for_unsaved_specs
     dispatcher = Prism::Dispatcher.new
     stub_test_library("minitest")
     listener = RubyLsp::Requests::CodeLens.new(@global_state, document, dispatcher)
-    dispatcher.dispatch(document.parse_result.value)
+    dispatcher.dispatch(document.ast)
     response = listener.perform
 
     assert_empty(response)
@@ -267,7 +267,7 @@ def test_baz; end
 
     dispatcher = Prism::Dispatcher.new
     listener = RubyLsp::Requests::CodeLens.new(@global_state, document, dispatcher)
-    dispatcher.dispatch(document.parse_result.value)
+    dispatcher.dispatch(document.ast)
     response = listener.perform
 
     assert_equal(6, response.size)