feat(treesitter): support URLs (neovim#27132)

gpanders · web-flow · commit cb46f6e46726 · 2024-03-12T09:32:17.000-05:00
Tree-sitter queries can add URLs to a capture using the `#set!` directive, e.g. (inline_link (link_text) @text.reference (link_destination) @text.uri (#set! @text.reference "url" @text.uri)) The pattern above is included by default in the `markdown_inline` highlight query so that users with supporting terminals will see hyperlinks. For now, this creates a hyperlink for *all* Markdown URLs of the pattern [link text](link url), even if `link url` does not contain a valid protocol (e.g. if `link url` is a path to a file). We may wish to change this in the future to only linkify when the URL has a valid protocol scheme, but for now we delegate handling this to the terminal emulator. In order to support directives which reference other nodes, the highlighter must be updated to use `iter_matches` rather than `iter_captures`. The former provides the `match` table which maps capture IDs to nodes. However, this has its own challenges: - `iter_matches` does not guarantee the order in which patterns are iterated matches the order in the query file. So we must enforce ordering manually using "subpriorities" (neovim#27131). The pattern index of each match dictates the extmark's subpriority. - When injections are used, the highlighter contains multiple trees. The pattern indices of each tree must be offset relative to the maximum pattern index from all previous trees to ensure that extmarks appear in the correct order. - The `iter_captures` implementation currently has a bug where the "match" table is only returned for the first capture within a pattern (see neovim#27274). This bug means that `#set!` directives in a query apply only to the first capture within a pattern. Unfortunately, many queries in the wild have come to depend on this behavior. `iter_matches` does not share this flaw, so switching to `iter_matches` exposed bugs in existing highlight queries. These queries have been updated in this repo, but may still need to be updated by users. The `#set!` directive applies to the _entire_ query pattern when used without a capture argument. To make `#set!` apply only to a single capture, the capture must be given as an argument.
diff --git a/runtime/doc/news.txt b/runtime/doc/news.txt
@@ -254,6 +254,9 @@ The following new APIs and features were added.
     indexing.
   • |:InspectTree| shows root nodes
   • |:InspectTree| now supports |folding|
+  • The `#set!` directive can set the "url" property of a node to have the
+    node emit a hyperlink. Hyperlinks are UI specific: in the TUI, the OSC 8
+    control sequence is used.
 
 • |vim.ui.open()| opens URIs using the system default handler (macOS `open`,
   Windows `explorer`, Linux `xdg-open`, etc.)
diff --git a/runtime/lua/vim/treesitter/highlighter.lua b/runtime/lua/vim/treesitter/highlighter.lua
@@ -4,7 +4,7 @@ local Range = require('vim.treesitter._range')
 
 local ns = api.nvim_create_namespace('treesitter/highlighter')
 
----@alias vim.treesitter.highlighter.Iter fun(end_line: integer|nil): integer, TSNode, vim.treesitter.query.TSMetadata
+---@alias vim.treesitter.highlighter.Iter fun(): integer, table<integer, TSNode[]>, vim.treesitter.query.TSMetadata
 
 ---@class (private) vim.treesitter.highlighter.Query
 ---@field private _query vim.treesitter.Query?
@@ -248,6 +248,13 @@ end
 ---@param line integer
 ---@param is_spell_nav boolean
 local function on_line_impl(self, buf, line, is_spell_nav)
+  -- Track the maximum pattern index encountered in each tree. For subsequent
+  -- trees, the subpriority passed to nvim_buf_set_extmark is offset by the
+  -- largest pattern index from the prior tree. This ensures that extmarks
+  -- from subsequent trees always appear "on top of" extmarks from previous
+  -- trees (e.g. injections should always appear over base highlights).
+  local pattern_offset = 0
+
   self:for_each_highlight_state(function(state)
     local root_node = state.tstree:root()
     local root_start_row, _, root_end_row, _ = root_node:range()
@@ -258,22 +265,24 @@ local function on_line_impl(self, buf, line, is_spell_nav)
     end
 
     if state.iter == nil or state.next_row < line then
-      state.iter =
-        state.highlighter_query:query():iter_captures(root_node, self.bufnr, line, root_end_row + 1)
+      state.iter = state.highlighter_query
+        :query()
+        :iter_matches(root_node, self.bufnr, line, root_end_row + 1, { all = true })
     end
 
+    local max_pattern_index = -1
     while line >= state.next_row do
-      local capture, node, metadata = state.iter(line)
+      local pattern, match, metadata = state.iter()
 
-      local range = { root_end_row + 1, 0, root_end_row + 1, 0 }
-      if node then
-        range = vim.treesitter.get_range(node, buf, metadata and metadata[capture])
+      if pattern and pattern > max_pattern_index then
+        max_pattern_index = pattern
       end
-      local start_row, start_col, end_row, end_col = Range.unpack4(range)
 
-      if capture then
-        local hl = state.highlighter_query:get_hl_from_capture(capture)
+      if not match then
+        state.next_row = root_end_row + 1
+      end
 
+      for capture, nodes in pairs(match or {}) do
         local capture_name = state.highlighter_query:query().captures[capture]
         local spell = nil ---@type boolean?
         if capture_name == 'spell' then
@@ -282,28 +291,60 @@ local function on_line_impl(self, buf, line, is_spell_nav)
           spell = false
         end
 
+        local hl = state.highlighter_query:get_hl_from_capture(capture)
+
         -- Give nospell a higher priority so it always overrides spell captures.
         local spell_pri_offset = capture_name == 'nospell' and 1 or 0
 
-        if hl and end_row >= line and (not is_spell_nav or spell ~= nil) then
-          local priority = (tonumber(metadata.priority) or vim.highlight.priorities.treesitter)
-            + spell_pri_offset
-          api.nvim_buf_set_extmark(buf, ns, start_row, start_col, {
-            end_line = end_row,
-            end_col = end_col,
-            hl_group = hl,
-            ephemeral = true,
-            priority = priority,
-            conceal = metadata.conceal,
-            spell = spell,
-          })
+        -- The "priority" attribute can be set at the pattern level or on a particular capture
+        local priority = (
+          tonumber(metadata.priority or metadata[capture] and metadata[capture].priority)
+          or vim.highlight.priorities.treesitter
+        ) + spell_pri_offset
+
+        local url = metadata[capture] and metadata[capture].url ---@type string|number|nil
+        if type(url) == 'number' then
+          if match and match[url] then
+            -- Assume there is only one matching node. If there is more than one, take the URL
+            -- from the first.
+            local other_node = match[url][1]
+            url = vim.treesitter.get_node_text(other_node, buf, {
+              metadata = metadata[url],
+            })
+          else
+            url = nil
+          end
         end
-      end
 
-      if start_row > line then
-        state.next_row = start_row
+        -- The "conceal" attribute can be set at the pattern level or on a particular capture
+        local conceal = metadata.conceal or metadata[capture] and metadata[capture].conceal
+
+        for _, node in ipairs(nodes) do
+          local range = vim.treesitter.get_range(node, buf, metadata[capture])
+          local start_row, start_col, end_row, end_col = Range.unpack4(range)
+
+          if hl and end_row >= line and (not is_spell_nav or spell ~= nil) then
+            api.nvim_buf_set_extmark(buf, ns, start_row, start_col, {
+              end_line = end_row,
+              end_col = end_col,
+              hl_group = hl,
+              ephemeral = true,
+              priority = priority,
+              _subpriority = pattern_offset + pattern,
+              conceal = conceal,
+              spell = spell,
+              url = url,
+            })
+          end
+
+          if start_row > line then
+            state.next_row = start_row
+          end
+        end
       end
     end
+
+    pattern_offset = pattern_offset + max_pattern_index
   end)
 end
 
diff --git a/runtime/queries/markdown_inline/highlights.scm b/runtime/queries/markdown_inline/highlights.scm
@@ -33,6 +33,11 @@
   ] @markup.link
   (#set! conceal ""))
 
+(inline_link
+  (link_text) @markup.link.label
+  (link_destination) @markup.link
+  (#set! @markup.link.label "url" @markup.link))
+
 ; Conceal image links
 (image
   [
diff --git a/runtime/queries/vimdoc/highlights.scm b/runtime/queries/vimdoc/highlights.scm
@@ -12,21 +12,30 @@
 
 (tag
   "*" @markup.heading.5.marker
-  (#set! conceal "")
-  text: (_) @label)
+  .
+  text: (_) @label
+  .
+  "*" @markup.heading.5.marker
+  (#set! @markup.heading.5.marker conceal ""))
 
 (taglink
-  "|" @markup.link
-  (#set! conceal "")
-  text: (_) @markup.link)
+  "|" @markup.link.delimiter
+  .
+  text: (_) @markup.link
+  .
+  "|" @markup.link.delimiter
+  (#set! @markup.link.delimiter conceal ""))
 
 (optionlink
   text: (_) @markup.link)
 
 (codespan
   "`" @markup.raw.delimiter
-  (#set! conceal "")
-  text: (_) @markup.raw)
+  .
+  text: (_) @markup.raw
+  .
+  "`" @markup.raw.delimiter
+  (#set! @markup.raw.delimiter conceal ""))
 
 ((codeblock) @markup.raw.block
   (#set! "priority" 90))
diff --git a/test/functional/treesitter/highlight_spec.lua b/test/functional/treesitter/highlight_spec.lua
@@ -681,6 +681,12 @@ describe('treesitter highlighting (C)', function()
         ((identifier) @Identifier
          (#set! conceal "")
          (#eq? @Identifier "lstate"))
+
+        ((call_expression
+            function: (identifier) @function
+            arguments: (argument_list) @arguments)
+         (#eq? @function "multiqueue_put")
+         (#set! @function conceal "V"))
       ]]}})
     ]=]
 
@@ -697,7 +703,7 @@ describe('treesitter highlighting (C)', function()
                                                                        |
         LuaRef cb = nlua_ref(, 1);                                     |
                                                                        |
-        multiqueue_put(main_loop.events, nlua_schedule_event,          |
+        {11:V}(main_loop.events, nlua_schedule_event,                       |
                        1, (void *)(ptrdiff_t)cb);                      |
         return 0;                                                      |
       ^}                                                                |
@@ -758,6 +764,44 @@ describe('treesitter highlighting (C)', function()
   end)
 end)
 
+describe('treesitter highlighting (lua)', function()
+  local screen
+
+  before_each(function()
+    screen = Screen.new(65, 18)
+    screen:attach()
+    screen:set_default_attr_ids {
+      [1] = { bold = true, foreground = Screen.colors.Blue },
+      [2] = { foreground = Screen.colors.DarkCyan },
+      [3] = { foreground = Screen.colors.Magenta },
+      [4] = { foreground = Screen.colors.SlateBlue },
+      [5] = { bold = true, foreground = Screen.colors.Brown },
+    }
+  end)
+
+  it('supports language injections', function()
+    insert [[
+      local ffi = require('ffi')
+      ffi.cdef("int (*fun)(int, char *);")
+    ]]
+
+    exec_lua [[
+      vim.bo.filetype = 'lua'
+      vim.treesitter.start()
+    ]]
+
+    screen:expect {
+      grid = [[
+        {5:local} {2:ffi} {5:=} {4:require(}{3:'ffi'}{4:)}                                     |
+        {2:ffi}{4:.}{2:cdef}{4:(}{3:"}{4:int}{3: }{4:(}{5:*}{3:fun}{4:)(int,}{3: }{4:char}{3: }{5:*}{4:);}{3:"}{4:)}                           |
+      ^                                                                 |
+      {1:~                                                                }|*14
+                                                                       |
+    ]],
+    }
+  end)
+end)
+
 describe('treesitter highlighting (help)', function()
   local screen
 
@@ -891,3 +935,46 @@ vim.cmd([[
     }
   end)
 end)
+
+describe('treesitter highlighting (markdown)', function()
+  local screen
+
+  before_each(function()
+    screen = Screen.new(40, 6)
+    screen:attach()
+    screen:set_default_attr_ids {
+      [1] = { foreground = Screen.colors.Blue1 },
+      [2] = { bold = true, foreground = Screen.colors.Blue1 },
+      [3] = { bold = true, foreground = Screen.colors.Brown },
+      [4] = { foreground = Screen.colors.Cyan4 },
+      [5] = { foreground = Screen.colors.Magenta1 },
+    }
+  end)
+
+  it('supports hyperlinks', function()
+    local url = 'https://example.com'
+    insert(string.format('[This link text](%s) is a hyperlink.', url))
+    exec_lua([[
+      vim.bo.filetype = 'markdown'
+      vim.treesitter.start()
+    ]])
+
+    screen:expect {
+      grid = [[
+      {4:[}{6:This link text}{4:](}{7:https://example.com}{4:)} is|
+       a hyperlink^.                           |
+      {2:~                                       }|*3
+                                              |
+    ]],
+      attr_ids = {
+        [1] = { foreground = Screen.colors.Blue1 },
+        [2] = { bold = true, foreground = Screen.colors.Blue1 },
+        [3] = { bold = true, foreground = Screen.colors.Brown },
+        [4] = { foreground = Screen.colors.Cyan4 },
+        [5] = { foreground = Screen.colors.Magenta },
+        [6] = { foreground = Screen.colors.Cyan4, url = url },
+        [7] = { underline = true, foreground = Screen.colors.SlateBlue },
+      },
+    }
+  end)
+end)
