Use byteidx()/utf16idx() for correct UTF-16 position conversion

autoload/lsp/utils.vim

@@ -327,19 +327,46 @@ endfunction
 " Convert a byte-index (1-based) to a character-index (0-based)
 " This function requires a buffer specifier (expr, see :help bufname()),
 " a line number (lnum, 1-based), and a byte-index (char, 1-based).
-function! lsp#utils#to_char(expr, lnum, col) abort
-    let l:lines = getbufline(a:expr, a:lnum)
-    if l:lines == []
-        if type(a:expr) != v:t_string || !filereadable(a:expr)
-            " invalid a:expr
-            return a:col - 1
+if exists('*utf16idx')
+    function! lsp#utils#to_char(expr, lnum, col) abort
+        let l:lines = getbufline(a:expr, a:lnum)
+        if l:lines == []
+            if type(a:expr) != v:t_string || !filereadable(a:expr)
+                " invalid a:expr
+                return a:col - 1
+            endif
+            " a:expr is a file that is not yet loaded as a buffer
+            let l:lines = readfile(a:expr, '', a:lnum)
         endif
-        " a:expr is a file that is not yet loaded as a buffer
-        let l:lines = readfile(a:expr, '', a:lnum)
-    endif
-    let l:linestr = l:lines[-1]
-    return strchars(strpart(l:linestr, 0, a:col - 1))
-endfunction
+        let l:linestr = l:lines[-1]
+        let l:byteidx = a:col - 1
+        if l:byteidx >= strlen(l:linestr)
+            return utf16idx(l:linestr, strlen(l:linestr))
+        endif
+        let l:utf16 = utf16idx(l:linestr, l:byteidx)
+        " If byteidx is in the middle of a multi-byte character, round up
+        " to match the old strchars(strpart()) behavior
+        let l:round_trip = byteidx(l:linestr, l:utf16, v:true)
+        if l:round_trip >= 0 && l:round_trip < l:byteidx
+            return l:utf16 + 1
+        endif
+        return l:utf16
+    endfunction
+else
+    function! lsp#utils#to_char(expr, lnum, col) abort
+        let l:lines = getbufline(a:expr, a:lnum)
+        if l:lines == []
+            if type(a:expr) != v:t_string || !filereadable(a:expr)
+                " invalid a:expr
+                return a:col - 1
+            endif
+            " a:expr is a file that is not yet loaded as a buffer
+            let l:lines = readfile(a:expr, '', a:lnum)
+        endif
+        let l:linestr = l:lines[-1]
+        return strchars(strpart(l:linestr, 0, a:col - 1))
+    endfunction
+endif
 
 function! s:get_base64_alphabet() abort
     let l:alphabet = []

autoload/lsp/utils/position.vim

@@ -3,39 +3,68 @@
 " Convert a character-index (0-based) to byte-index (1-based)
 " This function requires a buffer specifier (expr, see :help bufname()),
 " a line number (lnum, 1-based), and a character-index (char, 0-based).
-function! s:to_col(expr, lnum, char) abort
+"
+" When utf16idx()/byteidx() with UTF-16 support are available (Vim 9.0.1485+),
+" the character-index is treated as a UTF-16 code unit offset, which is
+" correct per the LSP specification. Otherwise falls back to Unicode
+" codepoint counting.
+
+function! s:_get_line(expr, lnum) abort
     let l:lines = getbufline(a:expr, a:lnum)
     if l:lines == []
         if type(a:expr) != v:t_string || !filereadable(a:expr)
-            " invalid a:expr
-            return a:char + 1
+            return []
         endif
-        " a:expr is a file that is not yet loaded as a buffer
         let l:lines = readfile(a:expr, '', a:lnum)
+    endif
+    return l:lines
+endfunction
+
+if exists('*utf16idx')
+    function! s:to_col(expr, lnum, char) abort
+        let l:lines = s:_get_line(a:expr, a:lnum)
         if l:lines == []
-            " when the file is empty. a:char should be 0 in the case
             return a:char + 1
         endif
-    endif
-    let l:linestr = l:lines[-1]
-    return strlen(strcharpart(l:linestr, 0, a:char)) + 1
-endfunction
+        return byteidx(l:lines[-1], a:char, v:true) + 1
+    endfunction
 
-" The inverse version of `s:to_col`.
-" Convert [lnum, col] to LSP's `Position`.
-function! s:to_char(expr, lnum, col) abort
-    let l:lines = getbufline(a:expr, a:lnum)
-    if l:lines == []
-        if type(a:expr) != v:t_string || !filereadable(a:expr)
-            " invalid a:expr
+    function! s:to_char(expr, lnum, col) abort
+        let l:lines = s:_get_line(a:expr, a:lnum)
+        if l:lines == []
             return a:col - 1
         endif
-        " a:expr is a file that is not yet loaded as a buffer
-        let l:lines = readfile(a:expr, '', a:lnum)
-    endif
-    let l:linestr = l:lines[-1]
-    return strchars(strpart(l:linestr, 0, a:col - 1))
-endfunction
+        let l:linestr = l:lines[-1]
+        let l:byteidx = a:col - 1
+        if l:byteidx >= strlen(l:linestr)
+            return utf16idx(l:linestr, strlen(l:linestr))
+        endif
+        let l:utf16 = utf16idx(l:linestr, l:byteidx)
+        " If byteidx is in the middle of a multi-byte character, round up
+        " to match the old strchars(strpart()) behavior
+        let l:round_trip = byteidx(l:linestr, l:utf16, v:true)
+        if l:round_trip >= 0 && l:round_trip < l:byteidx
+            return l:utf16 + 1
+        endif
+        return l:utf16
+    endfunction
+else
+    function! s:to_col(expr, lnum, char) abort
+        let l:lines = s:_get_line(a:expr, a:lnum)
+        if l:lines == []
+            return a:char + 1
+        endif
+        return strlen(strcharpart(l:lines[-1], 0, a:char)) + 1
+    endfunction
+
+    function! s:to_char(expr, lnum, col) abort
+        let l:lines = s:_get_line(a:expr, a:lnum)
+        if l:lines == []
+            return a:col - 1
+        endif
+        return strchars(strpart(l:lines[-1], 0, a:col - 1))
+    endfunction
+endif
 
 " @param expr = see :help bufname()
 " @param position = {
@@ -88,4 +117,3 @@ function! lsp#utils#position#vim_to_lsp(expr, pos) abort
          \   'character': s:to_char(a:expr, a:pos[0], a:pos[1])
          \ }
 endfunction
-