fix: handle unicode->snippetstring->unicode conversion correctly.

lua/luasnip/util/str.lua

@@ -199,8 +199,11 @@ function M.multiline_append(strmod, strappend)
 end
 
 -- turn a row+col-offset for a multiline-string (string[]) (where the column is
--- given in utf-codepoints and 0-based) into an offset (in bytes!, 1-based) for
+-- given in bytes and 0-based) into an offset (in bytes, 1-based) for
 -- the \n-concatenated version of that string.
+---
+---@param str string[], a multiline string
+---@param pos LuaSnip.ApiPosition, an api-position relative to the start of str.
 function M.multiline_to_byte_offset(str, pos)
 	if pos[1] < 0 or pos[1] + 1 > #str or pos[2] < 0 then
 		-- pos is trivially (row negative or beyond str, or col negative)
@@ -218,34 +221,33 @@ function M.multiline_to_byte_offset(str, pos)
 
 	-- allow positions one beyond the last character for all lines (even the
 	-- last line).
-	local pos_line_str = str[pos[1] + 1] .. "\n"
-
-	if pos[2] >= #pos_line_str then
+	if pos[2] >= #str[pos[1]+1] + 1 then
 		-- in this case, pos is outside of the multiline-region.
 		return nil
 	end
 
 	-- I think we can always assume utf-8?
-	byte_pos = byte_pos + vim.str_byteindex(pos_line_str, "utf-8", pos[2])
+	byte_pos = byte_pos + pos[2]
 
 	-- 0- to 1-based columns.
 	return byte_pos + 1
 end
 
--- inverse of multiline_to_byte_offset, 1-based byte to 0,0-based row,column, utf-aware.
+-- inverse of multiline_to_byte_offset, 1-based byte to 0,0-based row,column.
+---@param str string[], the multiline string
+---@param byte_pos number, a 1-based index into the \n-concatenated `str`.
 function M.byte_to_multiline_offset(str, byte_pos)
 	if byte_pos < 0 then
 		return nil
 	end
 
 	local byte_pos_so_far = 0
 	for i, line in ipairs(str) do
+		-- line-length + \n.
 		local line_i_end = byte_pos_so_far + #line + 1
 		if byte_pos <= line_i_end then
-			-- byte located in this line, find utf-index.
-			local utf16_index =
-				vim.str_utfindex(line .. "\n", byte_pos - byte_pos_so_far - 1)
-			return { i - 1, utf16_index }
+			-- byte is in this line, return it.
+			return { i - 1, byte_pos - byte_pos_so_far - 1 }
 		end
 		byte_pos_so_far = line_i_end
 	end

tests/integration/choice_spec.lua

@@ -370,8 +370,7 @@ describe("ChoiceNode", function()
 		ls_helpers.clear()
 		ls_helpers.session_setup_luasnip()
 
-		screen = Screen.new(50, 7)
-		screen:attach()
+		screen = ls_helpers.new_screen(50, 7)
 		screen:set_default_attr_ids({
 			[0] = { bold = true, foreground = Screen.colors.Blue },
 			[1] = { bold = true, foreground = Screen.colors.Brown },
@@ -604,7 +603,7 @@ describe("ChoiceNode", function()
 		screen:expect({ unchanged = true })
 
 		-- test some more wild stuff, just because.
-		feed("  <left> ")
+		feed("<space><space><left>")
 		exec_lua([[
 			ls.snip_expand(s("trig", {
 				t":",
@@ -622,52 +621,88 @@ describe("ChoiceNode", function()
 			}))
 		]])
 
-		screen:expect({
-			grid = [[
-    :.ccee e :^c{3:c}eeee:eee ee.:                         |
-    {0:~                                                 }|
-    {2:-- SELECT --}                                      |
-  ]],
-		})
+screen:expect([[
+  :.ccee ee :^c{3:c}eeee: ee ee.:                        |
+  {0:~                                                 }|
+  {2:-- SELECT --}                                      |
+]])
 		exec_lua("ls.jump(1)")
 		feed("<esc><right><right>i aa <left><left>")
 		exec_lua("ls.set_choice(2)")
-		screen:expect({
-			grid = [[
-    :.ccee e :.ccee ee^ee ee.:eee ee.:                 |
-    {0:~                                                 }|
-    {2:-- INSERT --}                                      |
-  ]],
-		})
+
+screen:expect([[
+  :.ccee ee :.ccee ee^ee ee.: ee ee.:                |
+  {0:~                                                 }|
+  {2:-- INSERT --}                                      |
+]])
 
 		-- reselect outer choiceNode
 		exec_lua("ls.jump(-1)")
 		exec_lua("ls.jump(-1)")
 		exec_lua("ls.jump(-1)")
 		exec_lua("ls.jump(1)")
-		screen:expect({
-			grid = [[
-    :.cc^e{3:e e :.ccee eeee ee.:eee ee}.:                 |
-    {0:~                                                 }|
-    {2:-- SELECT --}                                      |
-  ]],
-		})
+screen:expect([[
+  :.cc^e{3:e ee :.ccee eeee ee.: ee ee}.:                |
+  {0:~                                                 }|
+  {2:-- SELECT --}                                      |
+]])
 		exec_lua("ls.change_choice(1)")
-		screen:expect({
-			grid = [[
-    :cc^e{3:e e :.ccee eeee ee.:eee ee}:                   |
-    {0:~                                                 }|
-    {2:-- SELECT --}                                      |
-  ]],
-		})
+screen:expect([[
+  :cc^e{3:e ee :.ccee eeee ee.: ee ee}:                  |
+  {0:~                                                 }|
+  {2:-- SELECT --}                                      |
+]])
 		exec_lua("ls.jump(1)")
 		exec_lua("ls.jump(1)")
-		screen:expect({
-			grid = [[
-    :ccee e :.cc^e{3:e eeee ee}.:eee ee:                   |
-    {0:~                                                 }|
-    {2:-- SELECT --}                                      |
-  ]],
-		})
+screen:expect([[
+  :ccee ee :.cc^e{3:e eeee ee}.: ee ee:                  |
+  {0:~                                                 }|
+  {2:-- SELECT --}                                      |
+]])
+	end)
+
+	it("correctly handles unicode when storing and restoring.", function()
+		exec_lua([=[
+			ls.snip_expand(
+				s("choice", {
+					c(1, {
+						{t"a ", r(1, "k", i(1)), t" a"},
+						{t"bb ", r(1, "k"), t" bb"}
+					}, {restore_cursor = true})
+				}))
+		]=])
+screen:expect([[
+  a ^ a                                              |
+  {0:~                                                 }|
+  {2:-- INSERT --}                                      |
+]])
+		feed("a  a<left><left>")
+		exec_lua([=[
+			ls.snip_expand(s("bad", {i(1, "i…i")}))
+		]=])
+screen:expect([[
+  a a ^i{3:…i} a a                                       |
+  {0:~                                                 }|
+  {2:-- SELECT --}                                      |
+]])
+
+		exec_lua("ls.change_choice(1)")
+screen:expect([[
+  bb a ^i{3:…i} a bb                                     |
+  {0:~                                                 }|
+  {2:-- SELECT --}                                      |
+]])
+		feed("<Esc>la")
+screen:expect([[
+  bb a i…^i a bb                                     |
+  {0:~                                                 }|
+  {2:-- INSERT --}                                      |
+]])
+		exec_lua("ls.change_choice(1)")
+screen:expect([[
+  a a i…^i a a                                       |
+  {0:~                                                 }|
+  {2:-- INSERT --}                                      |
+]])
 	end)
 end)

tests/unit/str_spec.lua

@@ -298,6 +298,9 @@ describe("str.multiline_to_byte_offset", function()
 	check("on linebreak of last line", { "asdf", "qwer" }, { 1, 4 }, 10)
 	check_is_nil("negative row", { "asdf", "qwer" }, { -1, 0 })
 	check_is_nil("negative col", { "asdf", "qwer" }, { 0, -2 })
+	check("unicode1", { "aa … aa" }, { 0, 6 }, 7)
+	check("unicode2", { "aa …a… aa" }, { 0, 6 }, 7)
+	check("unicode3", { "aa …a… aa", "aa …a… aa" }, { 1, 6 }, 21)
 end)
 
 describe("byte_to_multiline_offset", function()
@@ -342,4 +345,7 @@ describe("byte_to_multiline_offset", function()
 	check("multiple lines middle linebreak", { "asdf", "qwer" }, 10, { 1, 4 })
 	check_is_nil("before string", { "asdf", "qwer" }, -1)
 	check_is_nil("multiple lines behind string", { "asdf", "qwer" }, 11)
+	check("unicode1", { "aa … aa" }, 7, { 0, 6 })
+	check("unicode2", { "aa …a… aa" }, 7, { 0, 6 })
+	check("unicode3", { "aa …a… aa", "aa …a… aa" }, 21, { 1, 6 })
 end)