fix: Handle special characters in branch names (#945)

git-spice failed to work with branch names containing
non-ASCII characters like θ, ✅, 👨‍💻, café, or 日本語.
The issue occurred when reading branch metadata
from the state storage, which stores data as Git tree objects.

The root cause was that `git ls-tree` without the `-z` flag
returns quoted and escaped filenames for "unusual" characters.
For example, a branch named `feature/θ-theta` would be returned
as `feature/\316\270-theta` (escaped UTF-8 bytes).

This change updates both `ListTree` and `MakeTree` operations
to use the `-z` flag, which uses NUL-byte termination
and preserves UTF-8 characters exactly as they appear.

Additionally, ShamHub's find-by-branch API pattern was updated
from `{branch}` to `{branch...}` to match branch names
containing forward slashes, which are valid Git branch names.

Resolves #944

Co-Authored-By: Claude <noreply@anthropic.com>

Author: abhinav

.changes/unreleased/Fixed-20251120-200342.yaml

@@ -0,0 +1,5 @@
+kind: Fixed
+body: >-
+  Correctly handle unicode in branch names across various commands.
+  These would previously be stored in state, but not be accessible.
+time: 2025-11-20T20:03:42.420684-08:00

internal/forge/shamhub/find.go

@@ -12,7 +12,7 @@ import (
 
 var (
 	_ = shamhubRESTHandler("GET /{owner}/{repo}/change/{number}", (*ShamHub).handleGetChange)
-	_ = shamhubRESTHandler("GET /{owner}/{repo}/changes/by-branch/{branch}", (*ShamHub).handleFindChangesByBranch)
+	_ = shamhubRESTHandler("GET /{owner}/{repo}/changes/by-branch/{branch...}", (*ShamHub).handleFindChangesByBranch)
 )
 
 type getChangeRequest struct {

internal/git/integration_test.go

@@ -112,3 +112,75 @@ func TestIntegrationCommitListing(t *testing.T) {
 		}, msgs)
 	})
 }
+
+func TestIntegrationPeelToCommit_specialChars(t *testing.T) {
+	t.Parallel()
+
+	fixture, err := gittest.LoadFixtureScript([]byte(text.Dedent(`
+		as 'Test <test@example.com>'
+		at '2025-11-20T10:00:00Z'
+
+		git init
+		git commit --allow-empty -m 'Initial commit'
+
+		# Create branches with special characters
+		git checkout -b 'feature/θ-theta'
+		git add theta.txt
+		git commit -m 'Add theta feature'
+
+		git checkout -b 'feature/✅-checkmark'
+		git add checkmark.txt
+		git commit -m 'Add checkmark feature'
+
+		git checkout -b 'feature/👨‍💻-developer'
+		git add developer.txt
+		git commit -m 'Add developer feature'
+
+		git checkout -b 'feature/café'
+		git add cafe.txt
+		git commit -m 'Add café feature'
+
+		git checkout -b 'feature/日本語'
+		git add japanese.txt
+		git commit -m 'Add Japanese feature'
+
+		-- theta.txt --
+		Theta content
+		-- checkmark.txt --
+		Checkmark content
+		-- developer.txt --
+		Developer content
+		-- cafe.txt --
+		Café content
+		-- japanese.txt --
+		Japanese content
+	`)))
+	require.NoError(t, err)
+
+	ctx := t.Context()
+	repo, err := git.Open(ctx, fixture.Dir(), git.OpenOptions{
+		Log: silogtest.New(t),
+	})
+	require.NoError(t, err)
+
+	tests := []struct {
+		name   string
+		branch string
+	}{
+		{"Greek theta", "feature/θ-theta"},
+		{"Unicode checkmark", "feature/✅-checkmark"},
+		{"Combined emoji", "feature/👨‍💻-developer"},
+		{"Accented character", "feature/café"},
+		{"Japanese characters", "feature/日本語"},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			ctx := t.Context()
+			hash, err := repo.PeelToCommit(ctx, tt.branch)
+			require.NoError(t, err)
+			assert.NotEmpty(t, hash)
+			assert.NotEqual(t, git.ZeroHash, hash)
+		})
+	}
+}

internal/git/tree.go

@@ -13,6 +13,7 @@ import (
 	"strings"
 
 	"go.abhg.dev/gs/internal/must"
+	"go.abhg.dev/gs/internal/scanutil"
 )
 
 // Mode is the octal file mode of a Git tree entry.
@@ -72,7 +73,7 @@ type TreeEntry struct {
 // Returns the hash of the new tree and the number of entries written.
 func (r *Repository) MakeTree(ctx context.Context, ents iter.Seq2[TreeEntry, error]) (_ Hash, numEnts int, err error) {
 	var stdout bytes.Buffer
-	cmd := r.gitCmd(ctx, "mktree").Stdout(&stdout)
+	cmd := r.gitCmd(ctx, "mktree", "-z").Stdout(&stdout)
 	stdin, err := cmd.StdinPipe()
 	if err != nil {
 		return ZeroHash, numEnts, fmt.Errorf("pipe: %w", err)
@@ -111,9 +112,9 @@ func (r *Repository) MakeTree(ctx context.Context, ents iter.Seq2[TreeEntry, err
 			return ZeroHash, numEnts, fmt.Errorf("name %q contains a slash", ent.Name)
 		}
 
-		// mktree expects input in the form:
-		//	<mode> SP <type> SP <hash> TAB <name> NL
-		_, err := fmt.Fprintf(stdin, "%s %s %s\t%s\n", ent.Mode, ent.Type, ent.Hash, ent.Name)
+		// mktree -z expects input in the form:
+		//	<mode> SP <type> SP <hash> TAB <name> NUL
+		_, err := fmt.Fprintf(stdin, "%s %s %s\t%s\x00", ent.Mode, ent.Type, ent.Hash, ent.Name)
 		if err != nil {
 			return ZeroHash, numEnts, fmt.Errorf("write: %w", err)
 		}
@@ -153,6 +154,7 @@ func (r *Repository) ListTree(
 ) iter.Seq2[TreeEntry, error] {
 	args := []string{
 		"ls-tree",
+		"-z",          // NUL-terminate entries for proper handling of special characters
 		"--full-tree", // don't limit listing to the current working directory
 	}
 	if opts.Recurse {
@@ -162,14 +164,14 @@ func (r *Repository) ListTree(
 
 	return func(yield func(TreeEntry, error) bool) {
 		cmd := r.gitCmd(ctx, args...)
-		for line, err := range cmd.ScanLines(r.exec) {
+		for line, err := range cmd.Scan(r.exec, scanutil.SplitNull) {
 			if err != nil {
 				yield(TreeEntry{}, fmt.Errorf("git ls-tree: %w", err))
 				return
 			}
 
-			// ls-tree output is in the form:
-			//	<mode> SP <type> SP <hash> TAB <name> NL
+			// ls-tree -z output is in the form:
+			//	<mode> SP <type> SP <hash> TAB <name> NUL
 			modeTypeHash, name, ok := bytes.Cut(line, []byte{'\t'})
 			if !ok {
 				r.log.Warnf("ls-tree: skipping invalid line: %q", line)

internal/git/tree_test.go

@@ -2,6 +2,9 @@ package git_test
 
 import (
 	"bytes"
+	"os"
+	"os/exec"
+	"path/filepath"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -215,3 +218,130 @@ func TestIntegrationUpdateTree(t *testing.T) {
 		assert.Empty(t, ents)
 	})
 }
+
+func TestIntegrationListTree_specialCharsFromFiles(t *testing.T) {
+	t.Parallel()
+
+	ctx := t.Context()
+	dir := t.TempDir()
+	repo, _, err := git.Init(ctx, dir, git.InitOptions{
+		Log: silogtest.New(t),
+	})
+	require.NoError(t, err)
+
+	// Create actual files with special characters in their names
+	specialNames := []string{
+		"θ-theta",
+		"✅-checkmark",
+		"👨‍💻-developer",
+		"café",
+		"日本語",
+	}
+
+	// Create a subdirectory to hold the files
+	featureDir := filepath.Join(dir, "feature")
+	require.NoError(t, os.MkdirAll(featureDir, 0o755))
+
+	for _, name := range specialNames {
+		filePath := filepath.Join(featureDir, name)
+		require.NoError(t, os.WriteFile(filePath, []byte("content"), 0o644))
+	}
+
+	// Add files to git index and commit
+	addCmd := exec.Command("git", "add", "feature")
+	addCmd.Dir = dir
+	require.NoError(t, addCmd.Run())
+
+	commitCmd := exec.Command("git", "commit", "-m", "Add files with special characters")
+	commitCmd.Dir = dir
+	commitCmd.Env = append(os.Environ(),
+		"GIT_AUTHOR_NAME=Test",
+		"GIT_AUTHOR_EMAIL=test@example.com",
+		"GIT_COMMITTER_NAME=Test",
+		"GIT_COMMITTER_EMAIL=test@example.com",
+	)
+	require.NoError(t, commitCmd.Run())
+
+	// Get the commit hash
+	revParseCmd := exec.Command("git", "rev-parse", "HEAD")
+	revParseCmd.Dir = dir
+	commitHashBytes, err := revParseCmd.Output()
+	require.NoError(t, err)
+	commitHash := git.Hash(string(bytes.TrimSpace(commitHashBytes)))
+
+	// Get the tree from the commit
+	treeHash, err := repo.PeelToTree(ctx, commitHash.String())
+	require.NoError(t, err)
+
+	// List the tree recursively
+	ents, err := sliceutil.CollectErr(repo.ListTree(ctx, treeHash, git.ListTreeOptions{
+		Recurse: true,
+	}))
+	require.NoError(t, err)
+
+	// Extract names from the returned entries
+	var gotNames []string
+	for _, ent := range ents {
+		gotNames = append(gotNames, ent.Name)
+	}
+
+	// Build expected names with feature/ prefix
+	var expectedNames []string
+	for _, name := range specialNames {
+		expectedNames = append(expectedNames, "feature/"+name)
+	}
+
+	// Verify all special character names are returned exactly as they were created
+	assert.ElementsMatch(t, expectedNames, gotNames,
+		"ListTree should return file names with special characters exactly as they were created")
+}
+
+func TestIntegrationListTree_specialCharsFromMakeTree(t *testing.T) {
+	t.Parallel()
+
+	ctx := t.Context()
+	repo, _, err := git.Init(ctx, t.TempDir(), git.InitOptions{
+		Log: silogtest.New(t),
+	})
+	require.NoError(t, err)
+
+	emptyFile, err := repo.WriteObject(ctx, git.BlobType, bytes.NewReader(nil))
+	require.NoError(t, err)
+
+	// Create tree entries with special characters in their names
+	// (without slashes since MakeTree doesn't allow them in individual entry names)
+	specialNames := []string{
+		"θ-theta",
+		"✅-checkmark",
+		"👨‍💻-developer",
+		"café",
+		"日本語",
+	}
+
+	var entries []git.TreeEntry
+	for _, name := range specialNames {
+		entries = append(entries, git.TreeEntry{
+			Type: git.BlobType,
+			Name: name,
+			Hash: emptyFile,
+		})
+	}
+
+	treeHash, numEnts, err := repo.MakeTree(ctx, sliceutil.All2[error](entries))
+	require.NoError(t, err)
+	assert.Equal(t, len(specialNames), numEnts)
+
+	// List the tree and verify names match exactly
+	ents, err := sliceutil.CollectErr(repo.ListTree(ctx, treeHash, git.ListTreeOptions{}))
+	require.NoError(t, err)
+
+	// Extract names from the returned entries
+	var gotNames []string
+	for _, ent := range ents {
+		gotNames = append(gotNames, ent.Name)
+	}
+
+	// Verify all special character names are returned exactly as they were created
+	assert.ElementsMatch(t, specialNames, gotNames,
+		"ListTree should return names with special characters exactly as they were created")
+}

internal/spice/state/storage/git_test.go

@@ -186,3 +186,70 @@ func TestGitBackend_ConcurrentOperations(t *testing.T) {
 	require.NoError(t, err)
 	assert.Equal(t, "final_value", finalValue)
 }
+
+func TestGitBackend_SpecialCharacterKeys(t *testing.T) {
+	ctx := t.Context()
+	repo, _, err := git.Init(ctx, t.TempDir(), git.InitOptions{
+		Log: silogtest.New(t),
+	})
+	require.NoError(t, err)
+
+	backend := NewGitBackend(GitConfig{
+		Repo:        repo,
+		Ref:         "refs/data",
+		AuthorName:  "Test Author",
+		AuthorEmail: "test@example.com",
+		Log:         silogtest.New(t),
+	})
+
+	tests := []struct {
+		name  string
+		key   string
+		value string
+	}{
+		{"Greek theta", "branches/feature/θ-theta", "theta-value"},
+		{"Unicode checkmark", "branches/feature/✅-checkmark", "checkmark-value"},
+		{"Combined emoji", "branches/feature/👨‍💻-developer", "developer-value"},
+		{"Accented character", "branches/feature/café", "cafe-value"},
+		{"Japanese characters", "branches/feature/日本語", "japanese-value"},
+	}
+
+	db := NewDB(backend)
+
+	// Store all values
+	for _, tt := range tests {
+		t.Run("Set/"+tt.name, func(t *testing.T) {
+			err := db.Set(ctx, tt.key, tt.value, "add "+tt.name)
+			require.NoError(t, err)
+		})
+	}
+
+	// Retrieve all values
+	for _, tt := range tests {
+		t.Run("Get/"+tt.name, func(t *testing.T) {
+			var got string
+			err := backend.Get(ctx, tt.key, &got)
+			require.NoError(t, err)
+			assert.Equal(t, tt.value, got)
+		})
+	}
+
+	// List keys and verify all special character keys are present
+	t.Run("Keys", func(t *testing.T) {
+		keys, err := backend.Keys(ctx, "branches")
+		require.NoError(t, err)
+
+		expectedKeys := []string{
+			"feature/θ-theta",
+			"feature/✅-checkmark",
+			"feature/👨‍💻-developer",
+			"feature/café",
+			"feature/日本語",
+		}
+
+		for _, expectedKey := range expectedKeys {
+			assert.Contains(t, keys, expectedKey,
+				"Keys list should contain %q", expectedKey)
+		}
+	})
+}