feat: improve indexer performance

Author: shyim

.claude/settings.json

@@ -0,0 +1,19 @@
+{
+  "hooks": {
+    "SessionStart": [
+      {
+        "matcher": "startup",
+        "hooks": [
+          {
+            "type": "command",
+            "command": "if [ -f \"$CLAUDE_PROJECT_DIR/AGENTS.md\" ]; then cat \"$CLAUDE_PROJECT_DIR/AGENTS.md\"; fi"
+          },
+          {
+            "type": "command",
+            "command": "if [ -f \"$CLAUDE_PROJECT_DIR/docs/docs/contrib-code.md\" ]; then cat \"$CLAUDE_PROJECT_DIR/docs/docs/contrib-code.md\"; fi"
+          }
+        ]
+      }
+    ]
+  }
+}

.windsurfrules

@@ -0,0 +1,84 @@
+# Project Overview
+
+Shopware LSP is a Language Server Protocol implementation for Shopware and Symfony development. It provides IDE features (completion, go-to-definition, hover, diagnostics) for PHP, Twig, XML, and YAML files.
+
+**Tech Stack:** Go backend with tree-sitter parsing, BBolt embedded database for indexes, TypeScript VSCode extension.
+
+## Build & Test Commands
+
+```bash
+# Build
+go build                          # Build LSP server binary
+go build ./...                    # Build all packages
+
+# Test
+go test ./...                     # Run all tests
+go test -race ./internal/...      # Race detection (used in CI)
+go test ./internal/php/... -v     # Test specific package
+go test -run TestFeatureIndexer   # Run specific test
+
+# Lint
+golangci-lint run                 # Lint check (run before committing)
+
+# VSCode extension
+cd vscode-extension
+npm install && npm run compile    # Build extension
+npm run check-types               # Type check only
+```
+
+## Architecture
+
+### Entry Point
+`main.go` initializes the LSP server, registers all indexers and providers, then starts on stdin/stdout (JSON-RPC).
+
+### Key Packages (`internal/`)
+
+| Package | Purpose |
+|---------|---------|
+| `lsp/` | LSP protocol, server.go is the main handler |
+| `lsp/completion/` | 7 completion providers (services, routes, twig, snippets, features, system config, theme) |
+| `lsp/definition/` | 7 go-to-definition providers (same domains) |
+| `indexer/` | FileScanner for file watching, DataIndexer for BBolt persistence |
+| `symfony/` | Service container and route indexing from XML/YAML/PHP |
+| `php/` | PHP class/method indexing, type inference, alias resolution |
+| `twig/` | Template indexing, block tracking, extends/include parsing |
+| `snippet/` | Translation key indexing from JSON files |
+| `feature/` | Feature flag indexing from YAML |
+| `theme/` | Theme config and icon indexing |
+| `extension/` | Shopware extension metadata from composer.json |
+
+### Provider Pattern
+All LSP features use a provider interface pattern. Multiple providers can handle the same feature type, routed by document language/context.
+
+### Indexing Flow
+1. `FileScanner` detects file changes (fsnotify)
+2. Files parsed with tree-sitter based on type
+3. Each registered indexer processes AST nodes
+4. Data persisted to BBolt databases in cache directory
+
+## Testing Patterns
+
+Tests use `testify/assert` and `testify/require`. Common pattern:
+```go
+func TestSomething(t *testing.T) {
+    tempDir := t.TempDir()
+    indexer, err := NewIndexer(tempDir)
+    require.NoError(t, err)
+    defer indexer.Close()
+    // ... test logic
+}
+```
+
+## Commit Guidelines
+
+Use Conventional Commits: `type(scope): summary`
+- `fix:` bug fixes
+- `feat:` new features
+- `build:` build system changes
+- `docs:` documentation
+
+## Debug Tool
+
+```bash
+go run cmd/debug_ast/main.go path/to/file.php  # Visualize PHP AST
+```

AGENTS.md

@@ -7,7 +7,6 @@ replace github.com/tree-sitter-grammars/tree-sitter-xml => github.com/justinMBul
 replace github.com/tree-sitter-grammars/tree-sitter-scss => github.com/shyim/tree-sitter-scss v0.0.0-20250502122635-ca898ab73795
 
 require (
-	github.com/cespare/xxhash/v2 v2.3.0
 	github.com/fsnotify/fsnotify v1.9.0
 	github.com/sourcegraph/jsonrpc2 v0.2.1
 	github.com/stretchr/testify v1.10.0

AGENTS.md

@@ -1,5 +1,3 @@
-github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
-github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k=

CLAUDE.md

@@ -454,27 +454,41 @@ func (fs *FileScanner) RemoveFiles(ctx context.Context, paths []string) error {
 	return nil
 }
 
-// removeFileFromIndexers removes a single file from all indexers without affecting the hash database
-func (fs *FileScanner) removeFileFromIndexers(path string) error {
+func (fs *FileScanner) removeFilesFromIndexers(paths []string) error {
 	for _, indexer := range fs.indexer {
-		if err := indexer.RemovedFiles([]string{path}); err != nil {
+		if err := indexer.RemovedFiles(paths); err != nil {
 			return err
 		}
 	}
 	return nil
 }
 
-// updateFileState updates the stored file size and mtime for a file
-func (fs *FileScanner) updateFileState(path string, info os.FileInfo) error {
+func (fs *FileScanner) updateFileStates(files []fileState) error {
 	return fs.db.Update(func(tx *bbolt.Tx) error {
 		hashBucket := tx.Bucket([]byte("file_hashes"))
-		stateBytes := make([]byte, 16)
-		binary.LittleEndian.PutUint64(stateBytes[:8], uint64(info.Size()))
-		binary.LittleEndian.PutUint64(stateBytes[8:], uint64(info.ModTime().UnixNano()))
-		return hashBucket.Put([]byte(path), stateBytes)
+		for _, file := range files {
+			stateBytes := make([]byte, 16)
+			binary.LittleEndian.PutUint64(stateBytes[:8], uint64(file.info.Size()))
+			binary.LittleEndian.PutUint64(stateBytes[8:], uint64(file.info.ModTime().UnixNano()))
+			if err := hashBucket.Put([]byte(file.path), stateBytes); err != nil {
+				return err
+			}
+		}
+		return nil
 	})
 }
 
+type fileState struct {
+	path string
+	info os.FileInfo
+}
+
+type fileWork struct {
+	path    string
+	content []byte
+	info    os.FileInfo
+}
+
 // IndexFiles processes multiple files in parallel
 func (fs *FileScanner) IndexFiles(ctx context.Context, files []string) error {
 	if len(files) == 0 {
@@ -532,49 +546,81 @@ func (fs *FileScanner) IndexFiles(ctx context.Context, files []string) error {
 			defer wg.Done()
 
 			parsers := CreateTreesitterParsers()
+			const batchSize = 50
+			batch := make([]fileWork, 0, batchSize)
 
-			for path := range fileChan {
-				// Check if file needs indexing
-				needsIndexing, content, info, err := fs.fileNeedsIndexing(path)
-				if err != nil {
-					// We'll just skip file errors to reduce noise
-					continue
+			processBatch := func(items []fileWork) {
+				if len(items) == 0 {
+					return
 				}
 
-				// If file hasn't changed, skip it
-				if !needsIndexing {
-					continue
+				paths := make([]string, 0, len(items))
+				for _, item := range items {
+					paths = append(paths, item.path)
 				}
 
-				// Remove the file from all indexers since we're reindexing it
-				if err := fs.removeFileFromIndexers(path); err != nil {
+				if err := fs.removeFilesFromIndexers(paths); err != nil {
 					errChan <- err
-					continue
+					return
 				}
 
-				ext := strings.ToLower(filepath.Ext(path))
-
-				parser := parsers[ext]
-				if parser == nil {
-					panic(fmt.Sprintf("no parser found for file type: %s", ext))
-				}
+				for _, item := range items {
+					ext := strings.ToLower(filepath.Ext(item.path))
+					parser := parsers[ext]
+					if parser == nil {
+						panic(fmt.Sprintf("no parser found for file type: %s", ext))
+					}
 
-				tree := parser.Parse(content, nil)
+					tree := parser.Parse(item.content, nil)
 
-				for _, indexer := range fs.indexer {
-					if err := indexer.Index(path, tree.RootNode(), content); err != nil {
-						errChan <- err
+					for _, indexer := range fs.indexer {
+						if err := indexer.Index(item.path, tree.RootNode(), item.content); err != nil {
+							errChan <- err
+						}
 					}
+
+					tree.Close()
 				}
 
-				tree.Close()
+				fileStates := make([]fileState, 0, len(items))
+				for _, item := range items {
+					fileStates = append(fileStates, fileState{
+						path: item.path,
+						info: item.info,
+					})
+				}
 
-				// Update the file hash
-				if err := fs.updateFileState(path, info); err != nil {
+				if err := fs.updateFileStates(fileStates); err != nil {
 					errChan <- err
 				}
 			}
 
+			for path := range fileChan {
+				// Check if file needs indexing
+				needsIndexing, content, info, err := fs.fileNeedsIndexing(path)
+				if err != nil {
+					// We'll just skip file errors to reduce noise
+					continue
+				}
+
+				// If file hasn't changed, skip it
+				if !needsIndexing {
+					continue
+				}
+
+				batch = append(batch, fileWork{
+					path:    path,
+					content: content,
+					info:    info,
+				})
+				if len(batch) >= batchSize {
+					processBatch(batch)
+					batch = batch[:0]
+				}
+			}
+
+			processBatch(batch)
+
 			CloseTreesitterParsers(parsers)
 		}()
 	}