Merge pull request #170 from oraios/fix_deadlocks

Fix deadlocks

Author: MischaPanch

pyproject.toml

@@ -24,7 +24,6 @@ dependencies = [
   "python-dotenv>=1.0.0, <2",
   "mcp>=1.5.0",
   "fastapi>=0.115.12",
-  "fastmcp>=0.4.1",
   "sensai-utils>=1.4.0",
   "pydantic>=2.10.6",
   "types-pyyaml>=6.0.12.20241230",
@@ -107,7 +106,9 @@ exclude = "^build/|^docs/"
 PYDEVD_DISABLE_FILE_VALIDATION = "1"
 
 [tool.poe.tasks]
-test = "pytest test --color=yes"
+# Uses PYTEST_MARKERS env var for default markers (defaults to "not java and not rust")
+# Set PYTEST_MARKERS="" to run all tests
+test = "pytest test --color=yes -m \"${PYTEST_MARKERS:-not java and not rust}\""
 _black_check = "black --check --exclude src/multilspy/ src scripts test"
 _ruff_check = "ruff check --exclude .venv/ --exclude src/multilspy/ src scripts test"
 _black_format = "black --exclude .venv/|src/multilspy/ src scripts test"
@@ -259,4 +260,5 @@ markers = [
     "typescript: language server running for TypeScript",
     "php: language server running for PHP",
     "snapshot: snapshot tests",
+    "isolated_process: test runs with process isolated agent",
 ]

src/multilspy/language_server.py

@@ -317,7 +317,7 @@ def is_ignored_path(self, relative_path: str, ignore_unsupported_files: bool = T
 
         return False
 
-    async def _shutdown(self, timeout: float = 10.0):
+    async def _shutdown(self, timeout: float = 5.0):
         """
         A robust shutdown process designed to terminate cleanly on all platforms, including Windows,
         by explicitly closing all I/O pipes.
@@ -946,7 +946,7 @@ def visit_tree_nodes_and_build_tree_repr(node: GenericDocumentSymbol) -> List[mu
             self._document_symbols_cache[cache_key] = (file_data.content_hash, result)
             self._cache_has_changed = True
         return result
-    
+
     async def request_full_symbol_tree(self, within_relative_path: str | None = None, include_body: bool = False) -> List[multilspy_types.UnifiedSymbolInformation]:
         """
         Will go through all files in the project or within a relative path and build a tree of symbols. 
@@ -1750,41 +1750,44 @@ def start_server(self) -> Iterator["SyncLanguageServer"]:
 
         :return: None
         """
-        self.loop = asyncio.new_event_loop()
-        self.loop_thread = threading.Thread(target=self.loop.run_forever, daemon=True)
-        self.loop_thread.start()
-        ctx = self.language_server.start_server()
-        asyncio.run_coroutine_threadsafe(ctx.__aenter__(), loop=self.loop).result()
+        self.start()
         yield self
+        self.language_server.logger.log("SyncLS Startup: exiting LS context", logging.DEBUG)
         self.stop()
 
     def request_definition(self, file_path: str, line: int, column: int) -> List[multilspy_types.Location]:
         """
         Raise a [textDocument/definition](https://microsoft.github.io/language-server-protocol/specifications/lsp/3.17/specification/#textDocument_definition) request to the Language Server
         for the symbol at the given line and column in the given file. Wait for the response and return the result.
 
-        :param relative_file_path: The relative path of the file that has the symbol for which definition should be looked up
-        :param line: The line number of the symbol
-        :param column: The column number of the symbol
+        :param file_path: The path of the file relative to the repository root.
+        :param line: The line number (zero-indexed).
+        :param column: The column number (zero-indexed).
 
-        :return List[multilspy_types.Location]: A list of locations where the symbol is defined
+        :return: A list of Locations defining the symbol.
         """
+        if not self.is_running():
+            raise RuntimeError("Language server is not running. Cannot make requests to a stopped language server.")
+            
         result = asyncio.run_coroutine_threadsafe(
             self.language_server.request_definition(file_path, line, column), self.loop
         ).result(timeout=self.timeout)
         return result
 
     def request_references(self, file_path: str, line: int, column: int) -> List[multilspy_types.Location]:
         """
-        Raise a [textDocument/references](https://microsoft.github.io/language-server-protocol/specifications/lsp/3.17/specification/#textDocument_references) request to the Language Server
-        to find references to the symbol at the given line and column in the given file. Wait for the response and return the result.
+        Raises a [textDocument/references](https://microsoft.github.io/language-server-protocol/specifications/lsp/3.17/specification/#textDocument_references) request to the Language Server
+        for the symbol at the given line and column in the given file. Wait for the response and return the result.
 
-        :param relative_file_path: The relative path of the file that has the symbol for which references should be looked up
-        :param line: The line number of the symbol
-        :param column: The column number of the symbol
+        :param file_path: The path of the file relative to the repository root.
+        :param line: The line number (zero-indexed).
+        :param column: The column number (zero-indexed).
 
-        :return List[multilspy_types.Location]: A list of locations where the symbol is referenced
+        :return: A list of Locations referencing the symbol.
         """
+        if not self.is_running():
+            raise RuntimeError("Language server is not running. Cannot make requests to a stopped language server.")
+            
         try:
             result = asyncio.run_coroutine_threadsafe(
                 self.language_server.request_references(file_path, line, column), self.loop
@@ -1870,6 +1873,9 @@ def request_full_symbol_tree(self, within_relative_path: str | None = None, incl
 
         :return: A list of root symbols representing the top-level packages/modules in the project.
         """
+        if not self.is_running():
+            raise RuntimeError("Language server is not running. Cannot make requests to a stopped language server.")
+        
         result = asyncio.run_coroutine_threadsafe(
             self.language_server.request_full_symbol_tree(within_relative_path, include_body), self.loop
         ).result(timeout=self.timeout)
@@ -1882,7 +1888,9 @@ def request_dir_overview(self, relative_dir_path: str) -> dict[str, list[tuple[s
         Maps relative paths of all contained files to info about top-level symbols in the file
         (name, kind, line, column).
         """
-        assert self.loop
+        if not self.is_running():
+            raise RuntimeError("Language server is not running. Cannot make requests to a stopped language server.")
+            
         result = asyncio.run_coroutine_threadsafe(
             self.language_server.request_dir_overview(relative_dir_path), self.loop
         ).result(timeout=self.timeout)
@@ -2255,8 +2263,6 @@ def stop(self, shutdown_timeout: float = 5.0) -> None:
         shutdown_type = "Nuclear" if is_windows else "Graceful"
         self.language_server.logger.log(f"{shutdown_type} shutdown complete - all references cleared", logging.INFO)
 
-
-
     def save_cache(self):
         """
         Save the cache to a file.

src/multilspy/language_servers/pyright_language_server/pyright_server.py

@@ -2,10 +2,12 @@
 Provides Python specific instantiation of the LanguageServer class. Contains various configurations and settings specific to Python.
 """
 
+import asyncio
 import json
 import logging
 import os
 import pathlib
+import re
 from contextlib import asynccontextmanager
 from typing import AsyncIterator, Tuple
 
@@ -38,6 +40,10 @@ def __init__(self, config: MultilspyConfig, logger: MultilspyLogger, repository_
             "python",
         )
 
+        # Event to signal when initial workspace analysis is complete
+        self.analysis_complete = asyncio.Event()
+        self.found_source_files = False
+
     @override
     def is_ignored_dirname(self, dirname: str) -> bool:
         return super().is_ignored_dirname(dirname) or dirname in ["venv", "__pycache__"]
@@ -145,16 +151,19 @@ def _get_initialize_params(self, repository_absolute_path: str) -> InitializePar
     @asynccontextmanager
     async def start_server(self) -> AsyncIterator["PyrightServer"]:
         """
-        Starts the Pyright Language Server, waits for the server to be ready and yields the LanguageServer instance.
+        Starts the Pyright Language Server and waits for initial workspace analysis to complete.
+        
+        This prevents zombie processes by ensuring Pyright has finished its initial background
+        tasks before we consider the server ready.
 
         Usage:
         ```
         async with lsp.start_server():
-            # LanguageServer has been initialized and ready to serve requests
+            # LanguageServer has been initialized and workspace analysis is complete
             await lsp.request_definition(...)
             await lsp.request_references(...)
             # Shutdown the LanguageServer on exit from scope
-        # LanguageServer has been shutdown
+        # LanguageServer has been shutdown cleanly
         ```
         """
 
@@ -164,13 +173,33 @@ async def execute_client_command_handler(params):
         async def do_nothing(params):
             return
 
-        async def check_experimental_status(params):
-            if params["quiescent"] == True:
-                self.completions_available.set()
-
         async def window_log_message(msg):
-            self.logger.log(f"LSP: window/logMessage: {msg}", logging.INFO)
+            """
+            Monitor Pyright's log messages to detect when initial analysis is complete.
+            Pyright logs "Found X source files" when it finishes scanning the workspace.
+            """
+            message_text = msg.get("message", "")
+            self.logger.log(f"LSP: window/logMessage: {message_text}", logging.INFO)
+            
+            # Look for "Found X source files" which indicates workspace scanning is complete
+            # Unfortunately, pyright is unreliable and there seems to be no better way
+            if re.search(r"Found \d+ source files?", message_text):
+                self.logger.log("Pyright workspace scanning complete", logging.INFO)
+                self.found_source_files = True
+                self.analysis_complete.set()
+                self.completions_available.set()
 
+        async def check_experimental_status(params):
+            """
+            Also listen for experimental/serverStatus as a backup signal
+            """
+            if params.get("quiescent") == True:
+                self.logger.log("Received experimental/serverStatus with quiescent=true", logging.INFO)
+                if not self.found_source_files:
+                    self.analysis_complete.set()
+                    self.completions_available.set()
+
+        # Set up notification handlers
         self.server.on_request("client/registerCapability", do_nothing)
         self.server.on_notification("language/status", do_nothing)
         self.server.on_notification("window/logMessage", window_log_message)
@@ -195,6 +224,23 @@ async def window_log_message(msg):
             self.logger.log(f"Received initialize response from pyright server: {init_response}", logging.INFO)
 
             # Verify that the server supports our required features
+            assert "textDocumentSync" in init_response["capabilities"]
+            assert "completionProvider" in init_response["capabilities"]
+            assert "definitionProvider" in init_response["capabilities"]
+
+            # Complete the initialization handshake
             self.server.notify.initialized({})
+            
+            # Wait for Pyright to complete its initial workspace analysis
+            # This prevents zombie processes by ensuring background tasks finish
+            self.logger.log("Waiting for Pyright to complete initial workspace analysis...", logging.INFO)
+            try:
+                await asyncio.wait_for(self.analysis_complete.wait(), timeout=1.0)
+                self.logger.log("Pyright initial analysis complete, server ready", logging.INFO)
+            except asyncio.TimeoutError:
+                self.logger.log("Timeout waiting for Pyright analysis completion, proceeding anyway", logging.WARNING)
+                # Fallback: assume analysis is complete after timeout
+                self.analysis_complete.set()
+                self.completions_available.set()
 
-            yield self
+            yield self

src/multilspy/language_servers/typescript_language_server/typescript_language_server.py

@@ -186,12 +186,22 @@ async def do_nothing(params):
 
         async def window_log_message(msg):
             self.logger.log(f"LSP: window/logMessage: {msg}", logging.INFO)
+            
+        
+        async def check_experimental_status(params):
+            """
+            Also listen for experimental/serverStatus as a backup signal
+            """
+            if params.get("quiescent") == True:
+                self.server_ready.set()
+                self.completions_available.set()
 
         self.server.on_request("client/registerCapability", register_capability_handler)
         self.server.on_notification("window/logMessage", window_log_message)
         self.server.on_request("workspace/executeClientCommand", execute_client_command_handler)
         self.server.on_notification("$/progress", do_nothing)
         self.server.on_notification("textDocument/publishDiagnostics", do_nothing)
+        self.server.on_notification("experimental/serverStatus", check_experimental_status)
 
         async with super().start_server():
             self.logger.log("Starting TypeScript server process", logging.INFO)
@@ -213,11 +223,13 @@ async def window_log_message(msg):
             }
 
             self.server.notify.initialized({})
-            self.completions_available.set()
-
-            # TypeScript server is typically ready immediately after initialization
-            self.server_ready.set()
-            await self.server_ready.wait()
+            try:
+                await asyncio.wait_for(self.server_ready.wait(), timeout=1.0)
+            except asyncio.TimeoutError:
+                self.logger.log("Timeout waiting for TypeScript server to become ready, proceeding anyway", logging.INFO)
+                # Fallback: assume server is ready after timeout
+                self.server_ready.set()
+                self.completions_available.set()
 
             yield self
 

src/multilspy/multilspy_logger.py

@@ -30,7 +30,7 @@ def __init__(self, json_format: bool = False, log_level: int = logging.INFO) ->
         self.logger.setLevel(log_level)
         self.json_format = json_format
 
-    def log(self, debug_message: str, level: int, sanitized_error_message: str = "") -> None:
+    def log(self, debug_message: str, level: int, sanitized_error_message: str = "", stacklevel: int = 2) -> None:
         """
         Log the debug and santized messages using the logger
         """
@@ -59,6 +59,7 @@ def log(self, debug_message: str, level: int, sanitized_error_message: str = "")
             self.logger.log(
                 level=level,
                 msg=debug_log_line.json(),
+                stacklevel=stacklevel,
             )
         else:
-            self.logger.log(level, debug_message)
+            self.logger.log(level, debug_message, stacklevel=stacklevel)