diff --git a/.serena/project.yml b/.serena/project.yml index 84b6fcf89..c9fd4467c 100644 --- a/.serena/project.yml +++ b/.serena/project.yml @@ -3,15 +3,18 @@ project_name: "serena" # list of languages for which language servers are started; choose from: -# al bash clojure cpp csharp -# csharp_omnisharp dart elixir elm erlang -# fortran fsharp go groovy haskell -# java julia kotlin lua markdown -# matlab nix pascal perl php -# php_phpactor powershell python python_jedi r -# rego ruby ruby_solargraph rust scala -# swift terraform toml typescript typescript_vts -# vue yaml zig +# al ansible bash clojure cpp +# cpp_ccls crystal csharp csharp_omnisharp dart +# elixir elm erlang fortran fsharp +# go groovy haskell haxe hlsl +# java json julia kotlin lean4 +# lua luau markdown matlab msl +# nix ocaml pascal perl php +# php_phpactor powershell python python_jedi python_ty +# r rego ruby ruby_solargraph rust +# scala solidity swift systemverilog terraform +# toml typescript typescript_vts vue yaml +# zig # (This list may be outdated. For the current list, see values of Language enum here: # https://github.com/oraios/serena/blob/main/src/solidlsp/ls_config.py # For some languages, there are alternative language servers, e.g. csharp_omnisharp, ruby_solargraph.) @@ -44,49 +47,12 @@ read_only: false # list of tool names to exclude. # This extends the existing exclusions (e.g. from the global configuration) -# -# Below is the complete list of tools for convenience. -# To make sure you have the latest list of tools, and to view their descriptions, -# execute `uv run scripts/print_tool_overview.py`. -# * `activate_project`: Activates a project based on the project name or path. -# * `check_onboarding_performed`: Checks whether project onboarding was already performed. -# * `create_text_file`: Creates/overwrites a file in the project directory. -# * `delete_memory`: Delete a memory file. Should only happen if a user asks for it explicitly, -# for example by saying that the information retrieved from a memory file is no longer correct -# or no longer relevant for the project. -# * `edit_memory`: Replaces content matching a regular expression in a memory. -# * `execute_shell_command`: Executes a shell command. -# * `find_file`: Finds files in the given relative paths -# * `find_referencing_symbols`: Finds symbols that reference the given symbol using the language server backend -# * `find_symbol`: Performs a global (or local) search using the language server backend. -# * `get_current_config`: Prints the current configuration of the agent, including the active and available projects, tools, contexts, and modes. -# * `get_symbols_overview`: Gets an overview of the top-level symbols defined in a given file. -# * `initial_instructions`: Provides instructions Serena usage (i.e. the 'Serena Instructions Manual') -# for clients that do not read the initial instructions when the MCP server is connected. -# * `insert_after_symbol`: Inserts content after the end of the definition of a given symbol. -# * `insert_before_symbol`: Inserts content before the beginning of the definition of a given symbol. -# * `list_dir`: Lists files and directories in the given directory (optionally with recursion). -# * `list_memories`: List available memories. Any memory can be read using the `read_memory` tool. -# * `onboarding`: Performs onboarding (identifying the project structure and essential tasks, e.g. for testing or building). -# * `read_file`: Reads a file within the project directory. -# * `read_memory`: Read the content of a memory file. This tool should only be used if the information -# is relevant to the current task. You can infer whether the information -# is relevant from the memory file name. -# You should not read the same memory file multiple times in the same conversation. -# * `rename_memory`: Renames or moves a memory. Moving between project and global scope is supported -# (e.g., renaming "global/foo" to "bar" moves it from global to project scope). -# * `rename_symbol`: Renames a symbol throughout the codebase using language server refactoring capabilities. -# For JB, we use a separate tool. -# * `replace_content`: Replaces content in a file (optionally using regular expressions). -# * `replace_symbol_body`: Replaces the full definition of a symbol using the language server backend. -# * `safe_delete_symbol`: -# * `search_for_pattern`: Performs a search for a pattern in the project. -# * `write_memory`: Write some information (utf-8-encoded) about this project that can be useful for future tasks to a memory in md format. -# The memory name should be meaningful. +# Find the list of tools here: https://oraios.github.io/serena/01-about/035_tools.html excluded_tools: [] # list of tools to include that would otherwise be disabled (particularly optional tools that are disabled by default). # This extends the existing inclusions (e.g. from the global configuration). +# Find the list of tools here: https://oraios.github.io/serena/01-about/035_tools.html included_optional_tools: [] # initial prompt for the project. It will always be given to the LLM upon activating the project @@ -121,14 +87,17 @@ encoding: utf-8 base_modes: # list of mode names that are to be activated by default. -# The full set of modes to be activated is base_modes + default_modes. +# The full set of modes to be activated is base_modes (from global config) + default_modes + added_modes. # If the setting is undefined, the default_modes from the global configuration (serena_config.yml) apply. # Otherwise, this overrides the setting from the global configuration (serena_config.yml). # This setting can, in turn, be overridden by CLI parameters (--mode). +# Set this to [] to not use the default modes defined in the global config for this project. +# See https://oraios.github.io/serena/02-usage/050_configuration.html#modes default_modes: # fixed set of tools to use as the base tool set (if non-empty), replacing Serena's default set of tools. # This cannot be combined with non-empty excluded_tools or included_optional_tools. +# Find the list of tools here: https://oraios.github.io/serena/01-about/035_tools.html fixed_tools: [] # time budget (seconds) per tool call for the retrieval of additional symbol information @@ -166,3 +135,10 @@ ignored_memory_patterns: [] # Have a look at the docstring of the constructors of the LS implementations within solidlsp (e.g., for C# or PHP) to see which options are available. # No documentation on options means no options are available. ls_specific_settings: {} + +# list of mode names to be activated additionally for this project. +# The full set of modes to be activated is base_modes (from global config) + default_modes + added_modes. +# Otherwise, this setting overrides the global configuration. +# Set this to a list of mode names to always include the respective modes for this project. +# See https://oraios.github.io/serena/02-usage/050_configuration.html#modes +added_modes: diff --git a/CHANGELOG.md b/CHANGELOG.md index 96330eaa0..158a82b9b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,12 @@ Status of the `main` branch. Changes prior to the next official version change will appear here. +* General: + - Breaking change in mode definitions: Projects (project.yml) can no longer override `base_modes`. + Instead, they can define `added_modes` to add modes on top of base and default modes. + See updated [documentation on modes](https://oraios.github.io/serena/02-usage/050_configuration.html#modes). + - Serena's default configuration now uses `interactive` and `editing` as `base_modes` instead of as `default_modes`. + * Language Servers: - Java (`eclipse.jdt.ls`): Add upstream JDTLS mode for offline / restricted-network use. Setting both `jdtls_path` and `lombok_path` in `ls_specific_settings.java` makes Serena use an existing upstream JDTLS installation (e.g. `brew install jdtls`) and the system JDK 21+, skipping the ~500 MB vscode-java VSIX, Gradle, and IntelliCode downloads. New related setting `java_home` lets the user override the JDK used to launch JDTLS. Default behavior unchanged — the JDTLS workspace hash is preserved bit-for-bit for users on the default route, so existing project caches are reused without a one-time reindex; the launcher path is mixed into the hash only when `jdtls_path` is set, isolating upstream installations from the default workspace. #1415 diff --git a/docs/02-usage/050_configuration.md b/docs/02-usage/050_configuration.md index d0b4c3428..6efedcfc0 100644 --- a/docs/02-usage/050_configuration.md +++ b/docs/02-usage/050_configuration.md @@ -118,32 +118,26 @@ Examples of built-in modes include: Find the concrete definitions of these modes [here](https://github.com/oraios/serena/tree/main/src/serena/resources/config/modes). -Active modes are configured in (from lowest to highest precedence): +The modes to be activated are configured in: * the global configuration file (`serena_config.yml`) + - defines `base_modes`, which are always included + - defines `default_modes`, which can be overridden by projects or command line parameters * the project configuration file (`project.yml`) + - defines `default_modes` (overriding the default modes in the global configuration) + - defines `added_modes`, which are added on top * at startup via command-line parameters - -The two former sources define both **base modes** and **default modes**. -Ultimately, the active modes are the union of base modes and default modes (after applying all overrides). -Command-line parameters override default modes but not base modes. -Base modes should thus be used to define modes that you always want to be active, regardless of command-line parameters. - -Command-line parameters for overriding default modes: -When launching the MCP sever, specify modes using `--mode `; multiple modes can be specified, e.g. `--mode planning --mode no-onboarding`. - -:::{important} -By default, Serena activates the two modes `interactive` and `editing` (as defined in the global configuration). - -As soon as you start to specify modes via the command line, only the modes you explicitly specify will be active, however. -Therefore, if you want to keep the default modes, you must specify them as well. -For example, to add mode `no-memories` to the default behaviour, specify -```shell ---mode interactive --mode editing --mode no-memories -``` - -If you want to keep certain modes as always active, regardless of command-line parameters, -define them as *base modes* in the global or project configuration. -::: + - can override default modes with `--mode` + - can define modes to be added on top with `--add-mode` + +Ultimately, the active modes are given by the union of + * `base_modes` defined in the global configuration (always active) + * `default_modes` (defined in the global configuration, optionally overridden by the project/CLI) + * `added_modes` (defined in the project configuration/via CLI parameters) + +So you should + * define modes you definitely always want to use in `base_modes`, + * define modes that you typically want to use but sometimes want to override in `default_modes`, + * use `added_modes` to add modes that you need only for specific projects/sessions. :::{note} **Mode Compatibility**: While you can combine modes, some may be semantically incompatible (e.g., `interactive` and `one-shot`). diff --git a/src/serena/agent.py b/src/serena/agent.py index 2daf0b4eb..db67577d4 100644 --- a/src/serena/agent.py +++ b/src/serena/agent.py @@ -31,6 +31,8 @@ from serena.config.serena_config import ( LanguageBackend, ModeSelectionDefinition, + ModeSelectionDefinitionWithAddedModes, + ModeSelectionDefinitionWithBaseModes, NamedToolInclusionDefinition, RegisteredProject, SerenaConfig, @@ -206,18 +208,36 @@ class ActiveModes: def __init__(self) -> None: self._configured_base_modes: Sequence[str] | None = None self._configured_default_modes: Sequence[str] | None = None + self._added_modes: set[str] = set() + self._dynamically_activated_mode_names: set[str] = set() + """ + the subset of active mode names that are dynamically activated (not necessarily enabled after project change) + """ self._active_mode_names: Sequence[str] = [] + """ + the full list of active mode names + """ def apply(self, mode_selection: ModeSelectionDefinition) -> None: + log.debug("Applying mode selection definition %s", mode_selection) + # apply overrides - log.debug("Applying mode selection: default_modes=%s, base_modes=%s", mode_selection.default_modes, mode_selection.base_modes) - if mode_selection.base_modes is not None: - self._configured_base_modes = mode_selection.base_modes + if isinstance(mode_selection, ModeSelectionDefinitionWithBaseModes): + if mode_selection.base_modes is not None: + self._configured_base_modes = mode_selection.base_modes if mode_selection.default_modes is not None: self._configured_default_modes = mode_selection.default_modes log.debug("Current mode selection: base_modes=%s, default_modes=%s", self._configured_base_modes, self._configured_default_modes) - self._active_mode_names = sorted(set(self._configured_base_modes or []) | set(self._configured_default_modes or [])) + # apply added modes (if any) + if isinstance(mode_selection, ModeSelectionDefinitionWithAddedModes): + if mode_selection.added_modes: + log.debug("Adding modes: %s", mode_selection.added_modes) + self._added_modes.update(mode_selection.added_modes) + log.debug("Current added modes: %s", self._added_modes) + + self._dynamically_activated_mode_names = set(self._configured_default_modes or []) | self._added_modes + self._active_mode_names = sorted(set(self._configured_base_modes or []) | self._dynamically_activated_mode_names) def get_mode_names(self) -> Sequence[str]: return self._active_mode_names @@ -231,8 +251,8 @@ def get_mode_instance(cls, mode_name: str) -> SerenaAgentMode: def get_modes(self) -> Sequence[SerenaAgentMode]: return [self.get_mode_instance(mode_name) for mode_name in self._active_mode_names] - def get_default_modes(self) -> Sequence[SerenaAgentMode]: - return [self.get_mode_instance(mode_name) for mode_name in self._configured_default_modes or []] + def get_dynamically_activated_modes(self) -> Sequence[SerenaAgentMode]: + return [self.get_mode_instance(mode_name) for mode_name in self._dynamically_activated_mode_names] def get_base_modes(self) -> Sequence[SerenaAgentMode]: return [self.get_mode_instance(mode_name) for mode_name in self._configured_base_modes or []] @@ -511,8 +531,7 @@ def __init__( :param serena_config: the Serena configuration or None to read the configuration from the default location. :param context: the context in which the agent is operating, None for default context. The context may adjust prompts, tool availability, and tool descriptions. - :param modes: list of modes in which the agent is operating (they will be combined), None for default modes. - The modes may adjust prompts, tool availability, and tool descriptions. + :param modes: mode selection definition to apply for this session :param memory_log_handler: a MemoryLogHandler instance from which to read log messages; if None, a new one will be created if necessary. """ @@ -521,7 +540,7 @@ def __init__( self._gui_log_viewer: Optional["GuiLogViewer"] = None self._dashboard_manager: DashboardManager | None = None self._project_prompt_status = ProjectPromptProvisionStatus() - self._mode_overrides = modes + self._session_mode_selection_definition = modes self.version = serena_version() # obtain serena configuration using the decoupled factory function @@ -712,9 +731,11 @@ def _create_base_toolset( # * base modes: These cannot be changed, so they are fully applied for base_mode in modes.get_base_modes(): tool_inclusion_definitions.append(base_mode) - # * default modes: When not in a single-project context, these modes are dynamic (can later be turned off), - # so we consider only their inclusions (but not their exclusions, because these must not be hard) - for mode in modes.get_default_modes(): + # * dynamically activated modes: + # - When not in a single-project context, these modes can later be turned off, + # so we consider only their inclusions (but not their exclusions, because these must not be hard). + # - In a single-project context, we can consider them fully. + for mode in modes.get_dynamically_activated_modes(): if is_single_project: tool_inclusion_definitions.append(mode) else: @@ -986,8 +1007,8 @@ def _update_active_modes(self, log_message: bool = True) -> None: self._active_modes.apply(self.serena_config) if self._active_project: self._active_modes.apply(self._active_project.project_config) - if self._mode_overrides: - self._active_modes.apply(self._mode_overrides) + if self._session_mode_selection_definition: + self._active_modes.apply(self._session_mode_selection_definition) if log_message: active_mode_names = self._active_modes.get_mode_names() log.info(f"Active modes ({len(active_mode_names)}): {', '.join(active_mode_names)}") diff --git a/src/serena/cli.py b/src/serena/cli.py index e7407545d..8591d712e 100644 --- a/src/serena/cli.py +++ b/src/serena/cli.py @@ -23,6 +23,7 @@ from serena.config.serena_config import ( LanguageBackend, ModeSelectionDefinition, + ModeSelectionDefinitionWithAddedModes, ProjectConfig, RegisteredProject, SerenaConfig, @@ -37,7 +38,6 @@ ) from serena.prompt_factory import SerenaPromptFactory from serena.util.cli_util import AutoRegisteringGroup -from serena.util.dataclass import get_dataclass_default from serena.util.logging import MemoryLogHandler from solidlsp.ls_config import Language from solidlsp.ls_types import SymbolKind @@ -46,15 +46,17 @@ log = logging.getLogger(__name__) _MAX_CONTENT_WIDTH = 200 -_MODES_EXPLANATION = f"""\b\nBuilt-in mode names or paths to custom mode YAMLs with which to -override the default modes defined in the global Serena configuration or +_MODES_EXPLANATION = """\b\nBuilt-in mode names or paths to custom mode YAMLs with which to +override the default_modes defined in the global Serena configuration or the active project. For details on mode configuration, see https://oraios.github.io/serena/02-usage/050_configuration.html#modes. -If no configuration changes were made, the base defaults are: - {get_dataclass_default(SerenaConfig, "default_modes")}. -Overriding them means that they no longer apply, so you will need to -re-specify them in addition to further modes if you want to keep them.""" +""" +_ADD_MODES_EXPLANATION = """\b\nMode names or paths to custom mode YAMLs which shall +be added on top of the other modes specified by the global/project configuration. +For details on mode configuration, see + https://oraios.github.io/serena/02-usage/050_configuration.html#modes. +""" def find_project_root(root: str | Path | None = None) -> str | None: @@ -228,13 +230,22 @@ def setup(client: str) -> None: ) @click.option( "--mode", - "modes", + "default_modes", type=str, multiple=True, default=(), show_default=False, help=_MODES_EXPLANATION, ) + @click.option( + "--add-mode", + "added_modes", + type=str, + multiple=True, + default=(), + show_default=False, + help=_ADD_MODES_EXPLANATION, + ) @click.option( "--language-backend", type=click.Choice([lb.value for lb in LanguageBackend]), @@ -300,7 +311,8 @@ def start_mcp_server( project_file_arg: str | None, project_from_cwd: bool | None, context: str, - modes: Sequence[str], + default_modes: Sequence[str], + added_modes: Sequence[str], language_backend: str | None, transport: Literal["stdio", "sse", "streamable-http"], host: str, @@ -346,11 +358,15 @@ def start_mcp_server( project_file = project_file_arg or project + mode_selection_def: ModeSelectionDefinition | None = None + if default_modes or added_modes: + mode_selection_def = ModeSelectionDefinitionWithAddedModes(default_modes=default_modes or None, added_modes=added_modes or None) + factory = SerenaMCPFactory(context=context, project=project_file, memory_log_handler=memory_log_handler) server = factory.create_mcp_server( host=host, port=port, - modes=modes, + mode_selection_def=mode_selection_def, language_backend=LanguageBackend.from_str(language_backend) if language_backend else None, enable_web_dashboard=enable_web_dashboard, open_web_dashboard=open_web_dashboard, diff --git a/src/serena/config/serena_config.py b/src/serena/config/serena_config.py index 329e0e866..bade9a28c 100644 --- a/src/serena/config/serena_config.py +++ b/src/serena/config/serena_config.py @@ -168,10 +168,22 @@ def __str__(self) -> str: @dataclass class ModeSelectionDefinition: - base_modes: Sequence[str] | None = None default_modes: Sequence[str] | None = None +@dataclass +class ModeSelectionDefinitionWithBaseModes(ModeSelectionDefinition): + base_modes: Sequence[str] | None = ("interactive", "editing") + """ + the base modes to use, which are always guaranteed to be included + """ + + +@dataclass +class ModeSelectionDefinitionWithAddedModes(ModeSelectionDefinition): + added_modes: Sequence[str] | None = None + + class LanguageBackend(Enum): LSP = "LSP" """ @@ -228,7 +240,7 @@ def from_str(cls, value: str) -> "LineEnding": @dataclass -class SharedConfig(ModeSelectionDefinition, ToolInclusionDefinition, ToStringMixin): +class SharedConfig(ToolInclusionDefinition, ToStringMixin): """Shared between SerenaConfig and ProjectConfig, the latter used to override values in the form (same as in ModeSelectionDefinition). The defaults here shall be none and should be set to the global default values in SerenaConfig. @@ -255,7 +267,7 @@ class SerenaConfigError(Exception): @dataclass(kw_only=True) -class ProjectConfig(SharedConfig): +class ProjectConfig(SharedConfig, ModeSelectionDefinitionWithAddedModes): project_name: str languages: list[Language] ignored_paths: list[str] = field(default_factory=list) @@ -471,6 +483,9 @@ def _from_dict(cls, data: dict[str, Any], local_override_keys: list[str]) -> Sel excluded_tools = data["excluded_tools"] or [] included_optional_tools = data["included_optional_tools"] or [] + if "base_modes" in data and data["base_modes"] is not None: + log.warning("The base_modes setting in project.yml is deprecated and will be ignored.") + return cls( project_name=data["project_name"], languages=languages, @@ -486,7 +501,7 @@ def _from_dict(cls, data: dict[str, Any], local_override_keys: list[str]) -> Sel encoding=data["encoding"], line_ending=line_ending, language_backend=language_backend, - base_modes=data["base_modes"], + added_modes=data["added_modes"], default_modes=data["default_modes"], symbol_info_budget=symbol_info_budget, ls_specific_settings=data.get("ls_specific_settings", {}), @@ -677,7 +692,7 @@ def get_project_instance(self, serena_config: "SerenaConfig") -> "Project": @dataclass(kw_only=True) -class SerenaConfig(SharedConfig): +class SerenaConfig(SharedConfig, ModeSelectionDefinitionWithBaseModes): """ Holds the Serena agent configuration, which is typically loaded from a YAML configuration file (when instantiated via :method:`from_config_file`), which is updated when projects are added or removed. @@ -731,7 +746,6 @@ class SerenaConfig(SharedConfig): """ the language backend to use for code understanding features """ - default_modes: Sequence[str] | None = ("interactive", "editing") line_ending: LineEnding = LineEnding.NATIVE symbol_info_budget: float = 10.0 """ diff --git a/src/serena/mcp.py b/src/serena/mcp.py index 13b9a00c3..130f432a6 100644 --- a/src/serena/mcp.py +++ b/src/serena/mcp.py @@ -3,7 +3,7 @@ """ import sys -from collections.abc import AsyncIterator, Iterator, Sequence +from collections.abc import AsyncIterator, Iterator from contextlib import asynccontextmanager from copy import deepcopy from dataclasses import dataclass @@ -272,7 +272,7 @@ def create_mcp_server( self, host: str = "127.0.0.1", port: int = 8000, - modes: Sequence[str] = (), + mode_selection_def: ModeSelectionDefinition | None = None, language_backend: LanguageBackend | None = None, enable_web_dashboard: bool | None = None, enable_gui_log_window: bool | None = None, @@ -286,7 +286,7 @@ def create_mcp_server( :param host: The host to bind to :param port: The port to bind to - :param modes: List of mode names or paths to mode files + :param mode_selection_def: the mode selection definition to apply :param language_backend: the language backend to use, overriding the configuration setting. :param enable_web_dashboard: Whether to enable the web dashboard. If not specified, will take the value from the serena configuration. :param enable_gui_log_window: Whether to enable the GUI log window. It currently does not work on macOS, and setting this to True will be ignored then. @@ -318,9 +318,6 @@ def create_mcp_server( if language_backend is not None: config.language_backend = language_backend - mode_selection_def: ModeSelectionDefinition | None = None - if modes: - mode_selection_def = ModeSelectionDefinition(default_modes=modes) self.agent = self._create_serena_agent(config, mode_selection_def) except Exception as e: diff --git a/src/serena/resources/project.template.yml b/src/serena/resources/project.template.yml index ada6fd671..cfa204ece 100644 --- a/src/serena/resources/project.template.yml +++ b/src/serena/resources/project.template.yml @@ -3,16 +3,18 @@ project_name: "project_name" # list of languages for which language servers are started; choose from: -# al bash clojure cpp csharp -# csharp_omnisharp dart elixir elm erlang -# fortran fsharp go groovy haskell -# haxe java julia kotlin lua -# markdown -# matlab nix pascal perl php -# php_phpactor powershell python python_jedi r -# rego ruby ruby_solargraph rust scala -# swift terraform toml typescript typescript_vts -# vue yaml zig +# al ansible bash clojure cpp +# cpp_ccls crystal csharp csharp_omnisharp dart +# elixir elm erlang fortran fsharp +# go groovy haskell haxe hlsl +# java json julia kotlin lean4 +# lua luau markdown matlab msl +# nix ocaml pascal perl php +# php_phpactor powershell python python_jedi python_ty +# r rego ruby ruby_solargraph rust +# scala solidity swift systemverilog terraform +# toml typescript typescript_vts vue yaml +# zig # (This list may be outdated. For the current list, see values of Language enum here: # https://github.com/oraios/serena/blob/main/src/solidlsp/ls_config.py # For some languages, there are alternative language servers, e.g. csharp_omnisharp, ruby_solargraph.) @@ -65,71 +67,34 @@ read_only: false # list of tool names to exclude. # This extends the existing exclusions (e.g. from the global configuration) -# -# Below is the complete list of tools for convenience. -# To make sure you have the latest list of tools, and to view their descriptions, -# execute `uv run scripts/print_tool_overview.py`. -# -# * `activate_project`: Activates a project based on the project name or path. -# * `check_onboarding_performed`: Checks whether project onboarding was already performed. -# * `create_text_file`: Creates/overwrites a file in the project directory. -# * `delete_memory`: Delete a memory file. Should only happen if a user asks for it explicitly, -# for example by saying that the information retrieved from a memory file is no longer correct -# or no longer relevant for the project. -# * `edit_memory`: Replaces content matching a regular expression in a memory. -# * `execute_shell_command`: Executes a shell command. -# * `find_file`: Finds files in the given relative paths -# * `find_referencing_symbols`: Finds symbols that reference the given symbol using the language server backend -# * `find_symbol`: Performs a global (or local) search using the language server backend. -# * `get_current_config`: Prints the current configuration of the agent, including the active and available projects, tools, contexts, and modes. -# * `get_symbols_overview`: Gets an overview of the top-level symbols defined in a given file. -# * `initial_instructions`: Provides instructions Serena usage (i.e. the 'Serena Instructions Manual') -# for clients that do not read the initial instructions when the MCP server is connected. -# * `insert_after_symbol`: Inserts content after the end of the definition of a given symbol. -# * `insert_before_symbol`: Inserts content before the beginning of the definition of a given symbol. -# * `list_dir`: Lists files and directories in the given directory (optionally with recursion). -# * `list_memories`: List available memories. Any memory can be read using the `read_memory` tool. -# * `onboarding`: Performs onboarding (identifying the project structure and essential tasks, e.g. for testing or building). -# * `read_file`: Reads a file within the project directory. -# * `read_memory`: Read the content of a memory file. This tool should only be used if the information -# is relevant to the current task. You can infer whether the information -# is relevant from the memory file name. -# You should not read the same memory file multiple times in the same conversation. -# * `rename_memory`: Renames or moves a memory. Moving between project and global scope is supported -# (e.g., renaming "global/foo" to "bar" moves it from global to project scope). -# * `rename_symbol`: Renames a symbol throughout the codebase using language server refactoring capabilities. -# For JB, we use a separate tool. -# * `replace_content`: Replaces content in a file (optionally using regular expressions). -# * `replace_symbol_body`: Replaces the full definition of a symbol using the language server backend. -# * `safe_delete_symbol`: -# * `search_for_pattern`: Performs a search for a pattern in the project. -# * `write_memory`: Write some information (utf-8-encoded) about this project that can be useful for future tasks to a memory in md format. -# The memory name should be meaningful. +# Find the list of tools here: https://oraios.github.io/serena/01-about/035_tools.html excluded_tools: [] # list of tools to include that would otherwise be disabled (particularly optional tools that are disabled by default). # This extends the existing inclusions (e.g. from the global configuration). +# Find the list of tools here: https://oraios.github.io/serena/01-about/035_tools.html included_optional_tools: [] # fixed set of tools to use as the base tool set (if non-empty), replacing Serena's default set of tools. # This cannot be combined with non-empty excluded_tools or included_optional_tools. +# Find the list of tools here: https://oraios.github.io/serena/01-about/035_tools.html fixed_tools: [] -# list of mode names to that are always to be included in the set of active modes -# The full set of modes to be activated is base_modes + default_modes. -# If the setting is undefined, the base_modes from the global configuration (serena_config.yml) apply. -# Otherwise, this setting overrides the global configuration. -# Set this to [] to disable base modes for this project. -# Set this to a list of mode names to always include the respective modes for this project. -base_modes: - -# list of mode names that are to be activated by default. -# The full set of modes to be activated is base_modes + default_modes. -# If the setting is undefined, the default_modes from the global configuration (serena_config.yml) apply. +# list of mode names that are to be activated by default, overriding the setting in the global configuration. +# The full set of modes to be activated is base_modes (from global config) + default_modes + added_modes. +# If the setting is undefined/empty, the default_modes from the global configuration (serena_config.yml) apply. # Otherwise, this overrides the setting from the global configuration (serena_config.yml). +# Therefore, you can set this to [] if you do not want the default modes defined in the global config to apply +# for this project. # This setting can, in turn, be overridden by CLI parameters (--mode). +# See https://oraios.github.io/serena/02-usage/050_configuration.html#modes default_modes: +# list of mode names to be activated additionally for this project, e.g. ["query-projects"] +# The full set of modes to be activated is base_modes (from global config) + default_modes + added_modes. +# See https://oraios.github.io/serena/02-usage/050_configuration.html#modes +added_modes: + # initial prompt for the project. It will always be given to the LLM upon activating the project # (contrary to the memories, which are loaded on demand). initial_prompt: "" diff --git a/src/serena/resources/serena_config.template.yml b/src/serena/resources/serena_config.template.yml index 18974358e..93036b961 100644 --- a/src/serena/resources/serena_config.template.yml +++ b/src/serena/resources/serena_config.template.yml @@ -37,13 +37,15 @@ gui_log_window: False # Further information: https://oraios.github.io/serena/02-usage/060_dashboard.html web_dashboard: True -# whether to open the Dashboard window/browser tab when Serena starts (provided that web_dashboard is enabled). -# If set to false, you can still open the dashboard manually by clicking on the Serena icon in your system -# tray on Windows and macOS. On Linux, there is no system tray support, so you can only open the dashboard by -# a) telling the LLM to "open the dashboard" (provided that the open_dashboard tool is enabled) or by -# b) manually navigating to http://localhost:24282/dashboard/ in your web browser (actual port -# may be higher if you have multiple instances running; try ports 24283, 24284, etc.) -# See also: https://oraios.github.io/serena/02-usage/060_dashboard.html +# whether to open the Dashboard window/browser tab when Serena starts (provided that `web_dashboard` is enabled). +# If set to false, you can still open the dashboard manually: +# * When using an interface that supports a tray icon (see setting `web_dashboard_interface`), +# you can conveniently open the dashboard from the system tray. +# * When using the `browser` interface (no tray icon), so you can only open the dashboard by +# a) telling the LLM to "open the dashboard" (provided that the open_dashboard tool is enabled) or by +# b) manually navigating to http://localhost:24282/dashboard/ in your web browser (actual port +# may be higher if you have multiple instances running; try ports 24283, 24284, etc.) +# Further information: https://oraios.github.io/serena/02-usage/060_dashboard.html web_dashboard_open_on_launch: True # defines the interface (application mode) used for the web dashboard (if enabled). @@ -59,6 +61,7 @@ web_dashboard_open_on_launch: True # opening the dashboard in browser tabs when selected from the tray menu. # This is EXPERIMENTAL. It is tested on Windows only. We will establish macOS support, but it is yet untested. # On Linux, this cannot be universally supported, but it may work in some desktop environments. +# See https://oraios.github.io/serena/02-usage/060_dashboard.html web_dashboard_interface: # the address the web dashboard will listen on (bind address). @@ -112,18 +115,22 @@ included_optional_tools: [] # This cannot be combined with non-empty excluded_tools or included_optional_tools. fixed_tools: [] -# list of mode names to that are always to be included in the set of active modes -# The full set of modes to be activated is base_modes + default_modes. -# If this is undefined, no base modes are included. -# The project configuration (project.yml) may override this setting. +# list of mode names to that are always to be included in the set of active modes. +# The full set of modes to be activated is base_modes + default_modes + added_modes, +# where added_modes can be defined by projects/CLI parameters. +# If this is undefined/empty, no base modes are included. +# See https://oraios.github.io/serena/02-usage/050_configuration.html#modes base_modes: + - interactive + - editing -# list of mode names that are to be activated by default. -# The full set of modes to be activated is base_modes + default_modes. +# list of mode names that are to be activated by default (but can be overridden by projects/CLI params). +# The full set of modes to be activated is base_modes + default_modes + added_modes, +# where added_modes are defined by projects/CLI parameters. +# If this is undefined/empty, no default modes are defined. # These modes can be overridden by the project configuration (project.yml) or through the CLI (--mode). +# See https://oraios.github.io/serena/02-usage/050_configuration.html#modes default_modes: - - interactive - - editing # Used as default for tools where the apply method has a default maximal answer length. # Even though the value of the max_answer_chars can be changed when calling the tool, it may make sense to adjust this default diff --git a/src/solidlsp/ls_config.py b/src/solidlsp/ls_config.py index eb527dc34..a7353de9a 100644 --- a/src/solidlsp/ls_config.py +++ b/src/solidlsp/ls_config.py @@ -13,13 +13,21 @@ class FilenameMatcher: - def __init__(self, *patterns: str) -> None: + patterns: list[str] + + def __init__(self, *patterns: str, insensitive: bool = False) -> None: """ :param patterns: fnmatch-compatible patterns """ - self.patterns = patterns + self.insensitive = insensitive + # Pre-lowercase patterns if we are in insensitive mode + if self.insensitive: + self.patterns = [p.lower() for p in patterns] + else: + self.patterns = list[str](patterns) def is_relevant_filename(self, fn: str) -> bool: + fn = fn.lower() if self.insensitive else fn for pattern in self.patterns: if fnmatch.fnmatch(fn, pattern): return True @@ -231,8 +239,68 @@ def get_source_fn_matcher(self) -> FilenameMatcher: return FilenameMatcher("*.rb", "*.erb") case self.RUBY_SOLARGRAPH: return FilenameMatcher("*.rb") - case self.CPP | self.CPP_CCLS: - return FilenameMatcher("*.cpp", "*.h", "*.hpp", "*.c", "*.hxx", "*.cc", "*.cxx") + case self.CPP: + # From llvm-project/clang/lib/Driver/Types.cpp types::lookupTypeForExtension: + return FilenameMatcher( + # C + "*.c", + "*.h", + # C++ + "*.c++", + "*.cc", + "*.cp", + "*.cpp", + "*.cxx", + "*.hh", + "*.hpp", + "*.hxx", + # C++ include files + "*.inl", + "*.ipp", + "*.tpp", + "*.txx", + # Objective-C + "*.m", + "*.mm", + # C++20 module interface files + "*.c++m", + "*.cppm", + "*.cxxm", + "*.ixx", + # CUDA + "*.cu", + # HIP + "*.hip", + # OpenCL + "*.cl", + "*.clcpp", + insensitive=True, + ) + case self.CPP_CCLS: + # From llvm-project/clang/lib/Driver/Types.cpp types::lookupTypeForExtension: + return FilenameMatcher( + # C + "*.c", + "*.h", + # C++ + "*.c++", + "*.cc", + "*.cp", + "*.cpp", + "*.cxx", + "*.hh", + "*.hpp", + "*.hxx", + # C++ include files + "*.inl", + "*.ipp", + "*.tpp", + "*.txx", + # Objective-C + "*.m", + "*.mm", + insensitive=True, + ) case self.KOTLIN: return FilenameMatcher("*.kt", "*.kts") case self.DART: @@ -288,9 +356,7 @@ def get_source_fn_matcher(self) -> FilenameMatcher: case self.JULIA: return FilenameMatcher("*.jl") case self.FORTRAN: - return FilenameMatcher( - "*.f90", "*.F90", "*.f95", "*.F95", "*.f03", "*.F03", "*.f08", "*.F08", "*.f", "*.F", "*.for", "*.FOR", "*.fpp", "*.FPP" - ) + return FilenameMatcher("*.f90", "*.f95", "*.f03", "*.f08", "*.f", "*.for", "*.fpp", insensitive=True) case self.HASKELL: return FilenameMatcher("*.hs", "*.lhs") case self.HAXE: diff --git a/test/conftest.py b/test/conftest.py index 4d3f1154d..d09389242 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -298,6 +298,9 @@ def _determine_disabled_languages() -> list[Language]: # Disable CPP_CCLS tests if ccls is not available ccls_tests_enabled = _sh.which("ccls") is not None + # Skip ccls tests on Windows since no recent binary is available and version + # 0.20220729 from chocolatey crashes when parsing the test files. + ccls_tests_enabled = ccls_tests_enabled and not is_windows if not ccls_tests_enabled: result.append(Language.CPP_CCLS) diff --git a/test/resources/repos/cpp/test_repo/C/a.c b/test/resources/repos/cpp/test_repo/C/a.c new file mode 100644 index 000000000..fe5fbea73 --- /dev/null +++ b/test/resources/repos/cpp/test_repo/C/a.c @@ -0,0 +1,7 @@ +#include "b.h" + +int main() +{ + int x = add(3, 4); + return x; +} diff --git a/test/resources/repos/cpp/test_repo/C/b.h b/test/resources/repos/cpp/test_repo/C/b.h new file mode 100644 index 000000000..e404974dc --- /dev/null +++ b/test/resources/repos/cpp/test_repo/C/b.h @@ -0,0 +1,3 @@ +#pragma once + +int add(int a, int b); diff --git a/test/resources/repos/cpp/test_repo/CUDA/a.cu b/test/resources/repos/cpp/test_repo/CUDA/a.cu new file mode 100644 index 000000000..e32d79f48 --- /dev/null +++ b/test/resources/repos/cpp/test_repo/CUDA/a.cu @@ -0,0 +1,16 @@ +#include +#include + +__global__ auto +cuda_hello() -> void +{ + printf("Hello World from GPU!\n"); +} + +auto +main() -> int +{ + cuda_hello<<<1, 1>>>(); + cudaDeviceSynchronize(); + return EXIT_SUCCESS; +} diff --git a/test/resources/repos/cpp/test_repo/CXX20/a.cpp b/test/resources/repos/cpp/test_repo/CXX20/a.cpp new file mode 100644 index 000000000..79d860394 --- /dev/null +++ b/test/resources/repos/cpp/test_repo/CXX20/a.cpp @@ -0,0 +1,9 @@ +#include + +import b; + +auto +main() noexcept -> int +{ + return EXIT_SUCCESS; +} diff --git a/test/resources/repos/cpp/test_repo/CXX20/b.cpp b/test/resources/repos/cpp/test_repo/CXX20/b.cpp new file mode 100644 index 000000000..1b95a6798 --- /dev/null +++ b/test/resources/repos/cpp/test_repo/CXX20/b.cpp @@ -0,0 +1,17 @@ +module; + +#include + +module b; + +auto +add(auto x, auto y) -> decltype(x + y) +{ + return x + y; +} + +auto +add_f32(float x, float y) -> float +{ + return add(x, y); +} diff --git a/test/resources/repos/cpp/test_repo/CXX20/b.cppm b/test/resources/repos/cpp/test_repo/CXX20/b.cppm new file mode 100644 index 000000000..d1e7dcbe0 --- /dev/null +++ b/test/resources/repos/cpp/test_repo/CXX20/b.cppm @@ -0,0 +1,16 @@ +module; + +#include + +export module b; + +export auto +add(auto x, auto y) -> decltype(x + y); + +export auto +add_f32(float x, float y) -> float; + +auto +add_u32(std::int32_t x, std::int32_t y) -> std::int32_t { + return x + y; +} diff --git a/test/resources/repos/cpp/test_repo/HIP/a.hip b/test/resources/repos/cpp/test_repo/HIP/a.hip new file mode 100644 index 000000000..910b9876a --- /dev/null +++ b/test/resources/repos/cpp/test_repo/HIP/a.hip @@ -0,0 +1,10 @@ +// -*- mode: C -*- + +#include + +__global__ +void add_vectors(const float* A, const float* B, float* C) +{ + auto const idx = blockIdx.x * blockDim.x + threadIdx.x; + C[idx] = A[idx] + B[idx]; +} diff --git a/test/resources/repos/cpp/test_repo/Objective-C/a.m b/test/resources/repos/cpp/test_repo/Objective-C/a.m new file mode 100644 index 000000000..6487dfb88 --- /dev/null +++ b/test/resources/repos/cpp/test_repo/Objective-C/a.m @@ -0,0 +1,7 @@ +@interface Hello +- (void)hello; +@end + +int main(int argc, const char * argv[]) { + return 0; +} diff --git a/test/resources/repos/cpp/test_repo/OpenCL/a.cl b/test/resources/repos/cpp/test_repo/OpenCL/a.cl new file mode 100644 index 000000000..78fe1a68d --- /dev/null +++ b/test/resources/repos/cpp/test_repo/OpenCL/a.cl @@ -0,0 +1,8 @@ +// -*- mode: C -*- + +__kernel +void add_vectors(__global const float *A, __global const float *B, __global float *C) +{ + int idx = get_global_id(0); + C[idx] = A[idx] + B[idx]; +} diff --git a/test/resources/repos/cpp/test_repo/compile_commands.json b/test/resources/repos/cpp/test_repo/compile_commands.json index 21c02f626..7181b0587 100644 --- a/test/resources/repos/cpp/test_repo/compile_commands.json +++ b/test/resources/repos/cpp/test_repo/compile_commands.json @@ -8,5 +8,40 @@ "directory": ".", "command": "g++ -std=c++17 -I . -c b.cpp", "file": "b.cpp" + }, + { + "directory": "C", + "command": "clang -std=gnu23 -c a.c", + "file": "a.c" + }, + { + "directory": "CUDA", + "command": "nvcc -std=c++20 -c a.cu", + "file": "a.cu" + }, + { + "directory": "CXX20", + "command": "clang++ -std=gnu++20 -fmodule-file=b=b.pcm -c a.cpp", + "file": "a.cpp" + }, + { + "directory": "CXX20", + "command": "clang++ -std=gnu++20 -fmodule-file=b=b.pcm -c b.cpp", + "file": "b.cpp" + }, + { + "directory": "CXX20", + "command": "clang++ -std=gnu++20 -c b.cppm", + "file": "b.cppm" + }, + { + "directory": "OpenCL", + "command": "clang -std=cl3.0 -c a.cl", + "file": "a.cl" + }, + { + "directory": "HIP", + "command": "hipcc -c a.hip", + "file": "a.hip" } -] \ No newline at end of file +] diff --git a/test/serena/test_serena_agent.py b/test/serena/test_serena_agent.py index 896b030f1..25b58bf51 100644 --- a/test/serena/test_serena_agent.py +++ b/test/serena/test_serena_agent.py @@ -47,7 +47,7 @@ def serena_config(): Language.CLOJURE, Language.FSHARP, Language.POWERSHELL, - Language.CPP_CCLS, + Language.CPP, Language.HAXE, Language.LEAN4, Language.MSL, @@ -200,7 +200,7 @@ def test_find_symbol_within_php_file(self, serena_agent: SerenaAgent) -> None: pytest.param(Language.CLOJURE, "greet", "Function", clj.CORE_PATH, marks=pytest.mark.clojure), pytest.param(Language.CSHARP, "Calculator", "Class", "Program.cs", marks=pytest.mark.csharp), pytest.param(Language.POWERSHELL, "Greet-User", "Function", "main.ps1", marks=pytest.mark.powershell), - pytest.param(Language.CPP_CCLS, "add", "Function", "b.cpp", marks=pytest.mark.cpp), + pytest.param(Language.CPP, "add", "Function", "b.cpp", marks=pytest.mark.cpp), pytest.param(Language.HAXE, "Main", "Class", "Main.hx", marks=pytest.mark.haxe), pytest.param(Language.LEAN4, "add", "Method", "Helper.lean", marks=pytest.mark.lean4), pytest.param(Language.MSL, "greet", "Function", "main.mrc", marks=pytest.mark.msl), @@ -303,7 +303,7 @@ def contains_ref_with_relative_path(refs, relative_path): ), pytest.param(Language.CSHARP, "Calculator", "Program.cs", "Program.cs", marks=pytest.mark.csharp), pytest.param(Language.POWERSHELL, "Greet-User", "main.ps1", "main.ps1", marks=pytest.mark.powershell), - pytest.param(Language.CPP_CCLS, "add", "b.cpp", "a.cpp", marks=pytest.mark.cpp), + pytest.param(Language.CPP, "add", "b.cpp", "a.cpp", marks=pytest.mark.cpp), pytest.param( Language.HAXE, "addNumbers", diff --git a/test/solidlsp/cpp/test_ccls_languages.py b/test/solidlsp/cpp/test_ccls_languages.py new file mode 100644 index 000000000..70cfac6a9 --- /dev/null +++ b/test/solidlsp/cpp/test_ccls_languages.py @@ -0,0 +1,24 @@ +import os + +import pytest + +from solidlsp.ls_config import Language +from test.conftest import start_ls_context + + +@pytest.mark.cpp +class TestCCLSLanguages: + @pytest.mark.parametrize( + "lang, unit, names", + [ + ("C", "a.c", {"main"}), + ("Objective-C", "a.m", {"main", "Hello", "-hello"}), + ], + ) + def test_get_document_symbols(self, lang: str, unit: str, names: set[str]) -> None: + with start_ls_context(Language.CPP) as ccls: + path = os.path.join(lang, unit) + symbols = ccls.request_document_symbols(path).get_all_symbols_and_roots() + symbols = symbols[0] if symbols and isinstance(symbols[0], list) else symbols + symbols = {s.get("name") for s in symbols} + assert names == symbols, f"Expected '{names}' in document symbols, got: {symbols}" diff --git a/test/solidlsp/cpp/test_clangd_languages.py b/test/solidlsp/cpp/test_clangd_languages.py new file mode 100644 index 000000000..8bf3a30f2 --- /dev/null +++ b/test/solidlsp/cpp/test_clangd_languages.py @@ -0,0 +1,30 @@ +import os + +import pytest + +from solidlsp.ls_config import Language +from test.conftest import start_ls_context + + +@pytest.mark.cpp +class TestClangdLanguages: + @pytest.mark.parametrize( + "lang, unit, names", + [ + ("C", "a.c", {"main"}), + ("CUDA", "a.cu", {"main", "cuda_hello"}), + ("CXX20", "a.cpp", {"main"}), + ("CXX20", "b.cpp", {"add", "add_f32"}), + ("CXX20", "b.cppm", {"add", "add_f32", "add_u32"}), + ("HIP", "a.hip", {"add_vectors"}), + ("OpenCL", "a.cl", {"add_vectors"}), + ("Objective-C", "a.m", {"main", "Hello", "-hello"}), + ], + ) + def test_get_document_symbols(self, lang: str, unit: str, names: set[str]) -> None: + with start_ls_context(Language.CPP) as clangd: + path = os.path.join(lang, unit) + symbols = clangd.request_document_symbols(path).get_all_symbols_and_roots() + symbols = symbols[0] if symbols and isinstance(symbols[0], list) else symbols + symbols = {s.get("name") for s in symbols} + assert names == symbols, f"Expected '{names}' in document symbols, got: {symbols}" diff --git a/test/solidlsp/cpp/test_cpp_basic.py b/test/solidlsp/cpp/test_cpp_basic.py index dfb188811..21b904528 100644 --- a/test/solidlsp/cpp/test_cpp_basic.py +++ b/test/solidlsp/cpp/test_cpp_basic.py @@ -16,16 +16,11 @@ from solidlsp import SolidLanguageServer from solidlsp.ls_config import Language from solidlsp.ls_utils import SymbolUtils -from test.conftest import get_repo_path, start_ls_context +from test.conftest import get_repo_path, language_tests_enabled, start_ls_context from test.solidlsp.conftest import format_symbol_for_assert, has_malformed_name, request_all_symbols - -def _ccls_available() -> bool: - return shutil.which("ccls") is not None - - _cpp_servers: list[Language] = [Language.CPP] -if _ccls_available(): +if language_tests_enabled(Language.CPP_CCLS): _cpp_servers.append(Language.CPP_CCLS)