Provide Serena's system prompt (instruction manual) lazily

At MCP connection time, provide only a one-sentence bootstrap prompt

Author: opcode81

CHANGELOG.md

@@ -26,6 +26,9 @@ Status of the `main` branch. Changes prior to the next official version change w
      - `query_project`: Support use of project root instead of project name #1388
      - `list_queryable_projects`: Return both project names and project roots 
   - Fix: `search_for_pattern` tool returned 1-based line numbers (in contrast to all other tools); cause: implementation of `text_utils.search_text`
+  - Serena's system prompt (a.k.a. the 'Serena Instructions Manual') is now provided lazily. 
+    At MCP connection time, only a one-sentence bootstrap prompt is provided.
+    The `initial_instructions` tool provides the full prompt on demand, keeping the initial context lean.
 
 * JetBrains:
   - `Move` and `SafeDelete` tools: transform empty string to None (counteracts client errors)

src/serena/agent.py

@@ -891,8 +891,18 @@ def _format_prompt(self, prompt_template: str) -> str:
         template = JinjaTemplate(prompt_template)
         return template.render(available_tools=self._exposed_tools.tool_names, available_markers=self._exposed_tools.tool_marker_names)
 
+    def create_connection_prompt(self) -> str:
+        """
+        Returns the bootstrap prompt to be sent at MCP connection time.
+
+        :return: the prompt
+        """
+        return self.prompt_factory.create_connection_prompt()
+
     def create_system_prompt(self, session_id: str = "global") -> str:
         """
+        Returns the 'Serena Instructions Manual', i.e. Serena's system prompt.
+
         :param session_id: the client session ID for the case where this is run from a tool; "global" for the connection time case
         :return: the prompt
         """
@@ -925,7 +935,6 @@ def create_system_prompt(self, session_id: str = "global") -> str:
         if self._active_project is not None and not self._project_prompt_status.is_project_activation_message_already_provided(session_id):
             system_prompt += "\n\n" + self.get_project_activation_message(session_id)
 
-        log.info("System prompt:\n%s", system_prompt)
         return system_prompt
 
     def get_project_activation_message(self, session_id: str) -> str:

src/serena/generated/generated_prompt_factory.py

@@ -17,20 +17,8 @@ class PromptFactory(PromptFactoryBase):
     def create_onboarding_prompt(self, *, system: Any) -> str:
         return self._render_prompt("onboarding_prompt", locals())
 
-    def create_think_about_collected_information(self) -> str:
-        return self._render_prompt("think_about_collected_information", locals())
-
-    def create_think_about_task_adherence(self) -> str:
-        return self._render_prompt("think_about_task_adherence", locals())
-
-    def create_think_about_whether_you_are_done(self) -> str:
-        return self._render_prompt("think_about_whether_you_are_done", locals())
-
-    def create_summarize_changes(self) -> str:
-        return self._render_prompt("summarize_changes", locals())
-
-    def create_prepare_for_new_conversation(self) -> str:
-        return self._render_prompt("prepare_for_new_conversation", locals())
+    def create_connection_prompt(self) -> str:
+        return self._render_prompt("connection_prompt", locals())
 
     def create_system_prompt(
         self,

src/serena/mcp.py

@@ -332,6 +332,7 @@ def create_mcp_server(
         # retain only FASTMCP_ prefix for already set environment variables.
         Settings.model_config = SettingsConfigDict(env_prefix="FASTMCP_")
         instructions = self._get_initial_instructions()
+        log.info("MCP server initial instructions:\n%s", instructions)
         mcp = FastMCP(
             name="Serena",
             lifespan=self.server_lifespan,
@@ -357,4 +358,4 @@ async def server_lifespan(self, mcp_server: FastMCP) -> AsyncIterator[None]:
 
     def _get_initial_instructions(self) -> str:
         assert self.agent is not None
-        return self.agent.create_system_prompt()
+        return self.agent.create_connection_prompt()

src/serena/resources/config/prompt_templates/system_prompt.yml

@@ -2,6 +2,8 @@
 # in which case this prompt will be given as a regular message on the call of a simple tool which the agent
 # is encouraged (via the tool description) to call at the beginning of the conversation.
 prompts:
+  connection_prompt: |
+    CRITICAL: Before starting to work on a coding task, call the `initial_instructions` tool to read the 'Serena Instructions Manual'.
   system_prompt: |
     You have access to semantic coding tools upon which you rely heavily for all your work.
     You operate in a resource-efficient and intelligent manner, always keeping in mind to not read or generate