feat: add get_connector_builder_docs tool for LLM guidance

builder_mcp/_connector_builder.py

@@ -7,6 +7,7 @@
 import logging
 from typing import Annotated, Any, Literal
 
+import requests
 from airbyte_cdk.connector_builder.connector_builder_handler import (
     TestLimits,
     create_source,
@@ -218,6 +219,203 @@ def get_resolved_manifest(
         return "Failed to resolve manifest"
 
 
+def get_connector_builder_docs(
+    topic: Annotated[
+        str | None,
+        Field(
+            description="Specific YAML reference topic to get detailed documentation for. If not provided, returns high-level overview and topic list."
+        ),
+    ] = None,
+) -> str:
+    """Get connector builder documentation and guidance.
+
+    Args:
+        topic: Optional specific topic from YAML reference documentation
+
+    Returns:
+        High-level overview with topic list, or detailed topic-specific documentation
+    """
+    logger.info(f"Getting connector builder docs for topic: {topic}")
+
+    if topic is None:
+        return _get_high_level_overview()
+    else:
+        return _get_topic_specific_docs(topic)
+
+
+def _get_high_level_overview() -> str:
+    """Return high-level connector building process and available topics."""
+    overview = """# Connector Builder Documentation
+
+1. Use the manifest YAML JSON schema for high-level guidelines
+2. Use the validate manifest tool to confirm JSON schema is correct
+3. Start with one stream or (better) a stream template that maps to a single stream
+4. Make sure you have working authentication and data retrieval before moving onto pagination and other components
+5. When all is confirmed working on the first stream, you can add additional streams. It is generally safest to add one stream at a time, and test each one before moving to the next
+
+- `validate_manifest`: Validate connector manifest structure and configuration
+- `execute_stream_read`: Test reading from a connector stream
+- `get_resolved_manifest`: Get the resolved connector manifest
+
+We use the Declarative YAML Connector convention for building connectors. Note that we don't yet support custom Python components.
+
+For detailed documentation on specific components, request one of these topics:
+
+- overview - Connector Builder overview and introduction
+- tutorial - Step-by-step tutorial for building connectors
+- authentication - Authentication configuration
+- incremental-sync - Setting up incremental data synchronization
+- pagination - Handling paginated API responses
+- partitioning - Configuring partition routing for complex APIs
+- record-processing - Processing and transforming records
+- error-handling - Handling API errors and retries
+- ai-assist - Using AI assistance in the Connector Builder
+- stream-templates - Using stream templates for faster development
+- custom-components - Working with custom components
+- async-streams - Configuring asynchronous streams
+
+- yaml-overview - Understanding the YAML file structure
+- reference - Complete YAML reference documentation
+- yaml-incremental-syncs - Incremental sync configuration in YAML
+- yaml-pagination - Pagination configuration options
+- yaml-partition-router - Partition routing in YAML manifests
+- yaml-record-selector - Record selection and transformation
+- yaml-error-handling - Error handling configuration
+- yaml-authentication - Authentication methods in YAML
+- requester - HTTP requester configuration
+- request-options - Request parameter configuration
+- rate-limit-api-budget - Rate limiting and API budget management
+- file-syncing - File synchronization configuration
+- property-chunking - Property chunking for large datasets
+
+For detailed information on any topic, call this function again with the topic name.
+"""
+    return overview
+
+
+def _get_topic_specific_docs(topic: str) -> str:
+    """Get detailed documentation for a specific topic using raw GitHub URLs."""
+    logger.info(f"Fetching detailed docs for topic: {topic}")
+
+    topic_mapping = {
+        "overview": ("docs/platform/connector-development/connector-builder-ui/overview.md", "md"),
+        "tutorial": (
+            "docs/platform/connector-development/connector-builder-ui/tutorial.mdx",
+            "mdx",
+        ),
+        "authentication": (
+            "docs/platform/connector-development/connector-builder-ui/authentication.md",
+            "md",
+        ),
+        "incremental-sync": (
+            "docs/platform/connector-development/connector-builder-ui/incremental-sync.md",
+            "md",
+        ),
+        "pagination": (
+            "docs/platform/connector-development/connector-builder-ui/pagination.md",
+            "md",
+        ),
+        "partitioning": (
+            "docs/platform/connector-development/connector-builder-ui/partitioning.md",
+            "md",
+        ),
+        "record-processing": (
+            "docs/platform/connector-development/connector-builder-ui/record-processing.mdx",
+            "mdx",
+        ),
+        "error-handling": (
+            "docs/platform/connector-development/connector-builder-ui/error-handling.md",
+            "md",
+        ),
+        "ai-assist": (
+            "docs/platform/connector-development/connector-builder-ui/ai-assist.md",
+            "md",
+        ),
+        "stream-templates": (
+            "docs/platform/connector-development/connector-builder-ui/stream-templates.md",
+            "md",
+        ),
+        "custom-components": (
+            "docs/platform/connector-development/connector-builder-ui/custom-components.md",
+            "md",
+        ),
+        "async-streams": (
+            "docs/platform/connector-development/connector-builder-ui/async-streams.md",
+            "md",
+        ),
+        "yaml-overview": (
+            "docs/platform/connector-development/config-based/understanding-the-yaml-file/yaml-overview.md",
+            "md",
+        ),
+        "reference": (
+            "docs/platform/connector-development/config-based/understanding-the-yaml-file/reference.md",
+            "md",
+        ),
+        "yaml-incremental-syncs": (
+            "docs/platform/connector-development/config-based/understanding-the-yaml-file/incremental-syncs.md",
+            "md",
+        ),
+        "yaml-pagination": (
+            "docs/platform/connector-development/config-based/understanding-the-yaml-file/pagination.md",
+            "md",
+        ),
+        "yaml-partition-router": (
+            "docs/platform/connector-development/config-based/understanding-the-yaml-file/partition-router.md",
+            "md",
+        ),
+        "yaml-record-selector": (
+            "docs/platform/connector-development/config-based/understanding-the-yaml-file/record-selector.md",
+            "md",
+        ),
+        "yaml-error-handling": (
+            "docs/platform/connector-development/config-based/understanding-the-yaml-file/error-handling.md",
+            "md",
+        ),
+        "yaml-authentication": (
+            "docs/platform/connector-development/config-based/understanding-the-yaml-file/authentication.md",
+            "md",
+        ),
+        "requester": (
+            "docs/platform/connector-development/config-based/understanding-the-yaml-file/requester.md",
+            "md",
+        ),
+        "request-options": (
+            "docs/platform/connector-development/config-based/understanding-the-yaml-file/request-options.md",
+            "md",
+        ),
+        "rate-limit-api-budget": (
+            "docs/platform/connector-development/config-based/understanding-the-yaml-file/rate-limit-api-budget.md",
+            "md",
+        ),
+        "file-syncing": (
+            "docs/platform/connector-development/config-based/understanding-the-yaml-file/file-syncing.md",
+            "md",
+        ),
+        "property-chunking": (
+            "docs/platform/connector-development/config-based/understanding-the-yaml-file/property-chunking.md",
+            "md",
+        ),
+    }
+
+    if topic not in topic_mapping:
+        return f"# {topic} Documentation\n\nTopic '{topic}' not found. Please check the available topics list from the overview.\n\nAvailable topics: {', '.join(topic_mapping.keys())}"
+
+    relative_path, file_ext = topic_mapping[topic]
+    raw_github_url = f"https://raw.githubusercontent.com/airbytehq/airbyte/master/{relative_path}"
+
+    try:
+        response = requests.get(raw_github_url, timeout=30)
+        response.raise_for_status()
+
+        markdown_content = response.text
+        return f"# {topic} Documentation\n\n{markdown_content}"
+
+    except Exception as e:
+        logger.error(f"Error fetching documentation for topic {topic}: {e}")
+        docs_url = f"https://docs.airbyte.com/platform/connector-development/config-based/understanding-the-yaml-file/{topic}"
+        return f"# {topic} Documentation\n\nUnable to fetch detailed documentation from GitHub. Please refer to the full reference: {docs_url}\n\nError: {str(e)}"
+
+
 def register_connector_builder_tools(app: FastMCP) -> None:
     """Register connector builder tools with the FastMCP app.
 
@@ -227,3 +425,4 @@ def register_connector_builder_tools(app: FastMCP) -> None:
     app.tool(validate_manifest)
     app.tool(execute_stream_read)
     app.tool(get_resolved_manifest)
+    app.tool(get_connector_builder_docs)

pyproject.toml

@@ -10,17 +10,7 @@ dependencies = [
     "typing-extensions>=4.14.1",
     "fastmcp>=0.2.0",
     "pydantic>=2.7.0,<3.0",
-]
-
-[project.optional-dependencies]
-dev = [
-    "pytest>=8.0.0",
-    "pytest-asyncio>=0.24.0",
-    "ruff>=0.8.0",
-    "mypy>=1.13.0",
-    "pyyaml>=6.0.0",
-    "coverage>=7.0.0",
-    "poethepoet>=0.29.0",
+    "requests>=2.25.0",
 ]
 
 [dependency-groups]
@@ -33,6 +23,11 @@ dev = [
     "pytest-cov>=6.2.1",
     "ruff>=0.12.4",
     "ty>=0.0.1a15",
+    "mypy>=1.13.0",
+    "pyyaml>=6.0.0",
+    "coverage>=7.0.0",
+    "poethepoet>=0.29.0",
+    "types-requests>=2.25.0",
 ]
 
 [project.scripts]