feat: add reporting tools and misc fix bugs

connector_builder_agents/pyproject.toml

@@ -9,7 +9,7 @@ dependencies = [
     "openai-agents-mcp>=0.0.8",
     "openai-agents>=0.2.11",
     "mcp-agent>=0.1.15",  # Transitive dependency of openai-agents
-    "pydantic>=2.0.0",  # For phase data models
+    "pydantic>=2.11.9,<3.0",
 ]
 
 [dependency-groups]

connector_builder_agents/src/agents.py

@@ -6,6 +6,7 @@
     WebSearchTool,
     handoff,
 )
+from pydantic.main import BaseModel
 
 # from agents import OpenAIConversationsSession
 from .constants import (
@@ -16,10 +17,13 @@
 from .tools import (
     DEVELOPER_AGENT_TOOLS,
     MANAGER_AGENT_TOOLS,
+    get_latest_readiness_report,
+    get_progress_log_text,
     log_problem_encountered_by_developer,
     log_problem_encountered_by_manager,
     log_progress_milestone_from_developer,
     log_progress_milestone_from_manager,
+    log_tool_failure,
     mark_job_failed,
     mark_job_success,
     update_progress_log,
@@ -44,6 +48,7 @@ def create_developer_agent(
         tools=[
             log_progress_milestone_from_developer,
             log_problem_encountered_by_developer,
+            log_tool_failure,
             WebSearchTool(),
         ],
     )
@@ -103,5 +108,45 @@ async def on_phase3_handoff(ctx, input_data: Phase3Data) -> None:
             mark_job_failed,
             log_problem_encountered_by_manager,
             log_progress_milestone_from_manager,
+            log_tool_failure,
+            get_latest_readiness_report,
+            get_progress_log_text,
         ],
     )
+
+
+class ManagerHandoffInput(BaseModel):
+    """Input data for handoff from developer back to manager."""
+
+    short_status: str
+    detailed_progress_update: str
+    is_blocked: bool
+    is_partial_success: bool
+    is_full_success: bool
+
+
+async def on_manager_handback(ctx, input_data: ManagerHandoffInput) -> None:
+    update_progress_log(
+        f"🤝 Handing back control to manager."
+        f"\n Summary of status: {input_data.short_status}"
+        f"\n Partial success: {input_data.is_partial_success}"
+        f"\n Full success: {input_data.is_full_success}"
+        f"\n Blocked: {input_data.is_blocked}"
+        f"\n Detailed progress update: {input_data.detailed_progress_update}"
+    )
+
+
+def add_handback_to_manager(
+    developer_agent: OpenAIAgent,
+    manager_agent: OpenAIAgent,
+) -> None:
+    """Add a handoff from the developer back to the manager to report progress."""
+    developer_agent.handoffs.append(
+        handoff(
+            agent=manager_agent,
+            tool_name_override="report_back_to_manager",
+            tool_description_override="Report progress or issues back to the manager agent",
+            input_type=ManagerHandoffInput,
+            on_handoff=on_manager_handback,
+        )
+    )

connector_builder_agents/src/guidance.py

@@ -20,10 +20,13 @@
 
 Monitor progress and ensure each phase completes successfully before moving to the next.
 
-When checking on the progress of your developer, if the build is complete, summarize the results and
-evaluate whether they meet the requirements. If not, you can repeat a phase, calling out what they
-missed and suggesting next steps. Determine the next phase or next appropriate action based on their
-progress.
+When checking on the progress of your developer:
+- Use the `get_progress_log_text` tool to get the latest progress log.
+- Use the `get_latest_readiness_report` tool to get the latest readiness report.
+
+If the build is complete, summarize the results and evaluate whether they meet the requirements. If
+not, you can repeat a phase, calling out what they missed and suggesting next steps. Determine the
+next phase or next appropriate action based on their progress.
 
 ## Exit Criteria
 

connector_builder_agents/src/run.py

@@ -16,6 +16,7 @@
 # from agents import OpenAIConversationsSession
 from ._util import open_if_browser_available
 from .agents import (
+    add_handback_to_manager,
     create_developer_agent,
     create_manager_agent,
 )
@@ -165,6 +166,10 @@ async def run_manager_developer_build(
         api_name=api_name or "(see below)",
         additional_instructions=instructions or "",
     )
+    add_handback_to_manager(
+        developer_agent=developer_agent,
+        manager_agent=manager_agent,
+    )
 
     for server in [*MANAGER_AGENT_TOOLS, *DEVELOPER_AGENT_TOOLS]:
         print(f"🔗 Connecting to MCP server: {server.name}...")

connector_builder_agents/src/tools.py

@@ -3,6 +3,7 @@
 
 from datetime import datetime
 from enum import Enum
+from typing import Annotated
 
 from agents.mcp import (
     MCPServer,
@@ -11,6 +12,7 @@
 )
 from agents.mcp.util import create_static_tool_filter
 from agents.tool import function_tool
+from pydantic.fields import Field
 
 # from agents import OpenAIConversationsSession
 from .constants import HEADLESS_BROWSER, WORKSPACE_WRITE_DIR
@@ -194,6 +196,37 @@ def log_problem_encountered(
     update_progress_log(f"⚠️ {agent.value} Encountered a Problem: {description}")
 
 
+@function_tool()
+def log_tool_failure(
+    tool_name: Annotated[
+        str,
+        Field(description="Name of the tool that failed."),
+    ],
+    input_args: Annotated[str, Field(description="Input arguments for the tool.")],
+    # *,
+    summary_failure_description: Annotated[
+        str, Field(description="Summary description of the failure.")
+    ],
+    is_unexpected_input_args_error: bool = False,
+    is_unhelpful_error_message: bool = False,
+    additional_info: str | None = None,
+) -> None:
+    """Log a tool failure message.
+
+    This is a specialized version of `log_problem_encountered` to report tool failures.
+    """
+    msg = f"🛠️ Tool Failure in '{tool_name}': {summary_failure_description}"
+    if is_unexpected_input_args_error:
+        msg += " (🙈 Unexpected input arguments error)"
+    if is_unhelpful_error_message:
+        msg += " (🫤 Unhelpful error message)"
+    msg += f"\n Input args: '{input_args}'"
+    if additional_info:
+        msg += f"\n Additional info: '{additional_info}'"
+
+    update_progress_log(msg)
+
+
 @function_tool(name_override="log_problem_encountered")
 def log_problem_encountered_by_manager(description: str) -> None:
     """Log a problem encountered message from the manager agent."""
@@ -216,3 +249,19 @@ def log_progress_milestone_from_manager(message: str) -> None:
 def log_progress_milestone_from_developer(message: str) -> None:
     """Log a milestone message from the developer agent."""
     log_progress_milestone(message, AgentEnum.DEVELOPER_AGENT_NAME)
+
+
+@function_tool
+def get_progress_log_text() -> str:
+    """Get the current progress log text."""
+    return EXECUTION_LOG_FILE.absolute().read_text(encoding="utf-8")
+
+
+@function_tool
+def get_latest_readiness_report() -> str:
+    """Get the path to the latest connector readiness report, if it exists."""
+    report_path = WORKSPACE_WRITE_DIR / "connector-readiness-report.md"
+    if report_path.exists():
+        return report_path.absolute().read_text(encoding="utf-8")
+
+    return "No readiness report found."

connector_builder_mcp/validation_testing.py

@@ -511,12 +511,6 @@ def run_connector_readiness_test_report(  # noqa: PLR0912, PLR0914, PLR0915 (too
             description="Optional paths/URLs to local .env files or Privatebin.net URLs for secret hydration. Can be a single string, comma-separated string, or list of strings. Privatebin secrets may be created at privatebin.net, and must: contain text formatted as a dotenv file, use a password sent via the `PRIVATEBIN_PASSWORD` env var, and not include password text in the URL."
         ),
     ] = None,
-    save_to_project_dir: Annotated[
-        str | Path | None,
-        Field(
-            description="Optional path to the project directory where the report should be saved."
-        ),
-    ] = None,
 ) -> str:
     """Execute a connector readiness test and generate a comprehensive markdown report.
 
@@ -537,11 +531,14 @@ def run_connector_readiness_test_report(  # noqa: PLR0912, PLR0914, PLR0915 (too
     stream_results: dict[str, StreamSmokeTest] = {}
 
     manifest_dict, manifest_path = parse_manifest_input(manifest)
-    if not save_to_project_dir and manifest_path:
-        save_to_project_dir = Path(manifest_path).parent
+    session_artifacts_dir: Path | None = None
+    if manifest_path:
+        session_artifacts_dir = Path(manifest_path).parent
 
     report_output_path: Path | None = (
-        Path(save_to_project_dir) / "connector-readiness-report.md" if save_to_project_dir else None
+        Path(session_artifacts_dir) / "connector-readiness-report.md"
+        if session_artifacts_dir
+        else None
     )
 
     config = hydrate_config(
@@ -654,6 +651,7 @@ def run_connector_readiness_test_report(  # noqa: PLR0912, PLR0914, PLR0915 (too
         "# Connector Readiness Test Report",
         "",
         "## Summary",
+        "",
         f"- **Streams Tested**: {total_streams_tested} out of {total_available_streams} total streams",
         f"- **Successful Streams**: {total_streams_successful}/{total_streams_tested}",
         f"- **Total Records Extracted**: {total_records_count:,}",
@@ -682,7 +680,8 @@ def run_connector_readiness_test_report(  # noqa: PLR0912, PLR0914, PLR0915 (too
 
             report_lines.extend(
                 [
-                    f"### {stream_name} ✅",
+                    f"### `{stream_name}` ✅",
+                    "",
                     f"- **Records Extracted**: {smoke_result.records_read:,}",
                     f"- **Duration**: {smoke_result.duration_seconds:.2f}s",
                 ]
@@ -695,10 +694,15 @@ def run_connector_readiness_test_report(  # noqa: PLR0912, PLR0914, PLR0915 (too
 
             report_lines.append("")
         else:
-            error_msg = getattr(smoke_result, "error_message", "Unknown error")
+            error_msg = getattr(
+                smoke_result,
+                "error_message",
+                "Unknown error",
+            )
             report_lines.extend(
                 [
-                    f"### {stream_name} ❌",
+                    f"### `{stream_name}` ❌",
+                    "",
                     "- **Status**: Failed",
                     f"- **Error**: {error_msg}",
                     "",
@@ -707,7 +711,7 @@ def run_connector_readiness_test_report(  # noqa: PLR0912, PLR0914, PLR0915 (too
 
     return _as_saved_report(
         report_text="\n".join(report_lines),
-        file_path=save_to_project_dir,
+        file_path=report_output_path,
     )
 
 

poe_tasks.toml

@@ -2,7 +2,7 @@
 
 # Development Tasks
 build = { cmd = "bin/build_connector_feature_index.py", help = "Build connector feature index from Airbyte manifests" }
-install = { cmd = "uv sync --all-extras", help = "Install and sync project dependencies." }
+install = { shell = "uv sync --all-extras && uv sync --project=connector_builder_agents --all-extras", help = "Install and sync project dependencies." }
 sync = { cmd = "uv sync --all-extras", help = "Sync project dependencies. (Alias for install.)" }
 format = { cmd = "uv run ruff format .", help = "Format code with ruff" }
 format-check = { cmd = "uv run ruff format --check .", help = "Format check with ruff" }
@@ -13,7 +13,7 @@ which actionlint >/dev/null 2>&1 && actionlint || \
 (echo "ERROR: actionlint not found. Install with: 'brew install actionlint'" && exit 1)
 """
 check-gh-actions.help = "Lint GitHub Actions workflows"
-lock = { cmd = "uv lock", help = "Lock dependencies." }
+lock = { shell = "uv lock && uv lock --project=connector_builder_agents", help = "Lock dependencies." }
 mypy = { cmd = "uv run mypy connector_builder_mcp", help = "Type check code with mypy" }
 test = { cmd = "uv run pytest tests/ -v", help = "Run tests with verbose output" }
 test-fast = { cmd = "uv run pytest --exitfirst tests", help = "Run tests, stop on first failure" }

pyproject.toml

@@ -12,7 +12,7 @@ dependencies = [
     "fastmcp>=2.12.1,<3.0",
     "jsonschema>=4.0.0,<5.0",
     "privatebin>=0.3.0,<1.0",
-    "pydantic>=2.7.0,<3.0",
+    "pydantic>=2.11.9,<3.0",
     "python-dotenv>=1.1.1,<2.0",
     "pyyaml>=6.0.0,<7.0",
     "requests>=2.25.0,<3.0",