add tests (wip)

bin/test_mcp_tool.py

@@ -0,0 +1,98 @@
+#!/usr/bin/env python3
+"""
+One-liner CLI tool for testing MCP tools directly with JSON arguments.
+
+Usage:
+    poe test-tool execute_stream_test_read '{"manifest": "...", "config": {}, "stream_name": "users", "max_records": 3}'
+    poe test-tool run_connector_readiness_test_report '{"manifest": "...", "config": {}, "max_records": 10}'
+"""
+
+import json
+import sys
+from pathlib import Path
+from typing import Any, Dict
+
+from connector_builder_mcp._validation_testing import (
+    execute_stream_test_read,
+    run_connector_readiness_test_report,
+)
+
+
+def load_sample_manifest(name: str) -> str:
+    """Load a sample manifest from tests/resources."""
+    manifest_path = Path(__file__).parent.parent / "tests" / "resources" / f"{name}.yaml"
+    if manifest_path.exists():
+        return manifest_path.read_text()
+    raise FileNotFoundError(f"Sample manifest not found: {manifest_path}")
+
+
+def main() -> None:
+    """Main entry point for the MCP tool tester."""
+    if len(sys.argv) < 3:
+        print("Usage: python test_mcp_tool.py <tool_name> '<json_args>'", file=sys.stderr)
+        print("", file=sys.stderr)
+        print("Available tools:", file=sys.stderr)
+        print("  - execute_stream_test_read", file=sys.stderr)
+        print("  - run_connector_readiness_test_report", file=sys.stderr)
+        print("", file=sys.stderr)
+        print("Sample manifests (use @sample_name in manifest field):", file=sys.stderr)
+        print("  - @rick_and_morty_manifest", file=sys.stderr)
+        print("  - @simple_api_manifest", file=sys.stderr)
+        print("", file=sys.stderr)
+        print("Example:", file=sys.stderr)
+        print('  poe test-tool execute_stream_test_read \'{"manifest": "@simple_api_manifest", "config": {}, "stream_name": "users", "max_records": 3}\'', file=sys.stderr)
+        sys.exit(1)
+
+    tool_name = sys.argv[1]
+    json_args = sys.argv[2]
+
+    try:
+        args: Dict[str, Any] = json.loads(json_args)
+    except json.JSONDecodeError as e:
+        print(f"Error parsing JSON arguments: {e}", file=sys.stderr)
+        sys.exit(1)
+
+    if "manifest" in args and isinstance(args["manifest"], str) and args["manifest"].startswith("@"):
+        sample_name = args["manifest"][1:]
+        try:
+            manifest_path = Path(__file__).parent.parent / "tests" / "resources" / f"{sample_name}.yaml"
+            if manifest_path.exists():
+                args["manifest"] = str(manifest_path)
+            else:
+                raise FileNotFoundError(f"Sample manifest not found: {manifest_path}")
+        except FileNotFoundError as e:
+            print(f"Error loading sample manifest: {e}", file=sys.stderr)
+            sys.exit(1)
+
+    try:
+        if tool_name == "execute_stream_test_read":
+            result = execute_stream_test_read(**args)
+            print(f"Success: {result.success}")
+            print(f"Message: {result.message}")
+            print(f"Records read: {result.records_read}")
+            if result.record_stats:
+                print(f"Record stats: {json.dumps(result.record_stats, indent=2)}")
+            if result.errors:
+                print(f"Errors: {result.errors}")
+
+        elif tool_name == "run_connector_readiness_test_report":
+            result = run_connector_readiness_test_report(**args)
+            print("Readiness Test Report:")
+            print("=" * 60)
+            print(result)
+            print("=" * 60)
+
+        else:
+            print(f"Unknown tool: {tool_name}", file=sys.stderr)
+            print("Available tools: execute_stream_test_read, run_connector_readiness_test_report", file=sys.stderr)
+            sys.exit(1)
+
+    except Exception as e:
+        print(f"Error executing tool '{tool_name}': {e}", file=sys.stderr)
+        import traceback
+        traceback.print_exc()
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

connector_builder_mcp/_validation_testing.py

@@ -1,5 +1,4 @@
 """Validation and testing tools for Airbyte connector manifests."""
-
 import logging
 import pkgutil
 import time
@@ -15,17 +14,21 @@
     get_limits,
     resolve_manifest,
 )
-from airbyte_cdk.models import AirbyteMessage, ConfiguredAirbyteCatalog, Type
+from airbyte_cdk.models import AirbyteMessage, ConfiguredAirbyteCatalog, SyncMode, Type
 from airbyte_cdk.sources.declarative.parsers.manifest_component_transformer import (
     ManifestComponentTransformer,
 )
 from airbyte_cdk.sources.declarative.parsers.manifest_reference_resolver import (
     ManifestReferenceResolver,
 )
-from airbyte_cdk.test.catalog_builder import CatalogBuilder
+from airbyte_cdk.test.catalog_builder import CatalogBuilder, ConfiguredAirbyteStreamBuilder
 from airbyte_cdk.test.entrypoint_wrapper import EntrypointOutput, read
 from airbyte_cdk.test.state_builder import StateBuilder
-
+from airbyte_protocol_dataclasses.models.airbyte_protocol import (
+    AirbyteStream,
+    ConfiguredAirbyteStream,
+    DestinationSyncMode,
+)
 from connector_builder_mcp._secrets import hydrate_config
 from connector_builder_mcp._util import parse_manifest_input, validate_manifest_structure
 
@@ -76,16 +79,26 @@ class MultiStreamSmokeTest(BaseModel):
     stream_results: dict[str, StreamSmokeTest]
 
 
-def _get_dummy_catalog(manifest_dict: dict[str, Any]) -> ConfiguredAirbyteCatalog:
+def _get_dummy_catalog(
+    # manifest_dict: dict[str, Any],
+    stream_name: str,
+) -> ConfiguredAirbyteCatalog:
     """Create a dummy catalog for testing purposes."""
-    catalog_builder = CatalogBuilder()
-
-    streams = manifest_dict.get("streams", [])
-    for stream in streams:
-        stream_name = stream.get("name", "unknown_stream")
-        catalog_builder.with_stream(stream_name, {})
-
-    return catalog_builder.build()
+    return ConfiguredAirbyteCatalog(
+        streams=[
+            ConfiguredAirbyteStream(
+                sync_mode=SyncMode.full_refresh,
+                destination_sync_mode=DestinationSyncMode.append,
+                stream=AirbyteStream(
+                    name=stream_name,
+                    json_schema={},
+                    supported_sync_modes=[
+                        SyncMode.full_refresh,
+                    ],
+                )
+            )
+        ]
+    )
 
 
 def _get_declarative_component_schema() -> dict[str, Any]:
@@ -302,7 +315,10 @@ def execute_stream_test_read(
         limits = get_limits(config_with_manifest)
         source = create_source(config_with_manifest, limits)
 
-        catalog = _get_dummy_catalog(manifest_dict)
+        catalog = _get_dummy_catalog(
+            # manifest_dict,
+            stream_name=stream_name,
+        )
 
         for configured_stream in catalog.streams:
             if configured_stream.stream.name == stream_name:
@@ -388,8 +404,11 @@ def execute_stream_test_read(
         )
 
     except Exception as e:
-        logger.error(f"Error testing stream read: {e}")
-        error_msg = f"Stream test error: {str(e)}"
+        import traceback
+
+        tb = traceback.format_exc()
+        logger.error(f"Error testing stream read: {e}\n{tb}")
+        error_msg = f"Stream test error: {str(e)}\n{tb}"
 
         raw_responses_data = None
         if include_raw_responses_data is not False:

poe_tasks.toml

@@ -17,6 +17,7 @@ mcp-serve-local = { cmd = "uv run connector-builder-mcp", help = "Start the MCP
 mcp-serve-http = { cmd = "uv run python -c \"from connector_builder_mcp.server import app; app.run(transport='http', host='127.0.0.1', port=8000)\"", help = "Start the MCP server with HTTP transport" }
 mcp-serve-sse = { cmd = "uv run python -c \"from connector_builder_mcp.server import app; app.run(transport='sse', host='127.0.0.1', port=8000)\"", help = "Start the MCP server with SSE transport" }
 inspect = { cmd = "uv run fastmcp inspect connector_builder_mcp/server.py:app", help = "Inspect MCP tools and resources (supports --tools, --health, etc.)" }
+test-tool = { cmd = "uv run python bin/test_mcp_tool.py", help = "Test MCP tools directly with JSON arguments: poe test-tool <tool_name> '<json_args>'" }
 
 # Documentation tasks
 docs-generate = { cmd = "uv run pdoc connector_builder_mcp --output-dir docs/generated", help = "Generate API documentation" }

pyproject.toml

@@ -6,7 +6,7 @@ authors = [{name = "Airbyte", email = "[email protected]"}]
 readme = "README.md"
 requires-python = ">=3.11"
 dependencies = [
-    "airbyte-cdk>=6.0,<7.0",
+    "airbyte-cdk>=6.60.14,<7.0",
     "fastmcp>=0.2.0",
     "pydantic>=2.7.0,<3.0",
     "requests>=2.25.0",

tests/test_dummy_catalog.py

@@ -0,0 +1,38 @@
+# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
+"""Test dummy catalog creation."""
+from airbyte_cdk.models import SyncMode
+from connector_builder_mcp._validation_testing import _get_dummy_catalog
+
+
+def test_get_dummy_catalog_basic() -> None:
+    manifest_dict = {
+        "streams": [
+            {"name": "users"},
+            {"name": "items"},
+        ]
+    }
+    catalog = _get_dummy_catalog(
+        manifest_dict,
+    )
+    stream_names = [s.stream.name for s in catalog.streams]
+    assert stream_names == ["users", "items"]
+    for s in catalog.streams:
+        assert s.sync_mode == SyncMode.full_refresh
+
+def test_get_dummy_catalog_empty() -> None:
+    manifest_dict = {"streams": []}
+    catalog = _get_dummy_catalog(manifest_dict)
+    assert catalog.streams == []
+
+def test_get_dummy_catalog_missing_name() -> None:
+    manifest_dict = {
+        "streams": [
+            {},
+            {"name": "foo"},
+        ]
+    }
+    catalog = _get_dummy_catalog(manifest_dict)
+    stream_names = [s.stream.name for s in catalog.streams]
+    assert stream_names == ["unknown_stream", "foo"]
+    for s in catalog.streams:
+        assert s.sync_mode == SyncMode.full_refresh