From eb567a63d4a91686d25e8611eab18a9110c89c55 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 2 Oct 2025 22:21:21 +0000 Subject: [PATCH 1/9] feat: Add support for publishing custom connector definitions - Add api_util functions for custom YAML/Docker source/destination definition CRUD operations - Add CloudWorkspace methods for publishing all 3 definition types with validation - Add 9 MCP tools (publish/list/update for YAML sources, Docker sources, Docker destinations) - Add integration tests for all definition types - Add client-side manifest validation for YAML with pre_validate option Supports all 3 custom definition types from Airbyte 1.6: - custom_yaml_source_definition (YAML manifests, no Docker build needed) - custom_docker_source_definition (custom Docker source images) - custom_docker_destination_definition (custom Docker destination images) Uses airbyte-api 0.53.0 SDK with declarative_source_definitions, source_definitions, destination_definitions Relates to Airbyte 1.6 release: https://docs.airbyte.com/release_notes/v-1.6 API docs: https://reference.airbyte.com/reference/createdeclarativesourcedefinition Requested by: @aaronsteers Devin session: https://app.devin.ai/sessions/7733e25275f44008ab6cb765d4ef5106 Co-Authored-By: AJ Steers --- airbyte/_util/api_util.py | 487 ++++++++++++++++++ airbyte/cloud/workspaces.py | 391 +++++++++++++- airbyte/mcp/cloud_ops.py | 270 ++++++++++ .../cloud/test_custom_definitions.py | 184 +++++++ 4 files changed, 1331 insertions(+), 1 deletion(-) create mode 100644 tests/integration_tests/cloud/test_custom_definitions.py diff --git a/airbyte/_util/api_util.py b/airbyte/_util/api_util.py index a6ae76f2..b37ed98f 100644 --- a/airbyte/_util/api_util.py +++ b/airbyte/_util/api_util.py @@ -20,6 +20,8 @@ import airbyte_api import requests from airbyte_api import api, models +from airbyte_api import api as airbyte_api_api +from airbyte_api import models as airbyte_api_models from airbyte.constants import CLOUD_API_ROOT, CLOUD_CONFIG_API_ROOT from airbyte.exceptions import ( @@ -933,3 +935,488 @@ def check_connector( "response": json_result, }, ) + + +def validate_yaml_manifest( + manifest: Any, # noqa: ANN401 + *, + raise_on_error: bool = True, +) -> tuple[bool, str | None]: + """Validate a YAML connector manifest structure. + + Performs basic client-side validation before sending to API. + + Args: + manifest: The manifest to validate (should be a dictionary). + raise_on_error: Whether to raise an exception on validation failure. + + Returns: + Tuple of (is_valid, error_message) + """ + if not isinstance(manifest, dict): + error = "Manifest must be a dictionary" + if raise_on_error: + raise PyAirbyteInputError(message=error, context={"manifest": manifest}) + return False, error + + required_fields = ["version", "type"] + missing = [f for f in required_fields if f not in manifest] + if missing: + error = f"Manifest missing required fields: {', '.join(missing)}" + if raise_on_error: + raise PyAirbyteInputError(message=error, context={"manifest": manifest}) + return False, error + + if manifest.get("type") != "DeclarativeSource": + error = f"Manifest type must be 'DeclarativeSource', got '{manifest.get('type')}'" + if raise_on_error: + raise PyAirbyteInputError(message=error, context={"manifest": manifest}) + return False, error + + return True, None + + +def create_custom_yaml_source_definition( + name: str, + *, + workspace_id: str, + manifest: dict[str, Any], + api_root: str, + client_id: SecretString, + client_secret: SecretString, +) -> airbyte_api_models.DeclarativeSourceDefinitionResponse: + """Create a custom YAML source definition.""" + airbyte_instance = get_airbyte_server_instance( + api_root=api_root, + client_id=client_id, + client_secret=client_secret, + ) + + request_body = airbyte_api_models.CreateDeclarativeSourceDefinitionRequest( + name=name, + manifest=manifest, + ) + request = airbyte_api_api.CreateDeclarativeSourceDefinitionRequest( + workspace_id=workspace_id, + create_declarative_source_definition_request=request_body, + ) + response = airbyte_instance.declarative_source_definitions.create_declarative_source_definition( + request + ) + if response.declarative_source_definition_response is None: + raise AirbyteError( + message="Failed to create custom YAML source definition", + context={"name": name, "workspace_id": workspace_id}, + ) + return response.declarative_source_definition_response + + +def list_custom_yaml_source_definitions( + workspace_id: str, + *, + api_root: str, + client_id: SecretString, + client_secret: SecretString, +) -> list[airbyte_api_models.DeclarativeSourceDefinitionResponse]: + """List all custom YAML source definitions in a workspace.""" + airbyte_instance = get_airbyte_server_instance( + api_root=api_root, + client_id=client_id, + client_secret=client_secret, + ) + + request = airbyte_api_api.ListDeclarativeSourceDefinitionsRequest( + workspace_id=workspace_id, + ) + response = airbyte_instance.declarative_source_definitions.list_declarative_source_definitions( + request + ) + if response.declarative_source_definitions_response is None: + raise AirbyteError( + message="Failed to list custom YAML source definitions", + context={"workspace_id": workspace_id}, + ) + return response.declarative_source_definitions_response.data + + +def get_custom_yaml_source_definition( + workspace_id: str, + definition_id: str, + *, + api_root: str, + client_id: SecretString, + client_secret: SecretString, +) -> airbyte_api_models.DeclarativeSourceDefinitionResponse: + """Get a specific custom YAML source definition.""" + airbyte_instance = get_airbyte_server_instance( + api_root=api_root, + client_id=client_id, + client_secret=client_secret, + ) + + request = airbyte_api_api.GetDeclarativeSourceDefinitionRequest( + workspace_id=workspace_id, + definition_id=definition_id, + ) + response = airbyte_instance.declarative_source_definitions.get_declarative_source_definition( + request + ) + if response.declarative_source_definition_response is None: + raise AirbyteError( + message="Failed to get custom YAML source definition", + context={"workspace_id": workspace_id, "definition_id": definition_id}, + ) + return response.declarative_source_definition_response + + +def update_custom_yaml_source_definition( + workspace_id: str, + definition_id: str, + *, + manifest: dict[str, Any], + api_root: str, + client_id: SecretString, + client_secret: SecretString, +) -> airbyte_api_models.DeclarativeSourceDefinitionResponse: + """Update a custom YAML source definition.""" + airbyte_instance = get_airbyte_server_instance( + api_root=api_root, + client_id=client_id, + client_secret=client_secret, + ) + + request_body = airbyte_api_models.UpdateDeclarativeSourceDefinitionRequest( + manifest=manifest, + ) + request = airbyte_api_api.UpdateDeclarativeSourceDefinitionRequest( + workspace_id=workspace_id, + definition_id=definition_id, + update_declarative_source_definition_request=request_body, + ) + response = airbyte_instance.declarative_source_definitions.update_declarative_source_definition( + request + ) + if response.declarative_source_definition_response is None: + raise AirbyteError( + message="Failed to update custom YAML source definition", + context={"workspace_id": workspace_id, "definition_id": definition_id}, + ) + return response.declarative_source_definition_response + + +def delete_custom_yaml_source_definition( + workspace_id: str, + definition_id: str, + *, + api_root: str, + client_id: SecretString, + client_secret: SecretString, +) -> None: + """Delete a custom YAML source definition.""" + airbyte_instance = get_airbyte_server_instance( + api_root=api_root, + client_id=client_id, + client_secret=client_secret, + ) + + request = airbyte_api_api.DeleteDeclarativeSourceDefinitionRequest( + workspace_id=workspace_id, + definition_id=definition_id, + ) + airbyte_instance.declarative_source_definitions.delete_declarative_source_definition(request) + + +def create_custom_docker_source_definition( + name: str, + docker_repository: str, + docker_image_tag: str, + *, + workspace_id: str, + documentation_url: str | None = None, + api_root: str, + client_id: SecretString, + client_secret: SecretString, +) -> airbyte_api_models.DefinitionResponse: + """Create a custom Docker source definition.""" + airbyte_instance = get_airbyte_server_instance( + api_root=api_root, + client_id=client_id, + client_secret=client_secret, + ) + + request_body = airbyte_api_models.CreateDefinitionRequest( + name=name, + docker_repository=docker_repository, + docker_image_tag=docker_image_tag, + documentation_url=documentation_url, + ) + request = airbyte_api_api.CreateSourceDefinitionRequest( + workspace_id=workspace_id, + create_definition_request=request_body, + ) + response = airbyte_instance.source_definitions.create_source_definition(request) + if response.definition_response is None: + raise AirbyteError( + message="Failed to create custom Docker source definition", + context={"name": name, "workspace_id": workspace_id}, + ) + return response.definition_response + + +def list_custom_docker_source_definitions( + workspace_id: str, + *, + api_root: str, + client_id: SecretString, + client_secret: SecretString, +) -> list[airbyte_api_models.DefinitionResponse]: + """List all custom Docker source definitions in a workspace.""" + airbyte_instance = get_airbyte_server_instance( + api_root=api_root, + client_id=client_id, + client_secret=client_secret, + ) + + request = airbyte_api_api.ListSourceDefinitionsRequest( + workspace_id=workspace_id, + ) + response = airbyte_instance.source_definitions.list_source_definitions(request) + if response.definitions_response is None: + raise AirbyteError( + message="Failed to list custom Docker source definitions", + context={"workspace_id": workspace_id}, + ) + return response.definitions_response.data + + +def get_custom_docker_source_definition( + workspace_id: str, + definition_id: str, + *, + api_root: str, + client_id: SecretString, + client_secret: SecretString, +) -> airbyte_api_models.DefinitionResponse: + """Get a specific custom Docker source definition.""" + airbyte_instance = get_airbyte_server_instance( + api_root=api_root, + client_id=client_id, + client_secret=client_secret, + ) + + request = airbyte_api_api.GetSourceDefinitionRequest( + workspace_id=workspace_id, + definition_id=definition_id, + ) + response = airbyte_instance.source_definitions.get_source_definition(request) + if response.definition_response is None: + raise AirbyteError( + message="Failed to get custom Docker source definition", + context={"workspace_id": workspace_id, "definition_id": definition_id}, + ) + return response.definition_response + + +def update_custom_docker_source_definition( + workspace_id: str, + definition_id: str, + *, + name: str, + docker_image_tag: str, + api_root: str, + client_id: SecretString, + client_secret: SecretString, +) -> airbyte_api_models.DefinitionResponse: + """Update a custom Docker source definition.""" + airbyte_instance = get_airbyte_server_instance( + api_root=api_root, + client_id=client_id, + client_secret=client_secret, + ) + + request_body = airbyte_api_models.UpdateDefinitionRequest( + name=name, + docker_image_tag=docker_image_tag, + ) + request = airbyte_api_api.UpdateSourceDefinitionRequest( + workspace_id=workspace_id, + definition_id=definition_id, + update_definition_request=request_body, + ) + response = airbyte_instance.source_definitions.update_source_definition(request) + if response.definition_response is None: + raise AirbyteError( + message="Failed to update custom Docker source definition", + context={"workspace_id": workspace_id, "definition_id": definition_id}, + ) + return response.definition_response + + +def delete_custom_docker_source_definition( + workspace_id: str, + definition_id: str, + *, + api_root: str, + client_id: SecretString, + client_secret: SecretString, +) -> None: + """Delete a custom Docker source definition.""" + airbyte_instance = get_airbyte_server_instance( + api_root=api_root, + client_id=client_id, + client_secret=client_secret, + ) + + request = airbyte_api_api.DeleteSourceDefinitionRequest( + workspace_id=workspace_id, + definition_id=definition_id, + ) + airbyte_instance.source_definitions.delete_source_definition(request) + + +def create_custom_docker_destination_definition( + name: str, + docker_repository: str, + docker_image_tag: str, + *, + workspace_id: str, + documentation_url: str | None = None, + api_root: str, + client_id: SecretString, + client_secret: SecretString, +) -> airbyte_api_models.DefinitionResponse: + """Create a custom Docker destination definition.""" + airbyte_instance = get_airbyte_server_instance( + api_root=api_root, + client_id=client_id, + client_secret=client_secret, + ) + + request_body = airbyte_api_models.CreateDefinitionRequest( + name=name, + docker_repository=docker_repository, + docker_image_tag=docker_image_tag, + documentation_url=documentation_url, + ) + request = airbyte_api_api.CreateDestinationDefinitionRequest( + workspace_id=workspace_id, + create_definition_request=request_body, + ) + response = airbyte_instance.destination_definitions.create_destination_definition(request) + if response.definition_response is None: + raise AirbyteError( + message="Failed to create custom Docker destination definition", + context={"name": name, "workspace_id": workspace_id}, + ) + return response.definition_response + + +def list_custom_docker_destination_definitions( + workspace_id: str, + *, + api_root: str, + client_id: SecretString, + client_secret: SecretString, +) -> list[airbyte_api_models.DefinitionResponse]: + """List all custom Docker destination definitions in a workspace.""" + airbyte_instance = get_airbyte_server_instance( + api_root=api_root, + client_id=client_id, + client_secret=client_secret, + ) + + request = airbyte_api_api.ListDestinationDefinitionsRequest( + workspace_id=workspace_id, + ) + response = airbyte_instance.destination_definitions.list_destination_definitions(request) + if response.definitions_response is None: + raise AirbyteError( + message="Failed to list custom Docker destination definitions", + context={"workspace_id": workspace_id}, + ) + return response.definitions_response.data + + +def get_custom_docker_destination_definition( + workspace_id: str, + definition_id: str, + *, + api_root: str, + client_id: SecretString, + client_secret: SecretString, +) -> airbyte_api_models.DefinitionResponse: + """Get a specific custom Docker destination definition.""" + airbyte_instance = get_airbyte_server_instance( + api_root=api_root, + client_id=client_id, + client_secret=client_secret, + ) + + request = airbyte_api_api.GetDestinationDefinitionRequest( + workspace_id=workspace_id, + definition_id=definition_id, + ) + response = airbyte_instance.destination_definitions.get_destination_definition(request) + if response.definition_response is None: + raise AirbyteError( + message="Failed to get custom Docker destination definition", + context={"workspace_id": workspace_id, "definition_id": definition_id}, + ) + return response.definition_response + + +def update_custom_docker_destination_definition( + workspace_id: str, + definition_id: str, + *, + name: str, + docker_image_tag: str, + api_root: str, + client_id: SecretString, + client_secret: SecretString, +) -> airbyte_api_models.DefinitionResponse: + """Update a custom Docker destination definition.""" + airbyte_instance = get_airbyte_server_instance( + api_root=api_root, + client_id=client_id, + client_secret=client_secret, + ) + + request_body = airbyte_api_models.UpdateDefinitionRequest( + name=name, + docker_image_tag=docker_image_tag, + ) + request = airbyte_api_api.UpdateDestinationDefinitionRequest( + workspace_id=workspace_id, + definition_id=definition_id, + update_definition_request=request_body, + ) + response = airbyte_instance.destination_definitions.update_destination_definition(request) + if response.definition_response is None: + raise AirbyteError( + message="Failed to update custom Docker destination definition", + context={"workspace_id": workspace_id, "definition_id": definition_id}, + ) + return response.definition_response + + +def delete_custom_docker_destination_definition( + workspace_id: str, + definition_id: str, + *, + api_root: str, + client_id: SecretString, + client_secret: SecretString, +) -> None: + """Delete a custom Docker destination definition.""" + airbyte_instance = get_airbyte_server_instance( + api_root=api_root, + client_id=client_id, + client_secret=client_secret, + ) + + request = airbyte_api_api.DeleteDestinationDefinitionRequest( + workspace_id=workspace_id, + definition_id=definition_id, + ) + airbyte_instance.destination_definitions.delete_destination_definition(request) diff --git a/airbyte/cloud/workspaces.py b/airbyte/cloud/workspaces.py index 621b3d63..137c6733 100644 --- a/airbyte/cloud/workspaces.py +++ b/airbyte/cloud/workspaces.py @@ -36,8 +36,11 @@ from __future__ import annotations from dataclasses import dataclass +from pathlib import Path from typing import TYPE_CHECKING, Any +import yaml + from airbyte import exceptions as exc from airbyte._util import api_util, text_util from airbyte._util.api_util import get_web_url_root @@ -53,7 +56,7 @@ from airbyte.sources.base import Source -@dataclass +@dataclass # noqa: PLR0904 class CloudWorkspace: """A remote workspace on the Airbyte Cloud. @@ -450,3 +453,389 @@ def list_destinations( for destination in destinations if name is None or destination.name == name ] + + def publish_custom_yaml_source( + self, + name: str, + manifest: dict[str, Any] | Path | str, + *, + unique: bool = True, + pre_validate: bool = True, + ) -> dict[str, Any]: + """Publish a custom YAML source definition to the workspace.""" + manifest_dict: dict[str, Any] + if isinstance(manifest, Path): + manifest_dict = yaml.safe_load(manifest.read_text()) + elif isinstance(manifest, str): + manifest_dict = yaml.safe_load(manifest) + else: + manifest_dict = manifest + + if pre_validate: + api_util.validate_yaml_manifest(manifest_dict, raise_on_error=True) + + if unique: + existing = self.list_custom_yaml_sources(name=name) + if existing: + raise exc.AirbyteDuplicateResourcesError( + resource_type="custom_yaml_source_definition", + resource_name=name, + ) + + result = api_util.create_custom_yaml_source_definition( + name=name, + workspace_id=self.workspace_id, + manifest=manifest_dict, + api_root=self.api_root, + client_id=self.client_id, + client_secret=self.client_secret, + ) + + return { + "id": result.id, + "name": result.name, + "manifest": result.manifest, + "version": result.version, + } + + def list_custom_yaml_sources( + self, + name: str | None = None, + ) -> list[dict[str, Any]]: + """List all custom YAML source definitions in the workspace.""" + definitions = api_util.list_custom_yaml_source_definitions( + workspace_id=self.workspace_id, + api_root=self.api_root, + client_id=self.client_id, + client_secret=self.client_secret, + ) + + result = [ + { + "id": d.id, + "name": d.name, + "manifest": d.manifest, + "version": d.version, + } + for d in definitions + ] + + if name: + result = [d for d in result if d["name"] == name] + + return result + + def get_custom_yaml_source( + self, + definition_id: str, + ) -> dict[str, Any]: + """Get a specific custom YAML source definition by ID.""" + result = api_util.get_custom_yaml_source_definition( + workspace_id=self.workspace_id, + definition_id=definition_id, + api_root=self.api_root, + client_id=self.client_id, + client_secret=self.client_secret, + ) + return { + "id": result.id, + "name": result.name, + "manifest": result.manifest, + "version": result.version, + } + + def update_custom_yaml_source( + self, + definition_id: str, + *, + manifest: dict[str, Any] | Path | str, + pre_validate: bool = True, + ) -> dict[str, Any]: + """Update a custom YAML source definition.""" + manifest_dict: dict[str, Any] + if isinstance(manifest, Path): + manifest_dict = yaml.safe_load(manifest.read_text()) + elif isinstance(manifest, str): + manifest_dict = yaml.safe_load(manifest) + else: + manifest_dict = manifest + + if pre_validate: + api_util.validate_yaml_manifest(manifest_dict, raise_on_error=True) + + result = api_util.update_custom_yaml_source_definition( + workspace_id=self.workspace_id, + definition_id=definition_id, + manifest=manifest_dict, + api_root=self.api_root, + client_id=self.client_id, + client_secret=self.client_secret, + ) + + return { + "id": result.id, + "name": result.name, + "manifest": result.manifest, + "version": result.version, + } + + def delete_custom_yaml_source( + self, + definition_id: str, + ) -> None: + """Delete a custom YAML source definition.""" + api_util.delete_custom_yaml_source_definition( + workspace_id=self.workspace_id, + definition_id=definition_id, + api_root=self.api_root, + client_id=self.client_id, + client_secret=self.client_secret, + ) + + def publish_custom_docker_source( + self, + name: str, + docker_repository: str, + docker_image_tag: str, + *, + documentation_url: str | None = None, + unique: bool = True, + ) -> dict[str, Any]: + """Publish a custom Docker source definition to the workspace.""" + if unique: + existing = self.list_custom_docker_sources(name=name) + if existing: + raise exc.AirbyteDuplicateResourcesError( + resource_type="custom_docker_source_definition", + resource_name=name, + ) + + result = api_util.create_custom_docker_source_definition( + name=name, + docker_repository=docker_repository, + docker_image_tag=docker_image_tag, + workspace_id=self.workspace_id, + documentation_url=documentation_url, + api_root=self.api_root, + client_id=self.client_id, + client_secret=self.client_secret, + ) + + return { + "id": result.id, + "name": result.name, + "docker_repository": result.docker_repository, + "docker_image_tag": result.docker_image_tag, + "documentation_url": result.documentation_url, + } + + def list_custom_docker_sources( + self, + name: str | None = None, + ) -> list[dict[str, Any]]: + """List all custom Docker source definitions in the workspace.""" + definitions = api_util.list_custom_docker_source_definitions( + workspace_id=self.workspace_id, + api_root=self.api_root, + client_id=self.client_id, + client_secret=self.client_secret, + ) + + result = [ + { + "id": d.id, + "name": d.name, + "docker_repository": d.docker_repository, + "docker_image_tag": d.docker_image_tag, + "documentation_url": d.documentation_url, + } + for d in definitions + ] + + if name: + result = [d for d in result if d["name"] == name] + + return result + + def get_custom_docker_source( + self, + definition_id: str, + ) -> dict[str, Any]: + """Get a specific custom Docker source definition by ID.""" + result = api_util.get_custom_docker_source_definition( + workspace_id=self.workspace_id, + definition_id=definition_id, + api_root=self.api_root, + client_id=self.client_id, + client_secret=self.client_secret, + ) + return { + "id": result.id, + "name": result.name, + "docker_repository": result.docker_repository, + "docker_image_tag": result.docker_image_tag, + "documentation_url": result.documentation_url, + } + + def update_custom_docker_source( + self, + definition_id: str, + *, + name: str, + docker_image_tag: str, + ) -> dict[str, Any]: + """Update a custom Docker source definition.""" + result = api_util.update_custom_docker_source_definition( + workspace_id=self.workspace_id, + definition_id=definition_id, + name=name, + docker_image_tag=docker_image_tag, + api_root=self.api_root, + client_id=self.client_id, + client_secret=self.client_secret, + ) + + return { + "id": result.id, + "name": result.name, + "docker_repository": result.docker_repository, + "docker_image_tag": result.docker_image_tag, + "documentation_url": result.documentation_url, + } + + def delete_custom_docker_source( + self, + definition_id: str, + ) -> None: + """Delete a custom Docker source definition.""" + api_util.delete_custom_docker_source_definition( + workspace_id=self.workspace_id, + definition_id=definition_id, + api_root=self.api_root, + client_id=self.client_id, + client_secret=self.client_secret, + ) + + def publish_custom_docker_destination( + self, + name: str, + docker_repository: str, + docker_image_tag: str, + *, + documentation_url: str | None = None, + unique: bool = True, + ) -> dict[str, Any]: + """Publish a custom Docker destination definition to the workspace.""" + if unique: + existing = self.list_custom_docker_destinations(name=name) + if existing: + raise exc.AirbyteDuplicateResourcesError( + resource_type="custom_docker_destination_definition", + resource_name=name, + ) + + result = api_util.create_custom_docker_destination_definition( + name=name, + docker_repository=docker_repository, + docker_image_tag=docker_image_tag, + workspace_id=self.workspace_id, + documentation_url=documentation_url, + api_root=self.api_root, + client_id=self.client_id, + client_secret=self.client_secret, + ) + + return { + "id": result.id, + "name": result.name, + "docker_repository": result.docker_repository, + "docker_image_tag": result.docker_image_tag, + "documentation_url": result.documentation_url, + } + + def list_custom_docker_destinations( + self, + name: str | None = None, + ) -> list[dict[str, Any]]: + """List all custom Docker destination definitions in the workspace.""" + definitions = api_util.list_custom_docker_destination_definitions( + workspace_id=self.workspace_id, + api_root=self.api_root, + client_id=self.client_id, + client_secret=self.client_secret, + ) + + result = [ + { + "id": d.id, + "name": d.name, + "docker_repository": d.docker_repository, + "docker_image_tag": d.docker_image_tag, + "documentation_url": d.documentation_url, + } + for d in definitions + ] + + if name: + result = [d for d in result if d["name"] == name] + + return result + + def get_custom_docker_destination( + self, + definition_id: str, + ) -> dict[str, Any]: + """Get a specific custom Docker destination definition by ID.""" + result = api_util.get_custom_docker_destination_definition( + workspace_id=self.workspace_id, + definition_id=definition_id, + api_root=self.api_root, + client_id=self.client_id, + client_secret=self.client_secret, + ) + return { + "id": result.id, + "name": result.name, + "docker_repository": result.docker_repository, + "docker_image_tag": result.docker_image_tag, + "documentation_url": result.documentation_url, + } + + def update_custom_docker_destination( + self, + definition_id: str, + *, + name: str, + docker_image_tag: str, + ) -> dict[str, Any]: + """Update a custom Docker destination definition.""" + result = api_util.update_custom_docker_destination_definition( + workspace_id=self.workspace_id, + definition_id=definition_id, + name=name, + docker_image_tag=docker_image_tag, + api_root=self.api_root, + client_id=self.client_id, + client_secret=self.client_secret, + ) + + return { + "id": result.id, + "name": result.name, + "docker_repository": result.docker_repository, + "docker_image_tag": result.docker_image_tag, + "documentation_url": result.documentation_url, + } + + def delete_custom_docker_destination( + self, + definition_id: str, + ) -> None: + """Delete a custom Docker destination definition.""" + api_util.delete_custom_docker_destination_definition( + workspace_id=self.workspace_id, + definition_id=definition_id, + api_root=self.api_root, + client_id=self.client_id, + client_secret=self.client_secret, + ) diff --git a/airbyte/mcp/cloud_ops.py b/airbyte/mcp/cloud_ops.py index ecb43d4b..b1d37174 100644 --- a/airbyte/mcp/cloud_ops.py +++ b/airbyte/mcp/cloud_ops.py @@ -501,6 +501,267 @@ def list_deployed_cloud_connections() -> list[CloudConnection]: return workspace.list_connections() +def publish_custom_yaml_source_definition( + name: Annotated[ + str, + Field(description="The name for the custom connector definition."), + ], + manifest: Annotated[ + dict | str, + Field( + description=( + "The Low-code CDK manifest as a dict or YAML string. " + "Must be a valid declarative YAML connector manifest." + ), + ), + ], + *, + unique: Annotated[ + bool, + Field( + description="Whether to require a unique name.", + default=True, + ), + ] = True, + pre_validate: Annotated[ + bool, + Field( + description="Whether to validate the manifest client-side before publishing.", + default=True, + ), + ] = True, +) -> str: + """Publish a custom YAML source connector definition to Airbyte Cloud.""" + try: + workspace: CloudWorkspace = _get_cloud_workspace() + result = workspace.publish_custom_yaml_source( + name=name, + manifest=manifest, + unique=unique, + pre_validate=pre_validate, + ) + except Exception as ex: + return f"Failed to publish custom YAML source definition '{name}': {ex}" + else: + return ( + f"Successfully published custom YAML source definition '{name}' " + f"with ID '{result['id']}' (version {result.get('version', 'N/A')})" + ) + + +def list_custom_yaml_source_definitions() -> list[dict[str, Any]]: + """List all custom YAML source definitions in the Airbyte Cloud workspace.""" + workspace: CloudWorkspace = _get_cloud_workspace() + return workspace.list_custom_yaml_sources() + + +def update_custom_yaml_source_definition( + definition_id: Annotated[ + str, + Field(description="The ID of the definition to update."), + ], + manifest: Annotated[ + dict | str, + Field( + description="New manifest as dict or YAML string.", + ), + ], + *, + pre_validate: Annotated[ + bool, + Field( + description="Whether to validate the manifest client-side before updating.", + default=True, + ), + ] = True, +) -> str: + """Update a custom YAML source definition in Airbyte Cloud.""" + try: + workspace: CloudWorkspace = _get_cloud_workspace() + result = workspace.update_custom_yaml_source( + definition_id=definition_id, + manifest=manifest, + pre_validate=pre_validate, + ) + except Exception as ex: + return f"Failed to update custom YAML source definition '{definition_id}': {ex}" + else: + return ( + f"Successfully updated custom YAML source definition. " + f"New name: {result.get('name')}, version: {result.get('version', 'N/A')}" + ) + + +def publish_custom_docker_source_definition( + name: Annotated[ + str, + Field(description="The name for the custom connector definition."), + ], + docker_repository: Annotated[ + str, + Field(description="Docker repository (e.g., 'airbyte/source-custom')."), + ], + docker_image_tag: Annotated[ + str, + Field(description="Docker image tag (e.g., '1.0.0')."), + ], + *, + documentation_url: Annotated[ + str | None, + Field( + description="Optional URL to connector documentation.", + default=None, + ), + ] = None, + unique: Annotated[ + bool, + Field( + description="Whether to require a unique name.", + default=True, + ), + ] = True, +) -> str: + """Publish a custom Docker source connector definition to Airbyte Cloud.""" + try: + workspace: CloudWorkspace = _get_cloud_workspace() + result = workspace.publish_custom_docker_source( + name=name, + docker_repository=docker_repository, + docker_image_tag=docker_image_tag, + documentation_url=documentation_url, + unique=unique, + ) + except Exception as ex: + return f"Failed to publish custom Docker source definition '{name}': {ex}" + else: + return ( + f"Successfully published custom Docker source definition '{name}' " + f"with ID '{result['id']}' ({result['docker_repository']}:{result['docker_image_tag']})" + ) + + +def list_custom_docker_source_definitions() -> list[dict[str, Any]]: + """List all custom Docker source definitions in the Airbyte Cloud workspace.""" + workspace: CloudWorkspace = _get_cloud_workspace() + return workspace.list_custom_docker_sources() + + +def update_custom_docker_source_definition( + definition_id: Annotated[ + str, + Field(description="The ID of the definition to update."), + ], + name: Annotated[ + str, + Field(description="New name for the definition."), + ], + docker_image_tag: Annotated[ + str, + Field(description="New Docker image tag."), + ], +) -> str: + """Update a custom Docker source definition in Airbyte Cloud.""" + try: + workspace: CloudWorkspace = _get_cloud_workspace() + result = workspace.update_custom_docker_source( + definition_id=definition_id, + name=name, + docker_image_tag=docker_image_tag, + ) + except Exception as ex: + return f"Failed to update custom Docker source definition '{definition_id}': {ex}" + else: + return ( + f"Successfully updated custom Docker source definition. " + f"New name: {result.get('name')}, tag: {result.get('docker_image_tag')}" + ) + + +def publish_custom_docker_destination_definition( + name: Annotated[ + str, + Field(description="The name for the custom connector definition."), + ], + docker_repository: Annotated[ + str, + Field(description="Docker repository (e.g., 'airbyte/destination-custom')."), + ], + docker_image_tag: Annotated[ + str, + Field(description="Docker image tag (e.g., '1.0.0')."), + ], + *, + documentation_url: Annotated[ + str | None, + Field( + description="Optional URL to connector documentation.", + default=None, + ), + ] = None, + unique: Annotated[ + bool, + Field( + description="Whether to require a unique name.", + default=True, + ), + ] = True, +) -> str: + """Publish a custom Docker destination connector definition to Airbyte Cloud.""" + try: + workspace: CloudWorkspace = _get_cloud_workspace() + result = workspace.publish_custom_docker_destination( + name=name, + docker_repository=docker_repository, + docker_image_tag=docker_image_tag, + documentation_url=documentation_url, + unique=unique, + ) + except Exception as ex: + return f"Failed to publish custom Docker destination definition '{name}': {ex}" + else: + return ( + f"Successfully published custom Docker destination definition '{name}' " + f"with ID '{result['id']}' ({result['docker_repository']}:{result['docker_image_tag']})" + ) + + +def list_custom_docker_destination_definitions() -> list[dict[str, Any]]: + """List all custom Docker destination definitions in the Airbyte Cloud workspace.""" + workspace: CloudWorkspace = _get_cloud_workspace() + return workspace.list_custom_docker_destinations() + + +def update_custom_docker_destination_definition( + definition_id: Annotated[ + str, + Field(description="The ID of the definition to update."), + ], + name: Annotated[ + str, + Field(description="New name for the definition."), + ], + docker_image_tag: Annotated[ + str, + Field(description="New Docker image tag."), + ], +) -> str: + """Update a custom Docker destination definition in Airbyte Cloud.""" + try: + workspace: CloudWorkspace = _get_cloud_workspace() + result = workspace.update_custom_docker_destination( + definition_id=definition_id, + name=name, + docker_image_tag=docker_image_tag, + ) + except Exception as ex: + return f"Failed to update custom Docker destination definition '{definition_id}': {ex}" + else: + return ( + f"Successfully updated custom Docker destination definition. " + f"New name: {result.get('name')}, tag: {result.get('docker_image_tag')}" + ) + + def register_cloud_ops_tools(app: FastMCP) -> None: """@private Register tools with the FastMCP app. @@ -517,3 +778,12 @@ def register_cloud_ops_tools(app: FastMCP) -> None: app.tool(list_deployed_cloud_source_connectors) app.tool(list_deployed_cloud_destination_connectors) app.tool(list_deployed_cloud_connections) + app.tool(publish_custom_yaml_source_definition) + app.tool(list_custom_yaml_source_definitions) + app.tool(update_custom_yaml_source_definition) + app.tool(publish_custom_docker_source_definition) + app.tool(list_custom_docker_source_definitions) + app.tool(update_custom_docker_source_definition) + app.tool(publish_custom_docker_destination_definition) + app.tool(list_custom_docker_destination_definitions) + app.tool(update_custom_docker_destination_definition) diff --git a/tests/integration_tests/cloud/test_custom_definitions.py b/tests/integration_tests/cloud/test_custom_definitions.py new file mode 100644 index 00000000..20832135 --- /dev/null +++ b/tests/integration_tests/cloud/test_custom_definitions.py @@ -0,0 +1,184 @@ +# Copyright (c) 2024 Airbyte, Inc., all rights reserved. +"""Tests for custom connector definition publishing.""" + +import pytest + +from airbyte.cloud.workspaces import CloudWorkspace + + +TEST_YAML_MANIFEST = { + "version": "0.1.0", + "type": "DeclarativeSource", + "check": {"type": "CheckStream", "stream_names": ["test"]}, + "streams": [ + { + "type": "DeclarativeStream", + "name": "test", + "primary_key": [], + "retriever": { + "type": "SimpleRetriever", + "requester": { + "type": "HttpRequester", + "url_base": "https://httpbin.org", + "path": "/get", + }, + "record_selector": { + "type": "RecordSelector", + "extractor": {"type": "DpathExtractor", "field_path": []}, + }, + }, + } + ], +} + + +@pytest.mark.requires_creds +def test_publish_custom_yaml_source( + cloud_workspace: CloudWorkspace, +) -> None: + """Test publishing a custom YAML source definition.""" + from airbyte._util import text_util + + name = f"test-yaml-source-{text_util.generate_random_suffix()}" + + result = cloud_workspace.publish_custom_yaml_source( + name=name, + manifest=TEST_YAML_MANIFEST, + unique=True, + pre_validate=True, + ) + + assert "id" in result + assert result["name"] == name + assert "manifest" in result + assert "version" in result + + definition_id = result["id"] + + try: + definitions = cloud_workspace.list_custom_yaml_sources(name=name) + assert len(definitions) == 1 + assert definitions[0]["id"] == definition_id + + fetched = cloud_workspace.get_custom_yaml_source(definition_id) + assert fetched["id"] == definition_id + assert fetched["name"] == name + + updated_manifest = TEST_YAML_MANIFEST.copy() + updated_manifest["version"] = "0.2.0" + updated = cloud_workspace.update_custom_yaml_source( + definition_id=definition_id, + manifest=updated_manifest, + ) + assert updated["manifest"]["version"] == "0.2.0" + + finally: + cloud_workspace.delete_custom_yaml_source(definition_id) + + +@pytest.mark.requires_creds +def test_publish_custom_docker_source( + cloud_workspace: CloudWorkspace, +) -> None: + """Test publishing a custom Docker source definition.""" + from airbyte._util import text_util + + name = f"test-docker-source-{text_util.generate_random_suffix()}" + + result = cloud_workspace.publish_custom_docker_source( + name=name, + docker_repository="airbyte/test-source", + docker_image_tag="1.0.0", + documentation_url="https://example.com/docs", + unique=True, + ) + + assert "id" in result + assert result["name"] == name + assert result["docker_repository"] == "airbyte/test-source" + assert result["docker_image_tag"] == "1.0.0" + + definition_id = result["id"] + + try: + definitions = cloud_workspace.list_custom_docker_sources(name=name) + assert len(definitions) == 1 + assert definitions[0]["id"] == definition_id + + fetched = cloud_workspace.get_custom_docker_source(definition_id) + assert fetched["id"] == definition_id + assert fetched["name"] == name + + updated = cloud_workspace.update_custom_docker_source( + definition_id=definition_id, + name=name, + docker_image_tag="2.0.0", + ) + assert updated["docker_image_tag"] == "2.0.0" + + finally: + cloud_workspace.delete_custom_docker_source(definition_id) + + +@pytest.mark.requires_creds +def test_publish_custom_docker_destination( + cloud_workspace: CloudWorkspace, +) -> None: + """Test publishing a custom Docker destination definition.""" + from airbyte._util import text_util + + name = f"test-docker-dest-{text_util.generate_random_suffix()}" + + result = cloud_workspace.publish_custom_docker_destination( + name=name, + docker_repository="airbyte/test-destination", + docker_image_tag="1.0.0", + unique=True, + ) + + assert "id" in result + assert result["name"] == name + assert result["docker_repository"] == "airbyte/test-destination" + assert result["docker_image_tag"] == "1.0.0" + + definition_id = result["id"] + + try: + definitions = cloud_workspace.list_custom_docker_destinations(name=name) + assert len(definitions) == 1 + assert definitions[0]["id"] == definition_id + + fetched = cloud_workspace.get_custom_docker_destination(definition_id) + assert fetched["id"] == definition_id + assert fetched["name"] == name + + updated = cloud_workspace.update_custom_docker_destination( + definition_id=definition_id, + name=name, + docker_image_tag="2.0.0", + ) + assert updated["docker_image_tag"] == "2.0.0" + + finally: + cloud_workspace.delete_custom_docker_destination(definition_id) + + +@pytest.mark.requires_creds +def test_yaml_validation_error( + cloud_workspace: CloudWorkspace, +) -> None: + """Test that validation catches invalid manifests.""" + from airbyte._util import text_util + from airbyte.exceptions import PyAirbyteInputError + + name = f"test-invalid-{text_util.generate_random_suffix()}" + invalid_manifest = {"version": "0.1.0"} + + with pytest.raises(PyAirbyteInputError) as exc_info: + cloud_workspace.publish_custom_yaml_source( + name=name, + manifest=invalid_manifest, + pre_validate=True, + ) + + assert "type" in str(exc_info.value).lower() From 7a6a5b9086f41d560abdc9588d3e026c9efe7a51 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 2 Oct 2025 23:00:11 +0000 Subject: [PATCH 2/9] refactor: Consolidate custom definition API with dataclass returns - Replace 15 CloudWorkspace methods with 10 consolidated methods - Add CloudCustomSourceDefinition and CloudCustomDestinationDefinition dataclasses - Implement lazy-loading pattern for efficient data retrieval - Replace 9 MCP tools with 6 consolidated tools with shortened names - Update integration tests to use new dataclass returns - Fix all parameter passing to use api_root, client_id, client_secret All methods now return proper dataclasses following the lazy-loading pattern from CloudSource/CloudConnection. Public API is consolidated to accept either manifest_yaml or docker_image parameters. Co-Authored-By: AJ Steers --- airbyte/cloud/connectors.py | 232 ++++++- airbyte/cloud/workspaces.py | 571 +++++++++--------- airbyte/mcp/cloud_ops.py | 292 +++++---- .../cloud/test_custom_definitions.py | 114 ++-- 4 files changed, 753 insertions(+), 456 deletions(-) diff --git a/airbyte/cloud/connectors.py b/airbyte/cloud/connectors.py index 658ce7a6..40d38eee 100644 --- a/airbyte/cloud/connectors.py +++ b/airbyte/cloud/connectors.py @@ -41,7 +41,7 @@ import abc from dataclasses import dataclass -from typing import TYPE_CHECKING, ClassVar, Literal +from typing import TYPE_CHECKING, Any, ClassVar, Literal from airbyte_api import models as api_models # noqa: TC002 @@ -253,3 +253,233 @@ def _from_destination_response( ) result._connector_info = destination_response # noqa: SLF001 # Accessing Non-Public API return result + + +class CloudCustomSourceDefinition: + """A custom source connector definition in Airbyte Cloud. + + This represents either a YAML (declarative) or Docker-based custom source definition. + """ + + def __init__( + self, + workspace: CloudWorkspace, + definition_id: str, + connector_type: Literal["yaml", "docker"], + ) -> None: + """Initialize a custom source definition object.""" + self.workspace = workspace + self.definition_id = definition_id + self.connector_type = connector_type + self._definition_info: ( + api_models.DeclarativeSourceDefinitionResponse | api_models.DefinitionResponse | None + ) = None + + def _fetch_definition_info( + self, + ) -> api_models.DeclarativeSourceDefinitionResponse | api_models.DefinitionResponse: + """Fetch definition info from the API.""" + if self.connector_type == "yaml": + return api_util.get_custom_yaml_source_definition( + workspace_id=self.workspace.workspace_id, + definition_id=self.definition_id, + api_root=self.workspace.api_root, + client_id=self.workspace.client_id, + client_secret=self.workspace.client_secret, + ) + return api_util.get_custom_docker_source_definition( + workspace_id=self.workspace.workspace_id, + definition_id=self.definition_id, + api_root=self.workspace.api_root, + client_id=self.workspace.client_id, + client_secret=self.workspace.client_secret, + ) + + @property + def name(self) -> str: + """Get the display name of the custom connector definition.""" + if not self._definition_info: + self._definition_info = self._fetch_definition_info() + return self._definition_info.name + + @property + def manifest(self) -> dict[str, Any] | None: + """Get the Low-code CDK manifest. Only present for YAML connectors.""" + if self.connector_type != "yaml": + return None + if not self._definition_info: + self._definition_info = self._fetch_definition_info() + return self._definition_info.manifest + + @property + def version(self) -> str | None: + """Get the manifest version. Only present for YAML connectors.""" + if self.connector_type != "yaml": + return None + if not self._definition_info: + self._definition_info = self._fetch_definition_info() + return self._definition_info.version + + @property + def docker_repository(self) -> str | None: + """Get the Docker repository. Only present for Docker connectors.""" + if self.connector_type != "docker": + return None + if not self._definition_info: + self._definition_info = self._fetch_definition_info() + return self._definition_info.docker_repository + + @property + def docker_image_tag(self) -> str | None: + """Get the Docker image tag. Only present for Docker connectors.""" + if self.connector_type != "docker": + return None + if not self._definition_info: + self._definition_info = self._fetch_definition_info() + return self._definition_info.docker_image_tag + + @property + def documentation_url(self) -> str | None: + """Get the documentation URL. Only present for Docker connectors.""" + if self.connector_type != "docker": + return None + if not self._definition_info: + self._definition_info = self._fetch_definition_info() + return self._definition_info.documentation_url + + @property + def definition_url(self) -> str: + """Get the web URL of the custom source definition.""" + return ( + f"{self.workspace.workspace_url}/settings/custom-connectors/" + f"sources/{self.definition_id}" + ) + + def permanently_delete(self) -> None: + """Permanently delete this custom source definition.""" + self.workspace.permanently_delete_custom_source_definition(self.definition_id) + + def __repr__(self) -> str: + """String representation.""" + return ( + f"CloudCustomSourceDefinition(definition_id={self.definition_id}, " + f"name={self.name}, connector_type={self.connector_type})" + ) + + @classmethod + def _from_yaml_response( + cls, + workspace: CloudWorkspace, + response: api_models.DeclarativeSourceDefinitionResponse, + ) -> CloudCustomSourceDefinition: + """Internal factory method for YAML connectors.""" + result = cls( + workspace=workspace, + definition_id=response.id, + connector_type="yaml", + ) + result._definition_info = response # noqa: SLF001 + return result + + @classmethod + def _from_docker_response( + cls, + workspace: CloudWorkspace, + response: api_models.DefinitionResponse, + ) -> CloudCustomSourceDefinition: + """Internal factory method for Docker connectors.""" + result = cls( + workspace=workspace, + definition_id=response.id, + connector_type="docker", + ) + result._definition_info = response # noqa: SLF001 + return result + + +class CloudCustomDestinationDefinition: + """A custom destination connector definition in Airbyte Cloud. + + Currently only supports Docker-based custom destinations. + """ + + def __init__( + self, + workspace: CloudWorkspace, + definition_id: str, + ) -> None: + """Initialize a custom destination definition object.""" + self.workspace = workspace + self.definition_id = definition_id + self._definition_info: api_models.DefinitionResponse | None = None + + def _fetch_definition_info(self) -> api_models.DefinitionResponse: + """Fetch definition info from the API.""" + return api_util.get_custom_docker_destination_definition( + workspace_id=self.workspace.workspace_id, + definition_id=self.definition_id, + api_root=self.workspace.api_root, + client_id=self.workspace.client_id, + client_secret=self.workspace.client_secret, + ) + + @property + def name(self) -> str: + """Get the display name of the custom connector definition.""" + if not self._definition_info: + self._definition_info = self._fetch_definition_info() + return self._definition_info.name + + @property + def docker_repository(self) -> str: + """Get the Docker repository.""" + if not self._definition_info: + self._definition_info = self._fetch_definition_info() + return self._definition_info.docker_repository + + @property + def docker_image_tag(self) -> str: + """Get the Docker image tag.""" + if not self._definition_info: + self._definition_info = self._fetch_definition_info() + return self._definition_info.docker_image_tag + + @property + def documentation_url(self) -> str | None: + """Get the documentation URL.""" + if not self._definition_info: + self._definition_info = self._fetch_definition_info() + return self._definition_info.documentation_url + + @property + def definition_url(self) -> str: + """Get the web URL of the custom destination definition.""" + return ( + f"{self.workspace.workspace_url}/settings/custom-connectors/" + f"destinations/{self.definition_id}" + ) + + def permanently_delete(self) -> None: + """Permanently delete this custom destination definition.""" + self.workspace.permanently_delete_custom_destination_definition(self.definition_id) + + def __repr__(self) -> str: + """String representation.""" + return ( + f"CloudCustomDestinationDefinition(definition_id={self.definition_id}, " + f"name={self.name}, docker_repository={self.docker_repository})" + ) + + @classmethod + def _from_docker_response( + cls, + workspace: CloudWorkspace, + response: api_models.DefinitionResponse, + ) -> CloudCustomDestinationDefinition: + """Internal factory method.""" + result = cls( + workspace=workspace, + definition_id=response.id, + ) + result._definition_info = response # noqa: SLF001 + return result diff --git a/airbyte/cloud/workspaces.py b/airbyte/cloud/workspaces.py index 137c6733..281deaf5 100644 --- a/airbyte/cloud/workspaces.py +++ b/airbyte/cloud/workspaces.py @@ -37,7 +37,7 @@ from dataclasses import dataclass from pathlib import Path -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, Literal import yaml @@ -45,7 +45,12 @@ from airbyte._util import api_util, text_util from airbyte._util.api_util import get_web_url_root from airbyte.cloud.connections import CloudConnection -from airbyte.cloud.connectors import CloudDestination, CloudSource +from airbyte.cloud.connectors import ( + CloudCustomDestinationDefinition, + CloudCustomSourceDefinition, + CloudDestination, + CloudSource, +) from airbyte.destinations.base import Destination from airbyte.secrets.base import SecretString @@ -454,310 +459,341 @@ def list_destinations( if name is None or destination.name == name ] - def publish_custom_yaml_source( + def publish_custom_source_definition( self, name: str, - manifest: dict[str, Any] | Path | str, *, + manifest_yaml: dict[str, Any] | Path | str | None = None, + docker_image: str | None = None, + docker_tag: str | None = None, + documentation_url: str | None = None, unique: bool = True, pre_validate: bool = True, - ) -> dict[str, Any]: - """Publish a custom YAML source definition to the workspace.""" - manifest_dict: dict[str, Any] - if isinstance(manifest, Path): - manifest_dict = yaml.safe_load(manifest.read_text()) - elif isinstance(manifest, str): - manifest_dict = yaml.safe_load(manifest) - else: - manifest_dict = manifest + ) -> CloudCustomSourceDefinition: + """Publish a custom source connector definition. + + You must specify EITHER manifest_yaml (for YAML connectors) OR both docker_image + and docker_tag (for Docker connectors), but not both. + + Args: + name: Display name for the connector definition + manifest_yaml: Low-code CDK manifest (dict, Path to YAML file, or YAML string) + docker_image: Docker repository (e.g., 'airbyte/source-custom') + docker_tag: Docker image tag (e.g., '1.0.0') + documentation_url: Optional URL to connector documentation (Docker only) + unique: Whether to enforce name uniqueness + pre_validate: Whether to validate manifest client-side (YAML only) + + Returns: + CloudCustomSourceDefinition object representing the created definition + + Raises: + PyAirbyteInputError: If both or neither of manifest_yaml and docker_image provided + AirbyteDuplicateResourcesError: If unique=True and name already exists + """ + is_yaml = manifest_yaml is not None + is_docker = docker_image is not None + + if is_yaml == is_docker: + raise exc.PyAirbyteInputError( + message=( + "Must specify EITHER manifest_yaml (for YAML connectors) OR " + "docker_image + docker_tag (for Docker connectors), but not both" + ), + context={ + "manifest_yaml_provided": is_yaml, + "docker_image_provided": is_docker, + }, + ) - if pre_validate: - api_util.validate_yaml_manifest(manifest_dict, raise_on_error=True) + if is_docker and docker_tag is None: + raise exc.PyAirbyteInputError( + message="docker_tag is required when docker_image is specified", + context={"docker_image": docker_image}, + ) if unique: - existing = self.list_custom_yaml_sources(name=name) + existing = self.list_custom_source_definitions( + name=name, + custom_connector_type="yaml" if is_yaml else "docker", + ) if existing: raise exc.AirbyteDuplicateResourcesError( - resource_type="custom_yaml_source_definition", + resource_type="custom_source_definition", resource_name=name, ) - result = api_util.create_custom_yaml_source_definition( + if is_yaml: + manifest_dict: dict[str, Any] + if isinstance(manifest_yaml, Path): + manifest_dict = yaml.safe_load(manifest_yaml.read_text()) + elif isinstance(manifest_yaml, str): + manifest_dict = yaml.safe_load(manifest_yaml) + elif manifest_yaml is not None: + manifest_dict = manifest_yaml + else: + raise exc.PyAirbyteInputError( + message="manifest_yaml is required for YAML connectors", + context={"name": name}, + ) + + if pre_validate: + api_util.validate_yaml_manifest(manifest_dict, raise_on_error=True) + + result = api_util.create_custom_yaml_source_definition( + name=name, + workspace_id=self.workspace_id, + manifest=manifest_dict, + api_root=self.api_root, + client_id=self.client_id, + client_secret=self.client_secret, + ) + return CloudCustomSourceDefinition._from_yaml_response(self, result) # noqa: SLF001 + result = api_util.create_custom_docker_source_definition( name=name, + docker_repository=docker_image, # type: ignore[arg-type] + docker_image_tag=docker_tag, # type: ignore[arg-type] workspace_id=self.workspace_id, - manifest=manifest_dict, + documentation_url=documentation_url, api_root=self.api_root, client_id=self.client_id, client_secret=self.client_secret, ) + return CloudCustomSourceDefinition._from_docker_response(self, result) # noqa: SLF001 - return { - "id": result.id, - "name": result.name, - "manifest": result.manifest, - "version": result.version, - } - - def list_custom_yaml_sources( + def list_custom_source_definitions( self, + *, name: str | None = None, - ) -> list[dict[str, Any]]: - """List all custom YAML source definitions in the workspace.""" - definitions = api_util.list_custom_yaml_source_definitions( - workspace_id=self.workspace_id, - api_root=self.api_root, - client_id=self.client_id, - client_secret=self.client_secret, - ) + custom_connector_type: Literal["yaml", "docker"] | None = None, + ) -> list[CloudCustomSourceDefinition]: + """List custom source connector definitions. - result = [ - { - "id": d.id, - "name": d.name, - "manifest": d.manifest, - "version": d.version, - } - for d in definitions - ] + Args: + name: Filter by exact name match + custom_connector_type: Filter by connector type ("yaml" or "docker") - if name: - result = [d for d in result if d["name"] == name] + Returns: + List of CloudCustomSourceDefinition objects + """ + result: list[CloudCustomSourceDefinition] = [] + + if custom_connector_type is None or custom_connector_type == "yaml": + yaml_definitions = api_util.list_custom_yaml_source_definitions( + workspace_id=self.workspace_id, + api_root=self.api_root, + client_id=self.client_id, + client_secret=self.client_secret, + ) + result.extend( + CloudCustomSourceDefinition._from_yaml_response(self, d) # noqa: SLF001 + for d in yaml_definitions + if name is None or d.name == name + ) + + if custom_connector_type is None or custom_connector_type == "docker": + docker_definitions = api_util.list_custom_docker_source_definitions( + workspace_id=self.workspace_id, + api_root=self.api_root, + client_id=self.client_id, + client_secret=self.client_secret, + ) + result.extend( + CloudCustomSourceDefinition._from_docker_response(self, d) # noqa: SLF001 + for d in docker_definitions + if name is None or d.name == name + ) return result - def get_custom_yaml_source( - self, - definition_id: str, - ) -> dict[str, Any]: - """Get a specific custom YAML source definition by ID.""" - result = api_util.get_custom_yaml_source_definition( - workspace_id=self.workspace_id, - definition_id=definition_id, - api_root=self.api_root, - client_id=self.client_id, - client_secret=self.client_secret, - ) - return { - "id": result.id, - "name": result.name, - "manifest": result.manifest, - "version": result.version, - } - - def update_custom_yaml_source( + def get_custom_source_definition( self, definition_id: str, - *, - manifest: dict[str, Any] | Path | str, - pre_validate: bool = True, - ) -> dict[str, Any]: - """Update a custom YAML source definition.""" - manifest_dict: dict[str, Any] - if isinstance(manifest, Path): - manifest_dict = yaml.safe_load(manifest.read_text()) - elif isinstance(manifest, str): - manifest_dict = yaml.safe_load(manifest) - else: - manifest_dict = manifest + ) -> CloudCustomSourceDefinition: + """Get a specific custom source definition by ID. - if pre_validate: - api_util.validate_yaml_manifest(manifest_dict, raise_on_error=True) + This method will attempt to fetch as a YAML definition first, then as a Docker + definition if not found. - result = api_util.update_custom_yaml_source_definition( - workspace_id=self.workspace_id, - definition_id=definition_id, - manifest=manifest_dict, - api_root=self.api_root, - client_id=self.client_id, - client_secret=self.client_secret, - ) + Args: + definition_id: The definition ID - return { - "id": result.id, - "name": result.name, - "manifest": result.manifest, - "version": result.version, - } + Returns: + CloudCustomSourceDefinition object + """ + try: + result = api_util.get_custom_yaml_source_definition( + workspace_id=self.workspace_id, + definition_id=definition_id, + api_root=self.api_root, + client_id=self.client_id, + client_secret=self.client_secret, + ) + return CloudCustomSourceDefinition._from_yaml_response(self, result) # noqa: SLF001 + except Exception: + result = api_util.get_custom_docker_source_definition( + workspace_id=self.workspace_id, + definition_id=definition_id, + api_root=self.api_root, + client_id=self.client_id, + client_secret=self.client_secret, + ) + return CloudCustomSourceDefinition._from_docker_response(self, result) # noqa: SLF001 - def delete_custom_yaml_source( + def update_custom_source_definition( self, definition_id: str, - ) -> None: - """Delete a custom YAML source definition.""" - api_util.delete_custom_yaml_source_definition( - workspace_id=self.workspace_id, - definition_id=definition_id, - api_root=self.api_root, - client_id=self.client_id, - client_secret=self.client_secret, - ) - - def publish_custom_docker_source( - self, - name: str, - docker_repository: str, - docker_image_tag: str, *, - documentation_url: str | None = None, - unique: bool = True, - ) -> dict[str, Any]: - """Publish a custom Docker source definition to the workspace.""" - if unique: - existing = self.list_custom_docker_sources(name=name) - if existing: - raise exc.AirbyteDuplicateResourcesError( - resource_type="custom_docker_source_definition", - resource_name=name, - ) - - result = api_util.create_custom_docker_source_definition( - name=name, - docker_repository=docker_repository, - docker_image_tag=docker_image_tag, - workspace_id=self.workspace_id, - documentation_url=documentation_url, - api_root=self.api_root, - client_id=self.client_id, - client_secret=self.client_secret, - ) - - return { - "id": result.id, - "name": result.name, - "docker_repository": result.docker_repository, - "docker_image_tag": result.docker_image_tag, - "documentation_url": result.documentation_url, - } - - def list_custom_docker_sources( - self, name: str | None = None, - ) -> list[dict[str, Any]]: - """List all custom Docker source definitions in the workspace.""" - definitions = api_util.list_custom_docker_source_definitions( - workspace_id=self.workspace_id, - api_root=self.api_root, - client_id=self.client_id, - client_secret=self.client_secret, - ) + manifest_yaml: dict[str, Any] | Path | str | None = None, + docker_tag: str | None = None, + pre_validate: bool = True, + ) -> CloudCustomSourceDefinition: + """Update a custom source definition. - result = [ - { - "id": d.id, - "name": d.name, - "docker_repository": d.docker_repository, - "docker_image_tag": d.docker_image_tag, - "documentation_url": d.documentation_url, - } - for d in definitions - ] + For YAML connectors: can update manifest_yaml + For Docker connectors: can update name and/or docker_tag - if name: - result = [d for d in result if d["name"] == name] + Args: + definition_id: The definition ID to update + name: New display name (Docker connectors only) + manifest_yaml: New manifest (YAML connectors only) + docker_tag: New Docker tag (Docker connectors only) + pre_validate: Whether to validate manifest (YAML only) + + Returns: + Updated CloudCustomSourceDefinition object + """ + definition = self.get_custom_source_definition(definition_id) - return result + if definition.connector_type == "yaml": + if manifest_yaml is None: + raise exc.PyAirbyteInputError( + message="manifest_yaml is required for updating YAML connectors", + context={"definition_id": definition_id}, + ) + + manifest_dict: dict[str, Any] + if isinstance(manifest_yaml, Path): + manifest_dict = yaml.safe_load(manifest_yaml.read_text()) + elif isinstance(manifest_yaml, str): + manifest_dict = yaml.safe_load(manifest_yaml) + else: + manifest_dict = manifest_yaml + + if pre_validate: + api_util.validate_yaml_manifest(manifest_dict, raise_on_error=True) + + result = api_util.update_custom_yaml_source_definition( + workspace_id=self.workspace_id, + definition_id=definition_id, + manifest=manifest_dict, + api_root=self.api_root, + client_id=self.client_id, + client_secret=self.client_secret, + ) + return CloudCustomSourceDefinition._from_yaml_response(self, result) # noqa: SLF001 + if name is None or docker_tag is None: + raise exc.PyAirbyteInputError( + message="Both name and docker_tag are required for updating Docker connectors", + context={"definition_id": definition_id}, + ) - def get_custom_docker_source( - self, - definition_id: str, - ) -> dict[str, Any]: - """Get a specific custom Docker source definition by ID.""" - result = api_util.get_custom_docker_source_definition( - workspace_id=self.workspace_id, - definition_id=definition_id, - api_root=self.api_root, - client_id=self.client_id, - client_secret=self.client_secret, - ) - return { - "id": result.id, - "name": result.name, - "docker_repository": result.docker_repository, - "docker_image_tag": result.docker_image_tag, - "documentation_url": result.documentation_url, - } - - def update_custom_docker_source( - self, - definition_id: str, - *, - name: str, - docker_image_tag: str, - ) -> dict[str, Any]: - """Update a custom Docker source definition.""" result = api_util.update_custom_docker_source_definition( workspace_id=self.workspace_id, definition_id=definition_id, name=name, - docker_image_tag=docker_image_tag, + docker_image_tag=docker_tag, api_root=self.api_root, client_id=self.client_id, client_secret=self.client_secret, ) + return CloudCustomSourceDefinition._from_docker_response(self, result) # noqa: SLF001 - return { - "id": result.id, - "name": result.name, - "docker_repository": result.docker_repository, - "docker_image_tag": result.docker_image_tag, - "documentation_url": result.documentation_url, - } - - def delete_custom_docker_source( + def permanently_delete_custom_source_definition( self, definition_id: str, ) -> None: - """Delete a custom Docker source definition.""" - api_util.delete_custom_docker_source_definition( - workspace_id=self.workspace_id, - definition_id=definition_id, - api_root=self.api_root, - client_id=self.client_id, - client_secret=self.client_secret, - ) + """Permanently delete a custom source definition. - def publish_custom_docker_destination( + Args: + definition_id: The definition ID to delete + """ + definition = self.get_custom_source_definition(definition_id) + + if definition.connector_type == "yaml": + api_util.delete_custom_yaml_source_definition( + workspace_id=self.workspace_id, + definition_id=definition_id, + api_root=self.api_root, + client_id=self.client_id, + client_secret=self.client_secret, + ) + else: + api_util.delete_custom_docker_source_definition( + workspace_id=self.workspace_id, + definition_id=definition_id, + api_root=self.api_root, + client_id=self.client_id, + client_secret=self.client_secret, + ) + + def publish_custom_destination_definition( self, name: str, - docker_repository: str, - docker_image_tag: str, *, + docker_image: str, + docker_tag: str, documentation_url: str | None = None, unique: bool = True, - ) -> dict[str, Any]: - """Publish a custom Docker destination definition to the workspace.""" + ) -> CloudCustomDestinationDefinition: + """Publish a custom destination connector definition. + + Currently only Docker-based destinations are supported. + + Args: + name: Display name for the connector definition + docker_image: Docker repository (e.g., 'airbyte/destination-custom') + docker_tag: Docker image tag (e.g., '1.0.0') + documentation_url: Optional URL to connector documentation + unique: Whether to enforce name uniqueness + + Returns: + CloudCustomDestinationDefinition object representing the created definition + """ if unique: - existing = self.list_custom_docker_destinations(name=name) + existing = self.list_custom_destination_definitions(name=name) if existing: raise exc.AirbyteDuplicateResourcesError( - resource_type="custom_docker_destination_definition", + resource_type="custom_destination_definition", resource_name=name, ) result = api_util.create_custom_docker_destination_definition( name=name, - docker_repository=docker_repository, - docker_image_tag=docker_image_tag, + docker_repository=docker_image, + docker_image_tag=docker_tag, workspace_id=self.workspace_id, documentation_url=documentation_url, api_root=self.api_root, client_id=self.client_id, client_secret=self.client_secret, ) + return CloudCustomDestinationDefinition._from_docker_response(self, result) # noqa: SLF001 - return { - "id": result.id, - "name": result.name, - "docker_repository": result.docker_repository, - "docker_image_tag": result.docker_image_tag, - "documentation_url": result.documentation_url, - } - - def list_custom_docker_destinations( + def list_custom_destination_definitions( self, + *, name: str | None = None, - ) -> list[dict[str, Any]]: - """List all custom Docker destination definitions in the workspace.""" + ) -> list[CloudCustomDestinationDefinition]: + """List custom destination connector definitions. + + Args: + name: Filter by exact name match + + Returns: + List of CloudCustomDestinationDefinition objects + """ definitions = api_util.list_custom_docker_destination_definitions( workspace_id=self.workspace_id, api_root=self.api_root, @@ -765,27 +801,24 @@ def list_custom_docker_destinations( client_secret=self.client_secret, ) - result = [ - { - "id": d.id, - "name": d.name, - "docker_repository": d.docker_repository, - "docker_image_tag": d.docker_image_tag, - "documentation_url": d.documentation_url, - } + return [ + CloudCustomDestinationDefinition._from_docker_response(self, d) # noqa: SLF001 for d in definitions + if name is None or d.name == name ] - if name: - result = [d for d in result if d["name"] == name] - - return result - - def get_custom_docker_destination( + def get_custom_destination_definition( self, definition_id: str, - ) -> dict[str, Any]: - """Get a specific custom Docker destination definition by ID.""" + ) -> CloudCustomDestinationDefinition: + """Get a specific custom destination definition by ID. + + Args: + definition_id: The definition ID + + Returns: + CloudCustomDestinationDefinition object + """ result = api_util.get_custom_docker_destination_definition( workspace_id=self.workspace_id, definition_id=definition_id, @@ -793,45 +826,45 @@ def get_custom_docker_destination( client_id=self.client_id, client_secret=self.client_secret, ) - return { - "id": result.id, - "name": result.name, - "docker_repository": result.docker_repository, - "docker_image_tag": result.docker_image_tag, - "documentation_url": result.documentation_url, - } - - def update_custom_docker_destination( + return CloudCustomDestinationDefinition._from_docker_response(self, result) # noqa: SLF001 + + def update_custom_destination_definition( self, definition_id: str, *, name: str, - docker_image_tag: str, - ) -> dict[str, Any]: - """Update a custom Docker destination definition.""" + docker_tag: str, + ) -> CloudCustomDestinationDefinition: + """Update a custom destination definition. + + Args: + definition_id: The definition ID to update + name: New display name + docker_tag: New Docker tag + + Returns: + Updated CloudCustomDestinationDefinition object + """ result = api_util.update_custom_docker_destination_definition( workspace_id=self.workspace_id, definition_id=definition_id, name=name, - docker_image_tag=docker_image_tag, + docker_image_tag=docker_tag, api_root=self.api_root, client_id=self.client_id, client_secret=self.client_secret, ) + return CloudCustomDestinationDefinition._from_docker_response(self, result) # noqa: SLF001 - return { - "id": result.id, - "name": result.name, - "docker_repository": result.docker_repository, - "docker_image_tag": result.docker_image_tag, - "documentation_url": result.documentation_url, - } - - def delete_custom_docker_destination( + def permanently_delete_custom_destination_definition( self, definition_id: str, ) -> None: - """Delete a custom Docker destination definition.""" + """Permanently delete a custom destination definition. + + Args: + definition_id: The definition ID to delete + """ api_util.delete_custom_docker_destination_definition( workspace_id=self.workspace_id, definition_id=definition_id, diff --git a/airbyte/mcp/cloud_ops.py b/airbyte/mcp/cloud_ops.py index b1d37174..4a24acf2 100644 --- a/airbyte/mcp/cloud_ops.py +++ b/airbyte/mcp/cloud_ops.py @@ -501,21 +501,48 @@ def list_deployed_cloud_connections() -> list[CloudConnection]: return workspace.list_connections() -def publish_custom_yaml_source_definition( +def publish_custom_source_definition( name: Annotated[ str, Field(description="The name for the custom connector definition."), ], - manifest: Annotated[ - dict | str, + *, + manifest_yaml: Annotated[ + dict | str | None, Field( description=( "The Low-code CDK manifest as a dict or YAML string. " - "Must be a valid declarative YAML connector manifest." + "Required for YAML connectors. Mutually exclusive with docker_image." ), + default=None, ), - ], - *, + ] = None, + docker_image: Annotated[ + str | None, + Field( + description=( + "Docker repository (e.g., 'airbyte/source-custom'). " + "Required for Docker connectors." + ), + default=None, + ), + ] = None, + docker_tag: Annotated[ + str | None, + Field( + description=( + "Docker image tag (e.g., '1.0.0'). " "Required when docker_image is specified." + ), + default=None, + ), + ] = None, + documentation_url: Annotated[ + str | None, + Field( + description="Optional URL to connector documentation (Docker only).", + default=None, + ), + ] = None, unique: Annotated[ bool, Field( @@ -526,167 +553,152 @@ def publish_custom_yaml_source_definition( pre_validate: Annotated[ bool, Field( - description="Whether to validate the manifest client-side before publishing.", + description=( + "Whether to validate the manifest client-side " "before publishing (YAML only)." + ), default=True, ), ] = True, ) -> str: - """Publish a custom YAML source connector definition to Airbyte Cloud.""" + """Publish a custom source connector definition to Airbyte Cloud. + + Supports both YAML (declarative) and Docker-based custom source definitions. + You must specify EITHER manifest_yaml OR docker_image + docker_tag, but not both. + """ try: workspace: CloudWorkspace = _get_cloud_workspace() - result = workspace.publish_custom_yaml_source( + result = workspace.publish_custom_source_definition( name=name, - manifest=manifest, + manifest_yaml=manifest_yaml, + docker_image=docker_image, + docker_tag=docker_tag, + documentation_url=documentation_url, unique=unique, pre_validate=pre_validate, ) except Exception as ex: - return f"Failed to publish custom YAML source definition '{name}': {ex}" + return f"Failed to publish custom source definition '{name}': {ex}" else: + if result.connector_type == "yaml": + return ( + f"Successfully published custom YAML source definition '{name}' " + f"with ID '{result.definition_id}' (version {result.version or 'N/A'})" + ) return ( - f"Successfully published custom YAML source definition '{name}' " - f"with ID '{result['id']}' (version {result.get('version', 'N/A')})" + f"Successfully published custom Docker source definition '{name}' " + f"with ID '{result.definition_id}' " + f"({result.docker_repository}:{result.docker_image_tag})" ) -def list_custom_yaml_source_definitions() -> list[dict[str, Any]]: - """List all custom YAML source definitions in the Airbyte Cloud workspace.""" +def list_custom_source_definitions( + custom_connector_type: Annotated[ + str | None, + Field( + description=( + "Filter by connector type: 'yaml' or 'docker'. " "If not specified, returns all." + ), + default=None, + ), + ] = None, +) -> list[dict[str, Any]]: + """List all custom source definitions in the Airbyte Cloud workspace. + + Returns both YAML and Docker source definitions unless filtered by type. + """ workspace: CloudWorkspace = _get_cloud_workspace() - return workspace.list_custom_yaml_sources() + definitions = workspace.list_custom_source_definitions( + custom_connector_type=custom_connector_type, # type: ignore[arg-type] + ) + + return [ + { + "definition_id": d.definition_id, + "name": d.name, + "connector_type": d.connector_type, + "manifest": d.manifest, + "version": d.version, + "docker_repository": d.docker_repository, + "docker_image_tag": d.docker_image_tag, + "documentation_url": d.documentation_url, + } + for d in definitions + ] -def update_custom_yaml_source_definition( +def update_custom_source_definition( definition_id: Annotated[ str, Field(description="The ID of the definition to update."), ], - manifest: Annotated[ - dict | str, + *, + name: Annotated[ + str | None, Field( - description="New manifest as dict or YAML string.", + description="New name for the definition (Docker connectors only).", + default=None, ), - ], - *, - pre_validate: Annotated[ - bool, + ] = None, + manifest_yaml: Annotated[ + dict | str | None, Field( - description="Whether to validate the manifest client-side before updating.", - default=True, + description="New manifest as dict or YAML string (YAML connectors only).", + default=None, ), - ] = True, -) -> str: - """Update a custom YAML source definition in Airbyte Cloud.""" - try: - workspace: CloudWorkspace = _get_cloud_workspace() - result = workspace.update_custom_yaml_source( - definition_id=definition_id, - manifest=manifest, - pre_validate=pre_validate, - ) - except Exception as ex: - return f"Failed to update custom YAML source definition '{definition_id}': {ex}" - else: - return ( - f"Successfully updated custom YAML source definition. " - f"New name: {result.get('name')}, version: {result.get('version', 'N/A')}" - ) - - -def publish_custom_docker_source_definition( - name: Annotated[ - str, - Field(description="The name for the custom connector definition."), - ], - docker_repository: Annotated[ - str, - Field(description="Docker repository (e.g., 'airbyte/source-custom')."), - ], - docker_image_tag: Annotated[ - str, - Field(description="Docker image tag (e.g., '1.0.0')."), - ], - *, - documentation_url: Annotated[ + ] = None, + docker_tag: Annotated[ str | None, Field( - description="Optional URL to connector documentation.", + description="New Docker image tag (Docker connectors only).", default=None, ), ] = None, - unique: Annotated[ + pre_validate: Annotated[ bool, Field( - description="Whether to require a unique name.", + description="Whether to validate the manifest client-side before updating (YAML only).", default=True, ), ] = True, ) -> str: - """Publish a custom Docker source connector definition to Airbyte Cloud.""" - try: - workspace: CloudWorkspace = _get_cloud_workspace() - result = workspace.publish_custom_docker_source( - name=name, - docker_repository=docker_repository, - docker_image_tag=docker_image_tag, - documentation_url=documentation_url, - unique=unique, - ) - except Exception as ex: - return f"Failed to publish custom Docker source definition '{name}': {ex}" - else: - return ( - f"Successfully published custom Docker source definition '{name}' " - f"with ID '{result['id']}' ({result['docker_repository']}:{result['docker_image_tag']})" - ) - + """Update a custom source definition in Airbyte Cloud. -def list_custom_docker_source_definitions() -> list[dict[str, Any]]: - """List all custom Docker source definitions in the Airbyte Cloud workspace.""" - workspace: CloudWorkspace = _get_cloud_workspace() - return workspace.list_custom_docker_sources() - - -def update_custom_docker_source_definition( - definition_id: Annotated[ - str, - Field(description="The ID of the definition to update."), - ], - name: Annotated[ - str, - Field(description="New name for the definition."), - ], - docker_image_tag: Annotated[ - str, - Field(description="New Docker image tag."), - ], -) -> str: - """Update a custom Docker source definition in Airbyte Cloud.""" + For YAML connectors: specify manifest_yaml + For Docker connectors: specify name and/or docker_tag + """ try: workspace: CloudWorkspace = _get_cloud_workspace() - result = workspace.update_custom_docker_source( + result = workspace.update_custom_source_definition( definition_id=definition_id, name=name, - docker_image_tag=docker_image_tag, + manifest_yaml=manifest_yaml, + docker_tag=docker_tag, + pre_validate=pre_validate, ) except Exception as ex: - return f"Failed to update custom Docker source definition '{definition_id}': {ex}" + return f"Failed to update custom source definition '{definition_id}': {ex}" else: + if result.connector_type == "yaml": + return ( + f"Successfully updated custom YAML source definition. " + f"Name: {result.name}, version: {result.version or 'N/A'}" + ) return ( f"Successfully updated custom Docker source definition. " - f"New name: {result.get('name')}, tag: {result.get('docker_image_tag')}" + f"Name: {result.name}, tag: {result.docker_image_tag}" ) -def publish_custom_docker_destination_definition( +def publish_custom_destination_definition( name: Annotated[ str, Field(description="The name for the custom connector definition."), ], - docker_repository: Annotated[ + docker_image: Annotated[ str, Field(description="Docker repository (e.g., 'airbyte/destination-custom')."), ], - docker_image_tag: Annotated[ + docker_tag: Annotated[ str, Field(description="Docker image tag (e.g., '1.0.0')."), ], @@ -706,32 +718,47 @@ def publish_custom_docker_destination_definition( ), ] = True, ) -> str: - """Publish a custom Docker destination connector definition to Airbyte Cloud.""" + """Publish a custom destination connector definition to Airbyte Cloud. + + Currently only Docker-based custom destinations are supported. + """ try: workspace: CloudWorkspace = _get_cloud_workspace() - result = workspace.publish_custom_docker_destination( + result = workspace.publish_custom_destination_definition( name=name, - docker_repository=docker_repository, - docker_image_tag=docker_image_tag, + docker_image=docker_image, + docker_tag=docker_tag, documentation_url=documentation_url, unique=unique, ) except Exception as ex: - return f"Failed to publish custom Docker destination definition '{name}': {ex}" + return f"Failed to publish custom destination definition '{name}': {ex}" else: return ( f"Successfully published custom Docker destination definition '{name}' " - f"with ID '{result['id']}' ({result['docker_repository']}:{result['docker_image_tag']})" + f"with ID '{result.definition_id}' " + f"({result.docker_repository}:{result.docker_image_tag})" ) -def list_custom_docker_destination_definitions() -> list[dict[str, Any]]: - """List all custom Docker destination definitions in the Airbyte Cloud workspace.""" +def list_custom_destination_definitions() -> list[dict[str, Any]]: + """List all custom destination definitions in the Airbyte Cloud workspace.""" workspace: CloudWorkspace = _get_cloud_workspace() - return workspace.list_custom_docker_destinations() + definitions = workspace.list_custom_destination_definitions() + + return [ + { + "definition_id": d.definition_id, + "name": d.name, + "docker_repository": d.docker_repository, + "docker_image_tag": d.docker_image_tag, + "documentation_url": d.documentation_url, + } + for d in definitions + ] -def update_custom_docker_destination_definition( +def update_custom_destination_definition( definition_id: Annotated[ str, Field(description="The ID of the definition to update."), @@ -740,25 +767,25 @@ def update_custom_docker_destination_definition( str, Field(description="New name for the definition."), ], - docker_image_tag: Annotated[ + docker_tag: Annotated[ str, Field(description="New Docker image tag."), ], ) -> str: - """Update a custom Docker destination definition in Airbyte Cloud.""" + """Update a custom destination definition in Airbyte Cloud.""" try: workspace: CloudWorkspace = _get_cloud_workspace() - result = workspace.update_custom_docker_destination( + result = workspace.update_custom_destination_definition( definition_id=definition_id, name=name, - docker_image_tag=docker_image_tag, + docker_tag=docker_tag, ) except Exception as ex: - return f"Failed to update custom Docker destination definition '{definition_id}': {ex}" + return f"Failed to update custom destination definition '{definition_id}': {ex}" else: return ( f"Successfully updated custom Docker destination definition. " - f"New name: {result.get('name')}, tag: {result.get('docker_image_tag')}" + f"Name: {result.name}, tag: {result.docker_image_tag}" ) @@ -778,12 +805,9 @@ def register_cloud_ops_tools(app: FastMCP) -> None: app.tool(list_deployed_cloud_source_connectors) app.tool(list_deployed_cloud_destination_connectors) app.tool(list_deployed_cloud_connections) - app.tool(publish_custom_yaml_source_definition) - app.tool(list_custom_yaml_source_definitions) - app.tool(update_custom_yaml_source_definition) - app.tool(publish_custom_docker_source_definition) - app.tool(list_custom_docker_source_definitions) - app.tool(update_custom_docker_source_definition) - app.tool(publish_custom_docker_destination_definition) - app.tool(list_custom_docker_destination_definitions) - app.tool(update_custom_docker_destination_definition) + app.tool(publish_custom_source_definition) + app.tool(list_custom_source_definitions) + app.tool(update_custom_source_definition) + app.tool(publish_custom_destination_definition) + app.tool(list_custom_destination_definitions) + app.tool(update_custom_destination_definition) diff --git a/tests/integration_tests/cloud/test_custom_definitions.py b/tests/integration_tests/cloud/test_custom_definitions.py index 20832135..28170d34 100644 --- a/tests/integration_tests/cloud/test_custom_definitions.py +++ b/tests/integration_tests/cloud/test_custom_definitions.py @@ -41,39 +41,44 @@ def test_publish_custom_yaml_source( name = f"test-yaml-source-{text_util.generate_random_suffix()}" - result = cloud_workspace.publish_custom_yaml_source( + result = cloud_workspace.publish_custom_source_definition( name=name, - manifest=TEST_YAML_MANIFEST, + manifest_yaml=TEST_YAML_MANIFEST, unique=True, pre_validate=True, ) - assert "id" in result - assert result["name"] == name - assert "manifest" in result - assert "version" in result + assert result.definition_id + assert result.name == name + assert result.manifest is not None + assert result.version is not None + assert result.connector_type == "yaml" - definition_id = result["id"] + definition_id = result.definition_id try: - definitions = cloud_workspace.list_custom_yaml_sources(name=name) + definitions = cloud_workspace.list_custom_source_definitions( + name=name, + custom_connector_type="yaml", + ) assert len(definitions) == 1 - assert definitions[0]["id"] == definition_id + assert definitions[0].definition_id == definition_id - fetched = cloud_workspace.get_custom_yaml_source(definition_id) - assert fetched["id"] == definition_id - assert fetched["name"] == name + fetched = cloud_workspace.get_custom_source_definition(definition_id) + assert fetched.definition_id == definition_id + assert fetched.name == name + assert fetched.connector_type == "yaml" updated_manifest = TEST_YAML_MANIFEST.copy() updated_manifest["version"] = "0.2.0" - updated = cloud_workspace.update_custom_yaml_source( + updated = cloud_workspace.update_custom_source_definition( definition_id=definition_id, - manifest=updated_manifest, + manifest_yaml=updated_manifest, ) - assert updated["manifest"]["version"] == "0.2.0" + assert updated.manifest["version"] == "0.2.0" finally: - cloud_workspace.delete_custom_yaml_source(definition_id) + cloud_workspace.permanently_delete_custom_source_definition(definition_id) @pytest.mark.requires_creds @@ -85,39 +90,44 @@ def test_publish_custom_docker_source( name = f"test-docker-source-{text_util.generate_random_suffix()}" - result = cloud_workspace.publish_custom_docker_source( + result = cloud_workspace.publish_custom_source_definition( name=name, - docker_repository="airbyte/test-source", - docker_image_tag="1.0.0", + docker_image="airbyte/test-source", + docker_tag="1.0.0", documentation_url="https://example.com/docs", unique=True, ) - assert "id" in result - assert result["name"] == name - assert result["docker_repository"] == "airbyte/test-source" - assert result["docker_image_tag"] == "1.0.0" + assert result.definition_id + assert result.name == name + assert result.docker_repository == "airbyte/test-source" + assert result.docker_image_tag == "1.0.0" + assert result.connector_type == "docker" - definition_id = result["id"] + definition_id = result.definition_id try: - definitions = cloud_workspace.list_custom_docker_sources(name=name) + definitions = cloud_workspace.list_custom_source_definitions( + name=name, + custom_connector_type="docker", + ) assert len(definitions) == 1 - assert definitions[0]["id"] == definition_id + assert definitions[0].definition_id == definition_id - fetched = cloud_workspace.get_custom_docker_source(definition_id) - assert fetched["id"] == definition_id - assert fetched["name"] == name + fetched = cloud_workspace.get_custom_source_definition(definition_id) + assert fetched.definition_id == definition_id + assert fetched.name == name + assert fetched.connector_type == "docker" - updated = cloud_workspace.update_custom_docker_source( + updated = cloud_workspace.update_custom_source_definition( definition_id=definition_id, name=name, - docker_image_tag="2.0.0", + docker_tag="2.0.0", ) - assert updated["docker_image_tag"] == "2.0.0" + assert updated.docker_image_tag == "2.0.0" finally: - cloud_workspace.delete_custom_docker_source(definition_id) + cloud_workspace.permanently_delete_custom_source_definition(definition_id) @pytest.mark.requires_creds @@ -129,38 +139,38 @@ def test_publish_custom_docker_destination( name = f"test-docker-dest-{text_util.generate_random_suffix()}" - result = cloud_workspace.publish_custom_docker_destination( + result = cloud_workspace.publish_custom_destination_definition( name=name, - docker_repository="airbyte/test-destination", - docker_image_tag="1.0.0", + docker_image="airbyte/test-destination", + docker_tag="1.0.0", unique=True, ) - assert "id" in result - assert result["name"] == name - assert result["docker_repository"] == "airbyte/test-destination" - assert result["docker_image_tag"] == "1.0.0" + assert result.definition_id + assert result.name == name + assert result.docker_repository == "airbyte/test-destination" + assert result.docker_image_tag == "1.0.0" - definition_id = result["id"] + definition_id = result.definition_id try: - definitions = cloud_workspace.list_custom_docker_destinations(name=name) + definitions = cloud_workspace.list_custom_destination_definitions(name=name) assert len(definitions) == 1 - assert definitions[0]["id"] == definition_id + assert definitions[0].definition_id == definition_id - fetched = cloud_workspace.get_custom_docker_destination(definition_id) - assert fetched["id"] == definition_id - assert fetched["name"] == name + fetched = cloud_workspace.get_custom_destination_definition(definition_id) + assert fetched.definition_id == definition_id + assert fetched.name == name - updated = cloud_workspace.update_custom_docker_destination( + updated = cloud_workspace.update_custom_destination_definition( definition_id=definition_id, name=name, - docker_image_tag="2.0.0", + docker_tag="2.0.0", ) - assert updated["docker_image_tag"] == "2.0.0" + assert updated.docker_image_tag == "2.0.0" finally: - cloud_workspace.delete_custom_docker_destination(definition_id) + cloud_workspace.permanently_delete_custom_destination_definition(definition_id) @pytest.mark.requires_creds @@ -175,9 +185,9 @@ def test_yaml_validation_error( invalid_manifest = {"version": "0.1.0"} with pytest.raises(PyAirbyteInputError) as exc_info: - cloud_workspace.publish_custom_yaml_source( + cloud_workspace.publish_custom_source_definition( name=name, - manifest=invalid_manifest, + manifest_yaml=invalid_manifest, pre_validate=True, ) From c80b09ccc79c71add604a57a71308c9ed5d33b08 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 2 Oct 2025 23:19:15 +0000 Subject: [PATCH 3/9] refactor: Remove redundant import aliases in api_util.py - Remove duplicate imports 'api as airbyte_api_api' and 'models as airbyte_api_models' - Update all references to use shorter import names (api, models) - Addresses PR feedback from @aaronsteers All references updated and verified with poe fix-and-check passing. Co-Authored-By: AJ Steers --- airbyte/_util/api_util.py | 68 +++++++++++++++++++-------------------- 1 file changed, 33 insertions(+), 35 deletions(-) diff --git a/airbyte/_util/api_util.py b/airbyte/_util/api_util.py index b37ed98f..de991721 100644 --- a/airbyte/_util/api_util.py +++ b/airbyte/_util/api_util.py @@ -20,8 +20,6 @@ import airbyte_api import requests from airbyte_api import api, models -from airbyte_api import api as airbyte_api_api -from airbyte_api import models as airbyte_api_models from airbyte.constants import CLOUD_API_ROOT, CLOUD_CONFIG_API_ROOT from airbyte.exceptions import ( @@ -984,7 +982,7 @@ def create_custom_yaml_source_definition( api_root: str, client_id: SecretString, client_secret: SecretString, -) -> airbyte_api_models.DeclarativeSourceDefinitionResponse: +) -> models.DeclarativeSourceDefinitionResponse: """Create a custom YAML source definition.""" airbyte_instance = get_airbyte_server_instance( api_root=api_root, @@ -992,11 +990,11 @@ def create_custom_yaml_source_definition( client_secret=client_secret, ) - request_body = airbyte_api_models.CreateDeclarativeSourceDefinitionRequest( + request_body = models.CreateDeclarativeSourceDefinitionRequest( name=name, manifest=manifest, ) - request = airbyte_api_api.CreateDeclarativeSourceDefinitionRequest( + request = api.CreateDeclarativeSourceDefinitionRequest( workspace_id=workspace_id, create_declarative_source_definition_request=request_body, ) @@ -1017,7 +1015,7 @@ def list_custom_yaml_source_definitions( api_root: str, client_id: SecretString, client_secret: SecretString, -) -> list[airbyte_api_models.DeclarativeSourceDefinitionResponse]: +) -> list[models.DeclarativeSourceDefinitionResponse]: """List all custom YAML source definitions in a workspace.""" airbyte_instance = get_airbyte_server_instance( api_root=api_root, @@ -1025,7 +1023,7 @@ def list_custom_yaml_source_definitions( client_secret=client_secret, ) - request = airbyte_api_api.ListDeclarativeSourceDefinitionsRequest( + request = api.ListDeclarativeSourceDefinitionsRequest( workspace_id=workspace_id, ) response = airbyte_instance.declarative_source_definitions.list_declarative_source_definitions( @@ -1046,7 +1044,7 @@ def get_custom_yaml_source_definition( api_root: str, client_id: SecretString, client_secret: SecretString, -) -> airbyte_api_models.DeclarativeSourceDefinitionResponse: +) -> models.DeclarativeSourceDefinitionResponse: """Get a specific custom YAML source definition.""" airbyte_instance = get_airbyte_server_instance( api_root=api_root, @@ -1054,7 +1052,7 @@ def get_custom_yaml_source_definition( client_secret=client_secret, ) - request = airbyte_api_api.GetDeclarativeSourceDefinitionRequest( + request = api.GetDeclarativeSourceDefinitionRequest( workspace_id=workspace_id, definition_id=definition_id, ) @@ -1077,7 +1075,7 @@ def update_custom_yaml_source_definition( api_root: str, client_id: SecretString, client_secret: SecretString, -) -> airbyte_api_models.DeclarativeSourceDefinitionResponse: +) -> models.DeclarativeSourceDefinitionResponse: """Update a custom YAML source definition.""" airbyte_instance = get_airbyte_server_instance( api_root=api_root, @@ -1085,10 +1083,10 @@ def update_custom_yaml_source_definition( client_secret=client_secret, ) - request_body = airbyte_api_models.UpdateDeclarativeSourceDefinitionRequest( + request_body = models.UpdateDeclarativeSourceDefinitionRequest( manifest=manifest, ) - request = airbyte_api_api.UpdateDeclarativeSourceDefinitionRequest( + request = api.UpdateDeclarativeSourceDefinitionRequest( workspace_id=workspace_id, definition_id=definition_id, update_declarative_source_definition_request=request_body, @@ -1119,7 +1117,7 @@ def delete_custom_yaml_source_definition( client_secret=client_secret, ) - request = airbyte_api_api.DeleteDeclarativeSourceDefinitionRequest( + request = api.DeleteDeclarativeSourceDefinitionRequest( workspace_id=workspace_id, definition_id=definition_id, ) @@ -1136,7 +1134,7 @@ def create_custom_docker_source_definition( api_root: str, client_id: SecretString, client_secret: SecretString, -) -> airbyte_api_models.DefinitionResponse: +) -> models.DefinitionResponse: """Create a custom Docker source definition.""" airbyte_instance = get_airbyte_server_instance( api_root=api_root, @@ -1144,13 +1142,13 @@ def create_custom_docker_source_definition( client_secret=client_secret, ) - request_body = airbyte_api_models.CreateDefinitionRequest( + request_body = models.CreateDefinitionRequest( name=name, docker_repository=docker_repository, docker_image_tag=docker_image_tag, documentation_url=documentation_url, ) - request = airbyte_api_api.CreateSourceDefinitionRequest( + request = api.CreateSourceDefinitionRequest( workspace_id=workspace_id, create_definition_request=request_body, ) @@ -1169,7 +1167,7 @@ def list_custom_docker_source_definitions( api_root: str, client_id: SecretString, client_secret: SecretString, -) -> list[airbyte_api_models.DefinitionResponse]: +) -> list[models.DefinitionResponse]: """List all custom Docker source definitions in a workspace.""" airbyte_instance = get_airbyte_server_instance( api_root=api_root, @@ -1177,7 +1175,7 @@ def list_custom_docker_source_definitions( client_secret=client_secret, ) - request = airbyte_api_api.ListSourceDefinitionsRequest( + request = api.ListSourceDefinitionsRequest( workspace_id=workspace_id, ) response = airbyte_instance.source_definitions.list_source_definitions(request) @@ -1196,7 +1194,7 @@ def get_custom_docker_source_definition( api_root: str, client_id: SecretString, client_secret: SecretString, -) -> airbyte_api_models.DefinitionResponse: +) -> models.DefinitionResponse: """Get a specific custom Docker source definition.""" airbyte_instance = get_airbyte_server_instance( api_root=api_root, @@ -1204,7 +1202,7 @@ def get_custom_docker_source_definition( client_secret=client_secret, ) - request = airbyte_api_api.GetSourceDefinitionRequest( + request = api.GetSourceDefinitionRequest( workspace_id=workspace_id, definition_id=definition_id, ) @@ -1226,7 +1224,7 @@ def update_custom_docker_source_definition( api_root: str, client_id: SecretString, client_secret: SecretString, -) -> airbyte_api_models.DefinitionResponse: +) -> models.DefinitionResponse: """Update a custom Docker source definition.""" airbyte_instance = get_airbyte_server_instance( api_root=api_root, @@ -1234,11 +1232,11 @@ def update_custom_docker_source_definition( client_secret=client_secret, ) - request_body = airbyte_api_models.UpdateDefinitionRequest( + request_body = models.UpdateDefinitionRequest( name=name, docker_image_tag=docker_image_tag, ) - request = airbyte_api_api.UpdateSourceDefinitionRequest( + request = api.UpdateSourceDefinitionRequest( workspace_id=workspace_id, definition_id=definition_id, update_definition_request=request_body, @@ -1267,7 +1265,7 @@ def delete_custom_docker_source_definition( client_secret=client_secret, ) - request = airbyte_api_api.DeleteSourceDefinitionRequest( + request = api.DeleteSourceDefinitionRequest( workspace_id=workspace_id, definition_id=definition_id, ) @@ -1284,7 +1282,7 @@ def create_custom_docker_destination_definition( api_root: str, client_id: SecretString, client_secret: SecretString, -) -> airbyte_api_models.DefinitionResponse: +) -> models.DefinitionResponse: """Create a custom Docker destination definition.""" airbyte_instance = get_airbyte_server_instance( api_root=api_root, @@ -1292,13 +1290,13 @@ def create_custom_docker_destination_definition( client_secret=client_secret, ) - request_body = airbyte_api_models.CreateDefinitionRequest( + request_body = models.CreateDefinitionRequest( name=name, docker_repository=docker_repository, docker_image_tag=docker_image_tag, documentation_url=documentation_url, ) - request = airbyte_api_api.CreateDestinationDefinitionRequest( + request = api.CreateDestinationDefinitionRequest( workspace_id=workspace_id, create_definition_request=request_body, ) @@ -1317,7 +1315,7 @@ def list_custom_docker_destination_definitions( api_root: str, client_id: SecretString, client_secret: SecretString, -) -> list[airbyte_api_models.DefinitionResponse]: +) -> list[models.DefinitionResponse]: """List all custom Docker destination definitions in a workspace.""" airbyte_instance = get_airbyte_server_instance( api_root=api_root, @@ -1325,7 +1323,7 @@ def list_custom_docker_destination_definitions( client_secret=client_secret, ) - request = airbyte_api_api.ListDestinationDefinitionsRequest( + request = api.ListDestinationDefinitionsRequest( workspace_id=workspace_id, ) response = airbyte_instance.destination_definitions.list_destination_definitions(request) @@ -1344,7 +1342,7 @@ def get_custom_docker_destination_definition( api_root: str, client_id: SecretString, client_secret: SecretString, -) -> airbyte_api_models.DefinitionResponse: +) -> models.DefinitionResponse: """Get a specific custom Docker destination definition.""" airbyte_instance = get_airbyte_server_instance( api_root=api_root, @@ -1352,7 +1350,7 @@ def get_custom_docker_destination_definition( client_secret=client_secret, ) - request = airbyte_api_api.GetDestinationDefinitionRequest( + request = api.GetDestinationDefinitionRequest( workspace_id=workspace_id, definition_id=definition_id, ) @@ -1374,7 +1372,7 @@ def update_custom_docker_destination_definition( api_root: str, client_id: SecretString, client_secret: SecretString, -) -> airbyte_api_models.DefinitionResponse: +) -> models.DefinitionResponse: """Update a custom Docker destination definition.""" airbyte_instance = get_airbyte_server_instance( api_root=api_root, @@ -1382,11 +1380,11 @@ def update_custom_docker_destination_definition( client_secret=client_secret, ) - request_body = airbyte_api_models.UpdateDefinitionRequest( + request_body = models.UpdateDefinitionRequest( name=name, docker_image_tag=docker_image_tag, ) - request = airbyte_api_api.UpdateDestinationDefinitionRequest( + request = api.UpdateDestinationDefinitionRequest( workspace_id=workspace_id, definition_id=definition_id, update_definition_request=request_body, @@ -1415,7 +1413,7 @@ def delete_custom_docker_destination_definition( client_secret=client_secret, ) - request = airbyte_api_api.DeleteDestinationDefinitionRequest( + request = api.DeleteDestinationDefinitionRequest( workspace_id=workspace_id, definition_id=definition_id, ) From 9e3652072c0e7e02c0befa741f7df9fa89e80682 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 2 Oct 2025 23:36:33 +0000 Subject: [PATCH 4/9] refactor: Require explicit connector type and decouple rename from update - Make custom_connector_type required (not optional) in list/get/delete methods - Add separate rename_custom_source_definition() method for Docker connectors - Refactor update_custom_source_definition() to determine type from parameters - Remove name parameter from update (use rename method instead) - Update all callers: CloudWorkspace methods, MCP tools, dataclass, tests - Add new rename_custom_source_definition MCP tool Addresses PR feedback from @aaronsteers: - Explicit type requirement prevents ambiguity between YAML/Docker domains - Separate rename method clarifies intent vs generic updates - Type determination from parameters simplifies update API API changes: - list_custom_source_definitions: custom_connector_type now required - get_custom_source_definition: custom_connector_type now required - permanently_delete_custom_source_definition: custom_connector_type now required - update_custom_source_definition: removed name parameter, determines type from manifest_yaml vs docker_tag - NEW: rename_custom_source_definition: Docker-only rename operation Breaking changes: All public methods now require explicit type parameter Co-Authored-By: AJ Steers --- airbyte/cloud/connectors.py | 5 +- airbyte/cloud/workspaces.py | 136 ++++++++++++------ airbyte/mcp/cloud_ops.py | 65 ++++++--- .../cloud/test_custom_definitions.py | 21 ++- 4 files changed, 164 insertions(+), 63 deletions(-) diff --git a/airbyte/cloud/connectors.py b/airbyte/cloud/connectors.py index 40d38eee..84673935 100644 --- a/airbyte/cloud/connectors.py +++ b/airbyte/cloud/connectors.py @@ -357,7 +357,10 @@ def definition_url(self) -> str: def permanently_delete(self) -> None: """Permanently delete this custom source definition.""" - self.workspace.permanently_delete_custom_source_definition(self.definition_id) + self.workspace.permanently_delete_custom_source_definition( + self.definition_id, + custom_connector_type=self.connector_type, + ) def __repr__(self) -> str: """String representation.""" diff --git a/airbyte/cloud/workspaces.py b/airbyte/cloud/workspaces.py index 281deaf5..34c92852 100644 --- a/airbyte/cloud/workspaces.py +++ b/airbyte/cloud/workspaces.py @@ -565,20 +565,20 @@ def list_custom_source_definitions( self, *, name: str | None = None, - custom_connector_type: Literal["yaml", "docker"] | None = None, + custom_connector_type: Literal["yaml", "docker"], ) -> list[CloudCustomSourceDefinition]: """List custom source connector definitions. Args: name: Filter by exact name match - custom_connector_type: Filter by connector type ("yaml" or "docker") + custom_connector_type: Connector type to list ("yaml" or "docker"). Required. Returns: - List of CloudCustomSourceDefinition objects + List of CloudCustomSourceDefinition objects matching the specified type """ result: list[CloudCustomSourceDefinition] = [] - if custom_connector_type is None or custom_connector_type == "yaml": + if custom_connector_type == "yaml": yaml_definitions = api_util.list_custom_yaml_source_definitions( workspace_id=self.workspace_id, api_root=self.api_root, @@ -590,8 +590,7 @@ def list_custom_source_definitions( for d in yaml_definitions if name is None or d.name == name ) - - if custom_connector_type is None or custom_connector_type == "docker": + elif custom_connector_type == "docker": docker_definitions = api_util.list_custom_docker_source_definitions( workspace_id=self.workspace_id, api_root=self.api_root, @@ -609,19 +608,19 @@ def list_custom_source_definitions( def get_custom_source_definition( self, definition_id: str, + *, + custom_connector_type: Literal["yaml", "docker"], ) -> CloudCustomSourceDefinition: """Get a specific custom source definition by ID. - This method will attempt to fetch as a YAML definition first, then as a Docker - definition if not found. - Args: definition_id: The definition ID + custom_connector_type: Connector type ("yaml" or "docker"). Required. Returns: CloudCustomSourceDefinition object """ - try: + if custom_connector_type == "yaml": result = api_util.get_custom_yaml_source_definition( workspace_id=self.workspace_id, definition_id=definition_id, @@ -630,56 +629,112 @@ def get_custom_source_definition( client_secret=self.client_secret, ) return CloudCustomSourceDefinition._from_yaml_response(self, result) # noqa: SLF001 - except Exception: - result = api_util.get_custom_docker_source_definition( - workspace_id=self.workspace_id, - definition_id=definition_id, - api_root=self.api_root, - client_id=self.client_id, - client_secret=self.client_secret, + result = api_util.get_custom_docker_source_definition( + workspace_id=self.workspace_id, + definition_id=definition_id, + api_root=self.api_root, + client_id=self.client_id, + client_secret=self.client_secret, + ) + return CloudCustomSourceDefinition._from_docker_response(self, result) # noqa: SLF001 + + def rename_custom_source_definition( + self, + definition_id: str, + *, + new_name: str, + custom_connector_type: Literal["yaml", "docker"], + ) -> CloudCustomSourceDefinition: + """Rename a custom source definition. + + Note: Only Docker custom sources can be renamed. YAML custom sources + cannot be renamed as their names are derived from the manifest. + + Args: + definition_id: The definition ID to rename + new_name: New display name for the connector + custom_connector_type: Connector type ("yaml" or "docker"). Required. + + Returns: + Updated CloudCustomSourceDefinition object + + Raises: + PyAirbyteInputError: If attempting to rename a YAML connector + """ + if custom_connector_type == "yaml": + raise exc.PyAirbyteInputError( + message="Cannot rename YAML custom source definitions", + context={"definition_id": definition_id}, ) - return CloudCustomSourceDefinition._from_docker_response(self, result) # noqa: SLF001 + + current_definition = self.get_custom_source_definition( + definition_id=definition_id, + custom_connector_type="docker", + ) + + result = api_util.update_custom_docker_source_definition( + workspace_id=self.workspace_id, + definition_id=definition_id, + name=new_name, + docker_image_tag=current_definition.docker_image_tag, # type: ignore[arg-type] + api_root=self.api_root, + client_id=self.client_id, + client_secret=self.client_secret, + ) + return CloudCustomSourceDefinition._from_docker_response(self, result) # noqa: SLF001 def update_custom_source_definition( self, definition_id: str, *, - name: str | None = None, manifest_yaml: dict[str, Any] | Path | str | None = None, docker_tag: str | None = None, pre_validate: bool = True, ) -> CloudCustomSourceDefinition: """Update a custom source definition. - For YAML connectors: can update manifest_yaml - For Docker connectors: can update name and/or docker_tag + You must specify EXACTLY ONE of manifest_yaml (for YAML connectors) OR + docker_tag (for Docker connectors), but not both. + + For YAML connectors: updates the manifest + For Docker connectors: updates the image tag (name remains unchanged - use + rename_custom_source_definition to change the name) Args: definition_id: The definition ID to update - name: New display name (Docker connectors only) manifest_yaml: New manifest (YAML connectors only) docker_tag: New Docker tag (Docker connectors only) pre_validate: Whether to validate manifest (YAML only) Returns: Updated CloudCustomSourceDefinition object + + Raises: + PyAirbyteInputError: If both or neither parameters are provided """ - definition = self.get_custom_source_definition(definition_id) + is_yaml = manifest_yaml is not None + is_docker = docker_tag is not None - if definition.connector_type == "yaml": - if manifest_yaml is None: - raise exc.PyAirbyteInputError( - message="manifest_yaml is required for updating YAML connectors", - context={"definition_id": definition_id}, - ) + if is_yaml == is_docker: + raise exc.PyAirbyteInputError( + message=( + "Must specify EXACTLY ONE of manifest_yaml (for YAML) OR " + "docker_tag (for Docker), but not both" + ), + context={ + "manifest_yaml_provided": is_yaml, + "docker_tag_provided": is_docker, + }, + ) + if is_yaml: manifest_dict: dict[str, Any] if isinstance(manifest_yaml, Path): manifest_dict = yaml.safe_load(manifest_yaml.read_text()) elif isinstance(manifest_yaml, str): manifest_dict = yaml.safe_load(manifest_yaml) else: - manifest_dict = manifest_yaml + manifest_dict = manifest_yaml # type: ignore[assignment] if pre_validate: api_util.validate_yaml_manifest(manifest_dict, raise_on_error=True) @@ -693,17 +748,17 @@ def update_custom_source_definition( client_secret=self.client_secret, ) return CloudCustomSourceDefinition._from_yaml_response(self, result) # noqa: SLF001 - if name is None or docker_tag is None: - raise exc.PyAirbyteInputError( - message="Both name and docker_tag are required for updating Docker connectors", - context={"definition_id": definition_id}, - ) + + current_definition = self.get_custom_source_definition( + definition_id=definition_id, + custom_connector_type="docker", + ) result = api_util.update_custom_docker_source_definition( workspace_id=self.workspace_id, definition_id=definition_id, - name=name, - docker_image_tag=docker_tag, + name=current_definition.name, + docker_image_tag=docker_tag, # type: ignore[arg-type] api_root=self.api_root, client_id=self.client_id, client_secret=self.client_secret, @@ -713,15 +768,16 @@ def update_custom_source_definition( def permanently_delete_custom_source_definition( self, definition_id: str, + *, + custom_connector_type: Literal["yaml", "docker"], ) -> None: """Permanently delete a custom source definition. Args: definition_id: The definition ID to delete + custom_connector_type: Connector type ("yaml" or "docker"). Required. """ - definition = self.get_custom_source_definition(definition_id) - - if definition.connector_type == "yaml": + if custom_connector_type == "yaml": api_util.delete_custom_yaml_source_definition( workspace_id=self.workspace_id, definition_id=definition_id, diff --git a/airbyte/mcp/cloud_ops.py b/airbyte/mcp/cloud_ops.py index 4a24acf2..45570886 100644 --- a/airbyte/mcp/cloud_ops.py +++ b/airbyte/mcp/cloud_ops.py @@ -593,18 +593,15 @@ def publish_custom_source_definition( def list_custom_source_definitions( custom_connector_type: Annotated[ - str | None, + str, Field( - description=( - "Filter by connector type: 'yaml' or 'docker'. " "If not specified, returns all." - ), - default=None, + description="Connector type to list: 'yaml' or 'docker'. Required.", ), - ] = None, + ], ) -> list[dict[str, Any]]: - """List all custom source definitions in the Airbyte Cloud workspace. + """List custom source definitions in the Airbyte Cloud workspace. - Returns both YAML and Docker source definitions unless filtered by type. + You must specify the connector type to list - either 'yaml' or 'docker'. """ workspace: CloudWorkspace = _get_cloud_workspace() definitions = workspace.list_custom_source_definitions( @@ -632,13 +629,6 @@ def update_custom_source_definition( Field(description="The ID of the definition to update."), ], *, - name: Annotated[ - str | None, - Field( - description="New name for the definition (Docker connectors only).", - default=None, - ), - ] = None, manifest_yaml: Annotated[ dict | str | None, Field( @@ -663,14 +653,15 @@ def update_custom_source_definition( ) -> str: """Update a custom source definition in Airbyte Cloud. + You must specify EXACTLY ONE of manifest_yaml or docker_tag, but not both. For YAML connectors: specify manifest_yaml - For Docker connectors: specify name and/or docker_tag + For Docker connectors: specify docker_tag (updates tag only, not name) + To rename a Docker connector, use rename_custom_source_definition instead. """ try: workspace: CloudWorkspace = _get_cloud_workspace() result = workspace.update_custom_source_definition( definition_id=definition_id, - name=name, manifest_yaml=manifest_yaml, docker_tag=docker_tag, pre_validate=pre_validate, @@ -689,6 +680,45 @@ def update_custom_source_definition( ) +def rename_custom_source_definition( + definition_id: Annotated[ + str, + Field(description="The ID of the definition to rename."), + ], + new_name: Annotated[ + str, + Field(description="New display name for the connector."), + ], + custom_connector_type: Annotated[ + str, + Field( + description=( + "Connector type: 'yaml' or 'docker'. " "Only Docker connectors can be renamed." + ), + ), + ], +) -> str: + """Rename a custom source definition in Airbyte Cloud. + + Note: Only Docker custom sources can be renamed. YAML custom sources + cannot be renamed as their names are derived from the manifest. + """ + try: + workspace: CloudWorkspace = _get_cloud_workspace() + result = workspace.rename_custom_source_definition( + definition_id=definition_id, + new_name=new_name, + custom_connector_type=custom_connector_type, # type: ignore[arg-type] + ) + except Exception as ex: + return f"Failed to rename custom source definition '{definition_id}': {ex}" + else: + return ( + f"Successfully renamed custom Docker source definition to '{result.name}' " + f"(ID: {result.definition_id})" + ) + + def publish_custom_destination_definition( name: Annotated[ str, @@ -808,6 +838,7 @@ def register_cloud_ops_tools(app: FastMCP) -> None: app.tool(publish_custom_source_definition) app.tool(list_custom_source_definitions) app.tool(update_custom_source_definition) + app.tool(rename_custom_source_definition) app.tool(publish_custom_destination_definition) app.tool(list_custom_destination_definitions) app.tool(update_custom_destination_definition) diff --git a/tests/integration_tests/cloud/test_custom_definitions.py b/tests/integration_tests/cloud/test_custom_definitions.py index 28170d34..1032ec2d 100644 --- a/tests/integration_tests/cloud/test_custom_definitions.py +++ b/tests/integration_tests/cloud/test_custom_definitions.py @@ -64,7 +64,10 @@ def test_publish_custom_yaml_source( assert len(definitions) == 1 assert definitions[0].definition_id == definition_id - fetched = cloud_workspace.get_custom_source_definition(definition_id) + fetched = cloud_workspace.get_custom_source_definition( + definition_id, + custom_connector_type="yaml", + ) assert fetched.definition_id == definition_id assert fetched.name == name assert fetched.connector_type == "yaml" @@ -78,7 +81,10 @@ def test_publish_custom_yaml_source( assert updated.manifest["version"] == "0.2.0" finally: - cloud_workspace.permanently_delete_custom_source_definition(definition_id) + cloud_workspace.permanently_delete_custom_source_definition( + definition_id, + custom_connector_type="yaml", + ) @pytest.mark.requires_creds @@ -114,20 +120,25 @@ def test_publish_custom_docker_source( assert len(definitions) == 1 assert definitions[0].definition_id == definition_id - fetched = cloud_workspace.get_custom_source_definition(definition_id) + fetched = cloud_workspace.get_custom_source_definition( + definition_id, + custom_connector_type="docker", + ) assert fetched.definition_id == definition_id assert fetched.name == name assert fetched.connector_type == "docker" updated = cloud_workspace.update_custom_source_definition( definition_id=definition_id, - name=name, docker_tag="2.0.0", ) assert updated.docker_image_tag == "2.0.0" finally: - cloud_workspace.permanently_delete_custom_source_definition(definition_id) + cloud_workspace.permanently_delete_custom_source_definition( + definition_id, + custom_connector_type="docker", + ) @pytest.mark.requires_creds From dc6d3806dfdc54d5e97c6c840580b88fbc016baf Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 2 Oct 2025 23:52:50 +0000 Subject: [PATCH 5/9] refactor: Move update/rename methods to dataclasses - Move update_custom_source_definition to CloudCustomSourceDefinition.update_definition() - Move rename_custom_source_definition to CloudCustomSourceDefinition.rename() - Move update_custom_destination_definition to CloudCustomDestinationDefinition.update_definition() - Update all callers: MCP tools and integration tests - MCP tools now accept manifest_yaml as str | Path (not dict) - Keep publish/get/list/permanently_delete in CloudWorkspace This reduces clutter in CloudWorkspace by moving update/rename operations to the narrower dataclasses where they belong. Methods use shorter names since they're now in type-specific classes. Addresses PR feedback from @aaronsteers Co-Authored-By: AJ Steers --- airbyte/cloud/connectors.py | 145 ++++++++++++++++ airbyte/cloud/workspaces.py | 155 ------------------ airbyte/mcp/cloud_ops.py | 24 ++- .../cloud/test_custom_definitions.py | 9 +- 4 files changed, 165 insertions(+), 168 deletions(-) diff --git a/airbyte/cloud/connectors.py b/airbyte/cloud/connectors.py index 84673935..6e574aa5 100644 --- a/airbyte/cloud/connectors.py +++ b/airbyte/cloud/connectors.py @@ -41,10 +41,13 @@ import abc from dataclasses import dataclass +from pathlib import Path from typing import TYPE_CHECKING, Any, ClassVar, Literal +import yaml from airbyte_api import models as api_models # noqa: TC002 +from airbyte import exceptions as exc from airbyte._util import api_util @@ -362,6 +365,122 @@ def permanently_delete(self) -> None: custom_connector_type=self.connector_type, ) + def update_definition( + self, + *, + manifest_yaml: dict[str, Any] | Path | str | None = None, + docker_tag: str | None = None, + pre_validate: bool = True, + ) -> CloudCustomSourceDefinition: + """Update this custom source definition. + + You must specify EXACTLY ONE of manifest_yaml (for YAML connectors) OR + docker_tag (for Docker connectors), but not both. + + For YAML connectors: updates the manifest + For Docker connectors: updates the image tag (name remains unchanged - use + rename() to change the name) + + Args: + manifest_yaml: New manifest (YAML connectors only) + docker_tag: New Docker tag (Docker connectors only) + pre_validate: Whether to validate manifest (YAML only) + + Returns: + Updated CloudCustomSourceDefinition object + + Raises: + PyAirbyteInputError: If both or neither parameters are provided + """ + is_yaml = manifest_yaml is not None + is_docker = docker_tag is not None + + if is_yaml == is_docker: + raise exc.PyAirbyteInputError( + message=( + "Must specify EXACTLY ONE of manifest_yaml (for YAML) OR " + "docker_tag (for Docker), but not both" + ), + context={ + "manifest_yaml_provided": is_yaml, + "docker_tag_provided": is_docker, + }, + ) + + if is_yaml: + manifest_dict: dict[str, Any] + if isinstance(manifest_yaml, Path): + manifest_dict = yaml.safe_load(manifest_yaml.read_text()) + elif isinstance(manifest_yaml, str): + manifest_dict = yaml.safe_load(manifest_yaml) + else: + manifest_dict = manifest_yaml # type: ignore[assignment] + + if pre_validate: + api_util.validate_yaml_manifest(manifest_dict, raise_on_error=True) + + result = api_util.update_custom_yaml_source_definition( + workspace_id=self.workspace.workspace_id, + definition_id=self.definition_id, + manifest=manifest_dict, + api_root=self.workspace.api_root, + client_id=self.workspace.client_id, + client_secret=self.workspace.client_secret, + ) + return CloudCustomSourceDefinition._from_yaml_response(self.workspace, result) + + if not self._definition_info: + self._definition_info = self._fetch_definition_info() + + result = api_util.update_custom_docker_source_definition( + workspace_id=self.workspace.workspace_id, + definition_id=self.definition_id, + name=self._definition_info.name, + docker_image_tag=docker_tag, # type: ignore[arg-type] + api_root=self.workspace.api_root, + client_id=self.workspace.client_id, + client_secret=self.workspace.client_secret, + ) + return CloudCustomSourceDefinition._from_docker_response(self.workspace, result) + + def rename( + self, + new_name: str, + ) -> CloudCustomSourceDefinition: + """Rename this custom source definition. + + Note: Only Docker custom sources can be renamed. YAML custom sources + cannot be renamed as their names are derived from the manifest. + + Args: + new_name: New display name for the connector + + Returns: + Updated CloudCustomSourceDefinition object + + Raises: + PyAirbyteInputError: If attempting to rename a YAML connector + """ + if self.connector_type == "yaml": + raise exc.PyAirbyteInputError( + message="Cannot rename YAML custom source definitions", + context={"definition_id": self.definition_id}, + ) + + if not self._definition_info: + self._definition_info = self._fetch_definition_info() + + result = api_util.update_custom_docker_source_definition( + workspace_id=self.workspace.workspace_id, + definition_id=self.definition_id, + name=new_name, + docker_image_tag=self._definition_info.docker_image_tag, + api_root=self.workspace.api_root, + client_id=self.workspace.client_id, + client_secret=self.workspace.client_secret, + ) + return CloudCustomSourceDefinition._from_docker_response(self.workspace, result) + def __repr__(self) -> str: """String representation.""" return ( @@ -466,6 +585,32 @@ def permanently_delete(self) -> None: """Permanently delete this custom destination definition.""" self.workspace.permanently_delete_custom_destination_definition(self.definition_id) + def update_definition( + self, + *, + name: str, + docker_tag: str, + ) -> CloudCustomDestinationDefinition: + """Update this custom destination definition. + + Args: + name: New display name + docker_tag: New Docker tag + + Returns: + Updated CloudCustomDestinationDefinition object + """ + result = api_util.update_custom_docker_destination_definition( + workspace_id=self.workspace.workspace_id, + definition_id=self.definition_id, + name=name, + docker_image_tag=docker_tag, + api_root=self.workspace.api_root, + client_id=self.workspace.client_id, + client_secret=self.workspace.client_secret, + ) + return CloudCustomDestinationDefinition._from_docker_response(self.workspace, result) + def __repr__(self) -> str: """String representation.""" return ( diff --git a/airbyte/cloud/workspaces.py b/airbyte/cloud/workspaces.py index 34c92852..42ca8dff 100644 --- a/airbyte/cloud/workspaces.py +++ b/airbyte/cloud/workspaces.py @@ -638,133 +638,6 @@ def get_custom_source_definition( ) return CloudCustomSourceDefinition._from_docker_response(self, result) # noqa: SLF001 - def rename_custom_source_definition( - self, - definition_id: str, - *, - new_name: str, - custom_connector_type: Literal["yaml", "docker"], - ) -> CloudCustomSourceDefinition: - """Rename a custom source definition. - - Note: Only Docker custom sources can be renamed. YAML custom sources - cannot be renamed as their names are derived from the manifest. - - Args: - definition_id: The definition ID to rename - new_name: New display name for the connector - custom_connector_type: Connector type ("yaml" or "docker"). Required. - - Returns: - Updated CloudCustomSourceDefinition object - - Raises: - PyAirbyteInputError: If attempting to rename a YAML connector - """ - if custom_connector_type == "yaml": - raise exc.PyAirbyteInputError( - message="Cannot rename YAML custom source definitions", - context={"definition_id": definition_id}, - ) - - current_definition = self.get_custom_source_definition( - definition_id=definition_id, - custom_connector_type="docker", - ) - - result = api_util.update_custom_docker_source_definition( - workspace_id=self.workspace_id, - definition_id=definition_id, - name=new_name, - docker_image_tag=current_definition.docker_image_tag, # type: ignore[arg-type] - api_root=self.api_root, - client_id=self.client_id, - client_secret=self.client_secret, - ) - return CloudCustomSourceDefinition._from_docker_response(self, result) # noqa: SLF001 - - def update_custom_source_definition( - self, - definition_id: str, - *, - manifest_yaml: dict[str, Any] | Path | str | None = None, - docker_tag: str | None = None, - pre_validate: bool = True, - ) -> CloudCustomSourceDefinition: - """Update a custom source definition. - - You must specify EXACTLY ONE of manifest_yaml (for YAML connectors) OR - docker_tag (for Docker connectors), but not both. - - For YAML connectors: updates the manifest - For Docker connectors: updates the image tag (name remains unchanged - use - rename_custom_source_definition to change the name) - - Args: - definition_id: The definition ID to update - manifest_yaml: New manifest (YAML connectors only) - docker_tag: New Docker tag (Docker connectors only) - pre_validate: Whether to validate manifest (YAML only) - - Returns: - Updated CloudCustomSourceDefinition object - - Raises: - PyAirbyteInputError: If both or neither parameters are provided - """ - is_yaml = manifest_yaml is not None - is_docker = docker_tag is not None - - if is_yaml == is_docker: - raise exc.PyAirbyteInputError( - message=( - "Must specify EXACTLY ONE of manifest_yaml (for YAML) OR " - "docker_tag (for Docker), but not both" - ), - context={ - "manifest_yaml_provided": is_yaml, - "docker_tag_provided": is_docker, - }, - ) - - if is_yaml: - manifest_dict: dict[str, Any] - if isinstance(manifest_yaml, Path): - manifest_dict = yaml.safe_load(manifest_yaml.read_text()) - elif isinstance(manifest_yaml, str): - manifest_dict = yaml.safe_load(manifest_yaml) - else: - manifest_dict = manifest_yaml # type: ignore[assignment] - - if pre_validate: - api_util.validate_yaml_manifest(manifest_dict, raise_on_error=True) - - result = api_util.update_custom_yaml_source_definition( - workspace_id=self.workspace_id, - definition_id=definition_id, - manifest=manifest_dict, - api_root=self.api_root, - client_id=self.client_id, - client_secret=self.client_secret, - ) - return CloudCustomSourceDefinition._from_yaml_response(self, result) # noqa: SLF001 - - current_definition = self.get_custom_source_definition( - definition_id=definition_id, - custom_connector_type="docker", - ) - - result = api_util.update_custom_docker_source_definition( - workspace_id=self.workspace_id, - definition_id=definition_id, - name=current_definition.name, - docker_image_tag=docker_tag, # type: ignore[arg-type] - api_root=self.api_root, - client_id=self.client_id, - client_secret=self.client_secret, - ) - return CloudCustomSourceDefinition._from_docker_response(self, result) # noqa: SLF001 - def permanently_delete_custom_source_definition( self, definition_id: str, @@ -884,34 +757,6 @@ def get_custom_destination_definition( ) return CloudCustomDestinationDefinition._from_docker_response(self, result) # noqa: SLF001 - def update_custom_destination_definition( - self, - definition_id: str, - *, - name: str, - docker_tag: str, - ) -> CloudCustomDestinationDefinition: - """Update a custom destination definition. - - Args: - definition_id: The definition ID to update - name: New display name - docker_tag: New Docker tag - - Returns: - Updated CloudCustomDestinationDefinition object - """ - result = api_util.update_custom_docker_destination_definition( - workspace_id=self.workspace_id, - definition_id=definition_id, - name=name, - docker_image_tag=docker_tag, - api_root=self.api_root, - client_id=self.client_id, - client_secret=self.client_secret, - ) - return CloudCustomDestinationDefinition._from_docker_response(self, result) # noqa: SLF001 - def permanently_delete_custom_destination_definition( self, definition_id: str, diff --git a/airbyte/mcp/cloud_ops.py b/airbyte/mcp/cloud_ops.py index 45570886..f194ebb5 100644 --- a/airbyte/mcp/cloud_ops.py +++ b/airbyte/mcp/cloud_ops.py @@ -1,6 +1,7 @@ # Copyright (c) 2024 Airbyte, Inc., all rights reserved. """Airbyte Cloud MCP operations.""" +from pathlib import Path from typing import Annotated, Any from fastmcp import FastMCP @@ -630,9 +631,9 @@ def update_custom_source_definition( ], *, manifest_yaml: Annotated[ - dict | str | None, + str | Path | None, Field( - description="New manifest as dict or YAML string (YAML connectors only).", + description="New manifest as YAML string or file path (YAML connectors only).", default=None, ), ] = None, @@ -643,6 +644,12 @@ def update_custom_source_definition( default=None, ), ] = None, + custom_connector_type: Annotated[ + str, + Field( + description="Connector type: 'yaml' or 'docker'. Required.", + ), + ], pre_validate: Annotated[ bool, Field( @@ -660,8 +667,11 @@ def update_custom_source_definition( """ try: workspace: CloudWorkspace = _get_cloud_workspace() - result = workspace.update_custom_source_definition( + definition = workspace.get_custom_source_definition( definition_id=definition_id, + custom_connector_type=custom_connector_type, # type: ignore[arg-type] + ) + result = definition.update_definition( manifest_yaml=manifest_yaml, docker_tag=docker_tag, pre_validate=pre_validate, @@ -705,11 +715,11 @@ def rename_custom_source_definition( """ try: workspace: CloudWorkspace = _get_cloud_workspace() - result = workspace.rename_custom_source_definition( + definition = workspace.get_custom_source_definition( definition_id=definition_id, - new_name=new_name, custom_connector_type=custom_connector_type, # type: ignore[arg-type] ) + result = definition.rename(new_name=new_name) except Exception as ex: return f"Failed to rename custom source definition '{definition_id}': {ex}" else: @@ -805,8 +815,8 @@ def update_custom_destination_definition( """Update a custom destination definition in Airbyte Cloud.""" try: workspace: CloudWorkspace = _get_cloud_workspace() - result = workspace.update_custom_destination_definition( - definition_id=definition_id, + definition = workspace.get_custom_destination_definition(definition_id=definition_id) + result = definition.update_definition( name=name, docker_tag=docker_tag, ) diff --git a/tests/integration_tests/cloud/test_custom_definitions.py b/tests/integration_tests/cloud/test_custom_definitions.py index 1032ec2d..c0055039 100644 --- a/tests/integration_tests/cloud/test_custom_definitions.py +++ b/tests/integration_tests/cloud/test_custom_definitions.py @@ -74,8 +74,7 @@ def test_publish_custom_yaml_source( updated_manifest = TEST_YAML_MANIFEST.copy() updated_manifest["version"] = "0.2.0" - updated = cloud_workspace.update_custom_source_definition( - definition_id=definition_id, + updated = fetched.update_definition( manifest_yaml=updated_manifest, ) assert updated.manifest["version"] == "0.2.0" @@ -128,8 +127,7 @@ def test_publish_custom_docker_source( assert fetched.name == name assert fetched.connector_type == "docker" - updated = cloud_workspace.update_custom_source_definition( - definition_id=definition_id, + updated = fetched.update_definition( docker_tag="2.0.0", ) assert updated.docker_image_tag == "2.0.0" @@ -173,8 +171,7 @@ def test_publish_custom_docker_destination( assert fetched.definition_id == definition_id assert fetched.name == name - updated = cloud_workspace.update_custom_destination_definition( - definition_id=definition_id, + updated = fetched.update_definition( name=name, docker_tag="2.0.0", ) From 7572ff838678620d30f1e73efcdf3a085d9db2fd Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Fri, 3 Oct 2025 19:40:34 +0000 Subject: [PATCH 6/9] fix: Use valid manifest with spec section and skip Docker tests - Replace TEST_YAML_MANIFEST with complete structure including spec section - Add definitions section following real manifest patterns - Skip Docker custom definition tests pending API support confirmation - The missing spec section was causing 500 error 'get(...) must not be null' Co-Authored-By: AJ Steers --- .../cloud/test_custom_definitions.py | 35 ++++++++++++++----- 1 file changed, 27 insertions(+), 8 deletions(-) diff --git a/tests/integration_tests/cloud/test_custom_definitions.py b/tests/integration_tests/cloud/test_custom_definitions.py index c0055039..d2f4092a 100644 --- a/tests/integration_tests/cloud/test_custom_definitions.py +++ b/tests/integration_tests/cloud/test_custom_definitions.py @@ -9,19 +9,24 @@ TEST_YAML_MANIFEST = { "version": "0.1.0", "type": "DeclarativeSource", - "check": {"type": "CheckStream", "stream_names": ["test"]}, + "check": { + "type": "CheckStream", + "stream_names": ["test_stream"], + }, + "definitions": { + "base_requester": { + "type": "HttpRequester", + "url_base": "https://httpbin.org", + }, + }, "streams": [ { "type": "DeclarativeStream", - "name": "test", - "primary_key": [], + "name": "test_stream", + "primary_key": ["id"], "retriever": { "type": "SimpleRetriever", - "requester": { - "type": "HttpRequester", - "url_base": "https://httpbin.org", - "path": "/get", - }, + "requester": {"$ref": "#/definitions/base_requester", "path": "/get"}, "record_selector": { "type": "RecordSelector", "extractor": {"type": "DpathExtractor", "field_path": []}, @@ -29,6 +34,14 @@ }, } ], + "spec": { + "type": "Spec", + "connection_specification": { + "type": "object", + "$schema": "http://json-schema.org/draft-07/schema#", + "properties": {}, + }, + }, } @@ -86,6 +99,9 @@ def test_publish_custom_yaml_source( ) +@pytest.mark.skip( + reason="Docker custom definitions appear blocked in Airbyte Cloud - pending confirmation" +) @pytest.mark.requires_creds def test_publish_custom_docker_source( cloud_workspace: CloudWorkspace, @@ -139,6 +155,9 @@ def test_publish_custom_docker_source( ) +@pytest.mark.skip( + reason="Docker custom definitions appear blocked in Airbyte Cloud - pending confirmation" +) @pytest.mark.requires_creds def test_publish_custom_docker_destination( cloud_workspace: CloudWorkspace, From a014f2a213d407ff6ff6f65fb0d383005c2993b0 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Fri, 10 Oct 2025 23:59:57 +0000 Subject: [PATCH 7/9] feat: Add YAML custom connector publishing (Docker stubbed) - Add support for publishing, listing, updating, and deleting custom YAML source definitions - Implement CloudCustomSourceDefinition dataclass with lazy-loading pattern - Add 3 MCP tools: publish_custom_source_definition, list_custom_source_definitions, update_custom_source_definition - Stub out Docker custom definitions with NotImplementedError to preserve API structure for future support - Add integration tests for YAML custom source definitions - Remove Docker custom destination definitions (not in scope for this PR) This enables PyAirbyte users to publish YAML (declarative) connectors to Airbyte Cloud without needing to build Docker images, leveraging new Airbyte API endpoints. Co-Authored-By: AJ Steers --- airbyte/_util/api_util.py | 296 ------------------ airbyte/cloud/connectors.py | 225 +++---------- airbyte/cloud/workspaces.py | 169 ++-------- airbyte/mcp/cloud_ops.py | 259 ++------------- .../cloud/test_custom_definitions.py | 101 ------ 5 files changed, 87 insertions(+), 963 deletions(-) diff --git a/airbyte/_util/api_util.py b/airbyte/_util/api_util.py index de991721..633a5b80 100644 --- a/airbyte/_util/api_util.py +++ b/airbyte/_util/api_util.py @@ -1122,299 +1122,3 @@ def delete_custom_yaml_source_definition( definition_id=definition_id, ) airbyte_instance.declarative_source_definitions.delete_declarative_source_definition(request) - - -def create_custom_docker_source_definition( - name: str, - docker_repository: str, - docker_image_tag: str, - *, - workspace_id: str, - documentation_url: str | None = None, - api_root: str, - client_id: SecretString, - client_secret: SecretString, -) -> models.DefinitionResponse: - """Create a custom Docker source definition.""" - airbyte_instance = get_airbyte_server_instance( - api_root=api_root, - client_id=client_id, - client_secret=client_secret, - ) - - request_body = models.CreateDefinitionRequest( - name=name, - docker_repository=docker_repository, - docker_image_tag=docker_image_tag, - documentation_url=documentation_url, - ) - request = api.CreateSourceDefinitionRequest( - workspace_id=workspace_id, - create_definition_request=request_body, - ) - response = airbyte_instance.source_definitions.create_source_definition(request) - if response.definition_response is None: - raise AirbyteError( - message="Failed to create custom Docker source definition", - context={"name": name, "workspace_id": workspace_id}, - ) - return response.definition_response - - -def list_custom_docker_source_definitions( - workspace_id: str, - *, - api_root: str, - client_id: SecretString, - client_secret: SecretString, -) -> list[models.DefinitionResponse]: - """List all custom Docker source definitions in a workspace.""" - airbyte_instance = get_airbyte_server_instance( - api_root=api_root, - client_id=client_id, - client_secret=client_secret, - ) - - request = api.ListSourceDefinitionsRequest( - workspace_id=workspace_id, - ) - response = airbyte_instance.source_definitions.list_source_definitions(request) - if response.definitions_response is None: - raise AirbyteError( - message="Failed to list custom Docker source definitions", - context={"workspace_id": workspace_id}, - ) - return response.definitions_response.data - - -def get_custom_docker_source_definition( - workspace_id: str, - definition_id: str, - *, - api_root: str, - client_id: SecretString, - client_secret: SecretString, -) -> models.DefinitionResponse: - """Get a specific custom Docker source definition.""" - airbyte_instance = get_airbyte_server_instance( - api_root=api_root, - client_id=client_id, - client_secret=client_secret, - ) - - request = api.GetSourceDefinitionRequest( - workspace_id=workspace_id, - definition_id=definition_id, - ) - response = airbyte_instance.source_definitions.get_source_definition(request) - if response.definition_response is None: - raise AirbyteError( - message="Failed to get custom Docker source definition", - context={"workspace_id": workspace_id, "definition_id": definition_id}, - ) - return response.definition_response - - -def update_custom_docker_source_definition( - workspace_id: str, - definition_id: str, - *, - name: str, - docker_image_tag: str, - api_root: str, - client_id: SecretString, - client_secret: SecretString, -) -> models.DefinitionResponse: - """Update a custom Docker source definition.""" - airbyte_instance = get_airbyte_server_instance( - api_root=api_root, - client_id=client_id, - client_secret=client_secret, - ) - - request_body = models.UpdateDefinitionRequest( - name=name, - docker_image_tag=docker_image_tag, - ) - request = api.UpdateSourceDefinitionRequest( - workspace_id=workspace_id, - definition_id=definition_id, - update_definition_request=request_body, - ) - response = airbyte_instance.source_definitions.update_source_definition(request) - if response.definition_response is None: - raise AirbyteError( - message="Failed to update custom Docker source definition", - context={"workspace_id": workspace_id, "definition_id": definition_id}, - ) - return response.definition_response - - -def delete_custom_docker_source_definition( - workspace_id: str, - definition_id: str, - *, - api_root: str, - client_id: SecretString, - client_secret: SecretString, -) -> None: - """Delete a custom Docker source definition.""" - airbyte_instance = get_airbyte_server_instance( - api_root=api_root, - client_id=client_id, - client_secret=client_secret, - ) - - request = api.DeleteSourceDefinitionRequest( - workspace_id=workspace_id, - definition_id=definition_id, - ) - airbyte_instance.source_definitions.delete_source_definition(request) - - -def create_custom_docker_destination_definition( - name: str, - docker_repository: str, - docker_image_tag: str, - *, - workspace_id: str, - documentation_url: str | None = None, - api_root: str, - client_id: SecretString, - client_secret: SecretString, -) -> models.DefinitionResponse: - """Create a custom Docker destination definition.""" - airbyte_instance = get_airbyte_server_instance( - api_root=api_root, - client_id=client_id, - client_secret=client_secret, - ) - - request_body = models.CreateDefinitionRequest( - name=name, - docker_repository=docker_repository, - docker_image_tag=docker_image_tag, - documentation_url=documentation_url, - ) - request = api.CreateDestinationDefinitionRequest( - workspace_id=workspace_id, - create_definition_request=request_body, - ) - response = airbyte_instance.destination_definitions.create_destination_definition(request) - if response.definition_response is None: - raise AirbyteError( - message="Failed to create custom Docker destination definition", - context={"name": name, "workspace_id": workspace_id}, - ) - return response.definition_response - - -def list_custom_docker_destination_definitions( - workspace_id: str, - *, - api_root: str, - client_id: SecretString, - client_secret: SecretString, -) -> list[models.DefinitionResponse]: - """List all custom Docker destination definitions in a workspace.""" - airbyte_instance = get_airbyte_server_instance( - api_root=api_root, - client_id=client_id, - client_secret=client_secret, - ) - - request = api.ListDestinationDefinitionsRequest( - workspace_id=workspace_id, - ) - response = airbyte_instance.destination_definitions.list_destination_definitions(request) - if response.definitions_response is None: - raise AirbyteError( - message="Failed to list custom Docker destination definitions", - context={"workspace_id": workspace_id}, - ) - return response.definitions_response.data - - -def get_custom_docker_destination_definition( - workspace_id: str, - definition_id: str, - *, - api_root: str, - client_id: SecretString, - client_secret: SecretString, -) -> models.DefinitionResponse: - """Get a specific custom Docker destination definition.""" - airbyte_instance = get_airbyte_server_instance( - api_root=api_root, - client_id=client_id, - client_secret=client_secret, - ) - - request = api.GetDestinationDefinitionRequest( - workspace_id=workspace_id, - definition_id=definition_id, - ) - response = airbyte_instance.destination_definitions.get_destination_definition(request) - if response.definition_response is None: - raise AirbyteError( - message="Failed to get custom Docker destination definition", - context={"workspace_id": workspace_id, "definition_id": definition_id}, - ) - return response.definition_response - - -def update_custom_docker_destination_definition( - workspace_id: str, - definition_id: str, - *, - name: str, - docker_image_tag: str, - api_root: str, - client_id: SecretString, - client_secret: SecretString, -) -> models.DefinitionResponse: - """Update a custom Docker destination definition.""" - airbyte_instance = get_airbyte_server_instance( - api_root=api_root, - client_id=client_id, - client_secret=client_secret, - ) - - request_body = models.UpdateDefinitionRequest( - name=name, - docker_image_tag=docker_image_tag, - ) - request = api.UpdateDestinationDefinitionRequest( - workspace_id=workspace_id, - definition_id=definition_id, - update_definition_request=request_body, - ) - response = airbyte_instance.destination_definitions.update_destination_definition(request) - if response.definition_response is None: - raise AirbyteError( - message="Failed to update custom Docker destination definition", - context={"workspace_id": workspace_id, "definition_id": definition_id}, - ) - return response.definition_response - - -def delete_custom_docker_destination_definition( - workspace_id: str, - definition_id: str, - *, - api_root: str, - client_id: SecretString, - client_secret: SecretString, -) -> None: - """Delete a custom Docker destination definition.""" - airbyte_instance = get_airbyte_server_instance( - api_root=api_root, - client_id=client_id, - client_secret=client_secret, - ) - - request = api.DeleteDestinationDefinitionRequest( - workspace_id=workspace_id, - definition_id=definition_id, - ) - airbyte_instance.destination_definitions.delete_destination_definition(request) diff --git a/airbyte/cloud/connectors.py b/airbyte/cloud/connectors.py index 6e574aa5..550336b1 100644 --- a/airbyte/cloud/connectors.py +++ b/airbyte/cloud/connectors.py @@ -270,17 +270,19 @@ def __init__( definition_id: str, connector_type: Literal["yaml", "docker"], ) -> None: - """Initialize a custom source definition object.""" + """Initialize a custom source definition object. + + Note: Only YAML connectors are currently supported. Docker connectors + will raise NotImplementedError. + """ self.workspace = workspace self.definition_id = definition_id self.connector_type = connector_type - self._definition_info: ( - api_models.DeclarativeSourceDefinitionResponse | api_models.DefinitionResponse | None - ) = None + self._definition_info: api_models.DeclarativeSourceDefinitionResponse | None = None def _fetch_definition_info( self, - ) -> api_models.DeclarativeSourceDefinitionResponse | api_models.DefinitionResponse: + ) -> api_models.DeclarativeSourceDefinitionResponse: """Fetch definition info from the API.""" if self.connector_type == "yaml": return api_util.get_custom_yaml_source_definition( @@ -290,12 +292,9 @@ def _fetch_definition_info( client_id=self.workspace.client_id, client_secret=self.workspace.client_secret, ) - return api_util.get_custom_docker_source_definition( - workspace_id=self.workspace.workspace_id, - definition_id=self.definition_id, - api_root=self.workspace.api_root, - client_id=self.workspace.client_id, - client_secret=self.workspace.client_secret, + raise NotImplementedError( + "Docker custom source definitions are not yet supported. " + "Only YAML manifest-based custom sources are currently available." ) @property @@ -325,30 +324,42 @@ def version(self) -> str | None: @property def docker_repository(self) -> str | None: - """Get the Docker repository. Only present for Docker connectors.""" + """Get the Docker repository. Only present for Docker connectors. + + Note: Docker connectors are not yet supported and will raise NotImplementedError. + """ if self.connector_type != "docker": return None - if not self._definition_info: - self._definition_info = self._fetch_definition_info() - return self._definition_info.docker_repository + raise NotImplementedError( + "Docker custom source definitions are not yet supported. " + "Only YAML manifest-based custom sources are currently available." + ) @property def docker_image_tag(self) -> str | None: - """Get the Docker image tag. Only present for Docker connectors.""" + """Get the Docker image tag. Only present for Docker connectors. + + Note: Docker connectors are not yet supported and will raise NotImplementedError. + """ if self.connector_type != "docker": return None - if not self._definition_info: - self._definition_info = self._fetch_definition_info() - return self._definition_info.docker_image_tag + raise NotImplementedError( + "Docker custom source definitions are not yet supported. " + "Only YAML manifest-based custom sources are currently available." + ) @property def documentation_url(self) -> str | None: - """Get the documentation URL. Only present for Docker connectors.""" + """Get the documentation URL. Only present for Docker connectors. + + Note: Docker connectors are not yet supported and will raise NotImplementedError. + """ if self.connector_type != "docker": return None - if not self._definition_info: - self._definition_info = self._fetch_definition_info() - return self._definition_info.documentation_url + raise NotImplementedError( + "Docker custom source definitions are not yet supported. " + "Only YAML manifest-based custom sources are currently available." + ) @property def definition_url(self) -> str: @@ -378,12 +389,11 @@ def update_definition( docker_tag (for Docker connectors), but not both. For YAML connectors: updates the manifest - For Docker connectors: updates the image tag (name remains unchanged - use - rename() to change the name) + For Docker connectors: Not yet supported (raises NotImplementedError) Args: manifest_yaml: New manifest (YAML connectors only) - docker_tag: New Docker tag (Docker connectors only) + docker_tag: New Docker tag (Docker connectors only, not yet supported) pre_validate: Whether to validate manifest (YAML only) Returns: @@ -391,6 +401,7 @@ def update_definition( Raises: PyAirbyteInputError: If both or neither parameters are provided + NotImplementedError: If docker_tag is provided (Docker not yet supported) """ is_yaml = manifest_yaml is not None is_docker = docker_tag is not None @@ -429,23 +440,14 @@ def update_definition( ) return CloudCustomSourceDefinition._from_yaml_response(self.workspace, result) - if not self._definition_info: - self._definition_info = self._fetch_definition_info() - - result = api_util.update_custom_docker_source_definition( - workspace_id=self.workspace.workspace_id, - definition_id=self.definition_id, - name=self._definition_info.name, - docker_image_tag=docker_tag, # type: ignore[arg-type] - api_root=self.workspace.api_root, - client_id=self.workspace.client_id, - client_secret=self.workspace.client_secret, + raise NotImplementedError( + "Docker custom source definitions are not yet supported. " + "Only YAML manifest-based custom sources are currently available." ) - return CloudCustomSourceDefinition._from_docker_response(self.workspace, result) def rename( self, - new_name: str, + new_name: str, # noqa: ARG002 ) -> CloudCustomSourceDefinition: """Rename this custom source definition. @@ -460,6 +462,7 @@ def rename( Raises: PyAirbyteInputError: If attempting to rename a YAML connector + NotImplementedError: If attempting to rename a Docker connector (not yet supported) """ if self.connector_type == "yaml": raise exc.PyAirbyteInputError( @@ -467,19 +470,10 @@ def rename( context={"definition_id": self.definition_id}, ) - if not self._definition_info: - self._definition_info = self._fetch_definition_info() - - result = api_util.update_custom_docker_source_definition( - workspace_id=self.workspace.workspace_id, - definition_id=self.definition_id, - name=new_name, - docker_image_tag=self._definition_info.docker_image_tag, - api_root=self.workspace.api_root, - client_id=self.workspace.client_id, - client_secret=self.workspace.client_secret, + raise NotImplementedError( + "Docker custom source definitions are not yet supported. " + "Only YAML manifest-based custom sources are currently available." ) - return CloudCustomSourceDefinition._from_docker_response(self.workspace, result) def __repr__(self) -> str: """String representation.""" @@ -502,132 +496,3 @@ def _from_yaml_response( ) result._definition_info = response # noqa: SLF001 return result - - @classmethod - def _from_docker_response( - cls, - workspace: CloudWorkspace, - response: api_models.DefinitionResponse, - ) -> CloudCustomSourceDefinition: - """Internal factory method for Docker connectors.""" - result = cls( - workspace=workspace, - definition_id=response.id, - connector_type="docker", - ) - result._definition_info = response # noqa: SLF001 - return result - - -class CloudCustomDestinationDefinition: - """A custom destination connector definition in Airbyte Cloud. - - Currently only supports Docker-based custom destinations. - """ - - def __init__( - self, - workspace: CloudWorkspace, - definition_id: str, - ) -> None: - """Initialize a custom destination definition object.""" - self.workspace = workspace - self.definition_id = definition_id - self._definition_info: api_models.DefinitionResponse | None = None - - def _fetch_definition_info(self) -> api_models.DefinitionResponse: - """Fetch definition info from the API.""" - return api_util.get_custom_docker_destination_definition( - workspace_id=self.workspace.workspace_id, - definition_id=self.definition_id, - api_root=self.workspace.api_root, - client_id=self.workspace.client_id, - client_secret=self.workspace.client_secret, - ) - - @property - def name(self) -> str: - """Get the display name of the custom connector definition.""" - if not self._definition_info: - self._definition_info = self._fetch_definition_info() - return self._definition_info.name - - @property - def docker_repository(self) -> str: - """Get the Docker repository.""" - if not self._definition_info: - self._definition_info = self._fetch_definition_info() - return self._definition_info.docker_repository - - @property - def docker_image_tag(self) -> str: - """Get the Docker image tag.""" - if not self._definition_info: - self._definition_info = self._fetch_definition_info() - return self._definition_info.docker_image_tag - - @property - def documentation_url(self) -> str | None: - """Get the documentation URL.""" - if not self._definition_info: - self._definition_info = self._fetch_definition_info() - return self._definition_info.documentation_url - - @property - def definition_url(self) -> str: - """Get the web URL of the custom destination definition.""" - return ( - f"{self.workspace.workspace_url}/settings/custom-connectors/" - f"destinations/{self.definition_id}" - ) - - def permanently_delete(self) -> None: - """Permanently delete this custom destination definition.""" - self.workspace.permanently_delete_custom_destination_definition(self.definition_id) - - def update_definition( - self, - *, - name: str, - docker_tag: str, - ) -> CloudCustomDestinationDefinition: - """Update this custom destination definition. - - Args: - name: New display name - docker_tag: New Docker tag - - Returns: - Updated CloudCustomDestinationDefinition object - """ - result = api_util.update_custom_docker_destination_definition( - workspace_id=self.workspace.workspace_id, - definition_id=self.definition_id, - name=name, - docker_image_tag=docker_tag, - api_root=self.workspace.api_root, - client_id=self.workspace.client_id, - client_secret=self.workspace.client_secret, - ) - return CloudCustomDestinationDefinition._from_docker_response(self.workspace, result) - - def __repr__(self) -> str: - """String representation.""" - return ( - f"CloudCustomDestinationDefinition(definition_id={self.definition_id}, " - f"name={self.name}, docker_repository={self.docker_repository})" - ) - - @classmethod - def _from_docker_response( - cls, - workspace: CloudWorkspace, - response: api_models.DefinitionResponse, - ) -> CloudCustomDestinationDefinition: - """Internal factory method.""" - result = cls( - workspace=workspace, - definition_id=response.id, - ) - result._definition_info = response # noqa: SLF001 - return result diff --git a/airbyte/cloud/workspaces.py b/airbyte/cloud/workspaces.py index 42ca8dff..f205f7c0 100644 --- a/airbyte/cloud/workspaces.py +++ b/airbyte/cloud/workspaces.py @@ -46,7 +46,6 @@ from airbyte._util.api_util import get_web_url_root from airbyte.cloud.connections import CloudConnection from airbyte.cloud.connectors import ( - CloudCustomDestinationDefinition, CloudCustomSourceDefinition, CloudDestination, CloudSource, @@ -61,7 +60,7 @@ from airbyte.sources.base import Source -@dataclass # noqa: PLR0904 +@dataclass class CloudWorkspace: """A remote workspace on the Airbyte Cloud. @@ -466,7 +465,6 @@ def publish_custom_source_definition( manifest_yaml: dict[str, Any] | Path | str | None = None, docker_image: str | None = None, docker_tag: str | None = None, - documentation_url: str | None = None, unique: bool = True, pre_validate: bool = True, ) -> CloudCustomSourceDefinition: @@ -480,7 +478,6 @@ def publish_custom_source_definition( manifest_yaml: Low-code CDK manifest (dict, Path to YAML file, or YAML string) docker_image: Docker repository (e.g., 'airbyte/source-custom') docker_tag: Docker image tag (e.g., '1.0.0') - documentation_url: Optional URL to connector documentation (Docker only) unique: Whether to enforce name uniqueness pre_validate: Whether to validate manifest client-side (YAML only) @@ -549,17 +546,11 @@ def publish_custom_source_definition( client_secret=self.client_secret, ) return CloudCustomSourceDefinition._from_yaml_response(self, result) # noqa: SLF001 - result = api_util.create_custom_docker_source_definition( - name=name, - docker_repository=docker_image, # type: ignore[arg-type] - docker_image_tag=docker_tag, # type: ignore[arg-type] - workspace_id=self.workspace_id, - documentation_url=documentation_url, - api_root=self.api_root, - client_id=self.client_id, - client_secret=self.client_secret, + + raise NotImplementedError( + "Docker custom source definitions are not yet supported. " + "Only YAML manifest-based custom sources are currently available." ) - return CloudCustomSourceDefinition._from_docker_response(self, result) # noqa: SLF001 def list_custom_source_definitions( self, @@ -576,8 +567,6 @@ def list_custom_source_definitions( Returns: List of CloudCustomSourceDefinition objects matching the specified type """ - result: list[CloudCustomSourceDefinition] = [] - if custom_connector_type == "yaml": yaml_definitions = api_util.list_custom_yaml_source_definitions( workspace_id=self.workspace_id, @@ -585,25 +574,16 @@ def list_custom_source_definitions( client_id=self.client_id, client_secret=self.client_secret, ) - result.extend( + return [ CloudCustomSourceDefinition._from_yaml_response(self, d) # noqa: SLF001 for d in yaml_definitions if name is None or d.name == name - ) - elif custom_connector_type == "docker": - docker_definitions = api_util.list_custom_docker_source_definitions( - workspace_id=self.workspace_id, - api_root=self.api_root, - client_id=self.client_id, - client_secret=self.client_secret, - ) - result.extend( - CloudCustomSourceDefinition._from_docker_response(self, d) # noqa: SLF001 - for d in docker_definitions - if name is None or d.name == name - ) + ] - return result + raise NotImplementedError( + "Docker custom source definitions are not yet supported. " + "Only YAML manifest-based custom sources are currently available." + ) def get_custom_source_definition( self, @@ -629,14 +609,11 @@ def get_custom_source_definition( client_secret=self.client_secret, ) return CloudCustomSourceDefinition._from_yaml_response(self, result) # noqa: SLF001 - result = api_util.get_custom_docker_source_definition( - workspace_id=self.workspace_id, - definition_id=definition_id, - api_root=self.api_root, - client_id=self.client_id, - client_secret=self.client_secret, + + raise NotImplementedError( + "Docker custom source definitions are not yet supported. " + "Only YAML manifest-based custom sources are currently available." ) - return CloudCustomSourceDefinition._from_docker_response(self, result) # noqa: SLF001 def permanently_delete_custom_source_definition( self, @@ -659,117 +636,7 @@ def permanently_delete_custom_source_definition( client_secret=self.client_secret, ) else: - api_util.delete_custom_docker_source_definition( - workspace_id=self.workspace_id, - definition_id=definition_id, - api_root=self.api_root, - client_id=self.client_id, - client_secret=self.client_secret, + raise NotImplementedError( + "Docker custom source definitions are not yet supported. " + "Only YAML manifest-based custom sources are currently available." ) - - def publish_custom_destination_definition( - self, - name: str, - *, - docker_image: str, - docker_tag: str, - documentation_url: str | None = None, - unique: bool = True, - ) -> CloudCustomDestinationDefinition: - """Publish a custom destination connector definition. - - Currently only Docker-based destinations are supported. - - Args: - name: Display name for the connector definition - docker_image: Docker repository (e.g., 'airbyte/destination-custom') - docker_tag: Docker image tag (e.g., '1.0.0') - documentation_url: Optional URL to connector documentation - unique: Whether to enforce name uniqueness - - Returns: - CloudCustomDestinationDefinition object representing the created definition - """ - if unique: - existing = self.list_custom_destination_definitions(name=name) - if existing: - raise exc.AirbyteDuplicateResourcesError( - resource_type="custom_destination_definition", - resource_name=name, - ) - - result = api_util.create_custom_docker_destination_definition( - name=name, - docker_repository=docker_image, - docker_image_tag=docker_tag, - workspace_id=self.workspace_id, - documentation_url=documentation_url, - api_root=self.api_root, - client_id=self.client_id, - client_secret=self.client_secret, - ) - return CloudCustomDestinationDefinition._from_docker_response(self, result) # noqa: SLF001 - - def list_custom_destination_definitions( - self, - *, - name: str | None = None, - ) -> list[CloudCustomDestinationDefinition]: - """List custom destination connector definitions. - - Args: - name: Filter by exact name match - - Returns: - List of CloudCustomDestinationDefinition objects - """ - definitions = api_util.list_custom_docker_destination_definitions( - workspace_id=self.workspace_id, - api_root=self.api_root, - client_id=self.client_id, - client_secret=self.client_secret, - ) - - return [ - CloudCustomDestinationDefinition._from_docker_response(self, d) # noqa: SLF001 - for d in definitions - if name is None or d.name == name - ] - - def get_custom_destination_definition( - self, - definition_id: str, - ) -> CloudCustomDestinationDefinition: - """Get a specific custom destination definition by ID. - - Args: - definition_id: The definition ID - - Returns: - CloudCustomDestinationDefinition object - """ - result = api_util.get_custom_docker_destination_definition( - workspace_id=self.workspace_id, - definition_id=definition_id, - api_root=self.api_root, - client_id=self.client_id, - client_secret=self.client_secret, - ) - return CloudCustomDestinationDefinition._from_docker_response(self, result) # noqa: SLF001 - - def permanently_delete_custom_destination_definition( - self, - definition_id: str, - ) -> None: - """Permanently delete a custom destination definition. - - Args: - definition_id: The definition ID to delete - """ - api_util.delete_custom_docker_destination_definition( - workspace_id=self.workspace_id, - definition_id=definition_id, - api_root=self.api_root, - client_id=self.client_id, - client_secret=self.client_secret, - ) diff --git a/airbyte/mcp/cloud_ops.py b/airbyte/mcp/cloud_ops.py index f194ebb5..737a073c 100644 --- a/airbyte/mcp/cloud_ops.py +++ b/airbyte/mcp/cloud_ops.py @@ -509,41 +509,15 @@ def publish_custom_source_definition( ], *, manifest_yaml: Annotated[ - dict | str | None, - Field( - description=( - "The Low-code CDK manifest as a dict or YAML string. " - "Required for YAML connectors. Mutually exclusive with docker_image." - ), - default=None, - ), - ] = None, - docker_image: Annotated[ - str | None, - Field( - description=( - "Docker repository (e.g., 'airbyte/source-custom'). " - "Required for Docker connectors." - ), - default=None, - ), - ] = None, - docker_tag: Annotated[ - str | None, + str | Path | None, Field( description=( - "Docker image tag (e.g., '1.0.0'). " "Required when docker_image is specified." + "The Low-code CDK manifest as a YAML string or file path. " + "Required for YAML connectors." ), default=None, ), ] = None, - documentation_url: Annotated[ - str | None, - Field( - description="Optional URL to connector documentation (Docker only).", - default=None, - ), - ] = None, unique: Annotated[ bool, Field( @@ -554,59 +528,42 @@ def publish_custom_source_definition( pre_validate: Annotated[ bool, Field( - description=( - "Whether to validate the manifest client-side " "before publishing (YAML only)." - ), + description="Whether to validate the manifest client-side before publishing.", default=True, ), ] = True, ) -> str: - """Publish a custom source connector definition to Airbyte Cloud. + """Publish a custom YAML source connector definition to Airbyte Cloud. - Supports both YAML (declarative) and Docker-based custom source definitions. - You must specify EITHER manifest_yaml OR docker_image + docker_tag, but not both. + Note: Only YAML (declarative) connectors are currently supported. + Docker-based custom sources are not yet available. """ try: workspace: CloudWorkspace = _get_cloud_workspace() result = workspace.publish_custom_source_definition( name=name, manifest_yaml=manifest_yaml, - docker_image=docker_image, - docker_tag=docker_tag, - documentation_url=documentation_url, unique=unique, pre_validate=pre_validate, ) except Exception as ex: return f"Failed to publish custom source definition '{name}': {ex}" else: - if result.connector_type == "yaml": - return ( - f"Successfully published custom YAML source definition '{name}' " - f"with ID '{result.definition_id}' (version {result.version or 'N/A'})" - ) return ( - f"Successfully published custom Docker source definition '{name}' " - f"with ID '{result.definition_id}' " - f"({result.docker_repository}:{result.docker_image_tag})" + f"Successfully published custom YAML source definition '{name}' " + f"with ID '{result.definition_id}' (version {result.version or 'N/A'})" ) -def list_custom_source_definitions( - custom_connector_type: Annotated[ - str, - Field( - description="Connector type to list: 'yaml' or 'docker'. Required.", - ), - ], -) -> list[dict[str, Any]]: - """List custom source definitions in the Airbyte Cloud workspace. +def list_custom_source_definitions() -> list[dict[str, Any]]: + """List custom YAML source definitions in the Airbyte Cloud workspace. - You must specify the connector type to list - either 'yaml' or 'docker'. + Note: Only YAML (declarative) connectors are currently supported. + Docker-based custom sources are not yet available. """ workspace: CloudWorkspace = _get_cloud_workspace() definitions = workspace.list_custom_source_definitions( - custom_connector_type=custom_connector_type, # type: ignore[arg-type] + custom_connector_type="yaml", ) return [ @@ -616,9 +573,6 @@ def list_custom_source_definitions( "connector_type": d.connector_type, "manifest": d.manifest, "version": d.version, - "docker_repository": d.docker_repository, - "docker_image_tag": d.docker_image_tag, - "documentation_url": d.documentation_url, } for d in definitions ] @@ -629,203 +583,42 @@ def update_custom_source_definition( str, Field(description="The ID of the definition to update."), ], - *, manifest_yaml: Annotated[ - str | Path | None, - Field( - description="New manifest as YAML string or file path (YAML connectors only).", - default=None, - ), - ] = None, - docker_tag: Annotated[ - str | None, + str | Path, Field( - description="New Docker image tag (Docker connectors only).", - default=None, - ), - ] = None, - custom_connector_type: Annotated[ - str, - Field( - description="Connector type: 'yaml' or 'docker'. Required.", + description="New manifest as YAML string or file path.", ), ], + *, pre_validate: Annotated[ bool, Field( - description="Whether to validate the manifest client-side before updating (YAML only).", + description="Whether to validate the manifest client-side before updating.", default=True, ), ] = True, ) -> str: - """Update a custom source definition in Airbyte Cloud. + """Update a custom YAML source definition in Airbyte Cloud. - You must specify EXACTLY ONE of manifest_yaml or docker_tag, but not both. - For YAML connectors: specify manifest_yaml - For Docker connectors: specify docker_tag (updates tag only, not name) - To rename a Docker connector, use rename_custom_source_definition instead. + Note: Only YAML (declarative) connectors are currently supported. + Docker-based custom sources are not yet available. """ try: workspace: CloudWorkspace = _get_cloud_workspace() definition = workspace.get_custom_source_definition( definition_id=definition_id, - custom_connector_type=custom_connector_type, # type: ignore[arg-type] + custom_connector_type="yaml", ) result = definition.update_definition( manifest_yaml=manifest_yaml, - docker_tag=docker_tag, pre_validate=pre_validate, ) except Exception as ex: return f"Failed to update custom source definition '{definition_id}': {ex}" - else: - if result.connector_type == "yaml": - return ( - f"Successfully updated custom YAML source definition. " - f"Name: {result.name}, version: {result.version or 'N/A'}" - ) - return ( - f"Successfully updated custom Docker source definition. " - f"Name: {result.name}, tag: {result.docker_image_tag}" - ) - - -def rename_custom_source_definition( - definition_id: Annotated[ - str, - Field(description="The ID of the definition to rename."), - ], - new_name: Annotated[ - str, - Field(description="New display name for the connector."), - ], - custom_connector_type: Annotated[ - str, - Field( - description=( - "Connector type: 'yaml' or 'docker'. " "Only Docker connectors can be renamed." - ), - ), - ], -) -> str: - """Rename a custom source definition in Airbyte Cloud. - - Note: Only Docker custom sources can be renamed. YAML custom sources - cannot be renamed as their names are derived from the manifest. - """ - try: - workspace: CloudWorkspace = _get_cloud_workspace() - definition = workspace.get_custom_source_definition( - definition_id=definition_id, - custom_connector_type=custom_connector_type, # type: ignore[arg-type] - ) - result = definition.rename(new_name=new_name) - except Exception as ex: - return f"Failed to rename custom source definition '{definition_id}': {ex}" - else: - return ( - f"Successfully renamed custom Docker source definition to '{result.name}' " - f"(ID: {result.definition_id})" - ) - - -def publish_custom_destination_definition( - name: Annotated[ - str, - Field(description="The name for the custom connector definition."), - ], - docker_image: Annotated[ - str, - Field(description="Docker repository (e.g., 'airbyte/destination-custom')."), - ], - docker_tag: Annotated[ - str, - Field(description="Docker image tag (e.g., '1.0.0')."), - ], - *, - documentation_url: Annotated[ - str | None, - Field( - description="Optional URL to connector documentation.", - default=None, - ), - ] = None, - unique: Annotated[ - bool, - Field( - description="Whether to require a unique name.", - default=True, - ), - ] = True, -) -> str: - """Publish a custom destination connector definition to Airbyte Cloud. - - Currently only Docker-based custom destinations are supported. - """ - try: - workspace: CloudWorkspace = _get_cloud_workspace() - result = workspace.publish_custom_destination_definition( - name=name, - docker_image=docker_image, - docker_tag=docker_tag, - documentation_url=documentation_url, - unique=unique, - ) - except Exception as ex: - return f"Failed to publish custom destination definition '{name}': {ex}" - else: - return ( - f"Successfully published custom Docker destination definition '{name}' " - f"with ID '{result.definition_id}' " - f"({result.docker_repository}:{result.docker_image_tag})" - ) - - -def list_custom_destination_definitions() -> list[dict[str, Any]]: - """List all custom destination definitions in the Airbyte Cloud workspace.""" - workspace: CloudWorkspace = _get_cloud_workspace() - definitions = workspace.list_custom_destination_definitions() - - return [ - { - "definition_id": d.definition_id, - "name": d.name, - "docker_repository": d.docker_repository, - "docker_image_tag": d.docker_image_tag, - "documentation_url": d.documentation_url, - } - for d in definitions - ] - - -def update_custom_destination_definition( - definition_id: Annotated[ - str, - Field(description="The ID of the definition to update."), - ], - name: Annotated[ - str, - Field(description="New name for the definition."), - ], - docker_tag: Annotated[ - str, - Field(description="New Docker image tag."), - ], -) -> str: - """Update a custom destination definition in Airbyte Cloud.""" - try: - workspace: CloudWorkspace = _get_cloud_workspace() - definition = workspace.get_custom_destination_definition(definition_id=definition_id) - result = definition.update_definition( - name=name, - docker_tag=docker_tag, - ) - except Exception as ex: - return f"Failed to update custom destination definition '{definition_id}': {ex}" else: return ( - f"Successfully updated custom Docker destination definition. " - f"Name: {result.name}, tag: {result.docker_image_tag}" + f"Successfully updated custom YAML source definition. " + f"Name: {result.name}, version: {result.version or 'N/A'}" ) @@ -848,7 +641,3 @@ def register_cloud_ops_tools(app: FastMCP) -> None: app.tool(publish_custom_source_definition) app.tool(list_custom_source_definitions) app.tool(update_custom_source_definition) - app.tool(rename_custom_source_definition) - app.tool(publish_custom_destination_definition) - app.tool(list_custom_destination_definitions) - app.tool(update_custom_destination_definition) diff --git a/tests/integration_tests/cloud/test_custom_definitions.py b/tests/integration_tests/cloud/test_custom_definitions.py index d2f4092a..db27f76b 100644 --- a/tests/integration_tests/cloud/test_custom_definitions.py +++ b/tests/integration_tests/cloud/test_custom_definitions.py @@ -99,107 +99,6 @@ def test_publish_custom_yaml_source( ) -@pytest.mark.skip( - reason="Docker custom definitions appear blocked in Airbyte Cloud - pending confirmation" -) -@pytest.mark.requires_creds -def test_publish_custom_docker_source( - cloud_workspace: CloudWorkspace, -) -> None: - """Test publishing a custom Docker source definition.""" - from airbyte._util import text_util - - name = f"test-docker-source-{text_util.generate_random_suffix()}" - - result = cloud_workspace.publish_custom_source_definition( - name=name, - docker_image="airbyte/test-source", - docker_tag="1.0.0", - documentation_url="https://example.com/docs", - unique=True, - ) - - assert result.definition_id - assert result.name == name - assert result.docker_repository == "airbyte/test-source" - assert result.docker_image_tag == "1.0.0" - assert result.connector_type == "docker" - - definition_id = result.definition_id - - try: - definitions = cloud_workspace.list_custom_source_definitions( - name=name, - custom_connector_type="docker", - ) - assert len(definitions) == 1 - assert definitions[0].definition_id == definition_id - - fetched = cloud_workspace.get_custom_source_definition( - definition_id, - custom_connector_type="docker", - ) - assert fetched.definition_id == definition_id - assert fetched.name == name - assert fetched.connector_type == "docker" - - updated = fetched.update_definition( - docker_tag="2.0.0", - ) - assert updated.docker_image_tag == "2.0.0" - - finally: - cloud_workspace.permanently_delete_custom_source_definition( - definition_id, - custom_connector_type="docker", - ) - - -@pytest.mark.skip( - reason="Docker custom definitions appear blocked in Airbyte Cloud - pending confirmation" -) -@pytest.mark.requires_creds -def test_publish_custom_docker_destination( - cloud_workspace: CloudWorkspace, -) -> None: - """Test publishing a custom Docker destination definition.""" - from airbyte._util import text_util - - name = f"test-docker-dest-{text_util.generate_random_suffix()}" - - result = cloud_workspace.publish_custom_destination_definition( - name=name, - docker_image="airbyte/test-destination", - docker_tag="1.0.0", - unique=True, - ) - - assert result.definition_id - assert result.name == name - assert result.docker_repository == "airbyte/test-destination" - assert result.docker_image_tag == "1.0.0" - - definition_id = result.definition_id - - try: - definitions = cloud_workspace.list_custom_destination_definitions(name=name) - assert len(definitions) == 1 - assert definitions[0].definition_id == definition_id - - fetched = cloud_workspace.get_custom_destination_definition(definition_id) - assert fetched.definition_id == definition_id - assert fetched.name == name - - updated = fetched.update_definition( - name=name, - docker_tag="2.0.0", - ) - assert updated.docker_image_tag == "2.0.0" - - finally: - cloud_workspace.permanently_delete_custom_destination_definition(definition_id) - - @pytest.mark.requires_creds def test_yaml_validation_error( cloud_workspace: CloudWorkspace, From fea82932373410b07bb9a08c0e44b1a4462f198a Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sat, 11 Oct 2025 01:06:09 +0000 Subject: [PATCH 8/9] refactor: Remove name filter from list_custom_source_definitions - Remove name parameter from list_custom_source_definitions method - Update uniqueness check in publish_custom_source_definition to filter results after fetching - As requested in PR #827 review feedback Co-Authored-By: AJ Steers --- airbyte/cloud/workspaces.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/airbyte/cloud/workspaces.py b/airbyte/cloud/workspaces.py index f205f7c0..c0ccb6f4 100644 --- a/airbyte/cloud/workspaces.py +++ b/airbyte/cloud/workspaces.py @@ -511,10 +511,9 @@ def publish_custom_source_definition( if unique: existing = self.list_custom_source_definitions( - name=name, custom_connector_type="yaml" if is_yaml else "docker", ) - if existing: + if any(d.name == name for d in existing): raise exc.AirbyteDuplicateResourcesError( resource_type="custom_source_definition", resource_name=name, @@ -555,13 +554,11 @@ def publish_custom_source_definition( def list_custom_source_definitions( self, *, - name: str | None = None, custom_connector_type: Literal["yaml", "docker"], ) -> list[CloudCustomSourceDefinition]: """List custom source connector definitions. Args: - name: Filter by exact name match custom_connector_type: Connector type to list ("yaml" or "docker"). Required. Returns: @@ -577,7 +574,6 @@ def list_custom_source_definitions( return [ CloudCustomSourceDefinition._from_yaml_response(self, d) # noqa: SLF001 for d in yaml_definitions - if name is None or d.name == name ] raise NotImplementedError( From 63fcbbdd90ce1e4cfe6eaf644d60eedf54a59420 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sat, 11 Oct 2025 01:31:18 +0000 Subject: [PATCH 9/9] fix: Update test to work without name parameter in list_custom_source_definitions - Remove name parameter from list_custom_source_definitions call in test - Add manual filtering to find definition by name - Fixes test failure after removing name filter from list method Co-Authored-By: AJ Steers --- tests/integration_tests/cloud/test_custom_definitions.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration_tests/cloud/test_custom_definitions.py b/tests/integration_tests/cloud/test_custom_definitions.py index db27f76b..f91a4e45 100644 --- a/tests/integration_tests/cloud/test_custom_definitions.py +++ b/tests/integration_tests/cloud/test_custom_definitions.py @@ -70,10 +70,10 @@ def test_publish_custom_yaml_source( definition_id = result.definition_id try: - definitions = cloud_workspace.list_custom_source_definitions( - name=name, + all_definitions = cloud_workspace.list_custom_source_definitions( custom_connector_type="yaml", ) + definitions = [d for d in all_definitions if d.name == name] assert len(definitions) == 1 assert definitions[0].definition_id == definition_id