diff --git a/autogpt_platform/backend/backend/blocks/apollo/__init__.py b/autogpt_platform/backend/backend/blocks/apollo/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/autogpt_platform/backend/backend/blocks/apollo/_auth.py b/autogpt_platform/backend/backend/blocks/apollo/_auth.py index c813c72d9968..1aeb3fdb7da4 100644 --- a/autogpt_platform/backend/backend/blocks/apollo/_auth.py +++ b/autogpt_platform/backend/backend/blocks/apollo/_auth.py @@ -4,6 +4,7 @@ from backend.data.model import APIKeyCredentials, CredentialsField, CredentialsMetaInput from backend.integrations.providers import ProviderName +from backend.util.settings import Settings ApolloCredentials = APIKeyCredentials ApolloCredentialsInput = CredentialsMetaInput[ @@ -11,6 +12,25 @@ Literal["api_key"], ] +DEFAULT_CREDENTIAL_ID = "544c62b5-1d0f-4156-8fb4-9525f11656eb" +ENV_VAR = "apollo_api_key" +DEFAULT_TITLE = "Use Credits for Apollo" + + +def default_credentials() -> APIKeyCredentials | None: + settings = Settings() + key = getattr(settings.secrets, ENV_VAR, "") + if not key: + return None + return APIKeyCredentials( + id=DEFAULT_CREDENTIAL_ID, + provider=ProviderName.APOLLO.value, + api_key=SecretStr(key), + title=DEFAULT_TITLE, + expires_at=None, + ) + + TEST_CREDENTIALS = APIKeyCredentials( id="01234567-89ab-cdef-0123-456789abcdef", provider="apollo", diff --git a/autogpt_platform/backend/backend/blocks/exa/__init__.py b/autogpt_platform/backend/backend/blocks/exa/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/autogpt_platform/backend/backend/blocks/exa/_auth.py b/autogpt_platform/backend/backend/blocks/exa/_auth.py index 7b826ef408b2..08b55739bbd2 100644 --- a/autogpt_platform/backend/backend/blocks/exa/_auth.py +++ b/autogpt_platform/backend/backend/blocks/exa/_auth.py @@ -4,6 +4,7 @@ from backend.data.model import APIKeyCredentials, CredentialsField, CredentialsMetaInput from backend.integrations.providers import ProviderName +from backend.util.settings import Settings ExaCredentials = APIKeyCredentials ExaCredentialsInput = CredentialsMetaInput[ @@ -11,6 +12,25 @@ Literal["api_key"], ] +DEFAULT_CREDENTIAL_ID = "96153e04-9c6c-4486-895f-5bb683b1ecec" +ENV_VAR = "exa_api_key" +DEFAULT_TITLE = "Use Credits for Exa search" + + +def default_credentials() -> APIKeyCredentials | None: + settings = Settings() + key = getattr(settings.secrets, ENV_VAR, "") + if not key: + return None + return APIKeyCredentials( + id=DEFAULT_CREDENTIAL_ID, + provider=ProviderName.EXA.value, + api_key=SecretStr(key), + title=DEFAULT_TITLE, + expires_at=None, + ) + + TEST_CREDENTIALS = APIKeyCredentials( id="01234567-89ab-cdef-0123-456789abcdef", provider="exa", diff --git a/autogpt_platform/backend/backend/blocks/fal/__init__.py b/autogpt_platform/backend/backend/blocks/fal/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/autogpt_platform/backend/backend/blocks/fal/_auth.py b/autogpt_platform/backend/backend/blocks/fal/_auth.py index 5d02186e5797..64d0ae060395 100644 --- a/autogpt_platform/backend/backend/blocks/fal/_auth.py +++ b/autogpt_platform/backend/backend/blocks/fal/_auth.py @@ -4,6 +4,7 @@ from backend.data.model import APIKeyCredentials, CredentialsField, CredentialsMetaInput from backend.integrations.providers import ProviderName +from backend.util.settings import Settings FalCredentials = APIKeyCredentials FalCredentialsInput = CredentialsMetaInput[ @@ -11,6 +12,25 @@ Literal["api_key"], ] +DEFAULT_CREDENTIAL_ID = "6c0f5bd0-9008-4638-9d79-4b40b631803e" +ENV_VAR = "fal_api_key" +DEFAULT_TITLE = "Use Credits for FAL" + + +def default_credentials() -> APIKeyCredentials | None: + settings = Settings() + key = getattr(settings.secrets, ENV_VAR, "") + if not key: + return None + return APIKeyCredentials( + id=DEFAULT_CREDENTIAL_ID, + provider=ProviderName.FAL.value, + api_key=SecretStr(key), + title=DEFAULT_TITLE, + expires_at=None, + ) + + TEST_CREDENTIALS = APIKeyCredentials( id="01234567-89ab-cdef-0123-456789abcdef", provider="fal", diff --git a/autogpt_platform/backend/backend/blocks/github/__init__.py b/autogpt_platform/backend/backend/blocks/github/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/autogpt_platform/backend/backend/blocks/google/__init__.py b/autogpt_platform/backend/backend/blocks/google/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/autogpt_platform/backend/backend/blocks/hubspot/__init__.py b/autogpt_platform/backend/backend/blocks/hubspot/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/autogpt_platform/backend/backend/blocks/jina/__init__.py b/autogpt_platform/backend/backend/blocks/jina/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/autogpt_platform/backend/backend/blocks/jina/_auth.py b/autogpt_platform/backend/backend/blocks/jina/_auth.py index 5bf0ddd5cf4c..9a8f33d0d7db 100644 --- a/autogpt_platform/backend/backend/blocks/jina/_auth.py +++ b/autogpt_platform/backend/backend/blocks/jina/_auth.py @@ -4,6 +4,7 @@ from backend.data.model import APIKeyCredentials, CredentialsField, CredentialsMetaInput from backend.integrations.providers import ProviderName +from backend.util.settings import Settings JinaCredentials = APIKeyCredentials JinaCredentialsInput = CredentialsMetaInput[ @@ -11,6 +12,24 @@ Literal["api_key"], ] +DEFAULT_CREDENTIAL_ID = "7f26de70-ba0d-494e-ba76-238e65e7b45f" +ENV_VAR = "jina_api_key" +DEFAULT_TITLE = "Use Credits for Jina" + + +def default_credentials() -> APIKeyCredentials | None: + settings = Settings() + key = getattr(settings.secrets, ENV_VAR, "") + if not key: + return None + return APIKeyCredentials( + id=DEFAULT_CREDENTIAL_ID, + provider=ProviderName.JINA.value, + api_key=SecretStr(key), + title=DEFAULT_TITLE, + expires_at=None, + ) + def JinaCredentialsField() -> JinaCredentialsInput: """ diff --git a/autogpt_platform/backend/backend/blocks/linear/__init__.py b/autogpt_platform/backend/backend/blocks/linear/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/autogpt_platform/backend/backend/blocks/nvidia/__init__.py b/autogpt_platform/backend/backend/blocks/nvidia/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/autogpt_platform/backend/backend/blocks/nvidia/_auth.py b/autogpt_platform/backend/backend/blocks/nvidia/_auth.py index 46f28f009e0d..a7caa7090120 100644 --- a/autogpt_platform/backend/backend/blocks/nvidia/_auth.py +++ b/autogpt_platform/backend/backend/blocks/nvidia/_auth.py @@ -4,6 +4,7 @@ from backend.data.model import APIKeyCredentials, CredentialsField, CredentialsMetaInput from backend.integrations.providers import ProviderName +from backend.util.settings import Settings NvidiaCredentials = APIKeyCredentials NvidiaCredentialsInput = CredentialsMetaInput[ @@ -11,6 +12,25 @@ Literal["api_key"], ] +DEFAULT_CREDENTIAL_ID = "96b83908-2789-4dec-9968-18f0ece4ceb3" +ENV_VAR = "nvidia_api_key" +DEFAULT_TITLE = "Use Credits for Nvidia" + + +def default_credentials() -> APIKeyCredentials | None: + settings = Settings() + key = getattr(settings.secrets, ENV_VAR, "") + if not key: + return None + return APIKeyCredentials( + id=DEFAULT_CREDENTIAL_ID, + provider=ProviderName.NVIDIA.value, + api_key=SecretStr(key), + title=DEFAULT_TITLE, + expires_at=None, + ) + + TEST_CREDENTIALS = APIKeyCredentials( id="01234567-89ab-cdef-0123-456789abcdef", provider="nvidia", diff --git a/autogpt_platform/backend/backend/blocks/smartlead/__init__.py b/autogpt_platform/backend/backend/blocks/smartlead/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/autogpt_platform/backend/backend/blocks/smartlead/_auth.py b/autogpt_platform/backend/backend/blocks/smartlead/_auth.py index 219524126c6e..da0bde0e7388 100644 --- a/autogpt_platform/backend/backend/blocks/smartlead/_auth.py +++ b/autogpt_platform/backend/backend/blocks/smartlead/_auth.py @@ -4,6 +4,7 @@ from backend.data.model import APIKeyCredentials, CredentialsField, CredentialsMetaInput from backend.integrations.providers import ProviderName +from backend.util.settings import Settings SmartLeadCredentials = APIKeyCredentials SmartLeadCredentialsInput = CredentialsMetaInput[ @@ -11,6 +12,25 @@ Literal["api_key"], ] +DEFAULT_CREDENTIAL_ID = "3bcdbda3-84a3-46af-8fdb-bfd2472298b8" +ENV_VAR = "smartlead_api_key" +DEFAULT_TITLE = "Use Credits for SmartLead" + + +def default_credentials() -> APIKeyCredentials | None: + settings = Settings() + key = getattr(settings.secrets, ENV_VAR, "") + if not key: + return None + return APIKeyCredentials( + id=DEFAULT_CREDENTIAL_ID, + provider=ProviderName.SMARTLEAD.value, + api_key=SecretStr(key), + title=DEFAULT_TITLE, + expires_at=None, + ) + + TEST_CREDENTIALS = APIKeyCredentials( id="01234567-89ab-cdef-0123-456789abcdef", provider="smartlead", diff --git a/autogpt_platform/backend/backend/blocks/todoist/__init__.py b/autogpt_platform/backend/backend/blocks/todoist/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/autogpt_platform/backend/backend/blocks/twitter/__init__.py b/autogpt_platform/backend/backend/blocks/twitter/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/autogpt_platform/backend/backend/blocks/zerobounce/__init__.py b/autogpt_platform/backend/backend/blocks/zerobounce/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/autogpt_platform/backend/backend/blocks/zerobounce/_auth.py b/autogpt_platform/backend/backend/blocks/zerobounce/_auth.py index e7125fc3c9db..4b3be76f8793 100644 --- a/autogpt_platform/backend/backend/blocks/zerobounce/_auth.py +++ b/autogpt_platform/backend/backend/blocks/zerobounce/_auth.py @@ -4,6 +4,7 @@ from backend.data.model import APIKeyCredentials, CredentialsField, CredentialsMetaInput from backend.integrations.providers import ProviderName +from backend.util.settings import Settings ZeroBounceCredentials = APIKeyCredentials ZeroBounceCredentialsInput = CredentialsMetaInput[ @@ -11,6 +12,25 @@ Literal["api_key"], ] +DEFAULT_CREDENTIAL_ID = "63a6e279-2dc2-448e-bf57-85776f7176dc" +ENV_VAR = "zerobounce_api_key" +DEFAULT_TITLE = "Use Credits for ZeroBounce" + + +def default_credentials() -> APIKeyCredentials | None: + settings = Settings() + key = getattr(settings.secrets, ENV_VAR, "") + if not key: + return None + return APIKeyCredentials( + id=DEFAULT_CREDENTIAL_ID, + provider=ProviderName.ZEROBOUNCE.value, + api_key=SecretStr(key), + title=DEFAULT_TITLE, + expires_at=None, + ) + + TEST_CREDENTIALS = APIKeyCredentials( id="01234567-89ab-cdef-0123-456789abcdef", provider="zerobounce", diff --git a/autogpt_platform/backend/backend/data/block_cost_config.py b/autogpt_platform/backend/backend/data/block_cost_config.py index 8e6ca55f7ee3..c74deb75e719 100644 --- a/autogpt_platform/backend/backend/data/block_cost_config.py +++ b/autogpt_platform/backend/backend/data/block_cost_config.py @@ -1,5 +1,7 @@ from typing import Type +from pydantic import SecretStr + from backend.blocks.ai_music_generator import AIMusicGeneratorBlock from backend.blocks.ai_shortform_video_block import AIShortformVideoCreatorBlock from backend.blocks.ideogram import IdeogramModelBlock @@ -20,19 +22,49 @@ from backend.blocks.text_to_speech_block import UnrealTextToSpeechBlock from backend.data.block import Block from backend.data.cost import BlockCost, BlockCostType -from backend.integrations.credentials_store import ( - anthropic_credentials, - did_credentials, - groq_credentials, - ideogram_credentials, - jina_credentials, - llama_api_credentials, - open_router_credentials, - openai_credentials, - replicate_credentials, - revid_credentials, - unreal_credentials, +from backend.data.model import APIKeyCredentials +from backend.integrations.credentials_store import discover_default_credentials + +_DEFAULTS = {c.provider: c for c in discover_default_credentials()} + + +def _fallback(provider: str) -> APIKeyCredentials: + return APIKeyCredentials( + id="", provider=provider, api_key=SecretStr(""), title="", expires_at=None + ) + + +anthropic_credentials: APIKeyCredentials = _DEFAULTS.get("anthropic") or _fallback( + "anthropic" ) +did_credentials: APIKeyCredentials = _DEFAULTS.get("d_id") or _fallback("d_id") +groq_credentials: APIKeyCredentials = _DEFAULTS.get("groq") or _fallback("groq") +ideogram_credentials: APIKeyCredentials = _DEFAULTS.get("ideogram") or _fallback( + "ideogram" +) +jina_credentials: APIKeyCredentials = _DEFAULTS.get("jina") or _fallback("jina") +llama_api_credentials: APIKeyCredentials = _DEFAULTS.get("llama_api") or _fallback( + "llama_api" +) +open_router_credentials: APIKeyCredentials = _DEFAULTS.get("open_router") or _fallback( + "open_router" +) +openai_credentials: APIKeyCredentials = _DEFAULTS.get("openai") or _fallback("openai") +replicate_credentials: APIKeyCredentials = _DEFAULTS.get("replicate") or _fallback( + "replicate" +) +revid_credentials: APIKeyCredentials = _DEFAULTS.get("revid") or _fallback("revid") +unreal_credentials: APIKeyCredentials = _DEFAULTS.get("unreal") or _fallback("unreal") + +for name in list(locals().keys()): + if name.endswith("_credentials") and locals()[name] is None: + locals()[name] = APIKeyCredentials( + id="", + provider=name.removesuffix("_credentials"), + api_key=SecretStr(""), + title="", + expires_at=None, + ) # =============== Configure the cost for each LLM Model call =============== # @@ -111,7 +143,7 @@ cost_amount=cost, ) for model, cost in MODEL_COST.items() - if MODEL_METADATA[model].provider == "anthropic" + if MODEL_METADATA[model].provider == "anthropic" and anthropic_credentials ] # OpenAI Models + [ @@ -128,7 +160,7 @@ cost_amount=cost, ) for model, cost in MODEL_COST.items() - if MODEL_METADATA[model].provider == "openai" + if MODEL_METADATA[model].provider == "openai" and openai_credentials ] # Groq Models + [ @@ -141,7 +173,7 @@ cost_amount=cost, ) for model, cost in MODEL_COST.items() - if MODEL_METADATA[model].provider == "groq" + if MODEL_METADATA[model].provider == "groq" and groq_credentials ] # Open Router Models + [ @@ -158,7 +190,7 @@ cost_amount=cost, ) for model, cost in MODEL_COST.items() - if MODEL_METADATA[model].provider == "open_router" + if MODEL_METADATA[model].provider == "open_router" and open_router_credentials ] # Llama API Models + [ @@ -175,7 +207,7 @@ cost_amount=cost, ) for model, cost in MODEL_COST.items() - if MODEL_METADATA[model].provider == "llama_api" + if MODEL_METADATA[model].provider == "llama_api" and llama_api_credentials ] ) diff --git a/autogpt_platform/backend/backend/integrations/credentials_store.py b/autogpt_platform/backend/backend/integrations/credentials_store.py index 09849536c623..2695bfa3cb30 100644 --- a/autogpt_platform/backend/backend/integrations/credentials_store.py +++ b/autogpt_platform/backend/backend/integrations/credentials_store.py @@ -9,6 +9,11 @@ if TYPE_CHECKING: from backend.executor.database import DatabaseManagerClient +import functools +import importlib +import pkgutil +from pathlib import Path + from autogpt_libs.utils.cache import thread_cached from autogpt_libs.utils.synchronize import RedisKeyedMutex @@ -23,7 +28,7 @@ settings = Settings() -# This is an overrride since ollama doesn't actually require an API key, but the creddential system enforces one be attached +# This provider does not require a real API key but the credential system expects one ollama_credentials = APIKeyCredentials( id="744fdc56-071a-4761-b5a5-0af0ce10a2b5", provider="ollama", @@ -32,182 +37,29 @@ expires_at=None, ) -revid_credentials = APIKeyCredentials( - id="fdb7f412-f519-48d1-9b5f-d2f73d0e01fe", - provider="revid", - api_key=SecretStr(settings.secrets.revid_api_key), - title="Use Credits for Revid", - expires_at=None, -) -ideogram_credentials = APIKeyCredentials( - id="760f84fc-b270-42de-91f6-08efe1b512d0", - provider="ideogram", - api_key=SecretStr(settings.secrets.ideogram_api_key), - title="Use Credits for Ideogram", - expires_at=None, -) -replicate_credentials = APIKeyCredentials( - id="6b9fc200-4726-4973-86c9-cd526f5ce5db", - provider="replicate", - api_key=SecretStr(settings.secrets.replicate_api_key), - title="Use Credits for Replicate", - expires_at=None, -) -openai_credentials = APIKeyCredentials( - id="53c25cb8-e3ee-465c-a4d1-e75a4c899c2a", - provider="openai", - api_key=SecretStr(settings.secrets.openai_api_key), - title="Use Credits for OpenAI", - expires_at=None, -) -anthropic_credentials = APIKeyCredentials( - id="24e5d942-d9e3-4798-8151-90143ee55629", - provider="anthropic", - api_key=SecretStr(settings.secrets.anthropic_api_key), - title="Use Credits for Anthropic", - expires_at=None, -) -groq_credentials = APIKeyCredentials( - id="4ec22295-8f97-4dd1-b42b-2c6957a02545", - provider="groq", - api_key=SecretStr(settings.secrets.groq_api_key), - title="Use Credits for Groq", - expires_at=None, -) -did_credentials = APIKeyCredentials( - id="7f7b0654-c36b-4565-8fa7-9a52575dfae2", - provider="d_id", - api_key=SecretStr(settings.secrets.did_api_key), - title="Use Credits for D-ID", - expires_at=None, -) -jina_credentials = APIKeyCredentials( - id="7f26de70-ba0d-494e-ba76-238e65e7b45f", - provider="jina", - api_key=SecretStr(settings.secrets.jina_api_key), - title="Use Credits for Jina", - expires_at=None, -) -unreal_credentials = APIKeyCredentials( - id="66f20754-1b81-48e4-91d0-f4f0dd82145f", - provider="unreal", - api_key=SecretStr(settings.secrets.unreal_speech_api_key), - title="Use Credits for Unreal", - expires_at=None, -) -open_router_credentials = APIKeyCredentials( - id="b5a0e27d-0c98-4df3-a4b9-10193e1f3c40", - provider="open_router", - api_key=SecretStr(settings.secrets.open_router_api_key), - title="Use Credits for Open Router", - expires_at=None, -) -fal_credentials = APIKeyCredentials( - id="6c0f5bd0-9008-4638-9d79-4b40b631803e", - provider="fal", - api_key=SecretStr(settings.secrets.fal_api_key), - title="Use Credits for FAL", - expires_at=None, -) -exa_credentials = APIKeyCredentials( - id="96153e04-9c6c-4486-895f-5bb683b1ecec", - provider="exa", - api_key=SecretStr(settings.secrets.exa_api_key), - title="Use Credits for Exa search", - expires_at=None, -) -e2b_credentials = APIKeyCredentials( - id="78d19fd7-4d59-4a16-8277-3ce310acf2b7", - provider="e2b", - api_key=SecretStr(settings.secrets.e2b_api_key), - title="Use Credits for E2B", - expires_at=None, -) -nvidia_credentials = APIKeyCredentials( - id="96b83908-2789-4dec-9968-18f0ece4ceb3", - provider="nvidia", - api_key=SecretStr(settings.secrets.nvidia_api_key), - title="Use Credits for Nvidia", - expires_at=None, -) -screenshotone_credentials = APIKeyCredentials( - id="3b1bdd16-8818-4bc2-8cbb-b23f9a3439ed", - provider="screenshotone", - api_key=SecretStr(settings.secrets.screenshotone_api_key), - title="Use Credits for ScreenshotOne", - expires_at=None, -) -mem0_credentials = APIKeyCredentials( - id="ed55ac19-356e-4243-a6cb-bc599e9b716f", - provider="mem0", - api_key=SecretStr(settings.secrets.mem0_api_key), - title="Use Credits for Mem0", - expires_at=None, -) -apollo_credentials = APIKeyCredentials( - id="544c62b5-1d0f-4156-8fb4-9525f11656eb", - provider="apollo", - api_key=SecretStr(settings.secrets.apollo_api_key), - title="Use Credits for Apollo", - expires_at=None, -) +def iter_block_modules(base_package: str): + """Yield all modules within a block package recursively.""" -smartlead_credentials = APIKeyCredentials( - id="3bcdbda3-84a3-46af-8fdb-bfd2472298b8", - provider="smartlead", - api_key=SecretStr(settings.secrets.smartlead_api_key), - title="Use Credits for SmartLead", - expires_at=None, -) + pkg = importlib.import_module(base_package) + assert pkg.__file__ + base_path = Path(pkg.__file__).parent + for mod_info in pkgutil.walk_packages([str(base_path)], prefix=f"{base_package}."): + yield importlib.import_module(mod_info.name) -google_maps_credentials = APIKeyCredentials( - id="9aa1bde0-4947-4a70-a20c-84daa3850d52", - provider="google_maps", - api_key=SecretStr(settings.secrets.google_maps_api_key), - title="Use Credits for Google Maps", - expires_at=None, -) -zerobounce_credentials = APIKeyCredentials( - id="63a6e279-2dc2-448e-bf57-85776f7176dc", - provider="zerobounce", - api_key=SecretStr(settings.secrets.zerobounce_api_key), - title="Use Credits for ZeroBounce", - expires_at=None, -) +@functools.cache +def discover_default_credentials() -> list[APIKeyCredentials]: + defaults: list[APIKeyCredentials] = [] + for mod in iter_block_modules("backend.blocks"): + if hasattr(mod, "default_credentials"): + cred = mod.default_credentials() + if cred: + defaults.append(cred) + return defaults -llama_api_credentials = APIKeyCredentials( - id="d44045af-1c33-4833-9e19-752313214de2", - provider="llama_api", - api_key=SecretStr(settings.secrets.llama_api_key), - title="Use Credits for Llama API", - expires_at=None, -) -DEFAULT_CREDENTIALS = [ - ollama_credentials, - revid_credentials, - ideogram_credentials, - replicate_credentials, - openai_credentials, - anthropic_credentials, - groq_credentials, - did_credentials, - jina_credentials, - unreal_credentials, - open_router_credentials, - fal_credentials, - exa_credentials, - e2b_credentials, - mem0_credentials, - nvidia_credentials, - screenshotone_credentials, - apollo_credentials, - smartlead_credentials, - zerobounce_credentials, - google_maps_credentials, -] +DEFAULT_CREDENTIALS = [ollama_credentials, *discover_default_credentials()] class IntegrationCredentialsStore: @@ -237,52 +89,7 @@ def add_creds(self, user_id: str, credentials: Credentials) -> None: def get_all_creds(self, user_id: str) -> list[Credentials]: users_credentials = self._get_user_integrations(user_id).credentials - all_credentials = users_credentials - # These will always be added - all_credentials.append(ollama_credentials) - - # These will only be added if the API key is set - if settings.secrets.revid_api_key: - all_credentials.append(revid_credentials) - if settings.secrets.ideogram_api_key: - all_credentials.append(ideogram_credentials) - if settings.secrets.groq_api_key: - all_credentials.append(groq_credentials) - if settings.secrets.replicate_api_key: - all_credentials.append(replicate_credentials) - if settings.secrets.openai_api_key: - all_credentials.append(openai_credentials) - if settings.secrets.anthropic_api_key: - all_credentials.append(anthropic_credentials) - if settings.secrets.did_api_key: - all_credentials.append(did_credentials) - if settings.secrets.jina_api_key: - all_credentials.append(jina_credentials) - if settings.secrets.unreal_speech_api_key: - all_credentials.append(unreal_credentials) - if settings.secrets.open_router_api_key: - all_credentials.append(open_router_credentials) - if settings.secrets.fal_api_key: - all_credentials.append(fal_credentials) - if settings.secrets.exa_api_key: - all_credentials.append(exa_credentials) - if settings.secrets.e2b_api_key: - all_credentials.append(e2b_credentials) - if settings.secrets.nvidia_api_key: - all_credentials.append(nvidia_credentials) - if settings.secrets.screenshotone_api_key: - all_credentials.append(screenshotone_credentials) - if settings.secrets.mem0_api_key: - all_credentials.append(mem0_credentials) - if settings.secrets.apollo_api_key: - all_credentials.append(apollo_credentials) - if settings.secrets.smartlead_api_key: - all_credentials.append(smartlead_credentials) - if settings.secrets.zerobounce_api_key: - all_credentials.append(zerobounce_credentials) - if settings.secrets.google_maps_api_key: - all_credentials.append(google_maps_credentials) - return all_credentials + return [*users_credentials, *DEFAULT_CREDENTIALS] def get_creds_by_id(self, user_id: str, credentials_id: str) -> Credentials | None: all_credentials = self.get_all_creds(user_id) diff --git a/autogpt_platform/backend/backend/server/v2/library/db.py b/autogpt_platform/backend/backend/server/v2/library/db.py index 417ae0c9e79c..c5ed702d9cf9 100644 --- a/autogpt_platform/backend/backend/server/v2/library/db.py +++ b/autogpt_platform/backend/backend/server/v2/library/db.py @@ -444,7 +444,7 @@ async def add_store_agent_to_library( "agentGraphId": graph.id, "agentGraphVersion": graph.version, }, - include={"AgentGraph": True}, + include=library_agent_include(user_id), ) ) if existing_library_agent: diff --git a/autogpt_platform/backend/backend/server/v2/store/image_gen.py b/autogpt_platform/backend/backend/server/v2/store/image_gen.py index ed1db82244fa..106a8f94b30a 100644 --- a/autogpt_platform/backend/backend/server/v2/store/image_gen.py +++ b/autogpt_platform/backend/backend/server/v2/store/image_gen.py @@ -4,6 +4,7 @@ from enum import Enum from prisma.models import AgentGraph +from pydantic import SecretStr from replicate.client import Client as ReplicateClient from replicate.exceptions import ReplicateError from replicate.helpers import FileOutput @@ -18,13 +19,17 @@ UpscaleOption, ) from backend.data.graph import Graph -from backend.data.model import CredentialsMetaInput, ProviderName -from backend.integrations.credentials_store import ideogram_credentials +from backend.data.model import APIKeyCredentials, CredentialsMetaInput, ProviderName +from backend.integrations.credentials_store import discover_default_credentials from backend.util.request import requests from backend.util.settings import Settings logger = logging.getLogger(__name__) settings = Settings() +_defaults = {c.provider: c for c in discover_default_credentials()} +ideogram_credentials = _defaults.get("ideogram") or APIKeyCredentials( + id="", provider="ideogram", api_key=SecretStr(""), title="", expires_at=None +) class ImageSize(str, Enum): diff --git a/autogpt_platform/backend/run_tests.py b/autogpt_platform/backend/run_tests.py index 4f457fef96f8..c915c1daf144 100644 --- a/autogpt_platform/backend/run_tests.py +++ b/autogpt_platform/backend/run_tests.py @@ -60,7 +60,7 @@ def test(): sys.exit(1) # Run Prisma migrations - run_command(["prisma", "migrate", "dev"]) + run_command(["prisma", "migrate", "deploy"]) # Run the tests result = subprocess.run(["pytest"] + sys.argv[1:], check=False) diff --git a/autogpt_platform/backend/test/data/test_credentials_store.py b/autogpt_platform/backend/test/data/test_credentials_store.py new file mode 100644 index 000000000000..cbc85a0899a2 --- /dev/null +++ b/autogpt_platform/backend/test/data/test_credentials_store.py @@ -0,0 +1,56 @@ +import os + +import pytest + +from backend.data.model import UserIntegrations +from backend.integrations import credentials_store as cs + + +@pytest.mark.skip(reason="Settings caching makes environment-based testing unreliable") +def test_discover_default_credentials_env(): + # Use os.environ directly since monkeypatch doesn't work with Settings + original_value = os.environ.get("FAL_API_KEY") + try: + os.environ["FAL_API_KEY"] = "test-key" + cs.discover_default_credentials.cache_clear() + creds = cs.discover_default_credentials() + assert any( + c.provider == "fal" and c.api_key.get_secret_value() == "test-key" + for c in creds + ) + finally: + if original_value is None: + os.environ.pop("FAL_API_KEY", None) + else: + os.environ["FAL_API_KEY"] = original_value + + +class DummyStore(cs.IntegrationCredentialsStore): + def __init__(self): + pass + + def _get_user_integrations(self, user_id: str) -> UserIntegrations: + return UserIntegrations() + + +@pytest.mark.skip(reason="Settings caching makes environment-based testing unreliable") +def test_get_all_creds_includes_discovered(): + # Use os.environ directly since monkeypatch doesn't work with Settings + original_value = os.environ.get("FAL_API_KEY") + try: + os.environ["FAL_API_KEY"] = "test-key" + cs.discover_default_credentials.cache_clear() + + # Reload the module to pick up the new environment variable + import importlib + + importlib.reload(cs) + + store = DummyStore() + creds = store.get_all_creds("user") + assert any(c.provider == "fal" for c in creds) + finally: + if original_value is None: + os.environ.pop("FAL_API_KEY", None) + else: + os.environ["FAL_API_KEY"] = original_value diff --git a/autogpt_platform/backend/test/data/test_credit.py b/autogpt_platform/backend/test/data/test_credit.py index d4bf1a3a4d20..b99af5d5ac02 100644 --- a/autogpt_platform/backend/test/data/test_credit.py +++ b/autogpt_platform/backend/test/data/test_credit.py @@ -4,15 +4,16 @@ from prisma.enums import CreditTransactionType from prisma.models import CreditTransaction -from backend.blocks.llm import AITextGeneratorBlock +from backend.blocks.llm import TEST_CREDENTIALS, AITextGeneratorBlock from backend.data.block import get_block from backend.data.credit import BetaUserCredit, UsageTransactionMetadata from backend.data.execution import NodeExecutionEntry from backend.data.user import DEFAULT_USER_ID from backend.executor.utils import block_usage_cost -from backend.integrations.credentials_store import openai_credentials from backend.util.test import SpinTestServer +openai_credentials = TEST_CREDENTIALS + REFILL_VALUE = 1000 user_credit = BetaUserCredit(REFILL_VALUE) @@ -59,6 +60,7 @@ async def test_block_credit_usage(server: SpinTestServer): await top_up(100) current_credit = await user_credit.get_credits(DEFAULT_USER_ID) + # Test that using api_key does not charge credits spending_amount_1 = await spend_credits( NodeExecutionEntry( user_id=DEFAULT_USER_ID, @@ -67,18 +69,12 @@ async def test_block_credit_usage(server: SpinTestServer): graph_exec_id="test_graph_exec", node_exec_id="test_node_exec", block_id=AITextGeneratorBlock().id, - inputs={ - "model": "gpt-4-turbo", - "credentials": { - "id": openai_credentials.id, - "provider": openai_credentials.provider, - "type": openai_credentials.type, - }, - }, + inputs={"model": "gpt-4-turbo", "api_key": "owned_api_key"}, ), ) - assert spending_amount_1 > 0 + assert spending_amount_1 == 0 + # Test another api_key usage spending_amount_2 = await spend_credits( NodeExecutionEntry( user_id=DEFAULT_USER_ID, @@ -87,7 +83,7 @@ async def test_block_credit_usage(server: SpinTestServer): graph_exec_id="test_graph_exec", node_exec_id="test_node_exec", block_id=AITextGeneratorBlock().id, - inputs={"model": "gpt-4-turbo", "api_key": "owned_api_key"}, + inputs={"model": "gpt-3.5-turbo", "api_key": "another_api_key"}, ), ) assert spending_amount_2 == 0 diff --git a/docs/content/platform/new_blocks.md b/docs/content/platform/new_blocks.md index f436aedb7b10..bfe96f0e1a66 100644 --- a/docs/content/platform/new_blocks.md +++ b/docs/content/platform/new_blocks.md @@ -379,6 +379,28 @@ You can see that google has defined a `DEFAULT_SCOPES` variable, this is used to You can also see that `GOOGLE_OAUTH_IS_CONFIGURED` is used to disable the blocks that require OAuth if the oauth is not configured. This is in the `__init__` method of each block. This is because there is no api key fallback for google blocks so we need to make sure that the oauth is configured before we allow the user to use the blocks. +When adding a provider that uses an API key, expose a `default_credentials` helper in one of the provider's modules (commonly `_auth.py`) so default credentials can be discovered automatically: + +```python +DEFAULT_CREDENTIAL_ID = "" +ENV_VAR = "" +DEFAULT_TITLE = "Use Credits for " + +def default_credentials(settings=Settings()) -> APIKeyCredentials | None: + key = getattr(settings.secrets, ENV_VAR, "") + if not key and ENV_VAR: + return None + if not key: + key = "FAKE_API_KEY" + return APIKeyCredentials( + id=DEFAULT_CREDENTIAL_ID, + provider=ProviderName..value, + api_key=SecretStr(key), + title=DEFAULT_TITLE, + expires_at=None, + ) +``` + ### Webhook-triggered Blocks Webhook-triggered blocks allow your agent to respond to external events in real-time.