From de3436d02f86479292274fb7248432eb5346d9ae Mon Sep 17 00:00:00 2001 From: Devis Lucato Date: Mon, 8 May 2023 15:52:48 -0700 Subject: [PATCH] Python: fixes and release (#855) * Fix lint errors * Bump version number * Add retry loop to integration tests * Enable all integration tests: some tests will fail if OpenAI/Azure are throttling. --- python/pyproject.toml | 2 +- .../services/hf_text_completion.py | 10 ++- .../services/hf_text_embedding.py | 11 ++- python/semantic_kernel/kernel.py | 1 + .../completions/e2e_text_completion.py | 80 +++++++++++++------ .../test_azure_oai_chat_service.py | 8 +- .../test_azure_oai_text_service.py | 4 + .../completions/test_oai_chat_service.py | 8 +- .../completions/test_oai_text_service.py | 8 +- .../embeddings/test_oai_embedding_service.py | 1 - 10 files changed, 87 insertions(+), 46 deletions(-) diff --git a/python/pyproject.toml b/python/pyproject.toml index 2500ca5c62a0..58cbb9e31165 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "semantic-kernel" -version = "0.2.6.dev" +version = "0.2.7.dev" description = "" authors = ["Microsoft "] readme = "pip/README.md" diff --git a/python/semantic_kernel/connectors/ai/hugging_face/services/hf_text_completion.py b/python/semantic_kernel/connectors/ai/hugging_face/services/hf_text_completion.py index 9215f0b32af7..bc19d8886f4a 100644 --- a/python/semantic_kernel/connectors/ai/hugging_face/services/hf_text_completion.py +++ b/python/semantic_kernel/connectors/ai/hugging_face/services/hf_text_completion.py @@ -3,7 +3,6 @@ from logging import Logger from typing import Optional - from semantic_kernel.connectors.ai.ai_exception import AIException from semantic_kernel.connectors.ai.complete_request_settings import ( CompleteRequestSettings, @@ -48,11 +47,13 @@ def __init__( self._log = log if log is not None else NullLogger() try: - import transformers import torch + import transformers except (ImportError, ModuleNotFoundError): - raise ImportError("Please ensure that torch and transformers are installed to use HuggingFaceTextCompletion") - + raise ImportError( + "Please ensure that torch and transformers are installed to use HuggingFaceTextCompletion" + ) + self.device = ( "cuda:" + device if device >= 0 and torch.cuda.is_available() else "cpu" ) @@ -75,6 +76,7 @@ async def complete_async( """ try: import transformers + generation_config = transformers.GenerationConfig( temperature=request_settings.temperature, top_p=request_settings.top_p, diff --git a/python/semantic_kernel/connectors/ai/hugging_face/services/hf_text_embedding.py b/python/semantic_kernel/connectors/ai/hugging_face/services/hf_text_embedding.py index 867a1a02fe43..137ec90f33f0 100644 --- a/python/semantic_kernel/connectors/ai/hugging_face/services/hf_text_embedding.py +++ b/python/semantic_kernel/connectors/ai/hugging_face/services/hf_text_embedding.py @@ -2,6 +2,7 @@ from logging import Logger from typing import List, Optional + from numpy import array, ndarray from semantic_kernel.connectors.ai.ai_exception import AIException @@ -37,11 +38,13 @@ def __init__( self._log = log if log is not None else NullLogger() try: - import torch import sentence_transformers - except (ImportError): - raise ImportError("Please ensure that torch and sentence-transformers are installed to use HuggingFaceTextEmbedding") - + import torch + except ImportError: + raise ImportError( + "Please ensure that torch and sentence-transformers are installed to use HuggingFaceTextEmbedding" + ) + self.device = ( "cuda:" + device if device >= 0 and torch.cuda.is_available() else "cpu" ) diff --git a/python/semantic_kernel/kernel.py b/python/semantic_kernel/kernel.py index 652c495024c9..b6b5597e9fd5 100644 --- a/python/semantic_kernel/kernel.py +++ b/python/semantic_kernel/kernel.py @@ -49,6 +49,7 @@ T = TypeVar("T") + class Kernel(KernelBase, KernelExtensions): _log: Logger _skill_collection: SkillCollectionBase diff --git a/python/tests/integration/completions/e2e_text_completion.py b/python/tests/integration/completions/e2e_text_completion.py index 31999d45b77a..8a363c850d65 100644 --- a/python/tests/integration/completions/e2e_text_completion.py +++ b/python/tests/integration/completions/e2e_text_completion.py @@ -1,5 +1,25 @@ +import logging +import time + import semantic_kernel as sk +logging.basicConfig(level=logging.DEBUG) +logger = logging.getLogger() + + +async def retry(func, retries=15, delay=1): + for i in range(retries): + try: + result = str(await func()) + if "Error" in result: + raise ValueError(result) + return result + except Exception as e: + logger.error(f"Retry {i + 1}: {e}") + if i == retries - 1: # Last retry + raise + time.sleep(delay) + async def summarize_function_test(kernel: sk.Kernel): # Define semantic function using SK prompt template language @@ -33,68 +53,80 @@ async def summarize_function_test(kernel: sk.Kernel): print() # Summarize input string and print - summary = await kernel.run_async(tldr_function, input_str=text_to_summarize) - + summary = await retry( + lambda: kernel.run_async(tldr_function, input_str=text_to_summarize) + ) output = str(summary).strip() print(f"Summary using input string: '{output}'") - assert len(output.split(" ")) == 5 + assert "humans" in output or "Humans" in output or "preserve" in output + assert len(output) < 100 # Summarize input as context variable and print context_vars = sk.ContextVariables(text_to_summarize) - summary = await kernel.run_async(tldr_function, input_vars=context_vars) - + summary = await retry( + lambda: kernel.run_async(tldr_function, input_vars=context_vars) + ) output = str(summary).strip() print(f"Summary using context variables: '{output}'") - assert len(output.split(" ")) == 5 + assert "humans" in output or "Humans" in output or "preserve" in output + assert len(output) < 100 # Summarize input context and print context = kernel.create_new_context() context["input"] = text_to_summarize - summary = await kernel.run_async(tldr_function, input_context=context) - + summary = await retry( + lambda: kernel.run_async(tldr_function, input_context=context) + ) output = str(summary).strip() print(f"Summary using input context: '{output}'") - assert len(output.split(" ")) == 5 + assert "humans" in output or "Humans" in output or "preserve" in output + assert len(output) < 100 # Summarize input context with additional variables and print context = kernel.create_new_context() context["input"] = text_to_summarize context_vars = sk.ContextVariables("4) All birds are robots.") - summary = await kernel.run_async( - tldr_function, input_context=context, input_vars=context_vars + summary = await retry( + lambda: kernel.run_async( + tldr_function, input_context=context, input_vars=context_vars + ) ) - output = str(summary).strip() print(f"Summary using context and additional variables: '{output}'") - assert len(output.split(" ")) == 5 + assert "humans" in output or "Humans" in output or "preserve" in output + assert len(output) < 100 # Summarize input context with additional input string and print context = kernel.create_new_context() context["input"] = text_to_summarize - summary = await kernel.run_async( - tldr_function, input_context=context, input_str="4) All birds are robots." + summary = await retry( + lambda: kernel.run_async( + tldr_function, input_context=context, input_str="4) All birds are robots." + ) ) - output = str(summary).strip() print(f"Summary using context and additional string: '{output}'") - assert len(output.split(" ")) == 5 + assert "humans" in output or "Humans" in output or "preserve" in output + assert len(output) < 100 # Summarize input context with additional variables and string and print context = kernel.create_new_context() context["input"] = text_to_summarize context_vars = sk.ContextVariables(variables={"input2": "4) All birds are robots."}) - summary = await kernel.run_async( - tldr_function, - input_context=context, - input_vars=context_vars, - input_str="new text", + summary = await retry( + lambda: kernel.run_async( + tldr_function, + input_context=context, + input_vars=context_vars, + input_str="new text", + ) ) - output = str(summary).strip() print( f"Summary using context, additional variables, and additional string: '{output}'" ) - assert len(output.split(" ")) == 5 + assert "humans" in output or "Humans" in output or "preserve" in output + assert len(output) < 100 async def simple_summarization(kernel: sk.Kernel): diff --git a/python/tests/integration/completions/test_azure_oai_chat_service.py b/python/tests/integration/completions/test_azure_oai_chat_service.py index 6a62496de392..642797067e2e 100644 --- a/python/tests/integration/completions/test_azure_oai_chat_service.py +++ b/python/tests/integration/completions/test_azure_oai_chat_service.py @@ -11,10 +11,6 @@ @pytest.mark.asyncio -@pytest.mark.xfail( - raises=AssertionError, - reason="Azure OpenAI may throttle requests, preventing this test from passing", -) async def test_azure_chat_completion_with_skills(): kernel = sk.Kernel() @@ -27,6 +23,10 @@ async def test_azure_chat_completion_with_skills(): deployment_name, api_key, endpoint = sk.azure_openai_settings_from_dot_env() deployment_name = "gpt-4" + print("* Service: Azure OpenAI Chat Completion") + print(f"* Endpoint: {endpoint}") + print(f"* Deployment: {deployment_name}") + # Configure LLM service kernel.add_chat_service( "chat_completion", diff --git a/python/tests/integration/completions/test_azure_oai_text_service.py b/python/tests/integration/completions/test_azure_oai_text_service.py index a4fd8d23fe6e..c0d84c714373 100644 --- a/python/tests/integration/completions/test_azure_oai_text_service.py +++ b/python/tests/integration/completions/test_azure_oai_text_service.py @@ -23,6 +23,10 @@ async def test_azure_text_completion_with_skills(): deployment_name, api_key, endpoint = sk.azure_openai_settings_from_dot_env() deployment_name = "text-davinci-003" + print("* Service: Azure OpenAI Text Completion") + print(f"* Endpoint: {endpoint}") + print(f"* Deployment: {deployment_name}") + # Configure LLM service kernel.add_text_completion_service( "text_completion", diff --git a/python/tests/integration/completions/test_oai_chat_service.py b/python/tests/integration/completions/test_oai_chat_service.py index 3ccd118c6580..b07b4f9cf37f 100644 --- a/python/tests/integration/completions/test_oai_chat_service.py +++ b/python/tests/integration/completions/test_oai_chat_service.py @@ -11,10 +11,6 @@ @pytest.mark.asyncio -@pytest.mark.xfail( - raises=AssertionError, - reason="OpenAI may throttle requests, preventing this test from passing", -) async def test_oai_chat_service_with_skills(): kernel = sk.Kernel() @@ -25,6 +21,10 @@ async def test_oai_chat_service_with_skills(): # Load credentials from .env file api_key, org_id = sk.openai_settings_from_dot_env() + print("* Service: OpenAI Chat Completion") + print("* Endpoint: OpenAI") + print("* Model: gpt-3.5-turbo") + kernel.add_chat_service( "chat-gpt", sk_oai.OpenAIChatCompletion("gpt-3.5-turbo", api_key, org_id) ) diff --git a/python/tests/integration/completions/test_oai_text_service.py b/python/tests/integration/completions/test_oai_text_service.py index 2958ba5cfd4c..582f5e7d42c9 100644 --- a/python/tests/integration/completions/test_oai_text_service.py +++ b/python/tests/integration/completions/test_oai_text_service.py @@ -11,10 +11,6 @@ @pytest.mark.asyncio -@pytest.mark.xfail( - raises=AssertionError, - reason="OpenAI may throttle requests, preventing this test from passing", -) async def test_oai_text_completion_with_skills(): kernel = sk.Kernel() @@ -25,6 +21,10 @@ async def test_oai_text_completion_with_skills(): # Load credentials from .env file api_key, org_id = sk.openai_settings_from_dot_env() + print("* Service: OpenAI Text Completion") + print("* Endpoint: OpenAI") + print("* Model: text-davinci-003") + kernel.add_chat_service( "davinci-003", sk_oai.OpenAITextCompletion("text-davinci-003", api_key, org_id) ) diff --git a/python/tests/integration/embeddings/test_oai_embedding_service.py b/python/tests/integration/embeddings/test_oai_embedding_service.py index 65a4763fe6fb..b919267f5687 100644 --- a/python/tests/integration/embeddings/test_oai_embedding_service.py +++ b/python/tests/integration/embeddings/test_oai_embedding_service.py @@ -11,7 +11,6 @@ @pytest.mark.asyncio -# @pytest.mark.xfail(raises=AssertionError, reason="OpenAI may throttle requests, preventing this test from passing") async def test_oai_embedding_service_with_memories(): kernel = sk.Kernel()