Merge remote-tracking branch 'origin/main' into feature-declarative-agents

Author: markwallace-microsoft

dotnet/samples/Concepts/ChatCompletion/HybridCompletion_Fallback.cs

@@ -2,13 +2,15 @@
 
 using System.ClientModel;
 using System.ClientModel.Primitives;
-using System.ComponentModel;
 using System.Net;
 using System.Runtime.CompilerServices;
 using Azure.AI.OpenAI;
 using Azure.Identity;
 using Microsoft.Extensions.AI;
+using Microsoft.Extensions.DependencyInjection;
 using Microsoft.SemanticKernel;
+using Microsoft.SemanticKernel.ChatCompletion;
+using Microsoft.SemanticKernel.Connectors.AzureOpenAI;
 
 namespace ChatCompletion;
 
@@ -30,23 +32,33 @@ public class HybridCompletion_Fallback(ITestOutputHelper output) : BaseTest(outp
     [Fact]
     public async Task FallbackToAvailableModelAsync()
     {
-        // Create an unavailable chat client that fails with 503 Service Unavailable HTTP status code
-        IChatClient unavailableChatClient = CreateUnavailableOpenAIChatClient();
+        IKernelBuilder kernelBuilder = Kernel.CreateBuilder();
 
-        // Create a cloud available chat client
-        IChatClient availableChatClient = CreateAzureOpenAIChatClient();
+        // Create and register an unavailable chat client that fails with 503 Service Unavailable HTTP status code
+        kernelBuilder.Services.AddSingleton<IChatClient>(CreateUnavailableOpenAIChatClient());
 
-        // Create a fallback chat client that will fallback to the available chat client when unavailable chat client fails
-        IChatClient fallbackChatClient = new FallbackChatClient([unavailableChatClient, availableChatClient]);
+        // Create and register a cloud available chat client
+        kernelBuilder.Services.AddSingleton<IChatClient>(CreateAzureOpenAIChatClient());
+
+        // Create and register fallback chat client that will fallback to the available chat client when unavailable chat client fails
+        kernelBuilder.Services.AddSingleton<IChatCompletionService>((sp) =>
+        {
+            IEnumerable<IChatClient> chatClients = sp.GetServices<IChatClient>();
 
-        ChatOptions chatOptions = new() { Tools = [AIFunctionFactory.Create(GetWeather)] };
+            return new FallbackChatClient(chatClients.ToList()).AsChatCompletionService();
+        });
 
-        var result = await fallbackChatClient.GetResponseAsync("Do I need an umbrella?", chatOptions);
+        Kernel kernel = kernelBuilder.Build();
+        kernel.ImportPluginFromFunctions("Weather", [KernelFunctionFactory.CreateFromMethod(() => "It's sunny", "GetWeather")]);
 
-        Output.WriteLine(result);
+        AzureOpenAIPromptExecutionSettings settings = new()
+        {
+            FunctionChoiceBehavior = FunctionChoiceBehavior.Auto()
+        };
 
-        [Description("Gets the weather")]
-        string GetWeather() => "It's sunny";
+        FunctionResult result = await kernel.InvokePromptAsync("Do I need an umbrella?", new(settings));
+
+        Output.WriteLine(result);
     }
 
     /// <summary>
@@ -62,19 +74,22 @@ public async Task FallbackToAvailableModelStreamingAsync()
         IChatClient availableChatClient = CreateAzureOpenAIChatClient();
 
         // Create a fallback chat client that will fallback to the available chat client when unavailable chat client fails
-        IChatClient fallbackChatClient = new FallbackChatClient([unavailableChatClient, availableChatClient]);
+        IChatCompletionService fallbackCompletionService = new FallbackChatClient([unavailableChatClient, availableChatClient]).AsChatCompletionService();
 
-        ChatOptions chatOptions = new() { Tools = [AIFunctionFactory.Create(GetWeather)] };
+        Kernel kernel = new();
+        kernel.ImportPluginFromFunctions("Weather", [KernelFunctionFactory.CreateFromMethod(() => "It's sunny", "GetWeather")]);
 
-        var result = fallbackChatClient.GetStreamingResponseAsync("Do I need an umbrella?", chatOptions);
+        AzureOpenAIPromptExecutionSettings settings = new()
+        {
+            FunctionChoiceBehavior = FunctionChoiceBehavior.Auto()
+        };
+
+        IAsyncEnumerable<StreamingChatMessageContent> result = fallbackCompletionService.GetStreamingChatMessageContentsAsync("Do I need an umbrella?", settings, kernel);
 
         await foreach (var update in result)
         {
             Output.WriteLine(update);
         }
-
-        [Description("Gets the weather")]
-        string GetWeather() => "It's sunny";
     }
 
     private static IChatClient CreateUnavailableOpenAIChatClient()

dotnet/src/Functions/Functions.OpenApi/Model/RestApiOperation.cs

@@ -170,7 +170,7 @@ internal IDictionary<string, string> BuildHeaders(IDictionary<string, object?> a
                 throw new KernelException($"The headers parameter '{parameterStyle}' serialization style is not supported.");
             }
 
-            var node = OpenApiTypeConverter.Convert(parameter.Name, parameter.Type, argument);
+            var node = OpenApiTypeConverter.Convert(parameter.Name, parameter.Type, argument, parameter.Schema);
 
             //Serializing the parameter and adding it to the headers.
             headers.Add(parameter.Name, serializer.Invoke(parameter, node));
@@ -206,7 +206,7 @@ internal string BuildQueryString(IDictionary<string, object?> arguments)
                 throw new KernelException($"The query string parameter '{parameterStyle}' serialization style is not supported.");
             }
 
-            var node = OpenApiTypeConverter.Convert(parameter.Name, parameter.Type, argument);
+            var node = OpenApiTypeConverter.Convert(parameter.Name, parameter.Type, argument, parameter.Schema);
 
             // Serializing the parameter and adding it to the query string if there's an argument for it.
             segments.Add(serializer.Invoke(parameter, node));
@@ -274,7 +274,7 @@ private string BuildPath(string pathTemplate, IDictionary<string, object?> argum
                 throw new KernelException($"The path parameter '{parameterStyle}' serialization style is not supported.");
             }
 
-            var node = OpenApiTypeConverter.Convert(parameter.Name, parameter.Type, argument);
+            var node = OpenApiTypeConverter.Convert(parameter.Name, parameter.Type, argument, parameter.Schema);
 
             // Serializing the parameter and adding it to the path.
             pathTemplate = pathTemplate.Replace($"{{{parameter.Name}}}", HttpUtility.UrlEncode(serializer.Invoke(parameter, node)));

dotnet/src/Functions/Functions.OpenApi/RestApiOperationRunner.cs

@@ -427,13 +427,13 @@ private JsonObject BuildJsonObject(IList<RestApiPayloadProperty> properties, IDi
             // Use property argument name to look up the property value
             if (!string.IsNullOrEmpty(propertyMetadata.ArgumentName) && arguments.TryGetValue(propertyMetadata.ArgumentName!, out object? argument) && argument is not null)
             {
-                result.Add(propertyMetadata.Name, OpenApiTypeConverter.Convert(propertyMetadata.Name, propertyMetadata.Type, argument));
+                result.Add(propertyMetadata.Name, OpenApiTypeConverter.Convert(propertyMetadata.Name, propertyMetadata.Type, argument, propertyMetadata.Schema));
                 continue;
             }
             // Use property name to look up the property value
             else if (arguments.TryGetValue(argumentName, out argument) && argument is not null)
             {
-                result.Add(propertyMetadata.Name, OpenApiTypeConverter.Convert(propertyMetadata.Name, propertyMetadata.Type, argument));
+                result.Add(propertyMetadata.Name, OpenApiTypeConverter.Convert(propertyMetadata.Name, propertyMetadata.Type, argument, propertyMetadata.Schema));
                 continue;
             }
 

dotnet/src/Functions/Functions.OpenApi/Serialization/OpenApiTypeConverter.cs

@@ -4,6 +4,7 @@
 using System.Globalization;
 using System.Text.Json;
 using System.Text.Json.Nodes;
+using Json.Schema;
 
 namespace Microsoft.SemanticKernel.Plugins.OpenApi;
 
@@ -13,19 +14,20 @@ namespace Microsoft.SemanticKernel.Plugins.OpenApi;
 internal static class OpenApiTypeConverter
 {
     /// <summary>
-    /// Converts the given parameter argument to a JsonNode based on the specified type.
+    /// Converts the given parameter argument to a JsonNode based on the specified type or schema.
     /// </summary>
     /// <param name="name">The parameter name.</param>
     /// <param name="type">The parameter type.</param>
     /// <param name="argument">The argument to be converted.</param>
+    /// <param name="schema">The parameter schema.</param>
     /// <returns>A JsonNode representing the converted value.</returns>
-    public static JsonNode Convert(string name, string type, object argument)
+    public static JsonNode Convert(string name, string type, object argument, KernelJsonSchema? schema = null)
     {
         Verify.NotNull(argument);
 
         try
         {
-            JsonNode? converter = type switch
+            JsonNode? node = type switch
             {
                 "string" => JsonValue.Create(argument),
                 "array" => argument switch
@@ -52,10 +54,12 @@ string stringArgument when double.TryParse(stringArgument, out var doubleValue)
                     byte or sbyte or short or ushort or int or uint or long or ulong or float or double or decimal => JsonValue.Create(argument),
                     _ => null
                 },
-                _ => throw new NotSupportedException($"Unexpected type '{type}' of parameter '{name}' with argument '{argument}'."),
+                _ => schema is null
+                    ? JsonSerializer.SerializeToNode(argument)
+                    : ValidateSchemaAndConvert(name, schema, argument)
             };
 
-            return converter ?? throw new ArgumentOutOfRangeException(name, argument, $"Argument type '{argument.GetType()}' is not convertible to parameter type '{type}'.");
+            return node ?? throw new ArgumentOutOfRangeException(name, argument, $"Argument type '{argument.GetType()}' is not convertible to parameter type '{type}'.");
         }
         catch (ArgumentException ex)
         {
@@ -66,4 +70,25 @@ string stringArgument when double.TryParse(stringArgument, out var doubleValue)
             throw new ArgumentOutOfRangeException(name, argument, ex.Message);
         }
     }
+
+    /// <summary>
+    /// Validates the argument against the parameter schema and converts it to a JsonNode if valid.
+    /// </summary>
+    /// <param name="parameterName">The parameter name.</param>
+    /// <param name="parameterSchema">The parameter schema.</param>
+    /// <param name="argument">The argument to be validated and converted.</param>
+    /// <returns>A JsonNode representing the converted value.</returns>
+    private static JsonNode? ValidateSchemaAndConvert(string parameterName, KernelJsonSchema parameterSchema, object argument)
+    {
+        var jsonSchema = JsonSchema.FromText(JsonSerializer.Serialize(parameterSchema));
+
+        var node = JsonSerializer.SerializeToNode(argument);
+
+        if (jsonSchema.Evaluate(node).IsValid)
+        {
+            return node;
+        }
+
+        throw new ArgumentOutOfRangeException(parameterName, argument, $"Argument type '{argument.GetType()}' does not match the schema.");
+    }
 }

dotnet/src/Functions/Functions.UnitTests/OpenApi/Serialization/OpenApiTypeConverterTests.cs

@@ -3,6 +3,7 @@
 using System;
 using System.Collections.Generic;
 using System.Globalization;
+using Microsoft.SemanticKernel;
 using Microsoft.SemanticKernel.Plugins.OpenApi;
 using Microsoft.VisualBasic;
 using Xunit;
@@ -112,4 +113,50 @@ public void ItShouldConvertCollections()
 
         Assert.Equal("[1,2,3]", OpenApiTypeConverter.Convert("id", "array", "[1, 2, 3]").ToJsonString());
     }
+
+    [Fact]
+    public void ItShouldConvertWithNoTypeAndNoSchema()
+    {
+        // Act
+        var result = OpenApiTypeConverter.Convert("lat", null!, 51.8985136);
+
+        // Assert
+        Assert.Equal(51.8985136, result.GetValue<double>());
+    }
+
+    [Fact]
+    public void ItShouldConvertWithNoTypeAndValidSchema()
+    {
+        // Arrange
+        var schema = KernelJsonSchema.Parse(
+        """
+        {
+            "type": "number",
+            "format": "double",
+            "nullable": false
+        }
+        """);
+
+        // Act
+        var result = OpenApiTypeConverter.Convert("lat", null!, 51.8985136, schema);
+
+        // Assert
+        Assert.Equal(51.8985136, result.GetValue<double>());
+    }
+
+    [Fact]
+    public void ItShouldThrowExceptionWhenNoTypeAndInvalidSchema()
+    {
+        // Arrange
+        var schema = KernelJsonSchema.Parse(
+        """
+        {
+            "type": "boolean",
+            "nullable": false
+        }
+        """);
+
+        // Act & Assert
+        Assert.Throws<ArgumentOutOfRangeException>(() => OpenApiTypeConverter.Convert("lat", null!, 51.8985136, schema));
+    }
 }

python/samples/concepts/README.md

@@ -4,20 +4,51 @@
 
 ### Agents - Creating and using [agents](../../semantic_kernel/agents/) in Semantic Kernel
 
-- [Assistant Agent Chart Maker](./agents/assistant_agent/assistant_agent_chart_maker.py)
-- [Assistant Agent File Manipulation](./agents/assistant_agent/assistant_agent_file_manipulation.py)
-- [Assistant Agent File Manipulation Streaming](./agents/assistant_agent/assistant_agent_file_manipulation_streaming.py)
-- [Assistant Agent Retrieval](./agents/assistant_agent/assistant_agent_retrieval.py)
-- [Assistant Agent Streaming](./agents/assistant_agent/assistant_agent_streaming.py)
+#### [OpenAI Assistant Agent](../../semantic_kernel/agents/open_ai/open_ai_assistant_agent.py)
+
+- [OpenAI Assistant Chart Maker Streaming](./agents/openai_assistant/openai_assistant_chart_maker_streaming.py)
+- [OpenAI Assistant Chart Maker](./agents/openai_assistant/openai_assistant_chart_maker.py)
+- [OpenAI Assistant File Manipulation Streaming](./agents/openai_assistant/openai_assistant_file_manipulation_streaming.py)
+- [OpenAI Assistant File Manipulation](./agents/openai_assistant/openai_assistant_file_manipulation.py)
+- [OpenAI Assistant File Manipulation Streaming](./agents/openai_assistant/openai_assistant_file_manipulation_streaming.py)
+- [OpenAI Assistant Retrieval](./agents/openai_assistant/openai_assistant_retrieval.py)
+- [OpenAI Assistant Streaming](./agents/openai_assistant/openai_assistant_streaming.py)
+- [OpenAI Assistant Structured Outputs](./agents/openai_assistant/openai_assistant_structured_outputs.py)
+- [OpenAI Assistant Templating Streaming](./agents/openai_assistant/openai_assistant_templating_streaming.py)
+- [OpenAI Assistant Vision Streaming](./agents/openai_assistant/openai_assistant_vision_streaming.py)
+
+#### [Azure AI Agent](../../semantic_kernel/agents/azure_ai/azure_ai_agent.py)
+
+- [Azure AI Agent with Azure AI Search](./agents/azure_ai_agent/azure_ai_agent_azure_ai_search.py)
+- [Azure AI Agent File Manipulation](./agents/azure_ai_agent/azure_ai_agent_file_manipulation.py)
+- [Azure AI Agent Streaming](./agents/azure_ai_agent/azure_ai_agent_streaming.py)
+
+#### [Bedrock Agent](../../semantic_kernel/agents/bedrock/bedrock_agent.py)
+- [Bedrock Agent Simple Chat Streaming](./agents/bedrock_agent/bedrock_agent_simple_chat_streaming.py)
+- [Bedrock Agent Simple Chat](./agents/bedrock_agent/bedrock_agent_simple_chat.py)
+- [Bedrock Agent Update Agent](./agents/bedrock_agent/bedrock_agent_update_agent.py)
+- [Bedrock Agent Use Existing](./agents/bedrock_agent/bedrock_agent_use_existing.py)
+- [Bedrock Agent With Code Interpreter Streaming](./agents/bedrock_agent/bedrock_agent_with_code_interpreter_streaming.py)
+- [Bedrock Agent With Code Interpreter](./agents/bedrock_agent/bedrock_agent_with_code_interpreter.py)
+- [Bedrock Agent With Kernel Function Streaming](./agents/bedrock_agent/bedrock_agent_with_kernel_function_streaming.py)
+- [Bedrock Agent With Kernel Function](./agents/bedrock_agent/bedrock_agent_with_kernel_function.py)
+- [Bedrock Agent Mixed Chat Agents Streaming](./agents/bedrock_agent/bedrock_mixed_chat_agents_streaming.py)
+- [Bedrock Agent Mixed Chat Agents](./agents/bedrock_agent/bedrock_mixed_chat_agents.py)
+
+#### [Chat Completion Agent](../../semantic_kernel/agents/chat_completion/chat_completion_agent.py)
+
 - [Chat Completion Function Termination](./agents/chat_completion_agent/chat_completion_function_termination.py)
 - [Chat Completion History Reducer](./agents/chat_completion_agent/chat_completion_history_reducer.py)
-- [Mixed Chat Agents](./agents/mixed_chat/mixed_chat_agents.py)
+- [Chat Completion Templating](./agents/chat_completion_agent/chat_completion_templating.py)
+
+#### [Mixed Agent Group Chat](../../semantic_kernel/agents/group_chat/agent_group_chat.py)
+
 - [Mixed Chat Agents Plugins](./agents/mixed_chat/mixed_chat_agents_plugins.py)
+- [Mixed Chat Agents](./agents/mixed_chat/mixed_chat_agents.py)
 - [Mixed Chat Files](./agents/mixed_chat/mixed_chat_files.py)
 - [Mixed Chat Images](./agents/mixed_chat/mixed_chat_images.py)
 - [Mixed Chat Reset](./agents/mixed_chat/mixed_chat_reset.py)
 - [Mixed Chat Streaming](./agents/mixed_chat/mixed_chat_streaming.py)
-- [Bedrock Agent](./agents/bedrock_agent/README.md)
 
 ### Audio - Using services that support audio-to-text and text-to-audio conversion
 

python/samples/concepts/agents/README.md

@@ -7,7 +7,10 @@ This project contains a step by step guide to get started with _Semantic Kernel
 - For the use of Chat Completion agents, the minimum allowed Semantic Kernel pypi version is 1.3.0.
 - For the use of OpenAI Assistant agents, the minimum allowed Semantic Kernel pypi version is 1.4.0.
 - For the use of Agent Group Chat, the minimum allowed Semantic kernel pypi version is 1.6.0.
-- For the use of Streaming OpenAI Assistant agents, the minimum allowed Semantic Kernel pypi version is 1.11.0
+- For the use of Streaming OpenAI Assistant agents, the minimum allowed Semantic Kernel pypi version is 1.11.0.
+- For the use of AzureAI and Bedrock agents, the minimum allowed Semantic Kernel pypi version is 1.21.0.
+- For the use of Crew.AI as a plugin, the minimum allowed Semantic Kernel pypi version is 1.21.1.
+
 
 ## Source
 
@@ -20,12 +23,12 @@ The concept agents examples are grouped by prefix:
 Prefix|Description
 ---|---
 assistant|How to use agents based on the [Open AI Assistant API](https://platform.openai.com/docs/assistants).
-chat_completion|How to use Semantic Kernel Chat Completion agents.
-bedrock|How to use AWS Bedrock agents in Semantic Kernel.
+autogen_conversable_agent| How to use [AutoGen 0.2 Conversable Agents](https://microsoft.github.io/autogen/0.2/docs/Getting-Started) within Semantic Kernel.
+azure_ai_agent|How to use an [Azure AI Agent](https://learn.microsoft.com/en-us/azure/ai-services/agents/quickstart?pivots=programming-language-python-azure) within Semantic Kernel.
+chat_completion_agent|How to use Semantic Kernel Chat Completion agents that leverage AI Connector Chat Completion APIs.
+bedrock|How to use [AWS Bedrock agents](https://aws.amazon.com/bedrock/agents/) in Semantic Kernel.
 mixed_chat|How to combine different agent types.
-complex_chat|**Coming Soon**
-
-_Note: As we strive for parity with .NET, more getting_started_with_agent samples will be added. The current steps and names may be revised to further align with our .NET counterpart._
+openai_assistant|How to use [OpenAI Assistants](https://platform.openai.com/docs/assistants/overview) in Semantic Kernel.
 
 ## Configuring the Kernel