Skip to content

Commit 3cd7067

Browse files
benbpbenbp
authored
[azsdk-cli] Use scoped tools and helpers, add micro agent example (#12017)
* Inject token usage helper * Move azure client logging to error level * Change ai project endpoint and default model for agent service * Fix log level filter for cli tools classes * Track token usage for micro agent service calls * Add micro agent service demo to ExampleTool * Remove cost estimates from token usage helper
1 parent 6868637 commit 3cd7067

11 files changed

Lines changed: 202 additions & 144 deletions

File tree

‎tools/azsdk-cli/Azure.Sdk.Tools.Cli.Contract/MCPTool.cs‎

Lines changed: 20 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -3,32 +3,31 @@
33
using System.CommandLine;
44
using System.CommandLine.Invocation;
55

6-
namespace Azure.Sdk.Tools.Cli.Contract
6+
namespace Azure.Sdk.Tools.Cli.Contract;
7+
8+
/// <summary>
9+
/// This is the base class defining how an MCP enabled tool will interface with the server.
10+
///
11+
/// This covers:
12+
/// - route registration/disambiguation
13+
/// - compilation trim avoidance for reflection-included MCP tools
14+
/// </summary>
15+
public abstract class MCPTool
716
{
8-
/// <summary>
9-
/// This is the base class defining how an MCP enabled tool will interface with the server.
10-
///
11-
/// This covers:
12-
/// - route registration/disambiguation
13-
/// - compilation trim avoidance for reflection-included MCP tools
14-
/// </summary>
15-
public abstract class MCPTool
16-
{
17-
public MCPTool() { }
17+
public MCPTool() { }
1818

19-
public Command? Command;
19+
public Command? Command;
2020

21-
public int ExitCode { get; set; } = 0;
21+
public int ExitCode { get; set; } = 0;
2222

23-
public void SetFailure(int exitCode = 1)
24-
{
25-
ExitCode = exitCode;
26-
}
23+
public void SetFailure(int exitCode = 1)
24+
{
25+
ExitCode = exitCode;
26+
}
2727

28-
public CommandGroup[] CommandHierarchy { get; set; } = [];
28+
public CommandGroup[] CommandHierarchy { get; set; } = [];
2929

30-
public abstract Command GetCommand();
30+
public abstract Command GetCommand();
3131

32-
public abstract Task HandleCommand(InvocationContext ctx, CancellationToken ct);
33-
}
32+
public abstract Task HandleCommand(InvocationContext ctx, CancellationToken ct);
3433
}

‎tools/azsdk-cli/Azure.Sdk.Tools.Cli.Tests/Microagents/MicroagentHostServiceTests.cs‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,8 @@
33
using Azure.AI.OpenAI;
44
using Azure.Sdk.Tools.Cli.Microagents;
55
using Azure.Sdk.Tools.Cli.Microagents.Tools;
6+
using Azure.Sdk.Tools.Cli.Helpers;
7+
using NUnit.Framework;
68
using Moq;
79
using OpenAI.Chat;
810

@@ -23,7 +25,8 @@ public void Setup()
2325
chatClientMock = new Mock<ChatClient>();
2426
openAIClientMock.Setup(client => client.GetChatClient(It.IsAny<string>()))
2527
.Returns(chatClientMock.Object);
26-
microagentHostService = new MicroagentHostService(openAIClientMock.Object, loggerMock.Object);
28+
var tokenUsageHelper = new TokenUsageHelper(Mock.Of<Azure.Sdk.Tools.Cli.Helpers.IOutputHelper>());
29+
microagentHostService = new MicroagentHostService(openAIClientMock.Object, loggerMock.Object, tokenUsageHelper);
2730
}
2831

2932
[Test]

‎tools/azsdk-cli/Azure.Sdk.Tools.Cli.Tests/Tools/ExampleToolTests.cs‎

Lines changed: 19 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
// Copyright (c) Microsoft Corporation.
22
// Licensed under the MIT License.
33
using Moq;
4+
using NUnit.Framework;
45
using NUnit.Framework.Internal;
56
using Azure.Core;
67
using Azure.Sdk.Tools.Cli.Helpers;
@@ -22,6 +23,7 @@ internal class ExampleToolTests
2223
private MockGitHubService? mockGitHubService;
2324
private Mock<IProcessHelper>? mockProcessHelper;
2425
private Mock<IPowershellHelper>? mockPowershellHelper;
26+
private Mock<Azure.Sdk.Tools.Cli.Microagents.IMicroagentHostService>? mockMicroagentHostService;
2527

2628
[SetUp]
2729
public void Setup()
@@ -33,6 +35,7 @@ public void Setup()
3335
mockGitHubService = new MockGitHubService();
3436
mockProcessHelper = new Mock<IProcessHelper>();
3537
mockPowershellHelper = new Mock<IPowershellHelper>();
38+
mockMicroagentHostService = new Mock<Azure.Sdk.Tools.Cli.Microagents.IMicroagentHostService>();
3639

3740
// Set up Azure service mock to return a mock credential
3841
var mockCredential = new Mock<TokenCredential>();
@@ -58,6 +61,8 @@ public void Setup()
5861
mockGitHubService,
5962
mockProcessHelper.Object,
6063
mockPowershellHelper.Object,
64+
tokenUsageHelper: new TokenUsageHelper(mockOutput.Object),
65+
mockMicroagentHostService.Object,
6166
#pragma warning disable CS8625 // Cannot convert null literal to non-nullable reference type.
6267
null
6368
#pragma warning restore CS8625 // Cannot convert null literal to non-nullable reference type.
@@ -153,7 +158,7 @@ public void GetCommand_ReturnsCommandWithCorrectSubCommands()
153158

154159
Assert.That(command.Name, Is.EqualTo("demo"));
155160
Assert.That(command.Description, Does.Contain("Comprehensive demonstration"));
156-
Assert.That(command.Subcommands.Count, Is.EqualTo(7));
161+
Assert.That(command.Subcommands.Count, Is.EqualTo(8));
157162

158163
var subCommandNames = command.Subcommands.Select(sc => sc.Name).ToList();
159164
Assert.That(subCommandNames, Does.Contain("azure"));
@@ -163,6 +168,7 @@ public void GetCommand_ReturnsCommandWithCorrectSubCommands()
163168
Assert.That(subCommandNames, Does.Contain("error"));
164169
Assert.That(subCommandNames, Does.Contain("process"));
165170
Assert.That(subCommandNames, Does.Contain("powershell"));
171+
Assert.That(subCommandNames, Does.Contain("microagent"));
166172
}
167173

168174
[Test]
@@ -236,7 +242,6 @@ public async Task DemonstratePowershellExecution_Success()
236242
{
237243
mockPowershellHelper!.Setup(p => p.Run(It.IsAny<PowershellOptions>(), It.IsAny<CancellationToken>()))
238244
.ReturnsAsync(new ProcessResult { ExitCode = 0, });
239-
240245
var result = await tool.DemonstratePowershellExecution("foobar");
241246

242247
Assert.That(result.ResponseError, Is.Null);
@@ -245,4 +250,16 @@ public async Task DemonstratePowershellExecution_Success()
245250
Assert.That(result.Result, Is.Empty);
246251
Assert.That(result.Details?["exit_code"], Is.EqualTo("0"));
247252
}
253+
254+
[Test]
255+
public async Task DemonstrateMicroagentFibonacci_Success()
256+
{
257+
mockMicroagentHostService!.Setup(m => m.RunAgentToCompletion(It.IsAny<Azure.Sdk.Tools.Cli.Microagents.Microagent<int>>(), It.IsAny<CancellationToken>()))
258+
.ReturnsAsync(13);
259+
260+
var response = await tool.DemonstrateMicroagentFibonacci(7);
261+
262+
Assert.That(response.ResponseError, Is.Null);
263+
Assert.That(response.Result as string, Does.Contain("Fibonacci(7) = 13"));
264+
}
248265
}

‎tools/azsdk-cli/Azure.Sdk.Tools.Cli/Helpers/TokenUsageHelper.cs‎

Lines changed: 14 additions & 70 deletions
Original file line numberDiff line numberDiff line change
@@ -1,80 +1,24 @@
11
namespace Azure.Sdk.Tools.Cli.Helpers;
22

3-
public class TokenUsageHelper
3+
public class TokenUsageHelper(IOutputHelper outputHelper)
44
{
5-
protected double PromptTokens { get; set; }
6-
protected double CompletionTokens { get; set; }
7-
protected double InputCost { get; set; }
8-
protected double OutputCost { get; set; }
9-
protected double TotalCost { get; set; }
10-
public List<string> Models { get; set; } = [];
5+
protected double PromptTokens { get; set; } = 0;
6+
protected double CompletionTokens { get; set; } = 0;
7+
protected IEnumerable<string> ModelsUsed { get; set; } = [];
118

12-
public TokenUsageHelper(string model, long inputTokens, long outputTokens)
9+
public void Add(string model, long inputTokens, long outputTokens)
1310
{
14-
PromptTokens = inputTokens;
15-
CompletionTokens = outputTokens;
16-
Models = [model];
17-
SetCost(model);
11+
ModelsUsed = ModelsUsed.Union([model]);
12+
PromptTokens += inputTokens;
13+
CompletionTokens += outputTokens;
1814
}
1915

20-
protected TokenUsageHelper() { }
21-
22-
private void SetCost(string model)
16+
public void LogUsage()
2317
{
24-
var oneMillion = 1000000;
25-
double inputPrice, outputPrice;
26-
27-
// Prices assume the slightly more expensive regional model pricing
28-
if (model == "gpt-4o")
29-
{
30-
(inputPrice, outputPrice) = (2.75, 11);
31-
}
32-
else if (model == "gpt-4o-mini")
33-
{
34-
(inputPrice, outputPrice) = (0.165, 0.66);
35-
}
36-
if (model == "gpt-4.1")
37-
{
38-
(inputPrice, outputPrice) = (2, 8);
39-
}
40-
else if (model == "gpt-4.1-mini")
41-
{
42-
(inputPrice, outputPrice) = (0.4, 1.60);
43-
}
44-
else if (model == "o3-mini")
45-
{
46-
(inputPrice, outputPrice) = (1.21, 4.84);
47-
}
48-
else
49-
{
50-
return;
51-
}
52-
53-
54-
InputCost = PromptTokens / oneMillion * inputPrice;
55-
OutputCost = CompletionTokens / oneMillion * outputPrice;
56-
}
18+
var models = string.Join(", ", ModelsUsed);
5719

58-
public void LogCost()
59-
{
60-
var _inputCost = InputCost == 0 ? "?" : InputCost.ToString("F3");
61-
var _outputCost = OutputCost == 0 ? "?" : OutputCost.ToString("F3");
62-
var _totalCost = (InputCost + OutputCost) == 0 ? "?" : (InputCost + OutputCost).ToString("F3");
63-
var models = string.Join(", ", Models);
64-
Console.WriteLine("--------------------------------------------------------------------------------");
65-
Console.WriteLine($"[{models}] Usage (cost / tokens):");
66-
Console.WriteLine($" Input: ${_inputCost} / {PromptTokens}");
67-
Console.WriteLine($" Output: ${_outputCost} / {CompletionTokens}");
68-
Console.WriteLine($" Total: ${_totalCost} / {PromptTokens + CompletionTokens}");
69-
Console.WriteLine("--------------------------------------------------------------------------------");
20+
outputHelper.OutputConsole("--------------------------------------------------------------------------------");
21+
outputHelper.OutputConsole($"[token usage][{models}] input: {PromptTokens}, output: {CompletionTokens}, total: {PromptTokens + CompletionTokens}");
22+
outputHelper.OutputConsole("--------------------------------------------------------------------------------");
7023
}
71-
72-
public static TokenUsageHelper operator +(TokenUsageHelper a, TokenUsageHelper? b) => new()
73-
{
74-
Models = a.Models.Union(b?.Models ?? []).ToList(),
75-
PromptTokens = a.PromptTokens + (b?.PromptTokens ?? 0),
76-
CompletionTokens = a.CompletionTokens + (b?.CompletionTokens ?? 0),
77-
InputCost = a.InputCost + (b?.InputCost ?? 0),
78-
OutputCost = a.OutputCost + (b?.OutputCost ?? 0),
79-
};
80-
}
24+
}

‎tools/azsdk-cli/Azure.Sdk.Tools.Cli/Microagents/MicroagentHostService.cs‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,11 @@
11
using System.ComponentModel;
22
using Azure.AI.OpenAI;
3+
using Azure.Sdk.Tools.Cli.Helpers;
34
using OpenAI.Chat;
45

56
namespace Azure.Sdk.Tools.Cli.Microagents;
67

7-
public class MicroagentHostService(AzureOpenAIClient openAI, ILogger<MicroagentHostService> logger) : IMicroagentHostService
8+
public class MicroagentHostService(AzureOpenAIClient openAI, ILogger<MicroagentHostService> logger, TokenUsageHelper tokenUsageHelper) : IMicroagentHostService
89
{
910
private const string ExitToolName = "Exit";
1011

@@ -60,6 +61,10 @@ public async Task<TResult> RunAgentToCompletion<TResult>(Microagent<TResult> age
6061
// Request the chat completion
6162
logger.LogDebug("Sending conversation history with {MessageCount} messages to model '{Model}'", conversationHistory.Count, agentDefinition.Model);
6263
var response = await chatClient.CompleteChatAsync(conversationHistory, chatCompletionOptions, ct);
64+
if (null != response.Value.Usage)
65+
{
66+
tokenUsageHelper.Add(agentDefinition.Model, response.Value.Usage.InputTokenCount, response.Value.Usage.OutputTokenCount);
67+
}
6368

6469
var toolCall = response.Value.ToolCalls.Single();
6570
logger.LogInformation("Model called tool '{ToolName}'", toolCall.FunctionName);
@@ -104,7 +109,7 @@ public async Task<TResult> RunAgentToCompletion<TResult>(Microagent<TResult> age
104109
conversationHistory.Add(ChatMessage.CreateToolMessage(toolCall.Id, toolResult));
105110
}
106111

107-
throw new Exception("Agent did not return a result within the maximum number of iterations");
112+
throw new Exception($"Agent did not return a result within the maximum number of {agentDefinition.MaxToolCalls} iterations");
108113
}
109114

110115
/// <summary>

‎tools/azsdk-cli/Azure.Sdk.Tools.Cli/Program.cs‎

Lines changed: 10 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,8 @@ public static WebApplicationBuilder CreateAppBuilder(string[] args)
4646
WebApplicationBuilder builder = WebApplication.CreateBuilder(args);
4747

4848
builder.Services.AddOpenTelemetry()
49-
.WithTracing(b => {
49+
.WithTracing(b =>
50+
{
5051
b.AddSource(Constants.TOOLS_ACTIVITY_SOURCE)
5152
.AddAspNetCoreInstrumentation()
5253
.AddHttpClientInstrumentation()
@@ -55,24 +56,21 @@ public static WebApplicationBuilder CreateAppBuilder(string[] args)
5556
})
5657
.UseOtlpExporter();
5758

58-
// Log everything to stderr in mcp mode so the client doesn't try to interpret stdout messages that aren't json rpc
59-
var logErrorThreshold = isCLI ? LogLevel.Error : LogLevel.Debug;
60-
6159
builder.Logging.AddConsole(consoleLogOptions =>
6260
{
61+
// Log everything to stderr in mcp mode so the client doesn't try to interpret stdout messages that aren't json rpc
62+
var logErrorThreshold = isCLI ? LogLevel.Error : LogLevel.Debug;
6363
consoleLogOptions.LogToStandardErrorThreshold = logErrorThreshold;
6464
});
6565

66-
// Skip verbose azure client logging
66+
// Skip azure client logging noise
6767
builder.Logging.AddFilter((category, level) =>
6868
{
69-
var isAzureClient = category!.StartsWith("Azure.", StringComparison.Ordinal);
70-
var isToolsClient = category!.StartsWith("Azure.Sdk.Tools.", StringComparison.Ordinal);
71-
if (isAzureClient && !isToolsClient)
72-
{
73-
return level >= LogLevel.Warning;
74-
}
75-
return level >= logErrorThreshold;
69+
if (debug || null == category) { return level >= logLevel; }
70+
var isAzureClient = category.StartsWith("Azure.", StringComparison.Ordinal);
71+
var isToolsClient = category.StartsWith("Azure.Sdk.Tools.", StringComparison.Ordinal);
72+
if (isAzureClient && !isToolsClient) { return level >= LogLevel.Error; }
73+
return level >= logLevel;
7674
});
7775

7876
// add the console logger

0 commit comments

Comments
 (0)