Skip to content

Commit 3b7adbe

Browse files
[AdvancedPaste]Improved telemetry (#33520)
<!-- Enter a brief description/summary of your PR here. What does it fix/what does it change/how was it tested (even manually, if necessary)? --> ## Summary of the Pull Request This PR improves advanced paste telemetry. Here's what's changed - Added `AdvancedPasteClipboardItemClicked` event - Changed `CustomFormatEvent` to only fire on successful completion and include the number of tokens used, and the model name Here are the goals of adding this telemtry: - `AdvancedPasteClipboardItemClicked` helps us estimate the total number of user who are using the clipboard history feature, which helps us prioritize future investments and improvements in PowerToys. (This is just regular feature usage data). - `CustomFormatEvent` now includes number of tokens used, and the model name. We are considering using alternative models to power Advanced Paste (as we've heard feedback about this), and these different models could have different token lengths. Understanding the average usage will help us make good decisions that will benefit the most users. As well, the model name is hard coded right now, but in the future if we do add different AI models for completion then this will help us compare their performance. <!-- Please review the items on the PR checklist before submitting--> ## PR Checklist - [X] Communication: I've discussed this with core contributors already. If work hasn't been agreed, this work might be rejected <!-- Provide a more detailed description of the PR, other things fixed or any additional comments/features here --> ## Detailed Description of the Pull Request / Additional comments The details above are detailed enough since this change is super small. <!-- Describe how you validated the behavior. Add automated tests wherever possible, but list manual validation steps taken as well --> ## Validation Steps Performed Ensured that PowerToys successfully built (Need help verifying the events fire off correctly).
1 parent 8b8c75b commit 3b7adbe

File tree

5 files changed

+42
-6
lines changed

5 files changed

+42
-6
lines changed

src/modules/AdvancedPaste/AdvancedPaste/AdvancedPasteXAML/Controls/PromptBox.xaml.cs

-2
Original file line numberDiff line numberDiff line change
@@ -78,8 +78,6 @@ private void GenerateCustom()
7878
{
7979
Logger.LogTrace();
8080

81-
PowerToysTelemetry.Log.WriteEvent(new Telemetry.AdvancedPasteGenerateCustomFormatEvent());
82-
8381
VisualStateManager.GoToState(this, "LoadingState", true);
8482
string inputInstructions = InputTxtBox.Text;
8583
ViewModel.SaveQuery(inputInstructions);

src/modules/AdvancedPaste/AdvancedPaste/AdvancedPasteXAML/Pages/MainPage.xaml.cs

+1
Original file line numberDiff line numberDiff line change
@@ -231,6 +231,7 @@ private async void ClipboardHistory_ItemClick(object sender, ItemClickEventArgs
231231
var item = e.ClickedItem as ClipboardItem;
232232
if (item is not null)
233233
{
234+
PowerToysTelemetry.Log.WriteEvent(new Telemetry.AdvancedPasteClipboardItemClicked());
234235
if (!string.IsNullOrEmpty(item.Content))
235236
{
236237
ClipboardHelper.SetClipboardTextContent(item.Content);

src/modules/AdvancedPaste/AdvancedPaste/Helpers/AICompletionsHelper.cs

+12-4
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,8 @@ public AICompletionsResponse(string response, int apiRequestStatus)
3333

3434
private string _openAIKey;
3535

36+
private string _modelName = "gpt-3.5-turbo-instruct";
37+
3638
public bool IsAIEnabled => !string.IsNullOrEmpty(this._openAIKey);
3739

3840
public AICompletionsHelper()
@@ -69,14 +71,14 @@ public static string LoadOpenAIKey()
6971
return string.Empty;
7072
}
7173

72-
public string GetAICompletion(string systemInstructions, string userMessage)
74+
private Response<Completions> GetAICompletion(string systemInstructions, string userMessage)
7375
{
7476
OpenAIClient azureAIClient = new OpenAIClient(_openAIKey);
7577

7678
var response = azureAIClient.GetCompletions(
7779
new CompletionsOptions()
7880
{
79-
DeploymentName = "gpt-3.5-turbo-instruct",
81+
DeploymentName = _modelName,
8082
Prompts =
8183
{
8284
systemInstructions + "\n\n" + userMessage,
@@ -90,7 +92,7 @@ public string GetAICompletion(string systemInstructions, string userMessage)
9092
Console.WriteLine("Cut off due to length constraints");
9193
}
9294

93-
return response.Value.Choices[0].Text;
95+
return response;
9496
}
9597

9698
public AICompletionsResponse AIFormatString(string inputInstructions, string inputString)
@@ -109,10 +111,16 @@ public AICompletionsResponse AIFormatString(string inputInstructions, string inp
109111
";
110112

111113
string aiResponse = null;
114+
Response<Completions> rawAIResponse = null;
112115
int apiRequestStatus = (int)HttpStatusCode.OK;
113116
try
114117
{
115-
aiResponse = this.GetAICompletion(systemInstructions, userMessage);
118+
rawAIResponse = this.GetAICompletion(systemInstructions, userMessage);
119+
aiResponse = rawAIResponse.Value.Choices[0].Text;
120+
121+
int promptTokens = rawAIResponse.Value.Usage.PromptTokens;
122+
int completionTokens = rawAIResponse.Value.Usage.CompletionTokens;
123+
PowerToysTelemetry.Log.WriteEvent(new Telemetry.AdvancedPasteGenerateCustomFormatEvent(promptTokens, completionTokens, _modelName));
116124
}
117125
catch (Azure.RequestFailedException error)
118126
{
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
// Copyright (c) Microsoft Corporation
2+
// The Microsoft Corporation licenses this file to you under the MIT license.
3+
// See the LICENSE file in the project root for more information.
4+
5+
using System.Diagnostics.Tracing;
6+
using Microsoft.PowerToys.Telemetry;
7+
using Microsoft.PowerToys.Telemetry.Events;
8+
9+
namespace AdvancedPaste.Telemetry
10+
{
11+
[EventData]
12+
public class AdvancedPasteClipboardItemClicked : EventBase, IEvent
13+
{
14+
public PartA_PrivTags PartA_PrivTags => PartA_PrivTags.ProductAndServiceUsage;
15+
}
16+
}

src/modules/AdvancedPaste/AdvancedPaste/Telemetry/AdvancedPasteGenerateCustomFormatEvent.cs

+13
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,19 @@ namespace AdvancedPaste.Telemetry
1111
[EventData]
1212
public class AdvancedPasteGenerateCustomFormatEvent : EventBase, IEvent
1313
{
14+
public int PromptTokens { get; set; }
15+
16+
public int CompletionTokens { get; set; }
17+
18+
public string ModelName { get; set; }
19+
20+
public AdvancedPasteGenerateCustomFormatEvent(int promptTokens, int completionTokens, string modelName)
21+
{
22+
this.PromptTokens = promptTokens;
23+
this.CompletionTokens = completionTokens;
24+
ModelName = modelName;
25+
}
26+
1427
public PartA_PrivTags PartA_PrivTags => PartA_PrivTags.ProductAndServiceUsage;
1528
}
1629
}

0 commit comments

Comments
 (0)