diff --git a/dotnet/src/Grafana.Sigil/SigilClient.cs b/dotnet/src/Grafana.Sigil/SigilClient.cs index 43e6ce2..0ca5e8c 100644 --- a/dotnet/src/Grafana.Sigil/SigilClient.cs +++ b/dotnet/src/Grafana.Sigil/SigilClient.cs @@ -54,6 +54,7 @@ public sealed class SigilClient : IAsyncDisposable internal const string SpanAttrToolDescription = "gen_ai.tool.description"; internal const string SpanAttrToolCallArguments = "gen_ai.tool.call.arguments"; internal const string SpanAttrToolCallResult = "gen_ai.tool.call.result"; + internal const string SpanAttrToolCallCount = "sigil.gen_ai.tool_call_count"; private const int MaxRatingConversationIdLen = 255; private const int MaxRatingIdLen = 128; private const int MaxRatingGenerationIdLen = 255; @@ -1187,6 +1188,12 @@ internal static void ApplyGenerationSpanAttributes(Activity activity, Generation { activity.SetTag(SpanAttrReasoningTokens, generation.Usage.ReasoningTokens); } + + var toolCallCount = CountToolCallParts(generation.Output); + if (toolCallCount != 0) + { + activity.SetTag(SpanAttrToolCallCount, toolCallCount); + } } internal static void ApplyEmbeddingStartSpanAttributes(Activity activity, EmbeddingStart start) diff --git a/dotnet/tests/Grafana.Sigil.Tests/ConformanceTests.cs b/dotnet/tests/Grafana.Sigil.Tests/ConformanceTests.cs index 5571025..5ef816b 100644 --- a/dotnet/tests/Grafana.Sigil.Tests/ConformanceTests.cs +++ b/dotnet/tests/Grafana.Sigil.Tests/ConformanceTests.cs @@ -149,6 +149,7 @@ public async Task SyncRoundtripSemantics() Assert.Equal("generateText", span.GetTagItem("gen_ai.operation.name")?.ToString()); Assert.Equal("Roundtrip conversation", span.GetTagItem("sigil.conversation.title")?.ToString()); Assert.Equal("user-roundtrip", span.GetTagItem("user.id")?.ToString()); + Assert.Equal(1L, span.GetTagItem("sigil.gen_ai.tool_call_count")); Assert.Contains("gen_ai.client.operation.duration", env.MetricNames); Assert.Contains("gen_ai.client.token.usage", env.MetricNames); Assert.DoesNotContain("gen_ai.client.time_to_first_token", env.MetricNames); diff --git a/go/sigil/client.go b/go/sigil/client.go index 5b4ff22..b40bcc0 100644 --- a/go/sigil/client.go +++ b/go/sigil/client.go @@ -157,6 +157,7 @@ const ( spanAttrToolDescription = "gen_ai.tool.description" spanAttrToolCallArguments = "gen_ai.tool.call.arguments" spanAttrToolCallResult = "gen_ai.tool.call.result" + spanAttrToolCallCount = "sigil.gen_ai.tool_call_count" metricOperationDuration = "gen_ai.client.operation.duration" metricTokenUsage = "gen_ai.client.token.usage" @@ -1298,6 +1299,10 @@ func generationSpanAttributes(g Generation) []attribute.KeyValue { attrs = append(attrs, attribute.Int64(spanAttrReasoningTokens, g.Usage.ReasoningTokens)) } + if count := countToolCalls(g.Output); count != 0 { + attrs = append(attrs, attribute.Int64(spanAttrToolCallCount, int64(count))) + } + return attrs } diff --git a/go/sigil/conformance_helpers_test.go b/go/sigil/conformance_helpers_test.go index 6268d53..45c3acb 100644 --- a/go/sigil/conformance_helpers_test.go +++ b/go/sigil/conformance_helpers_test.go @@ -68,6 +68,7 @@ const ( spanAttrCacheWriteTokens = "gen_ai.usage.cache_write_input_tokens" spanAttrCacheCreationTokens = "gen_ai.usage.cache_creation_input_tokens" spanAttrReasoningTokens = "gen_ai.usage.reasoning_tokens" + spanAttrToolCallCount = "sigil.gen_ai.tool_call_count" metricOperationDuration = "gen_ai.client.operation.duration" metricTokenUsage = "gen_ai.client.token.usage" metricTimeToFirstToken = "gen_ai.client.time_to_first_token" diff --git a/go/sigil/conformance_test.go b/go/sigil/conformance_test.go index 9f8e64d..4cf65e6 100644 --- a/go/sigil/conformance_test.go +++ b/go/sigil/conformance_test.go @@ -204,6 +204,7 @@ func TestConformance_FullGenerationRoundtrip(t *testing.T) { requireSpanInt64Attr(t, attrs, spanAttrCacheWriteTokens, 4) requireSpanInt64Attr(t, attrs, spanAttrCacheCreationTokens, 6) requireSpanInt64Attr(t, attrs, spanAttrReasoningTokens, 9) + requireSpanInt64Attr(t, attrs, spanAttrToolCallCount, 1) duration := findHistogram[float64](t, metrics, metricOperationDuration) durationPoint := findHistogramPoint(t, duration, map[string]string{ diff --git a/java/core/src/main/java/com/grafana/sigil/sdk/SigilClient.java b/java/core/src/main/java/com/grafana/sigil/sdk/SigilClient.java index 251a831..0712203 100644 --- a/java/core/src/main/java/com/grafana/sigil/sdk/SigilClient.java +++ b/java/core/src/main/java/com/grafana/sigil/sdk/SigilClient.java @@ -76,6 +76,7 @@ public final class SigilClient implements AutoCloseable { static final String SPAN_ATTR_TOOL_DESCRIPTION = "gen_ai.tool.description"; static final String SPAN_ATTR_TOOL_CALL_ARGUMENTS = "gen_ai.tool.call.arguments"; static final String SPAN_ATTR_TOOL_CALL_RESULT = "gen_ai.tool.call.result"; + static final String SPAN_ATTR_TOOL_CALL_COUNT = "sigil.gen_ai.tool_call_count"; private static final int MAX_RATING_CONVERSATION_ID_LEN = 255; private static final int MAX_RATING_ID_LEN = 128; private static final int MAX_RATING_GENERATION_ID_LEN = 255; @@ -1028,6 +1029,11 @@ static void setGenerationSpanAttributes(Span span, Generation generation) { span.setAttribute(SPAN_ATTR_CACHE_CREATION_TOKENS, usage.getCacheCreationInputTokens()); span.setAttribute(SPAN_ATTR_REASONING_TOKENS, usage.getReasoningTokens()); } + + long toolCallCount = countToolCalls(generation.getOutput()); + if (toolCallCount != 0) { + span.setAttribute(SPAN_ATTR_TOOL_CALL_COUNT, toolCallCount); + } } static void setToolSpanAttributes(Span span, ToolExecutionStart seed) { diff --git a/java/core/src/test/java/com/grafana/sigil/sdk/ConformanceTest.java b/java/core/src/test/java/com/grafana/sigil/sdk/ConformanceTest.java index eb4161c..47e326d 100644 --- a/java/core/src/test/java/com/grafana/sigil/sdk/ConformanceTest.java +++ b/java/core/src/test/java/com/grafana/sigil/sdk/ConformanceTest.java @@ -147,6 +147,8 @@ void syncRoundtripSemantics() throws Exception { .isEqualTo("Roundtrip conversation"); assertThat(span.getAttributes().get(AttributeKey.stringKey(SigilClient.SPAN_ATTR_USER_ID))) .isEqualTo("user-roundtrip"); + assertThat(span.getAttributes().get(AttributeKey.longKey(SigilClient.SPAN_ATTR_TOOL_CALL_COUNT))) + .isEqualTo(1L); assertThat(metricNames).contains(SigilClient.METRIC_OPERATION_DURATION, SigilClient.METRIC_TOKEN_USAGE); assertThat(metricNames).doesNotContain(SigilClient.METRIC_TTFT); } diff --git a/js/src/client.ts b/js/src/client.ts index bef40de..e61e5e7 100644 --- a/js/src/client.ts +++ b/js/src/client.ts @@ -114,6 +114,7 @@ const spanAttrToolType = 'gen_ai.tool.type'; const spanAttrToolDescription = 'gen_ai.tool.description'; const spanAttrToolCallArguments = 'gen_ai.tool.call.arguments'; const spanAttrToolCallResult = 'gen_ai.tool.call.result'; +const spanAttrToolCallCount = 'sigil.gen_ai.tool_call_count'; const maxRatingConversationIdLen = 255; const maxRatingIdLen = 128; const maxRatingGenerationIdLen = 255; @@ -1240,6 +1241,7 @@ function setGenerationSpanAttributes( responseId?: string; responseModel?: string; stopReason?: string; + output?: Message[]; usage?: { inputTokens?: number; outputTokens?: number; @@ -1338,6 +1340,11 @@ function setGenerationSpanAttributes( span.setAttribute(spanAttrFinishReasons, [generation.stopReason]); } + const toolCallCount = countToolCallParts(generation.output ?? []); + if (toolCallCount !== 0) { + span.setAttribute(spanAttrToolCallCount, toolCallCount); + } + const usage = generation.usage; if (usage === undefined) { return; diff --git a/js/test/conformance.test.mjs b/js/test/conformance.test.mjs index 55b0682..3424578 100644 --- a/js/test/conformance.test.mjs +++ b/js/test/conformance.test.mjs @@ -143,6 +143,7 @@ test('conformance sync roundtrip semantics', async () => { assert.equal(span.attributes['gen_ai.operation.name'], 'generateText'); assert.equal(span.attributes['sigil.conversation.title'], 'Roundtrip conversation'); assert.equal(span.attributes['user.id'], 'user-roundtrip'); + assert.equal(span.attributes['sigil.gen_ai.tool_call_count'], 1); assert.ok(metricNames.includes('gen_ai.client.operation.duration')); assert.ok(metricNames.includes('gen_ai.client.token.usage')); assert.ok(!metricNames.includes('gen_ai.client.time_to_first_token')); diff --git a/python/sigil_sdk/client.py b/python/sigil_sdk/client.py index e543cfa..f699449 100644 --- a/python/sigil_sdk/client.py +++ b/python/sigil_sdk/client.py @@ -100,6 +100,7 @@ _span_attr_tool_description = "gen_ai.tool.description" _span_attr_tool_call_arguments = "gen_ai.tool.call.arguments" _span_attr_tool_call_result = "gen_ai.tool.call.result" +_span_attr_tool_call_count = "sigil.gen_ai.tool_call_count" _max_rating_conversation_id_len = 255 _max_rating_id_len = 128 _max_rating_generation_id_len = 255 @@ -1251,6 +1252,10 @@ def _set_generation_span_attributes(span: Span, generation: Generation) -> None: if usage.reasoning_tokens: span.set_attribute(_span_attr_reasoning_tokens, usage.reasoning_tokens) + tool_call_count = _count_tool_call_parts(generation.output) + if tool_call_count: + span.set_attribute(_span_attr_tool_call_count, tool_call_count) + def _set_embedding_start_span_attributes(span: Span, start: EmbeddingStart) -> None: span.set_attribute(_span_attr_operation_name, _default_embedding_operation_name) diff --git a/python/tests/test_conformance.py b/python/tests/test_conformance.py index c8d4188..6f8db24 100644 --- a/python/tests/test_conformance.py +++ b/python/tests/test_conformance.py @@ -321,6 +321,7 @@ def test_conformance_sync_roundtrip_semantics() -> None: assert span.attributes["gen_ai.operation.name"] == "generateText" assert span.attributes[_span_attr_conversation_title] == "Roundtrip conversation" assert span.attributes[_span_attr_user_id] == "user-roundtrip" + assert span.attributes["sigil.gen_ai.tool_call_count"] == 1 assert "gen_ai.client.operation.duration" in metrics assert "gen_ai.client.token.usage" in metrics assert "gen_ai.client.time_to_first_token" not in metrics