Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions dotnet/src/Grafana.Sigil/SigilClient.cs
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,7 @@ public sealed class SigilClient : IAsyncDisposable
internal const string SpanAttrToolDescription = "gen_ai.tool.description";
internal const string SpanAttrToolCallArguments = "gen_ai.tool.call.arguments";
internal const string SpanAttrToolCallResult = "gen_ai.tool.call.result";
internal const string SpanAttrToolCallCount = "sigil.gen_ai.tool_call_count";
private const int MaxRatingConversationIdLen = 255;
private const int MaxRatingIdLen = 128;
private const int MaxRatingGenerationIdLen = 255;
Expand Down Expand Up @@ -1187,6 +1188,12 @@ internal static void ApplyGenerationSpanAttributes(Activity activity, Generation
{
activity.SetTag(SpanAttrReasoningTokens, generation.Usage.ReasoningTokens);
}

var toolCallCount = CountToolCallParts(generation.Output);
if (toolCallCount != 0)
{
activity.SetTag(SpanAttrToolCallCount, toolCallCount);
}
}

internal static void ApplyEmbeddingStartSpanAttributes(Activity activity, EmbeddingStart start)
Expand Down
1 change: 1 addition & 0 deletions dotnet/tests/Grafana.Sigil.Tests/ConformanceTests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -149,6 +149,7 @@ public async Task SyncRoundtripSemantics()
Assert.Equal("generateText", span.GetTagItem("gen_ai.operation.name")?.ToString());
Assert.Equal("Roundtrip conversation", span.GetTagItem("sigil.conversation.title")?.ToString());
Assert.Equal("user-roundtrip", span.GetTagItem("user.id")?.ToString());
Assert.Equal(1L, span.GetTagItem("sigil.gen_ai.tool_call_count"));
Assert.Contains("gen_ai.client.operation.duration", env.MetricNames);
Assert.Contains("gen_ai.client.token.usage", env.MetricNames);
Assert.DoesNotContain("gen_ai.client.time_to_first_token", env.MetricNames);
Expand Down
5 changes: 5 additions & 0 deletions go/sigil/client.go
Original file line number Diff line number Diff line change
Expand Up @@ -139,6 +139,7 @@ const (
spanAttrToolDescription = "gen_ai.tool.description"
spanAttrToolCallArguments = "gen_ai.tool.call.arguments"
spanAttrToolCallResult = "gen_ai.tool.call.result"
spanAttrToolCallCount = "sigil.gen_ai.tool_call_count"

metricOperationDuration = "gen_ai.client.operation.duration"
metricTokenUsage = "gen_ai.client.token.usage"
Expand Down Expand Up @@ -1262,6 +1263,10 @@ func generationSpanAttributes(g Generation) []attribute.KeyValue {
attrs = append(attrs, attribute.Int64(spanAttrReasoningTokens, g.Usage.ReasoningTokens))
}

if count := countToolCalls(g.Output); count != 0 {
attrs = append(attrs, attribute.Int64(spanAttrToolCallCount, int64(count)))
}

return attrs
}

Expand Down
1 change: 1 addition & 0 deletions go/sigil/conformance_helpers_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,7 @@ const (
spanAttrCacheWriteTokens = "gen_ai.usage.cache_write_input_tokens"
spanAttrCacheCreationTokens = "gen_ai.usage.cache_creation_input_tokens"
spanAttrReasoningTokens = "gen_ai.usage.reasoning_tokens"
spanAttrToolCallCount = "sigil.gen_ai.tool_call_count"
metricOperationDuration = "gen_ai.client.operation.duration"
metricTokenUsage = "gen_ai.client.token.usage"
metricTimeToFirstToken = "gen_ai.client.time_to_first_token"
Expand Down
1 change: 1 addition & 0 deletions go/sigil/conformance_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -204,6 +204,7 @@ func TestConformance_FullGenerationRoundtrip(t *testing.T) {
requireSpanInt64Attr(t, attrs, spanAttrCacheWriteTokens, 4)
requireSpanInt64Attr(t, attrs, spanAttrCacheCreationTokens, 6)
requireSpanInt64Attr(t, attrs, spanAttrReasoningTokens, 9)
requireSpanInt64Attr(t, attrs, spanAttrToolCallCount, 1)

duration := findHistogram[float64](t, metrics, metricOperationDuration)
durationPoint := findHistogramPoint(t, duration, map[string]string{
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,7 @@ public final class SigilClient implements AutoCloseable {
static final String SPAN_ATTR_TOOL_DESCRIPTION = "gen_ai.tool.description";
static final String SPAN_ATTR_TOOL_CALL_ARGUMENTS = "gen_ai.tool.call.arguments";
static final String SPAN_ATTR_TOOL_CALL_RESULT = "gen_ai.tool.call.result";
static final String SPAN_ATTR_TOOL_CALL_COUNT = "sigil.gen_ai.tool_call_count";
private static final int MAX_RATING_CONVERSATION_ID_LEN = 255;
private static final int MAX_RATING_ID_LEN = 128;
private static final int MAX_RATING_GENERATION_ID_LEN = 255;
Expand Down Expand Up @@ -1028,6 +1029,11 @@ static void setGenerationSpanAttributes(Span span, Generation generation) {
span.setAttribute(SPAN_ATTR_CACHE_CREATION_TOKENS, usage.getCacheCreationInputTokens());
span.setAttribute(SPAN_ATTR_REASONING_TOKENS, usage.getReasoningTokens());
}

long toolCallCount = countToolCalls(generation.getOutput());
if (toolCallCount != 0) {
span.setAttribute(SPAN_ATTR_TOOL_CALL_COUNT, toolCallCount);
}
}

static void setToolSpanAttributes(Span span, ToolExecutionStart seed) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -147,6 +147,8 @@ void syncRoundtripSemantics() throws Exception {
.isEqualTo("Roundtrip conversation");
assertThat(span.getAttributes().get(AttributeKey.stringKey(SigilClient.SPAN_ATTR_USER_ID)))
.isEqualTo("user-roundtrip");
assertThat(span.getAttributes().get(AttributeKey.longKey(SigilClient.SPAN_ATTR_TOOL_CALL_COUNT)))
.isEqualTo(1L);
assertThat(metricNames).contains(SigilClient.METRIC_OPERATION_DURATION, SigilClient.METRIC_TOKEN_USAGE);
assertThat(metricNames).doesNotContain(SigilClient.METRIC_TTFT);
}
Expand Down
7 changes: 7 additions & 0 deletions js/src/client.ts
Original file line number Diff line number Diff line change
Expand Up @@ -105,6 +105,7 @@ const spanAttrToolType = 'gen_ai.tool.type';
const spanAttrToolDescription = 'gen_ai.tool.description';
const spanAttrToolCallArguments = 'gen_ai.tool.call.arguments';
const spanAttrToolCallResult = 'gen_ai.tool.call.result';
const spanAttrToolCallCount = 'sigil.gen_ai.tool_call_count';
const maxRatingConversationIdLen = 255;
const maxRatingIdLen = 128;
const maxRatingGenerationIdLen = 255;
Expand Down Expand Up @@ -1222,6 +1223,7 @@ function setGenerationSpanAttributes(
responseId?: string;
responseModel?: string;
stopReason?: string;
output?: Message[];
usage?: {
inputTokens?: number;
outputTokens?: number;
Expand Down Expand Up @@ -1320,6 +1322,11 @@ function setGenerationSpanAttributes(
span.setAttribute(spanAttrFinishReasons, [generation.stopReason]);
}

const toolCallCount = countToolCallParts(generation.output ?? []);
if (toolCallCount !== 0) {
span.setAttribute(spanAttrToolCallCount, toolCallCount);
}

const usage = generation.usage;
if (usage === undefined) {
return;
Expand Down
1 change: 1 addition & 0 deletions js/test/conformance.test.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -138,6 +138,7 @@ test('conformance sync roundtrip semantics', async () => {
assert.equal(span.attributes['gen_ai.operation.name'], 'generateText');
assert.equal(span.attributes['sigil.conversation.title'], 'Roundtrip conversation');
assert.equal(span.attributes['user.id'], 'user-roundtrip');
assert.equal(span.attributes['sigil.gen_ai.tool_call_count'], 1);
assert.ok(metricNames.includes('gen_ai.client.operation.duration'));
assert.ok(metricNames.includes('gen_ai.client.token.usage'));
assert.ok(!metricNames.includes('gen_ai.client.time_to_first_token'));
Expand Down
5 changes: 5 additions & 0 deletions python/sigil_sdk/client.py
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,7 @@
_span_attr_tool_description = "gen_ai.tool.description"
_span_attr_tool_call_arguments = "gen_ai.tool.call.arguments"
_span_attr_tool_call_result = "gen_ai.tool.call.result"
_span_attr_tool_call_count = "sigil.gen_ai.tool_call_count"
_max_rating_conversation_id_len = 255
_max_rating_id_len = 128
_max_rating_generation_id_len = 255
Expand Down Expand Up @@ -1243,6 +1244,10 @@ def _set_generation_span_attributes(span: Span, generation: Generation) -> None:
if usage.reasoning_tokens:
span.set_attribute(_span_attr_reasoning_tokens, usage.reasoning_tokens)

tool_call_count = _count_tool_call_parts(generation.output)
if tool_call_count:
span.set_attribute(_span_attr_tool_call_count, tool_call_count)


def _set_embedding_start_span_attributes(span: Span, start: EmbeddingStart) -> None:
span.set_attribute(_span_attr_operation_name, _default_embedding_operation_name)
Expand Down
1 change: 1 addition & 0 deletions python/tests/test_conformance.py
Original file line number Diff line number Diff line change
Expand Up @@ -324,6 +324,7 @@ def test_conformance_sync_roundtrip_semantics() -> None:
assert span.attributes["gen_ai.operation.name"] == "generateText"
assert span.attributes[_span_attr_conversation_title] == "Roundtrip conversation"
assert span.attributes[_span_attr_user_id] == "user-roundtrip"
assert span.attributes["sigil.gen_ai.tool_call_count"] == 1
assert "gen_ai.client.operation.duration" in metrics
assert "gen_ai.client.token.usage" in metrics
assert "gen_ai.client.time_to_first_token" not in metrics
Expand Down
Loading