Skip to content

Commit a01986f

Browse files
Han-Ya-Junclaude
andauthored
feat(mcp-proxy): add BKAIDev Agent Trace for MCP Gateway observability (#2665)
Why this change was needed: The MCP Gateway lacked independent observability for BKAIDev Agent interactions. Operations teams needed a way to trace agent-to-tool call flows, including caller identity, latency, error codes, and upstream agent metadata (via X-Bkapi-ItsmFlex header), without interfering with the existing project-level OpenTelemetry tracing. What changed: - Added independent OTLP/HTTP trace provider in pkg/infra/bkaidtrace with its own TracerProvider, propagator, and lifecycle management, fully isolated from the project's existing tracing infrastructure - Added Gin middleware (BkAIDevTraceContextMiddleware) to extract W3C traceparent from incoming requests or generate fresh span contexts when absent - Added MCP-level middleware (BkAIDevTraceMiddleware) that creates spans per MCP method call with rich attributes: app_code, bk_username, client_ip, client_id, gateway_name, mcp_server_name, tool_name, request_id, x_request_id, trace_id, caller_executor, agent_code, latency_ms, status, and error_code - Added X-Bkapi-ItsmFlex header parsing to extract agent identity fields (agent code, agent name, caller executor, executor) - Added BkAIDevTrace config struct with enable/endpoint/token/service name fields - Added graceful shutdown of bkaidtrace provider in server shutdown - Comprehensive unit tests covering all new modules (25+ test cases) - Applied gofumpt/goimports-reviser formatting fixes to existing files Problem solved: BKAIDev Agent trace spans are now independently reported to a dedicated OTLP endpoint, enabling full observability of agent-driven MCP tool calls without coupling to or polluting the project's own tracing pipeline. Co-authored-by: claude <noreply@anthropic.com>
1 parent 6007bd1 commit a01986f

25 files changed

Lines changed: 1223 additions & 95 deletions

src/mcp-proxy/cmd/init.go

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ import (
2424
"github.com/spf13/viper"
2525

2626
"mcp_proxy/pkg/config"
27+
"mcp_proxy/pkg/infra/bkaidevtrace"
2728
"mcp_proxy/pkg/infra/database"
2829
"mcp_proxy/pkg/infra/logging"
2930
sty "mcp_proxy/pkg/infra/sentry"
@@ -93,3 +94,17 @@ func initTracing() {
9394
}
9495
logging.GetLogger().Info("init tracing success")
9596
}
97+
98+
func initBkAIDevTrace() {
99+
if !globalConfig.BkAIDevTrace.Enable {
100+
logging.GetLogger().Info("bkai dev trace is not enabled, will not init it")
101+
return
102+
}
103+
logging.GetLogger().Info("enabling bkai dev trace")
104+
err := bkaidevtrace.Init(globalConfig.BkAIDevTrace)
105+
if err != nil {
106+
logging.GetLogger().Errorf("init bkai dev trace fail: %+v", err)
107+
return
108+
}
109+
logging.GetLogger().Info("init bkai dev trace success")
110+
}

src/mcp-proxy/cmd/root.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -89,6 +89,7 @@ func Start() {
8989
initLogger()
9090
initDatabase()
9191
initTracing()
92+
initBkAIDevTrace()
9293
initSentry()
9394
initMetrics()
9495

src/mcp-proxy/pkg/config/config.go

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -155,6 +155,15 @@ type Instrument struct {
155155
McpAPI bool
156156
}
157157

158+
// BkAIDevTrace is the config for BKAIDev agent trace reporting.
159+
// It uses an independent OTLP/HTTP endpoint, fully isolated from the project's own tracing.
160+
type BkAIDevTrace struct {
161+
Enable bool
162+
Endpoint string
163+
ServiceName string
164+
Token string
165+
}
166+
158167
// Transport is the config for the shared HTTP transport used by tool calls.
159168
type Transport struct {
160169
InsecureSkipVerify bool
@@ -306,9 +315,10 @@ type Config struct {
306315
Databases []Database
307316
DatabaseMap map[string]Database
308317

309-
Logger Logger
310-
Tracing Tracing
311-
Metric Metric
318+
Logger Logger
319+
Tracing Tracing
320+
Metric Metric
321+
BkAIDevTrace BkAIDevTrace
312322

313323
McpServer McpServer
314324
PProf Pprof

src/mcp-proxy/pkg/constant/system.go

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,9 @@ const (
3535
// BkApiAllowedHeadersKey is a key to set the allowed headers in header
3636
BkApiAllowedHeadersKey = "X-Bkapi-Allowed-Headers"
3737

38+
// BkApiItsmFlexKey is a key to set the itsm flex info in header
39+
BkApiItsmFlexKey = "X-Bkapi-ItsmFlex"
40+
3841
// BkApiMCPServerIDKey is a key to set the mcp server id in header
3942
BkApiMCPServerIDKey = "X-Bkapi-Mcp-Server-Id"
4043
// BkApiMCPServerNameKey is a key to set the mcp server name in header
@@ -60,6 +63,7 @@ const (
6063
TraceID CtxKey = "trace_id"
6164
BkApiTimeout CtxKey = "bk_api_timeout"
6265
BkApiAllowedHeaders CtxKey = "bk_api_allowed_headers"
66+
BkApiItsmFlexData CtxKey = "bk_api_itsm_flex_data"
6367
ClientIP CtxKey = "client_ip"
6468
ClientID CtxKey = "client_id"
6569
)

src/mcp-proxy/pkg/entity/model/mcp.go

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -40,16 +40,16 @@ const ToolNameSeparator = "@"
4040

4141
// MCPServer ...
4242
type MCPServer struct {
43-
ID int `gorm:"primaryKey;autoIncrement;column:id"`
44-
Name string `gorm:"column:name;size:64;uniqueIndex"`
45-
Description string `gorm:"column:description;size:512"`
46-
IsPublic bool `gorm:"column:is_public"`
47-
Labels ArrayString `gorm:"column:labels"`
48-
ResourceNames ArrayString `gorm:"column:resource_names"`
49-
Status int `gorm:"column:status"`
50-
GatewayID int `gorm:"column:gateway_id"`
51-
StageID int `gorm:"column:stage_id"`
52-
ProtocolType string `gorm:"column:protocol_type;size:32;default:sse"`
43+
ID int `gorm:"primaryKey;autoIncrement;column:id"`
44+
Name string `gorm:"column:name;size:64;uniqueIndex"`
45+
Description string `gorm:"column:description;size:512"`
46+
IsPublic bool `gorm:"column:is_public"`
47+
Labels ArrayString `gorm:"column:labels"`
48+
ResourceNames ArrayString `gorm:"column:resource_names"`
49+
Status int `gorm:"column:status"`
50+
GatewayID int `gorm:"column:gateway_id"`
51+
StageID int `gorm:"column:stage_id"`
52+
ProtocolType string `gorm:"column:protocol_type;size:32;default:sse"`
5353
RawResponseEnabled bool `gorm:"column:raw_response_enabled;default:false"`
5454
}
5555

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
/*
2+
* TencentBlueKing is pleased to support the open source community by making
3+
* 蓝鲸智云 - API 网关(BlueKing - APIGateway) available.
4+
* Copyright (C) 2025 Tencent. All rights reserved.
5+
* Licensed under the MIT License (the "License"); you may not use this file except
6+
* in compliance with the License. You may obtain a copy of the License at
7+
*
8+
* http://opensource.org/licenses/MIT
9+
*
10+
* Unless required by applicable law or agreed to in writing, software distributed under
11+
* the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
12+
* either express or implied. See the License for the specific language governing permissions and
13+
* limitations under the License.
14+
*
15+
* We undertake not to change the open source license (MIT license) applicable
16+
* to the current version of the project delivered to anyone in the future.
17+
*/
18+
19+
package bkaidevtrace
20+
21+
import (
22+
"sync"
23+
24+
"go.opentelemetry.io/otel/propagation"
25+
sdktrace "go.opentelemetry.io/otel/sdk/trace"
26+
)
27+
28+
// ResetForTest resets all global state so that Init can be called again in tests.
29+
// NOTE: This function is exported for cross-package test usage (e.g., middleware_test).
30+
// It is only intended for use in test code; production callers should never invoke it.
31+
func ResetForTest() {
32+
globalProvider = nil
33+
globalTracer = nil
34+
propagator = nil
35+
once = sync.Once{}
36+
}
37+
38+
// SetTestProvider injects a test TracerProvider and propagator directly.
39+
// NOTE: This function is exported for cross-package test usage only.
40+
func SetTestProvider(tp *sdktrace.TracerProvider, p propagation.TextMapPropagator) {
41+
globalProvider = tp
42+
globalTracer = tp.Tracer("test")
43+
propagator = p
44+
}
Lines changed: 187 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,187 @@
1+
/*
2+
* TencentBlueKing is pleased to support the open source community by making
3+
* 蓝鲸智云 - API 网关(BlueKing - APIGateway) available.
4+
* Copyright (C) 2025 Tencent. All rights reserved.
5+
* Licensed under the MIT License (the "License"); you may not use this file except
6+
* in compliance with the License. You may obtain a copy of the License at
7+
*
8+
* http://opensource.org/licenses/MIT
9+
*
10+
* Unless required by applicable law or agreed to in writing, software distributed under
11+
* the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
12+
* either express or implied. See the License for the specific language governing permissions and
13+
* limitations under the License.
14+
*
15+
* We undertake not to change the open source license (MIT license) applicable
16+
* to the current version of the project delivered to anyone in the future.
17+
*/
18+
19+
// Package bkaidevtrace provides an independent OpenTelemetry trace pipeline
20+
// for BKAIDev Agent trace reporting. It is fully isolated from the project's
21+
// own global tracing to avoid interference.
22+
package bkaidevtrace
23+
24+
import (
25+
"context"
26+
"crypto/rand"
27+
"encoding/binary"
28+
"fmt"
29+
mrand "math/rand"
30+
"sync"
31+
"time"
32+
33+
"go.opentelemetry.io/otel/attribute"
34+
"go.opentelemetry.io/otel/exporters/otlp/otlptrace"
35+
"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp"
36+
"go.opentelemetry.io/otel/propagation"
37+
"go.opentelemetry.io/otel/sdk/resource"
38+
sdktrace "go.opentelemetry.io/otel/sdk/trace"
39+
semconv "go.opentelemetry.io/otel/semconv/v1.10.0"
40+
tc "go.opentelemetry.io/otel/trace"
41+
42+
"mcp_proxy/pkg/config"
43+
)
44+
45+
var (
46+
globalProvider *sdktrace.TracerProvider
47+
globalTracer tc.Tracer
48+
propagator propagation.TextMapPropagator
49+
once sync.Once
50+
)
51+
52+
// Init initializes the independent BKAIDev trace pipeline.
53+
func Init(cfg config.BkAIDevTrace) error {
54+
var initErr error
55+
once.Do(func() {
56+
// WithInsecure uses HTTP instead of HTTPS for the OTLP exporter.
57+
// This is intentional: the BKAIDev trace collector runs in the same
58+
// internal network, so TLS is not required. The bk.data.token in the
59+
// resource attributes provides authentication.
60+
client := otlptracehttp.NewClient(
61+
otlptracehttp.WithEndpoint(cfg.Endpoint),
62+
otlptracehttp.WithInsecure(),
63+
)
64+
exporter, err := otlptrace.New(context.Background(), client)
65+
if err != nil {
66+
initErr = fmt.Errorf("create bkaidevtrace exporter: %w", err)
67+
return
68+
}
69+
70+
tp := sdktrace.NewTracerProvider(
71+
sdktrace.WithBatcher(exporter),
72+
sdktrace.WithResource(resource.NewWithAttributes(
73+
semconv.SchemaURL,
74+
semconv.ServiceNameKey.String(cfg.ServiceName),
75+
attribute.Key("bk.data.token").String(cfg.Token),
76+
)),
77+
sdktrace.WithSampler(sdktrace.AlwaysSample()),
78+
)
79+
80+
globalProvider = tp
81+
globalTracer = tp.Tracer(cfg.ServiceName)
82+
propagator = propagation.NewCompositeTextMapPropagator(
83+
propagation.TraceContext{},
84+
propagation.Baggage{},
85+
)
86+
})
87+
return initErr
88+
}
89+
90+
// Enabled returns whether the BKAIDev trace pipeline is initialized.
91+
func Enabled() bool {
92+
return globalTracer != nil
93+
}
94+
95+
// StartSpan starts a new span using the independent tracer.
96+
func StartSpan(ctx context.Context, name string, opts ...tc.SpanStartOption) (context.Context, tc.Span) {
97+
if globalTracer == nil {
98+
return ctx, nil
99+
}
100+
return globalTracer.Start(ctx, name, opts...)
101+
}
102+
103+
// Extract extracts trace context from carrier using the independent propagator.
104+
func Extract(ctx context.Context, carrier propagation.TextMapCarrier) context.Context {
105+
if propagator == nil {
106+
return ctx
107+
}
108+
return propagator.Extract(ctx, carrier)
109+
}
110+
111+
// Inject injects trace context into carrier using the independent propagator.
112+
func Inject(ctx context.Context, carrier propagation.TextMapCarrier) {
113+
if propagator == nil {
114+
return
115+
}
116+
propagator.Inject(ctx, carrier)
117+
}
118+
119+
// GetTraceIDFromContext extracts the trace ID from the active span in context.
120+
func GetTraceIDFromContext(ctx context.Context) string {
121+
span := tc.SpanFromContext(ctx)
122+
if span == nil {
123+
return ""
124+
}
125+
sc := span.SpanContext()
126+
if !sc.TraceID().IsValid() {
127+
return ""
128+
}
129+
return sc.TraceID().String()
130+
}
131+
132+
// GetSpanIDFromContext extracts the span ID from the active span in context.
133+
func GetSpanIDFromContext(ctx context.Context) string {
134+
span := tc.SpanFromContext(ctx)
135+
if span == nil {
136+
return ""
137+
}
138+
sc := span.SpanContext()
139+
if !sc.SpanID().IsValid() {
140+
return ""
141+
}
142+
return sc.SpanID().String()
143+
}
144+
145+
// NewSpanContext creates a span context with a randomly-generated trace ID and span ID.
146+
// This is used to carry a valid trace context without creating an actual span.
147+
func NewSpanContext() tc.SpanContext {
148+
var traceID tc.TraceID
149+
var spanID tc.SpanID
150+
151+
// Try crypto/rand first for both traceID and spanID.
152+
// If crypto/rand fails (extremely rare), fallback to a single math/rand instance
153+
// to avoid seed collision when two separate instances are created with the same
154+
// time-based seed under high concurrency.
155+
traceOK := true
156+
if _, err := rand.Read(traceID[:]); err != nil {
157+
traceOK = false
158+
}
159+
if _, err := rand.Read(spanID[:]); err != nil {
160+
if traceOK {
161+
// Only traceID succeeded via crypto/rand; generate spanID with math/rand
162+
r := mrand.New(mrand.NewSource(time.Now().UnixNano()))
163+
binary.BigEndian.PutUint64(spanID[:], uint64(r.Int63()))
164+
}
165+
}
166+
if !traceOK {
167+
// crypto/rand failed for traceID; use a single math/rand instance for both
168+
r := mrand.New(mrand.NewSource(time.Now().UnixNano()))
169+
binary.BigEndian.PutUint64(traceID[:8], uint64(r.Int63()))
170+
binary.BigEndian.PutUint64(traceID[8:], uint64(r.Int63()))
171+
binary.BigEndian.PutUint64(spanID[:], uint64(r.Int63()))
172+
}
173+
174+
return tc.NewSpanContext(tc.SpanContextConfig{
175+
TraceID: traceID,
176+
SpanID: spanID,
177+
TraceFlags: tc.FlagsSampled,
178+
})
179+
}
180+
181+
// Shutdown flushes and shuts down the tracer provider.
182+
func Shutdown(ctx context.Context) error {
183+
if globalProvider == nil {
184+
return nil
185+
}
186+
return globalProvider.Shutdown(ctx)
187+
}

0 commit comments

Comments
 (0)