forked from deepseek-ai/deepseek-harness
-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathindex.ts
More file actions
301 lines (280 loc) · 13.6 KB
/
Copy pathindex.ts
File metadata and controls
301 lines (280 loc) · 13.6 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
/**
* OpenTelemetry Service Provider for the DeepSeek Harness telemetry capability.
*
* Composes the OTel JS SDK as-is — a `LoggerProvider` with a
* `BatchLogRecordProcessor` and an OTLP/HTTP log exporter — and maps each
* record handed over by the capture coordinator onto `logger.emit()`. After that call,
* batching, retry, queueing, and loss policy use the SDK's documented behavior, configured
* verbatim through the `exporter`/`processor` passthroughs. This package owns
* capture mode and an outer shutdown deadline: the SDK's export timeout does
* not bound its preceding `forceFlush()` wait.
*
* @module @deepseek-ai/dsh-session-telemetry-otel
*/
import { createRequire } from 'node:module'
import z from '@deepseek-ai/schemastery'
import type { Context } from '@deepseek-ai/cordis'
import type {} from '@deepseek-ai/dsh-command-feedback'
import {
SessionTelemetryBackend,
SessionTelemetryCoordinator,
type SessionTelemetrySink,
type SessionTelemetryRecord,
type SessionTelemetrySeverity,
type SessionTelemetrySharingStatus,
} from '@deepseek-ai/dsh-session-telemetry'
import { APP_IDENTITY } from '@deepseek-ai/dsh-llm'
import { getOrCreateAnonymousUserId } from '@deepseek-ai/dsh-anonymous-user-id'
import {
BatchLogRecordProcessor,
LoggerProvider,
type BatchLogRecordProcessorOptions,
} from '@opentelemetry/sdk-logs'
import { OTLPLogExporter } from '@opentelemetry/exporter-logs-otlp-http'
import type { OTLPExporterNodeConfigBase } from '@opentelemetry/otlp-exporter-base'
import { SeverityNumber, type AnyValue, type Logger } from '@opentelemetry/api-logs'
import { resourceFromAttributes } from '@opentelemetry/resources'
// The package's own manifest is the single source of the instrumentation-scope
// version (same pattern as dsh-llm's attribution identity).
const { version } = createRequire(import.meta.url)('../package.json') as { version: string }
/** Session-sharing policy selected by {@link Config.mode}. */
export enum SessionTelemetryMode {
FULL = 'FULL',
FEEDBACK_ONLY = 'FEEDBACK_ONLY',
DISABLED = 'DISABLED',
}
/** Default session-sharing policy for schema and direct construction. */
export const DEFAULT_TELEMETRY_MODE = SessionTelemetryMode.DISABLED
const DISABLED_FEEDBACK_WARNING = 'session telemetry is DISABLED; nothing will be shared and this feedback remains local'
const NON_CANONICAL_FEEDBACK_WARNING = 'session telemetry ignored a feedback event absent from the canonical session log'
const DROP_RECORD: SessionTelemetrySink['emit'] = () => {}
/** Resolve the default and reject unknown runtime values before transport setup. */
function resolveMode(mode: SessionTelemetryMode | undefined): SessionTelemetryMode {
const resolved = mode ?? DEFAULT_TELEMETRY_MODE
switch (resolved) {
case SessionTelemetryMode.FULL:
case SessionTelemetryMode.FEEDBACK_ONLY:
case SessionTelemetryMode.DISABLED:
return resolved
default:
return assertNever(resolved)
}
}
/** Fail closed when direct construction bypasses the runtime config schema. */
function assertNever(value: never): never {
throw new Error(`session-telemetry-otel: unsupported mode ${JSON.stringify(value)}`)
}
/** Map the serialized mode onto the seam's backend-independent sharing vocabulary. */
function sharingStatusFor(mode: SessionTelemetryMode): SessionTelemetrySharingStatus {
switch (mode) {
case SessionTelemetryMode.FULL: return 'full'
case SessionTelemetryMode.FEEDBACK_ONLY: return 'feedback-only'
case SessionTelemetryMode.DISABLED: return 'disabled'
/* v8 ignore next 2 -- resolveMode already rejected unknown values before this switch; the closed enum cannot reach the default. */
default: return assertNever(mode)
}
}
/**
* Plugin configuration: one sharing policy, two verbatim SDK option objects,
* and one DSH-owned shutdown bound. Uploading modes validate their endpoint
* and shutdown deadline at plugin load; `DISABLED` reads neither.
*/
export interface Config {
/** Sharing policy; defaults to local-only `DISABLED` behavior. */
mode?: SessionTelemetryMode
/**
* Passed verbatim to the SDK's OTLP/HTTP log exporter — the complete
* `OTLPExporterNodeConfigBase` shape (`headers`, `timeoutMillis`,
* `compression`, `keepAlive`, …), owned and documented by the SDK. `url`
* is the one field this package requires and validates itself.
*/
exporter?: OTLPExporterNodeConfigBase & {
/** Full logs endpoint (e.g. `https://collector.example.com/v1/logs`). Required outside `DISABLED`; validated at load. */
url?: string
}
/**
* Passed verbatim to `BatchLogRecordProcessor` (minus the exporter slot,
* which this plugin fills); the SDK owns and documents these knobs.
*/
processor?: Omit<BatchLogRecordProcessorOptions, 'exporter'>
/** Maximum time spent awaiting the SDK provider's complete shutdown path. */
shutdownTimeoutMillis?: number
}
/**
* Schemastery validator for {@link Config}; cordis runs it before the plugin
* starts. It checks only the top-level fields; value checks live in the constructor
* so their errors name the fields. Both SDK option objects pass through unchanged:
* the SDK defines and validates their fields. Re-declaring them here would
* silently drop every field this plugin did not repeat.
*/
export const Config: z<Config> = z.object({
mode: z.union(Object.values(SessionTelemetryMode)).default(DEFAULT_TELEMETRY_MODE),
exporter: z.any(),
processor: z.any(),
shutdownTimeoutMillis: z.number(),
})
/** Default outer allowance for the SDK's complete shutdown sequence. */
export const DEFAULT_SHUTDOWN_TIMEOUT_MILLIS = 3_000
// Node clamps larger timer delays to one millisecond. This is a runtime
// protocol limit, not a deployment default.
const MAX_TIMER_DELAY_MILLIS = 2_147_483_647
/** Severity mapping from the Service Definition's three-level vocabulary to OTel severity numbers. */
const SEVERITY: Record<SessionTelemetrySeverity, { severityNumber: SeverityNumber; severityText: string }> = {
info: { severityNumber: SeverityNumber.INFO, severityText: 'INFO' },
warn: { severityNumber: SeverityNumber.WARN, severityText: 'WARN' },
error: { severityNumber: SeverityNumber.ERROR, severityText: 'ERROR' },
}
/**
* The backend plugin — the only entry a deployment loads. It always registers
* the `telemetry` service (duplicate load throws). Uploading modes wire the SDK
* pipeline and compose {@link SessionTelemetryCoordinator}; `DISABLED` constructs no
* SDK state and listens only to warn when recorded feedback stays local.
*/
export class OpenTelemetrySessionBackend extends SessionTelemetryBackend {
static inject = ['sessions']
static Config = Config
private readonly directEmit: SessionTelemetrySink['emit']
private readonly provider: LoggerProvider | undefined
private readonly shutdownTimeoutMillis: number
override readonly sharing: SessionTelemetrySharingStatus
constructor(ctx: Context, config: Config) {
const mode = resolveMode(config.mode)
super(ctx)
this.sharing = sharingStatusFor(mode)
if (mode === SessionTelemetryMode.DISABLED) {
this.directEmit = DROP_RECORD
this.provider = undefined
this.shutdownTimeoutMillis = DEFAULT_SHUTDOWN_TIMEOUT_MILLIS
ctx.on('session/event', (_session, event) => {
if (event.type === 'feedback/record') ctx.logger.warn(DISABLED_FEEDBACK_WARNING)
})
return
}
const url = config.exporter?.url
if (url === undefined || url.length === 0) {
throw new Error('session-telemetry-otel: exporter.url is required (the full OTLP logs endpoint)')
}
let parsed: URL
try {
parsed = new URL(url)
} catch {
// Re-thrown as a config error: the only way here is a malformed url string.
throw new Error(`session-telemetry-otel: exporter.url is not a valid URL: ${JSON.stringify(url)}`)
}
if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') {
throw new Error(`session-telemetry-otel: exporter.url must be http(s), got ${parsed.protocol}`)
}
// The one processor field checked beyond the SDK's own validation: the
// SDK accepts a non-positive batch size, but its shutdown drain then
// splices empty batches without consuming the queue — dispose would hang
// forever with records queued. Misconfiguration fails at load instead.
const batchSize = config.processor?.maxExportBatchSize
if (batchSize !== undefined && (!Number.isInteger(batchSize) || batchSize < 1)) {
throw new Error(`session-telemetry-otel: processor.maxExportBatchSize must be a positive integer, got ${String(batchSize)}`)
}
const shutdownTimeoutMillis = config.shutdownTimeoutMillis ?? DEFAULT_SHUTDOWN_TIMEOUT_MILLIS
if (!Number.isFinite(shutdownTimeoutMillis) || shutdownTimeoutMillis <= 0 || shutdownTimeoutMillis > MAX_TIMER_DELAY_MILLIS) {
throw new Error(`session-telemetry-otel: shutdownTimeoutMillis must be a positive finite number no greater than ${MAX_TIMER_DELAY_MILLIS}, got ${String(shutdownTimeoutMillis)}`)
}
this.shutdownTimeoutMillis = shutdownTimeoutMillis
this.provider = new LoggerProvider({
resource: resourceFromAttributes({
'service.name': APP_IDENTITY.product,
'service.version': APP_IDENTITY.version,
// OTel semconv's standard user attribute, carried once per export
// batch on the Resource rather than per record: the collector
// aggregates by Resource, and the id is process-stable anyway.
'user.id': getOrCreateAnonymousUserId(),
}),
processors: [
new BatchLogRecordProcessor({
...config.processor,
// The complete validated exporter object, verbatim: every SDK
// option (`timeoutMillis`, `compression`, `keepAlive`, …) reaches
// the exporter — rebuilding selected fields here would silently
// ignore the rest. App identity travels in the Resource
// (service.name/version); the transport-level user-agent is the
// SDK's own, per the axiom.
exporter: new OTLPLogExporter(config.exporter),
}),
],
})
const ledger = this.provider.getLogger('@deepseek-ai/dsh-session-telemetry-otel', version)
const ops = this.provider.getLogger('@deepseek-ai/dsh-session-telemetry-otel/ops', version)
const enqueue: SessionTelemetrySink['emit'] = (record) => {
const logger: Logger = record.channel === 'ops' ? ops : ledger
logger.emit({
timestamp: record.time,
observedTimestamp: record.time,
...SEVERITY[record.severity],
// JSON-serializable by the seam's contract (validated at Session.append),
// which is exactly the AnyValue subset.
body: record.body as AnyValue,
attributes: record.attributes,
})
}
const backend: SessionTelemetrySink = {
emit: enqueue,
shutdown: () => this.shutdown(),
}
if (mode === SessionTelemetryMode.FULL) {
this.directEmit = enqueue
new SessionTelemetryCoordinator(ctx, backend, 'live')
return
}
this.directEmit = DROP_RECORD
const coordinator = new SessionTelemetryCoordinator(ctx, backend, 'on-demand')
ctx.on('session/event', (session, event) => {
if (event.type !== 'feedback/record') return
// Consent is the committed record, not an independently emitted bus value.
if (session.events[event.seq] !== event) {
ctx.logger.warn(NON_CANONICAL_FEEDBACK_WARNING)
return
}
coordinator.captureSession(session, event.seq)
})
}
/**
* Hand a direct service record to the SDK only in `FULL`. Direct calls are
* no-ops in `FEEDBACK_ONLY` and `DISABLED`; feedback replay uses a private
* backend capability created only for the canonical feedback listener.
* @param record - the logical record offered directly to the service.
*/
emit(record: SessionTelemetryRecord): void {
this.directEmit(record)
}
// The Service Definition's optional flush() hint is deliberately NOT implemented. The
// batch processor exports on its own cadence (`processor.scheduledDelayMillis`,
// the SDK's documented knob), and this backend is the SDK pipeline's only
// caller — forwarding the hint to `forceFlush()` would be the sole source of
// concurrent flushes, whose undocumented interactions with shutdown's
// internal drain (concurrent-flush guard, provider-level flush timeout)
// silently drop tail records. Rationale and the revival trigger: the
// revival Agent Note.
/**
* Ask the SDK to drain and quiesce, but reject after the backend-owned
* deadline. OTel's processor export timeout wraps `exportCompleted` only;
* shutdown awaits `exporter.forceFlush()` first, which can remain pending
* when the transport never obtains a socket. The provider promise remains
* observed after the deadline so a later rejection cannot become unhandled.
* `DISABLED` has no provider and resolves immediately.
* @returns resolves when the SDK pipeline quiesces or is disabled, or rejects at the configured deadline.
*/
async shutdown(): Promise<void> {
if (this.provider === undefined) return
const providerShutdown = this.provider.shutdown()
let timer: ReturnType<typeof setTimeout> | undefined
const deadline = new Promise<never>((_resolve, reject) => {
timer = setTimeout(() => {
reject(new Error(`session-telemetry-otel: provider shutdown exceeded ${this.shutdownTimeoutMillis}ms`))
}, this.shutdownTimeoutMillis)
})
try {
await Promise.race([providerShutdown, deadline])
} finally {
/* v8 ignore else -- the Promise executor assigns timer synchronously before this race starts. */
if (timer !== undefined) clearTimeout(timer)
}
}
}
export default OpenTelemetrySessionBackend