diff --git a/docs/advanced-config.md b/docs/advanced-config.md index aa04d41e..0048004e 100644 --- a/docs/advanced-config.md +++ b/docs/advanced-config.md @@ -66,7 +66,6 @@ This distribution supports all the configuration options supported by the compon | `OTEL_SPAN_LINK_COUNT_LIMIT` | `1000`\* | Stable | | `OTEL_TRACES_EXPORTER`
`tracing.spanExporterFactory` | `otlp` | Stable | Chooses the trace exporters. Shortcut for setting `spanExporterFactory`. Comma-delimited list of exporters. Currently supported values: `otlp`, `console`, `none`. | `OTEL_EXPORTER_OTLP_TRACES_PROTOCOL` | `http/protobuf` | Stable | Metric exporter protocol. Supported values: `http/protobuf`, `grpc`. -| `OTEL_TRACES_SAMPLER` | `parentbased_always_on` | Stable | Sampler to be used for traces. See [Sampling](https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/trace/sdk.md#sampling) | `OTEL_TRACES_SAMPLER_ARG` | | Stable | String value to be used as the sampler argument. Only be used if OTEL_TRACES_SAMPLER is set. | `SPLUNK_ACCESS_TOKEN`
`accessToken` | | Stable | The optional access token for exporting signal data directly to SignalFx API. | `SPLUNK_REALM`
`realm` | | Stable | The name of your organization's realm, for example, ``us0``. When you set the realm, telemetry is sent directly to the ingest endpoint of Splunk Observability Cloud, bypassing the Splunk OpenTelemetry Collector. Overridden by settings that define a complete endpoint URL, like `OTEL_EXPORTER_OTLP_ENDPOINT`. @@ -75,6 +74,41 @@ This distribution supports all the configuration options supported by the compon \*: Overwritten default value +#### Sampling configuration + +| Environment variable | Default value | Support | Notes +| ------------------------ | ------------------------ | -------------- | ------- | ----------- | +| `OTEL_TRACES_SAMPLER` | `always_on` | Stable | Sampler to be used for traces. See [Sampling](https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/trace/sdk.md#sampling) + +Splunk Distribution of OpenTelemetry JS supports all standard samplers as provided by +[OpenTelemetry JS SDK](https://github.com/open-telemetry/opentelemetry-js/tree/main/packages/opentelemetry-sdk-trace-base#built-in-samplers). +In addition, the distribution adds the following samplers: + +### `rules` +This sampler allows to ignore individual endpoints and drop all traces that originate from them. +It applies only to spans with `SERVER` kind. + +For example, the following configuration results in all requests to `/healthcheck` to be excluded from monitoring: + +```shell +export OTEL_TRACES_SAMPLER=rules +export OTEL_TRACES_SAMPLER_ARG=drop=/healthcheck;fallback=parentbased_always_on +``` +All requests to downstream services that happen as a consequence of calling an excluded endpoint are also excluded. + +The value of `OTEL_TRACES_SAMPLER_ARG` is interpreted as a semicolon-separated list of rules. +The following types of rules are supported: + +- `drop=`: The sampler drops a span if its `url.path` (or `http.target` for instrumentations using older semantic conventions) attribute has a substring equal to the provided value. + You can provide as many `drop` rules as you want. +- `fallback=sampler`: Fallback sampler used if no `drop` rule matched a given span. + Supported fallback samplers are `always_on` and `parentbased_always_on`. + Probability samplers such as `traceidratio` are not supported. + +> If several `fallback` rules are provided, only the last one will be in effect. + +If `OTEL_TRACES_SAMPLER_ARG` is not provided or has en empty value, no `drop` rules are configured and `always_on` sampler is as default. + #### Additional `tracing` config options in `start()` The following config options can be set by passing them as tracing arguments to `start()`. diff --git a/package-lock.json b/package-lock.json index bbc71d35..c410b2a7 100644 --- a/package-lock.json +++ b/package-lock.json @@ -69,6 +69,7 @@ "@opentelemetry/propagator-b3": "2.5.1", "@opentelemetry/resource-detector-container": "0.8.3", "@opentelemetry/resources": "2.5.1", + "@opentelemetry/sampler-composite": "^0.212.0", "@opentelemetry/sdk-logs": "0.212.0", "@opentelemetry/sdk-metrics": "2.5.1", "@opentelemetry/sdk-trace-base": "2.5.1", @@ -2177,6 +2178,22 @@ "@opentelemetry/api": ">=1.3.0 <1.10.0" } }, + "node_modules/@opentelemetry/sampler-composite": { + "version": "0.212.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/sampler-composite/-/sampler-composite-0.212.0.tgz", + "integrity": "sha512-rhsSqJ8CzL6GjP66JNc1UMm3/IweISwqnwSplY2hqpsciWceIWL3grh+6xFnlk7QbG70DPsufPX0uJx6EWG0pg==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.5.1", + "@opentelemetry/sdk-trace-base": "2.5.1" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, "node_modules/@opentelemetry/sdk-logs": { "version": "0.212.0", "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-logs/-/sdk-logs-0.212.0.tgz", @@ -8089,6 +8106,15 @@ "@opentelemetry/semantic-conventions": "^1.29.0" } }, + "@opentelemetry/sampler-composite": { + "version": "0.212.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/sampler-composite/-/sampler-composite-0.212.0.tgz", + "integrity": "sha512-rhsSqJ8CzL6GjP66JNc1UMm3/IweISwqnwSplY2hqpsciWceIWL3grh+6xFnlk7QbG70DPsufPX0uJx6EWG0pg==", + "requires": { + "@opentelemetry/core": "2.5.1", + "@opentelemetry/sdk-trace-base": "2.5.1" + } + }, "@opentelemetry/sdk-logs": { "version": "0.212.0", "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-logs/-/sdk-logs-0.212.0.tgz", diff --git a/package.json b/package.json index c6d3a4d9..df164489 100644 --- a/package.json +++ b/package.json @@ -123,6 +123,7 @@ "@opentelemetry/propagator-b3": "2.5.1", "@opentelemetry/resource-detector-container": "0.8.3", "@opentelemetry/resources": "2.5.1", + "@opentelemetry/sampler-composite": "^0.212.0", "@opentelemetry/sdk-logs": "0.212.0", "@opentelemetry/sdk-metrics": "2.5.1", "@opentelemetry/sdk-trace-base": "2.5.1", diff --git a/src/tracing/RuleBasedSampler.ts b/src/tracing/RuleBasedSampler.ts new file mode 100644 index 00000000..c0ed1f1a --- /dev/null +++ b/src/tracing/RuleBasedSampler.ts @@ -0,0 +1,114 @@ +/* + * Copyright Splunk Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { SpanKind } from '@opentelemetry/api'; +import { Sampler } from '@opentelemetry/sdk-trace-base'; +import { + createComposableAlwaysOffSampler, + createComposableAlwaysOnSampler, + createComposableParentThresholdSampler, + createComposableRuleBasedSampler, + createCompositeSampler, + type ComposableSampler, +} from '@opentelemetry/sampler-composite'; +import { ATTR_URL_PATH } from '@opentelemetry/semantic-conventions'; +import { SamplingPredicate } from '@opentelemetry/sampler-composite/build/src/types'; + +/** + * Deprecated but some instrumentations still default to old semantic conventions + */ +const ATTR_HTTP_TARGET = 'http.target'; + +function selectComposableSampler(type: string): ComposableSampler { + switch (type) { + case 'always_on': + return createComposableAlwaysOnSampler(); + case 'always_off': + return createComposableAlwaysOffSampler(); + case 'parentbased_always_on': + return createComposableParentThresholdSampler( + createComposableAlwaysOnSampler() + ); + case 'parentbased_always_off': + return createComposableParentThresholdSampler( + createComposableAlwaysOffSampler() + ); + default: + throw new Error('Unsupported fallback sampler ' + type); + } +} + +/** + * Parses OTEL_TRACES_SAMPLER_ARG for the `rules` sampler and returns a + * composite sampler ready for use. + * + * Format: semicolon-separated list of key=value rules: + * drop=/healthcheck;fallback=parentbased_always_on + * + * Supported rules: + * drop= - drop SERVER spans whose url.path/http.target contains + * fallback= - fallback sampler; supported: always_on, parentbased_always_on + * + * If arg is not set, defaults to no drop rules and parentbased_always_on fallback. + */ +export function createRuleBasedSampler(arg: string | undefined): Sampler { + const dropValues: string[] = []; + let fallback: ComposableSampler | undefined = undefined; + + if (arg && arg.trim().length > 0) { + for (const rule of arg.split(';')) { + const eqIndex = rule.indexOf('='); + if (eqIndex === -1) { + continue; + } + const key = rule.substring(0, eqIndex).trim(); + const value = rule.substring(eqIndex + 1).trim(); + + if (key === 'drop' && value.length > 0) { + dropValues.push(value); + } else if (key === 'fallback') { + fallback = selectComposableSampler(value); + } + } + } + + const matchesDrops: SamplingPredicate = ( + _ctx, + _traceId, + _name, + spanKind, + attributes + ) => { + if (spanKind !== SpanKind.SERVER) { + return false; + } + + const target = + (attributes[ATTR_URL_PATH] as string | undefined) ?? + (attributes[ATTR_HTTP_TARGET] as string | undefined); + if (typeof target !== 'string') { + return false; + } + return dropValues.some((v) => target.includes(v)); + }; + + return createCompositeSampler( + createComposableRuleBasedSampler([ + [matchesDrops, createComposableAlwaysOffSampler()], + [() => true, fallback ?? createComposableAlwaysOnSampler()], + ]) + ); +} diff --git a/src/tracing/options.ts b/src/tracing/options.ts index fd63762b..2c393965 100644 --- a/src/tracing/options.ts +++ b/src/tracing/options.ts @@ -21,6 +21,7 @@ import { SpanExporter, SpanProcessor, } from '@opentelemetry/sdk-trace-base'; +import { createRuleBasedSampler } from './RuleBasedSampler'; import { B3Propagator, B3InjectEncoding } from '@opentelemetry/propagator-b3'; import { AWSXRayPropagator } from '@opentelemetry/propagator-aws-xray'; @@ -79,9 +80,15 @@ function createSampler(userConfig: NodeTracerConfig) { const configSampler = configGetSampler(); if (configSampler === undefined) { - if (getNonEmptyEnvVar('OTEL_TRACES_SAMPLER') === undefined) { + const envSampler = getNonEmptyEnvVar('OTEL_TRACES_SAMPLER'); + if (envSampler === undefined) { return new AlwaysOnSampler(); } + if (envSampler === 'rules') { + return createRuleBasedSampler( + getNonEmptyEnvVar('OTEL_TRACES_SAMPLER_ARG') + ); + } } return configSampler; diff --git a/src/types.ts b/src/types.ts index 5275482c..1f182e87 100644 --- a/src/types.ts +++ b/src/types.ts @@ -50,6 +50,7 @@ export type EnvVarKey = | 'OTEL_SPAN_LINK_COUNT_LIMIT' | 'OTEL_TRACES_EXPORTER' | 'OTEL_TRACES_SAMPLER' + | 'OTEL_TRACES_SAMPLER_ARG' | 'SPLUNK_ACCESS_TOKEN' | 'SPLUNK_AUTOINSTRUMENT_PACKAGE_NAMES' | 'SPLUNK_AUTOMATIC_LOG_COLLECTION' diff --git a/test/tracing/rule_based_sampler.test.ts b/test/tracing/rule_based_sampler.test.ts new file mode 100644 index 00000000..61c39307 --- /dev/null +++ b/test/tracing/rule_based_sampler.test.ts @@ -0,0 +1,165 @@ +/* + * Copyright Splunk Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { strict as assert } from 'assert'; +import { after, test } from 'node:test'; +import { ROOT_CONTEXT, SpanKind } from '@opentelemetry/api'; +import { + InMemorySpanExporter, + Sampler, + SamplingDecision, + SpanProcessor, +} from '@opentelemetry/sdk-trace-base'; +import { parseOptionsAndConfigureInstrumentations } from '../../src/instrumentations'; +import { createRuleBasedSampler } from '../../src/tracing/RuleBasedSampler'; +import { startTracing, stopTracing } from '../../src/tracing'; +import { defaultSpanProcessorFactory } from '../../src/tracing/options'; +import { doRequest, setupServer } from '../uri_parameter_capture/common'; + +const TRACE_ID = '00000000000000000000000000000001'; + +function sample(sampler: Sampler, kind: SpanKind, urlPath?: string) { + const attributes = urlPath ? { 'url.path': urlPath } : {}; + return sampler.shouldSample( + ROOT_CONTEXT, + TRACE_ID, + 'test-span', + kind, + attributes, + [] + ); +} + +test('RuleBasedSampler: drops SERVER span when url.path matches drop rule', () => { + const sampler = createRuleBasedSampler( + 'drop=/healthcheck;fallback=always_on' + ); + + const result = sample(sampler, SpanKind.SERVER, '/healthcheck'); + assert.equal(result.decision, SamplingDecision.NOT_RECORD); + + const result2 = sample(sampler, SpanKind.SERVER, '/api/users'); + assert.equal(result2.decision, SamplingDecision.RECORD_AND_SAMPLED); +}); + +test('RuleBasedSampler: drops SERVER span when url.path contains drop substring', () => { + const sampler = createRuleBasedSampler('drop=/health;fallback=always_on'); + + const result = sample(sampler, SpanKind.SERVER, '/healthcheck'); + assert.equal(result.decision, SamplingDecision.NOT_RECORD); +}); + +test('RuleBasedSampler: does not drop non-SERVER spans even if url.path matches', () => { + const sampler = createRuleBasedSampler( + 'drop=/healthcheck;fallback=always_on' + ); + + for (const kind of [ + SpanKind.CLIENT, + SpanKind.INTERNAL, + SpanKind.PRODUCER, + SpanKind.CONSUMER, + ]) { + const result = sample(sampler, kind, '/healthcheck'); + assert.equal( + result.decision, + SamplingDecision.RECORD_AND_SAMPLED, + `SpanKind ${kind} should not be dropped` + ); + } +}); + +test('RuleBasedSampler: delegates to fallback when no url.path attribute present', () => { + const sampler = createRuleBasedSampler( + 'drop=/healthcheck;fallback=always_on' + ); + + const result = sample(sampler, SpanKind.SERVER); + assert.equal(result.decision, SamplingDecision.RECORD_AND_SAMPLED); +}); + +test('RuleBasedSampler: supports multiple drop rules', () => { + const sampler = createRuleBasedSampler( + 'drop=/healthcheck;drop=/metrics;drop=/ready;fallback=always_on' + ); + + assert.equal( + sample(sampler, SpanKind.SERVER, '/healthcheck').decision, + SamplingDecision.NOT_RECORD + ); + assert.equal( + sample(sampler, SpanKind.SERVER, '/metrics').decision, + SamplingDecision.NOT_RECORD + ); + assert.equal( + sample(sampler, SpanKind.SERVER, '/ready').decision, + SamplingDecision.NOT_RECORD + ); + assert.equal( + sample(sampler, SpanKind.SERVER, '/foo').decision, + SamplingDecision.RECORD_AND_SAMPLED + ); +}); + +test('RuleBasedSampler: defaults to always_on when arg is undefined', () => { + const sampler = createRuleBasedSampler(undefined); + + assert.equal( + sample(sampler, SpanKind.SERVER, '/foo').decision, + SamplingDecision.RECORD_AND_SAMPLED + ); +}); + +test('Tracing: OTEL_TRACES_SAMPLER=rules uses composite sampler', async () => { + process.env.OTEL_TRACES_SAMPLER = 'rules'; + process.env.OTEL_TRACES_SAMPLER_ARG = + 'drop=/healthcheck;drop=/ready;fallback=parentbased_always_on'; + + const exporter = new InMemorySpanExporter(); + let spanProcessor: SpanProcessor; + const [server, url] = await setupServer(); + + after(async () => { + server.close(); + await stopTracing(); + + delete process.env.OTEL_TRACES_SAMPLER; + delete process.env.OTEL_TRACES_SAMPLER_ARG; + }); + + const { tracingOptions } = parseOptionsAndConfigureInstrumentations({ + tracing: { + spanExporterFactory: () => exporter, + spanProcessorFactory: (options) => { + return ([spanProcessor] = defaultSpanProcessorFactory(options)); + }, + }, + }); + + startTracing(tracingOptions); + + await Promise.all([ + doRequest(`${url}/healthcheck`), + doRequest(`${url}/foo`), + doRequest(`${url}/ready`), + ]); + + await spanProcessor!.forceFlush(); + const spans = exporter.getFinishedSpans(); + + assert.equal(spans.length, 1); + assert.equal(spans[0].attributes['http.target'], '/foo'); +});