Skip to content

Commit 97d90fa

Browse files
litianningdatadogclaude
litianningdatadog
and
claude
committed
feat(logging): add log capture with native transport injection
Implements automatic log forwarding for Winston, Bunyan, and Pino by injecting native HTTP transports/streams directly into logger instances. Features: - Zero-configuration automatic injection via diagnostic channels - Native performance using logger-specific transports - Full trace correlation (trace_id, span_id, service, env, version) - Non-invasive (logs still go to original destinations) - Single intake endpoint with automatic format detection - Configurable batching and buffering Implementation: - Winston: Uses native winston.transports.Http - Bunyan: Custom Writable stream in object mode - Pino: Custom transport with NDJSON parsing Configuration: 9 new options (DD_LOG_CAPTURE_*) Performance: ~1.5-2μs per log overhead Package size: +33 KB (+0.13%) Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com>
1 parent c589ad4 commit 97d90fa

24 files changed

Lines changed: 4583 additions & 720 deletions

‎TRANSPORT_INJECTION_DESIGN.md‎

Lines changed: 1460 additions & 0 deletions
Large diffs are not rendered by default.

‎integration-tests/network-transport-injection/benchmark-overhead.js‎

Lines changed: 166 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,166 @@
1+
#!/usr/bin/env node
2+
3+
/**
4+
* Benchmark to measure performance overhead of transport injection
5+
* Compares logging performance with and without transport injection
6+
*/
7+
8+
const { performance } = require('perf_hooks')
9+
10+
// Test configurations
11+
const NUM_LOGS = 10000
12+
const WARMUP_LOGS = 1000
13+
14+
console.log('\n=== Transport Injection Performance Benchmark ===\n')
15+
console.log(`Iterations: ${NUM_LOGS.toLocaleString()}`)
16+
console.log(`Warmup: ${WARMUP_LOGS.toLocaleString()}\n`)
17+
18+
// Helper to measure performance
19+
function benchmark(name, fn) {
20+
// Warmup
21+
for (let i = 0; i < WARMUP_LOGS; i++) {
22+
fn()
23+
}
24+
25+
// Force GC if available
26+
if (global.gc) global.gc()
27+
28+
// Actual benchmark
29+
const start = performance.now()
30+
for (let i = 0; i < NUM_LOGS; i++) {
31+
fn()
32+
}
33+
const end = performance.now()
34+
35+
const totalMs = end - start
36+
const perLogUs = (totalMs * 1000) / NUM_LOGS
37+
38+
console.log(`${name}:`)
39+
console.log(` Total: ${totalMs.toFixed(2)}ms`)
40+
console.log(` Per-log: ${perLogUs.toFixed(3)}μs`)
41+
console.log(` Rate: ${(NUM_LOGS / (totalMs / 1000)).toFixed(0)} logs/sec`)
42+
console.log('')
43+
44+
return { totalMs, perLogUs }
45+
}
46+
47+
// ==================== WINSTON ====================
48+
console.log('--- Winston ---\n')
49+
50+
// Without transport injection
51+
const winston1 = require('winston')
52+
const winstonLoggerBaseline = winston1.createLogger({
53+
level: 'info',
54+
format: winston1.format.json(),
55+
transports: [new winston1.transports.Console({ silent: true })]
56+
})
57+
58+
const winstonBaseline = benchmark('Winston (baseline)', () => {
59+
winstonLoggerBaseline.info('test message', { userId: 12345, action: 'test' })
60+
})
61+
62+
// With transport injection
63+
process.env.DD_LOG_CAPTURE_ENABLED = 'true'
64+
process.env.DD_LOG_CAPTURE_METHOD = 'transport'
65+
process.env.DD_LOG_CAPTURE_HOST = 'localhost'
66+
process.env.DD_LOG_CAPTURE_PORT = '9999' // Non-existent server (buffering only)
67+
process.env.DD_LOG_CAPTURE_FLUSH_INTERVAL_MS = '999999' // Very long (no flushing during test)
68+
process.env.DD_LOGS_INJECTION = 'true'
69+
70+
require('../../index').init({
71+
service: 'benchmark-test',
72+
env: 'test',
73+
version: '1.0.0'
74+
})
75+
76+
const winston2 = require('winston')
77+
const winstonLoggerWithTransport = winston2.createLogger({
78+
level: 'info',
79+
format: winston2.format.json(),
80+
transports: [new winston2.transports.Console({ silent: true })]
81+
})
82+
83+
const winstonWithTransport = benchmark('Winston (with transport)', () => {
84+
winstonLoggerWithTransport.info('test message', { userId: 12345, action: 'test' })
85+
})
86+
87+
const winstonOverheadUs = winstonWithTransport.perLogUs - winstonBaseline.perLogUs
88+
const winstonOverheadPct = ((winstonOverheadUs / winstonBaseline.perLogUs) * 100).toFixed(1)
89+
console.log(`Winston Overhead: ${winstonOverheadUs.toFixed(3)}μs (${winstonOverheadPct}%)\n`)
90+
91+
// ==================== BUNYAN ====================
92+
console.log('--- Bunyan ---\n')
93+
94+
// Without transport injection
95+
delete require.cache[require.resolve('bunyan')]
96+
const bunyan1 = require('bunyan')
97+
const bunyanLoggerBaseline = bunyan1.createLogger({
98+
name: 'benchmark-baseline',
99+
level: 'info',
100+
streams: [{ stream: { write: () => {} } }] // Null stream
101+
})
102+
103+
const bunyanBaseline = benchmark('Bunyan (baseline)', () => {
104+
bunyanLoggerBaseline.info({ userId: 12345, action: 'test' }, 'test message')
105+
})
106+
107+
// With transport injection (already initialized tracer above)
108+
delete require.cache[require.resolve('bunyan')]
109+
const bunyan2 = require('bunyan')
110+
const bunyanLoggerWithTransport = bunyan2.createLogger({
111+
name: 'benchmark-with-transport',
112+
level: 'info',
113+
streams: [{ stream: { write: () => {} } }] // Null stream
114+
})
115+
116+
const bunyanWithTransport = benchmark('Bunyan (with transport)', () => {
117+
bunyanLoggerWithTransport.info({ userId: 12345, action: 'test' }, 'test message')
118+
})
119+
120+
const bunyanOverheadUs = bunyanWithTransport.perLogUs - bunyanBaseline.perLogUs
121+
const bunyanOverheadPct = ((bunyanOverheadUs / bunyanBaseline.perLogUs) * 100).toFixed(1)
122+
console.log(`Bunyan Overhead: ${bunyanOverheadUs.toFixed(3)}μs (${bunyanOverheadPct}%)\n`)
123+
124+
// ==================== PINO ====================
125+
console.log('--- Pino ---\n')
126+
127+
// Without transport injection
128+
const pino1 = require('pino')
129+
const pinoLoggerBaseline = pino1({ level: 'info' }, { write: () => {} }) // Null destination
130+
131+
const pinoBaseline = benchmark('Pino (baseline)', () => {
132+
pinoLoggerBaseline.info({ userId: 12345, action: 'test' }, 'test message')
133+
})
134+
135+
// With transport injection (already initialized tracer above)
136+
delete require.cache[require.resolve('pino')]
137+
const pino2 = require('pino')
138+
const pinoLoggerWithTransport = pino2({ level: 'info' }, { write: () => {} })
139+
140+
const pinoWithTransport = benchmark('Pino (with transport)', () => {
141+
pinoLoggerWithTransport.info({ userId: 12345, action: 'test' }, 'test message')
142+
})
143+
144+
const pinoOverheadUs = pinoWithTransport.perLogUs - pinoBaseline.perLogUs
145+
const pinoOverheadPct = ((pinoOverheadUs / pinoBaseline.perLogUs) * 100).toFixed(1)
146+
console.log(`Pino Overhead: ${pinoOverheadUs.toFixed(3)}μs (${pinoOverheadPct}%)\n`)
147+
148+
// ==================== SUMMARY ====================
149+
console.log('=== Summary ===\n')
150+
console.log('Per-Log Overhead:')
151+
console.log(` Winston: ${winstonOverheadUs.toFixed(3)}μs (${winstonOverheadPct}% increase)`)
152+
console.log(` Bunyan: ${bunyanOverheadUs.toFixed(3)}μs (${bunyanOverheadPct}% increase)`)
153+
console.log(` Pino: ${pinoOverheadUs.toFixed(3)}μs (${pinoOverheadPct}% increase)`)
154+
console.log('')
155+
156+
// Memory info
157+
const memUsage = process.memoryUsage()
158+
console.log('Memory Usage:')
159+
console.log(` Heap Used: ${(memUsage.heapUsed / 1024 / 1024).toFixed(2)} MB`)
160+
console.log(` External: ${(memUsage.external / 1024 / 1024).toFixed(2)} MB`)
161+
console.log('')
162+
163+
console.log('Note: Run with --expose-gc for accurate memory measurements')
164+
console.log('Example: node --expose-gc benchmark-overhead.js\n')
165+
166+
process.exit(0)

‎integration-tests/network-transport-injection/run-transport-tests.sh‎

Lines changed: 97 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,97 @@
1+
#!/bin/bash
2+
3+
# Test runner for all three logger transport injection implementations
4+
#
5+
# Usage:
6+
# From this directory: ./run-transport-tests.sh
7+
# From repo root: ./integration-tests/network-transport-injection/run-transport-tests.sh
8+
#
9+
# Prerequisites:
10+
# 1. Start intake server in another terminal:
11+
# node integration-tests/network-transport-injection/test-intake-server.js
12+
# 2. Logger dependencies will be checked and installed if needed
13+
14+
set -e
15+
16+
# Get the directory where this script is located
17+
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
18+
REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
19+
20+
# Change to repo root for npm commands
21+
cd "$REPO_ROOT"
22+
23+
# Check and install logger dependencies if needed
24+
echo "Checking logger dependencies..."
25+
if ! npm list winston bunyan pino pino-pretty &>/dev/null; then
26+
echo "Installing logger packages (winston, bunyan, pino, pino-pretty)..."
27+
npm install --no-save winston bunyan pino pino-pretty
28+
echo "✓ Dependencies installed"
29+
else
30+
echo "✓ All dependencies present"
31+
fi
32+
echo ""
33+
34+
# Change back to script directory to run tests
35+
cd "$SCRIPT_DIR"
36+
37+
echo "=============================================="
38+
echo " Logger Transport Injection Test Suite"
39+
echo "=============================================="
40+
echo ""
41+
42+
# Check if intake server is running
43+
echo "Checking if intake server is running on port 8080..."
44+
if ! nc -z localhost 8080 2>/dev/null; then
45+
echo "⚠️ WARNING: Intake server not detected on port 8080"
46+
echo " Start it with: node integration-tests/network-transport-injection/test-intake-server.js"
47+
echo " Or from this directory: node test-intake-server.js"
48+
echo ""
49+
read -p "Continue anyway? (y/N) " -n 1 -r
50+
echo ""
51+
if [[ ! $REPLY =~ ^[Yy]$ ]]; then
52+
exit 1
53+
fi
54+
else
55+
echo "✓ Intake server is running"
56+
fi
57+
58+
echo ""
59+
echo "=============================================="
60+
echo " Test 1/3: Winston Transport Injection"
61+
echo "=============================================="
62+
node test-winston-transport.js
63+
64+
echo ""
65+
echo "=============================================="
66+
echo " Test 2/3: Bunyan Stream Injection"
67+
echo "=============================================="
68+
node test-bunyan-transport.js
69+
70+
echo ""
71+
echo "=============================================="
72+
echo " Test 3/4: Pino Simple (No User Transport)"
73+
echo "=============================================="
74+
node test-pino-simple.js
75+
76+
echo ""
77+
echo "=============================================="
78+
echo " Test 4/4: Pino with pino-pretty"
79+
echo "=============================================="
80+
node test-pino-transport.js
81+
82+
echo ""
83+
echo "=============================================="
84+
echo " All Tests Complete!"
85+
echo "=============================================="
86+
echo ""
87+
echo "Check the intake server output to verify:"
88+
echo " ✓ All 20 logs received (5 per test × 4 tests)"
89+
echo " ✓ Trace correlation present (trace_id, span_id)"
90+
echo " ✓ Service metadata included (service, env, version)"
91+
echo ""
92+
echo "Test coverage:"
93+
echo " ✓ Winston: Native HTTP transport"
94+
echo " ✓ Bunyan: Custom stream with timing fix"
95+
echo " ✓ Pino Simple: Basic HTTP injection"
96+
echo " ✓ Pino Pretty: Multistream auto-combination"
97+
echo ""

0 commit comments

Comments
 (0)