Skip to content

Commit 12ae601

Browse files
amychisholm03amychisholm03
authored
test: Refactor LangChain versioned tests (#3624)
1 parent d1e90e4 commit 12ae601

20 files changed

+2186
-2894
lines changed

‎test/versioned/langchain-aws/package.json‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
{
2+
"name": "langchain-aws-bedrock-tests",
3+
"targets": [
4+
{
5+
"name": "@langchain/core",
6+
"minSupported": "0.1.17",
7+
"minAgentVersion": "11.13.0"
8+
}
9+
],
10+
"version": "0.0.0",
11+
"private": true,
12+
"engines": {
13+
"node": ">=20"
14+
},
15+
"tests": [
16+
{
17+
"engines": {
18+
"node": ">=20"
19+
},
20+
"dependencies": {
21+
"@langchain/aws": ">=1.1.0",
22+
"@langchain/core": ">=1.0.0",
23+
"@langchain/community": ">=1.0.0",
24+
"@elastic/elasticsearch": "8.13.1"
25+
},
26+
"files": [
27+
"runnables.test.js",
28+
"runnables-streaming.test.js",
29+
"vectorstore.test.js"
30+
]
31+
}
32+
]
33+
}

‎test/versioned/langchain-aws/runnables-streaming.test.js‎

Lines changed: 117 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,117 @@
1+
/*
2+
* Copyright 2025 New Relic Corporation. All rights reserved.
3+
* SPDX-License-Identifier: Apache-2.0
4+
*/
5+
6+
'use strict'
7+
8+
const test = require('node:test')
9+
const assert = require('node:assert')
10+
11+
const { removeModules } = require('../../lib/cache-buster')
12+
const { match } = require('../../lib/custom-assertions')
13+
const {
14+
runStreamingEnabledTests,
15+
runStreamingDisabledTest,
16+
runAiMonitoringDisabledTests
17+
} = require('../langchain/runnables-streaming')
18+
const { FAKE_CREDENTIALS, getAiResponseServer } = require('../../lib/aws-server-stubs')
19+
const helper = require('../../lib/agent_helper')
20+
21+
const config = {
22+
ai_monitoring: {
23+
enabled: true,
24+
streaming: {
25+
enabled: true
26+
}
27+
}
28+
}
29+
const createAiResponseServer = getAiResponseServer(__dirname)
30+
31+
async function beforeEach({ enabled, ctx }) {
32+
ctx.nr = {}
33+
const { server, baseUrl } = await createAiResponseServer()
34+
ctx.nr.server = server
35+
ctx.nr.agent = helper.instrumentMockedAgent(config)
36+
ctx.nr.agent.config.ai_monitoring.streaming.enabled = enabled
37+
38+
const { ChatPromptTemplate } = require('@langchain/core/prompts')
39+
const { StringOutputParser, CommaSeparatedListOutputParser } = require('@langchain/core/output_parsers')
40+
const { BaseCallbackHandler } = require('@langchain/core/callbacks/base')
41+
const { ChatBedrockConverse } = require('@langchain/aws')
42+
const { BedrockRuntimeClient } = require('@aws-sdk/client-bedrock-runtime')
43+
ctx.nr.ChatPromptTemplate = ChatPromptTemplate
44+
ctx.nr.CommaSeparatedListOutputParser = CommaSeparatedListOutputParser
45+
ctx.nr.BaseCallbackHandler = BaseCallbackHandler
46+
ctx.nr.langchainCoreVersion = require('@langchain/core/package.json').version
47+
48+
// Create the BedrockRuntimeClient with our mock endpoint
49+
const bedrockClient = new BedrockRuntimeClient({
50+
region: 'us-east-1',
51+
credentials: FAKE_CREDENTIALS,
52+
endpoint: baseUrl,
53+
maxAttempts: 1
54+
})
55+
56+
ctx.nr.prompt = ChatPromptTemplate.fromMessages([['assistant', 'text converse ultimate question {topic}']])
57+
ctx.nr.model = new ChatBedrockConverse({
58+
streaming: true,
59+
model: 'anthropic.claude-instant-v1',
60+
region: 'us-east-1',
61+
client: bedrockClient
62+
})
63+
ctx.nr.outputParser = new StringOutputParser()
64+
}
65+
66+
async function afterEach(ctx) {
67+
ctx.nr?.server?.destroy()
68+
helper.unloadAgent(ctx.nr.agent)
69+
// bust the require-cache so it can re-instrument
70+
removeModules(['@langchain/core', '@langchain/aws', '@aws-sdk'])
71+
}
72+
73+
test('streaming enabled', async (t) => {
74+
t.beforeEach((ctx) => beforeEach({ enabled: true, ctx }))
75+
t.afterEach((ctx) => afterEach(ctx))
76+
77+
await runStreamingEnabledTests({
78+
inputData: { topic: 'streamed' },
79+
expectedInput: '{"topic":"streamed"}',
80+
expectedContent: () => 'This is a test.',
81+
errorPromptTemplate: ['assistant', 'text converse ultimate question streamed error'],
82+
errorFromStreamEventCount: 4,
83+
errorFromStreamLangchainEventCount: 2,
84+
errorFromStreamAssertion: (exceptions) => {
85+
assert.equal(exceptions.length, 2)
86+
for (const e of exceptions) {
87+
match(e, {
88+
customAttributes: {
89+
'error.message': /Internal server error during streaming/,
90+
completion_id: /[\w-]{36}/
91+
}
92+
})
93+
}
94+
}
95+
})(t)
96+
})
97+
98+
test('streaming disabled', async (t) => {
99+
t.beforeEach((ctx) => beforeEach({ enabled: false, ctx }))
100+
t.afterEach((ctx) => afterEach(ctx))
101+
102+
await runStreamingDisabledTest({
103+
inputData: { topic: 'streamed' },
104+
expectedContent: () => 'This is a test.',
105+
streamingDisabledMessage: 'should increment streaming disabled in both langchain and bedrock'
106+
})(t)
107+
})
108+
109+
test('ai_monitoring disabled', async (t) => {
110+
t.beforeEach((ctx) => beforeEach({ enabled: true, ctx }))
111+
t.afterEach((ctx) => afterEach(ctx))
112+
113+
await runAiMonitoringDisabledTests({
114+
inputData: { topic: 'streamed' },
115+
expectedContent: () => 'This is a test.'
116+
})(t)
117+
})

‎test/versioned/langchain-aws/runnables.test.js‎

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
/*
2+
* Copyright 2025 New Relic Corporation. All rights reserved.
3+
* SPDX-License-Identifier: Apache-2.0
4+
*/
5+
6+
'use strict'
7+
8+
const test = require('node:test')
9+
10+
const { removeModules } = require('../../lib/cache-buster')
11+
const { runRunnablesTests } = require('../langchain/runnables')
12+
const { FAKE_CREDENTIALS, getAiResponseServer } = require('../../lib/aws-server-stubs')
13+
const helper = require('../../lib/agent_helper')
14+
15+
const config = {
16+
ai_monitoring: {
17+
enabled: true
18+
}
19+
}
20+
const createAiResponseServer = getAiResponseServer(__dirname)
21+
22+
test.beforeEach(async (ctx) => {
23+
ctx.nr = {}
24+
const { server, baseUrl } = await createAiResponseServer()
25+
ctx.nr.server = server
26+
ctx.nr.agent = helper.instrumentMockedAgent(config)
27+
28+
const { ChatPromptTemplate } = require('@langchain/core/prompts')
29+
const { StringOutputParser, CommaSeparatedListOutputParser } = require('@langchain/core/output_parsers')
30+
const { BaseCallbackHandler } = require('@langchain/core/callbacks/base')
31+
const { ChatBedrockConverse } = require('@langchain/aws')
32+
const { BedrockRuntimeClient } = require('@aws-sdk/client-bedrock-runtime')
33+
ctx.nr.ChatPromptTemplate = ChatPromptTemplate
34+
ctx.nr.CommaSeparatedListOutputParser = CommaSeparatedListOutputParser
35+
ctx.nr.BaseCallbackHandler = BaseCallbackHandler
36+
ctx.nr.langchainCoreVersion = require('@langchain/core/package.json').version
37+
38+
// Create the BedrockRuntimeClient with our mock endpoint
39+
const bedrockClient = new BedrockRuntimeClient({
40+
region: 'us-east-1',
41+
credentials: FAKE_CREDENTIALS,
42+
endpoint: baseUrl,
43+
maxAttempts: 1
44+
})
45+
46+
ctx.nr.prompt = ChatPromptTemplate.fromMessages([['assistant', 'text converse ultimate {topic}']])
47+
ctx.nr.model = new ChatBedrockConverse({
48+
model: 'anthropic.claude-3-haiku-20240307-v1:0',
49+
region: 'us-east-1',
50+
client: bedrockClient
51+
})
52+
ctx.nr.outputParser = new StringOutputParser()
53+
})
54+
55+
test.afterEach(async (ctx) => {
56+
ctx.nr?.server?.destroy()
57+
helper.unloadAgent(ctx.nr.agent)
58+
// bust the require-cache so it can re-instrument
59+
removeModules(['@langchain/core', '@langchain/aws', '@aws-sdk'])
60+
})
61+
62+
runRunnablesTests({
63+
inputData: { topic: 'question' },
64+
expectedInput: '{"topic":"question"}',
65+
expectedOutput: 'This is a test.',
66+
errorPromptTemplate: ['assistant', 'text converse ultimate question error'],
67+
errorEventCount: 5,
68+
arrayParserOutput: '["This is a test."]'
69+
})

‎test/versioned/langchain-aws/vectorstore.test.js‎

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
/*
2+
* Copyright 2025 New Relic Corporation. All rights reserved.
3+
* SPDX-License-Identifier: Apache-2.0
4+
*/
5+
6+
'use strict'
7+
8+
const test = require('node:test')
9+
10+
const { removeModules } = require('../../lib/cache-buster')
11+
const { runVectorstoreTests } = require('../langchain/vectorstore')
12+
const { Document } = require('@langchain/core/documents')
13+
const { FAKE_CREDENTIALS, getAiResponseServer } = require('../../lib/aws-server-stubs')
14+
const params = require('../../lib/params')
15+
const helper = require('../../lib/agent_helper')
16+
17+
const config = {
18+
ai_monitoring: {
19+
enabled: true
20+
}
21+
}
22+
const createAiResponseServer = getAiResponseServer(__dirname)
23+
24+
test.beforeEach(async (ctx) => {
25+
ctx.nr = {}
26+
const { server, baseUrl } = await createAiResponseServer()
27+
ctx.nr.server = server
28+
ctx.nr.agent = helper.instrumentMockedAgent(config)
29+
30+
const { BedrockEmbeddings } = require('@langchain/aws')
31+
const { BedrockRuntimeClient } = require('@aws-sdk/client-bedrock-runtime')
32+
ctx.nr.langchainCoreVersion = require('@langchain/core/package.json').version
33+
34+
const { Client } = require('@elastic/elasticsearch')
35+
const clientArgs = {
36+
client: new Client({
37+
node: `http://${params.elastic_host}:${params.elastic_port}`
38+
}),
39+
indexName: 'test_langchain_aws_vectorstore'
40+
}
41+
const { ElasticVectorSearch } = require('@langchain/community/vectorstores/elasticsearch')
42+
43+
// Create the BedrockRuntimeClient with our mock endpoint
44+
const bedrockClient = new BedrockRuntimeClient({
45+
region: 'us-east-1',
46+
credentials: FAKE_CREDENTIALS,
47+
endpoint: baseUrl,
48+
maxAttempts: 1
49+
})
50+
51+
ctx.nr.embedding = new BedrockEmbeddings({
52+
model: 'amazon.titan-embed-text-v1',
53+
region: 'us-east-1',
54+
client: bedrockClient,
55+
maxRetries: 0
56+
})
57+
const docs = [
58+
new Document({
59+
metadata: { id: '2' },
60+
pageContent: 'embed text amazon token count callback response'
61+
})
62+
]
63+
const vectorStore = new ElasticVectorSearch(ctx.nr.embedding, clientArgs)
64+
await vectorStore.deleteIfExists()
65+
await vectorStore.addDocuments(docs)
66+
ctx.nr.vs = vectorStore
67+
})
68+
69+
test.afterEach(async (ctx) => {
70+
await ctx.nr?.vs?.deleteIfExists()
71+
ctx.nr?.server?.destroy()
72+
helper.unloadAgent(ctx.nr.agent)
73+
// bust the require-cache so it can re-instrument
74+
removeModules(['@langchain/core', '@langchain/aws', '@aws-sdk', '@elastic', '@langchain/community'])
75+
})
76+
77+
runVectorstoreTests({
78+
searchQuery: 'embed text amazon token count callback response',
79+
expectedQuery: 'embed text amazon token count callback response',
80+
expectedPageContent: 'embed text amazon token count callback response'
81+
})

‎test/versioned/langchain-openai/package.json‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
{
2+
"name": "langchain-openai-tests",
3+
"targets": [
4+
{
5+
"name": "@langchain/core",
6+
"minSupported": "0.1.17",
7+
"minAgentVersion": "11.13.0"
8+
}
9+
],
10+
"version": "0.0.0",
11+
"private": true,
12+
"engines": {
13+
"node": ">=20"
14+
},
15+
"tests": [
16+
{
17+
"engines": {
18+
"node": ">=20"
19+
},
20+
"dependencies": {
21+
"@langchain/core": ">=1.0.0",
22+
"@langchain/community": ">=1.0.0",
23+
"@langchain/openai": ">=1.0.0",
24+
"openai": "4.90.0",
25+
"@elastic/elasticsearch": "8.13.1"
26+
},
27+
"files": [
28+
"runnables.test.js",
29+
"runnables-streaming.test.js",
30+
"vectorstore.test.js"
31+
]
32+
}
33+
]
34+
}

0 commit comments

Comments
 (0)