Skip to content

Commit c1edc2e

Browse files
dwdoughertydwdougherty
authored
DOC-4039: add TCEs to the query pages - reissue (#2846)
1 parent 49fdb79 commit c1edc2e

8 files changed

+4617
-81
lines changed

‎doctests/data/query_vector.json

+3,909-80
Large diffs are not rendered by default.

‎doctests/package.json

+2-1
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,8 @@
66
"private": true,
77
"type": "module",
88
"dependencies": {
9-
"redis": "../"
9+
"redis": "../",
10+
"@xenova/transformers": "^2.17.2"
1011
}
1112
}
1213

‎doctests/query-agg.js

+139
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,139 @@
1+
// EXAMPLE: query_agg
2+
// HIDE_START
3+
import assert from 'node:assert';
4+
import fs from 'node:fs';
5+
import { createClient } from 'redis';
6+
import { SchemaFieldTypes, AggregateSteps, AggregateGroupByReducers } from '@redis/search';
7+
8+
const client = createClient();
9+
10+
await client.connect().catch(console.error);
11+
12+
// create index
13+
await client.ft.create('idx:bicycle', {
14+
'$.condition': {
15+
type: SchemaFieldTypes.TAG,
16+
AS: 'condition'
17+
},
18+
'$.price': {
19+
type: SchemaFieldTypes.NUMERIC,
20+
AS: 'price'
21+
}
22+
}, {
23+
ON: 'JSON',
24+
PREFIX: 'bicycle:'
25+
})
26+
27+
// load data
28+
const bicycles = JSON.parse(fs.readFileSync('data/query_em.json', 'utf8'));
29+
30+
await Promise.all(
31+
bicycles.map((bicycle, bid) => {
32+
return client.json.set(`bicycle:${bid}`, '$', bicycle);
33+
})
34+
);
35+
// HIDE_END
36+
37+
// STEP_START agg1
38+
const res1 = await client.ft.aggregate('idx:bicycle', '@condition:{new}', {
39+
LOAD: ['__key', 'price'],
40+
APPLY: {
41+
expression: '@price - (@price * 0.1)',
42+
AS: 'discounted'
43+
}
44+
});
45+
46+
console.log(res1.results.length); // >>> 5
47+
console.log(res1.results); // >>>
48+
//[
49+
// [Object: null prototype] { __key: 'bicycle:0', price: '270' },
50+
// [Object: null prototype] { __key: 'bicycle:5', price: '810' },
51+
// [Object: null prototype] { __key: 'bicycle:6', price: '2300' },
52+
// [Object: null prototype] { __key: 'bicycle:7', price: '430' },
53+
// [Object: null prototype] { __key: 'bicycle:8', price: '1200' }
54+
//]
55+
// REMOVE_START
56+
assert.strictEqual(res1.results.length, 5);
57+
// REMOVE_END
58+
// STEP_END
59+
60+
// STEP_START agg2
61+
const res2 = await client.ft.aggregate('idx:bicycle', '*', {
62+
LOAD: ['@price'],
63+
STEPS: [{
64+
type: AggregateSteps.APPLY,
65+
expression: '@price<1000',
66+
AS: 'price_category'
67+
},{
68+
type: AggregateSteps.GROUPBY,
69+
properties: '@condition',
70+
REDUCE:[{
71+
type: AggregateGroupByReducers.SUM,
72+
property: '@price_category',
73+
AS: 'num_affordable'
74+
}]
75+
}]
76+
});
77+
console.log(res2.results.length); // >>> 3
78+
console.log(res2.results); // >>>
79+
//[[Object: null prototype] { condition: 'refurbished', num_affordable: '1' },
80+
// [Object: null prototype] { condition: 'used', num_affordable: '1' },
81+
// [Object: null prototype] { condition: 'new', num_affordable: '3' }
82+
//]
83+
// REMOVE_START
84+
assert.strictEqual(res2.results.length, 3);
85+
// REMOVE_END
86+
// STEP_END
87+
88+
// STEP_START agg3
89+
const res3 = await client.ft.aggregate('idx:bicycle', '*', {
90+
STEPS: [{
91+
type: AggregateSteps.APPLY,
92+
expression: "'bicycle'",
93+
AS: 'type'
94+
}, {
95+
type: AggregateSteps.GROUPBY,
96+
properties: '@type',
97+
REDUCE: [{
98+
type: AggregateGroupByReducers.COUNT,
99+
property: null,
100+
AS: 'num_total'
101+
}]
102+
}]
103+
});
104+
console.log(res3.results.length); // >>> 1
105+
console.log(res3.results); // >>>
106+
//[ [Object: null prototype] { type: 'bicycle', num_total: '10' } ]
107+
// REMOVE_START
108+
assert.strictEqual(res3.results.length, 1);
109+
// REMOVE_END
110+
// STEP_END
111+
112+
// STEP_START agg4
113+
const res4 = await client.ft.aggregate('idx:bicycle', '*', {
114+
LOAD: ['__key'],
115+
STEPS: [{
116+
type: AggregateSteps.GROUPBY,
117+
properties: '@condition',
118+
REDUCE: [{
119+
type: AggregateGroupByReducers.TOLIST,
120+
property: '__key',
121+
AS: 'bicycles'
122+
}]
123+
}]
124+
});
125+
console.log(res4.results.length); // >>> 3
126+
console.log(res4.results); // >>>
127+
//[[Object: null prototype] {condition: 'refurbished', bicycles: [ 'bicycle:9' ]},
128+
// [Object: null prototype] {condition: 'used', bicycles: [ 'bicycle:1', 'bicycle:2', 'bicycle:3', 'bicycle:4' ]},
129+
// [Object: null prototype] {condition: 'new', bicycles: [ 'bicycle:5', 'bicycle:6', 'bicycle:7', 'bicycle:0', 'bicycle:8' ]}]
130+
// REMOVE_START
131+
assert.strictEqual(res4.results.length, 3);
132+
// REMOVE_END
133+
// STEP_END
134+
135+
// REMOVE_START
136+
// destroy index and data
137+
await client.ft.dropIndex('idx:bicycle', { DD: true });
138+
await client.disconnect();
139+
// REMOVE_END

‎doctests/query-combined.js

+192
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,192 @@
1+
// EXAMPLE: query_combined
2+
// HIDE_START
3+
import assert from 'node:assert';
4+
import fs from 'node:fs';
5+
import { createClient } from 'redis';
6+
import { SchemaFieldTypes, VectorAlgorithms } from '@redis/search';
7+
import { pipeline } from '@xenova/transformers';
8+
9+
function float32Buffer(arr) {
10+
const floatArray = new Float32Array(arr);
11+
const float32Buffer = Buffer.from(floatArray.buffer);
12+
return float32Buffer;
13+
}
14+
15+
async function embedText(sentence) {
16+
let modelName = 'Xenova/all-MiniLM-L6-v2';
17+
let pipe = await pipeline('feature-extraction', modelName);
18+
19+
let vectorOutput = await pipe(sentence, {
20+
pooling: 'mean',
21+
normalize: true,
22+
});
23+
24+
if (vectorOutput == null) {
25+
throw new Error('vectorOutput is undefined');
26+
}
27+
28+
const embedding = Object.values(vectorOutput.data);
29+
30+
return embedding;
31+
}
32+
33+
let query = 'Bike for small kids';
34+
let vector_query = float32Buffer(await embedText('That is a very happy person'));
35+
36+
const client = createClient();
37+
await client.connect().catch(console.error);
38+
39+
// create index
40+
await client.ft.create('idx:bicycle', {
41+
'$.description': {
42+
type: SchemaFieldTypes.TEXT,
43+
AS: 'description'
44+
},
45+
'$.condition': {
46+
type: SchemaFieldTypes.TAG,
47+
AS: 'condition'
48+
},
49+
'$.price': {
50+
type: SchemaFieldTypes.NUMERIC,
51+
AS: 'price'
52+
},
53+
'$.description_embeddings': {
54+
type: SchemaFieldTypes.VECTOR,
55+
TYPE: 'FLOAT32',
56+
ALGORITHM: VectorAlgorithms.FLAT,
57+
DIM: 384,
58+
DISTANCE_METRIC: 'COSINE',
59+
AS: 'vector',
60+
}
61+
}, {
62+
ON: 'JSON',
63+
PREFIX: 'bicycle:'
64+
});
65+
66+
// load data
67+
const bicycles = JSON.parse(fs.readFileSync('data/query_vector.json', 'utf8'));
68+
69+
await Promise.all(
70+
bicycles.map((bicycle, bid) => {
71+
return client.json.set(`bicycle:${bid}`, '$', bicycle);
72+
})
73+
);
74+
// HIDE_END
75+
76+
// STEP_START combined1
77+
const res1 = await client.ft.search('idx:bicycle', '@price:[500 1000] @condition:{new}');
78+
console.log(res1.total); // >>> 1
79+
console.log(res1); // >>>
80+
//{
81+
// total: 1,
82+
// documents: [ { id: 'bicycle:5', value: [Object: null prototype] } ]
83+
//}
84+
// REMOVE_START
85+
assert.strictEqual(res1.total, 1);
86+
// REMOVE_END
87+
// STEP_END
88+
89+
// STEP_START combined2
90+
const res2 = await client.ft.search('idx:bicycle', 'kids @price:[500 1000] @condition:{used}');
91+
console.log(res2.total); // >>> 1
92+
console.log(res2); // >>>
93+
// {
94+
// total: 1,
95+
// documents: [ { id: 'bicycle:2', value: [Object: null prototype] } ]
96+
// }
97+
// REMOVE_START
98+
assert.strictEqual(res2.total, 1);
99+
// REMOVE_END
100+
// STEP_END
101+
102+
// STEP_START combined3
103+
const res3 = await client.ft.search('idx:bicycle', '(kids | small) @condition:{used}');
104+
console.log(res3.total); // >>> 2
105+
console.log(res3); // >>>
106+
//{
107+
// total: 2,
108+
// documents: [
109+
// { id: 'bicycle:2', value: [Object: null prototype] },
110+
// { id: 'bicycle:1', value: [Object: null prototype] }
111+
// ]
112+
//}
113+
// REMOVE_START
114+
assert.strictEqual(res3.total, 2);
115+
// REMOVE_END
116+
// STEP_END
117+
118+
// STEP_START combined4
119+
const res4 = await client.ft.search('idx:bicycle', '@description:(kids | small) @condition:{used}');
120+
console.log(res4.total); // >>> 2
121+
console.log(res4); // >>>
122+
//{
123+
// total: 2,
124+
// documents: [
125+
// { id: 'bicycle:2', value: [Object: null prototype] },
126+
// { id: 'bicycle:1', value: [Object: null prototype] }
127+
// ]
128+
//}
129+
// REMOVE_START
130+
assert.strictEqual(res4.total, 2);
131+
// REMOVE_END
132+
// STEP_END
133+
134+
// STEP_START combined5
135+
const res5 = await client.ft.search('idx:bicycle', '@description:(kids | small) @condition:{new | used}');
136+
console.log(res5.total); // >>> 3
137+
console.log(res5); // >>>
138+
//{
139+
// total: 3,
140+
// documents: [
141+
// { id: 'bicycle:1', value: [Object: null prototype] },
142+
// { id: 'bicycle:0', value: [Object: null prototype] },
143+
// { id: 'bicycle:2', value: [Object: null prototype] }
144+
// ]
145+
//}
146+
// REMOVE_START
147+
assert.strictEqual(res5.total, 3);
148+
// REMOVE_END
149+
// STEP_END
150+
151+
// STEP_START combined6
152+
const res6 = await client.ft.search('idx:bicycle', '@price:[500 1000] -@condition:{new}');
153+
console.log(res6.total); // >>> 2
154+
console.log(res6); // >>>
155+
//{
156+
// total: 2,
157+
// documents: [
158+
// { id: 'bicycle:2', value: [Object: null prototype] },
159+
// { id: 'bicycle:9', value: [Object: null prototype] }
160+
// ]
161+
//}
162+
// REMOVE_START
163+
assert.strictEqual(res6.total, 2);
164+
// REMOVE_END
165+
// STEP_END
166+
167+
// STEP_START combined7
168+
const res7 = await client.ft.search('idx:bicycle',
169+
'(@price:[500 1000] -@condition:{new})=>[KNN 3 @vector $query_vector]', {
170+
PARAMS: { query_vector: vector_query },
171+
DIALECT: 2
172+
}
173+
);
174+
console.log(res7.total); // >>> 2
175+
console.log(res7); // >>>
176+
//{
177+
// total: 2,
178+
// documents: [
179+
// { id: 'bicycle:2', value: [Object: null prototype] },
180+
// { id: 'bicycle:9', value: [Object: null prototype] }
181+
// ]
182+
//}
183+
// REMOVE_START
184+
assert.strictEqual(res7.total, 2);
185+
// REMOVE_END
186+
// STEP_END
187+
188+
// REMOVE_START
189+
// destroy index and data
190+
await client.ft.dropIndex('idx:bicycle', { DD: true });
191+
await client.disconnect();
192+
// REMOVE_END

0 commit comments

Comments
 (0)