Skip to content

Commit acf2e47

Browse files
delthasdelthas
committed
Add functional test for oplog pipeline location stripping
Issue: BB-491
1 parent fdb4e94 commit acf2e47

File tree

3 files changed

+219
-0
lines changed

3 files changed

+219
-0
lines changed

‎.github/workflows/tests.yaml‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -288,6 +288,74 @@ jobs:
288288
cache: yarn
289289
- name: Install node dependencies
290290
run: yarn install --ignore-engines --frozen-lockfile --network-concurrency 1
291+
- name: Lint markdown
292+
run: yarn run --silent lint_md
293+
- name: Lint Javascript
294+
run: yarn run --silent lint
295+
296+
- name: Run unit tests
297+
run: yarn run cover
298+
env:
299+
BACKBEAT_CONFIG_FILE: tests/config.json
300+
TEST_SUITE: test
301+
302+
- uses: codecov/codecov-action@v5
303+
with:
304+
token: ${{ secrets.CODECOV_TOKEN }}
305+
directory: ./coverage/test
306+
flags: unit
307+
308+
- name: Run backbeat routes test
309+
uses: ./.github/actions/ft-test
310+
with:
311+
testsuite: api:routes
312+
token: ${{ secrets.CODECOV_TOKEN }}
313+
runner: run_server_tests.bash
314+
env:
315+
MANAGEMENT_BACKEND: operator
316+
317+
- name: Run backbeat retry tests with account authentication
318+
uses: ./.github/actions/ft-test
319+
with:
320+
testsuite: api:retry
321+
token: ${{ secrets.CODECOV_TOKEN }}
322+
runner: run_server_tests.bash
323+
324+
- name: run feature replication tests
325+
uses: ./.github/actions/ft-test
326+
with:
327+
testsuite: replication
328+
token: ${{ secrets.CODECOV_TOKEN }}
329+
330+
- name: run feature lifecycle tests
331+
uses: ./.github/actions/ft-test
332+
with:
333+
testsuite: lifecycle
334+
token: ${{ secrets.CODECOV_TOKEN }}
335+
336+
- name: run feature ingestion tests
337+
uses: ./.github/actions/ft-test
338+
with:
339+
testsuite: ingestion
340+
token: ${{ secrets.CODECOV_TOKEN }}
341+
342+
- name: run oplogPopulator functional tests
343+
uses: ./.github/actions/ft-test
344+
with:
345+
testsuite: oplogPopulator
346+
token: ${{ secrets.CODECOV_TOKEN }}
347+
348+
- name: run misc functional tests
349+
uses: ./.github/actions/ft-test
350+
with:
351+
testsuite: lib
352+
token: ${{ secrets.CODECOV_TOKEN }}
353+
354+
- name: run backbeat notification feature tests
355+
uses: ./.github/actions/ft-test
356+
with:
357+
testsuite: notification
358+
token: ${{ secrets.CODECOV_TOKEN }}
291359

292360
- name: run ballooning tests for lifecycle conductor
293361
run: yarn mocha tests/performance/lifecycle/conductor-check-memory-balloon.js --exit

‎package.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
"ft_test:lib": "mocha --recursive $(find tests/functional/lib -name '*.js') --timeout 30000 --exit",
2626
"ft_test:lifecycle": "mocha --recursive $(find tests/functional/lifecycle -name '*.js') --timeout 30000 --exit",
2727
"ft_test:ingestion": "mocha --recursive $(find tests/functional/ingestion -name '*.js') --timeout 30000 --exit",
28+
"ft_test:oplogPopulator": "mocha --recursive $(find tests/functional/oplogPopulator -name '*.js') --timeout 30000 --exit",
2829
"ft_test:api:routes": "mocha tests/functional/api/routes.js --timeout 30000 --exit",
2930
"ft_test:api:retry": "mocha tests/functional/api/retry.js --timeout 30000 --exit",
3031
"ft_test:queuepopulator": "mocha --recursive $(find tests/functional/queuePopulator -name '*.js') --timeout 30000 --exit",

‎tests/functional/oplogPopulator/PipelineFactory.js‎

Lines changed: 150 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,150 @@
1+
const assert = require('assert');
2+
const MongoClient = require('mongodb').MongoClient;
3+
4+
const testConfig = require('../../config.json');
5+
const MultipleBucketsPipelineFactory =
6+
require('../../../extensions/oplogPopulator/pipeline/MultipleBucketsPipelineFactory');
7+
8+
const mongoUrl =
9+
`mongodb://${testConfig.queuePopulator.mongo.replicaSetHosts}` +
10+
'/db?replicaSet=rs0';
11+
const client = new MongoClient(mongoUrl, {});
12+
const db = client.db('metadata', { ignoreUndefined: true });
13+
14+
const THRESHOLD = 100;
15+
16+
describe('PipelineFactory', function () {
17+
this.timeout(30000);
18+
19+
const collectionName = 'test-pipeline-stripping';
20+
let collection;
21+
let setStage;
22+
23+
before(async () => {
24+
await client.connect();
25+
collection = db.collection(collectionName);
26+
27+
const factory = new MultipleBucketsPipelineFactory(THRESHOLD);
28+
const pipeline = JSON.parse(factory.getPipeline(['test-bucket']));
29+
setStage = pipeline[1];
30+
});
31+
32+
afterEach(async () => {
33+
await collection.deleteMany({});
34+
});
35+
36+
after(async () => {
37+
if (collection) {
38+
await collection.drop().catch(() => {});
39+
}
40+
await client.close();
41+
});
42+
43+
describe('fullDocument path (insert/replace events)', () => {
44+
it('should preserve location when content-length is missing', async () => {
45+
const doc = { fullDocument: { value: { location: [{ key: 'a' }] } } };
46+
await collection.insertOne(doc);
47+
const results = await collection.aggregate([setStage]).toArray();
48+
assert.strictEqual(results.length, 1);
49+
assert.deepStrictEqual(results[0].fullDocument.value.location, [{ key: 'a' }]);
50+
});
51+
52+
it('should preserve location when content-length is below threshold', async () => {
53+
const doc = { fullDocument: { value: { 'content-length': 50, 'location': [{ key: 'a' }] } } };
54+
await collection.insertOne(doc);
55+
const results = await collection.aggregate([setStage]).toArray();
56+
assert.strictEqual(results.length, 1);
57+
assert.deepStrictEqual(results[0].fullDocument.value.location, [{ key: 'a' }]);
58+
});
59+
60+
it('should preserve location when content-length is zero', async () => {
61+
const doc = { fullDocument: { value: { 'content-length': 0, 'location': [{ key: 'a' }] } } };
62+
await collection.insertOne(doc);
63+
const results = await collection.aggregate([setStage]).toArray();
64+
assert.strictEqual(results.length, 1);
65+
assert.deepStrictEqual(results[0].fullDocument.value.location, [{ key: 'a' }]);
66+
});
67+
68+
it('should strip location when content-length equals threshold', async () => {
69+
const doc = { fullDocument: { value: { 'content-length': THRESHOLD, 'location': [{ key: 'a' }] } } };
70+
await collection.insertOne(doc);
71+
const results = await collection.aggregate([setStage]).toArray();
72+
assert.strictEqual(results.length, 1);
73+
assert.strictEqual(results[0].fullDocument.value.location, undefined);
74+
});
75+
76+
it('should strip location when content-length is above threshold', async () => {
77+
const doc = { fullDocument: { value: { 'content-length': 200, 'location': [{ key: 'a' }] } } };
78+
await collection.insertOne(doc);
79+
const results = await collection.aggregate([setStage]).toArray();
80+
assert.strictEqual(results.length, 1);
81+
assert.strictEqual(results[0].fullDocument.value.location, undefined);
82+
});
83+
});
84+
85+
describe('updateDescription path (update events)', () => {
86+
it('should preserve location when content-length is missing', async () => {
87+
const doc = { updateDescription: { updatedFields: { value: { location: [{ key: 'a' }] } } } };
88+
await collection.insertOne(doc);
89+
const results = await collection.aggregate([setStage]).toArray();
90+
assert.strictEqual(results.length, 1);
91+
assert.deepStrictEqual(
92+
results[0].updateDescription.updatedFields.value.location,
93+
[{ key: 'a' }]
94+
);
95+
});
96+
97+
it('should preserve location when content-length is below threshold', async () => {
98+
const doc = {
99+
updateDescription: { updatedFields: { value: { 'content-length': 50, 'location': [{ key: 'a' }] } } },
100+
};
101+
await collection.insertOne(doc);
102+
const results = await collection.aggregate([setStage]).toArray();
103+
assert.strictEqual(results.length, 1);
104+
assert.deepStrictEqual(
105+
results[0].updateDescription.updatedFields.value.location,
106+
[{ key: 'a' }]
107+
);
108+
});
109+
110+
it('should preserve location when content-length is zero', async () => {
111+
const doc = {
112+
updateDescription: { updatedFields: { value: { 'content-length': 0, 'location': [{ key: 'a' }] } } },
113+
};
114+
await collection.insertOne(doc);
115+
const results = await collection.aggregate([setStage]).toArray();
116+
assert.strictEqual(results.length, 1);
117+
assert.deepStrictEqual(
118+
results[0].updateDescription.updatedFields.value.location,
119+
[{ key: 'a' }]
120+
);
121+
});
122+
123+
it('should strip location when content-length equals threshold', async () => {
124+
const value = { 'content-length': THRESHOLD, 'location': [{ key: 'a' }] };
125+
const doc = {
126+
updateDescription: { updatedFields: { value } },
127+
};
128+
await collection.insertOne(doc);
129+
const results = await collection.aggregate([setStage]).toArray();
130+
assert.strictEqual(results.length, 1);
131+
assert.strictEqual(
132+
results[0].updateDescription.updatedFields.value.location,
133+
undefined
134+
);
135+
});
136+
137+
it('should strip location when content-length is above threshold', async () => {
138+
const doc = {
139+
updateDescription: { updatedFields: { value: { 'content-length': 200, 'location': [{ key: 'a' }] } } },
140+
};
141+
await collection.insertOne(doc);
142+
const results = await collection.aggregate([setStage]).toArray();
143+
assert.strictEqual(results.length, 1);
144+
assert.strictEqual(
145+
results[0].updateDescription.updatedFields.value.location,
146+
undefined
147+
);
148+
});
149+
});
150+
});

0 commit comments

Comments
 (0)