diff --git a/.changeset/hip-planes-see.md b/.changeset/hip-planes-see.md new file mode 100644 index 0000000000..78cd218a20 --- /dev/null +++ b/.changeset/hip-planes-see.md @@ -0,0 +1,6 @@ +--- +'@openfn/language-dhis2': patch +--- + +Default `async:false` in `tracker.import` and add paging examples to +`tracker.export()` diff --git a/packages/dhis2/src/tracker.js b/packages/dhis2/src/tracker.js index c4346145fd..8a7c501ac9 100644 --- a/packages/dhis2/src/tracker.js +++ b/packages/dhis2/src/tracker.js @@ -47,6 +47,7 @@ import * as util from './util.js'; * @param {string} strategy - The effect the import should have. Can either be CREATE, UPDATE, CREATE_AND_UPDATE and DELETE. * @param {object} payload - The data to be imported. * @param {TrackerOptions} [options] - An optional object containing parseAs, and apiVersion, and queries for the request + * @param {boolean} [options.async=false] - Whether to perform the import asynchronously. Defaults to false. * @state {DHIS2State} * @returns {Operation} */ @@ -57,7 +58,7 @@ function _import(strategy, payload, options = {}) { const [resolvedStrategy, resolvedPayload, resolvedOptions] = expandReferences(state, strategy, payload, options); - const { apiVersion, parseAs, ...query } = resolvedOptions; + const { apiVersion, parseAs, async = false, ...query } = resolvedOptions; const response = await util.request(state.configuration, { method: 'POST', @@ -67,14 +68,14 @@ function _import(strategy, payload, options = {}) { ...resolvedOptions, resolvedStrategy, }, - 'tracker' + 'tracker', ), options: { apiVersion, parseAs, query: { ...query, - async: false, + async, }, }, data: resolvedPayload, @@ -95,15 +96,41 @@ export { _import as import }; * @example Export all enrollment resources * tracker.export('enrollments', {orgUnit: 'TSyzvBiovKh'}); * @example Export all events - * tracker.export('events') + * tracker.export('events', { paging: false}) + * @example Export all events with pagination + * tracker.export('events', { totalPages: true, pageSize: 1e4 }); + * fn(state => { + * state.results = state.data.instances; + * const { page, pageSize, pageCount, total } = state.data.pager; + * const remainingPages = pageCount - page; + * + * state.pages = Array.from({ length: remainingPages }, (_, i) => page + i + 1); + * state.pageSize = pageSize; + * return state; + * }); + * + * each( + * $.pages, + * tracker + * .export('events', { pageSize: $.pageSize, page: $.data }) + * .then(state => { + * state.results = state.results.concat(state.data.instances); + * return state; + * }), + * ); * @function * @param {string} path - Path to the resource, relative to the /tracker endpoint - * @param {object} query - An object of query parameters to be encoded into the URL + * @param {object} query - An object of query parameters to be encoded into the URL. Can include pagination parameters, filters, etc. + * @param {number} [query.page=1] - Page number to return + * @param {number} [query.pageSize=50] - Number of results per page + * @param {boolean} [query.totalPages=false] - Whether to return total number of elements and pages + * @param {boolean} [query.paging=true] - Set to false to return all rows without paging + * @param {string} [query.order] - Comma-separated field:sortDirection pairs, e.g. `createdAt:desc` * @param {TrackerOptions} [options] - An optional object containing parseAs, and apiVersion for the request * @state {DHIS2State} * @returns {Operation} */ -function _export(path, query, options = {}) { +function _export(path, query = {}, options = {}) { return async state => { console.log('Preparing tracker export operation...'); @@ -111,7 +138,7 @@ function _export(path, query, options = {}) { state, path, query, - options + options, ); const response = await util.request(state.configuration, { @@ -119,13 +146,12 @@ function _export(path, query, options = {}) { path: util.prefixVersionToPath( state.configuration, resolvedOptions, - `tracker/${resolvedPath}` + `tracker/${resolvedPath}`, ), options: { ...resolvedOptions, query: { ...resolvedQuery, - async: false, }, }, }); diff --git a/packages/dhis2/test/integration.js b/packages/dhis2/test/integration.js index a250682fbb..c249a1d2b7 100644 --- a/packages/dhis2/test/integration.js +++ b/packages/dhis2/test/integration.js @@ -1,6 +1,16 @@ import { expect } from 'chai'; import crypto from 'node:crypto'; -import { execute, create, update, upsert, get } from '../dist/index.js'; +import { + execute, + tracker, + combine, + create, + update, + upsert, + each, + get, + fn, +} from '../src/index.js'; const getRandomProgramPayload = () => { const name = crypto.randomBytes(16).toString('hex'); @@ -471,4 +481,71 @@ describe('Integration tests', () => { ); }); }); + describe('tracker', () => { + it('should export 50 events by default', async () => { + // v2.41+ for older version `skipPaging: true` + const state = { + configuration, + }; + const finalState = await execute(tracker.export('events'))(state); + + expect(finalState.data.instances.length).to.eql(50); + }).timeout(2e4); + + it('should export 1000 events with pageSize 1000', async () => { + const state = { + configuration, + }; + const { data } = await execute( + tracker.export('events', { totalPages: true, pageSize: 1e3 }), + )(state); + + expect(Object.keys(data).sort()).to.eql([ + 'instances', + 'page', + 'pageCount', + 'pageSize', + 'total', + ]); + expect(data.instances.length).to.eql(1000); + }).timeout(2e4); + + it('should export all events with pagination', async () => { + const state = { + configuration, + }; + const { data, results } = await execute( + tracker.export('events', { totalPages: true, pageSize: 1e4 }), + fn(state => { + console.log(Object.keys(state.data)); + state.results = state.data.instances; + const { page, pageSize, pageCount, total } = state.data; + const remainingPages = pageCount - page; + + state.pages = Array.from( + { length: remainingPages }, + (_, i) => page + i + 1, + ); + state.pageSize = pageSize; + return state; + }), + + each( + state => state.pages, + combine( + tracker.export('events', state => ({ + pageSize: state.pageSize, + page: state.data, + })), + fn(state => { + state.results = state.results.concat(state.data.instances); + return state; + }), + ), + ), + )(state); + + expect(results).to.be.greaterThan(3e4); + }).timeout(5e4); + }); }); diff --git a/packages/dhis2/test/tracker.test.js b/packages/dhis2/test/tracker.test.js index e9ada1a1a5..328193e480 100644 --- a/packages/dhis2/test/tracker.test.js +++ b/packages/dhis2/test/tracker.test.js @@ -15,84 +15,178 @@ const configuration = { apiVersion: '42', }; -const getPath = path => { - return `/stable-2-40-7/api/42/${path}`; -}; +const getPath = path => `/stable-2-40-7/api/42/${path}`; -describe('tracker', () => { - const state = { - configuration, - data: { - program: 'program1', - orgUnit: 'org50', +const trackedEntityPayload = { + trackedEntities: [ + { + orgUnit: 'TSyzvBiovKh', trackedEntityType: 'nEenWmSyUEp', - status: 'COMPLETED', - date: '02-02-20', + attributes: [{ attribute: 'w75KJ2mc4zz', value: 'Gigiwe' }], }, - }; - - it('should import a trackedEntity', async () => { - testServer - .intercept({ - path: getPath('tracker'), - method: 'POST', - query: { async: false }, - }) - .reply(200, { - httpStatus: 'OK', - message: 'the response', - }); - - const finalState = await execute( - tracker.import('CREATE', { - trackedEntities: [ - { - orgUnit: 'TSyzvBiovKh', - trackedEntityType: 'nEenWmSyUEp', - attributes: [ - { - attribute: 'w75KJ2mc4zz', - value: 'Gigiwe', - }, - ], - }, - ], - }) - )(state); - - expect(finalState.data).to.eql({ - httpStatus: 'OK', - message: 'the response', + ], +}; + +describe('tracker', () => { + const state = { configuration, data: {} }; + + describe('import', () => { + it('should default to async: false', async () => { + testServer + .intercept({ + path: getPath('tracker'), + method: 'POST', + query: { async: false }, + }) + .reply(200, { httpStatus: 'OK', status: 'OK' }); + + const finalState = await execute( + tracker.import('CREATE', trackedEntityPayload), + )(state); + + expect(finalState.response.query.async).to.eql(false); + expect(finalState.data.httpStatus).to.eql('OK'); + }); + + it('should send async: true when specified in options', async () => { + testServer + .intercept({ + path: getPath('tracker'), + method: 'POST', + query: { async: true }, + }) + .reply(200, { + httpStatus: 'OK', + response: { id: 'abc123', jobType: 'TRACKER_IMPORT_JOB' }, + }); + + const finalState = await execute( + tracker.import('CREATE', trackedEntityPayload, { async: true }), + )(state); + + expect(finalState.response.query.async).to.eql(true); + expect(finalState.data.response.id).to.eql('abc123'); + }); + + it('should forward extra options as query params alongside async', async () => { + testServer + .intercept({ + path: getPath('tracker'), + method: 'POST', + query: { async: false, atomicMode: 'ALL' }, + }) + .reply(200, { httpStatus: 'OK' }); + + const finalState = await execute( + tracker.import('CREATE', trackedEntityPayload, { atomicMode: 'ALL' }), + )(state); + + expect(finalState.response.query.atomicMode).to.eql('ALL'); + expect(finalState.response.query.async).to.eql(false); }); }); - it('should export all enrollements', async () => { - const query = { - orgUnit: 'TSyzvBiovKh', - }; - testServer - .intercept({ - path: getPath('tracker/enrollments'), - method: 'GET', - query: { - ...query, - async: false, - }, - }) - .reply(200, { - httpStatus: 'OK', - message: 'the response', - }); - - const finalState = await execute( - tracker.export('enrollments', { - orgUnit: 'TSyzvBiovKh', - }) - )(state); - - expect(finalState.data).to.eql({ - httpStatus: 'OK', - message: 'the response', + describe('export', () => { + it('should export a resource by path with filter query params', async () => { + testServer + .intercept({ + path: getPath('tracker/enrollments'), + method: 'GET', + query: { orgUnit: 'TSyzvBiovKh' }, + }) + .reply(200, { + instances: [{ enrollment: 'abc123', orgUnit: 'TSyzvBiovKh' }], + pager: { page: 1, pageSize: 50 }, + }); + + const finalState = await execute( + tracker.export('enrollments', { orgUnit: 'TSyzvBiovKh' }), + )(state); + + expect(finalState.data.instances).to.have.lengthOf(1); + expect(finalState.data.instances[0].enrollment).to.eql('abc123'); + }); + + it('should send paging: false to fetch all results without paging', async () => { + const allEvents = Array.from({ length: 300 }, (_, i) => ({ + event: `ev${i}`, + })); + + testServer + .intercept({ + path: getPath('tracker/events'), + method: 'GET', + query: { paging: false }, + }) + .reply(200, { instances: allEvents }); + + const finalState = await execute( + tracker.export('events', { paging: false }), + )(state); + + expect(finalState.response.query.paging).to.eql(false); + expect(finalState.data.instances).to.have.lengthOf(300); + }); + + it('should send page and pageSize for paginated requests', async () => { + testServer + .intercept({ + path: getPath('tracker/events'), + method: 'GET', + query: { page: 2, pageSize: 10 }, + }) + .reply(200, { + instances: Array.from({ length: 10 }, (_, i) => ({ + event: `ev${i}`, + })), + pager: { page: 2, pageSize: 10 }, + }); + + const finalState = await execute( + tracker.export('events', { page: 2, pageSize: 10 }), + )(state); + + expect(finalState.response.query.page).to.eql(2); + expect(finalState.response.query.pageSize).to.eql(10); + expect(finalState.data.instances).to.have.lengthOf(10); + expect(finalState.data.pager.page).to.eql(2); + }); + + it('should send totalPages: true to include total count in pager', async () => { + testServer + .intercept({ + path: getPath('tracker/events'), + method: 'GET', + query: { totalPages: true, pageSize: 10000 }, + }) + .reply(200, { + instances: [], + pager: { page: 1, pageSize: 10000, pageCount: 3, total: 25000 }, + }); + + const finalState = await execute( + tracker.export('events', { totalPages: true, pageSize: 10000 }), + )(state); + + expect(finalState.response.query.totalPages).to.eql(true); + expect(finalState.data.pager).to.include({ pageCount: 3, total: 25000 }); + }); + + it('should send order param for sorted results', async () => { + testServer + .intercept({ + path: getPath('tracker/events'), + method: 'GET', + query: { order: 'createdAt:desc' }, + }) + .reply(200, { instances: [{ event: 'ev1' }] }); + + const finalState = await execute( + tracker.export('events', { order: 'createdAt:desc' }), + )(state); + + expect(finalState.response.query.order).to.eql('createdAt:desc'); + expect(finalState.data.instances).to.have.lengthOf(1); }); }); });