Adding paggination and logs for supporting doc migrations (#4494)

Author: sadmananik

packages/migration/migrations/hearth/20221017133354-restore-supporting-document.js

@@ -9,40 +9,75 @@
  * Copyright (C) The OpenCRVS Authors. OpenCRVS and the OpenCRVS
  * graphic logo are (registered/a) trademark(s) of Plan International.
  */
-import { getBirthEncounterCompositionCursor } from './../../utils/hearth-helper.js'
+import {
+  getBirthEncounterCompositionCursor,
+  getBirthEncounterCompositionCount
+  // eslint-disable-next-line import/no-relative-parent-imports
+} from './../../utils/hearth-helper.js'
 
 export const up = async (db, client) => {
   const session = client.startSession()
+  const limit = 50
+  let skip = 0
+  let processedDocCount = 0
   try {
     await session.withTransaction(async () => {
-      const compositionCursor = await getBirthEncounterCompositionCursor(db)
-      while (await compositionCursor.hasNext()) {
-        const composition = await compositionCursor.next()
-        const compositionHistory = await db
-          .collection('Composition_history')
-          .find({
-            id: composition.id
-          })
-          .toArray()
-        compositionHistory.push(composition)
-        const correctionIndex = compositionHistory.findIndex((composition) => {
-          return composition.section.find(
-            (section) =>
-              section.code.coding[0].code === 'birth-correction-encounters'
-          )
-        })
-        const immediatePrevComp = compositionHistory[correctionIndex - 1]
-        const hasDocumentSection = immediatePrevComp?.section.find(
-          (section) => section.code.coding[0].code === 'supporting-documents'
+      const totalCompositionCount = await getBirthEncounterCompositionCount(db)
+      while (totalCompositionCount > processedDocCount) {
+        const compositionCursor = await getBirthEncounterCompositionCursor(
+          db,
+          limit,
+          skip
+        )
+        const count = await compositionCursor.count()
+        // eslint-disable-next-line no-console
+        console.log(
+          `Migration - SupportingDocuments :: Processing ${
+            processedDocCount + 1
+          } - ${
+            processedDocCount + count
+          } of ${totalCompositionCount} compositions...`
         )
-        if (hasDocumentSection) {
-          await db
-            .collection('Composition')
-            .updateOne(
-              { id: compositionHistory[correctionIndex].id },
-              { $push: { section: hasDocumentSection } }
-            )
+        while (await compositionCursor.hasNext()) {
+          const composition = await compositionCursor.next()
+          const compositionHistory = await db
+            .collection('Composition_history')
+            .find({
+              id: composition.id
+            })
+            .toArray()
+          compositionHistory.push(composition)
+          const correctionIndex = compositionHistory.findIndex(
+            (composition) => {
+              return composition.section.find(
+                (section) =>
+                  section.code.coding[0].code === 'birth-correction-encounters'
+              )
+            }
+          )
+          const immediatePrevComp = compositionHistory[correctionIndex - 1]
+          const hasDocumentSection = immediatePrevComp?.section.find(
+            (section) => section.code.coding[0].code === 'supporting-documents'
+          )
+          if (hasDocumentSection) {
+            await db
+              .collection('Composition')
+              .updateOne(
+                { id: compositionHistory[correctionIndex].id },
+                { $push: { section: hasDocumentSection } }
+              )
+          }
         }
+        skip += limit
+        processedDocCount += count
+        const percentage = (
+          (processedDocCount / totalCompositionCount) *
+          100
+        ).toFixed(2)
+        // eslint-disable-next-line no-console
+        console.log(
+          `Migration - SupportingDocuments :: Processing done ${percentage}%`
+        )
       }
     })
   } finally {

packages/migration/utils/hearth-helper.js

@@ -20,10 +20,26 @@ export const COLLECTION_NAMES = {
   TASK: 'Task'
 }
 
-export async function getBirthEncounterCompositionCursor(db) {
-  return db.collection(COLLECTION_NAMES.COMPOSITION).find({
-    'section.code.coding': { $elemMatch: { code: 'birth-encounter' } }
-  })
+export async function getBirthEncounterCompositionCursor(
+  db,
+  limit = 50,
+  skip = 0
+) {
+  return db.collection(COLLECTION_NAMES.COMPOSITION).find(
+    {
+      'section.code.coding': { $elemMatch: { code: 'birth-encounter' } }
+    },
+    { limit, skip }
+  )
+}
+
+export async function getBirthEncounterCompositionCount(db) {
+  return db
+    .collection(COLLECTION_NAMES.COMPOSITION)
+    .find({
+      'section.code.coding': { $elemMatch: { code: 'birth-encounter' } }
+    })
+    .count()
 }
 
 export async function getCompositionCursor(db, limit = 50, skip = 0) {