Merge pull request #25 from opencrvs/adding-tests

Add tests and github action to run them

Author: tumbledwyer

.github/workflows/test-migration.yml

@@ -0,0 +1,66 @@
+name: Test Data Migration
+
+on:
+  pull_request:
+    paths:
+      - 'v1-to-v2-data-migration/**'
+      - '.github/workflows/test-migration.yml'
+
+jobs:
+  test:
+    name: Run Migration Tests
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Setup Deno
+        uses: denoland/setup-deno@v1
+        with:
+          deno-version: v2.x
+
+      - name: Cache Deno dependencies
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.deno
+            ~/.cache/deno
+          key: ${{ runner.os }}-deno-${{ hashFiles('v1-to-v2-data-migration/tests/deno.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-deno-
+
+      - name: Detect country and run tests
+        working-directory: v1-to-v2-data-migration/tests
+        run: |
+          echo "🧪 Running migration tests with automatic country detection..."
+          deno task test
+
+      - name: Test Summary
+        if: always()
+        working-directory: v1-to-v2-data-migration/tests
+        run: |
+          # Extract country code from addressResolver.ts
+          COUNTRY=$(deno eval "import { COUNTRY_CODE } from '../countryData/addressResolver.ts'; console.log(COUNTRY_CODE)")
+
+          echo "## 🧪 Migration Test Results" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+          echo "**Branch:** \`${{ github.ref_name }}\`" >> $GITHUB_STEP_SUMMARY
+          echo "**Country:** \`$COUNTRY\`" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+          echo "### 📦 Test Suites Executed" >> $GITHUB_STEP_SUMMARY
+          echo "- ✅ Common unit tests (\`unit/\`)" >> $GITHUB_STEP_SUMMARY
+          echo "- ✅ Country-specific tests (\`$COUNTRY/\`)" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+          echo "### 🎯 Tests Included" >> $GITHUB_STEP_SUMMARY
+          echo "**Common Tests:**" >> $GITHUB_STEP_SUMMARY
+          echo "- Birth resolver tests" >> $GITHUB_STEP_SUMMARY
+          echo "- Death resolver tests" >> $GITHUB_STEP_SUMMARY
+          echo "- Action mapping tests" >> $GITHUB_STEP_SUMMARY
+          echo "- Corrections tests" >> $GITHUB_STEP_SUMMARY
+          echo "- Transform function tests" >> $GITHUB_STEP_SUMMARY
+          echo "- Post-process tests" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+          echo "**Country-Specific Tests:**" >> $GITHUB_STEP_SUMMARY
+          echo "- Address resolution tests" >> $GITHUB_STEP_SUMMARY
+          echo "- Address corrections tests" >> $GITHUB_STEP_SUMMARY

v1-to-v2-data-migration/countryData/addressResolver.ts

@@ -38,7 +38,9 @@ export function resolveAddress(
   if (!address) {
     return null
   }
-  const lines = address.line.filter(Boolean).filter((line) => !['URBAN', 'RURAL'].includes(line))
+  const lines = address.line
+    .filter(Boolean)
+    .filter((line) => !['URBAN', 'RURAL'].includes(line))
   const international = address.country !== COUNTRY_CODE
   if (international) {
     return {

v1-to-v2-data-migration/countryData/countryResolvers.ts

@@ -1 +1,6 @@
 export const countryResolver = {}
+
+// The V1 response will populate both the informant and special informant fields
+// so we need to check if the informant is a special informant to avoid duplication
+export const birthSpecialInformants = ['MOTHER', 'FATHER']
+export const deathSpecialInformants = ['SPOUSE']

v1-to-v2-data-migration/helpers/dateUtils.ts

@@ -0,0 +1,31 @@
+/**
+ * @example
+ * normalizeDateString('2025-3-3') // returns '2025-03-03'
+ */
+export function normalizeDateString(
+  dateStr: string | undefined
+): string | undefined {
+  if (!dateStr) {
+    return dateStr
+  }
+
+  const datePattern = /^(\d{4})-(\d{1,2})-(\d{1,2})$/
+  const match = dateStr.match(datePattern)
+
+  if (!match) {
+    return dateStr
+  }
+
+  const [, year, month, day] = match
+
+  const paddedMonth = month.padStart(2, '0')
+  const paddedDay = day.padStart(2, '0')
+
+  return `${year}-${paddedMonth}-${paddedDay}`
+}
+
+export function isDateField(fieldId: string): boolean {
+  const dateFieldPatterns = ['BirthDate', 'birthDate', 'Date', 'deathDate']
+
+  return dateFieldPatterns.some((pattern) => fieldId.includes(pattern))
+}

v1-to-v2-data-migration/helpers/defaultMappings.ts

@@ -89,7 +89,7 @@ export const DEFAULT_FIELD_MAPPINGS = {
   'death.spouse.reasonNotApplying': 'spouse.reason',
   'death.spouse.spouseBirthDate': 'spouse.dob',
   'death.spouse.exactDateOfBirthUnknown': 'spouse.dobUnknown',
-  'death.spouse.ageOfIndividualInYears': 'spouse.age',
+  // age mapping handled by AGE_MAPPINGS to transform to object with asOfDateRef
   'death.spouse.nationality': 'spouse.nationality',
   'death.spouse.spouseNationalId': 'spouse.nid',
   'death.spouse.spousePassport': 'spouse.passport',
@@ -122,7 +122,7 @@ export const CUSTOM_FIELD_MAPPINGS = {
   'death.spouse.spouse-view-group.verified': 'spouse.verified',
 }
 
-export const VERIFIED_MAPPINGS : Record<
+export const VERIFIED_MAPPINGS: Record<
   string,
   (data: string) => Record<string, any>
 > = {
@@ -187,4 +187,3 @@ export const AGE_MAPPINGS: Record<
     },
   }),
 }
-

v1-to-v2-data-migration/helpers/resolverUtils.ts

@@ -1,4 +1,14 @@
-import { Identifier, Document, ProcessedDocumentWithOptionType, PersonWithIdentifiers, ProcessedDocument } from './types.ts'
+import {
+  birthSpecialInformants,
+  deathSpecialInformants,
+} from '../countryData/countryResolvers.ts'
+import {
+  Identifier,
+  Document,
+  ProcessedDocumentWithOptionType,
+  PersonWithIdentifiers,
+  ProcessedDocument,
+} from './types.ts'
 
 export const getIdentifier = (
   data: { identifier?: Identifier[] } | undefined,
@@ -16,7 +26,7 @@ export const getDocument = (
       return {
         path: doc.uri,
         originalFilename: doc.uri.replace('/ocrvs/', ''),
-        type: doc.contentType
+        type: doc.contentType,
       }
     })[0]
   if (!document) {
@@ -50,20 +60,21 @@ export function getCustomField(data: any, id: string): any {
   )?.value
 }
 
-
 /**
  * Special informants have their own special sections like `mother.`, `father.` or `spouse.`.
  */
-export const isSpecialInformant = (informant: PersonWithIdentifiers | undefined, eventType: 'birth' | 'death') => {
-  if (!informant) return false
+export const isSpecialInformant = (
+  informant: PersonWithIdentifiers | undefined,
+  eventType: 'birth' | 'death'
+) => {
+  if (!informant?.relationship) return false
 
-  if(eventType === 'birth') {
-    return informant.relationship === 'MOTHER'
-    || informant.relationship === 'FATHER'
+  if (eventType === 'birth') {
+    return birthSpecialInformants.includes(informant.relationship)
   }
 
-  if(eventType === 'death') {
-    return informant.relationship === 'SPOUSE'
+  if (eventType === 'death') {
+    return deathSpecialInformants.includes(informant.relationship)
   }
   return false
-}
+}

v1-to-v2-data-migration/helpers/transform.ts

@@ -6,6 +6,7 @@ import {
   AGE_MAPPINGS,
   VERIFIED_MAPPINGS,
 } from './defaultMappings.ts'
+import { normalizeDateString, isDateField } from './dateUtils.ts'
 import { COUNTRY_FIELD_MAPPINGS } from '../countryData/countryMappings.ts'
 import { NAME_MAPPINGS } from '../countryData/nameMappings.ts'
 import { ADDRESS_MAPPINGS } from '../countryData/addressMappings.ts'
@@ -124,7 +125,11 @@ export function transformCorrection(
 
   const v1OutputDeclaration =
     historyItem.output?.reduce((acc: Record<string, any>, curr: any) => {
-      acc[`${event}.${curr.valueCode}.${curr.valueId}`] = curr.value
+      // Normalize date strings in output to ensure proper zero-padding
+      const value = isDateField(curr.valueId)
+        ? normalizeDateString(curr.value)
+        : curr.value
+      acc[`${event}.${curr.valueCode}.${curr.valueId}`] = value
       return acc
     }, {}) || {}
 
@@ -469,34 +474,114 @@ export function transform(
   return postProcess(documents)
 }
 
+/**
+ * Deep merge two objects, with priority given to source values
+ */
+function deepMerge(target: any, source: any): any {
+  const result = { ...target }
+
+  for (const key in source) {
+    if (source.hasOwnProperty(key)) {
+      const sourceValue = source[key]
+      const targetValue = target[key]
+
+      // If both are plain objects, merge recursively
+      if (
+        sourceValue &&
+        typeof sourceValue === 'object' &&
+        !Array.isArray(sourceValue) &&
+        targetValue &&
+        typeof targetValue === 'object' &&
+        !Array.isArray(targetValue)
+      ) {
+        result[key] = deepMerge(targetValue, sourceValue)
+      } else {
+        // Otherwise, use source value
+        result[key] = sourceValue
+      }
+    }
+  }
+
+  return result
+}
+
 function postProcess(documents: TransformedDocument): TransformedDocument {
   const correctionResolverKeys = Object.keys(correctionResolver)
 
+  // Step 1: Build a map of current declaration state at each action
+  // We start from the final declaration and work backwards
+  let currentDeclaration: Record<string, any> = {}
+
+  // Find the final declaration by merging all DECLARE/REGISTER/VALIDATE declarations
+  for (const action of documents.actions) {
+    if (
+      (action.type === 'DECLARE' ||
+        action.type === 'REGISTER' ||
+        action.type === 'VALIDATE') &&
+      action.declaration &&
+      Object.keys(action.declaration).length > 0
+    ) {
+      currentDeclaration = deepMerge(currentDeclaration, action.declaration)
+    }
+  }
+
+  // Step 2: Process corrections in reverse order to reverse-engineer the original state
+  const corrections: Array<{ index: number; action: Action }> = []
+
   for (let i = 0; i < documents.actions.length; i++) {
-    const action = documents.actions[i]
+    if (documents.actions[i].type === 'REQUEST_CORRECTION') {
+      corrections.push({ index: i, action: documents.actions[i] })
+    }
+  }
 
-    if (action.type === 'REQUEST_CORRECTION' && action.annotation) {
-      // Filter out correctionResolver fields from the annotation
-      const filteredAnnotation = Object.fromEntries(
-        Object.entries(action.annotation).filter(
-          ([key]) => !correctionResolverKeys.includes(key)
-        )
+  // Process corrections from newest to oldest to reverse engineer
+  for (let i = corrections.length - 1; i >= 0; i--) {
+    const { action } = corrections[i]
+
+    if (!action.annotation || !action.declaration) {
+      continue
+    }
+
+    // Filter out correctionResolver metadata fields from annotation
+    const filteredAnnotation = Object.fromEntries(
+      Object.entries(action.annotation).filter(
+        ([key]) => !correctionResolverKeys.includes(key)
       )
+    )
 
-      // Find the first action before this one with a non-empty declaration
-      for (let j = i - 1; j >= 0; j--) {
-        const previousAction = documents.actions[j]
-
-        if (
-          previousAction.declaration &&
-          typeof previousAction.declaration === 'object' &&
-          Object.keys(previousAction.declaration).length > 0
-        ) {
-          previousAction.declaration = filteredAnnotation
-          break
-        }
+    // First, reverse the correction: for each field in the correction's declaration,
+    // replace the current state with the input value (from filteredAnnotation)
+    for (const key of Object.keys(action.declaration)) {
+      if (filteredAnnotation.hasOwnProperty(key)) {
+        currentDeclaration[key] = filteredAnnotation[key]
       }
     }
+
+    // Now set the annotation to the reversed state (which is the state BEFORE this correction)
+    const newAnnotation = {
+      ...currentDeclaration,
+      ...Object.fromEntries(
+        Object.entries(action.annotation).filter(([key]) =>
+          correctionResolverKeys.includes(key)
+        )
+      ),
+    }
+
+    action.annotation = newAnnotation
+  }
+
+  // Step 3: Update the base DECLARE/REGISTER/VALIDATE actions with the reverse-engineered state
+  // Find the first action with a non-empty declaration
+  for (const action of documents.actions) {
+    if (
+      (action.type === 'DECLARE' ||
+        action.type === 'REGISTER' ||
+        action.type === 'VALIDATE') &&
+      action.declaration &&
+      Object.keys(action.declaration).length > 0
+    ) {
+      action.declaration = { ...currentDeclaration }
+    }
   }
 
   return documents

v1-to-v2-data-migration/helpers/types.ts

@@ -40,6 +40,7 @@ export interface Document {
   uri: string
   contentType: string
   type: string
+  subject?: string
 }
 
 export interface ProcessedDocumentWithOptionType {
@@ -56,6 +57,7 @@ export interface ProcessedDocument {
 
 // Address types
 export interface AddressLine {
+  type?: string
   line: string[]
   city?: string
   district?: string
@@ -111,7 +113,7 @@ export interface Registration {
   contactEmail?: string
   informantsSignature?: string
   attachments?: Document[]
-  duplicates?: Array<{ compositionId: string }>
+  duplicates?: Array<{ compositionId: string; trackingId?: string }>
 }
 
 // History item types
@@ -177,7 +179,10 @@ export interface HistoryItem {
 }
 
 // Resolver types
-export type ResolverFunction<T = any> = (data: T, eventType: 'birth' | 'death') => any
+export type ResolverFunction<T = any> = (
+  data: T,
+  eventType: 'birth' | 'death'
+) => any
 
 export interface ResolverMap {
   [fieldId: string]: ResolverFunction
@@ -204,7 +209,7 @@ export interface EventRegistration {
   weightAtBirth?: number
   deathDate?: string
   deathDescription?: string
-  causeOfDeathEstablished?: "true" | "false"
+  causeOfDeathEstablished?: 'true' | 'false'
   causeOfDeathMethod?: string
   mannerOfDeath?: string
 }