Merge pull request #18434 from CDCgov/deployment/2025-08-21

Deployment of 2025-08-21

Author: JFisk42

.github/actions/build-backend/action.yml

@@ -18,6 +18,10 @@ inputs:
   skip-schema-validation:
     description: "Skip schema validation step"
     default: true
+  integration-test-forks:
+    description: "Number of parallel forks for integration tests (auto-detect if not specified)"
+    required: false
+    default: "auto"
 
 runs:
   using: "composite"
@@ -147,10 +151,62 @@ runs:
         ./gradlew :prime-router:reloadSettings
       shell: bash
 
+    - name: Configure integration test parallelization
+      if: inputs.run-integration-tests == 'true'
+      shell: bash
+      run: |
+        # Auto-detect optimal parallelization based on runner specs
+        CORES=$(nproc)
+        MEMORY_GB=$(free -g | grep '^Mem' | awk '{print $2}')
+        
+        if [ "${{ inputs.integration-test-forks }}" = "auto" ]; then
+          # Conservative approach: use half the cores, with safety limits
+          FORKS=$((CORES / 2))
+          FORKS=$(( FORKS > 3 ? 3 : FORKS ))  # Max 3 forks to reduce external API pressure
+          FORKS=$(( FORKS < 1 ? 1 : FORKS ))  # Min 1 fork
+        else
+          FORKS=${{ inputs.integration-test-forks }}
+        fi
+        
+        # Calculate memory per fork (leave some buffer for system)
+        MEMORY_PER_FORK=$(( (MEMORY_GB * 80 / 100) / FORKS ))  # 80% of total memory
+        MEMORY_PER_FORK=$(( MEMORY_PER_FORK > 3 ? 3 : MEMORY_PER_FORK ))  # Max 3GB per fork
+        MEMORY_PER_FORK=$(( MEMORY_PER_FORK < 1 ? 1 : MEMORY_PER_FORK ))  # Min 1GB per fork
+        
+        echo "INTEGRATION_TEST_FORKS=$FORKS" >> $GITHUB_ENV
+        echo "INTEGRATION_TEST_MEMORY=${MEMORY_PER_FORK}g" >> $GITHUB_ENV
+        
+        echo "🖥️  Runner specifications detected:"
+        echo "   CPU cores available: $CORES"
+        echo "   Total memory: ${MEMORY_GB}GB"
+        echo ""
+        echo "🧪 Integration test configuration:"
+        echo "   Parallel forks: $FORKS"
+        echo "   Memory per fork: ${MEMORY_PER_FORK}GB"
+        echo "   Expected speedup: ~$((100 - (100 / FORKS)))% (theoretical)"
+
     # Integrations tests require the metadata catalog
-    - name: Run Integration Tests
+    - name: Run Integration Tests (Parallel with Resource Optimization)
       if: inputs.run-integration-tests == 'true'
-      run: ./gradlew :prime-router:testIntegration -Pshowtests
+      run: |
+        # Create a temporary gradle.properties with parallel settings
+        echo "org.gradle.parallel=true" >> gradle.properties
+        echo "org.gradle.workers.max=$INTEGRATION_TEST_FORKS" >> gradle.properties
+        echo "org.gradle.jvmargs=-Xmx$INTEGRATION_TEST_MEMORY -XX:+UseParallelGC" >> gradle.properties
+        
+        echo "📝 Gradle parallel configuration:"
+        echo "   Max workers: $INTEGRATION_TEST_FORKS"
+        echo "   JVM args: -Xmx$INTEGRATION_TEST_MEMORY -XX:+UseParallelGC"
+        echo ""
+        
+        # Run integration tests with parallel configuration and retry logic
+        ./gradlew :prime-router:testIntegration -Pshowtests --continue || {
+          echo "⚠️  First attempt failed, waiting 30s before retry..."
+          sleep 30
+          echo "🔄 Retrying integration tests with reduced parallelism..."
+          echo "org.gradle.workers.max=1" >> gradle.properties
+          ./gradlew :prime-router:testIntegration -Pshowtests --rerun-tasks
+        }
       shell: bash
 
     - name: Publish Integration Test Results

.github/dependabot.yml

@@ -11,18 +11,44 @@ updates:
     directory: "/operations"
     schedule:
       interval: "weekly"
+      day: "sunday"
+      time: "05:00"
+      timezone: "US/Eastern"
   - package-ecosystem: "docker"
     directory: "/prime-router"
     schedule:
-      interval: "monthly"
+      interval: "weekly"
+      day: "sunday"
+      time: "05:00"
+      timezone: "US/Eastern"
   - package-ecosystem: "docker"
     directory: "/.environment/docker/docker-compose"
     schedule:
-      interval: "monthly"
+      interval: "weekly"
+      day: "sunday"
+      time: "05:00"
+      timezone: "US/Eastern"
   - package-ecosystem: "docker"
     directory: "/frontend-react"
     schedule:
-      interval: "monthly"
+      interval: "weekly"
+      day: "sunday"
+      time: "05:00"
+      timezone: "US/Eastern"
+  - package-ecosystem: "docker"
+    directory: "/operations/utils/postgres"
+    schedule:
+      interval: "weekly"
+      day: "sunday"
+      time: "05:00"
+      timezone: "US/Eastern"
+  - package-ecosystem: "docker"
+    directory: "/operations/dnsmasq"
+    schedule:
+      interval: "weekly"
+      day: "sunday"
+      time: "05:00"
+      timezone: "US/Eastern"
 
   # slack-boltjs-app (chatops)
   - package-ecosystem: "gitsubmodule"

.github/workflows/snyk.yml

@@ -7,9 +7,7 @@ on:
       - main
     paths:
       -  "prime-router/**"
-      -  "submissions/**"
       -  "shared/**"
-      -  "auth/**"
 
 jobs:
   pre_job:
@@ -31,7 +29,7 @@ jobs:
     strategy:
         fail-fast: false
         matrix:
-            folder: [prime-router, submissions, auth, shared]
+            folder: [prime-router, shared]
     steps:
       - uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938
       - uses: snyk/actions/setup@b98d498629f1c368650224d6d212bf7dfa89e4bf
@@ -44,6 +42,6 @@ jobs:
       - uses: gradle/actions/setup-gradle@d156388eb19639ec20ade50009f3d199ce1e2808
       - name: Snyk Monitor
         working-directory: ${{ matrix.folder }}
-        run: snyk monitor --org=prime-reportstream
+        run: snyk monitor --org=prime-reportstream --target-reference="main"
         env:
           SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}

.gitignore

@@ -18,6 +18,9 @@
 !gradle-wrapper.jar
 *.war
 
+# act testing
+.act-testing/
+
 # virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml
 hs_err_pid*
 
@@ -133,4 +136,4 @@ gitleaks.log
 storybook-static
 
 # dotnet
-**/.config
+**/.config

gradle.properties

@@ -1,3 +1,4 @@
 org.gradle.jvmargs=-Dfile.encoding=UTF8 -Xmx2048M
 # Set Kotlin version for all peices of the build environment
 systemProp.kotlinVersion=2.1.0
+org.jetbrains.dokka.experimental.gradle.pluginMode=V2Enabled

prime-router/build.gradle.kts

@@ -215,12 +215,14 @@ tasks.test {
 
 tasks.javadoc.configure {
     actions.clear()
-    dependsOn(tasks.dokkaHtml)
+    dependsOn(tasks.dokkaGenerate)
 }
 
-tasks.dokkaHtml.configure {
+dokka {
     val docsDir = File(buildDir, "/docs/dokka")
-    outputDirectory.set(docsDir)
+    dokkaPublications.html {
+        outputDirectory.set(docsDir)
+    }
 }
 
 tasks.jacocoTestReport {
@@ -358,6 +360,13 @@ tasks.register<Copy>("copyApiSwaggerUI") {
 
 tasks.withType<Test>().configureEach {
     maxParallelForks = (Runtime.getRuntime().availableProcessors() / 2).takeIf { it > 0 } ?: 1
+    if ("AzureWebJobsStorage" !in System.getenv()) {
+        environment["AzureWebJobsStorage"] = "test-AzureWebJobsStorage"
+        println(
+            "'AzureWebJobsStorage' was not defined for testing and has been set to: " +
+                environment["AzureWebJobsStorage"]
+        )
+    }
 }
 
 tasks.processResources {
@@ -622,7 +631,7 @@ tasks.register("quickPackage") {
     tasks["compileTestKotlin"].enabled = false
     tasks["migrate"].enabled = false
     tasks["flywayMigrate"].enabled = false
-    tasks["dokkaHtml"].enabled = false
+    tasks["dokkaGenerate"].enabled = false
 }
 
 /**
@@ -688,7 +697,7 @@ tasks.register("quickRun") {
     tasks["compileTestKotlin"].enabled = false
     tasks["migrate"].enabled = false
     tasks["flywayMigrate"].enabled = false
-    tasks["dokkaHtml"].enabled = false
+    tasks["dokkaGenerate"].enabled = false
 }
 
 tasks.register("tiQuickRun") {

prime-router/src/main/kotlin/Receiver.kt

@@ -1,11 +1,10 @@
 package gov.cdc.prime.router
 
 import com.fasterxml.jackson.annotation.JsonIgnore
-import gov.cdc.prime.router.azure.BlobAccess
 import gov.cdc.prime.router.common.DateUtilities
-import gov.cdc.prime.router.common.Environment
 import gov.cdc.prime.router.fhirengine.translation.hl7.FhirToHl7Converter
 import gov.cdc.prime.router.fhirengine.translation.hl7.SchemaException
+import gov.cdc.prime.router.fhirengine.translation.hl7.utils.helpers.SchemaReferenceResolverHelper
 import java.time.LocalTime
 import java.time.OffsetDateTime
 import java.time.ZoneId
@@ -174,7 +173,7 @@ open class Receiver(
      *
      * @param operation MERGE will combine all reports in the batch into a single batch
      * @param numberPerDay Number of batches per day must be 1 to 3600
-     * @param initialTime The time of the day to first send. Must be format of hh:mm.
+     * @param initialTime The time of the day to first send. Must be of format of hh:mm.
      * @param timeZone the time zone of the initial sending
      * @param timeBetweenBatches the delay interval in seconds to apply between consecutive batches, if specified
      */
@@ -263,11 +262,11 @@ open class Receiver(
             if (this.topic.isUniversalPipeline) {
                 try {
                     FhirToHl7Converter(
-                        translation.schemaName,
-                        BlobAccess.BlobContainerMetadata.build(
-                            "metadata",
-                            Environment.get().storageEnvVar
-                        )
+                        SchemaReferenceResolverHelper.retrieveHl7SchemaReference(translation.schemaName),
+                        strict = false,
+                        terser = null,
+                        errors = mutableListOf(),
+                        warnings = mutableListOf(),
                     )
                 } catch (e: SchemaException) {
                     return e.message
@@ -283,15 +282,15 @@ open class Receiver(
     }
 
     companion object {
-        const val fullNameSeparator = "."
+        const val FULL_NAME_SEPARATOR = "."
 
         /** Global function to create receiver fullNames using
          * the [organizationName] and the [receiverName].
          */
         fun createFullName(
             organizationName: String,
             receiverName: String,
-        ): String = "$organizationName$fullNameSeparator$receiverName"
+        ): String = "$organizationName$FULL_NAME_SEPARATOR$receiverName"
 
         fun parseFullName(fullName: String): Pair<String, String> {
             val splits = fullName.split(Sender.fullNameSeparator)

prime-router/src/main/kotlin/azure/SendFunction.kt

@@ -326,10 +326,6 @@ class SendFunction(
         }
 
         val fileSize = content?.size ?: 0
-        // The last mile failure event has a slightly different pattern because we do not generate a child
-        // report as an output for this event so the childReport is the input to the send step and the
-        // parent report is the input to the batch step
-        val parentReport = workflowEngine.db.fetchFirstParentReport(report.reportId)
 
         reportEventService.sendReportEvent(
             eventName = ReportStreamEventName.REPORT_LAST_MILE_FAILURE,
@@ -346,9 +342,10 @@ class SendFunction(
                     ReportStreamEventProperties.FILENAME to externalFileName
                 )
             )
-            if (parentReport != null) {
-                parentReportId(parentReport.reportId)
-            }
+            // The last mile failure event has a slightly different pattern because we do not generate a child
+            // report as an output for this event so the childReport is the input to the send step and the
+            // parent report is the input to the batch step
+            parentReportId(report.reportId)
         }
 
         actionHistory.trackItemSendState(

prime-router/src/main/kotlin/azure/observability/event/ReportStreamEventBuilder.kt

@@ -96,6 +96,8 @@ abstract class AbstractReportStreamEventBuilder<T : AzureCustomEvent>(
             queueMessage
         )
 
+    abstract fun getSubmissionEventData(): SubmissionEventData
+
     fun send() {
             val event = buildEvent()
             sendToAzure(event)
@@ -140,8 +142,14 @@ open class ReportStreamReportEventBuilder(
 
     override fun buildEvent(): ReportStreamReportEvent = ReportStreamReportEvent(
             getReportEventData(),
+            getSubmissionEventData(),
             theParams
         )
+
+    override fun getSubmissionEventData(): SubmissionEventData = reportEventService.getSubmissionEventData(
+        0,
+        theParentReportId
+    )
 }
 
 /**
@@ -199,8 +207,15 @@ open class ReportStreamItemEventBuilder(
         )
     }
 
+    override fun getSubmissionEventData(): SubmissionEventData = reportEventService.getSubmissionEventData(
+        theChildIndex,
+        theParentReportId,
+        true
+    )
+
     override fun buildEvent(): ReportStreamItemEvent = ReportStreamItemEvent(
             getReportEventData(),
+            getSubmissionEventData(),
             getItemEventData(),
             theParams
         )
@@ -231,6 +246,7 @@ class ReportStreamReportProcessingErrorEventBuilder(
 ) {
     override fun buildEvent(): ReportStreamReportEvent = ReportStreamReportEvent(
             getReportEventData(),
+            getSubmissionEventData(),
             theParams + mapOf(ReportStreamEventProperties.PROCESSING_ERROR to error)
         )
 }
@@ -260,6 +276,7 @@ class ReportStreamItemProcessingErrorEventBuilder(
 ) {
     override fun buildEvent(): ReportStreamItemEvent = ReportStreamItemEvent(
             getReportEventData(),
+            getSubmissionEventData(),
             getItemEventData(),
             theParams + mapOf(ReportStreamEventProperties.PROCESSING_ERROR to error)
         )