ref data import

Author: agnis-mateuss

drupal/Chart.yaml

@@ -1,6 +1,6 @@
 apiVersion: v2
 name: drupal
-version: 1.20.0
+version: 1.21.0
 dependencies:
 - name: mariadb
   version: 7.5.x

drupal/templates/_helpers.tpl

@@ -535,21 +535,51 @@ else
 fi
 {{- end }}
 
-{{- define "drupal.import-reference-db" -}}
+{{- define  "drupal.import-reference-db" -}}
 if [ "${REF_DATA_COPY_DB:-}" == "true" ]; then
   if [[ -f /app/reference-data/db.tar.gz || -f /app/reference-data/db.sql.gz ]]; then
     echo "Dropping old database"
     drush sql-drop -y
 
     app_ref_data=/app/reference-data
     tmp_ref_data=/tmp/reference-data
+    import_method={{ .Values.referenceData.databaseImportMethod }}
 
     # New way of importing.
     if [[ -f "${app_ref_data}/db.tar.gz" ]]; then
       echo "Importing reference database dump from db.tar.gz"
       mkdir "${tmp_ref_data}"
       tar -xzf "${app_ref_data}/db.tar.gz" -C "${tmp_ref_data}/"
-      find "${tmp_ref_data}/" -type f -name "*.sql" | xargs -P10 -I{} sh -c 'echo "Importing {}" && mysql -A --user="${DB_USER}" --password="${DB_PASS}" --host="${DB_HOST}" "${DB_NAME}" < {}'
+
+      if [[ "$import_method" == "parallel" ]]; then
+        echo "Importing SQL files in parallel. This setting can be changed in silta.yml using the referenceData.databaseImportMethod key."
+        find "${tmp_ref_data}/" -type f -name "*.sql" | xargs -P10 -I{} sh -c 'echo "Importing {}" && mysql -A --user="${DB_USER}" --password="${DB_PASS}" --host="${DB_HOST}" "${DB_NAME}" < {}'
+        pipeline_exit_code=$? # Capture exit code of the pipeline (most likely influenced by xargs)
+
+        # Check if xargs reported an error (any non-zero exit status)
+        if [ "$pipeline_exit_code" -ne 0 ]; then
+          echo "ERROR: One or more parallel imports failed. Check the logs above for specific mysql errors."
+          exit 1
+        fi
+
+        echo "Parallel import command finished."
+
+      elif [[ "$import_method" == "sequential" ]]; then
+        echo "Importing SQL files sequentially. This setting can be changed in silta.yml using the referenceData.databaseImportMethod key."
+        find "${tmp_ref_data}/" -type f -name "*.sql" | sort | while IFS= read -r sql_file; do
+          echo "Importing ${sql_file}"
+          if ! mysql -A --user="${DB_USER}" --password="${DB_PASS}" --host="${DB_HOST}" "${DB_NAME}" < "${sql_file}"; then
+            echo "ERROR: Failed to import ${sql_file}. Check the logs above for specific mysql errors."
+            exit 1
+          fi
+        done
+
+        echo "Sequential import command finished."
+
+      else
+        echo "Incompatible import method. Please use either 'parallel' or 'sequential' in referenceData.databaseImportMethod."
+        exit 1
+      fi
 
     # Backwards compatibility for old way of importing.
     elif [[ -f "${app_ref_data}/db.sql.gz" ]]; then

drupal/values.schema.json

@@ -8,7 +8,7 @@
     "branchName": { "type": "string" },
     "imagePullSecrets": { "type": "array" },
     "imagePullSecret": { "type": "string" },
-    "serviceAccount": { 
+    "serviceAccount": {
       "type": "object",
       "additionalProperties": false,
       "properties": {
@@ -450,6 +450,7 @@
         "enabled": { "type": "boolean" },
         "referenceEnvironment": { "type": "string" },
         "schedule": { "type": "string" },
+        "databaseImportMethod": { "type": "string" },
         "copyDatabase": { "type": "boolean" },
         "copyFiles": { "type": "boolean" },
         "updateAfterDeployment": { "type": "boolean" },

drupal/values.yaml

@@ -448,6 +448,10 @@ referenceData:
   # When to automatically update the reference data.
   schedule: '~ 3 * * *'
 
+  # The method used to import the database tables. Can be either 'parallel' or 'sequential'.
+  # Parallel is faster but might in some cases be unstable. In that case, use sequential instead.
+  databaseImportMethod: 'parallel'
+
   # Whether to copy the database, only used programmatically.
   copyDatabase: true