Merge pull request #2270 from broadinstitute/jb-anndata-ingest-bugs

Prevent errors when assembling ingestSummary properties (SCP-6015)

Author: bistline

app/models/ingest_job.rb

@@ -1064,9 +1064,29 @@ def anndata_summary_props
         pipeline_args.detect { |c| c == study_file.id.to_s } &&
         (pipeline_args & CORE_ACTIONS).any?
     end
+    if previous_jobs.empty?
+      run = get_ingest_run
+      perftime = (TimeDifference.between(
+        event_timestamp(run.create_time), event_timestamp(run.update_time)
+      ).in_seconds * 1000).to_i
+      status = run.status.state == "FAILED" ? 'failed' : 'success'
+      return {
+        perfTime: perftime,
+        fileName: study_file.name,
+        fileType: study_file.file_type,
+        fileSize: study_file.upload_file_size,
+        studyAccession: study.accession,
+        trigger: study_file.upload_trigger,
+        jobStatus: status,
+        numFilesExtracted: 0,
+        machineType: params_object.machine_type,
+        action: 'unknown',
+        exitCode: exit_code
+      }
+    end
     # get total runtime from initial extract to final parse
-    initial_extract = previous_jobs.last
-    final_parse = previous_jobs.first
+    initial_extract = previous_jobs.min_by(&:create_time)
+    final_parse = previous_jobs.max_by(&:create_time)
     start_time = event_timestamp(initial_extract.create_time)
     end_time = event_timestamp(final_parse.update_time) # update_time is a proxy for last event timestamp
     job_perftime = (TimeDifference.between(start_time, end_time).in_seconds * 1000).to_i
@@ -1081,7 +1101,7 @@ def anndata_summary_props
     if job_status == 'failed'
       first_failure = previous_jobs.reverse.detect { |job| client.job_error(job.name).present? }
       args = client.get_job_command_line(job: first_failure)
-      error_action = args.detect {|c| BatchApiClient::FILE_TYPES_BY_ACTION.keys.include?(c.to_sym) }
+      error_action = args.detect { |c| BatchApiClient::FILE_TYPES_BY_ACTION.keys.include?(c.to_sym) }
       code = client.exit_code_from_task(first_failure.name)
     end
     # count total number of files extracted

test/models/ingest_job_test.rb

@@ -628,6 +628,58 @@ class IngestJobTest < ActiveSupport::TestCase
     end
   end
 
+  test 'should handle not finding upstream jobs in ingestSummary' do
+    study = FactoryBot.create(:detached_study,
+                              name_prefix: 'Empty Jobs test',
+                              user: @user,
+                              test_array: @@studies_to_clean)
+    ann_data_file = FactoryBot.create(:ann_data_file, name: 'failed.h5ad', study:)
+    pipeline_name = SecureRandom.uuid
+    dummy_job = Google::Apis::BatchV1::Job.new(
+      name: pipeline_name,
+      create_time: (@now - 1.hour).to_s,
+      update_time: @now.to_s,
+      status: Google::Apis::BatchV1::JobStatus.new(
+        state: 'FAILED',
+        status_events: [
+          Google::Apis::BatchV1::StatusEvent.new(event_time: (@now - 1.hour).to_s),
+          Google::Apis::BatchV1::StatusEvent.new(event_time: @now.to_s)
+        ]
+      )
+    )
+    list_mock = Minitest::Mock.new
+    list_mock.expect :jobs, []
+    mock = Minitest::Mock.new
+    mock.expect :list_jobs, list_mock
+    mock.expect :exit_code_from_task, 1, [pipeline_name]
+    params_object = AnnDataIngestParameters.new(
+      anndata_file: "gs://#{study.bucket_id}/matrix.h5ad", file_size: ann_data_file.upload_file_size
+    )
+    job = IngestJob.new(
+      pipeline_name:, study:, study_file: ann_data_file, user: @user, action: :ingest_anndata, params_object:
+    )
+    expected_props = {
+      perfTime: 3_600_000,
+      fileName: ann_data_file.name,
+      fileType: ann_data_file.file_type,
+      fileSize: ann_data_file.upload_file_size,
+      studyAccession: study.accession,
+      trigger: ann_data_file.upload_trigger,
+      jobStatus: 'failed',
+      numFilesExtracted: 0,
+      machineType: params_object.machine_type,
+      action: 'unknown',
+      exitCode: 1
+    }.with_indifferent_access
+    job.stub :get_ingest_run, dummy_job do
+      ApplicationController.stub :batch_api_client, mock do
+        props = job.anndata_summary_props
+        assert_equal expected_props, props.with_indifferent_access
+        mock.verify
+      end
+    end
+  end
+
   test 'should limit size when reading error logfile for email' do
     job = IngestJob.new(study: @basic_study, study_file: @basic_study_exp_file, user: @user, action: :ingest_expression)
     file_location = @basic_study_exp_file.bucket_location