_run_assembly.uge-nextflow

#!/bin/bash -l

SCRIPT_NAME="$(basename ${0#_} .uge-nextflow)"

# Set profile
# Get node number: >=231 = old rosalind, <=230 = biolinux/aspen
# The new Rosalind RHEL8 uses "rosalind-<INT>" node names not "node<int>" instead
echo "jobid:  ${SCRIPT_NAME}"
echo "hostname:  ${HOSTNAME}"
echo "system:  $(uname -a)"
echo -e "distro and version:\n$(lsb_release -a | sed 's/^/    /')"

NODE_NUM="$(echo ${HOSTNAME%%.*} | sed 's/node//1')"
echo "node_num:  $NODE_NUM"
if [[ "${NODE_NUM}" =~ ^rosalind-[0-9]+$ ]]; then
  echo "INFO: using rosalind_hpc_age profile ..."
  HPC='rosalind_hpc_age'
elif [[ "${NODE_NUM}" =~ ^?[0-9]+$ ]] && [[ ${NODE_NUM} -ge 95 ]]; then
  echo "INFO: using rosalind_hpc profile ..."
  HPC='rosalind_hpc'
elif [[ "${NODE_NUM}" =~ ^?[0-9]+$ ]] && [[ ${NODE_NUM} -lt 95 ]]; then
  echo "INFO: using aspen_hpc profile ..."
  HPC='aspen_hpc'
else
  # No regex matches to node<int> Aspen, Biolinux, Rosalind or to rosalind-<INT>
  echo -e "\n${RED_TXT}Biolinux/Aspen/Rosalind HPC is not detected.\nSubmission cancelled. ${COLOR_OFF}"
  exit 1
fi

time_stamp="$(date '+%Y-%b-%d_%a_%H-%M-%S')"

module --ignore-cache load nextflow/25.04.7
nextflow \
  -log "${OUT}/pipeline_info/nextflow_log.${time_stamp}.txt" \
  run \
  "${LAB_HOME}/workflows/wf-paired-end-illumina-assembly/main.nf" \
  -profile "${HPC}" \
  --input "${IN}" \
  --outdir "${OUT}" \
  -ansi-log false \
  -N "${USER}@cdc.gov" \
  -w "${OUT}/.work" \
  -with-dag "${OUT}/pipeline_info/dag.${time_stamp}.html" \
  --blast_db "${LAB_HOME}/.databases/ncbi" \
  --checkm2_db "${LAB_HOME}/.databases/checkm2" \
  --kraken2_db "${LAB_HOME}/.databases/kraken2" \
  --sra_scrubber_db "${LAB_HOME}/.databases/sra-human-scrubber/data/human_filter.db" \
  --create_excel_outputs \
  -resume

# Give 15 sec for stdout file to contain the info we wanna parse
sleep 15

# Check for errors and add to errors.tsv
# Get nextflow run name
run_name=$(grep "^Launching" "${OUT}/pipeline_info/ASM_*.o${SCRIPT_NAME}" \
  | cut -d '[' -f 2 | cut -d ']' -f 1)

# Regex for validating "string_string" pattern
if [[ ! "$run_name" =~ ^[a-zA-Z0-9]+_[a-zA-Z0-9]+$ ]]; then
    echo "INFO: extracted nextflow run_name is invalid: $run_name"
    exit 0
fi

###
# Report exit status codes if they exist in the stdout
###
mapfile -t error_lines < <(grep "terminated with an error exit status" "${OUT}/pipeline_info/ASM_*.o${SCRIPT_NAME}")

# Hopefully no errors and we can just skip
if [ ${#error_lines[@]} -eq 0 ]; then
  echo "No errors found, so no errors.tsv created"
  echo -e "Assembly and QA finished on $(date)\n${OUT}" | mail \
    -s "${num_assemblies} assembled $(basename "${OUT}") [HPC]" \
    -S smtp="smtpgw.cdc.gov" \
    "${USER}@cdc.gov"
  exit 0
fi

# Otherwise, create the errors.tsv file and write headers
error_file="errors.tsv"
echo -e "Sample_name\tProcess_name\tExit_node\tError_path\tTime\tRun_name\tRetry_number" > errors.tsv

# Extract Nextflow's single workDir for the entire run (painful due to ASCII chars!)
work_path=$(grep -P '\x1b\[0;34mworkDir' "${OUT}/pipeline_info/ASM_*.o${SCRIPT_NAME}" \
  | cut -d : -f 2 | sed 's/\x1b\[[0-9;]*m//g' \
  | sed 's/^[[:space:]]*://; s/^[[:space:]]*//')

# Extract Nextflow's special run name for the entire run (painful due to ASCII chars!)
run_name=$(grep -P '\x1b\[0;34mrunName' "${OUT}/pipeline_info/ASM_*.o${SCRIPT_NAME}" \
  | cut -d : -f 2 | sed 's/\x1b\[[0-9;]*m//g' \
  | sed 's/^[[:space:]]*://; s/^[[:space:]]*//')

# Loop through the lines
for ln in "${error_lines[@]}"; do
  # Extract process_id (example: [b5/d89f89])
  process_id=$(echo "$ln" | grep -oP '\[\K[^]]+' || echo "")

  # If process_id is empty, set it to "."
  if [ -z "$process_id" ]; then
      error_path='.'
      time_stamp=''
  # If process_id is not empty, combine with work_path
  else
      error_path="${work_path}/${process_id}"
      time_stamp=$(ls -ld "${error_path}" 2> /dev/null \
        | awk '{print $6, $7, $8}' \
        | xargs -I {} date -d "{}" '+%Y-%b-%d %a %H:%M:%S')
      if [ -z "$time_stamp" ]; then
          time_stamp='.'
      fi
  fi

  # Extract Sample_Name (inside the parentheses within the backticks)
  sample_name=$(echo "$ln" | grep -oP '\`[^`]*\(\K[^\)]+(?=\))' | head -n 1)

  # Extract the exit code (number after 'exit status')
  exit_code=$(echo "$ln" | grep -oP 'exit status \(\K[0-9]+')

  # Extract Process ID from the square brackets (if present)
  process_id=$(echo "$ln" | grep -oP '\[([^\]]+)\]' | tr -d '[]')

  # Extract Process_Name (remove everything before the colon and also remove parentheses)
  process_name=$(echo "$ln" | sed -E 's/.*:([^(]*)\(.*/\1/')

  # Extract Retries (number after '-- Execution is retried')
  retries=$(echo "$ln" | grep -oP 'Execution is retried \(\K[0-9]+')
  if [ -z "$retries" ]; then
      retries='.'
  fi

  echo -e "$sample_name\t$process_name\t$exit_code\t$error_path\t$time_stamp\t$run_name\t$retries" >> errors.tsv
done

# Count samples (total lines - 1 header line) in Summary.Assembly_Depth.tsv
num_assemblies=0
if [[ -f "${OUT}/Summaries/Summary.Assembly_Depth.tsv" ]]; then
  num_assemblies=$(awk 'END {print NR-1}' "${OUT}/Summaries/Summary.Assembly_Depth.tsv")
fi

# E-mail completion status
if [[ -f "${OUT}/Summaries/Summary-Report.xlsx" ]] \
  && [[ -f "${OUT}/pipeline_info/errors.tsv" ]]; then
  echo -e "Assembly and QA finished on $(date)\n${OUT}" | mail \
  -s "${num_assemblies} assembled $(basename "${OUT}") [HPC]" \
  -S smtp="smtpgw.cdc.gov" \
  -a "${OUT}/Summaries/Summary-Report.xlsx" \
  -a "${OUT}/pipeline_info/errors.tsv" \
  "${USER}@cdc.gov"

elif [[ -f "${OUT}/Summaries/Summary.Assembly_Depth.tsv" ]] \
  && [[ -f "${OUT}/pipeline_info/errors.tsv" ]]; then
  echo -e "Assembly and QA finished on $(date)\n${OUT}" | mail \
  -s "${num_assemblies} assembled $(basename "${OUT}") [HPC]" \
  -S smtp="smtpgw.cdc.gov" \
  -a "${OUT}/Summaries/Summary.Assembly_Depth.tsv" \
  -a "${OUT}/pipeline_info/errors.tsv" \
  "${USER}@cdc.gov"

elif [[ -f "${OUT}/Summaries/Summary-Report.xlsx" ]]; then
  echo -e "Assembly and QA finished on $(date)\n${OUT}" | mail \
  -s "${num_assemblies} assembled $(basename "${OUT}") [HPC]" \
  -S smtp="smtpgw.cdc.gov" \
  -a "${OUT}/Summaries/Summary-Report.xlsx" \
  "${USER}@cdc.gov"

elif [[ -f "${OUT}/Summaries/Summary.Assembly_Depth.tsv" ]]; then
  echo -e "Assembly and QA finished on $(date)\n${OUT}" | mail \
  -s "${num_assemblies} assembled $(basename "${OUT}") [HPC]" \
  -S smtp="smtpgw.cdc.gov" \
  -a "${OUT}/Summaries/Summary.Assembly_Depth.tsv" \
  "${USER}@cdc.gov"

elif [[ -f "${OUT}/pipeline_info/errors.tsv" ]]; then
  echo -e "Assembly and QA could not be completed on $(date)\n${OUT}" | mail \
  -s "No assemblies found $(basename "${OUT}") [HPC]" \
  -S smtp="smtpgw.cdc.gov" \
  -a "${OUT}/pipeline_info/errors.tsv" \
  "${USER}@cdc.gov"
fi

# Move and symlink work directory
# date=$(date '+%Y-%m-%d_%H-%M-%S')
# mv ${OUT}/.work /scicomp/scratch/${USER}/work/ASM_${date}
# ln -s /scicomp/scratch/${USER}/work/ASM_${date} ${OUT}/.work