Added cost commands. (#163)

* Added commands to calculate the cost for workflows (GCS backend only).

Author: jonn-smith

README.md

@@ -98,6 +98,16 @@ requires `column`, `curl`, `mail`, and [jq](https://stedolan.github.io/jq/)
    #### Update cromwell server
    * `update-server`
      * Change the cromwell server that new jobs will be submitted to.
+
+   #### Get cost for a workflow
+	 * `cost`
+	   * Get the cost for a workflow.
+		 * Will only work for workflows that completed more than 8 hours ago on GCS.
+		 * Requires the `gcp_bq_cost_table.config` configuration file to exist and contain the big query cost table for your organization.
+	 * `cost-detailed`
+	   * Get the cost for a workflow at the task level.
+		 * Will only work for workflows that completed more than 8 hours ago on GCS.
+		 * Requires the `gcp_bq_cost_table.config` configuration file to exist and contain the big query cost table for your organization.
 
  ### Features:
  * Running `submit` will create a new folder in the `~/.cromshell/${CROMWELL_URL}/` directory named with the cromwell job id of the newly submitted job.  

cromshell

@@ -44,6 +44,9 @@ mkdir -p ${CROMSHELL_CONFIG_DIR}
 CROMWELL_SUBMISSIONS_FILE="${CROMSHELL_CONFIG_DIR}/all.workflow.database.tsv"
 [[ ! -f ${CROMWELL_SUBMISSIONS_FILE} ]] && echo -e "DATE\tCROMWELL_SERVER\tRUN_ID\tWDL_NAME\tSTATUS\tALIAS" > ${CROMWELL_SUBMISSIONS_FILE}
 
+# Set the GCP big query cost table file:
+BQ_COST_TABLE_FILE=${CROMSHELL_CONFIG_DIR}/gcp_bq_cost_table.config
+
 # Update cromshell submissions file if it needs updating:
 grep -q 'ALIAS$' ${CROMWELL_SUBMISSIONS_FILE} 
 r=$?
@@ -207,6 +210,20 @@ function usage()
   echo -e "  Update cromwell server:"
   echo -e "   update-server          Change which cromwell server jobs will be submitted to."
   echo -e ""
+  echo -e "  Get cost for a workflow"
+  echo -e "   cost [workflow-id] [[workflow-id]...]                  Get the cost for a workflow."
+  echo -e "                                                          Only works for workflows that completed"
+  echo -e "                                                          more than 8 hours ago on GCS."
+  echo -e "                                                          Requires the 'gcp_bq_cost_table.config'"
+  echo -e "                                                          configuration file to exist and contain"
+  echo -e "                                                          the big query cost table for your organization."
+  echo -e "   cost-detailed [workflow-id] [[workflow-id]...]         Get the cost for a workflow at the task level."
+  echo -e "                                                          Only works for workflows that completed"
+  echo -e "                                                          more than 8 hours ago on GCS."
+  echo -e "                                                          Requires the 'gcp_bq_cost_table.config'"
+  echo -e "                                                          configuration file to exist and contain"
+  echo -e "                                                          the big query cost table for your organization."
+  echo -e ""
   echo -e "Return values:"
   echo -e "  0                  SUCCESS"
   echo -e "  ANYTHING_BUT_ZERO  FAILURE/ERROR"
@@ -1471,6 +1488,148 @@ function fetch-all()
   return 0
 }
 
+function _cost_helper() 
+{
+  local id=$1
+  local svr=$2
+
+	turtle
+	which bq &>/dev/null   
+  local r=$?
+  [[ ${r} -ne 0 ]] && error "bq does not exist.  Must install the big query command-line client." && exit 8
+
+	# Check for gdate:
+  if [[ "$(uname)" == "Darwin" ]] ; then
+		which gdate &> /dev/null
+		r=$?
+		[ $r -ne 0 ] && error "Must have coreutils installed for 'gdate'" && exit 13
+	fi
+
+  [ ! -e ${BQ_COST_TABLE_FILE} ] && error "Big Query cost table file does not exist.  Must populate ${BQ_COST_TABLE_FILE} with big query cost table information." && exit 9
+
+	# Make sure the given ID is actually in our file:
+	grep -q "${id}" ${CROMWELL_SUBMISSIONS_FILE} 
+	r=$?
+	[ $r -ne 0 ] && error "Given ID is not in your cromwell submissions file (${CROMWELL_SUBMISSIONS_FILE}): ${id}" && exit 10
+
+  COST_TABLE=$(head -n1 ${BQ_COST_TABLE_FILE})
+  
+	# Get the time that the workflow finished:
+	error "Fetching workflow finish time..."
+	tmpMetadata=$( makeTemp )
+  curl --compressed -s "${svr}/api/workflows/v1/${id}/metadata?includeKey=workflowProcessingEvents" > ${tmpMetadata}
+
+	[ ! -s ${tmpMetadata} ] && error "Could not communicate with server.  Perhaps try a longer timeout." && exit 15
+
+	grep -q '"description":"Finished",' ${tmpMetadata}
+	r=$?
+	[ $r -ne 0 ] && error "Workflow ${id} is not finished yet." && exit 11
+
+	STARTED_TIME=$( jq '.workflowProcessingEvents | map(select(.description == "PickedUp")) | .[].timestamp' ${tmpMetadata} | tr -d '"')
+	FINISHED_TIME=$( jq '.workflowProcessingEvents | map(select(.description == "Finished")) | .[].timestamp' ${tmpMetadata} | tr -d '"')
+
+	# Make sure that at least 8h have passed since the workflow finished:
+  if [[ "$(uname)" == "Darwin" ]] ; then
+    local DATE_CMD=gdate
+  else
+    local DATE_CMD=date
+  fi  
+
+	local ts1=$( $DATE_CMD +%s -d "${FINISHED_TIME}" )
+	local ts2=$( date +%s )
+	local can_check_cost=$( echo "(${ts2} - ${ts1}) >= (3600 * 8)" | bc )
+	[ $can_check_cost -ne 1 ] && error "Workflow finished less than 8 hours ago.  Cannot check cost.  Please try again later." && exit 12
+
+	# Generate the start and end dates for our query:
+	START_DATE=$($DATE_CMD +%Y-%m-%d -d "${STARTED_TIME} -1 day")
+	END_DATE=$($DATE_CMD +%Y-%m-%d -d "${FINISHED_TIME} +1 day")
+
+  error "Using cost table: ${COST_TABLE}"
+  error ""
+}
+
+# Get the cost for a workflow ID:
+function cost()
+{
+  local id=$1
+  local svr=$2
+	_cost_helper $id $svr
+
+  local tmp_cost_file=$( makeTemp )
+
+  # Get the cost from Big Query:
+  bq query --use_legacy_sql=false "SELECT sum(cost) FROM \`${COST_TABLE}\`, UNNEST(labels) WHERE value = \"cromwell-${id}\" AND _PARTITIONDATE BETWEEN \"${START_DATE}\" AND \"${END_DATE}\";" > ${tmp_cost_file}
+  r=$?
+
+  # Display the cost:
+	total_cost=$( head -n4 ${tmp_cost_file} | tail -n1 | tr -d '| \t')
+	[[ "${total_cost}" == NULL ]] && error "Could not retrieve cost - no cost entries found." && exit 14
+	echo -n '$'
+	echo "scale=2;${total_cost}/1" | bc
+
+	error ""
+	error "Costs rounded to nearest cent (approximately)."
+	error ""
+	error "WARNING: Costs here DO NOT include any call cached tasks."
+
+  return $r
+}
+
+# Get the cost for a workflow ID:
+function cost-detailed()
+{
+  local id=$1
+  local svr=$2
+	_cost_helper $id $svr
+
+  local tmp_cost_file=$( makeTemp )
+
+  # Get the cost from Big Query:
+  bq query \
+    --use_legacy_sql=false \
+    "SELECT
+    wfid.value, service.description, task.value as task_name, sum(cost) as cost
+    FROM
+    \`${COST_TABLE}\` as billing, UNNEST(labels) as wfid, UNNEST(labels) as task
+    WHERE
+    cost > 0
+    AND task.key LIKE \"wdl-task-name\"
+    AND wfid.key LIKE \"cromwell-workflow-id\"
+    AND wfid.value like \"%${id}\"
+    AND _PARTITIONDATE BETWEEN \"${START_DATE}\" AND \"${END_DATE}\"
+    GROUP BY 1,2,3
+    ORDER BY 4 DESC
+    ;" | tail -n+4 | grep -v '^+' | tr -d '|' | awk 'BEGIN{OFS="\t"}{print $(NF-1), $NF}' | sort > ${tmp_cost_file}
+
+  r=$?
+
+	local total_cost=$(awk '{print $2}' ${tmp_cost_file} | tr '\n' '+' | sed 's#$#0#' | bc)
+	local total_cost=$(echo "scale=2;${total_cost}/1" | bc)
+
+	local tmpf2=$(makeTemp)
+	echo -e "TASK\tCOST" > ${tmpf2}
+	while read line ; do
+		local task=$(echo $line | awk '{print $1}')
+		local task_cost=$(echo $line | awk '{print $2}')
+		task_cost=$(echo "scale=2;if ( ${task_cost} >= 0.01 ) { ${task_cost}/1; } else { 0.01 }" | bc)
+		printf "${task}\t$%02.2f\n" ${task_cost}
+	done < ${tmp_cost_file} >> ${tmpf2}
+
+	column -t ${tmpf2} | head -n1
+	local bar_width=$( column -t ${tmpf2} | head -n1 | wc -c )	
+	python -c "print('=' * ${bar_width})"
+	column -t ${tmpf2} | tail -n+2
+	python -c "print('=' * ${bar_width})"
+	echo "Total Cost: \$${total_cost}"
+
+	error ""
+	error "Costs rounded to nearest cent (approximately)."
+	error ""
+	error "WARNING: Costs here DO NOT include any call cached tasks."
+
+  return $r
+}
+
 function assertValidEmail() 
 {
   # Make sure the user gave us a good email address:
@@ -1885,7 +2044,7 @@ if ${ISINTERACTIVESHELL} ; then
 
   # Validate our sub-command:
   case ${SUB_COMMAND} in
-    cleanup|submit|status|logs|execution-status-count|counts|metadata|slim-metadata|timing|abort|notify|list|fetch-all|fetch-logs|list-outputs|alias)
+    cleanup|submit|status|cost|cost-detailed|logs|execution-status-count|counts|metadata|slim-metadata|timing|abort|notify|list|fetch-all|fetch-logs|list-outputs|alias)
       # This is a good sub-command, so we do not need to do anything. 
       ;;
     _rawNotify)