Merge pull request #35 from redhat-performance/jsonification

Jsonification of fio output and updates to built-in data reduction to handle it

Author: malucius-rh

fio/fio_run

@@ -254,22 +254,58 @@ report_info()
 	echo $1 >> /tmp/log
 }
 
+#
+# Averaging function for json formatted bw and iops
+#
 obtain_avg()
 {
-	value=`grep "${1}" fio-result.txt | grep max | cut -d'=' -f ${2} | grep -v percentiles | cut -d',' -f 1`
-	nitems=0
-	calc="scale=2;("
-	field_separ=""
-	for val in $value; do
-		calc="${calc}${field_separ}$val"
-		let "nitems=$nitems+1"
-		field_separ="+"
-	done
-	calc="${calc})/${nitems}"
+        FIELD=${1}
+        value=`jq '.. | select(type == "object" and has('\"$FIELD\"')).'\"$FIELD\"'' fio-results.json`
+        nitems=0
+        calc="scale=2;("
+        field_separ=""
+        for val in $value; do
+                calc="${calc}${field_separ}$val"
+                let "nitems=$nitems+1"
+                field_separ="+"
+        done
+        if [ ${nitems} -eq 0]; then
+		calc=0
+	else
+		# Divide by 3 since the json output includes all three I/O types and we only care about one
+		calc="${calc})/(${nitems}/3)"
+	fi
 	rtval=`echo $calc | bc`
-	echo $rtval
+        echo $rtval
 }
 
+#
+# A latency-specific averaging function to handle how the json output is formatted for clat, lat, and slat
+#
+obtain_avg_lat()
+{
+        FIELD=${1}
+        value=`jq '.. | select(type == "object" and has('\"$FIELD\"')).'\"$FIELD\"'.mean' fio-results.json`
+        nitems=0
+        calc="scale=2;("
+        field_separ=""
+        for val in $value; do
+                calc="${calc}${field_separ}$val"
+                let "nitems=$nitems+1"
+                field_separ="+"
+        done
+        if [ ${nitems} -eq 0]; then
+		calc=0
+	else
+		# Divide by 3 since the json output includes all three I/O types and we only care about one
+        	# Divide by 1000 because the json output is nsec but usec is a bit more friendly for now
+        	calc="${calc})/(${nitems}/3)/1000"
+	fi
+        rtval=`echo $calc | bc`
+        echo $rtval
+}
+
+
 reduce_data()
 {
 	file=`ls -d fio* | head -n 1`
@@ -300,11 +336,11 @@ reduce_data()
 			if [ -d "$dir" ]; then
 				fname=`echo $dir | cut -d'-' -f 2-30`
 				cd $dir
-				bw=$(obtain_avg "bw" 5)
-				iops=$(obtain_avg "iops" 4)
-				clat=$(obtain_avg "clat" 4)
-				slat=$(obtain_avg "slat" 4)
-				lat=$(obtain_avg " lat (usec" 4)
+				bw=$(obtain_avg "bw_mean")
+				iops=$(obtain_avg "iops")
+				clat=$(obtain_avg_lat "clat_ns")
+                                slat=$(obtain_avg_lat "slat_ns")
+                                lat=$(obtain_avg_lat "lat_ns")
 				echo "${njobs}:${ndisks}:${iodepth}:${bw}" >> $rdir/results_${fname}/results_bw.csv
 				echo "${njobs}:${ndisks}:${iodepth}:${iops}" >> $rdir/results_${fname}/results_iops.csv
 				echo "${njobs}:${ndisks}:${iodepth}:${slat}" >> $rdir/results_${fname}/results_slat.csv
@@ -652,9 +688,9 @@ loop_io_tests()
 		build_run_file $ios $ioe $iodepth $njobs $run_time $disk_list $io_test
 		cp /tmp/fio_run ${out_dir}/file_run_${test_index}
 		let "test_index=${test_index}+1"
-		fio /tmp/fio_run >> $rdir/fio-result.txt
+		fio --output-format=json /tmp/fio_run >> $rdir/fio-results.json
 		mv *.log $rdir 		# preserve raw data
-		lines=`wc -l $rdir/fio-result.txt | cut -d' ' -f 1`
+		lines=`wc -l $rdir/fio-results.json | cut -d' ' -f 1`
 		if [ $lines -lt 2 ]; then
 			run_results="Failed"
 		fi