Merge pull request #55 from redhat-performance/output_formatting

Switch to group reporting and adapt data reduction

Author: malucius-rh

fio/fio_run

@@ -245,54 +245,25 @@ report_info()
 }
 
 #
-# Averaging function for json formatted bw and iops
+# Summing function for json formatted bw and iops
 #
-obtain_avg()
+obtain_val()
 {
-        FIELD=${1}
-        value=`jq '.. | select(type == "object" and has('\"$FIELD\"')).'\"$FIELD\"'' fio-results.json`
-        nitems=0
-        calc="scale=2;("
-        field_separ=""
-        for val in $value; do
-                calc="${calc}${field_separ}$val"
-                let "nitems=$nitems+1"
-                field_separ="+"
-        done
-        if [ ${nitems} -eq 0 ]; then
-		calc=0
-	else
-		# Divide by 3 since the json output includes all three I/O types and we only care about one
-		calc="${calc})/(${nitems}/3)"
-	fi
-	rtval=`echo $calc | bc`
-        echo $rtval
+	FIELD=${1}
+	value=`jq '.. | select(type == "object" and has('\"$FIELD\"')).'\"$FIELD\"'' fio-results.json | grep -w -v 0.000000`
+	echo ${value}
 }
 
 #
 # A latency-specific averaging function to handle how the json output is formatted for clat, lat, and slat
 #
 obtain_avg_lat()
 {
-        FIELD=${1}
-        value=`jq '.. | select(type == "object" and has('\"$FIELD\"')).'\"$FIELD\"'.mean' fio-results.json`
-        nitems=0
-        calc="scale=2;("
-        field_separ=""
-        for val in $value; do
-                calc="${calc}${field_separ}$val"
-                let "nitems=$nitems+1"
-                field_separ="+"
-        done
-        if [ ${nitems} -eq 0 ]; then
-		calc=0
-	else
-		# Divide by 3 since the json output includes all three I/O types and we only care about one
-        	# Divide by 1000 because the json output is nsec but usec is a bit more friendly for now
-        	calc="${calc})/(${nitems}/3)/1000"
-	fi
-        rtval=`echo $calc | bc`
-        echo $rtval
+	FIELD=${1}
+	value=`jq '.. | select(type == "object" and has('\"$FIELD\"')).'\"$FIELD\"'.mean' fio-results.json | grep -w -v 0.000000`
+	calc="${value}/1000"		# ns is a little cumbersome for now, use us
+	rtval=`echo ${calc} | bc`
+	echo ${rtval}
 }
 
 
@@ -326,11 +297,11 @@ reduce_data()
 			if [ -d "$dir" ]; then
 				fname=`echo $dir | cut -d'-' -f 2-30`
 				cd $dir
-				bw=$(obtain_avg "bw_mean")
-				iops=$(obtain_avg "iops")
+				bw=$(obtain_val "bw_mean")
+				iops=$(obtain_val "iops")
 				clat=$(obtain_avg_lat "clat_ns")
-                                slat=$(obtain_avg_lat "slat_ns")
-                                lat=$(obtain_avg_lat "lat_ns")
+                slat=$(obtain_avg_lat "slat_ns")
+                lat=$(obtain_avg_lat "lat_ns")
 				echo "${njobs}:${ndisks}:${iodepth}:${bw}" >> $rdir/results_${fname}/results_bw.csv
 				echo "${njobs}:${ndisks}:${iodepth}:${iops}" >> $rdir/results_${fname}/results_iops.csv
 				echo "${njobs}:${ndisks}:${iodepth}:${slat}" >> $rdir/results_${fname}/results_slat.csv
@@ -346,41 +317,54 @@ reduce_data()
 	#
 	# Now sort the results csv files and place headers
 	#
-	$TOOLS_BIN/test_header_info --front_matter --results_file $working_dir/results_fio.csv --host $to_configuration --sys_type $to_sys_type --tuned $to_tuned_setting --results_version $results_version --test_name $test_name
+	$TOOLS_BIN/test_header_info --test_name ${test_name}  --info_in_dir_name "$file 2,3 4,5,6 9,10" --results_file ${working_dir}/results_fio.csv
 	for list in `ls -d results_*`; do
 		pushd $list > /dev/null
 		op_and_size=`echo $list | cut -d'_' -f 2`
 		op=`echo "$op_and_size" | cut -d'-' -f1`
 		size=`echo "$op_and_size" | cut -d'-' -f2`
 		for ifile in `ls res*`; do
-			$TOOLS_BIN/test_header_info --test_name ${test_name}  --info_in_dir_name "$file 2,3 4,5,6 7,8 9,10" --meta_output "op: $op" --meta_output "size: $size" --results_file results.csv
+			# Sort the metrics
 			if [[ $ifile == *"bw"* ]]; then
-				echo "njobs:ndisks:iodepth:bw" >> results.csv
+				sort -t : -k 3 -n $ifile >> tmp_bw.csv
 			fi
 			if [[ $ifile == *"iops"* ]]; then
-                                echo "njobs:ndisks:iodepth:iops" >> results.csv
+				sort -t : -k 3 -n $ifile | cut -d : -f 4 >> tmp_iops.csv
                         fi
 			if [[ $ifile == *"_lat"* ]]; then
-				echo "njobs:ndisks:iodepth:lat" >> results.csv
+				sort -t : -k 3 -n $ifile | cut -d : -f 4 >> tmp_lat.csv
 			fi
 			if [[ $ifile == *"_clat"* ]]; then
-				echo "njobs:ndisks:iodepth:clat" >> results.csv
+				sort -t : -k 3 -n $ifile | cut -d : -f 4 >> tmp_clat.csv
 			fi
 			if [[ $ifile == *"_slat"* ]]; then
-				echo "njobs:ndisks:iodepth:slat" >> results.csv
+				sort -t : -k 3 -n $ifile | cut -d : -f 4 >> tmp_slat.csv
 			fi
-			sort -t : -k 3 -n $ifile >> results.csv
-			echo "" >> results.csv
+		done
+			# Stage the op & size fields, one line for each line of results
+			num_lines=`wc -l tmp_bw.csv | awk {'print $1'}`
+			for ct in `seq 1 ${num_lines}`; do
+				echo "${op}:${size}" | sed 's/...$//' >> tmp_ons.csv
+			done
+
+			# Now merge them together with colons as separators
+			pr -m -T -s: tmp_ons.csv tmp_bw.csv tmp_iops.csv tmp_clat.csv tmp_lat.csv tmp_slat.csv >> results.csv
 
 			cat  results.csv >> $working_dir/results_fio.csv
+
+			# Clean up after ourselves
 			rm results.csv
-		done
+			rm tmp_*.csv
+
 		popd > /dev/null
 	done
 }
 
 reduce_io_data()
 {
+	$TOOLS_BIN/test_header_info --front_matter --results_file $working_dir/results_fio.csv --host $to_configuration --sys_type $to_sys_type --tuned $to_tuned_setting --results_version $results_version --test_name $test_name
+	echo "op:blocksize_KiB:njobs:ndisks:iodepth:bw_KiB_s:iops:clat_us:lat_us:slat_us" >> ${working_dir}/results_fio.csv
+
 	aio_type=`ls -d fio* | cut -d'_' -f 10 | sort -u`
 
 	#
@@ -651,7 +635,7 @@ loop_io_tests()
 		build_run_file $ios $ioe $iodepth $njobs $run_time $disk_list $io_test
 		cp /tmp/fio_run ${out_dir}/file_run_${test_index}
 		let "test_index=${test_index}+1"
-		fio --output-format=json /tmp/fio_run >> $rdir/fio-results.json
+		fio --output-format=json --group_reporting /tmp/fio_run >> $rdir/fio-results.json
 		mv *.log $rdir 		# preserve raw data
 		lines=`wc -l $rdir/fio-results.json | cut -d' ' -f 1`
 		if [ $lines -lt 2 ]; then