Skip to content

Commit cb0e3f3

Browse files
gujguj
committed
fixed shellcheck errors
1 parent 0d03b90 commit cb0e3f3

2 files changed

Lines changed: 124 additions & 97 deletions

File tree

‎source/utils/profiler_simplified/ReadMe.md‎

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,21 @@ Bytes are reported in "transport_0.wbytes"
1818

1919
For how to plot a json file, see test/commands
2020

21-
* ./writeSummary.sh <f1.json> [<f2.json> .. ] – produces a high-level summary of write metrics for each JSON file provided.
21+
* ./writeSummary.sh <f1.json> [<f2.json> .. ] – produces a high-level summary of write metrics for each JSON file provided. e.g.
22+
23+
```bash
24+
./writeSummary.sh data/v/v.json
25+
======= High level Summary of data/v/v.json =======
26+
Num Ranks: 4. NumSteps: 10
27+
DataSizeBased + SelectiveAggregationMetadata
28+
t1 bytes: 202.56 KiB
29+
t0 bytes: 12.50 GiB
30+
Max [ES + PDW]: = 1.602 sec at rank: 3
31+
Min [ES + PDW]: = 1.592 sec at rank: 2
32+
```
33+
34+
35+
2236
* python3 plot_json.py <f1.json> [<f2.json> .. ] - generates quick, informative plots for top level time-measurement tags in the provided JSON files.
2337

2438
The example plots are in ``python_plots/`. All plots reflect per-rank measurements with the BP5 engine. Other engines may omit some or all of these metrics.

‎source/utils/profiler_simplified/writeSummary.sh‎

Lines changed: 109 additions & 96 deletions
Original file line numberDiff line numberDiff line change
@@ -3,127 +3,140 @@
33
## summarizing for ADIOS writer ##
44
####################################
55
print_bytes() {
6-
local bytes=${1:-0}
7-
local precision=${2:-2} # default: 2 decimal places
8-
9-
if ((bytes < 1024)); then
10-
printf "%d B\n" "$bytes"
11-
elif ((bytes < 1048576)); then
12-
printf "%.${precision}f KiB\n" "$(echo "$bytes / 1024" | bc -l)"
13-
elif ((bytes < 1073741824)); then
14-
printf "%.${precision}f MiB\n" "$(echo "$bytes / 1048576" | bc -l)"
15-
elif ((bytes < 1099511627776)); then
16-
printf "%.${precision}f GiB\n" "$(echo "$bytes / 1073741824" | bc -l)"
17-
else
18-
printf "%.${precision}f TiB\n" "$(echo "$bytes / 1099511627776" | bc -l)"
19-
fi
6+
local bytes=${1:-0}
7+
local precision=${2:-2} # default: 2 decimal places
8+
9+
if ((bytes < 1024)); then
10+
printf "%d B\n" "$bytes"
11+
elif ((bytes < 1048576)); then
12+
printf "%.${precision}f KiB\n" "$(echo "$bytes / 1024" | bc -l)"
13+
elif ((bytes < 1073741824)); then
14+
printf "%.${precision}f MiB\n" "$(echo "$bytes / 1048576" | bc -l)"
15+
elif ((bytes < 1099511627776)); then
16+
printf "%.${precision}f GiB\n" "$(echo "$bytes / 1073741824" | bc -l)"
17+
else
18+
printf "%.${precision}f TiB\n" "$(echo "$bytes / 1099511627776" | bc -l)"
19+
fi
2020
}
2121

2222
print_time() {
23-
local tt=${1:-0}
24-
local precision=${2:-3} # default: 2 decimal places
25-
26-
if ((tt < 1000)); then
27-
printf "%d mus\n" "$tt"
28-
elif ((tt < 60000000)); then # secs if < 1 minute
29-
printf "%.${precision}f sec\n" "$(echo "$tt / 1000000" | bc -l)"
30-
elif ((tt < 600000000)); then # minutes if < 60 minutes
31-
printf "%.${precision}f min\n" "$(echo "$tt / 60000000" | bc -l)"
32-
else # >= 1 hr
33-
printf "%.${precision}f hr\n" "$(echo "$tt/ 3600000000" | bc -l)"
34-
fi
23+
local tt=${1:-0}
24+
local precision=${2:-3} # default: 2 decimal places
25+
26+
if ((tt < 1000)); then
27+
printf "%d mus\n" "$tt"
28+
elif ((tt < 60000000)); then # secs if < 1 minute
29+
printf "%.${precision}f sec\n" "$(echo "$tt / 1000000" | bc -l)"
30+
elif ((tt < 600000000)); then # minutes if < 60 minutes
31+
printf "%.${precision}f min\n" "$(echo "$tt / 60000000" | bc -l)"
32+
else # >= 1 hr
33+
printf "%.${precision}f hr\n" "$(echo "$tt/ 3600000000" | bc -l)"
34+
fi
3535
}
3636

3737
get_min() {
38-
targetFileName="$1"
39-
ok=$(jq 'map(. as $obj | {
38+
targetFileName="$1"
39+
ok=$(jq 'map(. as $obj | {
4040
rank: $obj.rank,
4141
sum: ([ $obj.ES_mus, $obj.PDW_mus, $obj.PP_mus ]
4242
| map(select(type=="number"))
43-
| add // 0) } ) | min_by(.sum) ' $targetFileName)
44-
45-
echo $ok
43+
| add // 0) } ) | min_by(.sum) ' "$targetFileName")
44+
echo "$ok"
4645
}
4746

4847
get_max() {
49-
targetFileName="$1"
50-
ok=$(jq 'map(. as $obj | {
48+
targetFileName="$1"
49+
ok=$(jq 'map(. as $obj | {
5150
rank: $obj.rank,
5251
sum: ([ $obj.ES_mus, $obj.PDW_mus, $obj.PP_mus ]
5352
| map(select(type=="number"))
54-
| add // 0) } ) | max_by(.sum) ' $targetFileName)
53+
| add // 0) } ) | max_by(.sum) ' "$targetFileName")
5554

56-
echo $ok
55+
echo "$ok"
5756
}
5857

5958
process_file() {
60-
targetFileName="$1"
61-
62-
# Check if the file exists and is readable
63-
if [[ ! -f "$targetFileName" ]]; then
64-
echo "Error: File '$targetFileName' does not exist."
65-
exit 1
66-
fi
67-
68-
if [[ ! -r "$targetFileName" ]]; then
69-
echo "Error: File '$targetFileName' is not readable."
70-
exit 1
71-
fi
72-
73-
rank_count=$(jq '. | length' ${targetFileName})
74-
75-
meta_agg_type="TwoLevelAggregationMetadata"
76-
77-
if grep -q '"ES_MDAgg_AggInfo"' "$targetFileName"; then
78-
meta_agg_type="SelectiveAggregationMetadata"
79-
fi
80-
81-
data_agg_type="EveryoneWrites"
82-
if grep -q '"InitAgg-ews"' "$targetFileName"; then
83-
data_agg_type="EveryoneWritesSerial"
84-
elif grep -q '"InitAgg-tls"' "$targetFileName"; then
85-
data_agg_type="TwoLevelShared"
86-
elif grep -q '"InitAgg-dsb"' "$targetFileName"; then
87-
data_agg_type="DataSizeBased"
88-
fi
89-
90-
steps=$(jq -r '.[0].ES.nCalls // empty' $targetFileName)
91-
92-
md_bytes=$(jq -r '.[0].transport_1.wbytes// empty' $targetFileName)
93-
94-
bytes=$(jq '[ .[] | .transport_0.wbytes | select(type == "number") ] | add // 0' "$targetFileName")
95-
96-
max_dur_mus=$(jq 'map([.ES_mus, .PDW_mus, .PP_mus] | map(select(type == "number")) | add // 0) | max // 0' "$targetFileName")
97-
min_dur_mus=$(jq 'map([.ES_mus, .PDW_mus, .PP_mus] | map(select(type == "number")) | add // 0) | min // 0' "$targetFileName")
98-
r0_dur_mus=$(jq 'map([.ES_mus, .PDW_mus, .PP_mus] | map(select(type == "number")) | add // 0) [0]' "$targetFileName")
99-
100-
min_dur_info=$(get_min $targetFileName)
101-
max_dur_info=$(get_max $targetFileName)
102-
103-
echo "======= High level Summary of ${targetFileName} ======="
104-
echo "Num Ranks: ${rank_count}. NumSteps: ${steps}"
105-
echo " ${data_agg_type} + ${meta_agg_type}"
106-
echo " t1 bytes: $(print_bytes ${md_bytes})"
107-
echo " t0 bytes: $(print_bytes ${bytes})"
108-
#print_bytes ${bytes}
109-
dur="[ES + PP]"
110-
if grep -q '"PDW"' "$targetFileName"; then
111-
dur="[ES + PDW]"
112-
fi
113-
echo " Max ${dur}: = $(print_time $(echo ${max_dur_info} | jq .sum)) at rank: $(echo ${max_dur_info} | jq .rank)"
114-
echo " Min ${dur}: = $(print_time $(echo ${min_dur_info} | jq .sum)) at rank: $(echo ${min_dur_info} | jq .rank)"
115-
116-
#echo " r0 Dur: = $(print_time ${r0_dur_mus})"
59+
targetFileName="$1"
60+
61+
# Check if the file exists and is readable
62+
if [[ ! -f "$targetFileName" ]]; then
63+
echo "Error: File '$targetFileName' does not exist."
64+
exit 1
65+
fi
66+
67+
if [[ ! -r "$targetFileName" ]]; then
68+
echo "Error: File '$targetFileName' is not readable."
69+
exit 1
70+
fi
71+
72+
rank_count=$(jq '. | length' "${targetFileName}")
73+
74+
meta_agg_type="TwoLevelAggregationMetadata"
75+
76+
if grep -q '"ES_MDAgg_AggInfo"' "$targetFileName"; then
77+
meta_agg_type="SelectiveAggregationMetadata"
78+
fi
79+
80+
data_agg_type="EveryoneWrites"
81+
if grep -q '"InitAgg-ews"' "$targetFileName"; then
82+
data_agg_type="EveryoneWritesSerial"
83+
elif grep -q '"InitAgg-tls"' "$targetFileName"; then
84+
data_agg_type="TwoLevelShared"
85+
elif grep -q '"InitAgg-dsb"' "$targetFileName"; then
86+
data_agg_type="DataSizeBased"
87+
fi
88+
89+
steps=$(jq -r '.[0].ES.nCalls // empty' "$targetFileName")
90+
91+
md_bytes=$(jq -r '.[0].transport_1.wbytes// empty' "$targetFileName")
92+
93+
bytes=$(jq '[ .[] | .transport_0.wbytes | select(type == "number") ] | add // 0' "$targetFileName")
94+
95+
#max_dur_mus=$(jq 'map([.ES_mus, .PDW_mus, .PP_mus] | map(select(type == "number")) | add // 0) | max // 0' "$targetFileName")
96+
#min_dur_mus=$(jq 'map([.ES_mus, .PDW_mus, .PP_mus] | map(select(type == "number")) | add // 0) | min // 0' "$targetFileName")
97+
#r0_dur_mus=$(jq 'map([.ES_mus, .PDW_mus, .PP_mus] | map(select(type == "number")) | add // 0) [0]' "$targetFileName")
98+
99+
min_dur_info=$(get_min "$targetFileName")
100+
max_dur_info=$(get_max "$targetFileName")
101+
102+
echo "======= High level Summary of ${targetFileName} ======="
103+
echo "Num Ranks: ${rank_count}. NumSteps: ${steps}"
104+
echo " ${data_agg_type} + ${meta_agg_type}"
105+
echo " t1 bytes: $(print_bytes "${md_bytes}")"
106+
echo " t0 bytes: $(print_bytes "${bytes}")"
107+
108+
dur="[ES + PP]"
109+
if grep -q '"PDW"' "$targetFileName"; then
110+
dur="[ES + PDW]"
111+
fi
112+
113+
#echo " Max ${dur}: = $(print_time $(echo "${max_dur_info}" | jq .sum)) at rank: $(echo "${max_dur_info}" | jq .rank)"
114+
max_sum=$(jq -r '.sum' <<<"$max_dur_info")
115+
max_rank=$(jq -r '.rank' <<<"$max_dur_info")
116+
printf ' Max %s: = %s at rank: %s\n' \
117+
"$dur" \
118+
"$(print_time "$max_sum")" \
119+
"$max_rank"
120+
121+
#echo " Min ${dur}: = $(print_time $(echo ${min_dur_info} | jq .sum)) at rank: $(echo ${min_dur_info} | jq .rank)"
122+
min_sum=$(jq -r '.sum' <<<"$min_dur_info")
123+
min_rank=$(jq -r '.rank' <<<"$min_dur_info")
124+
printf ' Min %s: = %s at rank: %s\n' \
125+
"$dur" \
126+
"$(print_time "$min_sum")" \
127+
"$min_rank"
128+
129+
#echo " r0 Dur: = $(print_time ${r0_dur_mus})"
117130
}
118131

119132
# main
120133
# Check if exactly one argument is provided
121134
if [[ $# -eq 0 ]]; then
122-
echo "Usage: $0 <path-to-json-file> [+]"
123-
echo "Example: $0 data.json"
124-
exit 1
135+
echo "Usage: $0 <path-to-json-file> [+]"
136+
echo "Example: $0 data.json"
137+
exit 1
125138
fi
126139

127140
for file in "$@"; do
128-
process_file $file
141+
process_file "$file"
129142
done

0 commit comments

Comments
 (0)