Skip to content

Commit 1b998ce

Browse files
alexcu2718alexcu2718
authored
Merge pull request #19 from alexcu2718/benchmarking_fix
fix: improve shellscripts to be A LOT MORE MAINTAINABLE
2 parents 7246d41 + f491a75 commit 1b998ce

33 files changed

Lines changed: 213 additions & 955 deletions

‎fd_benchmarks/cold-cache-simple-pattern.sh‎

Lines changed: 2 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -1,46 +1,8 @@
11
#!/usr/bin/env bash
22

3-
4-
set -euo pipefail
5-
# TODO WRITE THIS MESS INTO SOME FUNCTIONS
6-
source "prelude.sh"
3+
# shellcheck disable=SC1091
74
source "new_prelude.sh"
85
ask_for_sudo
9-
OUTPUT_DIR="./bench_results"
10-
mkdir -p "$OUTPUT_DIR"
116

127
pattern="'.*[0-9]\.jpg$'"
13-
14-
COMMAND_FD="fd -HI $pattern '$SEARCH_ROOT'"
15-
COMMAND_FIND="fdf -HI $pattern '$SEARCH_ROOT'"
16-
17-
echo -e "\nRunning cold cache benchmarks..."
18-
hyperfine \
19-
--min-runs 3 \
20-
--prepare "$RESET_CACHES" \
21-
"$COMMAND_FIND" \
22-
"$COMMAND_FD" \
23-
--export-markdown "$OUTPUT_DIR/results-cold-cache-simple-pattern.md"
24-
25-
echo -e "\nAnalysing differences..."
26-
eval "$COMMAND_FD" | sort > "$OUTPUT_DIR/fd_cold_pattern.lst"
27-
eval "$COMMAND_FIND" | sort > "$OUTPUT_DIR/fdf_cold_pattern.lst"
28-
29-
diff -u "$OUTPUT_DIR/fd_cold_pattern.lst" "$OUTPUT_DIR/fdf_cold_pattern.lst" > "$OUTPUT_DIR/fd_diff_cold_pattern.md"
30-
31-
differences=$(comm -3 "$OUTPUT_DIR/fd_cold_pattern.lst" "$OUTPUT_DIR/fdf_cold_pattern.lst" | wc -l)
32-
echo "Total lines differing: $differences"
33-
34-
if [[ $differences -gt 0 ]]; then
35-
echo -e "\nFiles only in fd:"
36-
comm -23 "$OUTPUT_DIR/fd_cold_pattern.lst" "$OUTPUT_DIR/fdf_cold_pattern.lst"
37-
38-
echo -e "\nFiles only in fdf:"
39-
comm -13 "$OUTPUT_DIR/fd_cold_pattern.lst" "$OUTPUT_DIR/fdf_cold_pattern.lst"
40-
41-
else
42-
echo "No differences found in direct execution"
43-
fi
44-
45-
echo -e "\nBenchmark results saved to $OUTPUT_DIR/results-cold-cache-simple-pattern.md"
46-
echo "Diff results saved to $OUTPUT_DIR/fd_diff_cold_pattern.md"
8+
run_cold_benchmark "simple-pattern" "-HI $pattern '$SEARCH_ROOT'" "-HI $pattern '$SEARCH_ROOT'" "cold_pattern"

‎fd_benchmarks/config.sh‎

Lines changed: 0 additions & 11 deletions
This file was deleted.

‎fd_benchmarks/new_prelude.sh‎

Lines changed: 140 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,29 @@
22

33

44

5+
cd "$(dirname "$0")" || exit
6+
7+
8+
9+
ask_for_sudo() {
10+
echo "This script will now ask for your password in order to gain root/sudo"
11+
echo "permissions. These are required to reset the harddisk caches in between"
12+
echo "benchmark runs."
13+
echo ""
14+
15+
sudo echo "Okay, acquired superpowers :-)" || exit
16+
17+
echo ""
18+
}
19+
20+
21+
22+
export WARMUP_COUNT=5
23+
24+
# command for "cold cache" benchmarks
25+
export RESET_CACHES="sync; echo 3 | sudo tee /proc/sys/vm/drop_caches"
26+
27+
528
TMP_DIR="${TMP:-/tmp}"
629

730
llvm_link=https://github.com/llvm/llvm-project
@@ -10,12 +33,10 @@ LLVM="$TMP_DIR/llvm-project"
1033
fdf_location="$TMP_DIR/fdf_test"
1134

1235
fdf_repo=https://github.com/alexcu2718/fdf
13-
36+
# shellcheck disable=SC2034
1437
SEARCH_ROOT="$LLVM"
1538

16-
17-
18-
alias sort='sort --parallel=$(nproc)' #speed up sorting speed
39+
EXCLUDE='paru/clone/.*/pkg|systemd-private|fd.*\.lst$'
1940

2041

2142

@@ -29,7 +50,7 @@ else
2950
:
3051
fi
3152

32-
# Clone and build fdf if not already installed
53+
3354
if [ ! -e "$fdf_location" ]; then
3455
echo "Cloning fdf to $fdf_location..."
3556
git clone "$fdf_repo" "$fdf_location" >/dev/null
@@ -43,3 +64,117 @@ else
4364
fi
4465

4566
export PATH="$fdf_location/target/release:$PATH"
67+
68+
run_warm_benchmark() {
69+
local benchmark_name="$1"
70+
local fdf_args="$2"
71+
local fd_args="$3"
72+
local output_basename="${4:-$benchmark_name}"
73+
local skip_diff="${5:-0}"
74+
75+
OUTPUT_DIR="./bench_results"
76+
mkdir -p "$OUTPUT_DIR"
77+
78+
79+
local COMMAND_FIND="fdf $fdf_args"
80+
local COMMAND_FD="fd $fd_args"
81+
82+
83+
if [[ "$skip_diff" == "0" ]]; then
84+
# (filter out paru and systemd temporary files (these are protected files so these would be false results anyway))
85+
echo -e "\nGetting accurate file counts..."
86+
fd_count=$(eval "$COMMAND_FD" | grep -vcE "$EXCLUDE" )
87+
fdf_count=$(eval "$COMMAND_FIND" | grep -vcE "$EXCLUDE")
88+
echo "fd count: $fd_count"
89+
echo "fdf count: $fdf_count"
90+
fi
91+
92+
93+
echo -e "\nRunning benchmarks..."
94+
hyperfine \
95+
--warmup "$WARMUP_COUNT" \
96+
--prepare 'sync; sleep 0.2' \
97+
"$COMMAND_FIND" \
98+
"$COMMAND_FD" \
99+
--export-markdown "$OUTPUT_DIR/results-warm-cache-${benchmark_name}.md"
100+
101+
if [[ "$skip_diff" == "0" ]]; then
102+
# sorted output lists (filter out paru and systemd files)
103+
eval "$COMMAND_FD" | grep -vE "$EXCLUDE" | sort --parallel="$(nproc)" > "$OUTPUT_DIR/fd_${output_basename}.lst"
104+
eval "$COMMAND_FIND" | grep -vE "$EXCLUDE" | sort --parallel="$(nproc)" > "$OUTPUT_DIR/fdf_${output_basename}.lst"
105+
106+
diff -u "$OUTPUT_DIR/fd_${output_basename}.lst" "$OUTPUT_DIR/fdf_${output_basename}.lst" > "$OUTPUT_DIR/fd_diff_${output_basename}.md"
107+
108+
differences=$(comm -3 "$OUTPUT_DIR/fd_${output_basename}.lst" "$OUTPUT_DIR/fdf_${output_basename}.lst" | wc -l)
109+
echo "Total lines differing: $differences"
110+
111+
if [[ $differences -gt 0 ]]; then
112+
echo -e "\nFiles only in fd (showing first 10):"
113+
echo -e "\n\n\n THESE FILES ARE USUALLY EXCLUDED DUE PERMISSIONS (fd usually being at /usr/bin/fd )"
114+
comm -23 "$OUTPUT_DIR/fd_${output_basename}.lst" "$OUTPUT_DIR/fdf_${output_basename}.lst" | head -n 10
115+
116+
echo -e "\nFiles only in fdf (showing first 10):"
117+
comm -13 "$OUTPUT_DIR/fd_${output_basename}.lst" "$OUTPUT_DIR/fdf_${output_basename}.lst" | head -n 10
118+
119+
echo -e "\nFull diff available in: $(realpath $OUTPUT_DIR/fd_diff_"${output_basename}".md)"
120+
else
121+
echo "No differences found in direct execution"
122+
fi
123+
124+
# Report results locations
125+
echo -e "\nBenchmark results saved to $OUTPUT_DIR/results-warm-cache-${benchmark_name}.md"
126+
echo "Diff results saved to $OUTPUT_DIR/fd_diff_${output_basename}.md"
127+
else
128+
echo -e "\nBenchmark results saved to $OUTPUT_DIR/results-warm-cache-${benchmark_name}.md"
129+
fi
130+
}
131+
132+
133+
run_cold_benchmark() {
134+
local benchmark_name="$1"
135+
local fdf_args="$2"
136+
local fd_args="$3"
137+
local output_basename="${4:-$benchmark_name}"
138+
local min_runs="${5:-3}"
139+
140+
OUTPUT_DIR="./bench_results"
141+
mkdir -p "$OUTPUT_DIR"
142+
143+
local COMMAND_FIND="fdf $fdf_args"
144+
local COMMAND_FD="fd $fd_args"
145+
146+
echo -e "\nRunning cold cache benchmarks..."
147+
hyperfine \
148+
--min-runs "$min_runs" \
149+
--prepare "$RESET_CACHES" \
150+
"$COMMAND_FIND" \
151+
"$COMMAND_FD" \
152+
--export-markdown "$OUTPUT_DIR/results-cold-cache-${benchmark_name}.md"
153+
154+
155+
156+
eval "$COMMAND_FD" | grep -vE "$EXCLUDE" | sort --parallel="$(nproc)" > "$OUTPUT_DIR/fd_${output_basename}.lst"
157+
eval "$COMMAND_FIND" | grep -vE "$EXCLUDE" | sort --parallel="$(nproc)" > "$OUTPUT_DIR/fdf_${output_basename}.lst"
158+
159+
diff -u "$OUTPUT_DIR/fd_${output_basename}.lst" "$OUTPUT_DIR/fdf_${output_basename}.lst" > "$OUTPUT_DIR/fd_diff_${output_basename}.md"
160+
161+
162+
differences=$(comm -3 "$OUTPUT_DIR/fd_${output_basename}.lst" "$OUTPUT_DIR/fdf_${output_basename}.lst" | wc -l)
163+
echo "Total lines differing: $differences"
164+
165+
if [[ $differences -gt 0 ]]; then
166+
echo -e "\nFiles only in fd (showing first 10):"
167+
comm -23 "$OUTPUT_DIR/fd_${output_basename}.lst" "$OUTPUT_DIR/fdf_${output_basename}.lst" | head -n 10
168+
169+
echo -e "\nFiles only in fdf (showing first 10):"
170+
comm -13 "$OUTPUT_DIR/fd_${output_basename}.lst" "$OUTPUT_DIR/fdf_${output_basename}.lst" | head -n 10
171+
172+
echo -e "\nFull diff available in: $(realpath $OUTPUT_DIR/fd_diff_"${output_basename}".md)"
173+
else
174+
echo "No differences found in direct execution"
175+
fi
176+
177+
178+
echo -e "\nBenchmark results saved to $OUTPUT_DIR/results-cold-cache-${benchmark_name}.md"
179+
echo "Diff results saved to $OUTPUT_DIR/fd_diff_${output_basename}.md"
180+
}

‎fd_benchmarks/prelude.sh‎

Lines changed: 0 additions & 18 deletions
This file was deleted.

‎fd_benchmarks/run_all_tests_USE_ME.sh‎

Lines changed: 7 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -57,17 +57,14 @@ fi
5757
echo -e "\n\nTHERE WILL BE A SMALL DISPARITY IN THESE TESTS DUE TO fd being located in /usr/bin (USUALLY) ((different permissions!))\n, differences are expected to be very small";
5858
echo -e "there will also be predictable temporary files created!\n\n"
5959
echo "these tests will take a while!"
60-
read -rp 'Do you want to run speed/correctness benchmarks for home directory? [y/n]:' run_benchmarks_home
6160

62-
if [[ "$run_benchmarks_home" =~ ^[Yy]$ ]]; then
63-
for script in ./warm*_home_dir.sh; do
64-
echo "Running $script"
65-
./"$script"
66-
sleep 2
67-
done
68-
else
69-
echo "Skipping benchmarks for home directory."
70-
fi
61+
62+
for script in ./warm*_home_dir.sh; do
63+
echo "Running $script"
64+
./"$script"
65+
sleep 2
66+
done
67+
7168

7269

7370
if [[ "$(uname -s)" == "Linux" ]]; then

‎fd_benchmarks/syscalltest.sh‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
#!/usr/bin/env bash
22

33

4+
# shellcheck disable=SC1091
45
set -euo pipefail
56
source "new_prelude.sh"
67
FDF="fdf"

‎fd_benchmarks/warm-cache-depth-test.sh‎

Lines changed: 2 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -1,53 +1,9 @@
11
#!/usr/bin/env bash
22

33

4-
source "prelude.sh"
4+
# shellcheck disable=SC1091
55
source "new_prelude.sh"
66

7-
#i dont use gitignore so -HI is equivalent on both tools
8-
OUTPUT_DIR="./bench_results"
9-
mkdir -p "$OUTPUT_DIR"
10-
117
DEPTH_LIMIT=2
12-
COMMAND_FIND="fdf '.' '$SEARCH_ROOT' -HI -d $DEPTH_LIMIT"
13-
COMMAND_FD="fd '.' '$SEARCH_ROOT' -HI -d $DEPTH_LIMIT"
14-
15-
# First get accurate baseline counts
16-
echo -e "\nGetting accurate file counts..."
17-
fd_count=$(eval "$COMMAND_FD" | wc -l)
18-
fdf_count=$(eval "$COMMAND_FIND" | wc -l)
19-
echo "fd count: $fd_count"
20-
echo "fdf count: $fdf_count"
21-
228
echo -e "\nRunning depth-limited benchmarks (depth=$DEPTH_LIMIT)..."
23-
hyperfine \
24-
--warmup "$WARMUP_COUNT" \
25-
--prepare 'sync; sleep 0.2' \
26-
"$COMMAND_FIND" \
27-
"$COMMAND_FD" \
28-
--export-markdown "$OUTPUT_DIR/results-warm-cache-depth-test.md"
29-
30-
eval "$COMMAND_FD" | sort > "$OUTPUT_DIR/fd_depth.lst"
31-
eval "$COMMAND_FIND" | sort > "$OUTPUT_DIR/fdf_depth.lst"
32-
33-
diff -u "$OUTPUT_DIR/fd_depth.lst" "$OUTPUT_DIR/fdf_depth.lst" > "$OUTPUT_DIR/fd_diff_depth.md"
34-
35-
differences=$(comm -3 "$OUTPUT_DIR/fd_depth.lst" "$OUTPUT_DIR/fdf_depth.lst" | wc -l)
36-
echo "Total files found by fd: $(wc -l < "$OUTPUT_DIR/fd_depth.lst")"
37-
echo "Total files found by fdf: $(wc -l < "$OUTPUT_DIR/fdf_depth.lst")"
38-
echo "Total files differing: $differences"
39-
40-
if [[ $differences -gt 0 ]]; then
41-
echo -e "\nFiles only in fd:"
42-
comm -23 "$OUTPUT_DIR/fd_depth.lst" "$OUTPUT_DIR/fdf_depth.lst"
43-
44-
echo -e "\nFiles only in fdf:"
45-
comm -13 "$OUTPUT_DIR/fd_depth.lst" "$OUTPUT_DIR/fdf_depth.lst"
46-
47-
48-
else
49-
echo "No differences found in direct execution"
50-
fi
51-
52-
echo -e "\nBenchmark results saved to $OUTPUT_DIR/results-warm-cache-depth-test.md"
53-
echo "Diff results saved to $OUTPUT_DIR/fd_diff_depth.md"
9+
run_warm_benchmark "depth-test" "'.' '$SEARCH_ROOT' -HI -d $DEPTH_LIMIT" "'.' '$SEARCH_ROOT' -HI -d $DEPTH_LIMIT" "depth"

‎fd_benchmarks/warm-cache-depth-test_home_dir.sh‎

Lines changed: 2 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -1,54 +1,8 @@
11
#!/usr/bin/env bash
22

33

4-
5-
6-
7-
source "prelude.sh"
4+
# shellcheck disable=SC1091
85
source "new_prelude.sh"
9-
#i dont use gitignore so -HI is equivalent on both tools
10-
OUTPUT_DIR="./bench_results"
11-
mkdir -p "$OUTPUT_DIR"
126

137
DEPTH_LIMIT=4
14-
COMMAND_FIND="fdf '.' '$HOME' -HI -d $DEPTH_LIMIT"
15-
COMMAND_FD="fd '.' '$HOME' -HI -d $DEPTH_LIMIT"
16-
17-
echo -e "\nGetting accurate file counts..."
18-
fd_count=$(eval "$COMMAND_FD" | wc -l)
19-
fdf_count=$(eval "$COMMAND_FIND" | wc -l)
20-
echo "fd count: $fd_count"
21-
echo "fdf count: $fdf_count"
22-
23-
echo -e "\nRunning depth-limited benchmarks (depth=$DEPTH_LIMIT)..."
24-
hyperfine \
25-
--warmup "$WARMUP_COUNT" \
26-
--prepare 'sync; sleep 0.2' \
27-
"$COMMAND_FIND" \
28-
"$COMMAND_FD" \
29-
--export-markdown "$OUTPUT_DIR/results-warm-cache-depth-test_home_dir.md"
30-
31-
eval "$COMMAND_FD" | sort > "$OUTPUT_DIR/fd_depth_home_dir.lst"
32-
eval "$COMMAND_FIND" | sort > "$OUTPUT_DIR/fdf_depth_home_dir.lst"
33-
34-
diff -u "$OUTPUT_DIR/fd_depth_home_dir.lst" "$OUTPUT_DIR/fdf_depth_home_dir.lst" > "$OUTPUT_DIR/fd_diff_depth_home_dir.md"
35-
36-
differences=$(comm -3 "$OUTPUT_DIR/fd_depth_home_dir.lst" "$OUTPUT_DIR/fdf_depth_home_dir.lst" | wc -l)
37-
echo "Total files found by fd: $(wc -l < "$OUTPUT_DIR/fd_depth_home_dir.lst")"
38-
echo "Total files found by fdf: $(wc -l < "$OUTPUT_DIR/fdf_depth_home_dir.lst")"
39-
echo "Total files differing: $differences"
40-
41-
if [[ $differences -gt 0 ]]; then
42-
echo -e "\nFiles only in fd:"
43-
comm -23 "$OUTPUT_DIR/fd_depth_home_dir.lst" "$OUTPUT_DIR/fdf_depth_home_dir.lst"
44-
45-
echo -e "\nFiles only in fdf:"
46-
comm -13 "$OUTPUT_DIR/fd_depth_home_dir.lst" "$OUTPUT_DIR/fdf_depth_home_dir.lst"
47-
48-
49-
else
50-
echo "No differences found in direct execution"
51-
fi
52-
53-
echo -e "\nBenchmark results saved to $OUTPUT_DIR/results-warm-cache-depth-test_home_dir.md"
54-
echo "Diff results saved to $OUTPUT_DIR/fd_diff_depth_home_dir.md"
8+
run_warm_benchmark "depth-test_home_dir" "'.' '$HOME' -HI -d $DEPTH_LIMIT" "'.' '$HOME' -HI -d $DEPTH_LIMIT" "depth_home_dir"

0 commit comments

Comments
 (0)