Merge pull request #213 from Shopify/emily/memory-bench

Memory comparisons between Prism and Sorbet

Author: egiurleo

prism_benchmarks/memory/data/parser/shopify/2024-08-22-0739d1462.txt

@@ -0,0 +1,49 @@
+	Command being timed: "bazel-bin/main/sorbet --parser=prism --stop-after=parser ../../Shopify/shopify"
+	User time (seconds): 308.82
+	System time (seconds): 10.04
+	Percent of CPU this job got: 113%
+	Elapsed (wall clock) time (h:mm:ss or m:ss): 4:40.27
+	Average shared text size (kbytes): 0
+	Average unshared data size (kbytes): 0
+	Average stack size (kbytes): 0
+	Average total size (kbytes): 0
+	Maximum resident set size (kbytes): 881064
+	Average resident set size (kbytes): 0
+	Major (requiring I/O) page faults: 0
+	Minor (reclaiming a frame) page faults: 226461
+	Voluntary context switches: 14702
+	Involuntary context switches: 80406
+	Swaps: 0
+	File system inputs: 0
+	File system outputs: 0
+	Socket messages sent: 0
+	Socket messages received: 0
+	Signals delivered: 0
+	Page size (bytes): 16384
+	Exit status: 0
+
+Command exited with non-zero status 100
+	Command being timed: "bazel-bin/main/sorbet --parser=sorbet --stop-after=parser ../../Shopify/shopify"
+	User time (seconds): 476.12
+	System time (seconds): 9.49
+	Percent of CPU this job got: 160%
+	Elapsed (wall clock) time (h:mm:ss or m:ss): 5:02.08
+	Average shared text size (kbytes): 0
+	Average unshared data size (kbytes): 0
+	Average stack size (kbytes): 0
+	Average total size (kbytes): 0
+	Maximum resident set size (kbytes): 1355112
+	Average resident set size (kbytes): 0
+	Major (requiring I/O) page faults: 0
+	Minor (reclaiming a frame) page faults: 343679
+	Voluntary context switches: 231
+	Involuntary context switches: 77417
+	Swaps: 0
+	File system inputs: 0
+	File system outputs: 0
+	Socket messages sent: 0
+	Socket messages received: 0
+	Signals delivered: 0
+	Page size (bytes): 16384
+	Exit status: 100
+

prism_benchmarks/memory/data/parser/yjit-bench/2024-08-22-0739d1462.txt

@@ -0,0 +1,48 @@
+	Command being timed: "bazel-bin/main/sorbet --parser=prism --stop-after=parser ../yjit-bench/benchmarks"
+	User time (seconds): 1.41
+	System time (seconds): 0.11
+	Percent of CPU this job got: 115%
+	Elapsed (wall clock) time (h:mm:ss or m:ss): 0:01.31
+	Average shared text size (kbytes): 0
+	Average unshared data size (kbytes): 0
+	Average stack size (kbytes): 0
+	Average total size (kbytes): 0
+	Maximum resident set size (kbytes): 137440
+	Average resident set size (kbytes): 0
+	Major (requiring I/O) page faults: 24
+	Minor (reclaiming a frame) page faults: 38722
+	Voluntary context switches: 406
+	Involuntary context switches: 2968
+	Swaps: 0
+	File system inputs: 0
+	File system outputs: 0
+	Socket messages sent: 0
+	Socket messages received: 0
+	Signals delivered: 0
+	Page size (bytes): 16384
+	Exit status: 0
+
+	Command being timed: "bazel-bin/main/sorbet --parser=sorbet --stop-after=parser ../yjit-bench/benchmarks"
+	User time (seconds): 4.07
+	System time (seconds): 0.08
+	Percent of CPU this job got: 198%
+	Elapsed (wall clock) time (h:mm:ss or m:ss): 0:02.09
+	Average shared text size (kbytes): 0
+	Average unshared data size (kbytes): 0
+	Average stack size (kbytes): 0
+	Average total size (kbytes): 0
+	Maximum resident set size (kbytes): 197452
+	Average resident set size (kbytes): 0
+	Major (requiring I/O) page faults: 0
+	Minor (reclaiming a frame) page faults: 51716
+	Voluntary context switches: 0
+	Involuntary context switches: 1299
+	Swaps: 0
+	File system inputs: 0
+	File system outputs: 0
+	Socket messages sent: 0
+	Socket messages received: 0
+	Signals delivered: 0
+	Page size (bytes): 16384
+	Exit status: 0
+

prism_benchmarks/memory/data/pipeline/2024-08-22-0739d1462.txt

@@ -0,0 +1,50 @@
+Command exited with non-zero status 100
+	Command being timed: "bazel-bin/main/sorbet --parser=prism test/prism_regression"
+	User time (seconds): 1.07
+	System time (seconds): 0.03
+	Percent of CPU this job got: 117%
+	Elapsed (wall clock) time (h:mm:ss or m:ss): 0:00.93
+	Average shared text size (kbytes): 0
+	Average unshared data size (kbytes): 0
+	Average stack size (kbytes): 0
+	Average total size (kbytes): 0
+	Maximum resident set size (kbytes): 64996
+	Average resident set size (kbytes): 0
+	Major (requiring I/O) page faults: 0
+	Minor (reclaiming a frame) page faults: 18361
+	Voluntary context switches: 0
+	Involuntary context switches: 893
+	Swaps: 0
+	File system inputs: 0
+	File system outputs: 0
+	Socket messages sent: 0
+	Socket messages received: 0
+	Signals delivered: 0
+	Page size (bytes): 16384
+	Exit status: 100
+
+Command exited with non-zero status 100
+	Command being timed: "bazel-bin/main/sorbet --parser=sorbet test/prism_regression"
+	User time (seconds): 1.06
+	System time (seconds): 0.03
+	Percent of CPU this job got: 119%
+	Elapsed (wall clock) time (h:mm:ss or m:ss): 0:00.91
+	Average shared text size (kbytes): 0
+	Average unshared data size (kbytes): 0
+	Average stack size (kbytes): 0
+	Average total size (kbytes): 0
+	Maximum resident set size (kbytes): 65492
+	Average resident set size (kbytes): 0
+	Major (requiring I/O) page faults: 0
+	Minor (reclaiming a frame) page faults: 18334
+	Voluntary context switches: 0
+	Involuntary context switches: 997
+	Swaps: 0
+	File system inputs: 0
+	File system outputs: 0
+	Socket messages sent: 0
+	Socket messages received: 0
+	Signals delivered: 0
+	Page size (bytes): 16384
+	Exit status: 100
+

prism_benchmarks/memory/run_benchmarks.sh

@@ -0,0 +1,64 @@
+#!/bin/bash
+
+# This script runs memory comparisons for the Prism parser in the Sorbet type checker.
+# It assumes the presence of a few directories and tools:
+# - The yjit-bench directory, which contains the benchmarks for YJIT (https://github.com/Shopify/yjit-bench)
+# - The Shopify directory, which contains the Shopify codebase (https://github.com/Shopify/shopify)
+# - gtime, a command-line time and memory analysis tool (https://www.gnu.org/software/time/)
+
+# ----- Setup -----
+
+YJIT_BENCH_DIR="../yjit-bench"
+SHOPIFY_DIR="../../Shopify/shopify"
+export SORBET_SILENCE_DEV_MESSAGE=1
+
+# Check if required directories exist
+for dir in "$YJIT_BENCH_DIR" "$SHOPIFY_DIR"; do
+    if [ ! -d "$dir" ]; then
+        echo "Please clone the required directories before running this script."
+        exit 1
+    fi
+done
+
+# Check and install gtime if necessary
+if ! command -v gtime &> /dev/null; then
+    echo "Installing gtime..."
+    if [[ "$OSTYPE" == "darwin"* ]]; then
+        brew install gnu-time
+    else
+        echo "Please install gtime: https://www.gnu.org/software/time/"
+        exit 1
+    fi
+fi
+
+output_file_name="$(date '+%Y-%m-%d')-$(git rev-parse --short HEAD).txt"
+parsers=("prism" "sorbet")
+
+# ----- Run Benchmarks -----
+
+run_benchmark() {
+    local title="$1"
+    local output_dir="$2"
+    local command="$3"
+
+    echo "#### $title ####"
+    output_file="prism_benchmarks/memory/data/$output_dir/${output_file_name}"
+
+    for parser in "${parsers[@]}"; do
+        gtime --verbose --output="$output_file" --append \
+            bazel-bin/main/sorbet --parser=$parser $command &> /dev/null
+        echo "" >> "$output_file"
+        # Calculate peak memory usage:
+        # - Search for lines that contain peak memory usage info
+        # - Take the last one
+        # - Extract the memory usage value
+        # - Convert it to MB
+        peak_memory=$(grep "Maximum resident set size (kbytes):" "$output_file" | tail -n 1 | awk '{print $6/1024 " MB"}')
+        echo "-> Peak Memory Usage with $parser parser: $peak_memory"
+    done
+    echo ""
+}
+
+run_benchmark "Memory Check 1: yjit-bench, parser only" "parser/yjit-bench" "--stop-after=parser $YJIT_BENCH_DIR/benchmarks"
+run_benchmark "Memory Check 2: shopify, parser only" "parser/shopify" "--stop-after=parser $SHOPIFY_DIR"
+run_benchmark "Memory Check 3: prism regression tests, whole pipeline" "pipeline" "test/prism_regression"

prism_benchmarks/data/parser/sorbet-tests/2024-07-18-f75db5824.json renamed to prism_benchmarks/time/data/parser/sorbet-tests/2024-07-18-f75db5824.json

@@ -34,15 +34,15 @@ echo "#### Benchmark 1: yjit-bench, parser only"
 # These benchmarks compare the performance of the Prism parser with the Sorbet parser without
 # any other parts of the type checker enabled.
 hyperfine \
-  --warmup=10 --export-json="prism_benchmarks/data/parser/yjit-bench/$file_name" --parameter-list parser sorbet,prism \
+  --warmup=10 --export-json="prism_benchmarks/time/data/parser/yjit-bench/$file_name" --parameter-list parser sorbet,prism \
   "bazel-bin/main/sorbet --parser={parser} --stop-after=parser ../yjit-bench/benchmarks"
 
 echo "#### Benchmark 2: sorbet-test, parser only"
 
 # Benchmark against sorbet tests to capture performance around parser errors
 # Pass the --ignore-failure flag to prevent hyperfine from exiting on parser errors
 hyperfine \
-  --warmup=10 --export-json="prism_benchmarks/data/parser/sorbet-tests/$file_name" --parameter-list parser sorbet,prism \
+  --warmup=10 --export-json="prism_benchmarks/time/data/parser/sorbet-tests/$file_name" --parameter-list parser sorbet,prism \
   "bazel-bin/main/sorbet --parser={parser} --stop-after=parser test/testdata" --ignore-failure
 
 echo "#### Benchmark 3: prism regression tests, whole pipeline"
@@ -52,5 +52,5 @@ echo "#### Benchmark 3: prism regression tests, whole pipeline"
 # the Prism parser enabled and the Sorbet parser enabled. These benchmarks must be run on a
 # smaller set of files that only contain nodes supported by the Prism --> Sorbet translation layer.
 hyperfine \
-  --warmup=10 --export-json="prism_benchmarks/data/pipeline/$file_name" --parameter-list parser sorbet,prism \
+  --warmup=10 --export-json="prism_benchmarks/time/data/pipeline/$file_name" --parameter-list parser sorbet,prism \
   "bazel-bin/main/sorbet --parser={parser} test/prism_regression"