[WIP] fib benchmark

Makefile

@@ -11,6 +11,13 @@ $(CAIRO_VM_CLI):
 # NOTE: This is super flaky, DO NOT move this section below the non proof mode one
 # or things will go wrong.
 
+BENCH_DIR=cairo_programs/benchmarks
+BENCH_FILES:=$(wildcard $(BENCH_DIR)/*.cairo)
+COMPILED_BENCHES:=$(patsubst $(BENCH_DIR)/%.cairo, $(BENCH_DIR)/%.json, $(BENCH_FILES))
+
+$(BENCH_DIR)/%.json: $(BENCH_DIR)/%.cairo
+	cairo-compile --cairo_path="$(BENCH_DIR)" $< --output $@ --proof_mode
+
 TEST_PROOF_DIR=cairo_programs/proof_programs
 TEST_PROOF_FILES:=$(wildcard $(TEST_PROOF_DIR)/*.cairo)
 COMPILED_PROOF_TESTS:=$(patsubst $(TEST_PROOF_DIR)/%.cairo, $(TEST_PROOF_DIR)/%.json, $(TEST_PROOF_FILES))
@@ -83,7 +90,7 @@ build:
 	@cp pkg/lambdaworks/lib/lambdaworks/target/release/liblambdaworks.a pkg/lambdaworks/lib
 	@cd pkg/starknet_crypto/lib/starknet_crypto && cargo build --release
 	@cp pkg/starknet_crypto/lib/starknet_crypto/target/release/libstarknet_crypto.a pkg/starknet_crypto/lib
-	@go build ./...
+	@go build -ldflags "-w -s" ./...
 
 fmt:
 	gofmt -w pkg
@@ -95,6 +102,10 @@ clean:
 	rm -f $(TEST_DIR)/*.json
 	rm -f $(TEST_DIR)/*.memory
 	rm -f $(TEST_DIR)/*.trace
+	rm -f $(TEST_PROOF_DIR)/*.json
+	rm -f $(TEST_PROOF_DIR)/*.memory
+	rm -f $(TEST_PROOF_DIR)/*.trace
+	rm -f $(BENCH_DIR)/*.json
 	cd pkg/lambdaworks/lib/lambdaworks && cargo clean
 	rm -f pkg/lambdaworks/lib/liblambdaworks.a
 	cd pkg/starknet_crypto/lib/starknet_crypto && cargo clean
@@ -104,9 +115,12 @@ clean:
 
 clean_files:
 	rm -f $(TEST_DIR)/*.json
-	rm -f $(TEST_DIR)/proof_programs/*.json
 	rm -f $(TEST_DIR)/*.memory
 	rm -f $(TEST_DIR)/*.trace
+	rm -f $(TEST_PROOF_DIR)/*.json
+	rm -f $(TEST_PROOF_DIR)/*.memory
+	rm -f $(TEST_PROOF_DIR)/*.trace
+	rm -f $(BENCH_DIR)/*.json
 
 demo_fibonacci: clean_files build_cairo_vm_cli build
 	@echo "Compiling fibonacci program..."
@@ -172,3 +186,9 @@ compare_proof_memory: build_cairo_vm_cli $(CAIRO_RS_PROOF_MEM) $(CAIRO_GO_PROOF_
 
 clean_trace_and_memory_files:
 	rm -f $(TEST_DIR)/*.rs.* && rm -f $(TEST_DIR)/*.go.* && rm -f $(TEST_PROOF_DIR)/*.rs.* && rm -f $(TEST_PROOF_DIR)/*.go.*
+
+compare_benchmarks: $(COMPILED_BENCHES) build
+	sh scripts/run_benchmarks.sh
+
+compare_benchmarks_light: $(COMPILED_BENCHES) build
+	sh scripts/run_benchmarks_light.sh

cairo_programs/benchmarks/fibonacci_100k.cairo

@@ -0,0 +1,14 @@
+func main() {
+    fib(1, 1, 100000);
+
+    ret;
+}
+
+func fib(first_element, second_element, n) -> (res: felt) {
+    if (n == 0) {
+        return (second_element,);
+    }
+
+    tempvar y = first_element + second_element;
+    return fib(second_element, y, n - 1);
+}

cairo_programs/benchmarks/fibonacci_10K.cairo

@@ -0,0 +1,14 @@
+func main() {
+    fib(1, 1, 10000);
+
+    ret;
+}
+
+func fib(first_element, second_element, n) -> (res: felt) {
+    if (n == 0) {
+        return (second_element,);
+    }
+
+    tempvar y = first_element + second_element;
+    return fib(second_element, y, n - 1);
+}

cairo_programs/benchmarks/fibonacci_500k.cairo

@@ -0,0 +1,14 @@
+func main() {
+    fib(1, 1, 500000);
+
+    ret;
+}
+
+func fib(first_element, second_element, n) -> (res: felt) {
+    if (n == 0) {
+        return (second_element,);
+    }
+
+    tempvar y = first_element + second_element;
+    return fib(second_element, y, n - 1);
+}

cairo_programs/benchmarks/fibonacci_50k.cairo

@@ -0,0 +1,14 @@
+func main() {
+    fib(1, 1, 50000);
+
+    ret;
+}
+
+func fib(first_element, second_element, n) -> (res: felt) {
+    if (n == 0) {
+        return (second_element,);
+    }
+
+    tempvar y = first_element + second_element;
+    return fib(second_element, y, n - 1);
+}

scripts/run_benchmarks.sh

@@ -0,0 +1,16 @@
+#!/usr/bin/env sh
+tests_path="cairo_programs/benchmarks"
+
+set -e
+
+for file in $(ls $tests_path | grep .cairo | sed -E 's/\.cairo//'); do
+    echo "Running $file benchmark"
+
+    export PATH="$(pyenv root)/shims:$PATH"
+
+    hyperfine -w 5 \
+        -n "Cairo VM (CPython)" "PYENV_VERSION=3.9.15 cairo-run --proof_mode --memory_file /dev/null --trace_file /dev/null --layout starknet_with_keccak --program $tests_path/$file.json" \
+	    -n "Cairo VM (PyPy)" "PYENV_VERSION=pypy3.9-7.3.9 cairo-run --proof_mode --memory_file /dev/null --trace_file /dev/null --layout starknet_with_keccak --program $tests_path/$file.json" \
+	    -n "cairo-vm (Rust)" "cairo-vm/target/release/cairo-vm-cli $tests_path/$file.json --proof_mode --memory_file /dev/null --trace_file /dev/null --layout starknet_with_keccak" \
+        -n "cairo-vm.go (Go)"  "go run cmd/cli/main.go $tests_path/$file.json --proof_mode --memory_file /dev/null --trace_file /dev/null --layout starknet_with_keccak" 
+done

scripts/run_benchmarks_light.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env sh
+tests_path="cairo_programs/benchmarks"
+
+set -e
+
+for file in $(ls $tests_path | grep .cairo | sed -E 's/\.cairo//'); do
+    echo "Running $file benchmark"
+
+    export PATH="$(pyenv root)/shims:$PATH"
+
+    hyperfine -w 5 \
+	    -n "cairo-vm (Rust)" "cairo-vm/target/release/cairo-vm-cli $tests_path/$file.json --proof_mode --memory_file /dev/null --trace_file /dev/null --layout starknet_with_keccak" \
+        -n "cairo-vm.go (Go)"  "go run cmd/cli/main.go $tests_path/$file.json --proof_mode --memory_file /dev/null --trace_file /dev/null --layout starknet_with_keccak" 
+done