diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index b8ced981..0ff3b358 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -238,6 +238,63 @@ cargo test -- --nocapture - Integration tests: none currently - Doc tests: in doc comments, tested with `cargo test --doc` +### Example Testing Framework + +All examples in the `examples/` directory follow a standardized test framework: + +**Directory Structure:** +``` +examples/ + example_name/ + main.zr # The example source code + Makefile # Standard Makefile (see below) + test/ + stdout.txt # Expected stdout output + stderr.txt # Expected stderr output (optional) + exitcode.txt # Expected exit code (optional, defaults to 0) + args.txt # Command-line arguments (optional) + stdin.txt # Standard input (optional) +``` + +**Standard Makefile Test Target:** + +All examples MUST use this exact test implementation: + +```makefile +.PHONY: test +test: build + set +e; \ + if [ -f test/args.txt ]; then args=$$(xargs < test/args.txt); else args=""; fi; \ + if [ -f test/stdin.txt ]; then stdin_file=test/stdin.txt; else stdin_file=/dev/null; fi; \ + ./$(OUTDIR)/run $$args < $$stdin_file > test/stdout.actual 2> test/stderr.actual; \ + if [ -f test/exitcode.txt ]; then expected_exitcode=$$(cat test/exitcode.txt); else expected_exitcode=0; fi; \ + exitcode=$$?; \ + status=0; \ + if [ $$exitcode -ne $$expected_exitcode ]; then \ + echo "Expected exit code $$expected_exitcode but got $$exitcode"; \ + status=1; \ + fi; \ + if [ -f test/stdout.txt ]; then \ + diff -u test/stdout.txt test/stdout.actual || { echo "stdout mismatch"; status=1; }; \ + fi; \ + if [ -f test/stderr.txt ]; then \ + diff -u test/stderr.txt test/stderr.actual || { echo "stderr mismatch"; status=1; }; \ + fi; \ + set -e; \ + rm test/stdout.actual test/stderr.actual; \ + exit $$status +``` + +**How it works:** +1. Runs the compiled example with optional arguments from `test/args.txt` +2. Provides optional stdin from `test/stdin.txt` (defaults to /dev/null) +3. Captures stdout to `test/stdout.actual` and stderr to `test/stderr.actual` +4. Compares actual output with expected files using `diff -u` +5. Checks exit code matches `test/exitcode.txt` (defaults to 0) +6. Cleans up temporary files and reports status + +**Do NOT create custom test implementations** - always use the standard framework above. Reference examples: `hello_world`, `fibonacci`, `struct_example`. + ## Common Issues and Workarounds ### LLVM-related Build Failures diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml new file mode 100644 index 00000000..79e4b3d7 --- /dev/null +++ b/.github/workflows/benchmark.yml @@ -0,0 +1,168 @@ +name: "benchmark" + +on: + pull_request: + workflow_dispatch: + +permissions: + contents: write + pull-requests: write + +jobs: + benchmark: + runs-on: ubuntu-latest + + steps: + - name: Checkout base branch + uses: actions/checkout@v5 + with: + ref: ${{ github.base_ref || 'main' }} + + - name: Set up Rust + uses: actions-rs/toolchain@v1 + with: + toolchain: stable + override: true + + - name: Install LLVM dependencies + run: | + sudo apt-get update -qq + sudo apt-get install -y llvm-16 llvm-16-dev libpolly-16-dev + + - name: Use dependency cache + uses: Swatinem/rust-cache@v2 + + - name: Run benchmarks on base branch + run: | + cargo bench --bench compilation -- --save-baseline base + + - name: Checkout PR branch + uses: actions/checkout@v5 + with: + ref: ${{ github.head_ref }} + repository: ${{github.event.pull_request.head.repo.full_name || github.repository }} + clean: false + + - name: Run benchmarks on PR branch + run: | + cargo bench --bench compilation -- --baseline base 2>&1 | tee benchmark_output.txt + + - name: Generate comparison report + run: | + echo "# šŸ“Š Benchmark Comparison Report" > benchmark_report.md + echo "" >> benchmark_report.md + echo "Comparing performance of PR against base branch (\`${{ github.base_ref || 'main' }}\`)" >> benchmark_report.md + echo "" >> benchmark_report.md + + # Check if we have any benchmark output + if [ -s benchmark_output.txt ]; then + # Extract benchmark results in diff format + echo "## Results" >> benchmark_report.md + echo "" >> benchmark_report.md + echo '```diff' >> benchmark_report.md + + # Process each benchmark - extract clean results in diff format + # Only highlight changes >= 5% + awk ' + /^[a-z_]+[[:space:]]+time:/ { + bench_name = $1; + # Extract the median time (middle value) + match($0, /\[[0-9.]+[[:space:]][Āµnm]?s[[:space:]]+([0-9.]+[[:space:]][Āµnm]?s)[[:space:]]+[0-9.]+[[:space:]][Āµnm]?s\]/, time_arr); + median_time = time_arr[1]; + # Read next line for change + getline; + if ($0 ~ /change:/) { + match($0, /\[[+-][0-9.]+%[[:space:]]+([+-][0-9.]+%)[[:space:]]+[+-][0-9.]+%\]/, change_arr); + median_change = change_arr[1]; + + # Extract numeric value from change percentage + match(median_change, /[+-]([0-9.]+)%/, num_arr); + change_value = num_arr[1] + 0; # Convert to number + + # Only highlight if change >= 5% + prefix = " "; + if (change_value >= 5.0) { + if (median_change ~ /^[+]/) { + prefix = "-"; # Regression (slower) - shows in red + } else if (median_change ~ /^[-]/) { + prefix = "+"; # Improvement (faster) - shows in green + } + } + + # Format: diff-style with change percentage + printf("%s %-25s %12s (%s)\n", prefix, bench_name, median_time, median_change); + } + } + ' benchmark_output.txt >> benchmark_report.md + + echo '```' >> benchmark_report.md + echo "" >> benchmark_report.md + + # Calculate summary statistics + improvements=$(grep -c "change:.*\[-[0-9]" benchmark_output.txt 2>/dev/null || echo "0") + regressions=$(grep -c "change:.*\[+[0-9]" benchmark_output.txt 2>/dev/null || echo "0") + total_benches=$(grep -c "^[a-z_][a-z_]*[[:space:]]*time:" benchmark_output.txt 2>/dev/null || echo "0") + + echo "### Summary" >> benchmark_report.md + echo "" >> benchmark_report.md + echo "- **Total benchmarks:** $total_benches" >> benchmark_report.md + echo "- **Improvements (faster):** $improvements šŸš€" >> benchmark_report.md + echo "- **Regressions (slower):** $regressions šŸ“‰" >> benchmark_report.md + echo "" >> benchmark_report.md + echo "> **Note:** Only changes ā‰„5% are highlighted in the diff. Smaller changes are shown but not color-coded, as they often represent normal variance." >> benchmark_report.md + else + echo "āŒ No benchmark output captured. The benchmark run may have failed." >> benchmark_report.md + fi + + echo "" >> benchmark_report.md + echo "---" >> benchmark_report.md + echo "" >> benchmark_report.md + echo "šŸ“„ **[Download Full Results & HTML Report](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}#artifacts)** - Click to view detailed criterion reports with charts" >> benchmark_report.md + + - name: Comment PR with benchmark results + if: github.event_name == 'pull_request' + uses: actions/github-script@v7 + with: + script: | + const fs = require('fs'); + const report = fs.readFileSync('benchmark_report.md', 'utf8'); + + // Find existing benchmark comment + const comments = await github.rest.issues.listComments({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + }); + + const botComment = comments.data.find(comment => + comment.user.type === 'Bot' && + comment.body.includes('Benchmark Comparison Report') + ); + + if (botComment) { + // Update existing comment + await github.rest.issues.updateComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: botComment.id, + body: report + }); + } else { + // Create new comment + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + body: report + }); + } + + - name: Upload benchmark results + uses: actions/upload-artifact@v4 + with: + name: benchmark-results + path: | + benchmark_report.md + benchmark_output.txt + target/criterion/ + retention-days: 30 diff --git a/Cargo.lock b/Cargo.lock index 9525ac4f..6650ade1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -20,6 +20,12 @@ dependencies = [ "libc", ] +[[package]] +name = "anes" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" + [[package]] name = "anstream" version = "0.6.8" @@ -147,6 +153,12 @@ version = "3.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f30e7476521f6f8af1a1c4c0b8cc94f0bee37d91763d0ca2665f299b6cd8aec" +[[package]] +name = "cast" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" + [[package]] name = "cc" version = "1.0.83" @@ -163,6 +175,33 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "ciborium" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" +dependencies = [ + "ciborium-io", + "ciborium-ll", + "serde", +] + +[[package]] +name = "ciborium-io" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" + +[[package]] +name = "ciborium-ll" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" +dependencies = [ + "ciborium-io", + "half", +] + [[package]] name = "clap" version = "4.5.28" @@ -262,6 +301,73 @@ dependencies = [ "libc", ] +[[package]] +name = "criterion" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2b12d017a929603d80db1831cd3a24082f8137ce19c69e6447f54f5fc8d692f" +dependencies = [ + "anes", + "cast", + "ciborium", + "clap", + "criterion-plot", + "is-terminal", + "itertools 0.10.5", + "num-traits", + "once_cell", + "oorandom", + "plotters", + "rayon", + "regex", + "serde", + "serde_derive", + "serde_json", + "tinytemplate", + "walkdir", +] + +[[package]] +name = "criterion-plot" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1" +dependencies = [ + "cast", + "itertools 0.10.5", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + [[package]] name = "crypto-common" version = "0.1.6" @@ -383,6 +489,16 @@ dependencies = [ "url", ] +[[package]] +name = "half" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "459196ed295495a68f7d7fe1d84f6c4b7ff0e21fe3017b2f283c6fac3ad803c9" +dependencies = [ + "cfg-if", + "crunchy", +] + [[package]] name = "hashbrown" version = "0.15.1" @@ -395,6 +511,12 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" +[[package]] +name = "hermit-abi" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" + [[package]] name = "home" version = "0.5.9" @@ -489,12 +611,32 @@ dependencies = [ "similar", ] +[[package]] +name = "is-terminal" +version = "0.4.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e04d7f318608d35d4b61ddd75cbdaee86b023ebe2bd5a66ee0915f0bf93095a9" +dependencies = [ + "hermit-abi", + "libc", + "windows-sys 0.52.0", +] + [[package]] name = "is_debug" version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06d198e9919d9822d5f7083ba8530e04de87841eaf21ead9af8f2304efd57c89" +[[package]] +name = "itertools" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +dependencies = [ + "either", +] + [[package]] name = "itertools" version = "0.14.0" @@ -521,10 +663,11 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.64" +version = "0.3.81" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5f195fe497f702db0f318b07fdd68edb16955aed830df8363d837542f8f935a" +checksum = "ec48937a97411dcb524a265206ccd4c90bb711fca92b2792c407f268825b9305" dependencies = [ + "once_cell", "wasm-bindgen", ] @@ -546,7 +689,7 @@ dependencies = [ "ascii-canvas", "bit-set", "ena", - "itertools", + "itertools 0.14.0", "lalrpop-util", "petgraph", "pico-args", @@ -711,6 +854,15 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + [[package]] name = "num_threads" version = "0.1.6" @@ -726,6 +878,12 @@ version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" +[[package]] +name = "oorandom" +version = "11.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" + [[package]] name = "parking_lot" version = "0.12.1" @@ -786,6 +944,34 @@ version = "0.3.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964" +[[package]] +name = "plotters" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aeb6f403d7a4911efb1e33402027fc44f29b5bf6def3effcc22d7bb75f2b747" +dependencies = [ + "num-traits", + "plotters-backend", + "plotters-svg", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "plotters-backend" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df42e13c12958a16b3f7f4386b9ab1f3e7933914ecea48da7139435263a4172a" + +[[package]] +name = "plotters-svg" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51bae2ac328883f7acdfea3d66a7c35751187f870bc81f94563733a154d7a670" +dependencies = [ + "plotters-backend", +] + [[package]] name = "powerfmt" version = "0.2.0" @@ -816,6 +1002,26 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "rayon" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "368f01d005bf8fd9b1206fb6fa653e6c4a81ceb1466406b81792d87c5677a58f" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + [[package]] name = "redox_syscall" version = "0.4.1" @@ -869,6 +1075,12 @@ version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ffc183a10b4478d04cbbbfc96d0873219d962dd5accaff2ffbd4ceb7df837f4" +[[package]] +name = "ryu" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" + [[package]] name = "same-file" version = "1.0.6" @@ -910,6 +1122,17 @@ dependencies = [ "syn", ] +[[package]] +name = "serde_json" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb0652c533506ad7a2e353cce269330d6afd8bdfb6d75e0ace5b35aacbd7b9e9" +dependencies = [ + "itoa", + "ryu", + "serde", +] + [[package]] name = "sha3" version = "0.10.8" @@ -1044,6 +1267,16 @@ dependencies = [ "time-core", ] +[[package]] +name = "tinytemplate" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" +dependencies = [ + "serde", + "serde_json", +] + [[package]] name = "tinyvec" version = "1.6.0" @@ -1168,23 +1401,25 @@ dependencies = [ [[package]] name = "wasm-bindgen" -version = "0.2.87" +version = "0.2.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7706a72ab36d8cb1f80ffbf0e071533974a60d0a308d01a5d0375bf60499a342" +checksum = "c1da10c01ae9f1ae40cbfac0bac3b1e724b320abfcf52229f80b547c0d250e2d" dependencies = [ "cfg-if", + "once_cell", + "rustversion", "wasm-bindgen-macro", + "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-backend" -version = "0.2.87" +version = "0.2.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ef2b6d3c510e9625e5fe6f509ab07d66a760f0885d858736483c32ed7809abd" +checksum = "671c9a5a66f49d8a47345ab942e2cb93c7d1d0339065d4f8139c486121b43b19" dependencies = [ "bumpalo", "log", - "once_cell", "proc-macro2", "quote", "syn", @@ -1193,9 +1428,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.87" +version = "0.2.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dee495e55982a3bd48105a7b947fd2a9b4a8ae3010041b9e0faab3f9cd028f1d" +checksum = "7ca60477e4c59f5f2986c50191cd972e3a50d8a95603bc9434501cf156a9a119" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -1203,9 +1438,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.87" +version = "0.2.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b" +checksum = "9f07d2f20d4da7b26400c9f4a0511e6e0345b040694e8a75bd41d578fa4421d7" dependencies = [ "proc-macro2", "quote", @@ -1216,9 +1451,22 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.87" +version = "0.2.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bad67dc8b2a1a6e5448428adec4c3e84c43e561d8c9ee8a9e5aabeb193ec41d1" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "web-sys" +version = "0.3.81" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca6ad05a4870b2bf5fe995117d3728437bd27d7cd5f06f13c17443ef369775a1" +checksum = "9367c417a924a74cae129e6a2ae3b47fabb1f8995595ab474029da749a8be120" +dependencies = [ + "js-sys", + "wasm-bindgen", +] [[package]] name = "winapi" @@ -1477,6 +1725,7 @@ version = "0.1.0" dependencies = [ "anyhow", "clap", + "criterion", "derive_more", "mimalloc", "shadow-rs", diff --git a/README.md b/README.md index 66c3f01a..1a74517b 100644 --- a/README.md +++ b/README.md @@ -77,6 +77,18 @@ You can compile a single Zirco file to a `.o` object with `zrc --emit object -o For more usage help, refer to `zrc --help`. +## Performance Testing + +The compiler includes comprehensive performance benchmarks to measure and track compilation performance. The benchmarks cover parsing, type checking, code generation, and end-to-end compilation. + +To run the benchmarks: + +```bash +cargo bench --bench compilation +``` + +Benchmark results are saved to `target/criterion/` with detailed HTML reports and statistical analysis. See the [zrc README](./compiler/zrc/README.md#performance-benchmarks) for more details on the benchmarking infrastructure. + ## A Note on Stability So that Zirco can continue to evolve at a rapid pace, there are **NO STABILITY GUARENTEES** on the current version of Zirco and `zrc`. diff --git a/compiler/zrc/Cargo.toml b/compiler/zrc/Cargo.toml index 2b01ae8e..40c07b74 100644 --- a/compiler/zrc/Cargo.toml +++ b/compiler/zrc/Cargo.toml @@ -17,5 +17,12 @@ shadow-rs = "0.36.0" derive_more = { version = "2.0.1", features = ["display"] } mimalloc = "0.1.48" +[dev-dependencies] +criterion = { version = "0.5", features = ["html_reports"] } + [build-dependencies] shadow-rs = "0.36.0" + +[[bench]] +name = "compilation" +harness = false diff --git a/compiler/zrc/README.md b/compiler/zrc/README.md index 43f868af..4ce74aaa 100644 --- a/compiler/zrc/README.md +++ b/compiler/zrc/README.md @@ -4,3 +4,93 @@ This crate serves as the frontend and binary for `zrc`, the official compiler fo programming language. When running `zrc`, you simply invoke it with one argument, which is the file to compile. + +## Performance Benchmarks + +This crate includes comprehensive performance benchmarks for measuring the compiler's performance +across different compilation stages. The benchmarks use [Criterion.rs](https://github.com/bheisler/criterion.rs) +to provide detailed statistical analysis. + +### Running Benchmarks + +To run all benchmarks: + +```bash +cargo bench --bench compilation +``` + +To run specific benchmark groups: + +```bash +# Only parsing benchmarks +cargo bench --bench compilation parsing + +# Only type checking benchmarks +cargo bench --bench compilation typechecking + +# Only code generation benchmarks +cargo bench --bench compilation codegen + +# Only end-to-end benchmarks +cargo bench --bench compilation end_to_end +``` + +To run a specific benchmark: + +```bash +cargo bench --bench compilation parse_simple +``` + +### Benchmark Categories + +The benchmark suite includes **22 benchmarks** organized into four categories: + +1. **Parsing** (6 benchmarks) - Measures lexing and parsing performance + - `parse_simple` - Simple hello world program + - `parse_fibonacci` - Recursive fibonacci function + - `parse_struct` - Struct definition and usage + - `parse_pointer` - Pointer operations and swap + - `parse_globals` - Global variables and constants + - `parse_loop` - For loop implementation + +2. **Type Checking** (6 benchmarks) - Measures type checking performance + - `typeck_simple` - Simple hello world program + - `typeck_fibonacci` - Recursive fibonacci function + - `typeck_struct` - Struct definition and usage + - `typeck_pointer` - Pointer operations and swap + - `typeck_globals` - Global variables and constants + - `typeck_loop` - For loop implementation + +3. **Code Generation** (6 benchmarks) - Measures LLVM IR generation performance + - `codegen_simple` - Simple hello world program + - `codegen_fibonacci` - Recursive fibonacci function + - `codegen_struct` - Struct definition and usage + - `codegen_pointer` - Pointer operations and swap + - `codegen_globals` - Global variables and constants + - `codegen_loop` - For loop implementation + +4. **End-to-End** (4 benchmarks) - Measures complete compilation pipeline performance + - `e2e_simple` - Simple hello world program (unoptimized) + - `e2e_fibonacci` - Recursive fibonacci function (unoptimized) + - `e2e_struct` - Struct definition and usage (unoptimized) + - `e2e_fibonacci_optimized` - Fibonacci with aggressive optimization + +### Benchmark Results + +Benchmark results are saved to `target/criterion/` and include: +- HTML reports with detailed statistics and charts +- Comparison with previous runs to detect performance regressions +- Statistical analysis including mean, median, and standard deviation + +Open `target/criterion/report/index.html` in a browser to view detailed results. + +### Automated Performance Tracking + +A GitHub Actions workflow automatically runs benchmarks on all pull requests to detect performance changes: + +- Compares PR performance against the base branch +- Posts results as a comment on the PR +- Archives full benchmark results as artifacts +- Helps identify performance regressions before merging + +The workflow can also be triggered manually from the Actions tab. diff --git a/compiler/zrc/benches/compilation.rs b/compiler/zrc/benches/compilation.rs new file mode 100644 index 00000000..b779c8c5 --- /dev/null +++ b/compiler/zrc/benches/compilation.rs @@ -0,0 +1,527 @@ +//! Compilation performance benchmarks for the Zirco compiler. +//! +//! This benchmark suite measures the performance of various compilation stages: +//! - Lexing and parsing +//! - Type checking +//! - Code generation +//! - End-to-end compilation +//! +//! Run with: `cargo bench --bench compilation` + +use criterion::{black_box, criterion_group, criterion_main, Criterion}; +use zrc_codegen::{DebugLevel, OptimizationLevel}; + +/// Sample Zirco code for benchmarking - simple hello world +const SIMPLE_CODE: &str = r#" +fn printf(format: *u8, ...) -> i32; + +fn main() { + printf("Hello, World!\n"); +} +"#; + +/// Sample Zirco code for benchmarking - fibonacci (recursive) +const FIBONACCI_CODE: &str = r#" +fn printf(format: *u8, ...) -> i32; + +fn fibonacci(n: i32) -> i32 { + if (n <= 1) { + return n; + } + return fibonacci(n - 1) + fibonacci(n - 2); +} + +fn main() { + let i = 0; + while (i < 10) { + printf("fibonacci(%d) = %d\n", i, fibonacci(i)); + i = i + 1; + } +} +"#; + +/// Sample Zirco code for benchmarking - struct example +const STRUCT_CODE: &str = r#" +fn printf(format: *u8, ...) -> i32; + +struct Point { + x: i32, + y: i32 +} + +fn distance_squared(p1: Point, p2: Point) -> i32 { + let dx = p2.x - p1.x; + let dy = p2.y - p1.y; + return dx * dx + dy * dy; +} + +fn main() { + let p1: Point; + p1.x = 0; + p1.y = 0; + + let p2: Point; + p2.x = 3; + p2.y = 4; + + let dist_sq = distance_squared(p1, p2); + printf("Distance squared: %d\n", dist_sq); +} +"#; + +/// Sample Zirco code for benchmarking - pointer operations +const POINTER_CODE: &str = r#" +fn printf(format: *u8, ...) -> i32; + +fn swap(a: *i32, b: *i32) { + let temp = *a; + *a = *b; + *b = temp; +} + +fn main() { + let x = 42; + let y = 17; + + printf("Before swap: x=%d, y=%d\n", x, y); + swap(&x, &y); + printf("After swap: x=%d, y=%d\n", x, y); +} +"#; + +/// Sample Zirco code for benchmarking - global variables +const GLOBAL_VARS_CODE: &str = r#" +fn printf(format: *u8, ...) -> i32; + +// Global constants with initializers +let MAX_COUNT: i32 = 100; +let ENABLED: bool = true; +let VERSION: i8 = 1i8; + +// Global variable without initializer (zero-initialized) +let counter: i32; + +fn increment_counter() { + counter = counter + 1; +} + +fn get_counter() -> i32 { + return counter; +} + +fn is_below_max() -> bool { + return counter < MAX_COUNT; +} + +fn main() { + printf("Initial counter: %d\n", counter); + printf("Max count: %d\n", MAX_COUNT); + printf("Version: %d\n", VERSION); + printf("Enabled: %d\n", ENABLED); + + increment_counter(); + increment_counter(); + increment_counter(); + + printf("After 3 increments: %d\n", get_counter()); + + if (is_below_max()) { + printf("Still below maximum\n"); + } +} +"#; + +/// Sample Zirco code for benchmarking - loops +const LOOP_CODE: &str = r#" +fn printf(format: *u8, ...) -> i32; + +fn print_numbers(n: i32) { + for (let i = 1; i <= n; i = i + 1) { + printf("%d\n", i); + } +} + +fn main() { + print_numbers(5); +} +"#; + +/// Benchmark parsing performance on simple code +fn bench_parse_simple(c: &mut Criterion) { + c.bench_function("parse_simple", |b| { + b.iter(|| { + let _ = zrc_parser::parser::parse_program(black_box(SIMPLE_CODE)); + }); + }); +} + +/// Benchmark parsing performance on fibonacci code +fn bench_parse_fibonacci(c: &mut Criterion) { + c.bench_function("parse_fibonacci", |b| { + b.iter(|| { + let _ = zrc_parser::parser::parse_program(black_box(FIBONACCI_CODE)); + }); + }); +} + +/// Benchmark parsing performance on struct code +fn bench_parse_struct(c: &mut Criterion) { + c.bench_function("parse_struct", |b| { + b.iter(|| { + let _ = zrc_parser::parser::parse_program(black_box(STRUCT_CODE)); + }); + }); +} + +/// Benchmark parsing performance on pointer operations +fn bench_parse_pointer(c: &mut Criterion) { + c.bench_function("parse_pointer", |b| { + b.iter(|| { + let _ = zrc_parser::parser::parse_program(black_box(POINTER_CODE)); + }); + }); +} + +/// Benchmark parsing performance on global variables +fn bench_parse_globals(c: &mut Criterion) { + c.bench_function("parse_globals", |b| { + b.iter(|| { + let _ = zrc_parser::parser::parse_program(black_box(GLOBAL_VARS_CODE)); + }); + }); +} + +/// Benchmark parsing performance on loops +fn bench_parse_loop(c: &mut Criterion) { + c.bench_function("parse_loop", |b| { + b.iter(|| { + let _ = zrc_parser::parser::parse_program(black_box(LOOP_CODE)); + }); + }); +} + +/// Benchmark type checking performance on simple code +fn bench_typeck_simple(c: &mut Criterion) { + let ast = zrc_parser::parser::parse_program(SIMPLE_CODE).unwrap(); + c.bench_function("typeck_simple", |b| { + b.iter(|| { + let _ = zrc_typeck::typeck::type_program(black_box(ast.clone())); + }); + }); +} + +/// Benchmark type checking performance on fibonacci code +fn bench_typeck_fibonacci(c: &mut Criterion) { + let ast = zrc_parser::parser::parse_program(FIBONACCI_CODE).unwrap(); + c.bench_function("typeck_fibonacci", |b| { + b.iter(|| { + let _ = zrc_typeck::typeck::type_program(black_box(ast.clone())); + }); + }); +} + +/// Benchmark type checking performance on struct code +fn bench_typeck_struct(c: &mut Criterion) { + let ast = zrc_parser::parser::parse_program(STRUCT_CODE).unwrap(); + c.bench_function("typeck_struct", |b| { + b.iter(|| { + let _ = zrc_typeck::typeck::type_program(black_box(ast.clone())); + }); + }); +} + +/// Benchmark type checking performance on pointer operations +fn bench_typeck_pointer(c: &mut Criterion) { + let ast = zrc_parser::parser::parse_program(POINTER_CODE).unwrap(); + c.bench_function("typeck_pointer", |b| { + b.iter(|| { + let _ = zrc_typeck::typeck::type_program(black_box(ast.clone())); + }); + }); +} + +/// Benchmark type checking performance on global variables +fn bench_typeck_globals(c: &mut Criterion) { + let ast = zrc_parser::parser::parse_program(GLOBAL_VARS_CODE).unwrap(); + c.bench_function("typeck_globals", |b| { + b.iter(|| { + let _ = zrc_typeck::typeck::type_program(black_box(ast.clone())); + }); + }); +} + +/// Benchmark type checking performance on loops +fn bench_typeck_loop(c: &mut Criterion) { + let ast = zrc_parser::parser::parse_program(LOOP_CODE).unwrap(); + c.bench_function("typeck_loop", |b| { + b.iter(|| { + let _ = zrc_typeck::typeck::type_program(black_box(ast.clone())); + }); + }); +} + +/// Benchmark code generation performance on simple code +fn bench_codegen_simple(c: &mut Criterion) { + let ast = zrc_parser::parser::parse_program(SIMPLE_CODE).unwrap(); + let typed_ast = zrc_typeck::typeck::type_program(ast).unwrap(); + + c.bench_function("codegen_simple", |b| { + b.iter(|| { + let _ = zrc_codegen::cg_program_to_string( + black_box("bench"), + black_box("."), + black_box("bench.zr"), + black_box("zrc bench.zr"), + black_box(SIMPLE_CODE), + black_box(typed_ast.clone()), + OptimizationLevel::None, + DebugLevel::None, + &zrc_codegen::get_native_triple(), + black_box("generic"), + ); + }); + }); +} + +/// Benchmark code generation performance on fibonacci code +fn bench_codegen_fibonacci(c: &mut Criterion) { + let ast = zrc_parser::parser::parse_program(FIBONACCI_CODE).unwrap(); + let typed_ast = zrc_typeck::typeck::type_program(ast).unwrap(); + + c.bench_function("codegen_fibonacci", |b| { + b.iter(|| { + let _ = zrc_codegen::cg_program_to_string( + black_box("bench"), + black_box("."), + black_box("bench.zr"), + black_box("zrc bench.zr"), + black_box(FIBONACCI_CODE), + black_box(typed_ast.clone()), + OptimizationLevel::None, + DebugLevel::None, + &zrc_codegen::get_native_triple(), + black_box("generic"), + ); + }); + }); +} + +/// Benchmark code generation performance on struct code +fn bench_codegen_struct(c: &mut Criterion) { + let ast = zrc_parser::parser::parse_program(STRUCT_CODE).unwrap(); + let typed_ast = zrc_typeck::typeck::type_program(ast).unwrap(); + + c.bench_function("codegen_struct", |b| { + b.iter(|| { + let _ = zrc_codegen::cg_program_to_string( + black_box("bench"), + black_box("."), + black_box("bench.zr"), + black_box("zrc bench.zr"), + black_box(STRUCT_CODE), + black_box(typed_ast.clone()), + OptimizationLevel::None, + DebugLevel::None, + &zrc_codegen::get_native_triple(), + black_box("generic"), + ); + }); + }); +} + +/// Benchmark code generation performance on pointer operations +fn bench_codegen_pointer(c: &mut Criterion) { + let ast = zrc_parser::parser::parse_program(POINTER_CODE).unwrap(); + let typed_ast = zrc_typeck::typeck::type_program(ast).unwrap(); + + c.bench_function("codegen_pointer", |b| { + b.iter(|| { + let _ = zrc_codegen::cg_program_to_string( + black_box("bench"), + black_box("."), + black_box("bench.zr"), + black_box("zrc bench.zr"), + black_box(POINTER_CODE), + black_box(typed_ast.clone()), + OptimizationLevel::None, + DebugLevel::None, + &zrc_codegen::get_native_triple(), + black_box("generic"), + ); + }); + }); +} + +/// Benchmark code generation performance on global variables +fn bench_codegen_globals(c: &mut Criterion) { + let ast = zrc_parser::parser::parse_program(GLOBAL_VARS_CODE).unwrap(); + let typed_ast = zrc_typeck::typeck::type_program(ast).unwrap(); + + c.bench_function("codegen_globals", |b| { + b.iter(|| { + let _ = zrc_codegen::cg_program_to_string( + black_box("bench"), + black_box("."), + black_box("bench.zr"), + black_box("zrc bench.zr"), + black_box(GLOBAL_VARS_CODE), + black_box(typed_ast.clone()), + OptimizationLevel::None, + DebugLevel::None, + &zrc_codegen::get_native_triple(), + black_box("generic"), + ); + }); + }); +} + +/// Benchmark code generation performance on loops +fn bench_codegen_loop(c: &mut Criterion) { + let ast = zrc_parser::parser::parse_program(LOOP_CODE).unwrap(); + let typed_ast = zrc_typeck::typeck::type_program(ast).unwrap(); + + c.bench_function("codegen_loop", |b| { + b.iter(|| { + let _ = zrc_codegen::cg_program_to_string( + black_box("bench"), + black_box("."), + black_box("bench.zr"), + black_box("zrc bench.zr"), + black_box(LOOP_CODE), + black_box(typed_ast.clone()), + OptimizationLevel::None, + DebugLevel::None, + &zrc_codegen::get_native_triple(), + black_box("generic"), + ); + }); + }); +} + +/// Benchmark end-to-end compilation on simple code +fn bench_e2e_simple(c: &mut Criterion) { + c.bench_function("e2e_simple", |b| { + b.iter(|| { + let ast = zrc_parser::parser::parse_program(black_box(SIMPLE_CODE)).unwrap(); + let typed_ast = zrc_typeck::typeck::type_program(ast).unwrap(); + let _ = zrc_codegen::cg_program_to_string( + black_box("bench"), + black_box("."), + black_box("bench.zr"), + black_box("zrc bench.zr"), + black_box(SIMPLE_CODE), + typed_ast, + OptimizationLevel::None, + DebugLevel::None, + &zrc_codegen::get_native_triple(), + black_box("generic"), + ); + }); + }); +} + +/// Benchmark end-to-end compilation on fibonacci code +fn bench_e2e_fibonacci(c: &mut Criterion) { + c.bench_function("e2e_fibonacci", |b| { + b.iter(|| { + let ast = zrc_parser::parser::parse_program(black_box(FIBONACCI_CODE)).unwrap(); + let typed_ast = zrc_typeck::typeck::type_program(ast).unwrap(); + let _ = zrc_codegen::cg_program_to_string( + black_box("bench"), + black_box("."), + black_box("bench.zr"), + black_box("zrc bench.zr"), + black_box(FIBONACCI_CODE), + typed_ast, + OptimizationLevel::None, + DebugLevel::None, + &zrc_codegen::get_native_triple(), + black_box("generic"), + ); + }); + }); +} + +/// Benchmark end-to-end compilation on struct code +fn bench_e2e_struct(c: &mut Criterion) { + c.bench_function("e2e_struct", |b| { + b.iter(|| { + let ast = zrc_parser::parser::parse_program(black_box(STRUCT_CODE)).unwrap(); + let typed_ast = zrc_typeck::typeck::type_program(ast).unwrap(); + let _ = zrc_codegen::cg_program_to_string( + black_box("bench"), + black_box("."), + black_box("bench.zr"), + black_box("zrc bench.zr"), + black_box(STRUCT_CODE), + typed_ast, + OptimizationLevel::None, + DebugLevel::None, + &zrc_codegen::get_native_triple(), + black_box("generic"), + ); + }); + }); +} + +/// Benchmark end-to-end compilation with optimization +fn bench_e2e_optimized(c: &mut Criterion) { + c.bench_function("e2e_fibonacci_optimized", |b| { + b.iter(|| { + let ast = zrc_parser::parser::parse_program(black_box(FIBONACCI_CODE)).unwrap(); + let typed_ast = zrc_typeck::typeck::type_program(ast).unwrap(); + let _ = zrc_codegen::cg_program_to_string( + black_box("bench"), + black_box("."), + black_box("bench.zr"), + black_box("zrc bench.zr"), + black_box(FIBONACCI_CODE), + typed_ast, + OptimizationLevel::Aggressive, + DebugLevel::None, + &zrc_codegen::get_native_triple(), + black_box("generic"), + ); + }); + }); +} + +criterion_group!( + parsing, + bench_parse_simple, + bench_parse_fibonacci, + bench_parse_struct, + bench_parse_pointer, + bench_parse_globals, + bench_parse_loop +); + +criterion_group!( + typechecking, + bench_typeck_simple, + bench_typeck_fibonacci, + bench_typeck_struct, + bench_typeck_pointer, + bench_typeck_globals, + bench_typeck_loop +); + +criterion_group!( + codegen, + bench_codegen_simple, + bench_codegen_fibonacci, + bench_codegen_struct, + bench_codegen_pointer, + bench_codegen_globals, + bench_codegen_loop +); + +criterion_group!( + end_to_end, + bench_e2e_simple, + bench_e2e_fibonacci, + bench_e2e_struct, + bench_e2e_optimized +); + +criterion_main!(parsing, typechecking, codegen, end_to_end); diff --git a/compiler/zrc/src/main.rs b/compiler/zrc/src/main.rs index 26af514f..6b182496 100644 --- a/compiler/zrc/src/main.rs +++ b/compiler/zrc/src/main.rs @@ -49,6 +49,8 @@ clippy::module_name_repetitions, clippy::doc_comment_double_space_linebreaks )] +// criterion is only used in benchmarks, not in the main binary or tests +#![cfg_attr(test, allow(unused_crate_dependencies))] use mimalloc::MiMalloc; /// Use the mimalloc allocator as the global allocator, as LLVM is heavy on heap diff --git a/examples/struct_construction/Makefile b/examples/struct_construction/Makefile index 25e8c9f0..9f9d6927 100644 --- a/examples/struct_construction/Makefile +++ b/examples/struct_construction/Makefile @@ -20,20 +20,25 @@ build: $(ZR_OUTPUTS) clean: rm -rf $(OUTDIR) -# Test that the example builds and runs correctly .PHONY: test test: build - @echo "Testing struct construction example..." - @./$(OUTDIR)/run > $(OUTDIR)/output.txt 2>&1 - @if grep -q "=== Struct Construction Examples ===" $(OUTDIR)/output.txt && \ - grep -q "Point { x: 0, y: 0 }" $(OUTDIR)/output.txt && \ - grep -q "Color { r: 255, g: 0, b: 0 }" $(OUTDIR)/output.txt && \ - grep -q "Distance squared" $(OUTDIR)/output.txt; then \ - echo "āœ“ Test passed: Example runs correctly"; \ - cat $(OUTDIR)/output.txt; \ - exit 0; \ - else \ - echo "āœ— Test failed: Expected output not found"; \ - cat $(OUTDIR)/output.txt; \ - exit 1; \ - fi + set +e; \ + if [ -f test/args.txt ]; then args=$$(xargs < test/args.txt); else args=""; fi; \ + if [ -f test/stdin.txt ]; then stdin_file=test/stdin.txt; else stdin_file=/dev/null; fi; \ + ./$(OUTDIR)/run $$args < $$stdin_file > test/stdout.actual 2> test/stderr.actual; \ + if [ -f test/exitcode.txt ]; then expected_exitcode=$$(cat test/exitcode.txt); else expected_exitcode=0; fi; \ + exitcode=$$?; \ + status=0; \ + if [ $$exitcode -ne $$expected_exitcode ]; then \ + echo "Expected exit code $$expected_exitcode but got $$exitcode"; \ + status=1; \ + fi; \ + if [ -f test/stdout.txt ]; then \ + diff -u test/stdout.txt test/stdout.actual || { echo "stdout mismatch"; status=1; }; \ + fi; \ + if [ -f test/stderr.txt ]; then \ + diff -u test/stderr.txt test/stderr.actual || { echo "stderr mismatch"; status=1; }; \ + fi; \ + set -e; \ + rm test/stdout.actual test/stderr.actual; \ + exit $$status