feat: switch to new preprocessor (#1031)

# Rationale for this change

Move to a new Yul preprocessor so Yul imports and snippets are handled
cleanly (rename to .presl), and to consolidate preprocessing logic under
versioned, testable Python code. The commit stream shows “add new Yul
preprocessor,” then renames and import fixes to align on the new
extension.

# What changes are included in this PR?

- New preprocessor module & tests
Adds solidity/preprocessor/yul_preprocessor.py with pytest tests and a
corpus of .presl fixtures covering imports, circular graphs, caching,
function dedup, etc.
- File/extension migration
Renames .pre.sol → .presl; updates Solidity imports and config
accordingly. Commits: “rename .pre.sol files to .presl”, “update imports
and config”, plus “fix Yul imports.”
- Build/Foundry config
foundry.toml now ignores .presl for match/coverage/doc, replacing the
old .pre.sol ignores.
- CI workflow
Adds a “Check Yul Preprocessor” job that sets up Python, runs black and
pytest for solidity/preprocessor. In the main lint/test job, it removes
preprocessor/test_files before coverage to avoid double counting.
- Helper scripts & misc
Adds solidity/scripts/pre_forge.sh and preprocess_yul_imports.sh;
touches Slither config and several .sol files to match the new import
pathing.

# Are these changes tested?
Yes!
Dedicated pytest suite runs in CI for the preprocessor. All existing
tests also pass.

---------

Co-authored-by: Claude <[email protected]>

Author: iajoiner

.github/workflows/lint-and-test.yml

@@ -197,6 +197,10 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.x'
       - name: Install Foundry
         uses: foundry-rs/foundry-toolchain@v1
         with:
@@ -211,6 +215,8 @@ jobs:
         run: solidity/scripts/pre_forge.sh test --optimize
       - name: Run tests with via-ir and optimization
         run: solidity/scripts/pre_forge.sh test --via-ir --optimize
+      - name: Remove preprocessor test files (for they are already tested in the check-yul-preprocessor job)
+        run: rm -rf solidity/preprocessor/test_files
       - name: Install lcov
         run: sudo apt-get update && sudo apt-get install -y lcov
       - name: Check code coverage
@@ -245,6 +251,10 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.x'
       - name: Install Foundry
         uses: foundry-rs/foundry-toolchain@v1
         with:

solidity/foundry.toml

@@ -4,11 +4,12 @@ out = "out"
 solc = "0.8.28"
 libs = ["dependencies"]
 deny-warnings = true
-no-match-path = "**/*.pre.sol"
-no-match-coverage = "pre.sol"
+no-match-path = "**/*.presl"
+no-match-coverage = "presl"
+ignored_paths = ["**/*.presl"]
 
 [doc]
-ignore = ["**/*.pre.sol"]
+ignore = ["**/*.presl"]
 
 [dependencies]
 forge-std = "1.9.5"

solidity/scripts/README.md

@@ -2,6 +2,7 @@
 set -euo pipefail
 SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
 cd $SCRIPT_DIR/..
-scripts/preprocess_yul_imports.sh src
-scripts/preprocess_yul_imports.sh test
+# Remove any existing .post.sol files
+find . -type f -name '*.post.sol' -delete
+python3 preprocessor/yul_preprocessor.py .
 forge "$@"

solidity/scripts/pre_forge.sh

@@ -28,10 +28,7 @@ library Array {
     /// @return __value The element at the specified index
     function __getArrayElement(uint256[][1] memory __array, uint256 __index) internal pure returns (uint256 __value) {
         assembly {
-            // IMPORT-YUL Errors.sol
-            function err(code) {
-                revert(0, 0)
-            }
+            // import err from Errors.sol
             function get_array_element(arr_ptr, index) -> value {
                 let arr := mload(arr_ptr)
                 let length := mload(arr)
@@ -62,10 +59,7 @@ library Array {
         returns (uint256[2][] memory __array)
     {
         assembly {
-            // IMPORT-YUL Errors.sol
-            function err(code) {
-                revert(0, 0)
-            }
+            // import err from Errors.sol
             function read_word_array_as_uint512_array(input_array_ptr) -> array_ptr {
                 array_ptr := mload(FREE_PTR)
 
@@ -126,10 +120,7 @@ library Array {
         returns (uint256 __upper, uint256 __lower)
     {
         assembly {
-            // IMPORT-YUL Errors.sol
-            function err(code) {
-                revert(0, 0)
-            }
+            // import err from Errors.sol
             function get_uint512_array_element(arr_ptr, index) -> upper, lower {
                 let arr := mload(arr_ptr)
                 let length := mload(arr)

solidity/scripts/preprocess_yul_imports.sh

@@ -3,7 +3,7 @@
 pragma solidity ^0.8.28;
 
 import "./Constants.sol";
-import "./SwitchUtil.pre.sol";
+import "./SwitchUtil.presl";
 
 /// @title DataType
 /// @dev Library providing parsing utilities for different data types
@@ -29,14 +29,8 @@ library DataType {
         returns (bytes calldata __exprOut, uint256 __entry)
     {
         assembly {
-            // IMPORT-YUL Errors.sol
-            function err(code) {
-                revert(0, 0)
-            }
-            // IMPORT-YUL SwitchUtil.pre.sol
-            function case_const(lhs, rhs) {
-                revert(0, 0)
-            }
+            // import err from Errors.sol
+            // import case_const from SwitchUtil.presl
             function read_binary(result_ptr) -> result_ptr_out, entry {
                 let free_ptr := mload(FREE_PTR)
                 let len := shr(UINT64_PADDING_BITS, calldataload(result_ptr))
@@ -84,7 +78,7 @@ library DataType {
                 result_ptr_out := add(result_ptr, len)
             }
 
-            // slither-disable-next-line cyclomatic-complexity
+            // slither-disable-start cyclomatic-complexity
             function read_entry(result_ptr, data_type_variant) -> result_ptr_out, entry {
                 result_ptr_out := result_ptr
                 switch data_type_variant
@@ -152,6 +146,7 @@ library DataType {
                 }
                 default { err(ERR_UNSUPPORTED_DATA_TYPE_VARIANT) }
             }
+            // slither-disable-end cyclomatic-complexity
             let __exprOutOffset
             __exprOutOffset, __entry := read_entry(__expr.offset, __dataTypeVariant)
             __exprOut.offset := __exprOutOffset
@@ -179,14 +174,8 @@ library DataType {
         returns (bytes calldata __exprOut, uint32 __dataType)
     {
         assembly {
-            // IMPORT-YUL Errors.sol
-            function err(code) {
-                revert(0, 0)
-            }
-            // IMPORT-YUL SwitchUtil.pre.sol
-            function case_const(lhs, rhs) {
-                revert(0, 0)
-            }
+            // import err from Errors.sol
+            // import case_const from SwitchUtil.presl
             function read_data_type(ptr) -> ptr_out, data_type {
                 data_type := shr(UINT32_PADDING_BITS, calldataload(ptr))
                 ptr_out := add(ptr, UINT32_SIZE)