chore: use llvm as submodule (#306)

Author: antonbaliasnikov

.github/workflows/ccache-regen.yaml

@@ -61,6 +61,24 @@ jobs:
     steps:
       - name: Checkout source
         uses: actions/checkout@v4
+        with:
+          submodules: 'recursive'
+
+      # This step is required to checkout submodules
+      # that are disabled in .gitmodules config
+      - name: Checkout submodules
+        run: |
+          git config --global --add safe.directory '*'
+          git submodule update --force --depth=1 --recursive --checkout
+
+      - name: Checkout host LLVM for musl build
+        uses: actions/checkout@v4
+        if: contains(matrix.target, 'musl')
+        with:
+          repository: 'llvm/llvm-project'
+          path: 'llvm-host'
+          ref: 'llvmorg-17.0.6'
+          fetch-depth: '1'
 
       - name: Prepare Windows env
         if: runner.os == 'Windows'
@@ -69,6 +87,7 @@ jobs:
       - name: Build LLVM for tests
         uses: matter-labs/era-compiler-ci/.github/actions/build-llvm@v1
         with:
+          clone-llvm: 'false'
           target-env: ${{ contains(matrix.target, 'musl') && 'musl' || 'gnu' }}
           enable-assertions: 'true'
           build-type: RelWithDebInfo

.github/workflows/check-tooling.yaml

@@ -33,11 +33,20 @@ jobs:
       - name: Checkout source
         uses: actions/checkout@v4
         with:
+          submodules: 'recursive'
           ref: ${{ inputs.zksolc-version || '' }}
 
+      # This step is required to checkout submodules
+      # that are disabled in .gitmodules config
+      - name: Checkout submodules
+        run: |
+          git config --global --add safe.directory '*'
+          git submodule update --force --depth=1 --recursive --checkout
+
       - name: Build LLVM
         uses: matter-labs/era-compiler-ci/.github/actions/build-llvm@v1
         with:
+          clone-llvm: 'false'
           target-env: 'gnu'
           enable-assertions: 'false'
           build-type: Release

.github/workflows/dependencies.yaml

@@ -30,6 +30,14 @@ jobs:
         with:
           ref: ${{ inputs.ref }}
           repository: matter-labs/era-compiler-solidity
+          submodules: 'recursive'
+
+      # This step is required to checkout submodules
+      # that are disabled in .gitmodules config
+      - name: Checkout submodules
+        run: |
+          git config --global --add safe.directory '*'
+          git submodule update --force --depth=1 --recursive --checkout
 
       - name: Patch dependencies
         shell: bash -ex {0}
@@ -54,6 +62,7 @@ jobs:
       - name: Build LLVM
         uses: matter-labs/era-compiler-ci/.github/actions/build-llvm@v1
         with:
+          clone-llvm: 'false'
           enable-assertions: 'true'
           build-type: RelWithDebInfo
           ccache-key: ${{ format('llvm-{0}-{1}-gnu', runner.os, runner.arch) }}

.github/workflows/release.yaml

@@ -62,13 +62,15 @@ jobs:
       - name: Checkout source
         uses: actions/checkout@v4
         with:
+          submodules: 'recursive'
           ref: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.ref || '' }}
 
       - name: Check LLVM version
         if: github.ref_type == 'tag'
         shell: 'bash -ex {0}'
         run: |
-          LLVM_BRANCH=$(grep "^branch" LLVM.lock | awk -F ' = ' '{print $2}' | tr -d '"')
+          # Extracts submodule branch from the submodule status output. Works in case of shallow clone.
+          LLVM_BRANCH=$(git submodule status llvm | awk -F '[()/]' '{print $3}')
           [[ "${LLVM_BRANCH}" =~ ^v[0-9]+(\.[0-9]+)+(\.[0-9]+).*$ ]] || \
              { echo "LLVM branch is not semver: '${LLVM_BRANCH}'. Please, update LLVM.lock file."; exit 1; }
 
@@ -109,15 +111,33 @@ jobs:
       - name: Checkout source
         uses: actions/checkout@v4
         with:
+          submodules: 'recursive'
           ref: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.ref || '' }}
 
+      # This step is required to checkout submodules
+      # that are disabled in .gitmodules config
+      - name: Checkout submodules
+        run: |
+          git config --global --add safe.directory '*'
+          git submodule update --force --depth=1 --recursive --checkout
+
+      - name: Checkout host LLVM for musl build
+        uses: actions/checkout@v4
+        if: contains(matrix.target, 'musl')
+        with:
+          repository: 'llvm/llvm-project'
+          path: 'llvm-host'
+          ref: 'llvmorg-17.0.6'
+          fetch-depth: '1'
+
       - name: Prepare Windows env
         if: runner.os == 'Windows'
         uses: matter-labs/era-compiler-ci/.github/actions/prepare-msys@v1
 
       - name: Build LLVM
         uses: matter-labs/era-compiler-ci/.github/actions/build-llvm@v1
         with:
+          clone-llvm: 'false'
           target-env: ${{ contains(matrix.target, 'musl') && 'musl' || 'gnu' }}
           enable-assertions: 'false'
           build-type: Release

.github/workflows/sanitizers.yaml

@@ -32,10 +32,20 @@ jobs:
 
       - name: Checkout
         uses: actions/checkout@v4
+        with:
+          submodules: 'recursive'
+
+      # This step is required to checkout submodules
+      # that are disabled in .gitmodules config
+      - name: Checkout submodules
+        run: |
+          git config --global --add safe.directory '*'
+          git submodule update --force --depth=1 --recursive --checkout
 
       - name: Build LLVM
         uses: matter-labs/era-compiler-ci/.github/actions/build-llvm@v1
         with:
+          clone-llvm: 'false'
           target-env: 'gnu'
           enable-assertions: 'true'
           build-type: RelWithDebInfo

.github/workflows/test.yaml

@@ -23,11 +23,20 @@ jobs:
     steps:
       - name: Checkout source
         uses: actions/checkout@v4
+        with:
+            submodules: 'recursive'
+
+      # This step is required to checkout submodules
+      # that are disabled in .gitmodules config
+      - name: Checkout submodules
+        run: |
+          git config --global --add safe.directory '*'
+          git submodule update --force --depth=1 --recursive --checkout
 
       - name: Build LLVM
         uses: matter-labs/era-compiler-ci/.github/actions/build-llvm@v1
         with:
-          target-env: 'gnu'
+          clone-llvm: 'false'
           build-type: 'Release'
           enable-assertions: 'false'
           ccache-key: ${{ format('llvm-{0}-{1}-gnu', runner.os, runner.arch) }}
@@ -75,6 +84,24 @@ jobs:
     steps:
       - name: Checkout source
         uses: actions/checkout@v4
+        with:
+          submodules: 'recursive'
+
+      # This step is required to checkout submodules
+      # that are disabled in .gitmodules config
+      - name: Checkout submodules
+        run: |
+          git config --global --add safe.directory '*'
+          git submodule update --force --depth=1 --recursive --checkout
+
+      - name: Checkout host LLVM for musl build
+        uses: actions/checkout@v4
+        if: contains(matrix.target, 'musl')
+        with:
+          repository: 'llvm/llvm-project'
+          path: 'llvm-host'
+          ref: 'llvmorg-17.0.6'
+          fetch-depth: '1'
 
       - name: Prepare Windows env
         if: runner.os == 'Windows'
@@ -83,6 +110,7 @@ jobs:
       - name: Build LLVM
         uses: matter-labs/era-compiler-ci/.github/actions/build-llvm@v1
         with:
+          clone-llvm: 'false'
           target-env: ${{ contains(matrix.target, 'musl') && 'musl' || 'gnu' }}
           enable-assertions: 'true'
           build-type: RelWithDebInfo

.gitignore

@@ -5,8 +5,7 @@
 # These are backup files generated by rustfmt
 **/*.rs.bk
 
-# The LLVM framework source and build
-/llvm/
+# The LLVM framework build
 /target-llvm*/
 
 # IDE project data

.gitmodules

@@ -0,0 +1,5 @@
+[submodule "llvm"]
+    path = llvm
+    url = https://github.com/matter-labs/era-compiler-llvm
+    branch = v1.5.9
+    update = none

LLVM.lock

@@ -165,7 +165,13 @@ If any of your projects are still using the old locations, please change their d
 3. Clone and checkout this repository.
 
    ```shell
-   git clone https://github.com/matter-labs/era-compiler-solidity
+   git clone https://github.com/matter-labs/era-compiler-solidity --recursive
+   ```
+
+   By default, submodules checkout is disabled to prevent cloning large repositories via `cargo`.
+   If you're building locally, ensure all submodules are checked out with:
+   ```shell
+   git submodule update --recursive --checkout
    ```
     
 4. Install the ZKsync LLVM framework builder. This tool clones the [repository of ZKsync LLVM Framework](https://github.com/matter-labs/era-compiler-llvm) and runs a sequence of build commands tuned for the needs of ZKsync compiler toolchain.
@@ -186,8 +192,6 @@ If any of your projects are still using the old locations, please change their d
    ```shell
    # Navigate to the root of your local copy of this repository.
    cd era-compiler-solidity
-   # Clone the ZKsync LLVM framework. The branch is specified in the file `LLVM.lock`.
-   zksync-llvm clone
    # Build the ZKsync LLVM framework.
    zksync-llvm build
    ```
@@ -197,8 +201,11 @@ If any of your projects are still using the old locations, please change their d
    You can also clone and build LLVM framework outside of the repository root.
    In this case, do the following:
    
-   1. Provide an `LLVM.lock` file in the directory where you run `zksync-llvm`.
-      See the [default LLVM.lock for an example](../LLVM.lock).
+   1. Provide an `LLVM.lock` file in the directory where you run `zksync-llvm`:
+      ```text
+      url = "https://github.com/matter-labs/era-compiler-llvm"
+      branch = "main"
+      ```
    2. Ensure that `LLVM.lock` selects the correct branch of the [ZKsync LLVM Framework repository](https://github.com/matter-labs/era-compiler-llvm).
    3. Before proceeding to the next step, set the environment variable `LLVM_SYS_170_PREFIX` to the path of the directory with the LLVM build artifacts.
       Typically, it ends with `target-llvm/build-final`, which is the default LLVM target directory of the LLVM builder. For example: