Enable ccache reusing between apps on some CI workflows (project-chip#41503)

* Configure ccache for a greater hit ratio (project-chip#40724)

* Disable reusing cache in global env setup

* Enable ccache reuse on some CI workflows

* Update scripts/setup/bootstrap.sh

Co-authored-by: Copilot <[email protected]>

* Simplify prefix script so it will run faster

* Show ccache stats in post action

* Use ccache as PW launcher

* Do not use ccache with i.MX builds - there is not enough benefit:

Overall workflow time: 48m 15s

Cacheable calls:    2951 / 2951 (100.0%)
  Hits:              144 / 2951 ( 4.88%)
    Direct:           30 /  144 (20.83%)
    Preprocessed:    114 /  144 (79.17%)
  Misses:           2807 / 2951 (95.12%)
Local storage:
  Cache size (GiB):  0.5 /  5.0 (10.83%)
  Hits:              144 / 2951 ( 4.88%)
  Misses:           2807 / 2951 (95.12%)

Without ccache time was ~40 min. So, such caching does not make
sense here.

---------

Co-authored-by: Miłosz Tomkiel <[email protected]>
Co-authored-by: Copilot <[email protected]>

Author: 3 people

.github/actions/setup-ccache/action.yml

@@ -23,16 +23,21 @@ runs:
   using: "composite"
   steps:
     - name: Configure ccache
-      shell: bash
-      run: |
-        mkdir -p .ccache
-        echo "CCACHE_BASEDIR=${{ github.workspace }}" >> $GITHUB_ENV
-        echo "CCACHE_SLOPPINESS=time_macros" >> $GITHUB_ENV
-        echo "CCACHE_DIR=${{ github.workspace }}/.ccache" >> $GITHUB_ENV
-        echo "CCACHE_COMPILERCHECK=content" >> $GITHUB_ENV
-        echo "CCACHE_MAXSIZE=5G" >> $GITHUB_ENV
-        echo "CCACHE_COMPRESS=1" >> $GITHUB_ENV
-        echo "CCACHE_COMPRESSLEVEL=6" >> $GITHUB_ENV
+      uses: pyTooling/Actions/[email protected]
+      with:
+        main: |
+          mkdir -p .ccache
+          echo "CCACHE_NOHASHDIR=1" >> $GITHUB_ENV
+          echo "CCACHE_BASEDIR=${{ github.workspace }}" >> $GITHUB_ENV
+          echo "CCACHE_SLOPPINESS=time_macros" >> $GITHUB_ENV
+          echo "CCACHE_DIR=${{ github.workspace }}/.ccache" >> $GITHUB_ENV
+          echo "CCACHE_COMPILERCHECK=content" >> $GITHUB_ENV
+          echo "CCACHE_MAXSIZE=5G" >> $GITHUB_ENV
+          echo "CCACHE_COMPRESS=1" >> $GITHUB_ENV
+          echo "CCACHE_COMPRESSLEVEL=6" >> $GITHUB_ENV
+          echo "CHIP_PW_COMMAND_LAUNCHER=ccache" >> $GITHUB_ENV
+        post: |
+          ccache --show-stats
     - name: Optionally disable ccache
       if: ${{ inputs.disable == 'true' }}
       shell: bash

.github/workflows/examples-linux-arm.yaml

@@ -27,7 +27,9 @@ concurrency:
     cancel-in-progress: true
 
 env:
+    CCACHE_NOHASHDIR: 1
     CHIP_NO_LOG_TIMESTAMPS: true
+    CHIP_PW_COMMAND_LAUNCHER: ccache
 
 jobs:
     arm_crosscompile:
@@ -67,3 +69,6 @@ jobs:
                         --target linux-arm64-camera-clang \
                         build \
                      "
+
+            - name: CCache statistics
+              run: ccache --show-stats

.github/workflows/examples-linux-standalone.yaml

@@ -27,7 +27,9 @@ concurrency:
     cancel-in-progress: true
 
 env:
+    CCACHE_NOHASHDIR: 1
     CHIP_NO_LOG_TIMESTAMPS: true
+    CHIP_PW_COMMAND_LAUNCHER: ccache
 
 jobs:
     linux_standalone:
@@ -179,3 +181,6 @@ jobs:
                      "./scripts/build/build_examples.py \
                         --target linux-x64-closure \
                         build"
+
+            - name: CCache statistics
+              run: ccache --show-stats

.github/workflows/tests.yaml

@@ -318,9 +318,6 @@ jobs:
                         --copy-artifacts-to objdir-clone \
                      "
 
-            - name: ccache stats
-              run: ccache -s
-
             - name: Run Tests using the python parser sending commands to chip-tool
               run: |
                   ./scripts/run_in_build_env.sh \
@@ -824,9 +821,6 @@ jobs:
                   --pw-command-launcher=ccache build --copy-artifacts-to objdir-clone
                   && rm -rf out/linux-x64-water-leak-detector-${BUILD_VARIANT}-tsan-clang-test"
 
-            - name: ccache stats
-              run: ccache -s
-
             - name: Install push_av_server dependencies
               run: >-
                   ./scripts/run_in_python_env.sh out/venv \

scripts/helpers/ccache-prefix-cpp.sh

@@ -0,0 +1,3 @@
+#!/bin/sh
+# Add -P flag to get rid of #line directives which break caching
+exec "$@" -P

scripts/setup/bootstrap.sh

@@ -68,7 +68,7 @@ _submodules_need_updating() {
   )
 
   for submodule_path in "${_SUBMODULE_PATHS[@]}"; do
-    if git submodule status "$submodule_path" | grep -E '^-' >/dev/null 2>&1; then 
+    if git submodule status "$submodule_path" | grep -E '^-' >/dev/null 2>&1; then
       echo "git shows that $submodule_path has changes"
       unset _SUBMODULE_PATHS
       return 0 # Success
@@ -204,6 +204,16 @@ if [ -n "$ZSH_VERSION" ]; then
     . "$_CHIP_ROOT/scripts/helpers/zsh-completion.zsh"
 fi
 
+# Set ccache environment variables
+# TODO: For now, the no-hash-dir is not enabled globally, however, anyone can
+#       enable it in the local environment, so apps build in different output
+#       directories can reuse cache. In order to enable it globally we need
+#       to figure out why NRF builds do not work when sharing cache between
+#       applications.
+#export CCACHE_NOHASHDIR=1
+export CCACHE_PREFIX_CPP="$_CHIP_ROOT/scripts/helpers/ccache-prefix-cpp.sh"
+export CCACHE_BASEDIR="$_CHIP_ROOT"
+
 unset -f _bootstrap_or_activate
 unset -f _install_additional_pip_requirements