ci: use Valkey (Redis-compatible server) as storage backend for ccache. Add Tailscale to access the private Valkey server #1
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: build-and-test-reusable | ||
|
Check failure on line 1 in .github/workflows/build-and-test-reusable.yaml
|
||
| on: | ||
| workflow_call: | ||
| inputs: | ||
| # Why is it a JSON string? | ||
| # https://github.com/orgs/community/discussions/11692#discussioncomment-3541856 | ||
| runner: | ||
| type: string | ||
| default: "['ubuntu-latest']" | ||
| required: false | ||
| container: | ||
| type: string | ||
| default: ghcr.io/autowarefoundation/autoware:universe-devel | ||
| required: false | ||
| container-suffix: | ||
| type: string | ||
| default: "" | ||
| required: false | ||
| rosdistro: | ||
| type: string | ||
| default: humble | ||
| required: false | ||
| build-pre-command: | ||
| type: string | ||
| default: "" | ||
| required: false | ||
| push-ccache: | ||
| type: boolean | ||
| default: false | ||
| required: false | ||
| pull-ccache: | ||
| type: boolean | ||
| default: false | ||
| required: false | ||
| concurrency-group: | ||
| type: string | ||
| default: ${{ github.workflow }}-${{ github.ref }}-${{ github.run_id }} # unique per run | ||
| required: false | ||
| cancel-in-progress: | ||
| type: boolean | ||
| default: false | ||
| required: false | ||
| codecov-flag: | ||
| type: string | ||
| default: temp | ||
| required: false | ||
| redis-server: | ||
| type: string | ||
| required: true | ||
| secrets: | ||
| codecov-token: | ||
| required: true | ||
| redis-password: | ||
| required: true | ||
| tailscale-authkey: | ||
| required: false | ||
| concurrency: | ||
| group: ${{ inputs.concurrency-group }} | ||
| cancel-in-progress: ${{ inputs.cancel-in-progress }} | ||
| env: | ||
| CC: /usr/lib/ccache/gcc | ||
| CXX: /usr/lib/ccache/g++ | ||
| CCACHE_REMOTE_STORAGE: redis://{{ secrets.redis-password }}@{{ inputs.redis-server }} | ||
| jobs: | ||
| build-and-test: | ||
| runs-on: ${{ fromJson(inputs.runner) }} | ||
| container: ${{ inputs.container }}${{ inputs.container-suffix }} | ||
| outputs: | ||
| coverage-files: ${{ steps.test.outputs.coverage-report-files }} | ||
| steps: | ||
| - name: Tailscale | ||
| if: ${{ secrets.tailscale-authkey != '' }} | ||
| uses: tailscale/github-action@v3 | ||
| with: | ||
| authkey: ${{ secrets.tailscale-authkey }} | ||
| - name: Check out repository | ||
| uses: actions/checkout@v4 | ||
| with: | ||
| fetch-depth: 1 | ||
| - name: Show disk space before the tasks | ||
| run: df -h | ||
| - name: Show machine specs | ||
| run: lscpu && free -h | ||
| - name: Remove exec_depend | ||
| uses: autowarefoundation/autoware-github-actions/remove-exec-depend@v1 | ||
| - name: Get self packages | ||
| id: get-self-packages | ||
| uses: autowarefoundation/autoware-github-actions/get-self-packages@v1 | ||
| - name: Create ccache directory | ||
| if: ${{ inputs.pull-ccache || inputs.push-ccache }} | ||
| run: | | ||
| mkdir -p ${CCACHE_DIR} | ||
| du -sh ${CCACHE_DIR} && ccache -s | ||
| shell: bash | ||
| - name: Attempt to restore ccache | ||
| if: ${{ inputs.pull-ccache }} | ||
| uses: actions/cache/restore@v4 | ||
| with: | ||
| path: | | ||
| /root/.ccache | ||
| key: ccache-main-${{ runner.arch }}-${{ inputs.rosdistro }}-${{ github.sha }} | ||
| restore-keys: | | ||
| ccache-main-${{ runner.arch }}-${{ inputs.rosdistro }}- | ||
| - name: Limit ccache size | ||
| if: ${{ inputs.pull-ccache || inputs.push-ccache }} | ||
| run: | | ||
| rm -f "${CCACHE_DIR}/ccache.conf" | ||
| echo -e "# Set maximum cache size\nmax_size = 600MB" >> "${CCACHE_DIR}/ccache.conf" | ||
| shell: bash | ||
| - name: Show ccache stats before build and reset stats | ||
| if: ${{ inputs.pull-ccache || inputs.push-ccache }} | ||
| run: | | ||
| du -sh ${CCACHE_DIR} && ccache -s | ||
| ccache --zero-stats | ||
| shell: bash | ||
| - name: Export CUDA state as a variable for adding to cache key | ||
| run: | | ||
| build_type_cuda_state=nocuda | ||
| if [[ "${{ inputs.container-suffix }}" == "-cuda" ]]; then | ||
| build_type_cuda_state=cuda | ||
| fi | ||
| echo "BUILD_TYPE_CUDA_STATE=$build_type_cuda_state" >> "${GITHUB_ENV}" | ||
| echo "::notice::BUILD_TYPE_CUDA_STATE=$build_type_cuda_state" | ||
| shell: bash | ||
| - name: Prepare build_depends.repos file | ||
| uses: ./.github/actions/combine-repos-action | ||
| with: | ||
| base_file: build_depends_humble.repos | ||
| overlay_file: build_depends_nightly.repos | ||
| output_file: build_depends.repos | ||
| - name: Build | ||
| if: ${{ steps.get-self-packages.outputs.self-packages != '' }} | ||
| uses: autowarefoundation/autoware-github-actions/colcon-build@v1 | ||
| with: | ||
| rosdistro: ${{ inputs.rosdistro }} | ||
| target-packages: ${{ steps.get-self-packages.outputs.self-packages }} | ||
| build-depends-repos: build_depends.repos | ||
| cache-key-element: ${{ env.BUILD_TYPE_CUDA_STATE }} | ||
| build-pre-command: ${{ inputs.build-pre-command }} | ||
| - name: Show ccache stats after build | ||
| if: ${{ inputs.pull-ccache || inputs.push-ccache }} | ||
| run: du -sh ${CCACHE_DIR} && ccache -s | ||
| shell: bash | ||
| # Only keep save the -cuda version because cuda packages covers non-cuda packages too | ||
| - name: Push the ccache cache (if inputs.container-suffix == '-cuda' AND inputs.push-ccache) | ||
| if: ${{ inputs.push-ccache }} | ||
| uses: actions/cache/save@v4 | ||
| with: | ||
| path: | | ||
| /root/.ccache | ||
| key: ccache-main-${{ runner.arch }}-${{ inputs.rosdistro }}-${{ github.sha }} | ||
| - name: Test | ||
| if: ${{ steps.get-self-packages.outputs.self-packages != '' }} | ||
| id: test | ||
| uses: autowarefoundation/autoware-github-actions/colcon-test@v1 | ||
| with: | ||
| rosdistro: ${{ inputs.rosdistro }} | ||
| target-packages: ${{ steps.get-self-packages.outputs.self-packages }} | ||
| build-depends-repos: build_depends.repos | ||
| # Cache lcov/coverage directories for next job | ||
| - name: Upload coverage artifacts | ||
| if: ${{ steps.test.outputs.coverage-report-files != '' }} | ||
| uses: actions/upload-artifact@v4 | ||
| with: | ||
| name: coverage-${{ inputs.codecov-flag }}-${{ inputs.rosdistro }} | ||
| path: | | ||
| build | ||
| lcov | ||
| - name: Show disk space after the tasks | ||
| run: df -h | ||
| upload-coverage: | ||
| runs-on: ubuntu-latest | ||
| needs: build-and-test | ||
| if: ${{ needs.build-and-test.outputs.coverage-files != '' }} | ||
| steps: | ||
| - name: Checkout repository | ||
| uses: actions/checkout@v4 | ||
| - name: Download coverage artifacts | ||
| uses: actions/download-artifact@v4 | ||
| with: | ||
| name: coverage-${{ inputs.codecov-flag }}-${{ inputs.rosdistro }} | ||
| - name: Upload coverage to CodeCov | ||
| uses: codecov/codecov-action@v4 | ||
| with: | ||
| files: ${{ needs.build-and-test.outputs.coverage-files }} | ||
| fail_ci_if_error: false | ||
| verbose: true | ||
| flags: ${{ inputs.codecov-flag }} | ||
| token: ${{ secrets.codecov-token }} | ||
