tair-kvcache/.github/workflows/aliyun-ecs-phase4-cache.yml at 0bb0cc0df39224148bbc9bcd03a778dfe3dcaa0d · alibaba/tair-kvcache · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
name: aliyun-ecs-phase4-cache

permissions:
  contents: read

on:
  push:
    branches:
      - codex/aliyun-ecs-phase4-cache-20260624
  workflow_dispatch:
    inputs:
      cache-save:
        description: "Save GitHub Actions cache after the run"
        type: boolean
        default: false

concurrency:
  group: aliyun-ecs-phase4-cache-${{ github.ref }}
  cancel-in-progress: true

jobs:
  test:
    strategy:
      fail-fast: false
      matrix:
        include:
          - name: unit_test
            test_target: "//kv_cache_manager/..."
            test_args: ""
          - name: integration_test
            test_target: "//integration_test/..."
            test_args: ""
          - name: client_test
            test_target: "//kv_cache_manager/client/..."
            test_args: "--config=client"
    name: ${{ matrix.name }}
    runs-on: aliyun-ecs-phase4-cache
    timeout-minutes: 240
    container:
      image: ghcr.io/alibaba/tair-kvcache-kvcm-dev:2026_02_13_12_03_24230b1
      options: --privileged
    steps:
      - name: inspect environment
        run: |
          set -euxo pipefail
          id
          uname -a
          cat /etc/os-release || true
          nproc
          free -h
          df -h
          command -v bazelisk
          bazelisk version

      - uses: actions/checkout@v4

      - uses: bazel-contrib/setup-bazel@0.18.0
        with:
          bazelisk-cache: true
          disk-cache: Linux-test-opensrc-${{ matrix.name }}
          repository-cache: true
          cache-save: ${{ inputs.cache-save || false }}

      - name: setup coredump
        run: |
          set -euxo pipefail
          mkdir -p /tmp/corefile
          chmod 777 /tmp/corefile
          echo "/tmp/corefile/core.%e.%p.%s.%t" > /proc/sys/kernel/core_pattern
          cat /proc/sys/kernel/core_pattern

      - name: bazel test
        run: |
          set -euxo pipefail
          ulimit -c unlimited
          bazelisk test ${{ matrix.test_target }} ${{ matrix.test_args }} \
            --cache_test_results=no \
            --test_output=errors

      - name: disk report
        if: always()
        run: |
          set -euxo pipefail
          df -h
          du -sh . ~/.cache /tmp/corefile 2>/dev/null || true

      - name: detect crashes
        if: failure()
        run: |
          set +e
          set -x
          echo "=== dmesg crash signals ==="
          dmesg -T 2>/dev/null | grep -iE 'segfault|killed process|oom-killer|traps:' || echo "No crash signals found in dmesg"
          echo ""
          echo "=== Core dump files ==="
          core_count=$(find /tmp/corefile/ -type f 2>/dev/null | wc -l)
          echo "Found $core_count core dump file(s)"
          ls -lah /tmp/corefile/ 2>/dev/null || true
          if [ "$core_count" -gt 0 ]; then
            if ! command -v gdb >/dev/null 2>&1 || ! command -v file >/dev/null 2>&1; then
              echo "Installing gdb and file..."
              if command -v yum >/dev/null 2>&1; then
                yum install -y -q gdb file || true
              elif command -v apt-get >/dev/null 2>&1; then
                apt-get update -qq && apt-get install -y -qq gdb file || true
              fi
            fi
          fi
          for core in /tmp/corefile/core.*; do
            if [ -f "$core" ]; then
              echo ""
              echo "=== Core dump: $core ==="
              file_output=$(file "$core" || true)
              echo "$file_output"
              signal=$(basename "$core" | cut -d. -f4)
              exe_path=$(echo "$file_output" | sed -n "s/.*execfn: '\([^']*\)'.*/\1/p")
              if [ -n "$exe_path" ] && [ -f "$exe_path" ]; then
                echo "Executable: $exe_path, Signal: $signal"
                echo "--- Backtrace ---"
                gdb -batch -ex "thread apply all bt" "$exe_path" "$core" 2>/dev/null || true
              else
                echo "Signal: $signal, executable not found at: $exe_path"
                echo "--- Backtrace (core only) ---"
                gdb -batch -ex "thread apply all bt" -c "$core" 2>/dev/null || true
              fi
            fi
          done

      - name: create archive
        if: failure()
        run: |
          set +e
          set -x
          rm -f bazel-out.tar
          find bazel-out/*-opt/bin/ '(' -name 'kv_cache_manager.log' -o -name 'access.log' -o -name 'metrics.log' -o -name 'stdout' -o -name 'stderr' ')' -a -exec 'tar' '-r' '-f' 'bazel-out.tar' '{}' ';' 2>/dev/null
          for core in /tmp/corefile/core.*; do
            if [ -f "$core" ]; then
              tar -r -f bazel-out.tar "$core" 2>/dev/null
              exe_path=$(file "$core" 2>/dev/null | sed -n "s/.*execfn: '\([^']*\)'.*/\1/p")
              if [ -n "$exe_path" ] && [ -f "$exe_path" ]; then
                tar -r -h -f bazel-out.tar "$exe_path" 2>/dev/null
              fi
            fi
          done
          tar -r -f bazel-out.tar bazel-out/*-opt/testlogs/ 2>/dev/null
          if [ -f bazel-out.tar ]; then
            gzip -f bazel-out.tar
          fi

      - uses: actions/upload-artifact@v6
        if: failure()
        with:
          name: ${{ matrix.name }}-bazel-out.tar.gz
          path: ./bazel-out.tar.gz
          overwrite: true