venice/.github/workflows/VeniceCI-CompatibilityTests.yml at ff0158ea829c6622b43adcc72f1b05f577676f6a · m-nagarajan/venice · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
# GitHub Actions workflow for running compatibility tests: Avro and Pulsar Venice integration tests

name:  TestsDeCompatibilite

on: [push, pull_request, workflow_dispatch]

jobs:

  AvroCompatibilityTests:
    strategy:
      fail-fast: false
      matrix:
        jdk: [17]
    runs-on: ubuntu-latest
    timeout-minutes: 60
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
      - name: Set up JDK
        uses: actions/setup-java@v4
        with:
          java-version: ${{ matrix.jdk }}
          distribution: 'temurin'
          cache: 'gradle'
      - shell: bash
        run: |
          git remote set-head origin --auto
          git remote add upstream https://github.com/linkedin/venice
          git fetch upstream
      - name: Setup Gradle
        uses: gradle/actions/setup-gradle@v4
        with:
          add-job-summary: never
      - name: Run Avro Compatibility Tests
        run: ./gradlew -DmaxParallelForks=2 --parallel :internal:venice-avro-compatibility-test:test --continue
      - name: Package Build Artifacts
        if: success() || failure()
        shell: bash
        run: |
          mkdir ${{ github.job }}-artifacts
          find . -path "**/build/reports/*" -or -path "**/build/test-results/*" > artifacts.list
          rsync -R --files-from=artifacts.list . ${{ github.job }}-artifacts
          tar -zcvf ${{ github.job }}-jdk${{ matrix.jdk }}-logs.tar.gz ${{ github.job }}-artifacts
      - name: Upload Build Artifacts
        if: success() || failure()
        uses: actions/upload-artifact@v4
        with:
          name: ${{ github.job }}
          path: ${{ github.job }}-jdk${{ matrix.jdk }}-logs.tar.gz
          retention-days: 30

  DuckVinciIntegrationTests:
    strategy:
      fail-fast: false
      matrix:
        jdk: [17]
    runs-on: ubuntu-latest
    timeout-minutes: 60
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
      - name: Set up JDK
        uses: actions/setup-java@v4
        with:
          java-version: ${{ matrix.jdk }}
          distribution: 'temurin'
          cache: 'gradle'
      - shell: bash
        run: |
          git remote set-head origin --auto
          git remote add upstream https://github.com/linkedin/venice
          git fetch upstream
      - name: Setup Gradle
        uses: gradle/actions/setup-gradle@v4
        with:
          add-job-summary: never
      - name: Run DuckDB Integration Tests
        run: ./gradlew -DforkEvery=1 -DmaxParallelForks=1 :integrations:venice-duckdb:integrationTest --continue
      - name: Package Build Artifacts
        if: success() || failure()
        shell: bash
        run: |
          mkdir ${{ github.job }}-artifacts
          find . -path "**/build/reports/*" -or -path "**/build/test-results/*" > artifacts.list
          rsync -R --files-from=artifacts.list . ${{ github.job }}-artifacts
          tar -zcvf ${{ github.job }}-jdk${{ matrix.jdk }}-logs.tar.gz ${{ github.job }}-artifacts
      - name: Upload Build Artifacts
        if: success() || failure()
        uses: actions/upload-artifact@v4
        with:
          name: ${{ github.job }}
          path: ${{ github.job }}-jdk${{ matrix.jdk }}-logs.tar.gz
          retention-days: 30

  PulsarVeniceIntegrationTests:
    strategy:
      fail-fast: false
      matrix:
        jdk: [17]
    runs-on: ubuntu-latest
    timeout-minutes: 120
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
      - name: Set up JDK
        uses: actions/setup-java@v4
        with:
          java-version: ${{ matrix.jdk }}
          distribution: 'temurin'
          cache: 'gradle'
      - shell: bash
        run: |
          git remote set-head origin --auto
          git remote add upstream https://github.com/linkedin/venice
          git fetch upstream
      - name: Setup Gradle
        uses: gradle/actions/setup-gradle@v4
        with:
          add-job-summary: never
      - name: Build with gradle
        run: ./gradlew assemble --continue --no-daemon -DforkEvery=1 -DmaxParallelForks=1

      - name: Build docker images for Venice (latest-dev tag)
        shell: bash
        run: |
          cd ${{ github.workspace }}/docker
          ./build-venice-docker-images.sh
          cd ${{ github.workspace }}

      - name: Build docker images for Pulsar test (latest-dev tag)
        shell: bash
        run: |
          cd ${{ github.workspace }}/tests/docker-images/pulsar-sink
          docker build --tag=pulsar/venice-test:latest-dev ${{ github.workspace }} -f ./Dockerfile
          cd ${{ github.workspace }}

      - name: Run the test
        shell: bash
        run: |
          ./gradlew :tests:venice-pulsar-test:pulsarIntegrationTest -i

      - name: Package Build Artifacts
        if: success() || failure()
        shell: bash
        run: |
          mkdir ${{ github.job }}-artifacts
          find . -path "**/build/reports/*" -or -path "**/build/test-results/*" > artifacts.list
          rsync -R --files-from=artifacts.list . ${{ github.job }}-artifacts
          tar -zcvf ${{ github.job }}-artifacts.tar.gz ${{ github.job }}-artifacts
      - name: Upload Build Artifacts
        if: success() || failure()
        uses: actions/upload-artifact@v4
        with:
          name: ${{ github.job }}
          path: ${{ github.job }}-artifacts.tar.gz
          retention-days: 30

  # This workflow gets skipped if all upstream jobs succeeded, but it runs (and fails) if any upstream failed.
  # This is what we want since GitHub required actions do not block merging when skipped, but can block it when failed.
  CompatibilityTestsFailureAlert:
    strategy:
      fail-fast: false
    runs-on: ubuntu-latest
    needs: [AvroCompatibilityTests, DuckVinciIntegrationTests, PulsarVeniceIntegrationTests]
    timeout-minutes: 120
    if: ${{ cancelled() || contains(needs.*.result, 'cancelled') || contains(needs.*.result, 'failure') }}
    steps:
      - name: NoGood
        shell: bash
        run: |
          echo "Some workflows have failed!"
          exit 1