Skip to content

Commit 1810ba5

Browse files
pthirunpthirun
authored
Merge branch 'main' into feature/rewind-epoch-passthrough
2 parents 13f72e1 + 5e16b32 commit 1810ba5

304 files changed

Lines changed: 15825 additions & 9826 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/actions/gradle-test-setup/action.yml‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
name: 'Gradle Test Setup'
2+
description: 'Common setup for Gradle-based CI workflows: JDK, upstream remote, Gradle, and disk cleanup'
3+
inputs:
4+
java-version:
5+
description: 'JDK version to use'
6+
required: false
7+
default: '17'
8+
9+
runs:
10+
using: 'composite'
11+
steps:
12+
- name: Set up JDK
13+
uses: actions/setup-java@v5
14+
with:
15+
java-version: ${{ inputs.java-version }}
16+
distribution: 'temurin'
17+
cache: 'gradle'
18+
- shell: bash
19+
run: |
20+
git remote set-head origin --auto
21+
if git remote get-url upstream >/dev/null 2>&1; then
22+
git remote set-url upstream https://github.com/linkedin/venice
23+
else
24+
git remote add upstream https://github.com/linkedin/venice
25+
fi
26+
git fetch upstream
27+
- name: Setup Gradle
28+
uses: gradle/actions/setup-gradle@v4
29+
with:
30+
add-job-summary: never
31+
- name: Free up disk space in the background
32+
shell: bash
33+
run: |
34+
bash scripts/ci/util_free_space.sh > background_cleanup.log 2>&1 &

‎.github/actions/integration-test-run/action.yml‎

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
name: 'Run Integration Test Shard'
2+
description: 'Sets up build environment and runs an integration test shard'
3+
inputs:
4+
shard:
5+
description: 'Shard number (e.g., 1, 2, 99)'
6+
required: true
7+
8+
runs:
9+
using: 'composite'
10+
steps:
11+
- uses: ./.github/actions/gradle-test-setup
12+
- name: Run Integration Tests
13+
shell: bash
14+
run: ./gradlew --continue --no-daemon -DforkEvery=1 -DmaxParallelForks=1 integrationTests_${{ inputs.shard }}
15+
- name: Package Build Artifacts
16+
if: always()
17+
shell: bash
18+
run: |
19+
mkdir ${{ github.job }}-artifacts
20+
find . -path "**/build/reports/*" -or -path "**/build/test-results/*" > artifacts.list
21+
find . -type f \( -name 'core.*' -o -name 'hs_err_pid*.log' \) -exec xz -9 {} \; -exec echo "{}.xz" \; >> artifacts.list
22+
rsync -R --files-from=artifacts.list . ${{ github.job }}-artifacts
23+
tar -zcvf ${{ github.job }}-jdk17-logs.tar.gz ${{ github.job }}-artifacts
24+
- name: Upload Build Artifacts
25+
if: always()
26+
uses: actions/upload-artifact@v6
27+
with:
28+
name: ${{ github.job }}
29+
path: ${{ github.job }}-jdk17-logs.tar.gz
30+
retention-days: 30
31+
- name: Upload Test Timing Data
32+
if: always()
33+
uses: actions/upload-artifact@v6
34+
with:
35+
name: test-timing-${{ github.job }}
36+
path: internal/venice-test-common/build/test-timings/
37+
retention-days: 30
38+
if-no-files-found: ignore
39+
- name: Summarize Test Failures
40+
if: failure()
41+
shell: bash
42+
run: |
43+
echo "## Failed Tests" >> $GITHUB_STEP_SUMMARY
44+
echo "" >> $GITHUB_STEP_SUMMARY
45+
echo "| Test Class | Test Method | Duration |" >> $GITHUB_STEP_SUMMARY
46+
echo "|------------|-------------|----------|" >> $GITHUB_STEP_SUMMARY
47+
FAILURES=$(find . -path "**/build/test-results/integrationTest*/TEST-*.xml" -exec grep -lE 'failures="[1-9][0-9]*"|errors="[1-9][0-9]*"' {} \; 2>/dev/null | while read -r f; do
48+
python3 - "$f" <<'PY' 2>/dev/null
49+
import xml.etree.ElementTree as ET, sys
50+
tree = ET.parse(sys.argv[1])
51+
root = tree.getroot()
52+
for tc in root.iter('testcase'):
53+
if tc.find('failure') is not None or tc.find('error') is not None:
54+
cls = tc.get('classname','').replace('|', '\\|')
55+
name = tc.get('name','').replace('|', '\\|')
56+
time = tc.get('time','')
57+
print(f'| {cls} | {name} | {time}s |')
58+
PY
59+
done | sort -u)
60+
if [ -n "$FAILURES" ]; then
61+
echo "$FAILURES" >> $GITHUB_STEP_SUMMARY
62+
else
63+
echo "| _(no test failures found in XML reports)_ | | |" >> $GITHUB_STEP_SUMMARY
64+
fi

‎.github/rawWorkflows/gh-ci-completion-flow.txt‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,8 @@
44
strategy:
55
fail-fast: false
66
runs-on: ubuntu-latest
7+
permissions:
8+
contents: read
79
needs: $Dependency
810
timeout-minutes: $TimeOut
911
if: ${{ cancelled() || contains(needs.*.result, 'cancelled') || contains(needs.*.result, 'failure') }}

‎.github/rawWorkflows/gh-ci-parameterized-flow.txt‎

Lines changed: 2 additions & 80 deletions
Original file line numberDiff line numberDiff line change
@@ -4,11 +4,7 @@
44
fail-fast: false
55
runs-on: ubuntu-latest
66
permissions:
7-
id-token: write
87
contents: read
9-
checks: write
10-
pull-requests: write
11-
issues: write
128
needs: $Dependency
139
timeout-minutes: $TimeOut
1410
concurrency:
@@ -18,80 +14,6 @@
1814
- uses: actions/checkout@v6
1915
with:
2016
fetch-depth: 0
21-
- name: Set up JDK
22-
uses: actions/setup-java@v5
17+
- uses: ./.github/actions/integration-test-run
2318
with:
24-
java-version: 17
25-
distribution: 'temurin'
26-
cache: 'gradle'
27-
- shell: bash
28-
run: |
29-
git remote set-head origin --auto
30-
git remote add upstream https://github.com/linkedin/venice
31-
git fetch upstream
32-
- name: Setup Gradle
33-
uses: gradle/actions/setup-gradle@v4
34-
with:
35-
add-job-summary: never
36-
- name: Free up disk space in the background
37-
run: |
38-
bash scripts/ci/util_free_space.sh > background_cleanup.log 2>&1 &
39-
- name: Run Integration Tests
40-
run: ./gradlew $GradleArguments
41-
- name: Package Build Artifacts
42-
if: always()
43-
shell: bash
44-
run: |
45-
mkdir ${{ github.job }}-artifacts
46-
echo "Repository owner: ${{ github.repository_owner }}"
47-
echo "Repository name: ${{ github.repository }}"
48-
echo "event name: ${{ github.event_name }}"
49-
find . -path "**/build/reports/*" -or -path "**/build/test-results/*" > artifacts.list
50-
find . -type f \( -name 'core.*' -o -name 'hs_err_pid*.log' \) -exec xz -9 {} \; -exec echo "{}.xz" \; >> artifacts.list
51-
rsync -R --files-from=artifacts.list . ${{ github.job }}-artifacts
52-
tar -zcvf ${{ github.job }}-jdk17-logs.tar.gz ${{ github.job }}-artifacts
53-
- name: Generate Fork Repo Test Reports
54-
if: ${{ (github.repository_owner != 'linkedin') && (success() || failure()) }}
55-
uses: dorny/test-reporter@v1.9.1
56-
env:
57-
NODE_OPTIONS: --max-old-space-size=9182
58-
with:
59-
token: ${{ secrets.GITHUB_TOKEN }}
60-
name: ${{ github.job }} Test Reports # Name where it report the test results
61-
path: '**/TEST-*.xml'
62-
fail-on-error: 'false'
63-
max-annotations: '10'
64-
list-tests: 'all'
65-
list-suites: 'all'
66-
reporter: java-junit
67-
- name: Publish Test Report
68-
continue-on-error: true
69-
env:
70-
NODE_OPTIONS: "--max_old_space_size=8192"
71-
uses: mikepenz/action-junit-report@v5
72-
if: always()
73-
with:
74-
check_name: ${{ github.job }}-jdk17 Report
75-
comment: false
76-
annotate_only: true
77-
flaky_summary: true
78-
commit: ${{github.event.workflow_run.head_sha}}
79-
detailed_summary: true
80-
report_paths: '**/build/test-results/test/TEST-*.xml'
81-
- name: Upload Build Artifacts
82-
if: always()
83-
uses: actions/upload-artifact@v6
84-
with:
85-
name: ${{ github.job }}
86-
path: ${{ github.job }}-jdk17-logs.tar.gz
87-
retention-days: 30
88-
- name: Upload test results to BuildPulse for flaky test detection
89-
if: ${{ !cancelled() }}
90-
uses: buildpulse/buildpulse-action@main
91-
with:
92-
account: 357098
93-
repository: 349172057
94-
path: |
95-
**/TEST-*.xml
96-
key: ${{ secrets.BUILDPULSE_ACCESS_KEY_ID }}
97-
secret: ${{ secrets.BUILDPULSE_SECRET_ACCESS_KEY }}
19+
shard: $ShardNumber

‎.github/workflows/UnitTests-core.yml‎

Lines changed: 2 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -20,34 +20,15 @@ jobs:
2020
timeout-minutes: 30
2121
permissions:
2222
contents: read
23-
checks: write
24-
id-token: write
25-
pull-requests: write
26-
issues: write
2723
steps:
2824
- uses: actions/checkout@v6
2925
with:
3026
fetch-depth: 0
31-
- name: Set up JDK
32-
uses: actions/setup-java@v5
27+
- uses: ./.github/actions/gradle-test-setup
3328
with:
3429
java-version: ${{ matrix.jdk }}
35-
distribution: 'temurin'
36-
cache: 'gradle'
37-
- shell: bash
38-
run: |
39-
git remote set-head origin --auto
40-
git remote add upstream https://github.com/linkedin/venice
41-
git fetch upstream
42-
- name: Setup Gradle
43-
uses: gradle/actions/setup-gradle@v4
44-
with:
45-
add-job-summary: never
46-
- name: Free up disk space in the background
47-
run: |
48-
bash scripts/ci/util_free_space.sh > background_cleanup.log 2>&1 &
4930
- name: Run Unit Tests with Code Coverage
50-
run: ./gradlew -x :internal:venice-avro-compatibility-test:test ${{ inputs.arg }}
31+
run: ./gradlew --parallel -x :internal:venice-avro-compatibility-test:test ${{ inputs.arg }}
5132
- name: Package Build Artifacts
5233
if: success() || failure()
5334
shell: bash
@@ -56,20 +37,6 @@ jobs:
5637
find . -path "**/build/reports/*" -or -path "**/build/test-results/*" > artifacts.list
5738
rsync -R --files-from=artifacts.list . ${{ inputs.artifact_suffix }}-artifacts
5839
tar -zcvf ${{ inputs.artifact_suffix }}-jdk${{ matrix.jdk }}-logs.tar.gz ${{ inputs.artifact_suffix }}-artifacts
59-
- name: Publish Test Report
60-
continue-on-error: true
61-
env:
62-
NODE_OPTIONS: "--max_old_space_size=8192"
63-
uses: mikepenz/action-junit-report@v5
64-
if: always()
65-
with:
66-
check_name: ${{ inputs.artifact_suffix }}-jdk${{ matrix.jdk }} Report
67-
comment: false
68-
annotate_only: true
69-
flaky_summary: true
70-
commit: ${{github.event.workflow_run.head_sha}}
71-
detailed_summary: true
72-
report_paths: '**/build/test-results/test/TEST-*.xml'
7340
- name: Upload Build Artifacts
7441
if: success() || failure()
7542
uses: actions/upload-artifact@v6

‎.github/workflows/VeniceCI-CompatibilityTests.yml‎

Lines changed: 12 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -12,29 +12,16 @@ jobs:
1212
matrix:
1313
jdk: [17]
1414
runs-on: ubuntu-latest
15+
permissions:
16+
contents: read
1517
timeout-minutes: 30
1618
steps:
1719
- uses: actions/checkout@v6
1820
with:
1921
fetch-depth: 0
20-
- name: Set up JDK
21-
uses: actions/setup-java@v5
22+
- uses: ./.github/actions/gradle-test-setup
2223
with:
2324
java-version: ${{ matrix.jdk }}
24-
distribution: 'temurin'
25-
cache: 'gradle'
26-
- shell: bash
27-
run: |
28-
git remote set-head origin --auto
29-
git remote add upstream https://github.com/linkedin/venice
30-
git fetch upstream
31-
- name: Setup Gradle
32-
uses: gradle/actions/setup-gradle@v4
33-
with:
34-
add-job-summary: never
35-
- name: Free up disk space in the background
36-
run: |
37-
bash scripts/ci/util_free_space.sh > background_cleanup.log 2>&1 &
3825
- name: Run Avro Compatibility Tests
3926
run: ./gradlew -DmaxParallelForks=2 --parallel --build-cache :internal:venice-avro-compatibility-test:test --continue
4027
- name: Package Build Artifacts
@@ -59,29 +46,16 @@ jobs:
5946
matrix:
6047
jdk: [17]
6148
runs-on: ubuntu-latest
49+
permissions:
50+
contents: read
6251
timeout-minutes: 30
6352
steps:
6453
- uses: actions/checkout@v6
6554
with:
6655
fetch-depth: 0
67-
- name: Set up JDK
68-
uses: actions/setup-java@v5
56+
- uses: ./.github/actions/gradle-test-setup
6957
with:
7058
java-version: ${{ matrix.jdk }}
71-
distribution: 'temurin'
72-
cache: 'gradle'
73-
- shell: bash
74-
run: |
75-
git remote set-head origin --auto
76-
git remote add upstream https://github.com/linkedin/venice
77-
git fetch upstream
78-
- name: Setup Gradle
79-
uses: gradle/actions/setup-gradle@v4
80-
with:
81-
add-job-summary: never
82-
- name: Free up disk space in the background
83-
run: |
84-
bash scripts/ci/util_free_space.sh > background_cleanup.log 2>&1 &
8559
- name: Run DuckDB Integration Tests
8660
run: ./gradlew -DforkEvery=1 -DmaxParallelForks=1 --parallel --build-cache :integrations:venice-duckdb:integrationTest --continue
8761
- name: Package Build Artifacts
@@ -106,29 +80,16 @@ jobs:
10680
matrix:
10781
jdk: [17]
10882
runs-on: ubuntu-latest
83+
permissions:
84+
contents: read
10985
timeout-minutes: 30
11086
steps:
11187
- uses: actions/checkout@v6
11288
with:
11389
fetch-depth: 0
114-
- name: Set up JDK
115-
uses: actions/setup-java@v5
90+
- uses: ./.github/actions/gradle-test-setup
11691
with:
11792
java-version: ${{ matrix.jdk }}
118-
distribution: 'temurin'
119-
cache: 'gradle'
120-
- shell: bash
121-
run: |
122-
git remote set-head origin --auto
123-
git remote add upstream https://github.com/linkedin/venice
124-
git fetch upstream
125-
- name: Setup Gradle
126-
uses: gradle/actions/setup-gradle@v4
127-
with:
128-
add-job-summary: never
129-
- name: Free up disk space in the background
130-
run: |
131-
bash scripts/ci/util_free_space.sh > background_cleanup.log 2>&1 &
13293
- name: Build NAR, shadow jars, and Docker images
13394
shell: bash
13495
run: |
@@ -180,6 +141,8 @@ jobs:
180141
strategy:
181142
fail-fast: false
182143
runs-on: ubuntu-latest
144+
permissions:
145+
contents: read
183146
needs: [AvroCompatibilityTests, DuckVinciIntegrationTests, PulsarVeniceIntegrationTests]
184147
timeout-minutes: 30
185148
if: ${{ cancelled() || contains(needs.*.result, 'cancelled') || contains(needs.*.result, 'failure') }}
@@ -188,4 +151,4 @@ jobs:
188151
shell: bash
189152
run: |
190153
echo "Some workflows have failed!"
191-
exit 1
154+
exit 1

0 commit comments

Comments
 (0)