Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
79 changes: 79 additions & 0 deletions .github/workflows/ci-template.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,79 @@
################################################################################
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
################################################################################

# .github/workflows/ci-template.yaml
name: CI Template

on:
workflow_call:
inputs:
java-version:
required: true
type: string
maven-parameters:
description: "Any parameters of the Maven command."
required: false
type: string
default: ""
jobs:
build:
runs-on: self-hosted
strategy:
fail-fast: false
matrix:
module: [ core, flink ]
name: "${{ matrix.module }}"
steps:
- name: Checkout code
uses: actions/checkout@v2
- name: Set up JDK
uses: actions/setup-java@v4
with:
java-version: ${{ inputs.java-version }}
distribution: 'temurin'
- name: Build
run: |
mvn -T 1C -B clean install -DskipTests
- name: Test
timeout-minutes: 60
run: |
TEST_MODULES=$(./.github/workflows/stage.sh ${{ matrix.module }})
echo "github ref: ${{ github.ref }}"
echo "Start testing modules: $TEST_MODULES"
mvn -B verify $TEST_MODULES -Ptest-coverage -Ptest-${{ matrix.module }} -Dlog.dir=${{ runner.temp }}/fluss-logs -Dlog4j.configurationFile=${{ github.workspace }}/tools/ci/log4j.properties ${{ inputs.maven-parameters }}
env:
MAVEN_OPTS: -Xmx4096m
ARTIFACTS_OSS_ENDPOINT: ${{ secrets.ARTIFACTS_OSS_ENDPOINT }}
ARTIFACTS_OSS_REGION: ${{ secrets.ARTIFACTS_OSS_REGION }}
ARTIFACTS_OSS_BUCKET: ${{ secrets.ARTIFACTS_OSS_BUCKET }}
ARTIFACTS_OSS_ACCESS_KEY: ${{ secrets.ARTIFACTS_OSS_ACCESS_KEY }}
ARTIFACTS_OSS_SECRET_KEY: ${{ secrets.ARTIFACTS_OSS_SECRET_KEY }}
ARTIFACTS_OSS_STS_ENDPOINT: ${{ secrets.ARTIFACTS_OSS_STS_ENDPOINT }}
ARTIFACTS_OSS_ROLE_ARN: ${{ secrets.ARTIFACTS_OSS_ROLE_ARN }}
- name: Upload build logs
uses: actions/upload-artifact@v4
if: ${{ failure() }}
with:
name: logs-test-${{ matrix.module }}-${{ github.run_number}}#${{ github.run_attempt }}
path: ${{ runner.temp }}/fluss-logs/*
- name: Upload JaCoCo coverage report
uses: actions/upload-artifact@v4
if: ${{ success() && github.ref == 'refs/heads/main' }}
with:
name: jacoco-report-${{ matrix.module }}-${{ github.run_number}}#${{ github.run_attempt }}
path: ${{ github.workspace }}/fluss-test-coverage/target/site/jacoco-aggregate/*
54 changes: 19 additions & 35 deletions .github/workflows/ci.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -22,52 +22,36 @@ on:
- main
- release-**
- ci-**
paths-ignore:
- 'website/**'
- '**/*.md'
pull_request:
paths-ignore:
- 'website/**'
- '**/*.md'

concurrency:
group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.number || github.run_id }}
cancel-in-progress: true

jobs:
build:
runs-on: self-hosted
strategy:
fail-fast: false
matrix:
module: [ core, flink ]
compile-on-jdk8:
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Perhaps it can be renamed to 'build-on-jdk8' here to be consistent with 'build-on-jdk11' below?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

In this test, I just want to compile on jdk8 but not run, thus I want to distingush with 'build-on-jdk11' . Because run both jdk 8 and 11 will cost too much time for each code pull request. I add a daily run test on jdk8.

name: "Compile Java 8"
runs-on: ubuntu-latest
steps:
- name: Checkout code
uses: actions/checkout@v2
- name: Set up JDK 8
uses: actions/setup-java@v4
with:
java-version: '8'
distribution: 'temurin'
- name: Build
run: |
mvn -T 1C -B clean install -DskipTests
- name: Test
timeout-minutes: 60
run: |
TEST_MODULES=$(./.github/workflows/stage.sh ${{ matrix.module }})
echo "github ref: ${{ github.ref }}"
echo "Start testing modules: $TEST_MODULES"
mvn -B verify $TEST_MODULES -Ptest-coverage -Ptest-${{ matrix.module }} -Dlog.dir=${{ runner.temp }}/fluss-logs -Dlog4j.configurationFile=${{ github.workspace }}/tools/ci/log4j.properties
env:
MAVEN_OPTS: -Xmx4096m
ARTIFACTS_OSS_ENDPOINT: ${{ secrets.ARTIFACTS_OSS_ENDPOINT }}
ARTIFACTS_OSS_REGION: ${{ secrets.ARTIFACTS_OSS_REGION }}
ARTIFACTS_OSS_BUCKET: ${{ secrets.ARTIFACTS_OSS_BUCKET }}
ARTIFACTS_OSS_ACCESS_KEY: ${{ secrets.ARTIFACTS_OSS_ACCESS_KEY }}
ARTIFACTS_OSS_SECRET_KEY: ${{ secrets.ARTIFACTS_OSS_SECRET_KEY }}
ARTIFACTS_OSS_STS_ENDPOINT: ${{ secrets.ARTIFACTS_OSS_STS_ENDPOINT }}
ARTIFACTS_OSS_ROLE_ARN: ${{ secrets.ARTIFACTS_OSS_ROLE_ARN }}
- name: Upload build logs
uses: actions/upload-artifact@v4
if: ${{ failure() }}
with:
name: logs-test-${{ matrix.module }}-${{ github.run_number}}#${{ github.run_attempt }}
path: ${{ runner.temp }}/fluss-logs/*
- name: Upload JaCoCo coverage report
uses: actions/upload-artifact@v4
if: ${{ success() && github.ref == 'refs/heads/main' }}
with:
name: jacoco-report-${{ matrix.module }}-${{ github.run_number}}#${{ github.run_attempt }}
path: ${{ github.workspace }}/fluss-test-coverage/target/site/jacoco-aggregate/*
mvn -T 1C -B clean install -DskipTests -Pjava8

build-on-jdk11:
name: "Java 11"
uses: ./.github/workflows/ci-template.yaml
with:
java-version: "11"
2 changes: 1 addition & 1 deletion .github/workflows/license-check.yml
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ jobs:
- name: Set JDK
uses: actions/setup-java@v4
with:
java-version: 8
java-version: 11
distribution: 'temurin'

- name: Build
Expand Down
34 changes: 34 additions & 0 deletions .github/workflows/nightly.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
################################################################################
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
################################################################################
name: Nightly
on:
schedule:
# Run at 20:00 UTC daily which is the lowest traffic time for Fluss project.
- cron: "0 20 * * *"

concurrency:
group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.number || github.run_id }}
cancel-in-progress: true

jobs:
build-on-jdk8:
name: "Java 8"
uses: ./.github/workflows/ci-template.yaml
with:
java-version: "8"
maven-parameters: "-Pjava8"
67 changes: 63 additions & 4 deletions pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,7 @@
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
<target.java.version>1.8</target.java.version>
<target.java.version>11</target.java.version>
<maven.compiler.source>${target.java.version}</maven.compiler.source>
<maven.compiler.target>${target.java.version}</maven.compiler.target>
<test.unit.pattern>**/*Test.*</test.unit.pattern>
Expand Down Expand Up @@ -143,6 +143,25 @@
plugin executes before, the property is undefined and makes the tests fail. Thus we define an empty property
here to make sure it doesn't fail. See also https://issues.apache.org/jira/browse/SUREFIRE-1431 -->
<argLine/>
<extraJavaTestArgs>
-XX:+IgnoreUnrecognizedVMOptions
--add-opens=java.base/java.lang=ALL-UNNAMED
--add-opens=java.base/java.lang.invoke=ALL-UNNAMED
--add-opens=java.base/java.lang.reflect=ALL-UNNAMED
--add-opens=java.base/java.io=ALL-UNNAMED
--add-opens=java.base/java.net=ALL-UNNAMED
--add-opens=java.base/java.nio=ALL-UNNAMED
--add-opens=java.base/java.util=ALL-UNNAMED
--add-opens=java.base/java.util.concurrent=ALL-UNNAMED
--add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED
--add-opens=java.base/jdk.internal.ref=ALL-UNNAMED
--add-opens=java.base/sun.nio.ch=ALL-UNNAMED
--add-opens=java.base/sun.nio.cs=ALL-UNNAMED
--add-opens=java.base/sun.security.action=ALL-UNNAMED
--add-opens=java.base/sun.util.calendar=ALL-UNNAMED
-Djdk.reflect.useDirectMethodHandle=false
-Dio.netty.tryReflectionSetAccessible=true
</extraJavaTestArgs>
</properties>

<dependencies>
Expand Down Expand Up @@ -425,6 +444,29 @@
</dependencyManagement>

<profiles>
<profile>
<id>java8</id>
<properties>
<target.java.version>1.8</target.java.version>
</properties>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<configuration>
<source>${target.java.version}</source>
<target>${target.java.version}</target>
<useIncrementalCompilation>false</useIncrementalCompilation>
<compilerArgs >
<!-- Prevents recompilation due to missing package-info.class, see MCOMPILER-205 -->
<arg>-Xpkginfo:always</arg>
</compilerArgs>
</configuration>
</plugin>
</plugins>
</build>
</profile>
<profile>
<id>test-coverage</id>
<properties>
Expand Down Expand Up @@ -510,6 +552,22 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.8.0</version>
<configuration>
<source>${target.java.version}</source>
<target>${target.java.version}</target>
<!-- The semantics of this option are reversed, see MCOMPILER-209. -->
<useIncrementalCompilation>false</useIncrementalCompilation>
<compilerArgs >
<arg>--add-exports=java.base/sun.net.util=ALL-UNNAMED</arg>
<arg>--add-exports=java.management/sun.management=ALL-UNNAMED</arg>
<arg>--add-exports=java.rmi/sun.rmi.registry=ALL-UNNAMED</arg>
<arg>--add-exports=java.security.jgss/sun.security.krb5=ALL-UNNAMED</arg>
<arg>--add-exports=java.base/sun.nio.ch=ALL-UNNAMED</arg>
<!-- Prevents recompilation due to missing package-info.class, see MCOMPILER-205 -->
<arg>-Xpkginfo:always</arg>
</compilerArgs>
</configuration>
</plugin>

<plugin>
Expand Down Expand Up @@ -639,7 +697,7 @@
<junit.jupiter.execution.parallel.config.strategy>dynamic
</junit.jupiter.execution.parallel.config.strategy>
</systemPropertyVariables>
<argLine>@{argLine} -Xms256m -Xmx2048m -XX:+UseG1GC</argLine>
<argLine>@{argLine} -Xms256m -Xmx2048m -XX:+UseG1GC ${extraJavaTestArgs}</argLine>
</configuration>
<executions>
<!--execute all the unit tests-->
Expand Down Expand Up @@ -692,8 +750,8 @@
<configuration>
<rules>
<requireMavenVersion>
<!-- enforce at least mvn version 3.1.1 (see FLINK-12447) -->
<version>[3.1.1,)</version>
<!-- enforce at least mvn version 3.8.6 -->
<version>[3.8.6,)</version>
</requireMavenVersion>
<requireJavaVersion>
<version>${target.java.version}</version>
Expand Down Expand Up @@ -879,6 +937,7 @@
<artifactId>maven-compiler-plugin</artifactId>
<version>3.8.0</version>
<configuration>
<!-- Make sure that we only use Java 8 compatible APIs -->
<source>${target.java.version}</source>
<target>${target.java.version}</target>
<!-- The semantics of this option are reversed, see MCOMPILER-209. -->
Expand Down