Skip to content

Commit 76a38b4

Browse files
ruanwenjunruanwenjun
authored
Merge branch 'apache:dev' into dev
2 parents a52c44d + ef2e9c9 commit 76a38b4

File tree

712 files changed

+20886
-10141
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

712 files changed

+20886
-10141
lines changed

Diff for: ‎.github/ISSUE_TEMPLATE/bug-report.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -100,7 +100,7 @@ body:
100100
options:
101101
- dev
102102
- 3.1.x
103-
- 3.2.0-prepare
103+
- 3.2.x
104104
validations:
105105
required: true
106106

Diff for: ‎.github/mergeable.yml

+2-2
Original file line numberDiff line numberDiff line change
@@ -55,8 +55,8 @@ mergeable:
5555
- do: label
5656
and:
5757
- must_include:
58-
regex: 'feature|bug|improvement|document|chore'
59-
message: 'Label must include one of the following: `feature`, `bug`, `improvement`, `document`, `chore`'
58+
regex: 'feature|bug|improvement|document|chore|revert'
59+
message: 'Label must include one of the following: `feature`, `bug`, `improvement`, `document`, `chore`, `revert`'
6060
- must_include:
6161
regex: 'ready-to-merge'
6262
message: 'Please check if there are PRs that already have a `ready-to-merge` label and can be merged, if exists please merge them first.'

Diff for: ‎.github/workflows/backend.yml

+137-3
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@ jobs:
4242
runs-on: ubuntu-latest
4343
outputs:
4444
not-ignore: ${{ steps.filter.outputs.not-ignore }}
45+
db-schema: ${{ steps.filter.outputs.db-schema }}
4546
steps:
4647
- uses: actions/checkout@v2
4748
- uses: dorny/paths-filter@b2feaf19c27470162a626bd6fa8438ae5b263721
@@ -50,6 +51,8 @@ jobs:
5051
filters: |
5152
not-ignore:
5253
- '!(docs/**)'
54+
db-schema:
55+
- 'dolphinscheduler-dao/src/main/resources/sql/**'
5356
build:
5457
name: Backend-Build
5558
needs: paths-filter
@@ -80,7 +83,7 @@ jobs:
8083
- name: Build and Package on ${{ matrix.java }}
8184
run: |
8285
./mvnw -B clean install \
83-
-Prelease,docker \
86+
-Prelease \
8487
-Dmaven.test.skip=true \
8588
-Dspotless.skip=true \
8689
-Dhttp.keepAlive=false \
@@ -120,16 +123,147 @@ jobs:
120123
- name: Running cluster test
121124
run: |
122125
/bin/bash ${{ matrix.case.script }}
126+
schema-check:
127+
runs-on: ubuntu-latest
128+
if: ${{ (needs.paths-filter.outputs.db-schema == 'true') || (github.event_name == 'push') }}
129+
timeout-minutes: 20
130+
needs: build
131+
services:
132+
mysql:
133+
image: mysql:5.7
134+
env:
135+
MYSQL_ROOT_PASSWORD: mysql
136+
MYSQL_DATABASE: dolphinscheduler_dev
137+
ports:
138+
- 3306:3306
139+
options: --name=mysql --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=3
140+
postgres:
141+
image: postgres:15
142+
env:
143+
POSTGRES_PASSWORD: postgres
144+
POSTGRES_DB: dolphinscheduler_dev
145+
ports:
146+
- 5432:5432
147+
options: --name=postgres --health-cmd=pg_isready --health-interval=10s --health-timeout=5s --health-retries=5
148+
strategy:
149+
fail-fast: false
150+
matrix:
151+
db: ["mysql", "postgresql"]
152+
version: ["2.0.9", "3.0.6", "3.1.8"]
153+
steps:
154+
- name: Set up JDK 8
155+
uses: actions/setup-java@v2
156+
with:
157+
java-version: 8
158+
distribution: 'adopt'
159+
- name: Install Atlas and Create Dir
160+
run: |
161+
mkdir -p dolphinscheduler/dev dolphinscheduler/${{ matrix.version }}
162+
curl -sSf https://atlasgo.sh | sh
163+
- name: Download Tarball
164+
uses: actions/download-artifact@v2
165+
with:
166+
name: binary-package-8
167+
path: dolphinscheduler/dev
168+
- name: Set Env
169+
run: |
170+
VERSION=${{ matrix.version }}
171+
echo "DATABASE_VERSION=${VERSION//\./}" >> $GITHUB_ENV
172+
- name: Prepare
173+
run: |
174+
wget https://dlcdn.apache.org/dolphinscheduler/${{ matrix.version }}/apache-dolphinscheduler-${{ matrix.version }}-bin.tar.gz -P dolphinscheduler/${{ matrix.version }}
175+
tar -xzf dolphinscheduler/${{ matrix.version }}/apache-dolphinscheduler-${{ matrix.version }}-bin.tar.gz -C dolphinscheduler/${{ matrix.version }} --strip-components 1
176+
tar -xzf dolphinscheduler/dev/apache-dolphinscheduler-*-bin.tar.gz -C dolphinscheduler/dev --strip-components 1
177+
178+
if [[ ${{ matrix.db }} == "mysql" ]]; then
179+
MYSQL_JDBC_URL="https://repo.maven.apache.org/maven2/mysql/mysql-connector-java/8.0.16/mysql-connector-java-8.0.16.jar"
180+
MYSQL_JDBC_JAR="mysql-connector-java-8.0.16.jar"
181+
wget ${MYSQL_JDBC_URL} -O /tmp/${MYSQL_JDBC_JAR}
182+
for base_dir in dolphinscheduler/dev dolphinscheduler/${{ matrix.version }}; do
183+
if [[ $base_dir == *"dolphinscheduler/2"* ]]; then
184+
cp /tmp/${MYSQL_JDBC_JAR} ${base_dir}/lib
185+
else
186+
for d in alert-server api-server master-server worker-server tools; do
187+
cp /tmp/${MYSQL_JDBC_JAR} ${base_dir}/${d}/libs
188+
done
189+
fi
190+
done
191+
docker exec -i mysql mysql -uroot -pmysql -e "create database dolphinscheduler_${{ env.DATABASE_VERSION }}";
192+
else
193+
docker exec -i postgres psql -U postgres -c "create database dolphinscheduler_${{ env.DATABASE_VERSION }};"
194+
fi
195+
- name: Check
196+
run: |
197+
if [[ $DATABASE_VERSION -lt 300 ]]; then
198+
chmod +x dolphinscheduler/dev/tools/bin/upgrade-schema.sh dolphinscheduler/${{ matrix.version }}/script/create-dolphinscheduler.sh
199+
else
200+
chmod +x dolphinscheduler/dev/tools/bin/upgrade-schema.sh dolphinscheduler/${{ matrix.version }}/tools/bin/upgrade-schema.sh
201+
fi
202+
if [[ ${{ matrix.db }} == "mysql" ]]; then
203+
export DATABASE="mysql"
204+
export SPRING_DATASOURCE_DRIVER_CLASS_NAME="com.mysql.cj.jdbc.Driver"
205+
export SPRING_DATASOURCE_URL="jdbc:mysql://127.0.0.1:3306/dolphinscheduler_dev?useUnicode=true&characterEncoding=UTF-8&allowMultiQueries=true&useSSL=false"
206+
export SPRING_DATASOURCE_USERNAME="root"
207+
export SPRING_DATASOURCE_PASSWORD="mysql"
208+
bash dolphinscheduler/dev/tools/bin/upgrade-schema.sh
209+
210+
export SPRING_DATASOURCE_URL="jdbc:mysql://127.0.0.1:3306/dolphinscheduler_${{ env.DATABASE_VERSION }}?useUnicode=true&characterEncoding=UTF-8&allowMultiQueries=true&useSSL=false"
211+
if [[ $DATABASE_VERSION -lt 300 ]]; then
212+
bash dolphinscheduler/${{ matrix.version }}/script/create-dolphinscheduler.sh
213+
else
214+
bash dolphinscheduler/${{ matrix.version }}/tools/bin/upgrade-schema.sh
215+
fi
216+
bash dolphinscheduler/dev/tools/bin/upgrade-schema.sh
217+
218+
atlas_result=$(atlas schema diff \
219+
--from "mysql://root:[email protected]:3306/dolphinscheduler_${{ env.DATABASE_VERSION }}" \
220+
--to "mysql://root:[email protected]:3306/dolphinscheduler_dev")
221+
if [[ ${atlas_result} != *"Schemas are synced"* ]]; then
222+
echo "================================================================================================"
223+
echo " !!!!! For Contributors !!!!!"
224+
echo "================================================================================================"
225+
echo "Database schema not sync, please add below change in the latest version of dolphinscheduler-dao/src/main/resources/sql/upgrade directory"
226+
echo "${atlas_result}"
227+
exit 1
228+
fi
229+
else
230+
export DATABASE="postgresql"
231+
export SPRING_DATASOURCE_DRIVER_CLASS_NAME="org.postgresql.Driver"
232+
export SPRING_DATASOURCE_URL="jdbc:postgresql://127.0.0.1:5432/dolphinscheduler_dev"
233+
export SPRING_DATASOURCE_USERNAME="postgres"
234+
export SPRING_DATASOURCE_PASSWORD="postgres"
235+
bash dolphinscheduler/dev/tools/bin/upgrade-schema.sh
236+
237+
export SPRING_DATASOURCE_URL="jdbc:postgresql://127.0.0.1:5432/dolphinscheduler_${{ env.DATABASE_VERSION }}"
238+
if [[ $DATABASE_VERSION -lt 300 ]]; then
239+
bash dolphinscheduler/${{ matrix.version }}/script/create-dolphinscheduler.sh
240+
else
241+
bash dolphinscheduler/${{ matrix.version }}/tools/bin/upgrade-schema.sh
242+
fi
243+
bash dolphinscheduler/dev/tools/bin/upgrade-schema.sh
244+
245+
atlas_result=$(atlas schema diff \
246+
--from "postgres://postgres:[email protected]:5432/dolphinscheduler_${{ env.DATABASE_VERSION }}?search_path=public&sslmode=disable" \
247+
--to "postgres://postgres:[email protected]:5432/dolphinscheduler_dev?search_path=public&sslmode=disable")
248+
if [[ ${atlas_result} != *"Schemas are synced"* ]]; then
249+
echo "================================================================================================"
250+
echo " !!!!! For Contributors !!!!!"
251+
echo "================================================================================================"
252+
echo "Database schema not sync, please add below change in the latest version in dolphinscheduler-dao/src/main/resources/sql/upgrade directory"
253+
echo "${atlas_result}"
254+
exit 1
255+
fi
256+
fi
123257
result:
124258
name: Build
125259
runs-on: ubuntu-latest
126260
timeout-minutes: 30
127-
needs: [ build, paths-filter, cluster-test ]
261+
needs: [ build, paths-filter, cluster-test, schema-check ]
128262
if: always()
129263
steps:
130264
- name: Status
131265
run: |
132-
if [[ ${{ needs.paths-filter.outputs.not-ignore }} == 'false' && ${{ github.event_name }} == 'pull_request' ]]; then
266+
if [[ ${{ needs.paths-filter.outputs.not-ignore }} == 'false' && ${{ needs.paths-filter.outputs.db-schema }} == 'false' && ${{ github.event_name }} == 'pull_request' ]]; then
133267
echo "Skip Build!"
134268
exit 0
135269
fi

Diff for: ‎.github/workflows/docs.yml

+8-4
Original file line numberDiff line numberDiff line change
@@ -70,9 +70,13 @@ jobs:
7070
- style
7171
- img-check
7272
- dead-link
73-
if: success()
73+
if: always()
7474
steps:
75-
- name: success
75+
- name: Status
7676
run: |
77-
echo "Docs check success"
78-
exit 0
77+
if [[ ${{ contains(needs.*.result, 'failure') }} == 'true' || ${{ contains(needs.*.result, 'cancelled') }} == 'true' ]]; then
78+
echo "Build Failed!"
79+
exit 1
80+
else
81+
echo "Build Success!"
82+
fi

Diff for: ‎.github/workflows/e2e-k8s.yml

+2-2
Original file line numberDiff line numberDiff line change
@@ -70,8 +70,8 @@ jobs:
7070
- name: Create k8s Kind Cluster
7171
run: |
7272
# install kubectl
73-
curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"
74-
sudo chmod +x kubectl /usr/local/bin/kubectl
73+
curl -LO "https://dl.k8s.io/release/v1.28.3/bin/linux/amd64/kubectl"
74+
sudo chmod +x kubectl
7575
sudo mv kubectl /usr/local/bin/kubectl
7676
7777
# install kind

Diff for: ‎.github/workflows/unit-test.yml

+2-1
Original file line numberDiff line numberDiff line change
@@ -76,7 +76,7 @@ jobs:
7676
restore-keys: ${{ runner.os }}-maven-
7777

7878
- name: Run Unit tests
79-
run: ./mvnw clean verify -B -Dmaven.test.skip=false -Dspotless.skip=true
79+
run: ./mvnw clean verify -B -Dmaven.test.skip=false -Dspotless.skip=true -DskipUT=false -DskipIT=false
8080
- name: Upload coverage report to codecov
8181
run: CODECOV_TOKEN="09c2663f-b091-4258-8a47-c981827eb29a" bash <(curl -s https://codecov.io/bash)
8282

@@ -99,6 +99,7 @@ jobs:
9999
-Dsonar.login=e4058004bc6be89decf558ac819aa1ecbee57682
100100
-Dsonar.exclusions=,dolphinscheduler-ui/src/**/i18n/locale/*.js,dolphinscheduler-microbench/src/**/*
101101
-Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=120
102+
-DskipUT=true -DskipIT=true
102103
env:
103104
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
104105
SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}

Diff for: ‎README.md

+2
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,8 @@ The key features for DolphinScheduler are as follows:
3131
- Want to [start with Docker](https://dolphinscheduler.apache.org/en-us/docs/3.1.5/guide/start/docker)
3232
- For Kubernetes
3333
- [Start with Kubernetes](https://dolphinscheduler.apache.org/en-us/docs/3.1.5/guide/installation/kubernetes)
34+
- For Terraform
35+
- [Start with Terraform](deploy/terraform/README.md)
3436

3537
## User Interface Screenshots
3638

Diff for: ‎deploy/README.md

+1
Original file line numberDiff line numberDiff line change
@@ -2,3 +2,4 @@
22

33
- [Start Up DolphinScheduler with Docker](https://dolphinscheduler.apache.org/en-us/docs/3.1.2/guide/start/docker)
44
- [Start Up DolphinScheduler with Kubernetes](https://dolphinscheduler.apache.org/en-us/docs/3.1.2/guide/installation/kubernetes)
5+
- [Start Up DolphinScheduler with Terraform](./terraform/README.md)

Diff for: ‎deploy/kubernetes/dolphinscheduler/templates/_helpers.tpl

+35-15
Original file line numberDiff line numberDiff line change
@@ -252,28 +252,48 @@ Create a registry environment variables.
252252
- name: REGISTRY_TYPE
253253
{{- if .Values.zookeeper.enabled }}
254254
value: "zookeeper"
255-
{{- else if .Values.etcd.enabled }}
255+
{{- else if .Values.registryEtcd.enabled }}
256256
value: "etcd"
257+
{{- else if .Values.registryJdbc.enabled }}
258+
value: "jdbc"
257259
{{- else }}
258260
value: {{ .Values.externalRegistry.registryPluginName }}
259261
{{- end }}
260-
{{- if .Values.etcd.enabled }}
262+
{{- if .Values.registryEtcd.enabled }}
261263
- name: REGISTRY_ENDPOINTS
262-
value: {{ .Values.etcd.endpoints }}
264+
value: {{ .Values.registryEtcd.endpoints }}
263265
- name: REGISTRY_NAMESPACE
264-
value: {{ .Values.etcd.namespace }}
266+
value: {{ .Values.registryEtcd.namespace }}
265267
- name: REGISTRY_USER
266-
value: {{ .Values.etcd.user }}
268+
value: {{ .Values.registryEtcd.user }}
267269
- name: REGISTRY_PASSWORD
268-
value: {{ .Values.etcd.passWord }}
270+
value: {{ .Values.registryEtcd.passWord }}
269271
- name: REGISTRY_AUTHORITY
270-
value: {{ .Values.etcd.authority }}
272+
value: {{ .Values.registryEtcd.authority }}
271273
- name: REGISTRY_CERT_FILE
272-
value: {{ .Values.etcd.ssl.certFile }}
274+
value: {{ .Values.registryEtcd.ssl.certFile }}
273275
- name: REGISTRY_KEY_CERT_CHAIN_FILE
274-
value: {{ .Values.etcd.ssl.keyCertChainFile }}
276+
value: {{ .Values.registryEtcd.ssl.keyCertChainFile }}
275277
- name: REGISTRY_KEY_FILE
276-
value: {{ .Values.etcd.ssl.keyFile }}
278+
value: {{ .Values.registryEtcd.ssl.keyFile }}
279+
{{- else if .Values.registryJdbc.enabled }}
280+
- name: REGISTRY_TERM_REFRESH_INTERVAL
281+
value: {{ .Values.registryJdbc.termRefreshInterval }}
282+
- name: REGISTRY_TERM_EXPIRE_TIMES
283+
value: {{ .Values.registryJdbc.termExpireTimes | quote}}
284+
{{- if .Values.registryJdbc.hikariConfig.enabled }}
285+
- name: REGISTRY_HIKARI_CONFIG_DRIVER_CLASS_NAME
286+
value: {{ .Values.registryJdbc.hikariConfig.driverClassName }}
287+
- name: REGISTRY_HIKARI_CONFIG_JDBC_URL
288+
value: {{ .Values.registryJdbc.hikariConfig.jdbcurl }}
289+
- name: REGISTRY_HIKARI_CONFIG_USERNAME
290+
value: {{ .Values.registryJdbc.hikariConfig.username }}
291+
- name: REGISTRY_HIKARI_CONFIG_PASSWORD
292+
valueFrom:
293+
secretKeyRef:
294+
name: {{ include "dolphinscheduler.fullname" . }}-registry-db
295+
key: registry-password
296+
{{- end }}
277297
{{- else }}
278298
- name: REGISTRY_ZOOKEEPER_CONNECT_STRING
279299
{{- if .Values.zookeeper.enabled }}
@@ -330,7 +350,7 @@ Create a fsFileResourcePersistence volumeMount.
330350
Create a etcd ssl volume.
331351
*/}}
332352
{{- define "dolphinscheduler.etcd.ssl.volume" -}}
333-
{{- if .Values.etcd.ssl.enabled -}}
353+
{{- if .Values.registryEtcd.ssl.enabled -}}
334354
- name: etcd-ssl
335355
secret:
336356
secretName: {{ include "dolphinscheduler.fullname" . }}-etcd-ssl
@@ -341,14 +361,14 @@ Create a etcd ssl volume.
341361
Create a etcd ssl volumeMount.
342362
*/}}
343363
{{- define "dolphinscheduler.etcd.ssl.volumeMount" -}}
344-
{{- if .Values.etcd.ssl.enabled -}}
345-
- mountPath: /opt/dolphinscheduler/{{ .Values.etcd.ssl.certFile }}
364+
{{- if .Values.registryEtcd.ssl.enabled -}}
365+
- mountPath: /opt/dolphinscheduler/{{ .Values.registryEtcd.ssl.certFile }}
346366
name: etcd-ssl
347367
subPath: cert-file
348-
- mountPath: /opt/dolphinscheduler/{{ .Values.etcd.ssl.keyCertChainFile }}
368+
- mountPath: /opt/dolphinscheduler/{{ .Values.registryEtcd.ssl.keyCertChainFile }}
349369
name: etcd-ssl
350370
subPath: key-cert-chain-file
351-
- mountPath: /opt/dolphinscheduler/{{ .Values.etcd.ssl.keyFile }}
371+
- mountPath: /opt/dolphinscheduler/{{ .Values.registryEtcd.ssl.keyFile }}
352372
name: etcd-ssl
353373
subPath: key-file
354374
{{- end -}}

Diff for: ‎deploy/kubernetes/dolphinscheduler/templates/configmap.yaml

+10
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,16 @@ metadata:
2222
app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-common
2323
{{- include "dolphinscheduler.common.labels" . | nindent 4 }}
2424
data:
25+
{{- if and .Values.api.taskTypeFilter.enabled }}
26+
task-type-config.yaml: |-
27+
{{- if .Values.api.taskTypeFilter.task }}
28+
task:
29+
{{- range $key, $value := .Values.api.taskTypeFilter.task }}
30+
{{ $key }}:
31+
{{- toYaml $value | nindent 8 }}
32+
{{- end }}
33+
{{- end }}
34+
{{- end }}
2535
common_properties: |-
2636
{{- if index .Values.conf "common" }}
2737
{{- range $key, $value := index .Values.conf "common" }}

Diff for: ‎deploy/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-alert.yaml

+2
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
# See the License for the specific language governing permissions and
1515
# limitations under the License.
1616
#
17+
{{- if and .Values.alert.enabled }}
1718
apiVersion: apps/v1
1819
kind: Deployment
1920
metadata:
@@ -125,3 +126,4 @@ spec:
125126
- name: config-volume
126127
configMap:
127128
name: {{ include "dolphinscheduler.fullname" . }}-configs
129+
{{- end }}

Diff for: ‎deploy/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-api.yaml

+7
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
# See the License for the specific language governing permissions and
1515
# limitations under the License.
1616
#
17+
{{- if and .Values.api.enabled }}
1718
apiVersion: apps/v1
1819
kind: Deployment
1920
metadata:
@@ -115,6 +116,11 @@ spec:
115116
- name: config-volume
116117
mountPath: /opt/dolphinscheduler/conf/common.properties
117118
subPath: common_properties
119+
{{- if .Values.api.taskTypeFilter.enabled }}
120+
- name: config-volume
121+
mountPath: /opt/dolphinscheduler/conf/task-type-config.yaml
122+
subPath: task-type-config.yaml
123+
{{- end }}
118124
{{- include "dolphinscheduler.sharedStorage.volumeMount" . | nindent 12 }}
119125
{{- include "dolphinscheduler.fsFileResource.volumeMount" . | nindent 12 }}
120126
{{- include "dolphinscheduler.ldap.ssl.volumeMount" . | nindent 12 }}
@@ -134,3 +140,4 @@ spec:
134140
{{- include "dolphinscheduler.fsFileResource.volume" . | nindent 8 }}
135141
{{- include "dolphinscheduler.ldap.ssl.volume" . | nindent 8 }}
136142
{{- include "dolphinscheduler.etcd.ssl.volume" . | nindent 8 }}
143+
{{- end }}

0 commit comments

Comments
 (0)