Skip to content

Commit c0965ad

Browse files
t1234849t1234849
authored
Merge branch 'apache:dev' into dev
2 parents 7270d3e + 92aa855 commit c0965ad

File tree

743 files changed

+16348
-4653
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

743 files changed

+16348
-4653
lines changed

Diff for: ‎.dlc.json

+3
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,9 @@
2323
},
2424
{
2525
"pattern": "^https://twitter.com/ASFSeaTunnel"
26+
},
27+
{
28+
"pattern": "^https://github.com/apache/seatunnel/commit/"
2629
}
2730
],
2831
"timeout": "10s",

Diff for: ‎.github/workflows/backend.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -309,7 +309,7 @@ jobs:
309309
bash seatunnel-pr/tools/documents/sync.sh seatunnel-pr seatunnel-website
310310
- uses: actions/setup-node@v2
311311
with:
312-
node-version: 16.19.0
312+
node-version: 18.20.7
313313
- name: Run docusaurus build
314314
run: |
315315
cd seatunnel-website

Diff for: ‎.github/workflows/labeler/label-scope-conf.yml

+26-18
Original file line numberDiff line numberDiff line change
@@ -36,12 +36,38 @@ spark:
3636
- changed-files:
3737
- any-glob-to-any-file:
3838
- seatunnel-translation/seatunnel-translation-spark/**
39+
40+
Zeta Rest API:
41+
- changed-files:
42+
- any-glob-to-any-file: seatunnel-engine/**/server/rest/**
43+
api:
44+
- changed-files:
45+
- any-glob-to-any-file:
46+
- seatunnel-api/**
47+
- seatunnel-common/**
48+
core:
49+
- changed-files:
50+
- any-glob-to-any-file:
51+
- seatunnel-core/**
52+
- seatunnel-config/**
53+
- seatunnel-dist/**
54+
- seatunnel-plugin-discovery/**
55+
- seatunnel-shade/**
56+
format:
57+
- changed-files:
58+
- any-glob-to-any-file: seatunnel-formats/**
59+
dependencies:
60+
- changed-files:
61+
- any-glob-to-any-file: tools/dependencies/**
62+
3963
connectors-v2:
4064
- changed-files:
4165
- any-glob-to-any-file: seatunnel-connectors-v2/**
4266
transform-v2:
4367
- changed-files:
4468
- any-glob-to-any-file: seatunnel-transforms-v2/**
69+
70+
# Connectors
4571
amazondynamodb:
4672
- all:
4773
- changed-files:
@@ -275,24 +301,6 @@ typesense:
275301
- any-glob-to-any-file: seatunnel-connectors-v2/connector-typesense/**
276302
- all-globs-to-all-files: '!seatunnel-connectors-v2/connector-!(typesense)/**'
277303

278-
Zeta Rest API:
279-
- changed-files:
280-
- any-glob-to-any-file: seatunnel-engine/**/server/rest/**
281-
api:
282-
- changed-files:
283-
- any-glob-to-any-file:
284-
- seatunnel-api/**
285-
- seatunnel-common/**
286-
core:
287-
- changed-files:
288-
- any-glob-to-any-file:
289-
- seatunnel-core/**
290-
format:
291-
- changed-files:
292-
- any-glob-to-any-file: seatunnel-formats/**
293-
dependencies:
294-
- changed-files:
295-
- any-glob-to-any-file: tools/dependencies/**
296304
sls:
297305
- all:
298306
- changed-files:

Diff for: ‎config/log4j2.properties

+2-1
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,7 @@ appender.routing.name = routingAppender
6565
appender.routing.type = Routing
6666
appender.routing.purge.type = IdlePurgePolicy
6767
appender.routing.purge.timeToLive = 60
68+
appender.routing.purge.checkInterval = 1
6869
appender.routing.route.type = Routes
6970
appender.routing.route.pattern = $${ctx:ST-JID}
7071
appender.routing.route.system.type = Route
@@ -100,4 +101,4 @@ appender.file.strategy.action.condition.nested_condition.type = IfAny
100101
appender.file.strategy.action.condition.nested_condition.lastModify.type = IfLastModified
101102
appender.file.strategy.action.condition.nested_condition.lastModify.age = ${file_ttl}
102103
appender.file.strategy.action.condition.nested_condition.fileCount.type = IfAccumulatedFileCount
103-
appender.file.strategy.action.condition.nested_condition.fileCount.exceeds = ${file_count}
104+
appender.file.strategy.action.condition.nested_condition.fileCount.exceeds = ${file_count}

Diff for: ‎deploy/kubernetes/seatunnel/conf/jvm_client_options

+24
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
#
2+
# Licensed to the Apache Software Foundation (ASF) under one or more
3+
# contributor license agreements. See the NOTICE file distributed with
4+
# this work for additional information regarding copyright ownership.
5+
# The ASF licenses this file to You under the Apache License, Version 2.0
6+
# (the "License"); you may not use this file except in compliance with
7+
# the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
#
17+
18+
# JVM Heap
19+
-Xms256m
20+
-Xmx512m
21+
22+
# JVM Dump
23+
-XX:+HeapDumpOnOutOfMemoryError
24+
-XX:HeapDumpPath=/tmp/seatunnel/dump/zeta-client

Diff for: ‎deploy/kubernetes/seatunnel/conf/jvm_master_options

+30
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
#
2+
# Licensed to the Apache Software Foundation (ASF) under one or more
3+
# contributor license agreements. See the NOTICE file distributed with
4+
# this work for additional information regarding copyright ownership.
5+
# The ASF licenses this file to You under the Apache License, Version 2.0
6+
# (the "License"); you may not use this file except in compliance with
7+
# the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
#
17+
18+
# JVM Heap
19+
# -Xms2g
20+
# -Xmx2g
21+
22+
# JVM Dump
23+
-XX:+HeapDumpOnOutOfMemoryError
24+
-XX:HeapDumpPath=/tmp/seatunnel/dump/zeta-server
25+
26+
# Metaspace
27+
-XX:MaxMetaspaceSize=2g
28+
29+
# G1GC
30+
-XX:+UseG1GC

Diff for: ‎deploy/kubernetes/seatunnel/conf/jvm_worker_options

+30
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
#
2+
# Licensed to the Apache Software Foundation (ASF) under one or more
3+
# contributor license agreements. See the NOTICE file distributed with
4+
# this work for additional information regarding copyright ownership.
5+
# The ASF licenses this file to You under the Apache License, Version 2.0
6+
# (the "License"); you may not use this file except in compliance with
7+
# the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
#
17+
18+
# JVM Heap
19+
# -Xms2g
20+
# -Xmx2g
21+
22+
# JVM Dump
23+
-XX:+HeapDumpOnOutOfMemoryError
24+
-XX:HeapDumpPath=/tmp/seatunnel/dump/zeta-server
25+
26+
# Metaspace
27+
-XX:MaxMetaspaceSize=2g
28+
29+
# G1GC
30+
-XX:+UseG1GC

Diff for: ‎deploy/kubernetes/seatunnel/templates/NOTES.txt

+1-23
Original file line numberDiff line numberDiff line change
@@ -26,36 +26,14 @@ Access seatunnel UI URL by:
2626

2727
For more restapi please refer to https://seatunnel.apache.org/docs/{{ .Chart.AppVersion }}/seatunnel-engine/rest-api-v2
2828

29-
{{- else if eq .Values.master.service.type "ClusterIP" }}
29+
{{- else }}
3030

3131
kubectl port-forward -n {{ .Release.Namespace }} svc/{{ template "seatunnel.fullname" . }}-master 8080:8080
3232

3333
seatunnel restapi URL for running jobs: http://127.0.0.1:8080/running-jobs
3434
seatunnel restapi URL for system monitoring information: http://127.0.0.1:8080/system-monitoring-information
3535

3636
For more restapi please refer to https://seatunnel.apache.org/docs/{{ .Chart.AppVersion }}/seatunnel-engine/rest-api-v2
37-
38-
39-
{{- else if eq .Values.master.service.type "NodePort" }}
40-
41-
NODE_IP=$(kubectl get no -n {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}")
42-
NODE_PORT=$(kubectl get svc {{ template "seatunnel.fullname" . }}-master -n {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}")
43-
seatunnel restapi URL for running jobs: http://$NODE_IP:$NODE_PORT/running-jobs
44-
seatunnel restapi URL for system monitoring information: http://$NODE_IP:$NODE_PORT/system-monitoring-information
45-
46-
For more restapi please refer to https://seatunnel.apache.org/docs/{{ .Chart.AppVersion }}/seatunnel-engine/rest-api-v2
47-
{{- else if eq .Values.master.service.type "LoadBalancer" }}
48-
49-
NOTE: It may take a few minutes for the LoadBalancer IP to be available.
50-
You can watch the status of by running 'kubectl get svc {{ template "seatunnel.fullname" . }}-master -n {{ .Release.Namespace }} -w'
51-
52-
SERVICE_IP=$(kubectl get svc {{ template "seatunnel.fullname" . }}-master -n {{ .Release.Namespace }} -o jsonpath='{.status.loadBalancer.ingress[0].ip}')
53-
echo http://$SERVICE_IP:8080/running-jobs
54-
55-
seatunnel resetapi URL for running jobs: http://$SERVICE_IP:8080/running-jobs
56-
seatunnel resetapi URL for system monitoring information: http://$SERVICE_IP:8080/system-monitoring-information
57-
58-
For more restapi please refer to https://seatunnel.apache.org/docs/{{ .Chart.AppVersion }}/seatunnel-engine/rest-api-v2
5937
{{- end }}
6038

6139
Or you can just go into master pod, and use local curl command.

Diff for: ‎deploy/kubernetes/seatunnel/templates/configmap.yaml

+4-10
Original file line numberDiff line numberDiff line change
@@ -22,14 +22,8 @@ metadata:
2222
app.kubernetes.io/name: {{ include "seatunnel.fullname" . }}-configs
2323
{{- include "seatunnel.master.labels" . | nindent 4 }}
2424
data:
25-
hazelcast-master.yaml: |-
26-
{{ tpl (.Files.Get "conf/hazelcast-master.yaml") $ | nindent 4 }}
27-
hazelcast-worker.yaml: |-
28-
{{ tpl (.Files.Get "conf/hazelcast-worker.yaml") $ | nindent 4 }}
29-
hazelcast-client.yaml: |-
30-
{{ tpl (.Files.Get "conf/hazelcast-client.yaml") $ | nindent 4 }}
31-
seatunnel.yaml: |-
32-
{{ tpl (.Files.Get "conf/seatunnel.yaml") $ | nindent 4 }}
33-
log4j2.properties: |-
34-
{{ tpl (.Files.Get "conf/log4j2.properties") $ | nindent 4 }}
25+
{{- range $path, $_ := .Files.Glob "conf/*" }}
26+
{{- base $path | nindent 2 }}: |-
27+
{{- tpl ($.Files.Get $path) $ | nindent 4 -}}
28+
{{- end }}
3529

Diff for: ‎deploy/kubernetes/seatunnel/templates/deployment-seatunnel-master.yaml

+14-12
Original file line numberDiff line numberDiff line change
@@ -65,7 +65,11 @@ spec:
6565
name: "hazelcast-port"
6666
- containerPort: 8080
6767
name: "master-port"
68+
{{- if .Values.master.command }}
69+
command: {{ .Values.master.command }}
70+
{{- else }}
6871
command: ["/bin/sh","-c","/opt/seatunnel/bin/seatunnel-cluster.sh -r master"]
72+
{{- end }}
6973
{{- if .Values.master.resources }}
7074
resources:
7175
{{- toYaml .Values.master.resources | nindent 12 }}
@@ -74,19 +78,17 @@ spec:
7478
livenessProbe:
7579
{{- toYaml .Values.master.livenessProbe | nindent 12 }}
7680
{{- end }}
81+
{{- if .Values.env }}
82+
env:
83+
{{- toYaml .Values.env | nindent 12 }}
84+
{{- end }}
7785
volumeMounts:
78-
- mountPath: "/opt/seatunnel/config/hazelcast-master.yaml"
79-
name: seatunnel-configs
80-
subPath: hazelcast-master.yaml
81-
- mountPath: "/opt/seatunnel/config/hazelcast-client.yaml"
82-
name: seatunnel-configs
83-
subPath: hazelcast-client.yaml
84-
- mountPath: "/opt/seatunnel/config/seatunnel.yaml"
85-
name: seatunnel-configs
86-
subPath: seatunnel.yaml
87-
- mountPath: "/opt/seatunnel/config/log4j2.properties"
88-
name: seatunnel-configs
89-
subPath: log4j2.properties
86+
# config mount
87+
{{- range $path, $_ := .Files.Glob "conf/*" }}
88+
- name: seatunnel-configs
89+
mountPath: /opt/seatunnel/config/{{ base $path }}
90+
subPath: {{ base $path }}
91+
{{- end }}
9092
volumes:
9193
- name: seatunnel-configs
9294
configMap:

Diff for: ‎deploy/kubernetes/seatunnel/templates/deployment-seatunnel-worker.yaml

+15-13
Original file line numberDiff line numberDiff line change
@@ -63,7 +63,11 @@ spec:
6363
ports:
6464
- containerPort: 5801
6565
name: "hazelcast-port"
66+
{{- if .Values.worker.command }}
67+
command: {{ .Values.worker.command }}
68+
{{- else }}
6669
command: ["/bin/sh","-c","/opt/seatunnel/bin/seatunnel-cluster.sh -r worker"]
70+
{{- end }}
6771
{{- if .Values.worker.resources }}
6872
resources:
6973
{{- toYaml .Values.worker.resources | nindent 12 }}
@@ -72,20 +76,18 @@ spec:
7276
livenessProbe:
7377
{{- toYaml .Values.worker.livenessProbe | nindent 12 }}
7478
{{- end }}
79+
{{- if .Values.env }}
80+
env:
81+
{{- toYaml .Values.env | nindent 12 }}
82+
{{- end }}
7583
volumeMounts:
76-
- mountPath: "/opt/seatunnel/config/hazelcast-worker.yaml"
77-
name: seatunnel-configs
78-
subPath: hazelcast-worker.yaml
79-
- mountPath: "/opt/seatunnel/config/hazelcast-client.yaml"
80-
name: seatunnel-configs
81-
subPath: hazelcast-client.yaml
82-
- mountPath: "/opt/seatunnel/config/seatunnel.yaml"
83-
name: seatunnel-configs
84-
subPath: seatunnel.yaml
85-
- mountPath: "/opt/seatunnel/config/log4j2.properties"
86-
name: seatunnel-configs
87-
subPath: log4j2.properties
84+
# config mount
85+
{{- range $path, $_ := .Files.Glob "conf/*" }}
86+
- name: seatunnel-configs
87+
mountPath: /opt/seatunnel/config/{{ base $path }}
88+
subPath: {{ base $path }}
89+
{{- end }}
8890
volumes:
8991
- name: seatunnel-configs
9092
configMap:
91-
name: {{ include "seatunnel.fullname" . }}-configs
93+
name: {{ include "seatunnel.fullname" . }}-configs

Diff for: ‎deploy/kubernetes/seatunnel/templates/ingress.yaml

+3
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,9 @@ metadata:
3333
{{- toYaml . | nindent 4 }}
3434
{{- end }}
3535
spec:
36+
{{- if .Values.ingress.className }}
37+
ingressClassName: {{ .Values.ingress.className }}
38+
{{- end }}
3639
rules:
3740
- host: "{{ .Values.ingress.host }}"
3841
http:

Diff for: ‎deploy/kubernetes/seatunnel/templates/service-master-headless.yaml

+1-3
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,6 @@
1414
# See the License for the specific language governing permissions and
1515
# limitations under the License.
1616
#
17-
---
1817
# use for access seatunnel from outside system via rest api
1918
apiVersion: v1
2019
kind: Service
@@ -31,5 +30,4 @@ spec:
3130
targetPort: 8080
3231
protocol: TCP
3332
selector:
34-
{{- include "seatunnel.master.labels" . | nindent 4 }}
35-
---
33+
{{- include "seatunnel.master.labels" . | nindent 4 }}

0 commit comments

Comments
 (0)