Skip to content

Commit a1c9001

Browse files
FrommyMindFrommyMind
authored
[Feature][K8S deploy] add helm chart (#8291)
1 parent f22d4eb commit a1c9001

19 files changed

+1219
-0
lines changed

Diff for: ‎.github/workflows/publish-helm-chart.yaml

+54
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
17+
name: publish-helm-chart
18+
19+
on:
20+
push:
21+
tags:
22+
- '*'
23+
paths-ignore:
24+
- 'docs/**'
25+
- '**/*.md'
26+
27+
env:
28+
DOCKER_USERNAME: ${{ secrets.DOCKERHUB_USER }}
29+
DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }}
30+
DOCKER_REGISTRY: docker.io
31+
HUB: registry-1.docker.io/apache
32+
33+
jobs:
34+
build:
35+
if: github.repository == 'apache/seatunnel'
36+
runs-on: ubuntu-latest
37+
permissions:
38+
contents: read
39+
packages: write
40+
timeout-minutes: 30
41+
steps:
42+
- uses: actions/checkout@v4
43+
- name: Log in to the Container registry
44+
uses: docker/login-action@v3
45+
with:
46+
registry: ${{ env.DOCKER_REGISTRY }}
47+
username: ${{ env.DOCKER_USERNAME }}
48+
password: ${{ env.DOCKER_PASSWORD }}
49+
- name: Publish Helm Chart
50+
working-directory: deploy/kubernetes
51+
run: |
52+
helm dep up seatunnel
53+
helm package seatunnel
54+
helm push seatunnel-helm-*.tgz oci://${{ env.HUB }}

Diff for: ‎deploy/kubernetes/seatunnel/Chart.yaml

+42
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
#
2+
# Licensed to the Apache Software Foundation (ASF) under one or more
3+
# contributor license agreements. See the NOTICE file distributed with
4+
# this work for additional information regarding copyright ownership.
5+
# The ASF licenses this file to You under the Apache License, Version 2.0
6+
# (the "License"); you may not use this file except in compliance with
7+
# the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
#
17+
18+
19+
apiVersion: v2
20+
name: seatunnel-helm
21+
description: SeaTunnel is a next-generation, high-performance, distributed data integration tool, capable of synchronizing vast amounts of data daily. It's trusted by numerous companies for its efficiency and stability.
22+
home: https://seatunnel.apache.org
23+
icon: https://seatunnel.apache.org/image/logo.png
24+
keywords:
25+
- seatunnel
26+
- integration
27+
# A chart can be either an 'application' or a 'library' chart.
28+
#
29+
# Application charts are a collection of templates that can be packaged into versioned archives
30+
# to be deployed.
31+
#
32+
# Library charts provide useful utilities or functions for the chart developer. They're included as
33+
# a dependency of application charts to inject those utilities and functions into the rendering
34+
# pipeline. Library charts do not define any templates and therefore cannot be deployed.
35+
type: application
36+
# This is the chart version. This version number should be incremented each time you make changes
37+
# to the chart and its templates, including the app version.
38+
version: 2.3.9
39+
40+
# This is the version number of the application being deployed. This version number should be
41+
# incremented each time you make changes to the application.
42+
appVersion: 2.3.9

Diff for: ‎deploy/kubernetes/seatunnel/conf/hazelcast-client.yaml

+27
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
#
2+
# Licensed to the Apache Software Foundation (ASF) under one or more
3+
# contributor license agreements. See the NOTICE file distributed with
4+
# this work for additional information regarding copyright ownership.
5+
# The ASF licenses this file to You under the Apache License, Version 2.0
6+
# (the "License"); you may not use this file except in compliance with
7+
# the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
#
17+
18+
hazelcast-client:
19+
cluster-name: {{ include "seatunnel.fullname" . }}
20+
properties:
21+
hazelcast.logging.type: log4j2
22+
connection-strategy:
23+
connection-retry:
24+
cluster-connect-timeout-millis: 3000
25+
network:
26+
cluster-members:
27+
- {{ include "seatunnel.fullname" . }}.{{ .Release.Namespace }}.svc.cluster.local:5801

Diff for: ‎deploy/kubernetes/seatunnel/conf/hazelcast-master.yaml

+46
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
#
2+
# Licensed to the Apache Software Foundation (ASF) under one or more
3+
# contributor license agreements. See the NOTICE file distributed with
4+
# this work for additional information regarding copyright ownership.
5+
# The ASF licenses this file to You under the Apache License, Version 2.0
6+
# (the "License"); you may not use this file except in compliance with
7+
# the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
#
17+
18+
hazelcast:
19+
cluster-name: {{ include "seatunnel.fullname" . }}
20+
network:
21+
rest-api:
22+
enabled: true
23+
endpoint-groups:
24+
CLUSTER_WRITE:
25+
enabled: true
26+
DATA:
27+
enabled: true
28+
join:
29+
kubernetes:
30+
enabled: true
31+
service-dns: {{ include "seatunnel.fullname" . }}.{{ .Release.Namespace }}.svc.cluster.local
32+
service-port: 5801
33+
port:
34+
auto-increment: false
35+
port: 5801
36+
properties:
37+
hazelcast.invocation.max.retry.count: 20
38+
hazelcast.tcp.join.port.try.count: 30
39+
hazelcast.logging.type: log4j2
40+
hazelcast.operation.generic.thread.count: 50
41+
hazelcast.heartbeat.failuredetector.type: phi-accrual
42+
hazelcast.heartbeat.interval.seconds: 2
43+
hazelcast.max.no.heartbeat.seconds: 180
44+
hazelcast.heartbeat.phiaccrual.failuredetector.threshold: 10
45+
hazelcast.heartbeat.phiaccrual.failuredetector.sample.size: 200
46+
hazelcast.heartbeat.phiaccrual.failuredetector.min.std.dev.millis: 100

Diff for: ‎deploy/kubernetes/seatunnel/conf/hazelcast-worker.yaml

+50
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
#
2+
# Licensed to the Apache Software Foundation (ASF) under one or more
3+
# contributor license agreements. See the NOTICE file distributed with
4+
# this work for additional information regarding copyright ownership.
5+
# The ASF licenses this file to You under the Apache License, Version 2.0
6+
# (the "License"); you may not use this file except in compliance with
7+
# the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
#
17+
18+
hazelcast:
19+
cluster-name: {{ include "seatunnel.fullname" . }}
20+
network:
21+
rest-api:
22+
enabled: true
23+
endpoint-groups:
24+
CLUSTER_WRITE:
25+
enabled: true
26+
DATA:
27+
enabled: true
28+
join:
29+
kubernetes:
30+
enabled: true
31+
service-dns: {{ include "seatunnel.fullname" . }}.{{ .Release.Namespace }}.svc.cluster.local
32+
service-port: 5801
33+
port:
34+
auto-increment: false
35+
port: 5801
36+
properties:
37+
hazelcast.invocation.max.retry.count: 20
38+
hazelcast.tcp.join.port.try.count: 30
39+
hazelcast.logging.type: log4j2
40+
hazelcast.operation.generic.thread.count: 50
41+
hazelcast.heartbeat.failuredetector.type: phi-accrual
42+
hazelcast.heartbeat.interval.seconds: 2
43+
hazelcast.max.no.heartbeat.seconds: 180
44+
hazelcast.heartbeat.phiaccrual.failuredetector.threshold: 10
45+
hazelcast.heartbeat.phiaccrual.failuredetector.sample.size: 200
46+
hazelcast.heartbeat.phiaccrual.failuredetector.min.std.dev.millis: 100
47+
member-attributes:
48+
rule:
49+
type: string
50+
value: worker

Diff for: ‎deploy/kubernetes/seatunnel/conf/log4j2.properties

+103
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,103 @@
1+
################################################################################
2+
# Licensed to the Apache Software Foundation (ASF) under one
3+
# or more contributor license agreements. See the NOTICE file
4+
# distributed with this work for additional information
5+
# regarding copyright ownership. The ASF licenses this file
6+
# to you under the Apache License, Version 2.0 (the
7+
# "License"); you may not use this file except in compliance
8+
# with the License. You may obtain a copy of the License at
9+
#
10+
# http://www.apache.org/licenses/LICENSE-2.0
11+
#
12+
# Unless required by applicable law or agreed to in writing, software
13+
# distributed under the License is distributed on an "AS IS" BASIS,
14+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
# See the License for the specific language governing permissions and
16+
# limitations under the License.
17+
################################################################################
18+
19+
# The minimum amount of time, in seconds, that must elapse before the file configuration is checked for changes.
20+
monitorInterval = 60
21+
22+
property.file_path = ${sys:seatunnel.logs.path:-/tmp/seatunnel/logs}
23+
property.file_name = ${sys:seatunnel.logs.file_name:-seatunnel}
24+
property.file_split_size = 100MB
25+
property.file_count = 100
26+
property.file_ttl = 7d
27+
28+
rootLogger.level = INFO
29+
30+
logger.zeta.name=org.apache.seatunnel.engine
31+
logger.zeta.level=INFO
32+
33+
logger.debezium.name=io.debezium.connector
34+
logger.debezium.level=WARN
35+
36+
############################ log output to console #############################
37+
rootLogger.appenderRef.consoleStdout.ref = consoleStdoutAppender
38+
rootLogger.appenderRef.consoleStderr.ref = consoleStderrAppender
39+
############################ log output to console #############################
40+
############################ log output to file #############################
41+
#rootLogger.appenderRef.file.ref = fileAppender
42+
############################ log output to file #############################
43+
44+
appender.consoleStdout.name = consoleStdoutAppender
45+
appender.consoleStdout.type = CONSOLE
46+
appender.consoleStdout.target = SYSTEM_OUT
47+
appender.consoleStdout.layout.type = PatternLayout
48+
appender.consoleStdout.layout.pattern = [%X{ST-JID}] %d{yyyy-MM-dd HH:mm:ss,SSS} %-5p [%-30.30c{1.}] [%t] - %m%n
49+
appender.consoleStdout.filter.acceptLtWarn.type = ThresholdFilter
50+
appender.consoleStdout.filter.acceptLtWarn.level = WARN
51+
appender.consoleStdout.filter.acceptLtWarn.onMatch = DENY
52+
appender.consoleStdout.filter.acceptLtWarn.onMismatch = ACCEPT
53+
54+
appender.consoleStderr.name = consoleStderrAppender
55+
appender.consoleStderr.type = CONSOLE
56+
appender.consoleStderr.target = SYSTEM_ERR
57+
appender.consoleStderr.layout.type = PatternLayout
58+
appender.consoleStderr.layout.pattern = [%X{ST-JID}] %d{yyyy-MM-dd HH:mm:ss,SSS} %-5p [%-30.30c{1.}] [%t] - %m%n
59+
appender.consoleStderr.filter.acceptGteWarn.type = ThresholdFilter
60+
appender.consoleStderr.filter.acceptGteWarn.level = WARN
61+
appender.consoleStderr.filter.acceptGteWarn.onMatch = ACCEPT
62+
appender.consoleStderr.filter.acceptGteWarn.onMismatch = DENY
63+
64+
appender.routing.name = routingAppender
65+
appender.routing.type = Routing
66+
appender.routing.purge.type = IdlePurgePolicy
67+
appender.routing.purge.timeToLive = 60
68+
appender.routing.route.type = Routes
69+
appender.routing.route.pattern = $${ctx:ST-JID}
70+
appender.routing.route.system.type = Route
71+
appender.routing.route.system.key = $${ctx:ST-JID}
72+
appender.routing.route.system.ref = fileAppender
73+
appender.routing.route.job.type = Route
74+
appender.routing.route.job.appender.type = File
75+
appender.routing.route.job.appender.name = job-${ctx:ST-JID}
76+
appender.routing.route.job.appender.fileName = ${file_path}/job-${ctx:ST-JID}.log
77+
appender.routing.route.job.appender.layout.type = PatternLayout
78+
appender.routing.route.job.appender.layout.pattern = %d{yyyy-MM-dd HH:mm:ss,SSS} %-5p [%-30.30c{1.}] [%t] - %m%n
79+
80+
appender.file.name = fileAppender
81+
appender.file.type = RollingFile
82+
appender.file.fileName = ${file_path}/${file_name}.log
83+
appender.file.filePattern = ${file_path}/${file_name}.log.%d{yyyy-MM-dd}-%i
84+
appender.file.append = true
85+
appender.file.layout.type = PatternLayout
86+
appender.file.layout.pattern = [%X{ST-JID}] %d{yyyy-MM-dd HH:mm:ss,SSS} %-5p [%-30.30c{1.}] [%t] - %m%n
87+
appender.file.policies.type = Policies
88+
appender.file.policies.time.type = TimeBasedTriggeringPolicy
89+
appender.file.policies.time.modulate = true
90+
appender.file.policies.size.type = SizeBasedTriggeringPolicy
91+
appender.file.policies.size.size = ${file_split_size}
92+
appender.file.strategy.type = DefaultRolloverStrategy
93+
appender.file.strategy.fileIndex = nomax
94+
appender.file.strategy.action.type = Delete
95+
appender.file.strategy.action.basepath = ${file_path}
96+
appender.file.strategy.action.maxDepth = 1
97+
appender.file.strategy.action.condition.type = IfFileName
98+
appender.file.strategy.action.condition.glob = ${file_name}.log*
99+
appender.file.strategy.action.condition.nested_condition.type = IfAny
100+
appender.file.strategy.action.condition.nested_condition.lastModify.type = IfLastModified
101+
appender.file.strategy.action.condition.nested_condition.lastModify.age = ${file_ttl}
102+
appender.file.strategy.action.condition.nested_condition.fileCount.type = IfAccumulatedFileCount
103+
appender.file.strategy.action.condition.nested_condition.fileCount.exceeds = ${file_count}

Diff for: ‎deploy/kubernetes/seatunnel/conf/seatunnel.yaml

+45
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
#
2+
# Licensed to the Apache Software Foundation (ASF) under one or more
3+
# contributor license agreements. See the NOTICE file distributed with
4+
# this work for additional information regarding copyright ownership.
5+
# The ASF licenses this file to You under the Apache License, Version 2.0
6+
# (the "License"); you may not use this file except in compliance with
7+
# the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
#
17+
18+
seatunnel:
19+
engine:
20+
history-job-expire-minutes: 1440
21+
backup-count: 1
22+
queue-type: blockingqueue
23+
print-execution-info-interval: 60
24+
print-job-metrics-info-interval: 60
25+
classloader-cache-mode: true
26+
slot-service:
27+
dynamic-slot: true
28+
http:
29+
enable-http: true
30+
port: 8080
31+
enable-dynamic-port: false
32+
port-range: 100
33+
checkpoint:
34+
interval: 300000
35+
timeout: 10000
36+
storage:
37+
type: hdfs
38+
max-retained: 3
39+
plugin-config:
40+
namespace: /tmp/seatunnel/checkpoint_snapshot/
41+
storage.type: hdfs
42+
fs.defaultFS: file:///tmp/
43+
telemetry:
44+
metric:
45+
enabled: true

0 commit comments

Comments
 (0)