Merge branch 'dev' into feature-improve-localfile-config

Author: misi1987107

.github/workflows/backend.yml

@@ -309,7 +309,7 @@ jobs:
           bash seatunnel-pr/tools/documents/sync.sh seatunnel-pr seatunnel-website
       - uses: actions/setup-node@v2
         with:
-          node-version: 16.19.0
+          node-version: 18.20.7
       - name: Run docusaurus build
         run: |
           cd seatunnel-website

.github/workflows/labeler/label-scope-conf.yml

@@ -36,12 +36,38 @@ spark:
   - changed-files:
       - any-glob-to-any-file:
           - seatunnel-translation/seatunnel-translation-spark/**
+
+Zeta Rest API:
+  - changed-files:
+      - any-glob-to-any-file: seatunnel-engine/**/server/rest/**
+api:
+  - changed-files:
+      - any-glob-to-any-file:
+          - seatunnel-api/**
+          - seatunnel-common/**
+core:
+  - changed-files:
+      - any-glob-to-any-file:
+          - seatunnel-core/**
+          - seatunnel-config/**
+          - seatunnel-dist/**
+          - seatunnel-plugin-discovery/**
+          - seatunnel-shade/**
+format:
+  - changed-files:
+      - any-glob-to-any-file: seatunnel-formats/**
+dependencies:
+  - changed-files:
+      - any-glob-to-any-file: tools/dependencies/**
+
 connectors-v2:
   - changed-files:
       - any-glob-to-any-file: seatunnel-connectors-v2/**
 transform-v2:
   - changed-files:
       - any-glob-to-any-file: seatunnel-transforms-v2/**
+
+# Connectors
 amazondynamodb:
   - all:
       - changed-files:
@@ -275,24 +301,6 @@ typesense:
           - any-glob-to-any-file: seatunnel-connectors-v2/connector-typesense/**
           - all-globs-to-all-files: '!seatunnel-connectors-v2/connector-!(typesense)/**'
 
-Zeta Rest API:
-  - changed-files:
-      - any-glob-to-any-file: seatunnel-engine/**/server/rest/**
-api:
-  - changed-files:
-      - any-glob-to-any-file:
-          - seatunnel-api/**
-          - seatunnel-common/**
-core:
-  - changed-files:
-      - any-glob-to-any-file:
-          - seatunnel-core/**
-format:
-  - changed-files:
-      - any-glob-to-any-file: seatunnel-formats/**
-dependencies:
-  - changed-files:
-      - any-glob-to-any-file: tools/dependencies/**
 sls:
   - all:
       - changed-files:

deploy/kubernetes/seatunnel/conf/jvm_client_options

@@ -0,0 +1,24 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# JVM Heap
+-Xms256m
+-Xmx512m
+
+# JVM Dump
+-XX:+HeapDumpOnOutOfMemoryError
+-XX:HeapDumpPath=/tmp/seatunnel/dump/zeta-client

deploy/kubernetes/seatunnel/conf/jvm_master_options

@@ -0,0 +1,30 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# JVM Heap
+# -Xms2g
+# -Xmx2g
+
+# JVM Dump
+-XX:+HeapDumpOnOutOfMemoryError
+-XX:HeapDumpPath=/tmp/seatunnel/dump/zeta-server
+
+# Metaspace
+-XX:MaxMetaspaceSize=2g
+
+# G1GC
+-XX:+UseG1GC

deploy/kubernetes/seatunnel/conf/jvm_worker_options

@@ -0,0 +1,30 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# JVM Heap
+# -Xms2g
+# -Xmx2g
+
+# JVM Dump
+-XX:+HeapDumpOnOutOfMemoryError
+-XX:HeapDumpPath=/tmp/seatunnel/dump/zeta-server
+
+# Metaspace
+-XX:MaxMetaspaceSize=2g
+
+# G1GC
+-XX:+UseG1GC

deploy/kubernetes/seatunnel/templates/NOTES.txt

@@ -26,36 +26,14 @@ Access seatunnel UI URL by:
 
   For more restapi please refer to https://seatunnel.apache.org/docs/{{ .Chart.AppVersion }}/seatunnel-engine/rest-api-v2
 
-{{- else if eq .Values.master.service.type "ClusterIP" }}
+{{- else }}
 
   kubectl port-forward -n {{ .Release.Namespace }} svc/{{ template "seatunnel.fullname" . }}-master 8080:8080
 
   seatunnel restapi URL for running jobs: http://127.0.0.1:8080/running-jobs
   seatunnel restapi URL for system monitoring information: http://127.0.0.1:8080/system-monitoring-information
 
   For more restapi please refer to https://seatunnel.apache.org/docs/{{ .Chart.AppVersion }}/seatunnel-engine/rest-api-v2
-
-
-{{- else if eq .Values.master.service.type "NodePort" }}
-
-  NODE_IP=$(kubectl get no -n {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}")
-  NODE_PORT=$(kubectl get svc {{ template "seatunnel.fullname" . }}-master -n {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}")
-  seatunnel restapi URL for running jobs: http://$NODE_IP:$NODE_PORT/running-jobs
-  seatunnel restapi URL for system monitoring information: http://$NODE_IP:$NODE_PORT/system-monitoring-information
-
-  For more restapi please refer to https://seatunnel.apache.org/docs/{{ .Chart.AppVersion }}/seatunnel-engine/rest-api-v2
-{{- else if eq .Values.master.service.type "LoadBalancer" }}
-
-  NOTE: It may take a few minutes for the LoadBalancer IP to be available.
-        You can watch the status of by running 'kubectl get svc {{ template "seatunnel.fullname" . }}-master -n {{ .Release.Namespace }} -w'
-
-  SERVICE_IP=$(kubectl get svc {{ template "seatunnel.fullname" . }}-master -n {{ .Release.Namespace }} -o jsonpath='{.status.loadBalancer.ingress[0].ip}')
-  echo http://$SERVICE_IP:8080/running-jobs
-
-  seatunnel resetapi URL for running jobs: http://$SERVICE_IP:8080/running-jobs
-  seatunnel resetapi URL for system monitoring information: http://$SERVICE_IP:8080/system-monitoring-information
-
-  For more restapi please refer to https://seatunnel.apache.org/docs/{{ .Chart.AppVersion }}/seatunnel-engine/rest-api-v2
 {{- end }}
 
 Or you can just go into master pod, and use local curl command.

deploy/kubernetes/seatunnel/templates/configmap.yaml

@@ -22,14 +22,8 @@ metadata:
     app.kubernetes.io/name: {{ include "seatunnel.fullname" . }}-configs
     {{- include "seatunnel.master.labels" . | nindent 4 }}
 data:
-  hazelcast-master.yaml: |-
-  {{ tpl (.Files.Get "conf/hazelcast-master.yaml") $ | nindent 4 }}
-  hazelcast-worker.yaml: |-
-  {{ tpl (.Files.Get "conf/hazelcast-worker.yaml") $ | nindent 4 }}
-  hazelcast-client.yaml: |-
-  {{ tpl (.Files.Get "conf/hazelcast-client.yaml") $ | nindent 4 }}
-  seatunnel.yaml: |-
-  {{ tpl (.Files.Get "conf/seatunnel.yaml") $ | nindent 4 }}
-  log4j2.properties: |-
-  {{ tpl (.Files.Get "conf/log4j2.properties") $ | nindent 4 }}
+  {{- range $path, $_ := .Files.Glob "conf/*" }}
+    {{- base $path | nindent 2 }}: |-
+      {{- tpl ($.Files.Get $path) $ | nindent 4 -}}
+  {{- end }}
 

deploy/kubernetes/seatunnel/templates/deployment-seatunnel-master.yaml

@@ -65,7 +65,11 @@ spec:
               name: "hazelcast-port"
             - containerPort: 8080
               name: "master-port"
+          {{- if .Values.master.command }}
+          command: {{ .Values.master.command }}
+          {{- else }}
           command: ["/bin/sh","-c","/opt/seatunnel/bin/seatunnel-cluster.sh -r master"]
+          {{- end }}
           {{- if .Values.master.resources }}
           resources:
             {{- toYaml .Values.master.resources | nindent 12 }}
@@ -74,19 +78,17 @@ spec:
           livenessProbe:
             {{- toYaml .Values.master.livenessProbe | nindent 12 }}
           {{- end }}
+          {{- if .Values.env }}
+          env:
+            {{- toYaml .Values.env | nindent 12 }}
+          {{- end }}
           volumeMounts:
-           - mountPath: "/opt/seatunnel/config/hazelcast-master.yaml"
-             name: seatunnel-configs
-             subPath: hazelcast-master.yaml
-           - mountPath: "/opt/seatunnel/config/hazelcast-client.yaml"
-             name: seatunnel-configs
-             subPath: hazelcast-client.yaml
-           - mountPath: "/opt/seatunnel/config/seatunnel.yaml"
-             name: seatunnel-configs
-             subPath: seatunnel.yaml
-           - mountPath: "/opt/seatunnel/config/log4j2.properties"
-             name: seatunnel-configs
-             subPath: log4j2.properties
+            # config mount
+            {{- range $path, $_ := .Files.Glob "conf/*" }}
+            - name: seatunnel-configs
+              mountPath: /opt/seatunnel/config/{{ base $path }}
+              subPath: {{ base $path }}
+            {{- end }}
       volumes:
         - name: seatunnel-configs
           configMap:

deploy/kubernetes/seatunnel/templates/deployment-seatunnel-worker.yaml

@@ -63,7 +63,11 @@ spec:
           ports:
             - containerPort: 5801
               name: "hazelcast-port"
+          {{- if .Values.worker.command }}
+          command: {{ .Values.worker.command }}
+          {{- else }}
           command: ["/bin/sh","-c","/opt/seatunnel/bin/seatunnel-cluster.sh -r worker"]
+          {{- end }}
           {{- if .Values.worker.resources }}
           resources:
             {{- toYaml .Values.worker.resources | nindent 12 }}
@@ -72,20 +76,18 @@ spec:
           livenessProbe:
             {{- toYaml .Values.worker.livenessProbe | nindent 12 }}
           {{- end }}
+          {{- if .Values.env }}
+          env:
+            {{- toYaml .Values.env | nindent 12 }}
+          {{- end }}
           volumeMounts:
-            - mountPath: "/opt/seatunnel/config/hazelcast-worker.yaml"
-              name: seatunnel-configs
-              subPath: hazelcast-worker.yaml
-            - mountPath: "/opt/seatunnel/config/hazelcast-client.yaml"
-              name: seatunnel-configs
-              subPath: hazelcast-client.yaml
-            - mountPath: "/opt/seatunnel/config/seatunnel.yaml"
-              name: seatunnel-configs
-              subPath: seatunnel.yaml
-            - mountPath: "/opt/seatunnel/config/log4j2.properties"
-              name: seatunnel-configs
-              subPath: log4j2.properties
+            # config mount
+            {{- range $path, $_ := .Files.Glob "conf/*" }}
+            - name: seatunnel-configs
+              mountPath: /opt/seatunnel/config/{{ base $path }}
+              subPath: {{ base $path }}
+            {{- end }}
       volumes:
         - name: seatunnel-configs
           configMap:
-            name: {{ include "seatunnel.fullname" . }}-configs
+            name: {{ include "seatunnel.fullname" . }}-configs

deploy/kubernetes/seatunnel/templates/ingress.yaml

@@ -33,6 +33,9 @@ metadata:
     {{- toYaml . | nindent 4 }}
   {{- end }}
 spec:
+  {{- if .Values.ingress.className }}
+  ingressClassName: {{ .Values.ingress.className }}
+  {{- end }}
   rules:
   - host: "{{ .Values.ingress.host }}"
     http:

deploy/kubernetes/seatunnel/templates/service-master-headless.yaml

@@ -14,7 +14,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
----
 # use for access seatunnel from outside system via rest api
 apiVersion: v1
 kind: Service
@@ -31,5 +30,4 @@ spec:
       targetPort: 8080
       protocol: TCP
   selector:
-    {{- include "seatunnel.master.labels" . | nindent 4 }}
----
+    {{- include "seatunnel.master.labels" . | nindent 4 }}

deploy/kubernetes/seatunnel/values.yaml

@@ -19,16 +19,20 @@
 # This is a YAML-formatted file.
 # Declare variables to be passed into your templates.
 
-timezone: "Asia/Shanghai"
-
-
 image:
   registry: "apache/seatunnel"
   tag: ""
   pullPolicy: "IfNotPresent"
   pullSecret: ""
 
+# The env for pod
+env:
+  - name: TZ
+    value: Asia/Shanghai
+
 master:
+  ## The command to start master.
+  command: []
   ## The deployment strategy to use to replace existing pods with new ones.
   strategy:
     type: RollingUpdate
@@ -67,11 +71,6 @@ master:
   #     cpu: "500m"
   ## Periodic probe of container liveness. Container will be restarted if the probe fails. Cannot be updated.
   ## More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes
-  
-  ## service config for restApi
-  service:
-    type: "ClusterIP"
-
   livenessProbe:
     tcpSocket:
       port: hazelcast-port
@@ -91,6 +90,8 @@ master:
     successThreshold: 1
 
 worker:
+  ## The command to start worker.
+  command: []
   ## The deployment strategy to use to replace existing pods with new ones.
   strategy:
     type: RollingUpdate
@@ -130,8 +131,6 @@ worker:
   #     cpu: "500m"
   ## Periodic probe of container liveness. Container will be restarted if the probe fails. Cannot be updated.
   ## More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes
-
-
   livenessProbe:
     tcpSocket:
       port: hazelcast-port
@@ -152,8 +151,9 @@ worker:
 
 ingress:
   enabled: false
-  host: "seatunnel.org"
-  path: "/"
+  className: ""
+  host: seatunnel.k8s.local
+  path: /
   annotations: {}
   tls:
     enabled: false