Merge branch 'apache:dev' into dev

Author: scienceyang

Diff for: .dlc.json

@@ -23,6 +23,9 @@
     },
     {
       "pattern": "^https://twitter.com/ASFSeaTunnel"
+    },
+    {
+      "pattern": "^https://github.com/apache/seatunnel/commit/"
     }
   ],
   "timeout": "10s",

Diff for: .github/workflows/backend.yml

@@ -59,6 +59,18 @@ jobs:
         env:
           MAVEN_OPTS: -Xmx512m
 
+  helm-chart-check:
+    name: Check Helm Chart Syntax
+    needs: [ license-header, code-style]
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Setup Helm
+        uses: azure/[email protected]
+        id: install
+      - name: Lint Chart
+        run: helm lint deploy/kubernetes/seatunnel
+
   dead-link:
     name: Dead links
     runs-on: ubuntu-latest
@@ -309,7 +321,7 @@ jobs:
           bash seatunnel-pr/tools/documents/sync.sh seatunnel-pr seatunnel-website
       - uses: actions/setup-node@v2
         with:
-          node-version: 16.19.0
+          node-version: 18.20.7
       - name: Run docusaurus build
         run: |
           cd seatunnel-website

Diff for: .github/workflows/codeql.yaml

@@ -43,17 +43,17 @@ jobs:
           java-version:  8
           distribution: 'adopt'
       - name: Cache local Maven repository
-        uses: actions/cache@v2
+        uses: actions/cache@v4
         with:
           path: ~/.m2/repository
           key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
           restore-keys: |
             ${{ runner.os }}-maven-
       - name: Initialize CodeQL
-        uses: github/codeql-action/init@v1
+        uses: github/codeql-action/init@v2
         with:
           languages: ${{ matrix.language }}
       - name: Autobuild
-        uses: github/codeql-action/autobuild@v1
+        uses: github/codeql-action/autobuild@v2
       - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@v1
+        uses: github/codeql-action/analyze@v2

Diff for: .github/workflows/labeler/label-scope-conf.yml

@@ -36,12 +36,38 @@ spark:
   - changed-files:
       - any-glob-to-any-file:
           - seatunnel-translation/seatunnel-translation-spark/**
+
+Zeta Rest API:
+  - changed-files:
+      - any-glob-to-any-file: seatunnel-engine/**/server/rest/**
+api:
+  - changed-files:
+      - any-glob-to-any-file:
+          - seatunnel-api/**
+          - seatunnel-common/**
+core:
+  - changed-files:
+      - any-glob-to-any-file:
+          - seatunnel-core/**
+          - seatunnel-config/**
+          - seatunnel-dist/**
+          - seatunnel-plugin-discovery/**
+          - seatunnel-shade/**
+format:
+  - changed-files:
+      - any-glob-to-any-file: seatunnel-formats/**
+dependencies:
+  - changed-files:
+      - any-glob-to-any-file: tools/dependencies/**
+
 connectors-v2:
   - changed-files:
       - any-glob-to-any-file: seatunnel-connectors-v2/**
 transform-v2:
   - changed-files:
       - any-glob-to-any-file: seatunnel-transforms-v2/**
+
+# Connectors
 amazondynamodb:
   - all:
       - changed-files:
@@ -275,26 +301,13 @@ typesense:
           - any-glob-to-any-file: seatunnel-connectors-v2/connector-typesense/**
           - all-globs-to-all-files: '!seatunnel-connectors-v2/connector-!(typesense)/**'
 
-Zeta Rest API:
-  - changed-files:
-      - any-glob-to-any-file: seatunnel-engine/**/server/rest/**
-api:
-  - changed-files:
-      - any-glob-to-any-file:
-          - seatunnel-api/**
-          - seatunnel-common/**
-core:
-  - changed-files:
-      - any-glob-to-any-file:
-          - seatunnel-core/**
-format:
-  - changed-files:
-      - any-glob-to-any-file: seatunnel-formats/**
-dependencies:
-  - changed-files:
-      - any-glob-to-any-file: tools/dependencies/**
 sls:
   - all:
       - changed-files:
           - any-glob-to-any-file: seatunnel-connectors-v2/connector-sls/**
           - all-globs-to-all-files: '!seatunnel-connectors-v2/connector-!(sls)/**'
+aerospike:
+  - all:
+      - changed-files:
+          - any-glob-to-any-file: seatunnel-connectors-v2/connector-aerospike/**
+          - all-globs-to-all-files: '!seatunnel-connectors-v2/connector-!(aerospike)/**'

Diff for: NOTICE

@@ -102,4 +102,9 @@ The class com.hazelcast.internal.util.ConcurrentReferenceHashMap contains code w
 and updated within the WildFly project (https://github.com/wildfly/wildfly).
 
 The class org.apache.calcite.linq4j.tree.ConstantExpression contains code
-originating from the Calcite project (https://github.com/apache/calcite).
+originating from the Calcite project (https://github.com/apache/calcite).
+
+Aerospike Sink Connector
+Copyright 2023 The original authors.
+Contains Aerospike Client Library (https://www.aerospike.com/)
+which is licensed under the AGPL 3.0 License (https://www.aerospike.com/terms/download/3rd-party-licenses)

Diff for: bin/install-plugin.cmd

@@ -22,8 +22,8 @@ REM Get seatunnel home
 set "SEATUNNEL_HOME=%~dp0..\"
 echo Set SEATUNNEL_HOME to [%SEATUNNEL_HOME%]
 
-REM Connector default version is 2.3.10, you can also choose a custom version. eg: 2.3.10:  install-plugin.bat 2.3.10
-set "version=2.3.10"
+REM Connector default version is 2.3.11, you can also choose a custom version. eg: 2.3.11:  install-plugin.bat 2.3.11
+set "version=2.3.11"
 if not "%~1"=="" set "version=%~1"
 
 REM Create the lib directory

Diff for: bin/install-plugin.sh

@@ -23,8 +23,8 @@
 # get seatunnel home
 SEATUNNEL_HOME=$(cd $(dirname $0);cd ../;pwd)
 
-# connector default version is 2.3.10, you can also choose a custom version. eg: 2.3.10:  sh install-plugin.sh 2.3.10
-version=2.3.10
+# connector default version is 2.3.11, you can also choose a custom version. eg: 2.3.11:  sh install-plugin.sh 2.3.11
+version=2.3.11
 
 if [ -n "$1" ]; then
     version="$1"

Diff for: config/log4j2.properties

@@ -65,6 +65,7 @@ appender.routing.name = routingAppender
 appender.routing.type = Routing
 appender.routing.purge.type = IdlePurgePolicy
 appender.routing.purge.timeToLive = 60
+appender.routing.purge.checkInterval = 1
 appender.routing.route.type = Routes
 appender.routing.route.pattern = $${ctx:ST-JID}
 appender.routing.route.system.type = Route
@@ -100,4 +101,4 @@ appender.file.strategy.action.condition.nested_condition.type = IfAny
 appender.file.strategy.action.condition.nested_condition.lastModify.type = IfLastModified
 appender.file.strategy.action.condition.nested_condition.lastModify.age = ${file_ttl}
 appender.file.strategy.action.condition.nested_condition.fileCount.type = IfAccumulatedFileCount
-appender.file.strategy.action.condition.nested_condition.fileCount.exceeds = ${file_count}
+appender.file.strategy.action.condition.nested_condition.fileCount.exceeds = ${file_count}

Diff for: deploy/kubernetes/seatunnel/conf/jvm_client_options

@@ -0,0 +1,24 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# JVM Heap
+-Xms256m
+-Xmx512m
+
+# JVM Dump
+-XX:+HeapDumpOnOutOfMemoryError
+-XX:HeapDumpPath=/tmp/seatunnel/dump/zeta-client

Diff for: deploy/kubernetes/seatunnel/conf/jvm_master_options

@@ -0,0 +1,30 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# JVM Heap
+# -Xms2g
+# -Xmx2g
+
+# JVM Dump
+-XX:+HeapDumpOnOutOfMemoryError
+-XX:HeapDumpPath=/tmp/seatunnel/dump/zeta-server
+
+# Metaspace
+-XX:MaxMetaspaceSize=2g
+
+# G1GC
+-XX:+UseG1GC

Diff for: deploy/kubernetes/seatunnel/conf/jvm_worker_options

@@ -0,0 +1,30 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# JVM Heap
+# -Xms2g
+# -Xmx2g
+
+# JVM Dump
+-XX:+HeapDumpOnOutOfMemoryError
+-XX:HeapDumpPath=/tmp/seatunnel/dump/zeta-server
+
+# Metaspace
+-XX:MaxMetaspaceSize=2g
+
+# G1GC
+-XX:+UseG1GC

Diff for: deploy/kubernetes/seatunnel/templates/NOTES.txt

@@ -26,36 +26,14 @@ Access seatunnel UI URL by:
 
   For more restapi please refer to https://seatunnel.apache.org/docs/{{ .Chart.AppVersion }}/seatunnel-engine/rest-api-v2
 
-{{- else if eq .Values.master.service.type "ClusterIP" }}
+{{- else }}
 
   kubectl port-forward -n {{ .Release.Namespace }} svc/{{ template "seatunnel.fullname" . }}-master 8080:8080
 
   seatunnel restapi URL for running jobs: http://127.0.0.1:8080/running-jobs
   seatunnel restapi URL for system monitoring information: http://127.0.0.1:8080/system-monitoring-information
 
   For more restapi please refer to https://seatunnel.apache.org/docs/{{ .Chart.AppVersion }}/seatunnel-engine/rest-api-v2
-
-
-{{- else if eq .Values.master.service.type "NodePort" }}
-
-  NODE_IP=$(kubectl get no -n {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}")
-  NODE_PORT=$(kubectl get svc {{ template "seatunnel.fullname" . }}-master -n {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}")
-  seatunnel restapi URL for running jobs: http://$NODE_IP:$NODE_PORT/running-jobs
-  seatunnel restapi URL for system monitoring information: http://$NODE_IP:$NODE_PORT/system-monitoring-information
-
-  For more restapi please refer to https://seatunnel.apache.org/docs/{{ .Chart.AppVersion }}/seatunnel-engine/rest-api-v2
-{{- else if eq .Values.master.service.type "LoadBalancer" }}
-
-  NOTE: It may take a few minutes for the LoadBalancer IP to be available.
-        You can watch the status of by running 'kubectl get svc {{ template "seatunnel.fullname" . }}-master -n {{ .Release.Namespace }} -w'
-
-  SERVICE_IP=$(kubectl get svc {{ template "seatunnel.fullname" . }}-master -n {{ .Release.Namespace }} -o jsonpath='{.status.loadBalancer.ingress[0].ip}')
-  echo http://$SERVICE_IP:8080/running-jobs
-
-  seatunnel resetapi URL for running jobs: http://$SERVICE_IP:8080/running-jobs
-  seatunnel resetapi URL for system monitoring information: http://$SERVICE_IP:8080/system-monitoring-information
-
-  For more restapi please refer to https://seatunnel.apache.org/docs/{{ .Chart.AppVersion }}/seatunnel-engine/rest-api-v2
 {{- end }}
 
 Or you can just go into master pod, and use local curl command.

Diff for: deploy/kubernetes/seatunnel/templates/configmap.yaml

@@ -22,14 +22,8 @@ metadata:
     app.kubernetes.io/name: {{ include "seatunnel.fullname" . }}-configs
     {{- include "seatunnel.master.labels" . | nindent 4 }}
 data:
-  hazelcast-master.yaml: |-
-  {{ tpl (.Files.Get "conf/hazelcast-master.yaml") $ | nindent 4 }}
-  hazelcast-worker.yaml: |-
-  {{ tpl (.Files.Get "conf/hazelcast-worker.yaml") $ | nindent 4 }}
-  hazelcast-client.yaml: |-
-  {{ tpl (.Files.Get "conf/hazelcast-client.yaml") $ | nindent 4 }}
-  seatunnel.yaml: |-
-  {{ tpl (.Files.Get "conf/seatunnel.yaml") $ | nindent 4 }}
-  log4j2.properties: |-
-  {{ tpl (.Files.Get "conf/log4j2.properties") $ | nindent 4 }}
+  {{- range $path, $_ := .Files.Glob "conf/*" }}
+    {{- base $path | nindent 2 }}: |-
+      {{- tpl ($.Files.Get $path) $ | nindent 4 -}}
+  {{- end }}
 

Diff for: deploy/kubernetes/seatunnel/templates/deployment-seatunnel-master.yaml

@@ -65,7 +65,11 @@ spec:
               name: "hazelcast-port"
             - containerPort: 8080
               name: "master-port"
+          {{- if .Values.master.command }}
+          command: {{ .Values.master.command }}
+          {{- else }}
           command: ["/bin/sh","-c","/opt/seatunnel/bin/seatunnel-cluster.sh -r master"]
+          {{- end }}
           {{- if .Values.master.resources }}
           resources:
             {{- toYaml .Values.master.resources | nindent 12 }}
@@ -74,19 +78,17 @@ spec:
           livenessProbe:
             {{- toYaml .Values.master.livenessProbe | nindent 12 }}
           {{- end }}
+          {{- if .Values.env }}
+          env:
+            {{- toYaml .Values.env | nindent 12 }}
+          {{- end }}
           volumeMounts:
-           - mountPath: "/opt/seatunnel/config/hazelcast-master.yaml"
-             name: seatunnel-configs
-             subPath: hazelcast-master.yaml
-           - mountPath: "/opt/seatunnel/config/hazelcast-client.yaml"
-             name: seatunnel-configs
-             subPath: hazelcast-client.yaml
-           - mountPath: "/opt/seatunnel/config/seatunnel.yaml"
-             name: seatunnel-configs
-             subPath: seatunnel.yaml
-           - mountPath: "/opt/seatunnel/config/log4j2.properties"
-             name: seatunnel-configs
-             subPath: log4j2.properties
+            # config mount
+            {{- range $path, $_ := .Files.Glob "conf/*" }}
+            - name: seatunnel-configs
+              mountPath: /opt/seatunnel/config/{{ base $path }}
+              subPath: {{ base $path }}
+            {{- end }}
       volumes:
         - name: seatunnel-configs
           configMap: