Skip to content

[Feature] Add RPM generation support #9181

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: dev
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 13 additions & 11 deletions seatunnel-core/seatunnel-starter/src/main/bin/seatunnel-cluster.sh
Original file line number Diff line number Diff line change
Expand Up @@ -34,13 +34,15 @@ while [ -h "$PRG" ] ; do
done

PRG_DIR=`dirname "$PRG"`
APP_DIR=`cd "$PRG_DIR/.." >/dev/null; pwd`
CONF_DIR=${APP_DIR}/config
APP_JAR=${APP_DIR}/starter/seatunnel-starter.jar
APP_DIR_DEFAULT=`cd "$PRG_DIR/.." >/dev/null; pwd`
APP_DIR=${APP_DIR:-"${APP_DIR_DEFAULT}"}
CONF_DIR=${CONF_DIR:-${APP_DIR}/config}
LOG_DIR=${LOG_DIR:-${APP_DIR}/logs}
MASTER_OUT="${LOG_DIR}/seatunnel-engine-master.out"
WORKER_OUT="${LOG_DIR}/seatunnel-engine-worker.out"
OUT="${LOG_DIR}/seatunnel-server.out"
APP_JAR="${APP_DIR}/starter/seatunnel-starter.jar"
APP_MAIN="org.apache.seatunnel.core.starter.seatunnel.SeaTunnelServer"
MASTER_OUT="${APP_DIR}/logs/seatunnel-engine-master.out"
WORKER_OUT="${APP_DIR}/logs/seatunnel-engine-worker.out"
OUT="${APP_DIR}/logs/seatunnel-server.out"
HELP=false
NODE_ROLE="master_and_worker"

Expand Down Expand Up @@ -86,7 +88,7 @@ JAVA_OPTS="${JAVA_OPTS} -Dlog4j2.contextSelector=org.apache.logging.log4j.core.a
JAVA_OPTS="${JAVA_OPTS} -Dlog4j2.isThreadContextMapInheritable=true -DAsyncLogger.ThreadNameStrategy=UNCACHED"
if [ -e "${CONF_DIR}/log4j2.properties" ]; then
JAVA_OPTS="${JAVA_OPTS} -Dhazelcast.logging.type=log4j2 -Dlog4j2.configurationFile=${CONF_DIR}/log4j2.properties"
JAVA_OPTS="${JAVA_OPTS} -Dseatunnel.logs.path=${APP_DIR}/logs"
JAVA_OPTS="${JAVA_OPTS} -Dseatunnel.logs.path=${LOG_DIR}"
fi

if [ "$NODE_ROLE" = "master" ]; then
Expand All @@ -96,7 +98,7 @@ if [ "$NODE_ROLE" = "master" ]; then
if [[ ! "$line" =~ ^# ]]; then
JAVA_OPTS="$JAVA_OPTS $line"
fi
done < ${APP_DIR}/config/jvm_master_options
done < "${CONF_DIR}"/jvm_master_options
# SeaTunnel Engine Config
if [ -z $HAZELCAST_CONFIG ]; then
HAZELCAST_CONFIG=${CONF_DIR}/hazelcast-master.yaml
Expand All @@ -118,7 +120,7 @@ elif [ "$NODE_ROLE" = "master_and_worker" ]; then
if [[ ! "$line" =~ ^# ]]; then
JAVA_OPTS="$JAVA_OPTS $line"
fi
done < ${APP_DIR}/config/jvm_options
done < "${CONF_DIR}"/jvm_options
if [ -z $HAZELCAST_CONFIG ]; then
HAZELCAST_CONFIG=${CONF_DIR}/hazelcast.yaml
fi
Expand Down Expand Up @@ -153,8 +155,8 @@ CLASS_PATH=${APP_DIR}/lib/*:${APP_JAR}
echo "start ${NODE_ROLE} node"

if [[ $DAEMON == true && $HELP == false ]]; then
if [[ ! -d ${APP_DIR}/logs ]]; then
mkdir -p ${APP_DIR}/logs
if [[ ! -d "${LOG_DIR}" ]]; then
mkdir -p "${LOG_DIR}"
fi
touch $OUT
nohup java ${JAVA_OPTS} -cp ${CLASS_PATH} ${APP_MAIN} ${args} > "$OUT" 200<&- 2>&1 < /dev/null &
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,10 @@
#

set -eu
if [ -f "/etc/seatunnel/conf/seatunnel-env.sh" ]; then
. "/etc/seatunnel/conf/seatunnel-env.sh"
echo "Using /etc/seatunnel/conf/seatunnel-env.sh."
fi
# resolve links - $0 may be a softlink
PRG="$0"

Expand All @@ -34,7 +38,8 @@ while [ -h "$PRG" ] ; do
done

PRG_DIR=$(dirname "$PRG")
APP_DIR=$(cd "$PRG_DIR/.." >/dev/null; pwd)
APP_DIR_DEFAULT=$(cd "$PRG_DIR/.." >/dev/null; pwd)
APP_DIR=${APP_DIR:-$APP_DIR_DEFAULT}
APP_JAR=${APP_DIR}/starter/seatunnel-starter.jar
LOAD_CLASS="org.apache.seatunnel.core.starter.seatunnel.SeaTunnelConnector"

Expand Down
18 changes: 13 additions & 5 deletions seatunnel-core/seatunnel-starter/src/main/bin/seatunnel.sh
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,12 @@
#

set -eu

if [ -f "/etc/seatunnel/conf/seatunnel-env.sh" ]; then
. "/etc/seatunnel/conf/seatunnel-env.sh"
echo "Using /etc/seatunnel/conf/seatunnel-env.sh."
fi

# resolve links - $0 may be a softlink
PRG="$0"

Expand All @@ -34,9 +40,11 @@ while [ -h "$PRG" ] ; do
done

PRG_DIR=`dirname "$PRG"`
APP_DIR=`cd "$PRG_DIR/.." >/dev/null; pwd`
SEATUNNEL_HOME=${APP_DIR}
CONF_DIR=${APP_DIR}/config
APP_DIR_DEFAULT=`cd "$PRG_DIR/.." >/dev/null; pwd`
APP_DIR=${APP_DIR:-$APP_DIR_DEFAULT}
LOG_DIR=${LOG_DIR:-${APP_DIR}/logs}
CONF_DIR=${CONF_DIR:-${APP_DIR}/config}

APP_JAR=${APP_DIR}/starter/seatunnel-starter.jar
APP_MAIN="org.apache.seatunnel.core.starter.seatunnel.SeaTunnelClient"

Expand Down Expand Up @@ -83,7 +91,7 @@ JAVA_OPTS="${JAVA_OPTS} -Dhazelcast.config=${HAZELCAST_CONFIG}"
JAVA_OPTS="${JAVA_OPTS} -Dlog4j2.isThreadContextMapInheritable=true"
if [ -e "${CONF_DIR}/log4j2_client.properties" ]; then
JAVA_OPTS="${JAVA_OPTS} -Dhazelcast.logging.type=log4j2 -Dlog4j2.configurationFile=${CONF_DIR}/log4j2_client.properties"
JAVA_OPTS="${JAVA_OPTS} -Dseatunnel.logs.path=${APP_DIR}/logs"
JAVA_OPTS="${JAVA_OPTS} -Dseatunnel.logs.path=${LOG_DIR}"
if [[ $args == *" -m local"* || $args == *" --master local"* || $args == *" -e local"* || $args == *" --deploy-mode local"* ]]; then
ntime=$(echo `date "+%N"`|sed -r 's/^0+//')
JAVA_OPTS="${JAVA_OPTS} -Dseatunnel.logs.file_name=seatunnel-starter-client-$((`date '+%s'`*1000+$ntime/1000000))"
Expand All @@ -98,7 +106,7 @@ while IFS= read -r line || [[ -n "$line" ]]; do
if [[ ! $line == \#* ]]; then
JAVA_OPTS="$JAVA_OPTS $line"
fi
done < ${APP_DIR}/config/jvm_client_options
done < "${CONF_DIR}"/jvm_client_options

# Parse JvmOption from command line, it should be parsed after jvm_client_options
for i in "$@"
Expand Down
111 changes: 111 additions & 0 deletions seatunnel-dist/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,11 @@
<properties>
<!-- disable mvn deploy to central maven repo by default -->
<maven.deploy.skip>true</maven.deploy.skip>
<rpm.build.disable>true</rpm.build.disable>
<rpm.release.number>1</rpm.release.number>
<rpm.stack.version>${project.version}-${rpm.release.number}</rpm.stack.version>
<rpm.stack.root>/usr/bigtop</rpm.stack.root>
<rpm.version>${project.version}-${rpm.release.number}</rpm.version>
</properties>

<build>
Expand Down Expand Up @@ -68,6 +73,112 @@
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-antrun-plugin</artifactId>
<version>1.8</version>
<executions>
<execution>
<id>postinstall_script</id>
<goals>
<goal>run</goal>
</goals>
<phase>generate-resources</phase>
<configuration>
<target>
<echo append="false" file="${project.build.directory}/postinstall.sh">#!/bin/sh
mkdir -p ${rpm.stack.root}/current
ln -sfn ${rpm.stack.root}/${rpm.version}/seatunnel ${rpm.stack.root}/current
chmod 755 ${rpm.stack.root}/${rpm.version}/seatunnel/bin/*.sh
chmod 755 /etc/seatunnel/conf/*.sh</echo>
</target>
</configuration>
</execution>
<execution>
<id>postremove_script</id>
<goals>
<goal>run</goal>
</goals>
<phase>generate-resources</phase>
<configuration>
<target>
<echo append="false" file="${project.build.directory}/postremove.sh">#!/bin/sh
rm -rf ${rpm.stack.root}/current/seatunnel
rm -rf ${rpm.stack.root}/${rpm.version}</echo>
</target>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>rpm-maven-plugin</artifactId>
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Some question:

  1. Do we need users to manually build rpm packages at present?
  2. Can the corresponding documentation be added?
  3. Can we push rpm to some third-party repositories in the future? For example, EPEL?

<version>2.3.0</version>
<executions>
<execution>
<id>generate-rpm</id>
<goals>
<goal>rpm</goal>
</goals>
<phase>package</phase>
<configuration>
<group>Development</group>
<name>seatunnel</name>
<version>${project.version}</version>
<release>${rpm.release.number}</release>
<needarch>x86_64</needarch>
<group>Application/Java</group>
<defaultFilemode>644</defaultFilemode>
<defaultDirmode>755</defaultDirmode>
<defaultUsername>root</defaultUsername>
<defaultGroupname>root</defaultGroupname>
<autoRequires>false</autoRequires>
<disabled>${rpm.build.disable}</disabled>
<mappings>
<mapping>
<directory>${rpm.stack.root}/${rpm.version}/seatunnel/</directory>
<sources>
<source>
<location>${project.build.directory}/apache-seatunnel-${project.version}-bin/apache-seatunnel-${project.version}/</location>
<excludes>
<exclude>config/**</exclude>
</excludes>
</source>
</sources>
</mapping>
<mapping>
<directory>/etc/seatunnel/conf/</directory>
<sources>
<source>
<location>${project.build.directory}/apache-seatunnel-${project.version}-bin/apache-seatunnel-${project.version}/config/</location>
</source>
</sources>
</mapping>
<mapping>
<directory>/usr/sbin</directory>
<filemode>755</filemode>
<username>root</username>
<groupname>root</groupname>
<directoryIncluded>false</directoryIncluded>
<sources>
<source>
<location>${project.basedir}/sbin/seatunnel</location>
</source>
</sources>
</mapping>
</mappings>
<postinstallScriptlet>
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

should we have a preInstallScriptlet, /usr/bigtop, which is defined as an rpm root, may not be present,

<preInstallScriptlet>
  <![CDATA[
  #!/bin/bash
  mkdir -p /usr/bigtop
  ]]>
</preInstallScriptlet>

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

During the installation of the RPM, if the rpm.stack.root directory (which is configurable but defaults to /usr/bigtop) does not exist, the yum install command will create it and then copy the necessary files and folders into it. Since we access the rpm.stack.root after the installation, it is guaranteed that the stack root will always be present.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

thanks for the clarification, if we can accommodate in RPM installation guide, it would be helpulf, -Drpm.stack.root=/usr/

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Sure, I will add the documentation later as part of resolving task #9196

<scriptFile>${project.build.directory}/postinstall.sh</scriptFile>
<fileEncoding>utf-8</fileEncoding>
</postinstallScriptlet>
<postremoveScriptlet>
<scriptFile>${project.build.directory}/postremove.sh</scriptFile>
<fileEncoding>utf-8</fileEncoding>
</postremoveScriptlet>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>

Expand Down
58 changes: 58 additions & 0 deletions seatunnel-dist/sbin/seatunnel
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
#!/bin/sh
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
set -

if [ -f "/etc/seatunnel/conf/seatunnel-env.sh" ]; then
. "/etc/seatunnel/conf/seatunnel-env.sh"
echo "Using /etc/seatunnel/conf/seatunnel-env.sh."
else
echo "File /etc/seatunnel/conf/seatunnel-env.sh does not exist."
exit 1
fi
if [ -z "${APP_DIR}" ]; then
echo "APP_DIR is not set. It must be set in seatunnel-env.sh"
exit 1
fi
BIN_DIR="${APP_DIR}/bin"

start() {
echo "starting seatunnel"
. "${BIN_DIR}/seatunnel-cluster.sh" -d
echo "seatunnel started"
}

stop() {
echo "stopping seatunnel..."
. "${BIN_DIR}/stop-seatunnel-cluster.sh"
echo "seatunnel stopped"
}

status() {
if jcmd | grep -q -i 'org.apache.seatunnel.core.starter.seatunnel.SeaTunnelServer'; then
echo "seatunnel is running"
else
echo "seatunnel is not running"
fi
}

case "$1" in
"start") start ;;
"stop") stop ;;
"status") status ;;
*) echo "Usage: seatunnel {start|stop|status}"; exit 1 ;;
esac
1 change: 1 addition & 0 deletions seatunnel-dist/src/main/assembly/assembly-bin-ci.xml
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
xsi:schemaLocation="http://maven.apache.org/ASSEMBLY/2.1.0 http://maven.apache.org/xsd/assembly-2.1.0.xsd">
<id>bin</id>
<formats>
<format>dir</format>
<format>tar.gz</format>
</formats>
<includeBaseDirectory>true</includeBaseDirectory>
Expand Down
Loading