[Dist] Adapt configuration files and CLI tooling

- Remove server.yaml and introduce common.yaml config file
- Introduce dedicated config files for coordinator and tablet-server to support different config options (host, port) for bind.listeners and Kafka
- Adapt config.sh to support common.yaml but ensuring backward compatibility for old server.yaml
- Adapt assemblies for newly introduced config files

Author: michaelkoepf

fluss-dist/src/main/assemblies/bin.xml

@@ -14,9 +14,9 @@
  limitations under the License.
 -->
 <assembly
-    xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0"
-    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-    xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0 http://maven.apache.org/xsd/assembly-1.1.0.xsd">
+        xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0"
+        xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+        xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0 http://maven.apache.org/xsd/assembly-1.1.0.xsd">
     <id>bin</id>
     <formats>
         <format>dir</format>
@@ -54,9 +54,19 @@
             <fileMode>0644</fileMode>
         </file>
 
-        <!-- copy the config file -->
+        <!-- copy config files -->
         <file>
-            <source>src/main/resources/server.yaml</source>
+            <source>src/main/resources/common.yaml</source>
+            <outputDirectory>conf</outputDirectory>
+            <fileMode>0644</fileMode>
+        </file>
+        <file>
+            <source>src/main/resources/coordinator-server.yaml</source>
+            <outputDirectory>conf</outputDirectory>
+            <fileMode>0644</fileMode>
+        </file>
+        <file>
+            <source>src/main/resources/tablet-server.yaml</source>
             <outputDirectory>conf</outputDirectory>
             <fileMode>0644</fileMode>
         </file>

fluss-dist/src/main/resources/bin/config.sh

@@ -164,11 +164,26 @@ findLakehousePaimonJar() {
   echo "$LAKEHOUSE_PAIMON"
 }
 
+findFlussConfFile() {
+  fluss_conf_dir=$1
 
+  if [[ -f "$fluss_conf_dir/server.yaml" && -f "$fluss_conf_dir/common.yaml" ]]; then
+    (>&2 echo "[ERROR] Misconfiguration detected. Only one of server.yaml and common.yaml must be present in $fluss_conf_dir.")
+    exit 1
+  elif [[ -f "$fluss_conf_dir/server.yaml" ]]; then
+    # backward compatability (< 0.7.0)
+    echo "server.yaml"
+  elif [[ -f "$fluss_conf_dir/common.yaml" ]]; then
+    # from 0.7.0 onwards
+    echo "common.yaml"
+  else
+    (>&2 echo "[ERROR] Could neither find server.yaml nor common.yaml in $fluss_conf_dir.")
+    exit 1
+  fi
+}
 
 
-# WARNING !!! , these values are only used if there is nothing else is specified in
-# conf/server.yaml
+# WARNING !!! , these values are only used if there is nothing else is specified in conf/common.yaml
 
 DEFAULT_ENV_PID_DIR="/tmp"                          # Directory to store *.pid files to
 DEFAULT_ENV_LOG_MAX=10                              # Maximum number of old log files to keep
@@ -181,7 +196,7 @@ DEFAULT_ENV_SSH_OPTS=""                             # Optional SSH parameters ru
 
 
 ########################################################################################################################
-# CONFIG KEYS: The default values can be overwritten by the following keys in conf/server.yaml
+# CONFIG KEYS: The default values can be overwritten by the following keys in conf/common.yaml
 ########################################################################################################################
 KEY_ENV_PID_DIR="env.pid.dir"
 KEY_ENV_LOG_DIR="env.log.dir"
@@ -238,7 +253,7 @@ if [ -z "$FLUSS_CONF_DIR" ]; then FLUSS_CONF_DIR=$FLUSS_HOME_DIR_MANGLED/conf; f
 FLUSS_BIN_DIR=$FLUSS_HOME_DIR_MANGLED/bin
 FLUSS_OPT_DIR=$FLUSS_HOME_DIR_MANGLED/opt
 DEFAULT_FLUSS_LOG_DIR=$FLUSS_HOME_DIR_MANGLED/log
-FLUSS_CONF_FILE="server.yaml"
+FLUSS_CONF_FILE=$(findFlussConfFile "${FLUSS_CONF_DIR}")
 YAML_CONF=${FLUSS_CONF_DIR}/${FLUSS_CONF_FILE}
 
 
@@ -262,7 +277,7 @@ if [ -z "${MY_JAVA_HOME}" ]; then
 fi
 # check if we have a valid JAVA_HOME and if java is not available
 if [ -z "${MY_JAVA_HOME}" ] && ! type java > /dev/null 2> /dev/null; then
-    echo "Please specify JAVA_HOME. Either in Fluss config ./conf/server.yaml or as system-wide JAVA_HOME."
+    echo "Please specify JAVA_HOME. Either in Fluss config ./conf/$FLUSS_CONF_FILE or as system-wide JAVA_HOME."
     exit 1
 else
     JAVA_HOME="${MY_JAVA_HOME}"

fluss-dist/src/main/resources/server.yaml renamed to fluss-dist/src/main/resources/common.yaml

@@ -14,14 +14,19 @@
 # limitations under the License.
 ################################################################################
 
+# Common default configuration file that is used by ALL start scripts, the Fluss Servers and the Lakehouse CLI.
+# In addition, there exist additional, dedicated default configuration files for some components that are also placed
+# in this folder. The dedicated configuration files are ONLY read by the respective Java applications but NOT by the
+# start scripts. In that case, configuration options in the dedicated files overwrite configuration options
+# in the common (i.e., this) file.
+
 # These parameters are required to run ./bin/lakehouse.sh for Java 17 support.
 # They can be safely removed when using Java 8/11.
 env.java.opts.all: --add-opens=java.base/java.util=ALL-UNNAMED
 
 #==============================================================================
 # Zookeeper
 #==============================================================================
-
 # Zookeeper connection string (see zookeeper docs for details).
 # This is a comma separated host:port pairs, each corresponding to a zk
 # server. e.g. "127.0.0.1:3000,127.0.0.1:3001,127.0.0.1:3002".
@@ -30,55 +35,27 @@ env.java.opts.all: --add-opens=java.base/java.util=ALL-UNNAMED
 zookeeper.address: localhost:2181
 
 #==============================================================================
-# Common
+# Data and FileSystems
 #==============================================================================
-
-# THe default bucket number to be used when creating tables if no bucket number
-# is specified for the table.
-default.bucket.number: 1
-
-# The default replication factor to be used when creating tables if no replication
-# factor is specified for the table.
-default.replication.factor: 1
-
-# The local data directory to be used for Fluss to storing kv and log data.
-data.dir: /tmp/fluss-data
-
 # The remote data directory to be used for Fluss. Now, it is only used for storing
 # kv snapshot data. Expected be set to a remote location like: oss://bucket/path for oss
 # filesystem or hdfs://namenode:port/path for hdfs filesystem
 remote.data.dir: /tmp/fluss-remote-data
 
-#==============================================================================
-# Listeners
-#==============================================================================
-
-# The network address and port to which the server binds for accepting connections.
-# The format is '{listener_name}://{host}:{port}', and multiple addresses can be specified, separated by commas.
-bind.listeners: FLUSS://localhost:9123
-
-
-#==============================================================================
-# Tablet Server
-#==============================================================================
-
-
-# The id of the tablet server to be run, must be set and should be different
-# when running multiple tablet servers.
-tablet-server.id: 0
-
-#==============================================================================
-# OSS FileSystem
-#==============================================================================
-
 # The configuration for oss filesystem when using oss as the remote data directory
 # fs.oss.endpoint: xxx
 # fs.oss.accessKeyId: xxx
 # fs.oss.accessKeySecret: xxx
 
+#==============================================================================
+# Lakehouse
+#==============================================================================
+#datalake.format: paimon
+#datalake.paimon.metastore: filesystem
+#datalake.paimon.warehouse: /tmp/paimon
+
 #==============================================================================
 # Fluss Kafka Configs
 #==============================================================================
 kafka.enabled: false
-kafka.port: 9092
 kafka.database: _kafka

fluss-dist/src/main/resources/coordinator-server.yaml

@@ -0,0 +1,40 @@
+################################################################################
+# Copyright (c) 2025 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+# Dedicated default configuration file for coordinator servers.
+
+#==============================================================================
+# Tables
+#==============================================================================
+# The default bucket number to be used when creating tables if no bucket number
+# is specified for the table.
+default.bucket.number: 1
+
+# The default replication factor to be used when creating tables if no replication
+# factor is specified for the table.
+default.replication.factor: 1
+
+#==============================================================================
+# Listeners
+#==============================================================================
+# The network address and port to which the server binds for accepting connections.
+# The format is '{listener_name}://{host}:{port}', and multiple addresses can be specified, separated by commas.
+bind.listeners: FLUSS://localhost:9123
+
+#==============================================================================
+# Fluss Kafka Configs
+#==============================================================================
+kafka.port: 9092

fluss-dist/src/main/resources/tablet-server.yaml

@@ -0,0 +1,42 @@
+################################################################################
+# Copyright (c) 2025 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+# Dedicated default configuration file for tablet servers.
+
+#==============================================================================
+# Tablet Server
+#==============================================================================
+# The id of the tablet server to be run, must be set and should be different
+# when running multiple tablet servers.
+tablet-server.id: 0
+
+#==============================================================================
+# Listeners
+#==============================================================================
+# The network address and port to which the server binds for accepting connections.
+# The format is '{listener_name}://{host}:{port}', and multiple addresses can be specified, separated by commas.
+bind.listeners: FLUSS://localhost:9124
+
+#==============================================================================
+# Data and FileSystems
+#==============================================================================
+# The local data directory to be used for Fluss to storing kv and log data.
+data.dir: /tmp/fluss-data
+
+#==============================================================================
+# Fluss Kafka Configs
+#==============================================================================
+kafka.port: 9093