Skip to content

Commit 06832fd

Browse files
michaelkoepfmichaelkoepf
committed
[Dist] Adapt configuration files and CLI tooling
- Remove server.yaml and introduce common.yaml config file - Introduce dedicated config files for coordinator and tablet-server to support different config options (host, port) for bind.listeners and Kafka - Adapt config.sh to support common.yaml but ensuring backward compatibility for old server.yaml - Adapt assemblies for newly introduced config files
1 parent be747bf commit 06832fd

File tree

5 files changed

+131
-47
lines changed

5 files changed

+131
-47
lines changed

‎fluss-dist/src/main/assemblies/bin.xml‎

Lines changed: 15 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -14,9 +14,9 @@
1414
limitations under the License.
1515
-->
1616
<assembly
17-
xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0"
18-
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
19-
xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0 http://maven.apache.org/xsd/assembly-1.1.0.xsd">
17+
xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0"
18+
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
19+
xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0 http://maven.apache.org/xsd/assembly-1.1.0.xsd">
2020
<id>bin</id>
2121
<formats>
2222
<format>dir</format>
@@ -54,9 +54,19 @@
5454
<fileMode>0644</fileMode>
5555
</file>
5656

57-
<!-- copy the config file -->
57+
<!-- copy config files -->
5858
<file>
59-
<source>src/main/resources/server.yaml</source>
59+
<source>src/main/resources/common.yaml</source>
60+
<outputDirectory>conf</outputDirectory>
61+
<fileMode>0644</fileMode>
62+
</file>
63+
<file>
64+
<source>src/main/resources/coordinator-server.yaml</source>
65+
<outputDirectory>conf</outputDirectory>
66+
<fileMode>0644</fileMode>
67+
</file>
68+
<file>
69+
<source>src/main/resources/tablet-server.yaml</source>
6070
<outputDirectory>conf</outputDirectory>
6171
<fileMode>0644</fileMode>
6272
</file>

‎fluss-dist/src/main/resources/bin/config.sh‎

Lines changed: 20 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -164,11 +164,26 @@ findLakehousePaimonJar() {
164164
echo "$LAKEHOUSE_PAIMON"
165165
}
166166

167+
findFlussConfFile() {
168+
fluss_conf_dir=$1
167169

170+
if [[ -f "$fluss_conf_dir/server.yaml" && -f "$fluss_conf_dir/common.yaml" ]]; then
171+
(>&2 echo "[ERROR] Misconfiguration detected. Only one of server.yaml and common.yaml must be present in $fluss_conf_dir.")
172+
exit 1
173+
elif [[ -f "$fluss_conf_dir/server.yaml" ]]; then
174+
# backward compatability (< 0.7.0)
175+
echo "server.yaml"
176+
elif [[ -f "$fluss_conf_dir/common.yaml" ]]; then
177+
# from 0.7.0 onwards
178+
echo "common.yaml"
179+
else
180+
(>&2 echo "[ERROR] Could neither find server.yaml nor common.yaml in $fluss_conf_dir.")
181+
exit 1
182+
fi
183+
}
168184

169185

170-
# WARNING !!! , these values are only used if there is nothing else is specified in
171-
# conf/server.yaml
186+
# WARNING !!! , these values are only used if there is nothing else is specified in conf/common.yaml
172187

173188
DEFAULT_ENV_PID_DIR="/tmp" # Directory to store *.pid files to
174189
DEFAULT_ENV_LOG_MAX=10 # Maximum number of old log files to keep
@@ -181,7 +196,7 @@ DEFAULT_ENV_SSH_OPTS="" # Optional SSH parameters ru
181196

182197

183198
########################################################################################################################
184-
# CONFIG KEYS: The default values can be overwritten by the following keys in conf/server.yaml
199+
# CONFIG KEYS: The default values can be overwritten by the following keys in conf/common.yaml
185200
########################################################################################################################
186201
KEY_ENV_PID_DIR="env.pid.dir"
187202
KEY_ENV_LOG_DIR="env.log.dir"
@@ -238,7 +253,7 @@ if [ -z "$FLUSS_CONF_DIR" ]; then FLUSS_CONF_DIR=$FLUSS_HOME_DIR_MANGLED/conf; f
238253
FLUSS_BIN_DIR=$FLUSS_HOME_DIR_MANGLED/bin
239254
FLUSS_OPT_DIR=$FLUSS_HOME_DIR_MANGLED/opt
240255
DEFAULT_FLUSS_LOG_DIR=$FLUSS_HOME_DIR_MANGLED/log
241-
FLUSS_CONF_FILE="server.yaml"
256+
FLUSS_CONF_FILE=$(findFlussConfFile "${FLUSS_CONF_DIR}")
242257
YAML_CONF=${FLUSS_CONF_DIR}/${FLUSS_CONF_FILE}
243258

244259

@@ -262,7 +277,7 @@ if [ -z "${MY_JAVA_HOME}" ]; then
262277
fi
263278
# check if we have a valid JAVA_HOME and if java is not available
264279
if [ -z "${MY_JAVA_HOME}" ] && ! type java > /dev/null 2> /dev/null; then
265-
echo "Please specify JAVA_HOME. Either in Fluss config ./conf/server.yaml or as system-wide JAVA_HOME."
280+
echo "Please specify JAVA_HOME. Either in Fluss config ./conf/$FLUSS_CONF_FILE or as system-wide JAVA_HOME."
266281
exit 1
267282
else
268283
JAVA_HOME="${MY_JAVA_HOME}"

fluss-dist/src/main/resources/server.yaml renamed to fluss-dist/src/main/resources/common.yaml

Lines changed: 14 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -14,14 +14,19 @@
1414
# limitations under the License.
1515
################################################################################
1616

17+
# Common default configuration file that is used by ALL start scripts, the Fluss Servers and the Lakehouse CLI.
18+
# In addition, there exist additional, dedicated default configuration files for some components that are also placed
19+
# in this folder. The dedicated configuration files are ONLY read by the respective Java applications but NOT by the
20+
# start scripts. In that case, configuration options in the dedicated files overwrite configuration options
21+
# in the common (i.e., this) file.
22+
1723
# These parameters are required to run ./bin/lakehouse.sh for Java 17 support.
1824
# They can be safely removed when using Java 8/11.
1925
env.java.opts.all: --add-opens=java.base/java.util=ALL-UNNAMED
2026

2127
#==============================================================================
2228
# Zookeeper
2329
#==============================================================================
24-
2530
# Zookeeper connection string (see zookeeper docs for details).
2631
# This is a comma separated host:port pairs, each corresponding to a zk
2732
# server. e.g. "127.0.0.1:3000,127.0.0.1:3001,127.0.0.1:3002".
@@ -30,55 +35,27 @@ env.java.opts.all: --add-opens=java.base/java.util=ALL-UNNAMED
3035
zookeeper.address: localhost:2181
3136

3237
#==============================================================================
33-
# Common
38+
# Data and FileSystems
3439
#==============================================================================
35-
36-
# THe default bucket number to be used when creating tables if no bucket number
37-
# is specified for the table.
38-
default.bucket.number: 1
39-
40-
# The default replication factor to be used when creating tables if no replication
41-
# factor is specified for the table.
42-
default.replication.factor: 1
43-
44-
# The local data directory to be used for Fluss to storing kv and log data.
45-
data.dir: /tmp/fluss-data
46-
4740
# The remote data directory to be used for Fluss. Now, it is only used for storing
4841
# kv snapshot data. Expected be set to a remote location like: oss://bucket/path for oss
4942
# filesystem or hdfs://namenode:port/path for hdfs filesystem
5043
remote.data.dir: /tmp/fluss-remote-data
5144

52-
#==============================================================================
53-
# Listeners
54-
#==============================================================================
55-
56-
# The network address and port to which the server binds for accepting connections.
57-
# The format is '{listener_name}://{host}:{port}', and multiple addresses can be specified, separated by commas.
58-
bind.listeners: FLUSS://localhost:9123
59-
60-
61-
#==============================================================================
62-
# Tablet Server
63-
#==============================================================================
64-
65-
66-
# The id of the tablet server to be run, must be set and should be different
67-
# when running multiple tablet servers.
68-
tablet-server.id: 0
69-
70-
#==============================================================================
71-
# OSS FileSystem
72-
#==============================================================================
73-
7445
# The configuration for oss filesystem when using oss as the remote data directory
7546
# fs.oss.endpoint: xxx
7647
# fs.oss.accessKeyId: xxx
7748
# fs.oss.accessKeySecret: xxx
7849

50+
#==============================================================================
51+
# Lakehouse
52+
#==============================================================================
53+
#datalake.format: paimon
54+
#datalake.paimon.metastore: filesystem
55+
#datalake.paimon.warehouse: /tmp/paimon
56+
7957
#==============================================================================
8058
# Fluss Kafka Configs
8159
#==============================================================================
8260
kafka.enabled: false
83-
kafka.port: 9092
8461
kafka.database: _kafka

‎fluss-dist/src/main/resources/coordinator-server.yaml‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
################################################################################
2+
# Copyright (c) 2025 Alibaba Group Holding Ltd.
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License");
5+
# you may not use this file except in compliance with the License.
6+
# You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
################################################################################
16+
17+
# Dedicated default configuration file for coordinator servers.
18+
19+
#==============================================================================
20+
# Tables
21+
#==============================================================================
22+
# The default bucket number to be used when creating tables if no bucket number
23+
# is specified for the table.
24+
default.bucket.number: 1
25+
26+
# The default replication factor to be used when creating tables if no replication
27+
# factor is specified for the table.
28+
default.replication.factor: 1
29+
30+
#==============================================================================
31+
# Listeners
32+
#==============================================================================
33+
# The network address and port to which the server binds for accepting connections.
34+
# The format is '{listener_name}://{host}:{port}', and multiple addresses can be specified, separated by commas.
35+
bind.listeners: FLUSS://localhost:9123
36+
37+
#==============================================================================
38+
# Fluss Kafka Configs
39+
#==============================================================================
40+
kafka.port: 9092

‎fluss-dist/src/main/resources/tablet-server.yaml‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
################################################################################
2+
# Copyright (c) 2025 Alibaba Group Holding Ltd.
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License");
5+
# you may not use this file except in compliance with the License.
6+
# You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
################################################################################
16+
17+
# Dedicated default configuration file for tablet servers.
18+
19+
#==============================================================================
20+
# Tablet Server
21+
#==============================================================================
22+
# The id of the tablet server to be run, must be set and should be different
23+
# when running multiple tablet servers.
24+
tablet-server.id: 0
25+
26+
#==============================================================================
27+
# Listeners
28+
#==============================================================================
29+
# The network address and port to which the server binds for accepting connections.
30+
# The format is '{listener_name}://{host}:{port}', and multiple addresses can be specified, separated by commas.
31+
bind.listeners: FLUSS://localhost:9124
32+
33+
#==============================================================================
34+
# Data and FileSystems
35+
#==============================================================================
36+
# The local data directory to be used for Fluss to storing kv and log data.
37+
data.dir: /tmp/fluss-data
38+
39+
#==============================================================================
40+
# Fluss Kafka Configs
41+
#==============================================================================
42+
kafka.port: 9093

0 commit comments

Comments
 (0)