feat(tablet-server): make controlled shutdown parameters configurable

Add configuration options for tablet server controlled shutdown retry behavior:
- tablet-server.controlled-shutdown.max-retries
- tablet-server.controlled-shutdown.retry-interval

Update documentation and add tests to verify the new configuration options work as expected

Author: hsavasere

fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java

@@ -418,6 +418,26 @@ public class ConfigOptions {
                                     + WRITER_ID_EXPIRATION_TIME.key()
                                     + " passing. The default value is 10 minutes.");
 
+    public static final ConfigOption<Integer> TABLET_SERVER_CONTROLLED_SHUTDOWN_MAX_RETRIES =
+            key("tablet-server.controlled-shutdown.max-retries")
+                    .intType()
+                    .defaultValue(3)
+                    .withDescription(
+                            "The maximum number of retries for controlled shutdown of the tablet server. "
+                                    + "During controlled shutdown, the tablet server attempts to transfer leadership "
+                                    + "of its buckets to other servers. If the transfer fails, it will retry up to "
+                                    + "this number of times before proceeding with shutdown. The default value is 3.");
+
+    public static final ConfigOption<Duration> TABLET_SERVER_CONTROLLED_SHUTDOWN_RETRY_INTERVAL =
+            key("tablet-server.controlled-shutdown.retry-interval")
+                    .durationType()
+                    .defaultValue(Duration.ofMillis(1000))
+                    .withDescription(
+                            "The interval between retries during controlled shutdown of the tablet server. "
+                                    + "When controlled shutdown fails to transfer bucket leadership, the tablet server "
+                                    + "will wait for this duration before attempting the next retry. "
+                                    + "The default value is 1000 milliseconds (1 second).");
+
     public static final ConfigOption<Integer> BACKGROUND_THREADS =
             key("server.background.threads")
                     .intType()

fluss-dist/src/main/resources/server.yaml

@@ -65,6 +65,10 @@ bind.listeners: FLUSS://localhost:9123
 # when running multiple tablet servers.
 tablet-server.id: 0
 
+# Controlled shutdown configuration for tablet servers
+# tablet-server.controlled-shutdown.max-retries: 3
+# tablet-server.controlled-shutdown.retry-interval: 1000ms
+
 #==============================================================================
 # OSS FileSystem
 #==============================================================================

fluss-server/src/main/java/org/apache/fluss/server/tablet/TabletServer.java

@@ -85,10 +85,6 @@ public class TabletServer extends ServerBase {
 
     private static final Logger LOG = LoggerFactory.getLogger(TabletServer.class);
 
-    // TODO, maybe need to make it configurable
-    private static final int CONTROLLED_SHUTDOWN_MAX_RETRIES = 3;
-    private static final long CONTROLLED_SHUTDOWN_RETRY_INTERVAL_MS = 1000L;
-
     private final int serverId;
 
     /**
@@ -452,7 +448,11 @@ private void controlledShutDown() {
         // a period of time and try again for a number of retries. If all the attempt fails, we
         // simply force the shutdown.
         boolean shutdownSucceeded = false;
-        int remainingRetries = CONTROLLED_SHUTDOWN_MAX_RETRIES;
+        int remainingRetries =
+                conf.getInt(ConfigOptions.TABLET_SERVER_CONTROLLED_SHUTDOWN_MAX_RETRIES);
+        long retryIntervalMs =
+                conf.get(ConfigOptions.TABLET_SERVER_CONTROLLED_SHUTDOWN_RETRY_INTERVAL).toMillis();
+
         while (!shutdownSucceeded && remainingRetries > 0) {
             remainingRetries--;
 
@@ -484,7 +484,7 @@ private void controlledShutDown() {
 
             if (!shutdownSucceeded && remainingRetries > 0) {
                 try {
-                    Thread.sleep(CONTROLLED_SHUTDOWN_RETRY_INTERVAL_MS);
+                    Thread.sleep(retryIntervalMs);
                 } catch (InterruptedException e) {
                     Thread.currentThread().interrupt();
                     break;

fluss-server/src/test/java/org/apache/fluss/server/tablet/TabletServerShutdownITCase.java

@@ -18,6 +18,7 @@
 package org.apache.fluss.server.tablet;
 
 import org.apache.fluss.config.ConfigOptions;
+import org.apache.fluss.config.Configuration;
 import org.apache.fluss.exception.RetriableException;
 import org.apache.fluss.metadata.Schema;
 import org.apache.fluss.metadata.TableBucket;
@@ -112,6 +113,35 @@ void testIOExceptionShouldStopTabletServer(boolean isLogTable) throws Exception
         FLUSS_CLUSTER_EXTENSION.startTabletServer(leader, true);
     }
 
+    @Test
+    void testControlledShutdownConfiguration() throws Exception {
+        // Test that the controlled shutdown configuration options are properly loaded
+        Configuration conf = new Configuration();
+
+        // Verify default values are loaded correctly
+        assertThat(conf.getInt(ConfigOptions.TABLET_SERVER_CONTROLLED_SHUTDOWN_MAX_RETRIES))
+                .isEqualTo(3);
+        assertThat(
+                        conf.get(ConfigOptions.TABLET_SERVER_CONTROLLED_SHUTDOWN_RETRY_INTERVAL)
+                                .toMillis())
+                .isEqualTo(1000L);
+
+        // Test custom configuration values
+        Configuration customConf = new Configuration();
+        customConf.set(ConfigOptions.TABLET_SERVER_CONTROLLED_SHUTDOWN_MAX_RETRIES, 5);
+        customConf.set(
+                ConfigOptions.TABLET_SERVER_CONTROLLED_SHUTDOWN_RETRY_INTERVAL,
+                Duration.ofMillis(2000));
+
+        assertThat(customConf.getInt(ConfigOptions.TABLET_SERVER_CONTROLLED_SHUTDOWN_MAX_RETRIES))
+                .isEqualTo(5);
+        assertThat(
+                        customConf
+                                .get(ConfigOptions.TABLET_SERVER_CONTROLLED_SHUTDOWN_RETRY_INTERVAL)
+                                .toMillis())
+                .isEqualTo(2000L);
+    }
+
     @Test
     void testControlledShutdown() throws Exception {
         FLUSS_CLUSTER_EXTENSION.assertHasTabletServerNumber(3);

website/docs/maintenance/operations/graceful-shutdown.md

@@ -69,8 +69,20 @@ kill -TERM <tablet-server-pid>
 
 #### Configuration Options
 
-- **Controlled Shutdown Retries**: Number of attempts to transfer leadership (`default:` 3 retries)
-- **Retry Interval**: Time between retry attempts (`default`: 1000L)
+The controlled shutdown process can be configured using the following options:
+
+- **`tablet-server.controlled-shutdown.max-retries`**: Maximum number of attempts to transfer leadership before proceeding with unclean shutdown (default: 3)
+- **`tablet-server.controlled-shutdown.retry-interval`**: Time interval between retry attempts (default: 1000ms)
+
+**Example Configuration:**
+
+```yaml
+# server.yaml
+tablet-server:
+  controlled-shutdown:
+    max-retries: 5
+    retry-interval: 2000ms
+```
 
 ## Monitoring Shutdown
 
@@ -110,8 +122,8 @@ Monitor shutdown-related metrics:
 
 | Configuration | Description | Default |
 |---------------|-------------|---------|
-| `controlled.shutdown.max.retries` | Maximum retries for controlled shutdown | 3 |
-| `controlled.shutdown.retry.interval.ms` | Interval between retry attempts | 5000 |
+| `tablet-server.controlled-shutdown.max-retries` | Maximum retries for controlled shutdown | 3 |
+| `tablet-server.controlled-shutdown.retry-interval` | Interval between retry attempts | 1000ms |
 | `shutdown.timeout.ms` | General shutdown timeout | 30000 |
 
 ## See Also