Skip to content

Commit a8cef41

Browse files
rionmonsterrionmonster
authored
[server] Updated Default RocksDB Log Directory to a separate directory (#1675)
1 parent 2a3381b commit a8cef41

File tree

3 files changed

+15
-6
lines changed

3 files changed

+15
-6
lines changed

‎fluss-server/src/main/java/org/apache/fluss/server/kv/rocksdb/RocksDBResourceContainer.java‎

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
import org.apache.fluss.config.ConfigOptions;
2323
import org.apache.fluss.config.Configuration;
2424
import org.apache.fluss.config.ReadableConfig;
25+
import org.apache.fluss.utils.FileUtils;
2526
import org.apache.fluss.utils.IOUtils;
2627

2728
import org.rocksdb.BlockBasedTableConfig;
@@ -42,6 +43,7 @@
4243
import javax.annotation.Nullable;
4344

4445
import java.io.File;
46+
import java.io.IOException;
4547
import java.util.ArrayList;
4648
import java.util.Collection;
4749
import java.util.List;
@@ -99,7 +101,7 @@ public RocksDBResourceContainer(
99101
}
100102

101103
/** Gets the RocksDB {@link DBOptions} to be used for RocksDB instances. */
102-
public DBOptions getDbOptions() {
104+
public DBOptions getDbOptions() throws IOException {
103105
// initial options from common profile
104106
DBOptions opt = createBaseCommonDBOptions();
105107
handlesToClose.add(opt);
@@ -175,7 +177,8 @@ private <T> T internalGetOption(ConfigOption<T> option) {
175177
}
176178

177179
@SuppressWarnings("ConstantConditions")
178-
private DBOptions setDBOptionsFromConfigurableOptions(DBOptions currentOptions) {
180+
private DBOptions setDBOptionsFromConfigurableOptions(DBOptions currentOptions)
181+
throws IOException {
179182
currentOptions.setMaxBackgroundJobs(
180183
internalGetOption(ConfigOptions.KV_MAX_BACKGROUND_THREADS));
181184

@@ -280,12 +283,16 @@ private ColumnFamilyOptions setColumnFamilyOptionsFromConfigurableOptions(
280283
*
281284
* @param dbOptions The RocksDB {@link DBOptions}.
282285
*/
283-
private void relocateDefaultDbLogDir(DBOptions dbOptions) {
286+
private void relocateDefaultDbLogDir(DBOptions dbOptions) throws IOException {
284287
String logFilePath = System.getProperty("log.file");
285288
if (logFilePath != null) {
286289
File logFile = resolveFileLocation(logFilePath);
287290
if (logFile != null && resolveFileLocation(logFile.getParent()) != null) {
288-
dbOptions.setDbLogDir(logFile.getParent());
291+
File logFileDirectory = logFile.getParentFile();
292+
File rocksDbLogDirectory = FileUtils.createDirectory(logFileDirectory, "rocksdb");
293+
if (resolveFileLocation(rocksDbLogDirectory.getAbsolutePath()) != null) {
294+
dbOptions.setDbLogDir(rocksDbLogDirectory.getAbsolutePath());
295+
}
289296
}
290297
}
291298
}

‎fluss-server/src/test/java/org/apache/fluss/server/kv/rocksdb/RocksDBResourceContainerTest.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -108,7 +108,9 @@ void testDefaultDbLogDir(@TempDir Path tempFolder) throws Exception {
108108
System.setProperty("log.file", logFile.getPath());
109109
try (RocksDBResourceContainer container = new RocksDBResourceContainer()) {
110110
assertThat(container.getDbOptions().infoLogLevel()).isEqualTo(InfoLogLevel.INFO_LEVEL);
111-
assertThat(container.getDbOptions().dbLogDir()).isEqualTo(logFile.getParent());
111+
File rocksDbLogDirectory = new File(logFile.getParent(), "rocksdb");
112+
assertThat(container.getDbOptions().dbLogDir())
113+
.isEqualTo(rocksDbLogDirectory.getAbsolutePath());
112114
} finally {
113115
logFile.delete();
114116
}

‎website/docs/maintenance/configuration.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -136,7 +136,7 @@ during the Fluss cluster working.
136136
| kv.rocksdb.files.open | Integer | -1 | The maximum number of open files (per bucket of table) that can be used by the DB, `-1` means no limit. The default value is `-1`. |
137137
| kv.rocksdb.log.max-file-size | MemorySize | 25mb | The maximum size of RocksDB's file used for information logging. If the log files becomes larger than this, a new file will be created. If 0, all logs will be written to one log file. The default maximum file size is `25MB`. |
138138
| kv.rocksdb.log.file-num | Integer | 4 | The maximum number of files RocksDB should keep for information logging (Default setting: 4). |
139-
| kv.rocksdb.log.dir | String | (None) | The directory for RocksDB's information logging files. If empty (Fluss default setting), log files will be in the same directory as the Fluss log. If non-empty, this directory will be used and the data directory's absolute path will be used as the prefix of the log file name. If setting this option as a non-existing location, e.g `/dev/null`, RocksDB will then create the log under its own database folder as before. |
139+
| kv.rocksdb.log.dir | String | `${FLUSS_HOME}/log/rocksdb` | The directory for RocksDB's information logging files. If empty (Fluss default setting), log files will be in the same directory as the Fluss log. If non-empty, this directory will be used and the data directory's absolute path will be used as the prefix of the log file name. If setting this option as a non-existing location, e.g `/dev/null`, RocksDB will then create the log under its own database folder as before. |
140140
| kv.rocksdb.log.level | Enum | INFO_LEVEL | The specified information logging level for RocksDB. Candidate log level is `DEBUG_LEVEL`, `INFO_LEVEL`, `WARN_LEVEL`, `ERROR_LEVEL`, `FATAL_LEVEL`, `HEADER_LEVEL`, NUM_INFO_LOG_LEVELS, . If unset, Fluss will use INFO_LEVEL. Note: RocksDB info logs will not be written to the Fluss's tablet server logs and there is no rolling strategy, unless you configure `kv.rocksdb.log.dir`, `kv.rocksdb.log.max-file-size` and `kv.rocksdb.log.file-num` accordingly. Without a rolling strategy, it may lead to uncontrolled disk space usage if configured with increased log levels! There is no need to modify the RocksDB log level, unless for troubleshooting RocksDB. |
141141
| kv.rocksdb.write-batch-size | MemorySize | 2mb | The max size of the consumed memory for RocksDB batch write, will flush just based on item count if this config set to 0. |
142142
| kv.rocksdb.compaction.style | Enum | LEVEL | The specified compaction style for DB. Candidate compaction style is LEVEL, FIFO, UNIVERSAL, or NONE, and Fluss chooses `LEVEL` as default style. |

0 commit comments

Comments
 (0)