KAFKA-19080 The constraint on segment.ms is not enforced at topic level #19371
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -575,6 +575,7 @@ final class KafkaMetadataLog private ( | |
| } | ||
|
|
||
| object KafkaMetadataLog extends Logging { | ||
|
|
||
| def apply( | ||
| topicPartition: TopicPartition, | ||
| topicId: Uuid, | ||
|
|
@@ -583,35 +584,72 @@ object KafkaMetadataLog extends Logging { | |
| scheduler: Scheduler, | ||
| config: MetadataLogConfig | ||
| ): KafkaMetadataLog = { | ||
| val props: Properties = settingLogProperties(config) | ||
| LogConfig.validate(props) | ||
| val defaultLogConfig = new LogConfig(props) | ||
|
|
||
|
There was a problem hiding this comment. IIUC, this logic has been replaced by the config validation system. SEGMENT_BYTES_CONFIG has minimum of 1 MB but can be overridden by INTERNAL_SEGMENT_BYTES_CONFIG which has no minimum. |
||
| validateConfig(config, defaultLogConfig) | ||
|
|
||
| val metadataLog: KafkaMetadataLog = createKafkaMetadataLog(topicPartition, topicId, dataDir, time, scheduler, config, defaultLogConfig) | ||
|
|
||
| printWarningMessage(config, metadataLog) | ||
|
|
||
| // When recovering, truncate fully if the latest snapshot is after the log end offset. This can happen to a follower | ||
| // when the follower crashes after downloading a snapshot from the leader but before it could truncate the log fully. | ||
| metadataLog.truncateToLatestSnapshot() | ||
|
|
||
| metadataLog | ||
| } | ||
|
|
||
| private def printWarningMessage(config: MetadataLogConfig, metadataLog: KafkaMetadataLog): Unit = { | ||
| // Print a warning if users have overridden the internal config | ||
| if (config.logSegmentMinBytes != KafkaRaftClient.MAX_BATCH_SIZE_BYTES) { | ||
| metadataLog.error(s"Overriding ${KRaftConfigs.METADATA_LOG_SEGMENT_MIN_BYTES_CONFIG} is only supported for testing. Setting " + | ||
| s"this value too low may lead to an inability to write batches of metadata records.") | ||
| } | ||
| } | ||
|
|
||
| // visible for testing | ||
| def internalApply( | ||
|
There was a problem hiding this comment. Instead of having an There was a problem hiding this comment. +1 to move the internal config to There was a problem hiding this comment. I just realized that metadata log uses a different approach to allow tests to use a smaller segment bytes than allowed in production. That approach defines the original segment byte config with a small minimal requirement, but adds METADATA_LOG_SEGMENT_MIN_BYTES_CONFIG to enforce the actual minimal requirement in production. This new config could be changed in tests to allow for smaller minimal bytes. The benefit of this approach is that it allows the existing config to be used directly to set a smaller value for tests. The downside is that the doc for min value is inaccurate and the validation is done through a customized logic. It would be useful to pick the same strategy between metadata log and regular log. The metadata log approach seems slightly better since it's less intrusive. We could fix the inaccurate min value description for production somehow. There was a problem hiding this comment. Excuse me, the strategy used by metadata log is to add a "internal" config ( There was a problem hiding this comment. I am just saying that we now have two different ways to achieve the same goal. In the metadata log approach, you set the desired value through the original config, which is segment.bytes. You then set an internal config to change the min constraint. The approach in this PR is to set the desired value through a different internal config. It would be useful to choose same approach for both the metadata log and the regular log. There was a problem hiding this comment. Personally, I prefer the approach of adding "internal".xxx config as it provide better user experience for public configs, allowing users to see the "correct" min value. Additionally, we can remove the customized logic of validation. In short, I suggest to add following changes to this PR.
public int logSegmentBytes() {
if (internalSogSegmentBytes != null) return internalSogSegmentBytes;
return logSegmentBytes;
}There was a problem hiding this comment. Hmm, why do we need |
||
| topicPartition: TopicPartition, | ||
| topicId: Uuid, | ||
| dataDir: File, | ||
| time: Time, | ||
| scheduler: Scheduler, | ||
| config: MetadataLogConfig | ||
| ): KafkaMetadataLog = { | ||
| val props: Properties = settingLogProperties(config) | ||
| props.remove(TopicConfig.SEGMENT_BYTES_CONFIG); | ||
| props.put(LogConfig.INTERNAL_SEGMENT_BYTES_CONFIG, config.logSegmentBytes.toString) | ||
| LogConfig.validate(props) | ||
| val defaultLogConfig = new LogConfig(props) | ||
|
|
||
| validateConfig(config, defaultLogConfig) | ||
|
|
||
| val metadataLog: KafkaMetadataLog = createKafkaMetadataLog(topicPartition, topicId, dataDir, time, scheduler, config, defaultLogConfig) | ||
|
|
||
| // Print a warning if users have overridden the internal config | ||
| printWarningMessage(config, metadataLog) | ||
|
|
||
| // When recovering, truncate fully if the latest snapshot is after the log end offset. This can happen to a follower | ||
| // when the follower crashes after downloading a snapshot from the leader but before it could truncate the log fully. | ||
| metadataLog.truncateToLatestSnapshot() | ||
|
|
||
| metadataLog | ||
| } | ||
|
|
||
| private def createKafkaMetadataLog( | ||
| topicPartition: TopicPartition, | ||
| topicId: Uuid, | ||
| dataDir: File, | ||
| time: Time, | ||
| scheduler: Scheduler, | ||
| metadataLogConfig: MetadataLogConfig, | ||
| logConfig: LogConfig | ||
| ): KafkaMetadataLog = { | ||
| val log = UnifiedLog.create( | ||
| dataDir, | ||
| logConfig, | ||
| 0L, | ||
| 0L, | ||
| scheduler, | ||
|
|
@@ -631,20 +669,38 @@ object KafkaMetadataLog extends Logging { | |
| scheduler, | ||
| recoverSnapshots(log), | ||
| topicPartition, | ||
| metadataLogConfig | ||
| ) | ||
| metadataLog | ||
| } | ||
|
|
||
|
There was a problem hiding this comment. Should we keep the error message if INTERNAL_SEGMENT_BYTES_CONFIG has been set? This was added to help operators avoid setting these testing-only configs in a production environment. |
||
| private def validateConfig(config: MetadataLogConfig, defaultLogConfig: LogConfig): Unit = { | ||
| if (config.logSegmentBytes < config.logSegmentMinBytes) { | ||
| throw new InvalidConfigurationException( | ||
| s"Cannot set ${KRaftConfigs.METADATA_LOG_SEGMENT_BYTES_CONFIG} below ${config.logSegmentMinBytes}: ${config.logSegmentBytes}" | ||
| ) | ||
| } else if (defaultLogConfig.retentionMs >= 0) { | ||
| throw new InvalidConfigurationException( | ||
| s"Cannot set ${TopicConfig.RETENTION_MS_CONFIG} above -1: ${defaultLogConfig.retentionMs}." | ||
| ) | ||
| } else if (defaultLogConfig.retentionSize >= 0) { | ||
| throw new InvalidConfigurationException( | ||
| s"Cannot set ${TopicConfig.RETENTION_BYTES_CONFIG} above -1: ${defaultLogConfig.retentionSize}." | ||
| ) | ||
| } | ||
| } | ||
|
|
||
| private def settingLogProperties(config: MetadataLogConfig) = { | ||
| val props = new Properties() | ||
| props.setProperty(TopicConfig.MAX_MESSAGE_BYTES_CONFIG, config.maxBatchSizeInBytes.toString) | ||
| props.setProperty(TopicConfig.SEGMENT_BYTES_CONFIG, config.logSegmentBytes.toString) | ||
| props.setProperty(TopicConfig.SEGMENT_MS_CONFIG, config.logSegmentMillis.toString) | ||
| props.setProperty(TopicConfig.FILE_DELETE_DELAY_MS_CONFIG, ServerLogConfigs.LOG_DELETE_DELAY_MS_DEFAULT.toString) | ||
|
|
||
| // Disable time and byte retention when deleting segments | ||
| props.setProperty(TopicConfig.RETENTION_MS_CONFIG, "-1") | ||
| props.setProperty(TopicConfig.RETENTION_BYTES_CONFIG, "-1") | ||
| props | ||
| } | ||
|
|
||
| private def recoverSnapshots( | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
this is a critical bug - open https://issues.apache.org/jira/browse/KAFKA-19392 to fix it