make tokeinzer size configurable

Author: gh123man

pkg/config/setup/config.go

@@ -1735,6 +1735,7 @@ func logsagent(config pkgconfigmodel.Setup) {
 	// Dynamic sampling configuration
 	config.BindEnvAndSetDefault("logs_config.dynamic_sampling_enabled", false)
 	config.BindEnvAndSetDefault("logs_config.dynamic_sampling.skip_json_logs", false)
+	config.BindEnvAndSetDefault("logs_config.dynamic_sampling.tokenizer_size", 10240)
 	config.BindEnvAndSetDefault("logs_config.dynamic_sampling.rate_ceiling", 1.0)
 	config.BindEnvAndSetDefault("logs_config.dynamic_sampling.fast_half_life_seconds", 2)
 	config.BindEnvAndSetDefault("logs_config.dynamic_sampling.slow_half_life_seconds", 60)

pkg/logs/internal/decoder/auto_multiline_detection/sample_aggregator.go

@@ -32,9 +32,10 @@ type TokenizedMessage struct {
 }
 
 func NewSampleAggregator(tailerInfo *status.InfoRegistry) *SampleAggregator {
+	tokenizerSize := pkgconfigsetup.Datadog().GetInt("logs_config.dynamic_sampling.tokenizer_size")
 
 	s := &SampleAggregator{
-		tokenizer: NewTokenizer(1024 * 10),
+		tokenizer: NewTokenizer(tokenizerSize),
 		sampler:   NewSampler(DefaultConfig()),
 	}
 	s.sampler.Start()