feat: add cleanup parameters in async logging

feat: add cleanup parameters in async logging

feat: add cleanup parameters in async logging

Author: JustinChengLZ

cmd/base/options/log.go

@@ -29,14 +29,17 @@ type LogsOptions struct {
 	SupportAsyncLogging bool
 	LogDir              string
 	LogBufferSizeMB     int
+	LogFileMaxAge       int
+	LogFileMaxBackups   int
 }
 
 func NewLogsOptions() *LogsOptions {
 	return &LogsOptions{
 		LogPackageLevel:    general.LoggingPKGFull,
 		LogFileMaxSizeInMB: 1800,
-		LogDir:             "/opt/tiger/toutiao/log/app",
 		LogBufferSizeMB:    10000,
+		LogFileMaxAge:      7,
+		LogFileMaxBackups:  10,
 	}
 }
 
@@ -47,6 +50,8 @@ func (o *LogsOptions) AddFlags(fs *pflag.FlagSet) {
 	fs.BoolVar(&o.SupportAsyncLogging, "support-async-logging", o.SupportAsyncLogging, "whether to support async logging")
 	fs.StringVar(&o.LogDir, "log-dir", o.LogDir, "directory of log file")
 	fs.IntVar(&o.LogBufferSizeMB, "log-buffer-size", o.LogBufferSizeMB, "size of the ring buffer to store async logs")
+	fs.IntVar(&o.LogFileMaxAge, "log-file-max-age", o.LogFileMaxAge, "max age of klog log file in days")
+	fs.IntVar(&o.LogFileMaxBackups, "log-file-max-backups", o.LogFileMaxBackups, "max number of klog log file backups")
 }
 
 func (o *LogsOptions) ApplyTo(c *generic.LogConfiguration) error {
@@ -56,5 +61,7 @@ func (o *LogsOptions) ApplyTo(c *generic.LogConfiguration) error {
 	c.LogDir = o.LogDir
 	c.LogFileMaxSize = int(o.LogFileMaxSizeInMB)
 	c.LogBufferSizeMB = o.LogBufferSizeMB
+	c.LogFileMaxAge = o.LogFileMaxAge
+	c.LogFileMaxBackups = o.LogFileMaxBackups
 	return nil
 }

cmd/katalyst-agent/app/agent.go

@@ -64,7 +64,7 @@ func Run(
 	}
 
 	if conf.SupportAsyncLogging {
-		asyncLogger := logging.NewAsyncLogger(genericCtx, conf.LogFileMaxSize, conf.LogBufferSizeMB)
+		asyncLogger := logging.NewAsyncLogger(genericCtx, conf.LogDir, conf.LogFileMaxSize, conf.LogFileMaxAge, conf.LogFileMaxBackups, conf.LogBufferSizeMB)
 		defer asyncLogger.Shutdown()
 	}
 

pkg/config/generic/log.go

@@ -21,6 +21,8 @@ type LogConfiguration struct {
 	LogDir              string
 	LogFileMaxSize      int
 	LogBufferSizeMB     int
+	LogFileMaxAge       int
+	LogFileMaxBackups   int
 }
 
 func NewLogConfiguration() *LogConfiguration {

pkg/util/logging/logger.go

@@ -17,6 +17,7 @@ limitations under the License.
 package logging
 
 import (
+	"path"
 	"time"
 
 	"github.com/rs/zerolog/diode"
@@ -44,10 +45,10 @@ const (
 )
 
 const (
-	defaultInfoLogFileName    = "/opt/tiger/toutiao/log/app/agent.info.log"
-	defaultWarningLogFileName = "/opt/tiger/toutiao/log/app/agent.warning.log"
-	defaultErrorLogFileName   = "/opt/tiger/toutiao/log/app/agent.error.log"
-	defaultFatalLogFileName   = "/opt/tiger/toutiao/log/app/agent.fatal.log"
+	defaultInfoLogFileName    = "agent.info.log"
+	defaultWarningLogFileName = "agent.warning.log"
+	defaultErrorLogFileName   = "agent.error.log"
+	defaultFatalLogFileName   = "agent.fatal.log"
 )
 
 type logInfo struct {
@@ -64,20 +65,23 @@ var logInfoMap = map[SeverityName]*logInfo{
 
 type AsyncLogger struct {
 	diodeWriters []diode.Writer
-	logFile      string
 }
 
 // NewAsyncLogger creates an async logger that produces an async writer for each of the severity levels.
 // The async writer spins up a goroutine that periodically flushes the buffered logs to disk.
-func NewAsyncLogger(agentCtx *agent.GenericContext, maxSizeMB int, bufferSizeMB int) *AsyncLogger {
+func NewAsyncLogger(
+	agentCtx *agent.GenericContext, logDir string, maxSizeMB, maxAge, maxBackups, bufferSizeMB int,
+) *AsyncLogger {
 	wrappedEmitter := agentCtx.EmitterPool.GetDefaultMetricsEmitter()
 
 	asyncLogger := &AsyncLogger{}
 	for severity, logInfo := range logInfoMap {
 		// lumberjackLogger is a logger that rotates log files
 		lumberjackLogger := &lumberjack.Logger{
-			Filename: logInfo.fileName,
-			MaxSize:  maxSizeMB,
+			Filename:   path.Join(logDir, logInfo.fileName),
+			MaxSize:    maxSizeMB,
+			MaxAge:     maxAge,
+			MaxBackups: maxBackups,
 		}
 
 		// diodeWriter is a writer that stores logs in a ring buffer and asynchronously flushes them

pkg/util/logging/logger_test.go

@@ -33,7 +33,7 @@ func TestNewAsyncLogger(t *testing.T) {
 			EmitterPool: metrics_pool.DummyMetricsEmitterPool{},
 		},
 	}
-	asyncLogger := NewAsyncLogger(agentCtx, 100, 100)
+	asyncLogger := NewAsyncLogger(agentCtx, "testDir", 100, 100, 100, 100)
 	assert.NotNil(t, asyncLogger)
 
 	assert.Equal(t, len(asyncLogger.diodeWriters), 4)