fix: content clean sql (#92)

* fix: content clean sql

* chore: request id null index

Author: zijiren233

README.md

@@ -110,7 +110,7 @@ docker-compose up -d
 - `LOG_DETAIL_REQUEST_BODY_MAX_SIZE`: Maximum size for request body in log details, default is `128KB`
 - `LOG_DETAIL_RESPONSE_BODY_MAX_SIZE`: Maximum size for response body in log details, default is `128KB`
 - `LOG_DETAIL_STORAGE_HOURS`: Hours to store log details, default is `72` (3 days)
-- `CLEAN_LOG_BATCH_SIZE`: Batch size for cleaning logs, default is `5000`
+- `CLEAN_LOG_BATCH_SIZE`: Batch size for cleaning logs, default is `2000`
 
 ### Service Control
 

README.zh.md

@@ -111,7 +111,7 @@ docker-compose up -d
 - `LOG_DETAIL_REQUEST_BODY_MAX_SIZE`: 日志详情请求体最大大小，默认 `128KB`
 - `LOG_DETAIL_RESPONSE_BODY_MAX_SIZE`: 日志详情响应体最大大小，默认 `128KB`
 - `LOG_DETAIL_STORAGE_HOURS`: 日志详情存储时间，默认 `72`（3 天）
-- `CLEAN_LOG_BATCH_SIZE`: 日志清理批量大小，默认 `5000`
+- `CLEAN_LOG_BATCH_SIZE`: 日志清理批量大小，默认 `2000`
 
 ### 服务控制
 

common/config/config.go

@@ -29,7 +29,7 @@ var (
 	logDetailRequestBodyMaxSize  int64 = 128 * 1024 // 128KB
 	logDetailResponseBodyMaxSize int64 = 128 * 1024 // 128KB
 	logDetailStorageHours        int64 = 3 * 24     // 3 days
-	cleanLogBatchSize            int64 = 5000
+	cleanLogBatchSize            int64 = 2000
 	internalToken                atomic.Value
 	notifyNote                   atomic.Value
 	ipGroupsThreshold            int64

main.go

@@ -251,7 +251,7 @@ func cleanLog(ctx context.Context) {
 		case <-ctx.Done():
 			return
 		case <-ticker.C:
-			optimize := trylock.Lock("optimizeLog", time.Minute*15)
+			optimize := trylock.Lock("optimizeLog", time.Hour*24)
 			err := model.CleanLog(int(config.GetCleanLogBatchSize()), optimize)
 			if err != nil {
 				notify.ErrorThrottle("cleanLog", time.Minute, "clean log failed", err.Error())

model/log.go

@@ -65,13 +65,13 @@ type Log struct {
 	Content              EmptyNullString `gorm:"type:text"                                                      json:"content,omitempty"`
 	GroupID              string          `json:"group,omitempty"`
 	Model                string          `json:"model"`
-	RequestID            EmptyNullString `gorm:"index"                                                          json:"request_id"`
+	RequestID            EmptyNullString `gorm:"index:,where:request_id is not null"                            json:"request_id"`
 	ID                   int             `gorm:"primaryKey"                                                     json:"id"`
 	TokenID              int             `gorm:"index"                                                          json:"token_id,omitempty"`
 	ChannelID            int             `json:"channel,omitempty"`
 	Code                 int             `gorm:"index"                                                          json:"code,omitempty"`
 	Mode                 int             `json:"mode,omitempty"`
-	IP                   EmptyNullString `gorm:"index"                                                          json:"ip,omitempty"`
+	IP                   EmptyNullString `gorm:"index:,where:ip is not null"                                    json:"ip,omitempty"`
 	RetryTimes           ZeroNullInt64   `json:"retry_times,omitempty"`
 	DownstreamResult     bool            `json:"downstream_result,omitempty"`
 	Price                Price           `gorm:"embedded"                                                       json:"price,omitempty"`
@@ -264,9 +264,9 @@ func cleanLog(batchSize int, optimize bool) error {
 		Model(&Log{}).
 		Where(
 			"created_at < ?",
-			time.Now().Add(-time.Duration(logContentStorageHours)*time.Hour),
+			time.Now().Truncate(time.Hour).Add(-time.Duration(logContentStorageHours)*time.Hour),
 		).
-		Where("endpoint IS NOT NULL OR ip IS NOT NULL OR content IS NOT NULL OR ttfb_milliseconds IS NOT NULL").
+		Where("endpoint IS NOT NULL").
 		Order("created_at DESC").
 		Limit(1).
 		Select("id").
@@ -282,7 +282,7 @@ func cleanLog(batchSize int, optimize bool) error {
 			Model(&Log{}).
 			Session(&gorm.Session{SkipDefaultTransaction: true}).
 			Where(
-				"id BETWEEN ? AND ?",
+				"id BETWEEN ? AND ? AND content IS NOT NULL",
 				id-int64(batchSize),
 				id,
 			).