executor: add slow-log file meta cache to avoid repeat read file meta information

domain/domain.go

@@ -31,6 +31,7 @@ import (
 	"github.com/pingcap/parser/model"
 	"github.com/pingcap/parser/mysql"
 	"github.com/pingcap/parser/terror"
+	"github.com/pingcap/sysutil/cache"
 	"github.com/pingcap/tidb/bindinfo"
 	"github.com/pingcap/tidb/config"
 	"github.com/pingcap/tidb/ddl"
@@ -86,6 +87,7 @@ type Domain struct {
 	statsUpdating        sync2.AtomicInt32
 	cancel               context.CancelFunc
 	indexUsageSyncLease  time.Duration
+	logFileMetaCache     *cache.LogFileMetaCache
 
 	serverID             uint64
 	serverIDSession      *concurrency.Session
@@ -689,6 +691,7 @@ func NewDomain(store kv.Storage, ddlLease time.Duration, statsLease time.Duratio
 		infoHandle:          infoschema.NewHandle(store),
 		slowQuery:           newTopNSlowQueries(30, time.Hour*24*7, 500),
 		indexUsageSyncLease: idxUsageSyncLease,
+		logFileMetaCache:    cache.NewLogFileMetaCache(),
 	}
 
 	do.SchemaValidator = NewSchemaValidator(ddlLease, do)
@@ -1330,6 +1333,11 @@ func (do *Domain) IsLostConnectionToPD() bool {
 	return do.isLostConnectionToPD.Get() != 0
 }
 
+// GetLogFileMetaCache gets slow-log file meta cache.
+func (do *Domain) GetLogFileMetaCache() *cache.LogFileMetaCache {
+	return do.logFileMetaCache
+}
+
 const (
 	serverIDEtcdPath               = "/tidb/server_id"
 	refreshServerIDRetryCnt        = 3

executor/slow_query.go

@@ -34,6 +34,8 @@ import (
 	"github.com/pingcap/parser/model"
 	"github.com/pingcap/parser/mysql"
 	"github.com/pingcap/parser/terror"
+	"github.com/pingcap/sysutil/cache"
+	"github.com/pingcap/tidb/domain"
 	"github.com/pingcap/tidb/infoschema"
 	plannercore "github.com/pingcap/tidb/planner/core"
 	"github.com/pingcap/tidb/privilege"
@@ -1002,6 +1004,13 @@ func (e *slowQueryRetriever) getAllFiles(ctx context.Context, sctx sessionctx.Co
 	if err != nil {
 		return nil, err
 	}
+	dom := domain.GetDomain(sctx)
+	var ca *cache.LogFileMetaCache
+	if dom != nil {
+		ca = dom.GetLogFileMetaCache()
+	} else {
+		ca = cache.NewLogFileMetaCache()
+	}
 	walkFn := func(path string, info os.FileInfo) error {
 		if info.IsDir() {
 			return nil
@@ -1024,8 +1033,23 @@ func (e *slowQueryRetriever) getAllFiles(ctx context.Context, sctx sessionctx.Co
 				terror.Log(file.Close())
 			}
 		}()
+		stat, err := file.Stat()
+		if err != nil {
+			return handleErr(err)
+		}
+		meta := ca.GetFileMata(stat)
+		if meta == nil {
+			meta = cache.NewLogFileMeta(stat)
+			defer ca.AddFileMataToCache(stat, meta)
+		} else {
+			if meta.CheckFileNotModified(stat) && meta.IsInValid() {
+				return nil
+			}
+		}
 		// Get the file start time.
-		fileStartTime, err := e.getFileStartTime(ctx, file)
+		fileStartTime, err := meta.GetStartTime(stat, func() (time.Time, error) {
+			return e.getFileStartTime(ctx, file)
+		})
 		if err != nil {
 			return handleErr(err)
 		}
@@ -1042,7 +1066,9 @@ func (e *slowQueryRetriever) getAllFiles(ctx context.Context, sctx sessionctx.Co
 		}
 
 		// Get the file end time.
-		fileEndTime, err := e.getFileEndTime(ctx, file)
+		fileEndTime, err := meta.GetEndTime(stat, func() (time.Time, error) {
+			return e.getFileEndTime(ctx, file)
+		})
 		if err != nil {
 			return handleErr(err)
 		}
@@ -1093,6 +1119,9 @@ func (e *slowQueryRetriever) getFileStartTime(ctx context.Context, file *os.File
 	for {
 		lineByte, err := getOneLine(reader)
 		if err != nil {
+			if err == io.EOF {
+				return t, cache.InvalidLogFile
+			}
 			return t, err
 		}
 		line := string(lineByte)
@@ -1107,7 +1136,7 @@ func (e *slowQueryRetriever) getFileStartTime(ctx context.Context, file *os.File
 			return t, ctx.Err()
 		}
 	}
-	return t, errors.Errorf("malform slow query file %v", file.Name())
+	return t, cache.InvalidLogFile
 }
 
 func (e *slowQueryRetriever) getRuntimeStats() execdetails.RuntimeStats {
@@ -1189,7 +1218,7 @@ func (e *slowQueryRetriever) getFileEndTime(ctx context.Context, file *os.File)
 			return t, ctx.Err()
 		}
 	}
-	return t, errors.Errorf("invalid slow query file %v", file.Name())
+	return t, cache.InvalidLogFile
 }
 
 const maxReadCacheSize = 1024 * 1024 * 64

go.mod

@@ -84,3 +84,5 @@ require (
 )
 
 go 1.13
+
+replace github.com/pingcap/sysutil => github.com/crazycs520/sysutil v0.0.0-20201207085517-9dd18ed31826

go.sum

@@ -173,6 +173,10 @@ github.com/couchbase/vellum v1.0.1/go.mod h1:FcwrEivFpNi24R3jLOs3n+fs5RnuQnQqCLB
 github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE=
 github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
 github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
+github.com/crazycs520/sysutil v0.0.0-20201203131959-c22c2028beb7 h1:6uU9EvYWskrDjxuS+x/qyiBvOz7E3If9gH/vqtUki7I=
+github.com/crazycs520/sysutil v0.0.0-20201203131959-c22c2028beb7/go.mod h1:EB/852NMQ+aRKioCpToQ94Wl7fktV+FNnxf3CX/TTXI=
+github.com/crazycs520/sysutil v0.0.0-20201207085517-9dd18ed31826 h1:t3zIDqYADdl/S4dtKEMCZMZ2GN131/qWjMEiXwhHQ7g=
+github.com/crazycs520/sysutil v0.0.0-20201207085517-9dd18ed31826/go.mod h1:EB/852NMQ+aRKioCpToQ94Wl7fktV+FNnxf3CX/TTXI=
 github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/creasty/defaults v1.3.0/go.mod h1:CIEEvs7oIVZm30R8VxtFJs+4k201gReYyuYHJxZc68I=