diff --git a/domain/domain.go b/domain/domain.go index f4692f71814cf..5ea495f1709a2 100644 --- a/domain/domain.go +++ b/domain/domain.go @@ -31,6 +31,7 @@ import ( "github.com/pingcap/parser/model" "github.com/pingcap/parser/mysql" "github.com/pingcap/parser/terror" + "github.com/pingcap/sysutil/cache" "github.com/pingcap/tidb/bindinfo" "github.com/pingcap/tidb/config" "github.com/pingcap/tidb/ddl" @@ -86,6 +87,7 @@ type Domain struct { statsUpdating sync2.AtomicInt32 cancel context.CancelFunc indexUsageSyncLease time.Duration + logFileMetaCache *cache.LogFileMetaCache serverID uint64 serverIDSession *concurrency.Session @@ -689,6 +691,7 @@ func NewDomain(store kv.Storage, ddlLease time.Duration, statsLease time.Duratio infoHandle: infoschema.NewHandle(store), slowQuery: newTopNSlowQueries(30, time.Hour*24*7, 500), indexUsageSyncLease: idxUsageSyncLease, + logFileMetaCache: cache.NewLogFileMetaCache(), } do.SchemaValidator = NewSchemaValidator(ddlLease, do) @@ -1330,6 +1333,11 @@ func (do *Domain) IsLostConnectionToPD() bool { return do.isLostConnectionToPD.Get() != 0 } +// GetLogFileMetaCache gets slow-log file meta cache. +func (do *Domain) GetLogFileMetaCache() *cache.LogFileMetaCache { + return do.logFileMetaCache +} + const ( serverIDEtcdPath = "/tidb/server_id" refreshServerIDRetryCnt = 3 diff --git a/executor/slow_query.go b/executor/slow_query.go index c6a5f8872f0e3..8e85864f5b0e1 100755 --- a/executor/slow_query.go +++ b/executor/slow_query.go @@ -34,6 +34,8 @@ import ( "github.com/pingcap/parser/model" "github.com/pingcap/parser/mysql" "github.com/pingcap/parser/terror" + "github.com/pingcap/sysutil/cache" + "github.com/pingcap/tidb/domain" "github.com/pingcap/tidb/infoschema" plannercore "github.com/pingcap/tidb/planner/core" "github.com/pingcap/tidb/privilege" @@ -1002,6 +1004,13 @@ func (e *slowQueryRetriever) getAllFiles(ctx context.Context, sctx sessionctx.Co if err != nil { return nil, err } + dom := domain.GetDomain(sctx) + var ca *cache.LogFileMetaCache + if dom != nil { + ca = dom.GetLogFileMetaCache() + } else { + ca = cache.NewLogFileMetaCache() + } walkFn := func(path string, info os.FileInfo) error { if info.IsDir() { return nil @@ -1024,8 +1033,23 @@ func (e *slowQueryRetriever) getAllFiles(ctx context.Context, sctx sessionctx.Co terror.Log(file.Close()) } }() + stat, err := file.Stat() + if err != nil { + return handleErr(err) + } + meta := ca.GetFileMata(stat) + if meta == nil { + meta = cache.NewLogFileMeta(stat) + defer ca.AddFileMataToCache(stat, meta) + } else { + if meta.CheckFileNotModified(stat) && meta.IsInValid() { + return nil + } + } // Get the file start time. - fileStartTime, err := e.getFileStartTime(ctx, file) + fileStartTime, err := meta.GetStartTime(stat, func() (time.Time, error) { + return e.getFileStartTime(ctx, file) + }) if err != nil { return handleErr(err) } @@ -1042,7 +1066,9 @@ func (e *slowQueryRetriever) getAllFiles(ctx context.Context, sctx sessionctx.Co } // Get the file end time. - fileEndTime, err := e.getFileEndTime(ctx, file) + fileEndTime, err := meta.GetEndTime(stat, func() (time.Time, error) { + return e.getFileEndTime(ctx, file) + }) if err != nil { return handleErr(err) } @@ -1093,6 +1119,9 @@ func (e *slowQueryRetriever) getFileStartTime(ctx context.Context, file *os.File for { lineByte, err := getOneLine(reader) if err != nil { + if err == io.EOF { + return t, cache.InvalidLogFile + } return t, err } line := string(lineByte) @@ -1107,7 +1136,7 @@ func (e *slowQueryRetriever) getFileStartTime(ctx context.Context, file *os.File return t, ctx.Err() } } - return t, errors.Errorf("malform slow query file %v", file.Name()) + return t, cache.InvalidLogFile } func (e *slowQueryRetriever) getRuntimeStats() execdetails.RuntimeStats { @@ -1189,7 +1218,7 @@ func (e *slowQueryRetriever) getFileEndTime(ctx context.Context, file *os.File) return t, ctx.Err() } } - return t, errors.Errorf("invalid slow query file %v", file.Name()) + return t, cache.InvalidLogFile } const maxReadCacheSize = 1024 * 1024 * 64 diff --git a/go.mod b/go.mod index fc4080ffa4790..5c71b9cf9c87f 100644 --- a/go.mod +++ b/go.mod @@ -84,3 +84,5 @@ require ( ) go 1.13 + +replace github.com/pingcap/sysutil => github.com/crazycs520/sysutil v0.0.0-20201207085517-9dd18ed31826 diff --git a/go.sum b/go.sum index 593d7d73545bd..07e8e0c6dd9a4 100644 --- a/go.sum +++ b/go.sum @@ -173,6 +173,10 @@ github.com/couchbase/vellum v1.0.1/go.mod h1:FcwrEivFpNi24R3jLOs3n+fs5RnuQnQqCLB github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/crazycs520/sysutil v0.0.0-20201203131959-c22c2028beb7 h1:6uU9EvYWskrDjxuS+x/qyiBvOz7E3If9gH/vqtUki7I= +github.com/crazycs520/sysutil v0.0.0-20201203131959-c22c2028beb7/go.mod h1:EB/852NMQ+aRKioCpToQ94Wl7fktV+FNnxf3CX/TTXI= +github.com/crazycs520/sysutil v0.0.0-20201207085517-9dd18ed31826 h1:t3zIDqYADdl/S4dtKEMCZMZ2GN131/qWjMEiXwhHQ7g= +github.com/crazycs520/sysutil v0.0.0-20201207085517-9dd18ed31826/go.mod h1:EB/852NMQ+aRKioCpToQ94Wl7fktV+FNnxf3CX/TTXI= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/creasty/defaults v1.3.0/go.mod h1:CIEEvs7oIVZm30R8VxtFJs+4k201gReYyuYHJxZc68I=