pkg/coveragedb: store information about covered file functions in db

Author: tarasmadan

dashboard/app/api.go

@@ -1936,7 +1936,7 @@ func apiCreateUploadURL(c context.Context, payload io.Reader) (interface{}, erro
 
 // apiSaveCoverage reads jsonl data from payload and stores it to coveragedb.
 // First payload jsonl line is a coveragedb.HistoryRecord (w/o session and time).
-// Second+ records are coveragedb.MergedCoverageRecord.
+// Second+ records are coveragedb.JSONLWrapper.
 func apiSaveCoverage(c context.Context, payload io.Reader) (interface{}, error) {
 	descr := new(coveragedb.HistoryRecord)
 	jsonDec := json.NewDecoder(payload)

pkg/coveragedb/coveragedb.go

@@ -22,22 +22,6 @@ import (
 	"google.golang.org/api/iterator"
 )
 
-type FilesRecord struct {
-	Session           string
-	FilePath          string
-	Instrumented      int64
-	Covered           int64
-	LinesInstrumented []int64
-	HitCounts         []int64
-	Manager           string // "*" means "collected from all managers"
-}
-
-type FileSubsystems struct {
-	Namespace  string
-	FilePath   string
-	Subsystems []string
-}
-
 type HistoryRecord struct {
 	Session   string
 	Time      time.Time
@@ -49,6 +33,25 @@ type HistoryRecord struct {
 	TotalRows int64
 }
 
+type MergedCoverageRecord struct {
+	Manager  string
+	FilePath string
+	FileData *Coverage
+}
+
+// FuncLines represents the 'functions' table records.
+// It could be used to maps 'hitcounts' from 'files' table to the function names.
+type FuncLines struct {
+	FilePath string
+	FuncName string
+	Lines    []int64 // List of lines we know belong to this function name according to the addr2line output.
+}
+
+type JSONLWrapper struct {
+	MCR *MergedCoverageRecord
+	FL  *FuncLines
+}
+
 type Coverage struct {
 	Instrumented      int64
 	Covered           int64
@@ -65,23 +68,27 @@ func (c *Coverage) AddLineHitCount(line int, hitCount int64) {
 	}
 }
 
-type MergedCoverageRecord struct {
-	Manager  string
-	FilePath string
-	FileData *Coverage
+type filesRecord struct {
+	Session           string
+	FilePath          string
+	Instrumented      int64
+	Covered           int64
+	LinesInstrumented []int64
+	HitCounts         []int64
+	Manager           string // "*" means "collected from all managers"
 }
 
-// FuncLines represents the 'functions' table records.
-// It could be used to maps 'hitcounts' from 'files' table to the function names.
-type FuncLines struct {
+type functionsRecord struct {
+	Session  string
 	FilePath string
 	FuncName string
-	Lines    []int64 // List of lines we know belong to this function name according to the addr2line output.
+	Lines    []int64
 }
 
-type JSONLWrapper struct {
-	MCR *MergedCoverageRecord
-	FL  *FuncLines
+type fileSubsystems struct {
+	Namespace  string
+	FilePath   string
+	Subsystems []string
 }
 
 func SaveMergeResult(ctx context.Context, client spannerclient.SpannerClient, descr *HistoryRecord, dec *json.Decoder,
@@ -94,23 +101,26 @@ func SaveMergeResult(ctx context.Context, client spannerclient.SpannerClient, de
 	ssCache := make(map[string][]string)
 
 	session := uuid.New().String()
-	mutations := []*spanner.Mutation{}
+	var mutations []*spanner.Mutation
 
 	for {
-		var mcr MergedCoverageRecord
-		err := dec.Decode(&mcr)
+		var wr JSONLWrapper
+		err := dec.Decode(&wr)
 		if err == io.EOF {
 			break
 		}
 		if err != nil {
 			return rowsCreated, fmt.Errorf("dec.Decode(MergedCoverageRecord): %w", err)
 		}
-		if mcr.FileData == nil {
-			return rowsCreated, errors.New("field MergedCoverageRecord.FileData can't be nil")
+		if mcr := wr.MCR; mcr != nil {
+			mutations = append(mutations, fileRecordMutation(session, mcr))
+			subsystems := getFileSubsystems(mcr.FilePath, ssMatcher, ssCache)
+			mutations = append(mutations, fileSubsystemsMutation(descr.Namespace, mcr.FilePath, subsystems))
+		} else if fl := wr.FL; fl != nil {
+			mutations = append(mutations, fileFunctionsMutation(session, fl))
+		} else {
+			return rowsCreated, errors.New("JSONLWrapper can't be empty")
 		}
-		mutations = append(mutations, fileRecordMutation(session, &mcr))
-		subsystems := fileSubsystems(mcr.FilePath, ssMatcher, ssCache)
-		mutations = append(mutations, fileSubsystemsMutation(descr.Namespace, mcr.FilePath, subsystems))
 		// There is a limit on the number of mutations per transaction (80k) imposed by the DB.
 		// This includes both explicit mutations of the fields (6 fields * 1k records = 6k mutations)
 		//   and implicit index mutations.
@@ -203,8 +213,21 @@ func historyMutation(session string, template *HistoryRecord) *spanner.Mutation
 	return historyInsert
 }
 
+func fileFunctionsMutation(session string, fl *FuncLines) *spanner.Mutation {
+	insert, err := spanner.InsertOrUpdateStruct("functions", &functionsRecord{
+		Session:  session,
+		FilePath: fl.FilePath,
+		FuncName: fl.FuncName,
+		Lines:    fl.Lines,
+	})
+	if err != nil {
+		panic(fmt.Sprintf("failed to fileFunctionsMutation: %v", err))
+	}
+	return insert
+}
+
 func fileRecordMutation(session string, mcr *MergedCoverageRecord) *spanner.Mutation {
-	insert, err := spanner.InsertOrUpdateStruct("files", &FilesRecord{
+	insert, err := spanner.InsertOrUpdateStruct("files", &filesRecord{
 		Session:           session,
 		FilePath:          mcr.FilePath,
 		Instrumented:      mcr.FileData.Instrumented,
@@ -220,7 +243,7 @@ func fileRecordMutation(session string, mcr *MergedCoverageRecord) *spanner.Muta
 }
 
 func fileSubsystemsMutation(ns, filePath string, subsystems []string) *spanner.Mutation {
-	insert, err := spanner.InsertOrUpdateStruct("file_subsystems", &FileSubsystems{
+	insert, err := spanner.InsertOrUpdateStruct("file_subsystems", &fileSubsystems{
 		Namespace:  ns,
 		FilePath:   filePath,
 		Subsystems: subsystems,
@@ -231,7 +254,7 @@ func fileSubsystemsMutation(ns, filePath string, subsystems []string) *spanner.M
 	return insert
 }
 
-func fileSubsystems(filePath string, ssMatcher *subsystem.PathMatcher, ssCache map[string][]string) []string {
+func getFileSubsystems(filePath string, ssMatcher *subsystem.PathMatcher, ssCache map[string][]string) []string {
 	sss, cached := ssCache[filePath]
 	if !cached {
 		for _, match := range ssMatcher.Match(filePath) {

pkg/coveragedb/coveragedb_mock_test.go

@@ -45,9 +45,10 @@ func TestSaveMergeResult(t *testing.T) {
 			jsonl:   strings.NewReader(`{a}`),
 			wantErr: true,
 		},
+		// nolint: dupl
 		{
-			name:     "1 record, Ok",
-			jsonl:    strings.NewReader(`{"FileData":{}}`),
+			name:     "1 MCR record, Ok",
+			jsonl:    strings.NewReader(`{"MCR":{"FileData":{}}}`),
 			descr:    &HistoryRecord{},
 			wantRows: 3, // 1 in files, 1 in file_subsystems and 1 in merge_history
 			mockTune: func(t *testing.T, m *mocks.SpannerClient) {
@@ -57,10 +58,23 @@ func TestSaveMergeResult(t *testing.T) {
 					Once()
 			},
 		},
+		// nolint: dupl
+		{
+			name:     "1 FC record, Ok",
+			jsonl:    strings.NewReader(`{"FL":{}}`),
+			descr:    &HistoryRecord{},
+			wantRows: 2, // 1 in functions and 1 in merge_history
+			mockTune: func(t *testing.T, m *mocks.SpannerClient) {
+				m.
+					On("Apply", mock.Anything, mock.Anything).
+					Return(time.Now(), nil).
+					Once()
+			},
+		},
 		{
 			name: "2 records, Ok",
-			jsonl: strings.NewReader(`	{"FileData":{}}
-																		{"FileData":{}}`),
+			jsonl: strings.NewReader(`	{"MCR":{"FileData":{}}}
+																		{"MCR":{"FileData":{}}}`),
 			descr:    &HistoryRecord{},
 			wantRows: 5,
 			mockTune: func(t *testing.T, m *mocks.SpannerClient) {
@@ -77,7 +91,7 @@ func TestSaveMergeResult(t *testing.T) {
 		},
 		{
 			name:     "2k records, Ok",
-			jsonl:    strings.NewReader(strings.Repeat("{\"FileData\":{}}\n", 2000)),
+			jsonl:    strings.NewReader(strings.Repeat("{\"MCR\":{\"FileData\":{}}}\n", 2000)),
 			descr:    &HistoryRecord{},
 			wantRows: 4001,
 			mockTune: func(t *testing.T, m *mocks.SpannerClient) {

pkg/covermerger/covermerger_test.go

@@ -55,8 +55,8 @@ func TestMergeCSVWriteJSONL_and_coveragedb_SaveMergeResult(t *testing.T) {
 		spannerMock := mocks.NewSpannerClient(t)
 		spannerMock.
 			On("Apply", mock.Anything, mock.MatchedBy(func(ms []*spanner.Mutation) bool {
-				// 1 file * (5 managers + 1 manager total) x 2 (to update files and subsystems) + 1 merge_history
-				return len(ms) == 13
+				// 1 file * (5 managers + 1 manager total) x 2 (to update files and subsystems) + 1 merge_history + 18 functions
+				return len(ms) == 13+18
 			})).
 			Return(time.Now(), nil).
 			Once()