syz-manager: add Rank column and show the bug consequences as a tooltip

pkg/build/netbsd.go

@@ -158,13 +158,13 @@ func (ctx netbsd) copyKernelToDisk(targetArch, vmType, outputDir, kernel string)
 	commands = append(commands, "sync") // Run sync so that the copied image is stored properly.
 	ctxTimeout, cancel := context.WithTimeout(context.Background(), time.Minute)
 	defer cancel()
-	_, rep, err := inst.Run(ctxTimeout, reporter, strings.Join(commands, ";"))
+	_, reps, err := inst.Run(ctxTimeout, reporter, strings.Join(commands, ";"))
 	if err != nil {
 		return fmt.Errorf("error syncing the instance %w", err)
 	}
 	// Make sure that the command has executed properly.
-	if rep != nil {
-		return fmt.Errorf("error executing sync: %v", rep.Title)
+	if len(reps) > 0 {
+		return fmt.Errorf("error executing sync: %v", reps[0].Title)
 	}
 	return nil
 }

pkg/html/pages/style.css

@@ -177,6 +177,12 @@ table td, table th {
 	text-align: right;
 }
 
+.list_table .rank {
+	width: 55pt;
+	font-family: monospace;
+	text-align: right;
+}
+
 .list_table .discussions {
 	font-family: monospace;
 	text-align: left;
@@ -493,3 +499,19 @@ aside {
 
 /* Change the background color of the dropdown button when the dropdown content is shown */
 .dropdown:hover .dropbtn {background-color: #ddd;}
+
+.rank .tooltiptext {
+	visibility: hidden;
+	background-color: black;
+	color: #fff;
+	text-align: left;
+	border-radius: 6px;
+	padding: 5px 0;
+
+	/* Position the tooltip */
+	position: absolute;
+	z-index: 1;
+}
+.rank:hover .tooltiptext {
+	visibility: visible;
+}

pkg/instance/execprog.go

@@ -126,10 +126,14 @@ func (inst *ExecProgInstance) runCommand(command string, duration time.Duration,
 	}
 	ctxTimeout, cancel := context.WithTimeout(context.Background(), duration)
 	defer cancel()
-	output, rep, err := inst.VMInstance.Run(ctxTimeout, inst.reporter, command,
+	output, reps, err := inst.VMInstance.Run(ctxTimeout, inst.reporter, command,
 		vm.WithExitCondition(exitCondition),
 		optionalBeforeContext,
 	)
+	var rep *report.Report
+	if len(reps) > 0 {
+		rep = reps[0]
+	}
 	if err != nil {
 		return nil, fmt.Errorf("failed to run command in VM: %w", err)
 	}

pkg/manager/crash.go

@@ -17,6 +17,7 @@ import (
 	"github.com/google/syzkaller/pkg/log"
 	"github.com/google/syzkaller/pkg/mgrconfig"
 	"github.com/google/syzkaller/pkg/osutil"
+	"github.com/google/syzkaller/pkg/report"
 	"github.com/google/syzkaller/prog"
 )
 
@@ -85,10 +86,14 @@ func (cs *CrashStore) SaveCrash(crash *Crash) (bool, error) {
 		}
 		osutil.WriteFile(filename, data)
 	}
+	reps := append([]*report.Report{crash.Report}, crash.TailReports...)
 	writeOrRemove("log", crash.Output)
 	writeOrRemove("tag", []byte(cs.Tag))
-	writeOrRemove("report", crash.Report.Report)
+	writeOrRemove("report", report.MergeReportBytes(reps))
 	writeOrRemove("machineInfo", crash.MachineInfo)
+	if err := report.AddTitleStat(filepath.Join(dir, "title-stat"), reps); err != nil {
+		return false, fmt.Errorf("report.AddTitleStat: %w", err)
+	}
 
 	return first, nil
 }
@@ -211,13 +216,15 @@ type CrashInfo struct {
 type BugInfo struct {
 	ID            string
 	Title         string
+	TailTitles    []*report.TitleFreqRank
 	FirstTime     time.Time
 	LastTime      time.Time
 	HasRepro      bool
 	HasCRepro     bool
 	StraceFile    string // relative to the workdir
 	ReproAttempts int
 	Crashes       []*CrashInfo
+	Rank          int
 }
 
 func (cs *CrashStore) BugInfo(id string, full bool) (*BugInfo, error) {
@@ -233,6 +240,16 @@ func (cs *CrashStore) BugInfo(id string, full bool) (*BugInfo, error) {
 		return nil, err
 	}
 	ret.Title = strings.TrimSpace(string(desc))
+
+	// Bug rank may go up over time if we observe higher ranked bugs as a consequence of the first failure.
+	ret.Rank = report.TitlesToImpact(ret.Title)
+	if titleStat, err := report.ReadStatFile(filepath.Join(dir, "title-stat")); err == nil {
+		ret.TailTitles = report.ExplainTitleStat(titleStat)
+		for _, ti := range ret.TailTitles {
+			ret.Rank = max(ret.Rank, ti.Rank)
+		}
+	}
+
 	ret.FirstTime = osutil.CreationTime(stat)
 	ret.LastTime = stat.ModTime()
 	files, err := osutil.ListDir(dir)

pkg/manager/diff.go

@@ -675,7 +675,7 @@ func (kc *kernelContext) runInstance(ctx context.Context, inst *vm.Instance,
 	cmd := fmt.Sprintf("%v runner %v %v %v", executorBin, inst.Index(), host, port)
 	ctxTimeout, cancel := context.WithTimeout(ctx, kc.cfg.Timeouts.VMRunningTime)
 	defer cancel()
-	_, rep, err := inst.Run(ctxTimeout, kc.reporter, cmd,
+	_, reps, err := inst.Run(ctxTimeout, kc.reporter, cmd,
 		vm.WithExitCondition(vm.ExitTimeout),
 		vm.WithInjectExecuting(injectExec),
 		vm.WithEarlyFinishCb(func() {
@@ -685,7 +685,10 @@ func (kc *kernelContext) runInstance(ctx context.Context, inst *vm.Instance,
 			kc.serv.StopFuzzing(inst.Index())
 		}),
 	)
-	return rep, err
+	if len(reps) > 0 {
+		return reps[0], err
+	}
+	return nil, err
 }
 
 func (kc *kernelContext) triageProgress() float64 {

pkg/manager/html/main.html

@@ -26,6 +26,7 @@
 	<thead>
 	<tr>
 		<th><a onclick="return sortTable(this, 'Description', textSort)" href="#">Description</a></th>
+		<th><a onclick="return sortTable(this, 'Rank', numSort)" href="#">Rank</a></th>
 		<th><a onclick="return sortTable(this, 'Count', numSort)" href="#">Count</a></th>
 		<th><a onclick="return sortTable(this, 'First Time', textSort, true)" href="#">First Time</a></th>
 		<th><a onclick="return sortTable(this, 'Last Time', textSort, true)" href="#">Last Time</a></th>
@@ -37,6 +38,14 @@
 	{{range $c := $.Crashes}}
 	<tr>
 		<td class="title"><a href="/crash?id={{$c.ID}}">{{$c.Title}}</a></td>
+		<td class="rank {{if not $c.Active}}inactive{{end}}">
+			{{if $c.RankTooltip}}
+				<b>{{$c.Rank}}</b>
+				<pre class="tooltiptext">{{$c.RankTooltip}}</pre>
+			{{else}}
+				{{$c.Rank}}
+			{{end}}
+		</td>
 		<td class="stat {{if not $c.Active}}inactive{{end}}">{{len $c.Crashes}}</td>
 		<td class="time {{if not $c.New}}inactive{{end}}">{{formatTime $c.FirstTime}}</td>
 		<td class="time {{if not $c.Active}}inactive{{end}}">{{formatTime $c.LastTime}}</td>

pkg/manager/http.go

@@ -29,6 +29,7 @@ import (
 	"github.com/google/syzkaller/pkg/html/pages"
 	"github.com/google/syzkaller/pkg/log"
 	"github.com/google/syzkaller/pkg/mgrconfig"
+	"github.com/google/syzkaller/pkg/report"
 	"github.com/google/syzkaller/pkg/stat"
 	"github.com/google/syzkaller/pkg/vcs"
 	"github.com/google/syzkaller/pkg/vminfo"
@@ -355,12 +356,33 @@ func makeUICrashType(info *BugInfo, startTime time.Time, repros map[string]bool)
 	triaged := reproStatus(info.HasRepro, info.HasCRepro, repros[info.Title],
 		info.ReproAttempts >= MaxReproAttempts)
 	return UICrashType{
-		BugInfo: *info,
-		New:     info.FirstTime.After(startTime),
-		Active:  info.LastTime.After(startTime),
-		Triaged: triaged,
-		Crashes: crashes,
+		BugInfo:     *info,
+		RankTooltip: higherRankTooltip(info.Title, info.TailTitles),
+		New:         info.FirstTime.After(startTime),
+		Active:      info.LastTime.After(startTime),
+		Triaged:     triaged,
+		Crashes:     crashes,
+	}
+}
+
+// higherRankTooltip generates the prioritized list of the titles with higher Rank
+// than the firstTitle has.
+func higherRankTooltip(firstTitle string, titlesInfo []*report.TitleFreqRank) string {
+	baseRank := report.TitlesToImpact(firstTitle)
+	res := ""
+	for _, ti := range titlesInfo {
+		if ti.Rank <= baseRank {
+			continue
+		}
+		res += fmt.Sprintf("[rank %2v, freq %5.1f%%] %s\n",
+			ti.Rank,
+			100*float32(ti.Count)/float32(ti.Total),
+			ti.Title)
+	}
+	if res != "" {
+		return fmt.Sprintf("[rank %2v,  originally] %s\n%s", baseRank, firstTitle, res)
 	}
+	return res
 }
 
 var crashIDRe = regexp.MustCompile(`^\w+$`)
@@ -1024,10 +1046,11 @@ type UICrashPage struct {
 
 type UICrashType struct {
 	BugInfo
-	New     bool // was first found in the current run
-	Active  bool // was found in the current run
-	Triaged string
-	Crashes []UICrash
+	RankTooltip string
+	New         bool // was first found in the current run
+	Active      bool // was found in the current run
+	Triaged     string
+	Crashes     []UICrash
 }
 
 type UICrash struct {

pkg/manager/repro.go

@@ -31,6 +31,7 @@ type Crash struct {
 	Manual        bool
 	FullRepro     bool // used by the diff fuzzer to do a full scale reproduction
 	*report.Report
+	TailReports []*report.Report
 }
 
 func (c *Crash) FullTitle() string {

pkg/report/impact_score.go

@@ -4,6 +4,8 @@
 package report
 
 import (
+	"sort"
+
 	"github.com/google/syzkaller/pkg/report/crash"
 )
 
@@ -62,3 +64,45 @@ func TitlesToImpact(title string, otherTitles ...string) int {
 	}
 	return maxImpact
 }
+
+type TitleFreqRank struct {
+	Title string
+	Count int
+	Total int
+	Rank  int
+}
+
+func ExplainTitleStat(ts *titleStat) []*TitleFreqRank {
+	titleCount := map[string]int{}
+	var totalCount int
+	ts.visit(func(count int, titles ...string) {
+		uniq := map[string]bool{}
+		for _, title := range titles {
+			uniq[title] = true
+		}
+		for title := range uniq {
+			titleCount[title] += count
+		}
+		totalCount += count
+	})
+	var res []*TitleFreqRank
+	for title, count := range titleCount {
+		res = append(res, &TitleFreqRank{
+			Title: title,
+			Count: count,
+			Total: totalCount,
+			Rank:  TitlesToImpact(title),
+		})
+	}
+	sort.Slice(res, func(l, r int) bool {
+		if res[l].Rank != res[r].Rank {
+			return res[l].Rank > res[r].Rank
+		}
+		lTitle, rTitle := res[l].Title, res[r].Title
+		if titleCount[lTitle] != titleCount[rTitle] {
+			return titleCount[lTitle] > titleCount[rTitle]
+		}
+		return lTitle < rTitle
+	})
+	return res
+}

pkg/report/report.go

@@ -943,3 +943,18 @@ func TitleToCrashType(title string) crash.Type {
 	}
 	return crash.UnknownType
 }
+
+const reportSeparator = "\n<<<<<<<<<<<<<<< tail report >>>>>>>>>>>>>>>\n\n"
+
+func MergeReportBytes(reps []*Report) []byte {
+	var res []byte
+	for _, rep := range reps {
+		res = append(res, rep.Report...)
+		res = append(res, []byte(reportSeparator)...)
+	}
+	return res
+}
+
+func SplitReportBytes(data []byte) [][]byte {
+	return bytes.Split(data, []byte(reportSeparator))
+}

pkg/report/report_test.go

@@ -510,3 +510,38 @@ BCDEF`), Truncate([]byte(`0123456789ABCDEF`), 0, 5))
 
 DEF`), Truncate([]byte(`0123456789ABCDEF`), 4, 3))
 }
+
+func TestSplitReportBytes(t *testing.T) {
+	tests := []struct {
+		name      string
+		input     []byte
+		wantFirst string
+	}{
+		{
+			name:      "empty",
+			input:     nil,
+			wantFirst: "",
+		},
+		{
+			name:      "single",
+			input:     []byte("report1"),
+			wantFirst: "report1",
+		},
+		{
+			name:      "split in the middle",
+			input:     []byte("report1" + reportSeparator + "report2"),
+			wantFirst: "report1",
+		},
+		{
+			name:      "split in the middle, save new line",
+			input:     []byte("report1\n" + reportSeparator + "report2"),
+			wantFirst: "report1\n",
+		},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			splitted := SplitReportBytes(test.input)
+			assert.Equal(t, test.wantFirst, string(splitted[0]))
+		})
+	}
+}