Skip to content

Commit ff8156b

Browse files
dvyukovdvyukov
committed
dashboard/app: add race harmfullness label
Add race:harmful/benign label. Set it automatically by confirmed AI jobs.
1 parent 54c44ce commit ff8156b

File tree

8 files changed

+183
-18
lines changed

8 files changed

+183
-18
lines changed

‎dashboard/app/ai.go‎

Lines changed: 85 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
package main
55

66
import (
7+
"bytes"
78
"context"
89
"encoding/json"
910
"fmt"
@@ -118,7 +119,7 @@ func handleAIJobPage(ctx context.Context, w http.ResponseWriter, r *http.Request
118119
default:
119120
job.Correct = spanner.NullBool{}
120121
}
121-
if err := aidb.UpdateJob(ctx, job); err != nil {
122+
if err := aiJobUpdate(ctx, job); err != nil {
122123
return err
123124
}
124125
}
@@ -284,10 +285,92 @@ func apiAIJobDone(ctx context.Context, req *dashapi.AIJobDoneReq) (any, error) {
284285
if len(req.Results) != 0 {
285286
job.Results = spanner.NullJSON{Value: req.Results, Valid: true}
286287
}
287-
err = aidb.UpdateJob(ctx, job)
288+
err = aiJobUpdate(ctx, job)
288289
return nil, err
289290
}
290291

292+
func aiJobUpdate(ctx context.Context, job *aidb.Job) error {
293+
if err := aidb.UpdateJob(ctx, job); err != nil {
294+
return err
295+
}
296+
if !job.BugID.Valid || !job.Finished.Valid || job.Error != "" {
297+
return nil
298+
}
299+
bug, err := loadBug(ctx, job.BugID.StringVal)
300+
if err != nil {
301+
return err
302+
}
303+
labelType, labelValue, labelAdd, err := aiBugLabel(job)
304+
if err != nil || labelType == EmptyLabel {
305+
return err
306+
}
307+
label := BugLabel{
308+
Label: labelType,
309+
Value: labelValue,
310+
Link: job.ID,
311+
}
312+
labelSet := makeLabelSet(ctx, bug)
313+
return updateSingleBug(ctx, bug.key(ctx), func(bug *Bug) error {
314+
if bug.HasUserLabel(labelType) {
315+
return nil
316+
}
317+
if labelAdd {
318+
return bug.SetLabels(labelSet, []BugLabel{label})
319+
}
320+
bug.UnsetLabels(labelType)
321+
return nil
322+
})
323+
}
324+
325+
func aiBugLabel(job *aidb.Job) (typ BugLabelType, value string, set bool, err0 error) {
326+
switch job.Type {
327+
case ai.WorkflowAssessmentKCSAN:
328+
// For now we require a manual correctness check,
329+
// later we may apply some labels w/o the manual check.
330+
if !job.Correct.Valid {
331+
return
332+
}
333+
if !job.Correct.Bool {
334+
return RaceLabel, "", false, nil
335+
}
336+
res, err := castJobResults[ai.AssessmentKCSANOutputs](job)
337+
if err != nil {
338+
err0 = err
339+
return
340+
}
341+
if !res.Confident {
342+
return
343+
}
344+
if res.Benign {
345+
return RaceLabel, BenignRace, true, nil
346+
}
347+
return RaceLabel, HarmfulRace, true, nil
348+
}
349+
return
350+
}
351+
352+
func castJobResults[T any](job *aidb.Job) (T, error) {
353+
var res T
354+
raw, ok := job.Results.Value.(map[string]any)
355+
if !ok || !job.Results.Valid {
356+
return res, fmt.Errorf("finished job %v %v does not have results", job.Type, job.ID)
357+
}
358+
// Database may store older versions of the output structs.
359+
// It's not possible to automatically handle all possible changes to the structs.
360+
// For now we just parse in some way. Later when we start changing output structs,
361+
// we may need to reconsider and use more careful parsing.
362+
data, err := json.Marshal(raw)
363+
if err != nil {
364+
return res, err
365+
}
366+
dec := json.NewDecoder(bytes.NewReader(data))
367+
dec.DisallowUnknownFields()
368+
if err := dec.Decode(&res); err != nil {
369+
return res, fmt.Errorf("failed to unmarshal %T: %w", res, err)
370+
}
371+
return res, nil
372+
}
373+
291374
func apiAITrajectoryLog(ctx context.Context, req *dashapi.AITrajectoryReq) (any, error) {
292375
err := aidb.StoreTrajectorySpan(ctx, req.JobID, req.Span)
293376
return nil, err

‎dashboard/app/ai_test.go‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,10 +4,12 @@
44
package main
55

66
import (
7+
"fmt"
78
"testing"
89
"time"
910

1011
"github.com/google/syzkaller/dashboard/dashapi"
12+
"github.com/google/syzkaller/pkg/aflow/ai"
1113
"github.com/google/syzkaller/pkg/aflow/trajectory"
1214
"github.com/google/syzkaller/prog"
1315
"github.com/stretchr/testify/require"
@@ -198,3 +200,58 @@ func TestAIJob(t *testing.T) {
198200
},
199201
}))
200202
}
203+
204+
func TestAIAssessmentKCSAN(t *testing.T) {
205+
c := NewSpannerCtx(t)
206+
defer c.Close()
207+
208+
build := testBuild(1)
209+
c.aiClient.UploadBuild(build)
210+
crash := testCrash(build, 1)
211+
crash.Title = "KCSAN: data-race in foo / bar"
212+
c.aiClient.ReportCrash(crash)
213+
extID := c.aiClient.pollEmailExtID()
214+
215+
resp, err := c.aiClient.AIJobPoll(&dashapi.AIJobPollReq{
216+
CodeRevision: prog.GitRevision,
217+
LLMModel: "smarty",
218+
Workflows: []dashapi.AIWorkflow{
219+
{Type: ai.WorkflowAssessmentKCSAN, Name: string(ai.WorkflowAssessmentKCSAN)},
220+
},
221+
})
222+
require.NoError(t, err)
223+
require.Equal(t, resp.Workflow, string(ai.WorkflowAssessmentKCSAN))
224+
225+
_, err = c.GET(fmt.Sprintf("/ai_job?id=%v", resp.ID))
226+
require.NoError(t, err)
227+
228+
// Since the job is not completed, setting correctness must fail.
229+
_, err = c.GET(fmt.Sprintf("/ai_job?id=%v&correct=%v", resp.ID, aiCorrectnessCorrect))
230+
require.Error(t, err)
231+
232+
require.NoError(t, c.aiClient.AIJobDone(&dashapi.AIJobDoneReq{
233+
ID: resp.ID,
234+
Results: map[string]any{
235+
"Confident": true,
236+
"Benign": true,
237+
"Explanation": "I don't care about races.",
238+
},
239+
}))
240+
241+
// Now setting correctness must not fail.
242+
_, err = c.GET(fmt.Sprintf("/ai_job?id=%v&correct=%v", resp.ID, aiCorrectnessCorrect))
243+
require.NoError(t, err)
244+
245+
bug, _, _ := c.loadBug(extID)
246+
labels := bug.LabelValues(RaceLabel)
247+
require.Len(t, labels, 1)
248+
require.Equal(t, labels[0].Value, BenignRace)
249+
250+
// Re-mark the result as incorrect, this should remove the label.
251+
_, err = c.GET(fmt.Sprintf("/ai_job?id=%v&correct=%v", resp.ID, aiCorrectnessIncorrect))
252+
require.NoError(t, err)
253+
254+
bug, _, _ = c.loadBug(extID)
255+
labels = bug.LabelValues(RaceLabel)
256+
require.Len(t, labels, 0)
257+
}

‎dashboard/app/entities_datastore.go‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -168,7 +168,7 @@ type BugLabel struct {
168168
// The email of the user who manually set this subsystem tag.
169169
// If empty, the label was set automatically.
170170
SetBy string
171-
// Link to the message.
171+
// Link to the message, or AI job ID for automatic labels.
172172
Link string
173173
}
174174

@@ -193,7 +193,7 @@ func (bug *Bug) SetAutoSubsystems(c context.Context, list []*subsystem.Subsystem
193193
for _, item := range list {
194194
objects = append(objects, BugLabel{Label: SubsystemLabel, Value: item.Name})
195195
}
196-
bug.SetLabels(makeLabelSet(c, bug.Namespace), objects)
196+
bug.SetLabels(makeLabelSet(c, bug), objects)
197197
}
198198

199199
func updateSingleBug(c context.Context, bugKey *db.Key, transform func(*Bug) error) error {
@@ -915,6 +915,15 @@ func bugKeyHash(c context.Context, ns, title string, seq int64) string {
915915
return hash.String([]byte(fmt.Sprintf("%v-%v-%v-%v", getNsConfig(c, ns).Key, ns, title, seq)))
916916
}
917917

918+
func loadBug(c context.Context, bugHash string) (*Bug, error) {
919+
bug := new(Bug)
920+
bugKey := db.NewKey(c, "Bug", bugHash, 0, nil)
921+
if err := db.Get(c, bugKey, bug); err != nil {
922+
return nil, fmt.Errorf("failed to load bug by hash %q: %w", bugHash, err)
923+
}
924+
return bug, nil
925+
}
926+
918927
func loadSimilarBugs(c context.Context, bug *Bug) ([]*Bug, error) {
919928
domain := getNsConfig(c, bug.Namespace).SimilarityDomain
920929
dedup := make(map[string]bool)

‎dashboard/app/label.go‎

Lines changed: 17 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,8 @@ import (
88
"fmt"
99
"sort"
1010
"strings"
11+
12+
"github.com/google/syzkaller/pkg/report/crash"
1113
)
1214

1315
const (
@@ -17,6 +19,7 @@ const (
1719
NoRemindersLabel BugLabelType = "no-reminders"
1820
OriginLabel BugLabelType = "origin"
1921
MissingBackportLabel BugLabelType = "missing-backport"
22+
RaceLabel BugLabelType = "race"
2023
)
2124

2225
type BugPrio string
@@ -27,11 +30,16 @@ const (
2730
HighPrioBug BugPrio = "high"
2831
)
2932

33+
const (
34+
BenignRace = "benign"
35+
HarmfulRace = "harmful"
36+
)
37+
3038
type oneOf []string
3139
type subsetOf []string
3240
type trueFalse struct{}
3341

34-
func makeLabelSet(c context.Context, ns string) *labelSet {
42+
func makeLabelSet(c context.Context, bug *Bug) *labelSet {
3543
ret := map[BugLabelType]any{
3644
PriorityLabel: oneOf([]string{
3745
string(LowPrioBug),
@@ -41,7 +49,12 @@ func makeLabelSet(c context.Context, ns string) *labelSet {
4149
NoRemindersLabel: trueFalse{},
4250
MissingBackportLabel: trueFalse{},
4351
}
44-
service := getNsConfig(c, ns).Subsystems.Service
52+
typ := crash.TitleToType(bug.Title)
53+
if typ == crash.KCSANDataRace {
54+
ret[RaceLabel] = oneOf([]string{BenignRace, HarmfulRace})
55+
}
56+
cfg := getNsConfig(c, bug.Namespace)
57+
service := cfg.Subsystems.Service
4558
if service != nil {
4659
names := []string{}
4760
for _, item := range service.List() {
@@ -51,7 +64,7 @@ func makeLabelSet(c context.Context, ns string) *labelSet {
5164
}
5265

5366
originLabels := []string{}
54-
for _, repo := range getNsConfig(c, ns).Repos {
67+
for _, repo := range cfg.Repos {
5568
if repo.LabelIntroduced != "" {
5669
originLabels = append(originLabels, repo.LabelIntroduced)
5770
}
@@ -66,7 +79,7 @@ func makeLabelSet(c context.Context, ns string) *labelSet {
6679

6780
return &labelSet{
6881
c: c,
69-
ns: ns,
82+
ns: bug.Namespace,
7083
labels: ret,
7184
}
7285
}

‎dashboard/app/reporting_email.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -973,7 +973,7 @@ Please contact the bot's maintainers.`
973973

974974
func handleSetCommand(c context.Context, bug *Bug, msg *email.Email,
975975
command *email.SingleCommand) string {
976-
labelSet := makeLabelSet(c, bug.Namespace)
976+
labelSet := makeLabelSet(c, bug)
977977

978978
match := setCmdRe.FindStringSubmatch(command.Args)
979979
if match == nil {

‎dashboard/app/tree.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -223,7 +223,7 @@ func (ctx *bugTreeContext) setOriginLabels() pollTreeJobResult {
223223
for _, label := range allLabels {
224224
labels = append(labels, BugLabel{Label: OriginLabel, Value: label})
225225
}
226-
ctx.bug.SetLabels(makeLabelSet(ctx.c, ctx.bug.Namespace), labels)
226+
ctx.bug.SetLabels(makeLabelSet(ctx.c, ctx.bug), labels)
227227
return pollResultSkip{}
228228
}
229229

@@ -330,7 +330,7 @@ func (ctx *bugTreeContext) missingBackports() pollTreeJobResult {
330330
}
331331
ctx.bug.UnsetLabels(MissingBackportLabel)
332332
if resultDone.Crashed {
333-
ctx.bug.SetLabels(makeLabelSet(ctx.c, ctx.bug.Namespace), []BugLabel{
333+
ctx.bug.SetLabels(makeLabelSet(ctx.c, ctx.bug), []BugLabel{
334334
{Label: MissingBackportLabel},
335335
})
336336
}

‎pkg/aflow/ai/ai.go‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,3 +13,12 @@ const (
1313
WorkflowModeration = WorkflowType("moderation")
1414
WorkflowAssessmentKCSAN = WorkflowType("assessment-kcsan")
1515
)
16+
17+
// Outputs of various workflow types.
18+
// Should be changed carefully since old outputs are stored in the dashboard database.
19+
20+
type AssessmentKCSANOutputs struct {
21+
Confident bool
22+
Benign bool
23+
Explanation string
24+
}

‎pkg/aflow/flow/assessment/kcsan.go‎

Lines changed: 1 addition & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -18,14 +18,8 @@ type kcsanInputs struct {
1818
CodesearchToolBin string
1919
}
2020

21-
type kcsanOutputs struct {
22-
Confident bool
23-
Benign bool
24-
Explanation string
25-
}
26-
2721
func init() {
28-
aflow.Register[kcsanInputs, kcsanOutputs](
22+
aflow.Register[kcsanInputs, ai.AssessmentKCSANOutputs](
2923
ai.WorkflowAssessmentKCSAN,
3024
"assess if a KCSAN report is about a benign race that only needs annotations or not",
3125
&aflow.Flow{

0 commit comments

Comments
 (0)