diff --git a/dashboard/app/ai.go b/dashboard/app/ai.go index c80c5005c8f2..80cc98b17bc7 100644 --- a/dashboard/app/ai.go +++ b/dashboard/app/ai.go @@ -39,6 +39,13 @@ type uiAIJobPage struct { Jobs []*uiAIJob CrashReport template.HTML Trajectory []*uiAITrajectorySpan + History []*uiJobReviewHistory +} + +type uiJobReviewHistory struct { + Date time.Time + User string + Correct string } type uiAIJob struct { @@ -141,6 +148,18 @@ func handleAIJobPage(ctx context.Context, w http.ResponseWriter, r *http.Request default: job.Correct = spanner.NullBool{} } + userEmail := "" + if user := currentUser(ctx); user != nil { + userEmail = user.Email + } + if err := aidb.AddJobReviewHistory(ctx, &aidb.JobReviewHistory{ + JobID: job.ID, + Date: timeNow(ctx), + User: userEmail, + Correct: job.Correct, + }); err != nil { + return err + } if err := aiJobUpdate(ctx, job); err != nil { return err } @@ -149,6 +168,10 @@ func handleAIJobPage(ctx context.Context, w http.ResponseWriter, r *http.Request if err != nil { return err } + history, err := aidb.LoadJobReviewHistory(ctx, job.ID) + if err != nil { + return err + } hdr, err := commonHeader(ctx, r, w, job.Namespace) if err != nil { return err @@ -172,6 +195,7 @@ func handleAIJobPage(ctx context.Context, w http.ResponseWriter, r *http.Request Jobs: []*uiAIJob{uiJob}, CrashReport: crashReport, Trajectory: makeUIAITrajectory(trajectory), + History: makeUIJobReviewHistory(history), } return serveTemplate(w, "ai_job.html", page) } @@ -258,6 +282,26 @@ func makeUIAITrajectory(trajetory []*aidb.TrajectorySpan) []*uiAITrajectorySpan return res } +func makeUIJobReviewHistory(history []*aidb.JobReviewHistory) []*uiJobReviewHistory { + var res []*uiJobReviewHistory + for _, h := range history { + val := aiCorrectnessUnset + if h.Correct.Valid { + if h.Correct.Bool { + val = aiCorrectnessCorrect + } else { + val = aiCorrectnessIncorrect + } + } + res = append(res, &uiJobReviewHistory{ + Date: h.Date, + User: h.User, + Correct: val, + }) + } + return res +} + func apiAIJobPoll(ctx context.Context, req *dashapi.AIJobPollReq) (any, error) { if len(req.Workflows) == 0 || req.CodeRevision == "" { return nil, fmt.Errorf("invalid request") diff --git a/dashboard/app/ai_test.go b/dashboard/app/ai_test.go index 5bdb9202a9e5..8c76b7451567 100644 --- a/dashboard/app/ai_test.go +++ b/dashboard/app/ai_test.go @@ -8,6 +8,7 @@ import ( "testing" "time" + "github.com/google/syzkaller/dashboard/app/aidb" "github.com/google/syzkaller/dashboard/dashapi" "github.com/google/syzkaller/pkg/aflow/ai" "github.com/google/syzkaller/pkg/aflow/trajectory" @@ -236,15 +237,29 @@ func TestAIAssessmentKCSAN(t *testing.T) { _, err = c.GET(fmt.Sprintf("/ai_job?id=%v&correct=%v", resp.ID, aiCorrectnessCorrect)) require.NoError(t, err) + history, err := aidb.LoadJobReviewHistory(c.ctx, resp.ID) + require.NoError(t, err) + require.Len(t, history, 1) + require.True(t, history[0].Correct.Bool) + require.NotEmpty(t, history[0].User) + bug, _, _ := c.loadBug(extID) labels := bug.LabelValues(RaceLabel) require.Len(t, labels, 1) require.Equal(t, labels[0].Value, BenignRace) + c.advanceTime(time.Second) + // Re-mark the result as incorrect, this should remove the label. _, err = c.GET(fmt.Sprintf("/ai_job?id=%v&correct=%v", resp.ID, aiCorrectnessIncorrect)) require.NoError(t, err) + history, err = aidb.LoadJobReviewHistory(c.ctx, resp.ID) + require.NoError(t, err) + require.Len(t, history, 2) + require.False(t, history[0].Correct.Bool) + require.True(t, history[1].Correct.Bool) + bug, _, _ = c.loadBug(extID) labels = bug.LabelValues(RaceLabel) require.Len(t, labels, 0) diff --git a/dashboard/app/aidb/crud.go b/dashboard/app/aidb/crud.go index 4b73a5c0ae20..a8e62da080ee 100644 --- a/dashboard/app/aidb/crud.go +++ b/dashboard/app/aidb/crud.go @@ -265,6 +265,34 @@ func selectTrajectorySpans() string { return selectAllFrom[TrajectorySpan]("TrajectorySpans") } +func selectJobReviewHistory() string { + return selectAllFrom[JobReviewHistory]("JobReviewHistory") +} + +func AddJobReviewHistory(ctx context.Context, history *JobReviewHistory) error { + history.ID = uuid.NewString() + client, err := dbClient(ctx) + if err != nil { + return err + } + defer client.Close() + mut, err := spanner.InsertStruct("JobReviewHistory", history) + if err != nil { + return err + } + _, err = client.Apply(ctx, []*spanner.Mutation{mut}) + return err +} + +func LoadJobReviewHistory(ctx context.Context, jobID string) ([]*JobReviewHistory, error) { + return selectAll[JobReviewHistory](ctx, spanner.Statement{ + SQL: selectJobReviewHistory() + `WHERE JobID = @jobID ORDER BY Date DESC`, + Params: map[string]any{ + "jobID": jobID, + }, + }) +} + func selectAllFrom[T any](table string) string { var fields []string for _, field := range reflect.VisibleFields(reflect.TypeFor[T]()) { diff --git a/dashboard/app/aidb/entities.go b/dashboard/app/aidb/entities.go index 4e99f14ca33a..75c8e58083cc 100644 --- a/dashboard/app/aidb/entities.go +++ b/dashboard/app/aidb/entities.go @@ -56,3 +56,11 @@ type TrajectorySpan struct { OutputTokens spanner.NullInt64 OutputThoughtsTokens spanner.NullInt64 } + +type JobReviewHistory struct { + ID string + JobID string + Date time.Time + User string + Correct spanner.NullBool +} diff --git a/dashboard/app/aidb/migrations/6_add_job_review_history.down.sql b/dashboard/app/aidb/migrations/6_add_job_review_history.down.sql new file mode 100644 index 000000000000..7a7ce3d180a3 --- /dev/null +++ b/dashboard/app/aidb/migrations/6_add_job_review_history.down.sql @@ -0,0 +1 @@ +DROP TABLE JobReviewHistory; diff --git a/dashboard/app/aidb/migrations/6_add_job_review_history.up.sql b/dashboard/app/aidb/migrations/6_add_job_review_history.up.sql new file mode 100644 index 000000000000..dc9e2f79feb1 --- /dev/null +++ b/dashboard/app/aidb/migrations/6_add_job_review_history.up.sql @@ -0,0 +1,8 @@ +CREATE TABLE JobReviewHistory ( + ID STRING(36) NOT NULL, + JobID STRING(36) NOT NULL, + Date TIMESTAMP NOT NULL, + User STRING(1000), + Correct BOOL, + CONSTRAINT FK_JobReviewHistory_Job FOREIGN KEY (JobID) REFERENCES Jobs (ID), +) PRIMARY KEY (ID); diff --git a/dashboard/app/templates/ai_job.html b/dashboard/app/templates/ai_job.html index e1971ac3dcc9..75de009a390f 100644 --- a/dashboard/app/templates/ai_job.html +++ b/dashboard/app/templates/ai_job.html @@ -31,6 +31,29 @@
{{end}} + {{if .History}} + + + + + + + + + + + {{range .History}} + + + + + + {{end}} + +
Decision History:
TimeUserDecision
{{formatTime .Date}}{{.User}}{{.Correct}}
+
+ {{end}} + {{range $res := .Job.Results}} {{if $res.IsBool}} {{$res.Name}}: