dashboard: remember ai job correctness review history

On the bug's page, display the history of job result changes.

Author: a-nogikh

dashboard/app/ai.go

@@ -39,6 +39,13 @@ type uiAIJobPage struct {
 	Jobs        []*uiAIJob
 	CrashReport template.HTML
 	Trajectory  []*uiAITrajectorySpan
+	History     []*uiJobReviewHistory
+}
+
+type uiJobReviewHistory struct {
+	Date    time.Time
+	User    string
+	Correct string
 }
 
 type uiAIJob struct {
@@ -141,6 +148,18 @@ func handleAIJobPage(ctx context.Context, w http.ResponseWriter, r *http.Request
 		default:
 			job.Correct = spanner.NullBool{}
 		}
+		userEmail := ""
+		if user := currentUser(ctx); user != nil {
+			userEmail = user.Email
+		}
+		if err := aidb.AddJobReviewHistory(ctx, &aidb.JobReviewHistory{
+			JobID:   job.ID,
+			Date:    timeNow(ctx),
+			User:    userEmail,
+			Correct: job.Correct,
+		}); err != nil {
+			return err
+		}
 		if err := aiJobUpdate(ctx, job); err != nil {
 			return err
 		}
@@ -149,6 +168,10 @@ func handleAIJobPage(ctx context.Context, w http.ResponseWriter, r *http.Request
 	if err != nil {
 		return err
 	}
+	history, err := aidb.LoadJobReviewHistory(ctx, job.ID)
+	if err != nil {
+		return err
+	}
 	hdr, err := commonHeader(ctx, r, w, job.Namespace)
 	if err != nil {
 		return err
@@ -172,6 +195,7 @@ func handleAIJobPage(ctx context.Context, w http.ResponseWriter, r *http.Request
 		Jobs:        []*uiAIJob{uiJob},
 		CrashReport: crashReport,
 		Trajectory:  makeUIAITrajectory(trajectory),
+		History:     makeUIJobReviewHistory(history),
 	}
 	return serveTemplate(w, "ai_job.html", page)
 }
@@ -258,6 +282,26 @@ func makeUIAITrajectory(trajetory []*aidb.TrajectorySpan) []*uiAITrajectorySpan
 	return res
 }
 
+func makeUIJobReviewHistory(history []*aidb.JobReviewHistory) []*uiJobReviewHistory {
+	var res []*uiJobReviewHistory
+	for _, h := range history {
+		val := aiCorrectnessUnset
+		if h.Correct.Valid {
+			if h.Correct.Bool {
+				val = aiCorrectnessCorrect
+			} else {
+				val = aiCorrectnessIncorrect
+			}
+		}
+		res = append(res, &uiJobReviewHistory{
+			Date:    h.Date,
+			User:    h.User,
+			Correct: val,
+		})
+	}
+	return res
+}
+
 func apiAIJobPoll(ctx context.Context, req *dashapi.AIJobPollReq) (any, error) {
 	if len(req.Workflows) == 0 || req.CodeRevision == "" {
 		return nil, fmt.Errorf("invalid request")

dashboard/app/ai_test.go

@@ -8,6 +8,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/google/syzkaller/dashboard/app/aidb"
 	"github.com/google/syzkaller/dashboard/dashapi"
 	"github.com/google/syzkaller/pkg/aflow/ai"
 	"github.com/google/syzkaller/pkg/aflow/trajectory"
@@ -236,15 +237,29 @@ func TestAIAssessmentKCSAN(t *testing.T) {
 	_, err = c.GET(fmt.Sprintf("/ai_job?id=%v&correct=%v", resp.ID, aiCorrectnessCorrect))
 	require.NoError(t, err)
 
+	history, err := aidb.LoadJobReviewHistory(c.ctx, resp.ID)
+	require.NoError(t, err)
+	require.Len(t, history, 1)
+	require.True(t, history[0].Correct.Bool)
+	require.NotEmpty(t, history[0].User)
+
 	bug, _, _ := c.loadBug(extID)
 	labels := bug.LabelValues(RaceLabel)
 	require.Len(t, labels, 1)
 	require.Equal(t, labels[0].Value, BenignRace)
 
+	c.advanceTime(time.Second)
+
 	// Re-mark the result as incorrect, this should remove the label.
 	_, err = c.GET(fmt.Sprintf("/ai_job?id=%v&correct=%v", resp.ID, aiCorrectnessIncorrect))
 	require.NoError(t, err)
 
+	history, err = aidb.LoadJobReviewHistory(c.ctx, resp.ID)
+	require.NoError(t, err)
+	require.Len(t, history, 2)
+	require.False(t, history[0].Correct.Bool)
+	require.True(t, history[1].Correct.Bool)
+
 	bug, _, _ = c.loadBug(extID)
 	labels = bug.LabelValues(RaceLabel)
 	require.Len(t, labels, 0)

dashboard/app/aidb/crud.go

@@ -265,6 +265,34 @@ func selectTrajectorySpans() string {
 	return selectAllFrom[TrajectorySpan]("TrajectorySpans")
 }
 
+func selectJobReviewHistory() string {
+	return selectAllFrom[JobReviewHistory]("JobReviewHistory")
+}
+
+func AddJobReviewHistory(ctx context.Context, history *JobReviewHistory) error {
+	history.ID = uuid.NewString()
+	client, err := dbClient(ctx)
+	if err != nil {
+		return err
+	}
+	defer client.Close()
+	mut, err := spanner.InsertStruct("JobReviewHistory", history)
+	if err != nil {
+		return err
+	}
+	_, err = client.Apply(ctx, []*spanner.Mutation{mut})
+	return err
+}
+
+func LoadJobReviewHistory(ctx context.Context, jobID string) ([]*JobReviewHistory, error) {
+	return selectAll[JobReviewHistory](ctx, spanner.Statement{
+		SQL: selectJobReviewHistory() + `WHERE JobID = @jobID ORDER BY Date DESC`,
+		Params: map[string]any{
+			"jobID": jobID,
+		},
+	})
+}
+
 func selectAllFrom[T any](table string) string {
 	var fields []string
 	for _, field := range reflect.VisibleFields(reflect.TypeFor[T]()) {

dashboard/app/aidb/entities.go

@@ -56,3 +56,11 @@ type TrajectorySpan struct {
 	OutputTokens         spanner.NullInt64
 	OutputThoughtsTokens spanner.NullInt64
 }
+
+type JobReviewHistory struct {
+	ID      string
+	JobID   string
+	Date    time.Time
+	User    string
+	Correct spanner.NullBool
+}

dashboard/app/aidb/migrations/6_add_job_review_history.down.sql

@@ -0,0 +1 @@
+DROP TABLE JobReviewHistory;

dashboard/app/aidb/migrations/6_add_job_review_history.up.sql

@@ -0,0 +1,8 @@
+CREATE TABLE JobReviewHistory (
+	ID			STRING(36) NOT NULL,
+	JobID			STRING(36) NOT NULL,
+	Date			TIMESTAMP NOT NULL,
+	User			STRING(1000),
+	Correct			BOOL,
+	CONSTRAINT FK_JobReviewHistory_Job FOREIGN KEY (JobID) REFERENCES Jobs (ID),
+) PRIMARY KEY (ID);

dashboard/app/templates/ai_job.html

@@ -31,6 +31,29 @@
 		<br>
 	{{end}}
 
+	{{if .History}}
+	<table class="list_table">
+		<caption>Decision History:</caption>
+		<thead>
+			<tr>
+				<th>Time</th>
+				<th>User</th>
+				<th>Decision</th>
+			</tr>
+		</thead>
+		<tbody>
+			{{range .History}}
+			<tr>
+				<td>{{formatTime .Date}}</td>
+				<td>{{.User}}</td>
+				<td>{{.Correct}}</td>
+			</tr>
+			{{end}}
+		</tbody>
+	</table>
+	<br>
+	{{end}}
+
 	{{range $res := .Job.Results}}
 		{{if $res.IsBool}}
 			<b>{{$res.Name}}:</b>