syz-cluster: collect information about base crashes

pkg/manager/diff.go

@@ -40,6 +40,7 @@ import (
 type DiffFuzzerConfig struct {
 	Debug        bool
 	PatchedOnly  chan *UniqueBug
+	BaseCrashes  chan string
 	Store        *DiffFuzzerStore
 	ArtifactsDir string // Where to store the artifacts that supplement the logs.
 	// The fuzzer waits no more than MaxTriageTime time until it starts taking VMs away
@@ -51,6 +52,11 @@ type DiffFuzzerConfig struct {
 	// trying to reach the modified code. The time is counted since the moment
 	// 99% of the corpus is triaged.
 	FuzzToReachPatched time.Duration
+	// The callback may be used to consult external systems on whether the
+	// base kernel has ever crashed with the given title.
+	// It may help reduce the false positive rate and prevent unnecessary
+	// bug reproductions.
+	BaseCrashKnown func(context.Context, string) (bool, error)
 }
 
 func (cfg *DiffFuzzerConfig) TriageDeadline() <-chan time.Time {
@@ -186,40 +192,40 @@ loop:
 			log.Logf(0, "STAT %s", data)
 		case rep := <-dc.base.crashes:
 			log.Logf(1, "base crash: %v", rep.Title)
-			dc.store.BaseCrashed(rep.Title, rep.Report)
+			dc.reportBaseCrash(ctx, rep)
 		case ret := <-runner.done:
 			// We have run the reproducer on the base instance.
 
 			// A sanity check: the base kernel might have crashed with the same title
 			// since the moment we have stared the reproduction / running on the repro base.
-			crashesOnBase := dc.store.EverCrashedBase(ret.origReport.Title)
+			crashesOnBase := dc.everCrashedBase(ctx, ret.reproReport.Title)
 			if ret.crashReport == nil && crashesOnBase {
 				// Report it as error so that we could at least find it in the logs.
-				log.Errorf("repro didn't crash base, but base itself crashed: %s", ret.origReport.Title)
+				log.Errorf("repro didn't crash base, but base itself crashed: %s", ret.reproReport.Title)
 			} else if ret.crashReport == nil {
-				dc.store.BaseNotCrashed(ret.origReport.Title)
+				dc.store.BaseNotCrashed(ret.reproReport.Title)
 				select {
 				case <-ctx.Done():
 				case dc.patchedOnly <- &UniqueBug{
-					Report: ret.origReport,
+					Report: ret.reproReport,
 					Repro:  ret.repro,
 				}:
 				}
-				log.Logf(0, "patched-only: %s", ret.origReport.Title)
+				log.Logf(0, "patched-only: %s", ret.reproReport.Title)
 				// Now that we know this bug only affects the patch kernel, we can spend more time
 				// generating a minimalistic repro and a C repro.
 				if !ret.fullRepro {
 					reproLoop.Enqueue(&Crash{
 						Report: &report.Report{
-							Title:  ret.origReport.Title,
+							Title:  ret.reproReport.Title,
 							Output: ret.repro.Prog.Serialize(),
 						},
 						FullRepro: true,
 					})
 				}
 			} else {
-				dc.store.BaseCrashed(ret.origReport.Title, ret.origReport.Report)
-				log.Logf(0, "crashes both: %s / %s", ret.origReport.Title, ret.crashReport.Title)
+				dc.reportBaseCrash(ctx, ret.crashReport)
+				log.Logf(0, "crashes both: %s / %s", ret.reproReport.Title, ret.crashReport.Title)
 			}
 		case ret := <-dc.doneRepro:
 			// We have finished reproducing a crash from the patched instance.
@@ -254,6 +260,36 @@ loop:
 	return g.Wait()
 }
 
+func (dc *diffContext) everCrashedBase(ctx context.Context, title string) bool {
+	if dc.store.EverCrashedBase(title) {
+		return true
+	}
+	// Let's try to ask the external systems about it as well.
+	if dc.cfg.BaseCrashKnown != nil {
+		known, err := dc.cfg.BaseCrashKnown(ctx, title)
+		if err != nil {
+			log.Logf(0, "a call to BaseCrashKnown failed: %v", err)
+		} else {
+			if known {
+				log.Logf(0, "base crash %q is already known", title)
+			}
+			return known
+		}
+	}
+	return false
+}
+
+func (dc *diffContext) reportBaseCrash(ctx context.Context, rep *report.Report) {
+	dc.store.BaseCrashed(rep.Title, rep.Report)
+	if dc.cfg.BaseCrashes == nil {
+		return
+	}
+	select {
+	case dc.cfg.BaseCrashes <- rep.Title:
+	case <-ctx.Done():
+	}
+}
+
 func (dc *diffContext) waitCorpusTriage(ctx context.Context, threshold float64) chan struct{} {
 	const backOffTime = 30 * time.Second
 	ret := make(chan struct{})
@@ -343,7 +379,10 @@ func (dc *diffContext) NeedRepro(crash *Crash) bool {
 	}
 	dc.mu.Lock()
 	defer dc.mu.Unlock()
-	if dc.store.EverCrashedBase(crash.Title) {
+
+	ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
+	defer cancel()
+	if dc.everCrashedBase(ctx, crash.Title) {
 		return false
 	}
 	if dc.reproAttempts[crash.Title] > maxReproAttempts {
@@ -678,7 +717,7 @@ type reproRunner struct {
 }
 
 type reproRunnerResult struct {
-	origReport  *report.Report
+	reproReport *report.Report
 	crashReport *report.Report
 	repro       *repro.Result
 	fullRepro   bool // whether this was a full reproduction
@@ -716,7 +755,7 @@ func (rr *reproRunner) Run(ctx context.Context, r *repro.Result, fullRepro bool)
 		rr.kernel.pool.ReserveForRun(min(cnt, pool.Total()))
 	}()
 
-	ret := reproRunnerResult{origReport: r.Report, repro: r, fullRepro: fullRepro}
+	ret := reproRunnerResult{reproReport: r.Report, repro: r, fullRepro: fullRepro}
 	for doneRuns := 0; doneRuns < needRuns; {
 		if ctx.Err() != nil {
 			return
@@ -742,7 +781,7 @@ func (rr *reproRunner) Run(ctx context.Context, r *repro.Result, fullRepro bool)
 				Opts:     opts,
 			})
 		})
-		logPrefix := fmt.Sprintf("attempt #%d to run %q on base", doneRuns, ret.origReport.Title)
+		logPrefix := fmt.Sprintf("attempt #%d to run %q on base", doneRuns, ret.reproReport.Title)
 		if errors.Is(runErr, context.Canceled) {
 			// Just exit without sending anything over the channel.
 			log.Logf(1, "%s: aborting due to context cancelation", logPrefix)

syz-cluster/pkg/api/client.go

@@ -114,6 +114,24 @@ func (client Client) UploadSession(ctx context.Context, req *NewSession) (*Uploa
 	return postJSON[NewSession, UploadSessionResp](ctx, client.baseURL+"/sessions/upload", req)
 }
 
+type BaseFindingInfo struct {
+	BuildID string `json:"buildID"`
+	Title   string `json:"title"`
+}
+
+func (client Client) UploadBaseFinding(ctx context.Context, req *BaseFindingInfo) error {
+	_, err := postJSON[BaseFindingInfo, any](ctx, client.baseURL+"/base_findings/upload", req)
+	return err
+}
+
+type BaseFindingStatus struct {
+	Observed bool `json:"observed"`
+}
+
+func (client Client) BaseFindingStatus(ctx context.Context, req *BaseFindingInfo) (*BaseFindingStatus, error) {
+	return postJSON[BaseFindingInfo, BaseFindingStatus](ctx, client.baseURL+"/base_findings/status", req)
+}
+
 const requestTimeout = time.Minute
 
 func finishRequest[Resp any](httpReq *http.Request) (*Resp, error) {

syz-cluster/pkg/controller/api.go

@@ -16,20 +16,22 @@ import (
 )
 
 type APIServer struct {
-	seriesService  *service.SeriesService
-	sessionService *service.SessionService
-	buildService   *service.BuildService
-	testService    *service.SessionTestService
-	findingService *service.FindingService
+	seriesService      *service.SeriesService
+	sessionService     *service.SessionService
+	buildService       *service.BuildService
+	testService        *service.SessionTestService
+	findingService     *service.FindingService
+	baseFindingService *service.BaseFindingService
 }
 
 func NewAPIServer(env *app.AppEnvironment) *APIServer {
 	return &APIServer{
-		seriesService:  service.NewSeriesService(env),
-		sessionService: service.NewSessionService(env),
-		buildService:   service.NewBuildService(env),
-		testService:    service.NewSessionTestService(env),
-		findingService: service.NewFindingService(env),
+		seriesService:      service.NewSeriesService(env),
+		sessionService:     service.NewSessionService(env),
+		buildService:       service.NewBuildService(env),
+		testService:        service.NewSessionTestService(env),
+		findingService:     service.NewFindingService(env),
+		baseFindingService: service.NewBaseFindingService(env),
 	}
 }
 
@@ -46,6 +48,8 @@ func (c APIServer) Mux() *http.ServeMux {
 	mux.HandleFunc("/tests/upload_artifacts", c.uploadTestArtifact)
 	mux.HandleFunc("/tests/upload", c.uploadTest)
 	mux.HandleFunc("/trees", c.getTrees)
+	mux.HandleFunc("/base_findings/upload", c.uploadBaseFinding)
+	mux.HandleFunc("/base_findings/status", c.baseFindingStatus)
 	return mux
 }
 
@@ -206,3 +210,32 @@ func (c APIServer) getTrees(w http.ResponseWriter, r *http.Request) {
 		FuzzConfigs: api.FuzzConfigs,
 	})
 }
+
+func (c APIServer) uploadBaseFinding(w http.ResponseWriter, r *http.Request) {
+	req := api.ParseJSON[api.BaseFindingInfo](w, r)
+	if req == nil {
+		return
+	}
+	err := c.baseFindingService.Upload(r.Context(), req)
+	if errors.Is(err, service.ErrBuildNotFound) {
+		http.Error(w, fmt.Sprint(err), http.StatusNotFound)
+		return
+	} else if err != nil {
+		http.Error(w, fmt.Sprint(err), http.StatusInternalServerError)
+		return
+	}
+	api.ReplyJSON[interface{}](w, nil)
+}
+
+func (c APIServer) baseFindingStatus(w http.ResponseWriter, r *http.Request) {
+	req := api.ParseJSON[api.BaseFindingInfo](w, r)
+	if req == nil {
+		return
+	}
+	resp, err := c.baseFindingService.Status(r.Context(), req)
+	if err != nil {
+		http.Error(w, fmt.Sprint(err), http.StatusInternalServerError)
+		return
+	}
+	api.ReplyJSON[*api.BaseFindingStatus](w, resp)
+}

syz-cluster/pkg/controller/api_test.go

@@ -139,6 +139,46 @@ func TestAPIUploadTestArtifacts(t *testing.T) {
 	assert.NoError(t, err)
 }
 
+func TestAPIBaseFindings(t *testing.T) {
+	env, ctx := app.TestEnvironment(t)
+	client := TestServer(t, env)
+	buildResp := UploadTestBuild(t, ctx, client, testBuild)
+
+	err := client.UploadBaseFinding(ctx, &api.BaseFindingInfo{
+		BuildID: buildResp.ID,
+		Title:   "title 1",
+	})
+	assert.NoError(t, err)
+
+	// Let's upload a different build for the same revision.
+	buildResp2 := UploadTestBuild(t, ctx, client, testBuild)
+	assert.NotEqual(t, buildResp.ID, buildResp2.ID)
+
+	resp, err := client.BaseFindingStatus(ctx, &api.BaseFindingInfo{
+		BuildID: buildResp2.ID,
+		Title:   "title 1",
+	})
+	assert.NoError(t, err)
+	assert.True(t, resp.Observed)
+
+	t.Run("unseen title", func(t *testing.T) {
+		resp, err := client.BaseFindingStatus(ctx, &api.BaseFindingInfo{
+			BuildID: buildResp2.ID,
+			Title:   "title 2",
+		})
+		assert.NoError(t, err)
+		assert.False(t, resp.Observed)
+	})
+
+	t.Run("invalid build id", func(t *testing.T) {
+		_, err := client.BaseFindingStatus(ctx, &api.BaseFindingInfo{
+			BuildID: "unknown id",
+			Title:   "title 1",
+		})
+		assert.Error(t, err)
+	})
+}
+
 var testSeries = &api.Series{
 	ExtID:       "ext-id",
 	AuthorEmail: "some@email.com",

syz-cluster/pkg/db/base_finding_repo.go

@@ -0,0 +1,49 @@
+// Copyright 2025 syzkaller project authors. All rights reserved.
+// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.
+
+package db
+
+import (
+	"context"
+
+	"cloud.google.com/go/spanner"
+)
+
+type BaseFindingRepository struct {
+	client *spanner.Client
+}
+
+func NewBaseFindingRepository(client *spanner.Client) *BaseFindingRepository {
+	return &BaseFindingRepository{
+		client: client,
+	}
+}
+
+func (repo *BaseFindingRepository) Save(ctx context.Context, info *BaseFinding) error {
+	_, err := repo.client.ReadWriteTransaction(ctx,
+		func(ctx context.Context, txn *spanner.ReadWriteTransaction) error {
+			m, err := spanner.InsertOrUpdateStruct("BaseFindings", info)
+			if err != nil {
+				return err
+			}
+			return txn.BufferWrite([]*spanner.Mutation{m})
+		})
+	return err
+}
+
+func (repo *BaseFindingRepository) Exists(ctx context.Context, info *BaseFinding) (bool, error) {
+	entity, err := readEntity[BaseFinding](ctx, repo.client.Single(), spanner.Statement{
+		SQL: `SELECT * FROM BaseFindings WHERE
+CommitHash = @commit AND
+Config = @config AND
+Arch = @arch AND
+Title = @title`,
+		Params: map[string]interface{}{
+			"commit": info.CommitHash,
+			"config": info.Config,
+			"arch":   info.Arch,
+			"title":  info.Title,
+		},
+	})
+	return entity != nil, err
+}

syz-cluster/pkg/db/base_finding_repo_test.go

@@ -0,0 +1,40 @@
+// Copyright 2025 syzkaller project authors. All rights reserved.
+// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.
+
+package db
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestBaseFindingRepository(t *testing.T) {
+	client, ctx := NewTransientDB(t)
+	repo := NewBaseFindingRepository(client)
+
+	// It works fine on unknown titles.
+	exists, err := repo.Exists(ctx, &BaseFinding{
+		CommitHash: "abcd",
+		Config:     "cfg",
+		Arch:       "x86",
+	})
+	require.NoError(t, err)
+	assert.False(t, exists)
+
+	// Add some new title.
+	finding := &BaseFinding{
+		CommitHash: "hash",
+		Config:     "config",
+		Arch:       "arch",
+		Title:      "title",
+	}
+	err = repo.Save(ctx, finding)
+	require.NoError(t, err)
+
+	// Verify it exists.
+	exists, err = repo.Exists(ctx, finding)
+	require.NoError(t, err)
+	assert.True(t, exists)
+}

syz-cluster/pkg/db/entities.go

@@ -161,3 +161,12 @@ type ReportReply struct {
 	ReportID  string    `spanner:"ReportID"`
 	Time      time.Time `spanner:"Time"`
 }
+
+// BaseFinding collects all crashes observed on the base kernel tree.
+// It will be used to avoid unnecessary bug reproduction attempts.
+type BaseFinding struct {
+	CommitHash string `spanner:"CommitHash"`
+	Config     string `spanner:"Config"`
+	Arch       string `spanner:"Arch"`
+	Title      string `spanner:"Title"`
+}

syz-cluster/pkg/db/migrations/6_base_findings.down.sql

@@ -0,0 +1 @@
+DROP TABLE BaseFindings;