syz-ci: upload syz progs with coverage to gcs

Author: tarasmadan

pkg/cover/html.go

@@ -279,6 +279,8 @@ type FileCoverage struct {
 }
 
 type ProgramCoverage struct {
+	Repo         string          `json:"repo,omitempty"`
+	Commit       string          `json:"commit,omitempty"`
 	Program      string          `json:"prog_base_64"`
 	CoveredFiles []*FileCoverage `json:"coverage"`
 }

pkg/cover/manager_to_ci.go

@@ -27,24 +27,24 @@ type dbCoverageRecord struct {
 	CoverageInfo
 }
 
-func writeJSLine(w io.Writer, covInfo dbCoverageRecord) error {
-	bs, err := json.Marshal(covInfo)
+func WriteJSLine[T any](w io.Writer, obj *T) error {
+	bs, err := json.Marshal(obj)
 	if err != nil {
-		return fmt.Errorf("failed to marshal covInfo: %w", err)
+		return fmt.Errorf("json.Marshal: %w", err)
 	}
 	bs = append(bs, '\n')
 	if _, err = w.Write(bs); err != nil {
-		return fmt.Errorf("failed to write js data: %w", err)
+		return fmt.Errorf("w.Write: %w", err)
 	}
 	return nil
 }
 
 func WriteCIJSONLine(w io.Writer, managerCover CoverageInfo, ciDetails CIDetails) error {
-	dbLine := dbCoverageRecord{
+	dbLine := &dbCoverageRecord{
 		CIDetails:    ciDetails,
 		CoverageInfo: managerCover,
 	}
-	if err := writeJSLine(w, dbLine); err != nil {
+	if err := WriteJSLine(w, dbLine); err != nil {
 		return fmt.Errorf("failed to serialize func line: %w", err)
 	}
 	return nil

syz-ci/manager.go

@@ -216,6 +216,9 @@ loop:
 			if err := mgr.uploadCoverReport(); err != nil {
 				mgr.Errorf("failed to upload cover report: %v", err)
 			}
+			if err := mgr.uploadProgramsWithCoverage(); err != nil {
+				mgr.Errorf("failed to upload programs with coverage: %v", err)
+			}
 			if err := mgr.uploadCoverStat(fuzzingMinutesBeforeCover); err != nil {
 				mgr.Errorf("failed to upload coverage stat: %v", err)
 			}
@@ -875,8 +878,8 @@ func (mgr *Manager) uploadCoverReport() error {
 	return nil
 }
 
-func (mgr *Manager) uploadCoverStat(fuzzingMinutes int) error {
-	if !mgr.managercfg.Cover || mgr.cfg.CoverPipelinePath == "" {
+func (mgr *Manager) uploadJSONLToGCS(mgrSrc, gcsDest string, f func(io.Writer, *json.Decoder, time.Time) error) error {
+	if !mgr.managercfg.Cover || gcsDest == "" {
 		return nil
 	}
 
@@ -889,19 +892,16 @@ func (mgr *Manager) uploadCoverStat(fuzzingMinutes int) error {
 	}
 	defer buildSem.Signal()
 
-	// Coverage report generation consumes and caches lots of memory.
-	// In the syz-ci context report generation won't be used after this point,
-	// so tell manager to flush report generator.
-	resp, err := mgr.httpGET("/cover?jsonl=1&flush=1")
+	resp, err := mgr.httpGET(mgrSrc)
 	if err != nil {
-		return fmt.Errorf("failed to httpGet /cover?jsonl=1 report: %w", err)
+		return fmt.Errorf("failed to httpGet %s: %w", mgrSrc, err)
 	}
 	defer resp.Body.Close()
 	if resp.StatusCode != http.StatusOK {
 		sb := new(strings.Builder)
 		io.Copy(sb, resp.Body)
-		return fmt.Errorf("failed to GET /cover?jsonl=1, httpStatus %d: %s",
-			resp.StatusCode, sb.String())
+		return fmt.Errorf("failed to GET %s, httpStatus %d: %s",
+			mgrSrc, resp.StatusCode, sb.String())
 	}
 
 	curTime := time.Now()
@@ -910,12 +910,35 @@ func (mgr *Manager) uploadCoverStat(fuzzingMinutes int) error {
 	go func() {
 		decoder := json.NewDecoder(resp.Body)
 		for decoder.More() {
-			var covInfo cover.CoverageInfo
-			if err := decoder.Decode(&covInfo); err != nil {
-				pw.CloseWithError(fmt.Errorf("failed to decode CoverageInfo: %w", err))
+			if err := f(pw, decoder, curTime); err != nil {
+				pw.CloseWithError(fmt.Errorf("callback: %w", err))
 				return
 			}
-			if err := cover.WriteCIJSONLine(pw, covInfo, cover.CIDetails{
+		}
+		pw.Close()
+	}()
+	fileName := fmt.Sprintf("%s/%s-%s-%d-%d.jsonl",
+		mgr.mgrcfg.DashboardClient,
+		mgr.name, curTime.Format(time.DateOnly),
+		curTime.Hour(), curTime.Minute())
+	if err := mgr.uploadFile(mgr.cfg.CoverPipelinePath, fileName, pr, false); err != nil {
+		return fmt.Errorf("failed to uploadFileGCS(): %w", err)
+	}
+	return nil
+}
+
+func (mgr *Manager) uploadCoverStat(fuzzingMinutes int) error {
+	// Coverage report generation consumes and caches lots of memory.
+	// In the syz-ci context report generation won't be used after this point,
+	// so tell manager to flush report generator.
+	if err := mgr.uploadJSONLToGCS("/cover?jsonl=1&flush=1",
+		mgr.cfg.CoverPipelinePath,
+		func(w io.Writer, dec *json.Decoder, curTime time.Time) error {
+			var covInfo cover.CoverageInfo
+			if err := dec.Decode(&covInfo); err != nil {
+				return fmt.Errorf("failed to decode CoverageInfo: %w", err)
+			}
+			if err := cover.WriteCIJSONLine(w, covInfo, cover.CIDetails{
 				Version:        1,
 				Timestamp:      curTime.Format(time.RFC3339Nano),
 				FuzzingMinutes: fuzzingMinutes,
@@ -926,18 +949,31 @@ func (mgr *Manager) uploadCoverStat(fuzzingMinutes int) error {
 				KernelBranch:   mgr.lastBuild.KernelBranch,
 				KernelCommit:   mgr.lastBuild.KernelCommit,
 			}); err != nil {
-				pw.CloseWithError(fmt.Errorf("failed to write CIJSONLine: %w", err))
-				return
+				return fmt.Errorf("failed to write CIJSONLine: %w", err)
 			}
-		}
-		pw.Close()
-	}()
-	fileName := fmt.Sprintf("%s/%s-%s-%d-%d.jsonl",
-		mgr.mgrcfg.DashboardClient,
-		mgr.name, curTime.Format(time.DateOnly),
-		curTime.Hour(), curTime.Minute())
-	if err := mgr.uploadFile(mgr.cfg.CoverPipelinePath, fileName, pr, false); err != nil {
-		return fmt.Errorf("failed to uploadFileGCS(): %w", err)
+			return nil
+		}); err != nil {
+		return fmt.Errorf("mgr.uploadJSONLToGCS: %w", err)
+	}
+	return nil
+}
+
+func (mgr *Manager) uploadProgramsWithCoverage() error {
+	if err := mgr.uploadJSONLToGCS("/coverprogs?jsonl=1",
+		mgr.cfg.CoverProgramsPath,
+		func(w io.Writer, dec *json.Decoder, curTime time.Time) error {
+			var programCoverage cover.ProgramCoverage
+			if err := dec.Decode(&programCoverage); err != nil {
+				return fmt.Errorf("cover.ProgramCoverage: %w", err)
+			}
+			programCoverage.Repo = mgr.lastBuild.KernelRepo
+			programCoverage.Commit = mgr.lastBuild.KernelCommit
+			if err := cover.WriteJSLine(w, &programCoverage); err != nil {
+				return fmt.Errorf("cover.WriteJSLine: %w", err)
+			}
+			return nil
+		}); err != nil {
+		return fmt.Errorf("mgr.uploadJSONLToGCS: %w", err)
 	}
 	return nil
 }

syz-ci/syz-ci.go

@@ -107,6 +107,8 @@ type Config struct {
 	// Path to upload coverage reports from managers (optional).
 	// Supported protocols: GCS (gs://) and HTTP PUT (http:// or https://).
 	CoverUploadPath string `json:"cover_upload_path"`
+	// Path to upload managers syz programs and their coverage in jsonl (optional).
+	CoverProgramsPath string `json:"cover_programs_path"`
 	// Path to upload json coverage reports from managers (optional).
 	CoverPipelinePath string `json:"cover_pipeline_path"`
 	// Path to upload corpus.db from managers (optional).