dashboard: export bug archives as .jsonl.gz to GCS

To facilitate syzbot statistics analysis, export jsonl files with
per-bug info to GCS for selected namespaces.

Author: a-nogikh

dashboard/app/config.go

@@ -139,6 +139,9 @@ type Config struct {
 	Coverage *CoverageConfig
 	// Reproducers export path.
 	ReproExportPath string
+	// Bug archive export path (a .jsonl.gz file).
+	// Should be of the form bucket/full_path.
+	BugArchiveExportPath string
 }
 
 // ACLItem is an Access Control List item.

dashboard/app/cron.yaml

@@ -39,3 +39,5 @@ cron:
 # Weekly update the kernel-file -> subsystems relationship
 - url: /cron/update_coverdb_subsystems
   schedule: every monday
+- url: /cron/export_bugs
+  schedule: every day 01:00

dashboard/app/export.go

@@ -0,0 +1,122 @@
+// Copyright 2025 syzkaller project authors. All rights reserved.
+// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.
+
+package main
+
+import (
+	"compress/gzip"
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"net/http"
+
+	"github.com/google/syzkaller/dashboard/api"
+	"github.com/google/syzkaller/pkg/gcs"
+	"golang.org/x/sync/errgroup"
+	"google.golang.org/appengine/v2"
+	db "google.golang.org/appengine/v2/datastore"
+	"google.golang.org/appengine/v2/log"
+)
+
+func handleExportBugs(w http.ResponseWriter, r *http.Request) {
+	c := appengine.NewContext(r)
+	for ns, nsConfig := range getConfig(c).Namespaces {
+		if nsConfig.BugArchiveExportPath == "" {
+			continue
+		}
+		log.Infof(c, "exporting bugs for %q", ns)
+		err := uploadBugsJSONL(c, ns, nsConfig.BugArchiveExportPath)
+		if err != nil {
+			log.Errorf(c, "failed to export %q bugs: %v", ns, err)
+			http.Error(w, "Internal Server Error", http.StatusInternalServerError)
+			return
+		}
+	}
+}
+
+const (
+	// With 16 threads, it takes 6-7 minutes to export 26k bugs
+	// from the upstream namespace. 32 threads do not accelerate it.
+	exportQueryThreads = 32
+	exportAccessLevel  = AccessPublic
+)
+
+func uploadBugsJSONL(c context.Context, ns, path string) error {
+	stream := make(chan *api.Bug, exportQueryThreads)
+	eg, ctx := errgroup.WithContext(c)
+	eg.Go(func() error {
+		defer close(stream)
+		return queryBugInfos(ctx, ns, stream)
+	})
+	eg.Go(func() error {
+		return uploadJSONL(ctx, path, stream)
+	})
+	return eg.Wait()
+}
+
+func queryBugInfos(c context.Context, ns string, stream chan<- *api.Bug) error {
+	bugs, _, err := loadAllBugs(c, func(query *db.Query) *db.Query {
+		return query.Filter("Namespace=", ns)
+	})
+	if err != nil {
+		return fmt.Errorf("failed to load bugs: %w", err)
+	}
+	log.Infof(c, "loaded %d bugs", len(bugs))
+	eg, ctx := errgroup.WithContext(c)
+	eg.SetLimit(exportQueryThreads)
+	for _, bug := range bugs {
+		if exportAccessLevel < bug.sanitizeAccess(ctx, exportAccessLevel) {
+			continue
+		}
+		eg.Go(func() error {
+			details, err := loadBugDetails(ctx, bug, exportAccessLevel)
+			if err != nil {
+				return err
+			}
+			select {
+			case stream <- getExtAPIDescrForBug(details):
+				return nil
+			case <-ctx.Done():
+				return ctx.Err()
+			}
+		})
+	}
+	return eg.Wait()
+}
+
+func uploadJSONL(c context.Context, path string, stream <-chan *api.Bug) error {
+	client, err := gcs.NewClient(c)
+	if err != nil {
+		return fmt.Errorf("failed create a gcs client: %w", err)
+	}
+	wc, err := client.FileWriter(path, "application/gzip", "")
+	if err != nil {
+		return fmt.Errorf("file writer ext failed: %w", err)
+	}
+	gzWriter := gzip.NewWriter(wc)
+	enc := json.NewEncoder(gzWriter)
+	count := 0
+	for bug := range stream {
+		err := enc.Encode(bug)
+		if err != nil {
+			return fmt.Errorf("failed to encode bug ID=%s: %w", bug.ID, err)
+		}
+		count++
+	}
+	log.Infof(c, "%d bugs exported", count)
+	// Save the file only if the context has not been canceled by now.
+	if ctxErr := c.Err(); ctxErr != nil {
+		if errors.Is(ctxErr, context.Canceled) {
+			return nil
+		}
+		return ctxErr
+	}
+	if err := gzWriter.Close(); err != nil {
+		return fmt.Errorf("failed to close gzip writer: %w", err)
+	}
+	if err := wc.Close(); err != nil {
+		return fmt.Errorf("unable to close writer: %w", err)
+	}
+	return client.Publish(path)
+}

dashboard/app/main.go

@@ -89,6 +89,7 @@ func initHTTPHandlers() {
 	http.HandleFunc("/cron/refresh_subsystems", handleRefreshSubsystems)
 	http.HandleFunc("/cron/subsystem_reports", handleSubsystemReports)
 	http.HandleFunc("/cron/update_coverdb_subsystems", handleUpdateCoverDBSubsystems)
+	http.HandleFunc("/cron/export_bugs", handleExportBugs)
 }
 
 func handleMovedPermanently(dest string) http.HandlerFunc {