dashboard/app: periodically remove coverage garbage from spanner

Author: tarasmadan

dashboard/app/batch_coverage.go

@@ -7,11 +7,13 @@ import (
 	"context"
 	"fmt"
 	"net/http"
+	"os"
 	"strconv"
 
 	"cloud.google.com/go/batch/apiv1/batchpb"
 	"cloud.google.com/go/bigquery"
 	"cloud.google.com/go/civil"
+	"cloud.google.com/go/spanner"
 	"github.com/google/syzkaller/pkg/coveragedb"
 	"google.golang.org/api/iterator"
 	"google.golang.org/appengine/v2"
@@ -151,3 +153,45 @@ func nsDataAvailable(ctx context.Context, ns string) ([]coveragedb.TimePeriod, [
 	}
 	return periods, recordsCount, nil
 }
+
+func handleBatchCoverageClean(w http.ResponseWriter, r *http.Request) {
+	err, deletedRows := cleanCoverageDB(20)
+	if err != nil {
+		w.WriteHeader(http.StatusInternalServerError)
+		w.Write([]byte(fmt.Sprintf("failed to cleanCoverageDB: %w", err.Error())))
+		return
+	}
+	w.Write([]byte(fmt.Sprintf("cleanCoverageDB deleted %d rows", deletedRows)))
+}
+
+func cleanCoverageDB(maxSessionsToDelete int) (error, int64) {
+	ctx := context.Background()
+	client, err := spanner.NewClient(ctx, os.Getenv("GOOGLE_CLOUD_PROJECT"))
+	if err != nil {
+		return fmt.Errorf("failed to coveragedb.NewClient: %w", err), 0
+	}
+	defer client.Close()
+	var rowCount int64
+
+	_, err = client.ReadWriteTransaction(ctx, func(ctx context.Context, txn *spanner.ReadWriteTransaction) error {
+		stmt := spanner.Statement{
+			SQL: fmt.Sprintf(`	delete from files
+																where files.session in (
+																	select
+																		distinct(files.session)
+																	from files
+																	left join merge_history
+																		on files.session = merge_history.session
+																	where merge_history.session is NULL
+																	limit %d
+																);`, maxSessionsToDelete),
+		}
+		var err error
+		rowCount, err = txn.Update(ctx, stmt)
+		if err != nil {
+			return fmt.Errorf("txn.Update: %w", err)
+		}
+		return nil
+	})
+	return err, rowCount
+}

dashboard/app/batch_main.go

@@ -18,6 +18,7 @@ import (
 func initBatchProcessors() {
 	http.HandleFunc("/cron/batch_coverage", handleBatchCoverage)
 	http.HandleFunc("/cron/batch_db_export", handleBatchDBExport)
+	http.HandleFunc("/cron/batch_coverage_clean", handleBatchCoverageClean)
 }
 
 // from https://cloud.google.com/batch/docs/samples/batch-create-script-job

dashboard/app/cron.yaml

@@ -24,7 +24,12 @@ cron:
   schedule: every sunday 00:00
 # Update other coverage numbers every day.
 - url: /cron/batch_coverage?days=true&months=true&steps=10
-  schedule: every 24 hours
+  schedule: every day 00:00
+# Clean up coverage db every week.
+# We're adding data w/o transactions.
+# It is important to run clean operation when there are no batch_coverage in progress.
+- url: /cron/batch_coverage_clean
+  schedule: every saturday 12:00
 # Export reproducers every week.
 - url: /cron/batch_db_export
   schedule: every saturday 00:00

tools/syz-covermerger/init_db.sh

@@ -23,6 +23,9 @@ CREATE TABLE
     (session, filepath) );')
 gcloud spanner databases ddl update $db --instance=syzbot --project=syzkaller \
  --ddl="$create_table"
+echo "creating 'files' indexes"
+gcloud spanner databases ddl update $db --instance=syzbot --project=syzkaller \
+ --ddl="CREATE INDEX files_session ON files (session);"
 
 echo "drop table 'merge_history' if exists"
 gcloud spanner databases ddl update $db --instance=syzbot --project=syzkaller \