Merge branch 'verifier-dev' into prepare-for-pr

Author: dBransky

.github/workflows/gemini-pr-review.yml

@@ -104,7 +104,7 @@ ifeq ("$(TARGETOS)", "trusty")
 endif
 
 .PHONY: all clean host target \
-	manager executor kfuzztest ci hub \
+	manager executor ci hub \
 	execprog mutate prog2c trace2syz repro upgrade db \
 	usbgen symbolize cover kconf syz-build crush \
 	bin/syz-extract bin/syz-fmt \
@@ -118,7 +118,7 @@ endif
 
 all: host target
 host: manager repro mutate prog2c db upgrade
-target: execprog executor check_syzos
+target: execprog executor
 
 executor: descriptions
 ifeq ($(TARGETOS),fuchsia)
@@ -217,14 +217,6 @@ syz-build:
 bisect: descriptions
 	GOOS=$(HOSTOS) GOARCH=$(HOSTARCH) $(HOSTGO) build $(GOHOSTFLAGS) -o ./bin/syz-bisect github.com/google/syzkaller/tools/syz-bisect
 
-ifeq ($(HOSTOS), linux)
-kfuzztest: descriptions
-	GOOS=$(HOSTOS) GOARCH=$(HOSTARCH) $(HOSTGO) build $(GOHOSTFLAGS) -o ./bin/syz-kfuzztest github.com/google/syzkaller/syz-kfuzztest
-else
-kfuzztest:
-	@echo "Skipping kfuzztest build (it's Linux-only)"
-endif
-
 verifier: descriptions
 	GOOS=$(HOSTOS) GOARCH=$(HOSTARCH) $(HOSTGO) build $(GOHOSTFLAGS) -o ./bin/syz-verifier github.com/google/syzkaller/syz-verifier
 
@@ -271,9 +263,7 @@ format_cpp:
 	clang-format --style=file -i executor/*.cc executor/*.h \
 		executor/android/android_seccomp.h \
 		tools/kcovtrace/*.c tools/kcovfuzzer/*.c tools/fops_probe/*.cc \
-		tools/clang/*.h \
-		tools/clang/declextract/*.h tools/clang/declextract/*.cpp \
-		tools/clang/codesearch/*.h tools/clang/codesearch/*.cpp
+		tools/syz-declextract/clangtool/*.cpp tools/syz-declextract/clangtool/*.h
 
 format_sys: bin/syz-fmt
 	bin/syz-fmt all
@@ -282,7 +272,7 @@ bin/syz-fmt:
 	$(HOSTGO) build $(GOHOSTFLAGS) -o $@ ./tools/syz-fmt
 
 configs: kconf
-	bin/syz-kconf -config dashboard/config/linux/main.yml -sourcedir $(SOURCEDIR) -instance=$(INSTANCE)
+	bin/syz-kconf -config dashboard/config/linux/main.yml -sourcedir $(SOURCEDIR)
 
 tidy: descriptions
 	clang-tidy -quiet -header-filter=executor/[^_].* -warnings-as-errors=* \
@@ -294,7 +284,7 @@ tidy: descriptions
 		executor/*.cc
 
 lint:
-	CGO_ENABLED=1 $(HOSTGO) install github.com/golangci/golangci-lint/v2/cmd/golangci-lint@v2.6.2
+	CGO_ENABLED=1 $(HOSTGO) install github.com/golangci/golangci-lint/v2/cmd/golangci-lint
 	CGO_ENABLED=1 $(HOSTGO) build -buildmode=plugin -o bin/syz-linter.so ./tools/syz-linter
 	bin/golangci-lint run ./...
 
@@ -436,9 +426,6 @@ check_links:
 check_html:
 	./tools/check-html.sh
 
-check_syzos: executor
-	./tools/check-syzos.sh 2>/dev/null
-
 # Check that the diff is empty. This is meant to be executed after generating
 # and formatting the code to make sure that everything is committed.
 check_diff:

Makefile

@@ -4,11 +4,11 @@
 package main
 
 import (
+	"bytes"
 	"context"
 	"encoding/json"
 	"fmt"
 	"net/mail"
-	"net/url"
 	"regexp"
 	"strings"
 	"time"
@@ -366,10 +366,12 @@ type CCConfig struct {
 type KcidbConfig struct {
 	// Origin is how this system identified in Kcidb, e.g. "syzbot_foobar".
 	Origin string
-	// RestURI is the REST API endpoint to which the Kcidb client will send data.
-	RestURI string
-	// Token is the authorization token to use for the Kcidb client.
-	Token string
+	// Project is Kcidb GCE project name, e.g. "kernelci-production".
+	Project string
+	// Topic is pubsub topic to publish messages to, e.g. "playground_kernelci_new".
+	Topic string
+	// Credentials is Google application credentials file contents to use for authorization.
+	Credentials []byte
 }
 
 // ThrottleConfig determines how many requests a single client can make in a period of time.
@@ -811,15 +813,14 @@ func checkKcidb(ns string, kcidb *KcidbConfig) {
 	if !regexp.MustCompile("^[a-z0-9_]+$").MatchString(kcidb.Origin) {
 		panic(fmt.Sprintf("%v: bad Kcidb origin %q", ns, kcidb.Origin))
 	}
-	if kcidb.RestURI == "" {
-		panic(fmt.Sprintf("%v: empty Kcidb RestURI", ns))
+	if kcidb.Project == "" {
+		panic(fmt.Sprintf("%v: empty Kcidb project", ns))
 	}
-	// Validate RestURI must be a valid URL.
-	if _, err := url.ParseRequestURI(kcidb.RestURI); err != nil {
-		panic(fmt.Sprintf("%v: invalid Kcidb RestURI %q: %v", ns, kcidb.RestURI, err))
+	if kcidb.Topic == "" {
+		panic(fmt.Sprintf("%v: empty Kcidb topic", ns))
 	}
-	if kcidb.Token == "" || len(kcidb.Token) < 8 {
-		panic(fmt.Sprintf("%v: bad Kcidb token %q", ns, kcidb.Token))
+	if !bytes.Contains(kcidb.Credentials, []byte("private_key")) {
+		panic(fmt.Sprintf("%v: empty Kcidb credentials", ns))
 	}
 }
 

dashboard/app/config.go

@@ -66,14 +66,13 @@ func TestJob(t *testing.T) {
 	c.incomingEmail(sender, "#syz test: repo",
 		EmailOptFrom("test@requester.com"), EmailOptSubject("my-subject"), EmailOptCC([]string{mailingList}))
 	msg := c.pollEmailBug()
-
-	c.expectEQ(strings.Contains(msg.Body, replyMalformedSyzTest), true)
+	c.expectEQ(strings.Contains(msg.Body, "want either no args or 2 args"), true)
 	c.expectEQ(msg.Subject, "Re: my-subject")
 
 	c.incomingEmail(sender, "#syz test: repo branch commit",
 		EmailOptFrom("test@requester.com"), EmailOptSubject("Re: my-subject"), EmailOptCC([]string{mailingList}))
 	msg = c.pollEmailBug()
-	c.expectEQ(strings.Contains(msg.Body, replyMalformedSyzTest), true)
+	c.expectEQ(strings.Contains(msg.Body, "want either no args or 2 args"), true)
 	c.expectEQ(msg.Subject, "Re: my-subject")
 
 	c.incomingEmail(sender, "#syz test: repo branch",

dashboard/app/jobs_test.go

@@ -32,7 +32,7 @@ func handleKcidbPoll(w http.ResponseWriter, r *http.Request) {
 }
 
 func handleKcidbNamespce(c context.Context, ns string, cfg *KcidbConfig) error {
-	client, err := kcidb.NewClient(c, cfg.Origin, cfg.RestURI, cfg.Token)
+	client, err := kcidb.NewClient(c, cfg.Origin, cfg.Project, cfg.Topic, cfg.Credentials)
 	if err != nil {
 		return err
 	}

dashboard/app/kcidb.go

@@ -74,12 +74,6 @@ const (
 		"The email is sent to  %[1]v address\n" +
 		"but the HASH does not correspond to any known bug.\n" +
 		"Please double check the address."
-	replyMalformedSyzTest = "I've failed to parse your command.\n" +
-		"Did you perhaps forget to provide the branch name, or added an extra ':'?\n" +
-		"Please use one of the two supported formats:\n" +
-		"1. #syz test\n" +
-		"2. #syz test: repo branch-or-commit-hash\n" +
-		"Note the lack of ':' in option 1."
 )
 
 var mailingLists map[string]bool
@@ -187,7 +181,7 @@ func sendNsCoverageReport(ctx context.Context, ns, email string,
 			coveragePageLink(ns, period[1].Type, period[1].DateTo.String(), minDrop, 2, true)),
 		Table: table,
 	}
-	title := fmt.Sprintf("%s coverage regression in %s", ns, periodTo)
+	title := fmt.Sprintf("%s coverage regression (%s)->(%s)", ns, periodFrom, periodTo)
 	err = sendMailTemplate(ctx, &mailSendParams{
 		templateName: "mail_ns_coverage.txt",
 		templateArg:  args,
@@ -903,7 +897,7 @@ func handleTestCommand(c context.Context, info *bugInfoResult,
 	msg *email.Email, command *email.SingleCommand) string {
 	args := strings.Fields(command.Args)
 	if len(args) != 0 && len(args) != 2 {
-		return replyMalformedSyzTest
+		return fmt.Sprintf("want either no args or 2 args (repo, branch), got %v", len(args))
 	}
 	repo, branch := "", ""
 	if len(args) == 2 {

dashboard/app/reporting_email.go

@@ -1406,7 +1406,7 @@ func TestCoverageRegression(t *testing.T) {
 	assert.Equal(t, 1, len(c.emailSink))
 	msg := <-c.emailSink
 	assert.Equal(t, []string{"test@test.test"}, msg.To)
-	assert.Equal(t, "coverage-tests coverage regression in December 1999", msg.Subject)
+	assert.Equal(t, "coverage-tests coverage regression (November 1999)->(December 1999)", msg.Subject)
 	wantLink := "https://testapp.appspot.com/coverage-tests/coverage?" +
 		"dateto=1999-12-31&min-cover-lines-drop=1&order-by-cover-lines-drop=1&period=month&period_count=2"
 	assert.Equal(t, `Regressions happened in 'coverage-tests' from November 1999 (30 days) to December 1999 (31 days).

dashboard/app/reporting_test.go

@@ -631,7 +631,7 @@ <h1>{{.Caption}}:</h1>
 			<td class="stat">{{link $item.KernelCommitLink (formatTagHash $item.KernelCommit)}}</td>
 			<td>
 				{{if $item.ReproCLink}}<a href="{{$item.ReproCLink}}">C</a>
-				{{else if $item.ReproSyzLink}}<a href="{{$item.ReproSyzLink}}">syz</a>{{end}}
+				{{else if $item.ReproSyzLink}}<a href="{{$item.ReproSyzLink}}">C</a>{{end}}
 			</td>
 			{{if ne $item.CrashTitle ""}}
 			<td class="status-crashed">

dashboard/app/templates/templates.html

@@ -63,7 +63,6 @@ func NewCtx(t *testing.T) *Ctx {
 	}
 	t.Parallel()
 	inst, err := aetest.NewInstance(&aetest.Options{
-		StartupTimeout: 120 * time.Second,
 		// Without this option datastore queries return data with slight delay,
 		// which fails reporting tests.
 		StronglyConsistentDatastore: true,

dashboard/app/util_test.go

@@ -101,7 +101,7 @@ The `baseline` config has `-base` suffix and has `baseline` and `base-config` fe
 config is meant to not contain majority of subsystem configs enabled (should just boot and have debugging
 configs enabled), however, that is fully controlled by config fragments based on the presence of the `baseline`
 feature. In order to facilitate even smaller `-base.config` files, the `base-config` feature is automatically
-set for them. Such configs are intended to be used by the config minimization procedure.
+set for them. Such configs are intended to be used by the config minization procedure.
 
 The only exception to the declarative nature of the process is USB configs. They are added procedurally, see
 `addUSBConfigs` function for details.
@@ -110,12 +110,10 @@ The only exception to the declarative nature of the process is USB configs. They
 [syz-env](/docs/contributing.md#using-syz-env), which provides correct versions of the compilers.
 
 To update kernel configs:
- - update kernel version in [linux-next.yml](bits/linux-next.yml) and [linux-upstream.yml](bits/linux-upstream.yml)
- - change config fragments as necessary (e.g. add additional configs to [subsystems.yml](bits/subsystems.yml))
- - run `./tools/syz-env make configs SOURCEDIR=/path/to/existing/linux/checkout`
-   * note: it will be mrproper-ed and a number of remotes will be added
-   * see [this](/docs/contributing.md#using-syz-env) on how to setup/use `syz-env`
- - resolve problems. You can iterate faster focusing on the specific instance problem.
-   * `./tools/syz-env make configs SOURCEDIR=/path/to/existing/linux/checkout INSTANCE=PROBLEM_INSTANCE_NAME`
+ - change config fragments as necessary (e.g. add additional configs to [subsystems.yml](bits/subsystems.yml) along
+   with minimal kernel version)
+ - run `syz-env make configs SOURCEDIR=/path/to/existing/linux/checkout`
+   (note: it will be mrproper-ed and a number of remotes will be added)
+   (see [this](/docs/contributing.md#using-syz-env) on how to setup/use `syz-env`)
  - check in config fragments and changed kernel configs and send a PR
  - changes will be deployed to `syzbot` within a day after merging