Skip to content

Commit b2d784d

Browse files
tarasmadantarasmadan
committed
dashboard/app: deserialize data directly from gz reader
1 parent b1b5e51 commit b2d784d

File tree

2 files changed

+60
-63
lines changed

2 files changed

+60
-63
lines changed

‎dashboard/app/api.go‎

Lines changed: 49 additions & 53 deletions
Original file line numberDiff line numberDiff line change
@@ -74,8 +74,8 @@ var apiNamespaceHandlers = map[string]APINamespaceHandler{
7474
}
7575

7676
type JSONHandler func(c context.Context, r *http.Request) (interface{}, error)
77-
type APIHandler func(c context.Context, r *http.Request, payload []byte) (interface{}, error)
78-
type APINamespaceHandler func(c context.Context, ns string, r *http.Request, payload []byte) (interface{}, error)
77+
type APIHandler func(c context.Context, r *http.Request, payload io.Reader) (interface{}, error)
78+
type APINamespaceHandler func(c context.Context, ns string, r *http.Request, payload io.Reader) (interface{}, error)
7979

8080
const (
8181
maxReproPerBug = 10
@@ -143,23 +143,19 @@ func handleAPI(c context.Context, r *http.Request) (reply interface{}, err error
143143
if err != nil {
144144
return nil, fmt.Errorf("checkClient('%s') error: %w", client, err)
145145
}
146-
var payload []byte
146+
var payloadReader io.Reader
147147
if str := r.PostFormValue("payload"); str != "" {
148148
gr, err := gzip.NewReader(strings.NewReader(str))
149149
if err != nil {
150150
return nil, fmt.Errorf("failed to ungzip payload: %w", err)
151151
}
152-
payload, err = io.ReadAll(gr)
153-
if err != nil {
154-
return nil, fmt.Errorf("failed to ungzip payload: %w", err)
155-
}
156-
if err := gr.Close(); err != nil {
157-
return nil, fmt.Errorf("failed to ungzip payload: %w", err)
158-
}
152+
payloadReader = gr
153+
// Ignore Close() error because we may not read all data.
154+
defer gr.Close()
159155
}
160156
handler := apiHandlers[method]
161157
if handler != nil {
162-
return handler(c, r, payload)
158+
return handler(c, r, payloadReader)
163159
}
164160
nsHandler := apiNamespaceHandlers[method]
165161
if nsHandler == nil {
@@ -168,21 +164,21 @@ func handleAPI(c context.Context, r *http.Request) (reply interface{}, err error
168164
if ns == "" {
169165
return nil, fmt.Errorf("method %q must be called within a namespace", method)
170166
}
171-
return nsHandler(c, ns, r, payload)
167+
return nsHandler(c, ns, r, payloadReader)
172168
}
173169

174-
func apiLogError(c context.Context, r *http.Request, payload []byte) (interface{}, error) {
170+
func apiLogError(c context.Context, r *http.Request, payload io.Reader) (interface{}, error) {
175171
req := new(dashapi.LogEntry)
176-
if err := json.Unmarshal(payload, req); err != nil {
172+
if err := json.NewDecoder(payload).Decode(req); err != nil {
177173
return nil, fmt.Errorf("failed to unmarshal request: %w", err)
178174
}
179175
log.Errorf(c, "%v: %v", req.Name, req.Text)
180176
return nil, nil
181177
}
182178

183-
func apiBuilderPoll(c context.Context, ns string, r *http.Request, payload []byte) (interface{}, error) {
179+
func apiBuilderPoll(c context.Context, ns string, r *http.Request, payload io.Reader) (interface{}, error) {
184180
req := new(dashapi.BuilderPollReq)
185-
if err := json.Unmarshal(payload, req); err != nil {
181+
if err := json.NewDecoder(payload).Decode(req); err != nil {
186182
return nil, fmt.Errorf("failed to unmarshal request: %w", err)
187183
}
188184
bugs, _, err := loadAllBugs(c, func(query *db.Query) *db.Query {
@@ -229,7 +225,7 @@ func reportEmail(c context.Context, ns string) string {
229225
return ""
230226
}
231227

232-
func apiCommitPoll(c context.Context, ns string, r *http.Request, payload []byte) (interface{}, error) {
228+
func apiCommitPoll(c context.Context, ns string, r *http.Request, payload io.Reader) (interface{}, error) {
233229
resp := &dashapi.CommitPollResp{
234230
ReportEmail: reportEmail(c, ns),
235231
}
@@ -295,9 +291,9 @@ func pollBackportCommits(c context.Context, ns string, count int) ([]string, err
295291
return backportTitles, nil
296292
}
297293

298-
func apiUploadCommits(c context.Context, ns string, r *http.Request, payload []byte) (interface{}, error) {
294+
func apiUploadCommits(c context.Context, ns string, r *http.Request, payload io.Reader) (interface{}, error) {
299295
req := new(dashapi.CommitPollResultReq)
300-
if err := json.Unmarshal(payload, req); err != nil {
296+
if err := json.NewDecoder(payload).Decode(req); err != nil {
301297
return nil, fmt.Errorf("failed to unmarshal request: %w", err)
302298
}
303299
// This adds fixing commits to bugs.
@@ -400,13 +396,13 @@ func addCommitInfoToBugImpl(c context.Context, bug *Bug, com dashapi.Commit) (bo
400396
return changed, nil
401397
}
402398

403-
func apiJobPoll(c context.Context, r *http.Request, payload []byte) (interface{}, error) {
399+
func apiJobPoll(c context.Context, r *http.Request, payload io.Reader) (interface{}, error) {
404400
if stop, err := emergentlyStopped(c); err != nil || stop {
405401
// The bot's operation was aborted. Don't accept new crash reports.
406402
return &dashapi.JobPollResp{}, err
407403
}
408404
req := new(dashapi.JobPollReq)
409-
if err := json.Unmarshal(payload, req); err != nil {
405+
if err := json.NewDecoder(payload).Decode(req); err != nil {
410406
return nil, fmt.Errorf("failed to unmarshal request: %w", err)
411407
}
412408
if len(req.Managers) == 0 {
@@ -416,28 +412,28 @@ func apiJobPoll(c context.Context, r *http.Request, payload []byte) (interface{}
416412
}
417413

418414
// nolint: dupl
419-
func apiJobDone(c context.Context, r *http.Request, payload []byte) (interface{}, error) {
415+
func apiJobDone(c context.Context, r *http.Request, payload io.Reader) (interface{}, error) {
420416
req := new(dashapi.JobDoneReq)
421-
if err := json.Unmarshal(payload, req); err != nil {
417+
if err := json.NewDecoder(payload).Decode(req); err != nil {
422418
return nil, fmt.Errorf("failed to unmarshal request: %w", err)
423419
}
424420
err := doneJob(c, req)
425421
return nil, err
426422
}
427423

428424
// nolint: dupl
429-
func apiJobReset(c context.Context, r *http.Request, payload []byte) (interface{}, error) {
425+
func apiJobReset(c context.Context, r *http.Request, payload io.Reader) (interface{}, error) {
430426
req := new(dashapi.JobResetReq)
431-
if err := json.Unmarshal(payload, req); err != nil {
427+
if err := json.NewDecoder(payload).Decode(req); err != nil {
432428
return nil, fmt.Errorf("failed to unmarshal request: %w", err)
433429
}
434430
err := resetJobs(c, req)
435431
return nil, err
436432
}
437433

438-
func apiUploadBuild(c context.Context, ns string, r *http.Request, payload []byte) (interface{}, error) {
434+
func apiUploadBuild(c context.Context, ns string, r *http.Request, payload io.Reader) (interface{}, error) {
439435
req := new(dashapi.Build)
440-
if err := json.Unmarshal(payload, req); err != nil {
436+
if err := json.NewDecoder(payload).Decode(req); err != nil {
441437
return nil, fmt.Errorf("failed to unmarshal request: %w", err)
442438
}
443439
now := timeNow(c)
@@ -706,9 +702,9 @@ func managerList(c context.Context, ns string) ([]string, error) {
706702
return managers, nil
707703
}
708704

709-
func apiReportBuildError(c context.Context, ns string, r *http.Request, payload []byte) (interface{}, error) {
705+
func apiReportBuildError(c context.Context, ns string, r *http.Request, payload io.Reader) (interface{}, error) {
710706
req := new(dashapi.BuildErrorReq)
711-
if err := json.Unmarshal(payload, req); err != nil {
707+
if err := json.NewDecoder(payload).Decode(req); err != nil {
712708
return nil, fmt.Errorf("failed to unmarshal request: %w", err)
713709
}
714710
now := timeNow(c)
@@ -740,13 +736,13 @@ const (
740736
suppressedReportTitle = "suppressed report"
741737
)
742738

743-
func apiReportCrash(c context.Context, ns string, r *http.Request, payload []byte) (interface{}, error) {
739+
func apiReportCrash(c context.Context, ns string, r *http.Request, payload io.Reader) (interface{}, error) {
744740
if stop, err := emergentlyStopped(c); err != nil || stop {
745741
// The bot's operation was aborted. Don't accept new crash reports.
746742
return &dashapi.ReportCrashResp{}, err
747743
}
748744
req := new(dashapi.Crash)
749-
if err := json.Unmarshal(payload, req); err != nil {
745+
if err := json.NewDecoder(payload).Decode(req); err != nil {
750746
return nil, fmt.Errorf("failed to unmarshal request: %w", err)
751747
}
752748
build, err := loadBuild(c, ns, req.BuildID)
@@ -1050,9 +1046,9 @@ func purgeOldCrashes(c context.Context, bug *Bug, bugKey *db.Key) {
10501046
log.Infof(c, "deleted %v crashes for bug %q", deleted, bug.Title)
10511047
}
10521048

1053-
func apiReportFailedRepro(c context.Context, ns string, r *http.Request, payload []byte) (interface{}, error) {
1049+
func apiReportFailedRepro(c context.Context, ns string, r *http.Request, payload io.Reader) (interface{}, error) {
10541050
req := new(dashapi.CrashID)
1055-
if err := json.Unmarshal(payload, req); err != nil {
1051+
if err := json.NewDecoder(payload).Decode(req); err != nil {
10561052
return nil, fmt.Errorf("failed to unmarshal request: %w", err)
10571053
}
10581054
req.Title = canonicalizeCrashTitle(req.Title, req.Corrupted, req.Suppressed)
@@ -1122,9 +1118,9 @@ func saveReproAttempt(c context.Context, bug *Bug, build *Build, log []byte) err
11221118
return nil
11231119
}
11241120

1125-
func apiNeedRepro(c context.Context, ns string, r *http.Request, payload []byte) (interface{}, error) {
1121+
func apiNeedRepro(c context.Context, ns string, r *http.Request, payload io.Reader) (interface{}, error) {
11261122
req := new(dashapi.CrashID)
1127-
if err := json.Unmarshal(payload, req); err != nil {
1123+
if err := json.NewDecoder(payload).Decode(req); err != nil {
11281124
return nil, fmt.Errorf("failed to unmarshal request: %w", err)
11291125
}
11301126
if req.Corrupted {
@@ -1174,9 +1170,9 @@ func normalizeCrashTitle(title string) string {
11741170
return strings.TrimSpace(limitLength(title, maxTextLen))
11751171
}
11761172

1177-
func apiManagerStats(c context.Context, ns string, r *http.Request, payload []byte) (interface{}, error) {
1173+
func apiManagerStats(c context.Context, ns string, r *http.Request, payload io.Reader) (interface{}, error) {
11781174
req := new(dashapi.ManagerStatsReq)
1179-
if err := json.Unmarshal(payload, req); err != nil {
1175+
if err := json.NewDecoder(payload).Decode(req); err != nil {
11801176
return nil, fmt.Errorf("failed to unmarshal request: %w", err)
11811177
}
11821178
now := timeNow(c)
@@ -1211,7 +1207,7 @@ func apiManagerStats(c context.Context, ns string, r *http.Request, payload []by
12111207
return nil, err
12121208
}
12131209

1214-
func apiBugList(c context.Context, ns string, r *http.Request, payload []byte) (interface{}, error) {
1210+
func apiBugList(c context.Context, ns string, r *http.Request, payload io.Reader) (interface{}, error) {
12151211
keys, err := db.NewQuery("Bug").
12161212
Filter("Namespace=", ns).
12171213
KeysOnly().
@@ -1226,9 +1222,9 @@ func apiBugList(c context.Context, ns string, r *http.Request, payload []byte) (
12261222
return resp, nil
12271223
}
12281224

1229-
func apiUpdateReport(c context.Context, ns string, r *http.Request, payload []byte) (interface{}, error) {
1225+
func apiUpdateReport(c context.Context, ns string, r *http.Request, payload io.Reader) (interface{}, error) {
12301226
req := new(dashapi.UpdateReportReq)
1231-
if err := json.Unmarshal(payload, req); err != nil {
1227+
if err := json.NewDecoder(payload).Decode(req); err != nil {
12321228
return nil, fmt.Errorf("failed to unmarshal request: %w", err)
12331229
}
12341230
bug := new(Bug)
@@ -1256,9 +1252,9 @@ func apiUpdateReport(c context.Context, ns string, r *http.Request, payload []by
12561252
return nil, runInTransaction(c, tx, nil)
12571253
}
12581254

1259-
func apiLoadBug(c context.Context, ns string, r *http.Request, payload []byte) (interface{}, error) {
1255+
func apiLoadBug(c context.Context, ns string, r *http.Request, payload io.Reader) (interface{}, error) {
12601256
req := new(dashapi.LoadBugReq)
1261-
if err := json.Unmarshal(payload, req); err != nil {
1257+
if err := json.NewDecoder(payload).Decode(req); err != nil {
12621258
return nil, fmt.Errorf("failed to unmarshal request: %w", err)
12631259
}
12641260
bug := new(Bug)
@@ -1272,9 +1268,9 @@ func apiLoadBug(c context.Context, ns string, r *http.Request, payload []byte) (
12721268
return loadBugReport(c, bug)
12731269
}
12741270

1275-
func apiLoadFullBug(c context.Context, r *http.Request, payload []byte) (interface{}, error) {
1271+
func apiLoadFullBug(c context.Context, r *http.Request, payload io.Reader) (interface{}, error) {
12761272
req := new(dashapi.LoadFullBugReq)
1277-
if err := json.Unmarshal(payload, req); err != nil {
1273+
if err := json.NewDecoder(payload).Decode(req); err != nil {
12781274
return nil, fmt.Errorf("failed to unmarshal request: %w", err)
12791275
}
12801276
bug, bugKey, err := findBugByReportingID(c, req.BugID)
@@ -1302,9 +1298,9 @@ func loadBugReport(c context.Context, bug *Bug) (*dashapi.BugReport, error) {
13021298
return createBugReport(c, bug, crash, crashKey, bugReporting, reporting)
13031299
}
13041300

1305-
func apiAddBuildAssets(c context.Context, ns string, r *http.Request, payload []byte) (interface{}, error) {
1301+
func apiAddBuildAssets(c context.Context, ns string, r *http.Request, payload io.Reader) (interface{}, error) {
13061302
req := new(dashapi.AddBuildAssetsReq)
1307-
if err := json.Unmarshal(payload, req); err != nil {
1303+
if err := json.NewDecoder(payload).Decode(req); err != nil {
13081304
return nil, fmt.Errorf("failed to unmarshal request: %w", err)
13091305
}
13101306
assets := []Asset{}
@@ -1347,7 +1343,7 @@ func parseIncomingAsset(c context.Context, newAsset dashapi.NewAsset, ns string)
13471343
}, nil
13481344
}
13491345

1350-
func apiNeededAssetsList(c context.Context, r *http.Request, payload []byte) (interface{}, error) {
1346+
func apiNeededAssetsList(c context.Context, r *http.Request, payload io.Reader) (interface{}, error) {
13511347
return queryNeededAssets(c)
13521348
}
13531349

@@ -1728,9 +1724,9 @@ func handleRefreshSubsystems(w http.ResponseWriter, r *http.Request) {
17281724
}
17291725
}
17301726

1731-
func apiSaveDiscussion(c context.Context, r *http.Request, payload []byte) (interface{}, error) {
1727+
func apiSaveDiscussion(c context.Context, r *http.Request, payload io.Reader) (interface{}, error) {
17321728
req := new(dashapi.SaveDiscussionReq)
1733-
if err := json.Unmarshal(payload, req); err != nil {
1729+
if err := json.NewDecoder(payload).Decode(req); err != nil {
17341730
return nil, fmt.Errorf("failed to unmarshal request: %w", err)
17351731
}
17361732
d := req.Discussion
@@ -1771,9 +1767,9 @@ func recordEmergencyStop(c context.Context) error {
17711767
// Share crash logs for non-reproduced bugs with syz-managers.
17721768
// In future, this can also take care of repro exchange between instances
17731769
// in the place of syz-hub.
1774-
func apiLogToReproduce(c context.Context, ns string, r *http.Request, payload []byte) (interface{}, error) {
1770+
func apiLogToReproduce(c context.Context, ns string, r *http.Request, payload io.Reader) (interface{}, error) {
17751771
req := new(dashapi.LogToReproReq)
1776-
if err := json.Unmarshal(payload, req); err != nil {
1772+
if err := json.NewDecoder(payload).Decode(req); err != nil {
17771773
return nil, fmt.Errorf("failed to unmarshal request: %w", err)
17781774
}
17791775
build, err := loadBuild(c, ns, req.BuildID)
@@ -1895,9 +1891,9 @@ func takeReproTask(c context.Context, ns, manager string) ([]byte, error) {
18951891
return log, err
18961892
}
18971893

1898-
func apiSaveCoverage(c context.Context, r *http.Request, payload []byte) (interface{}, error) {
1894+
func apiSaveCoverage(c context.Context, r *http.Request, payload io.Reader) (interface{}, error) {
18991895
req := new(dashapi.SaveCoverageReq)
1900-
if err := json.Unmarshal(payload, req); err != nil {
1896+
if err := json.NewDecoder(payload).Decode(req); err != nil {
19011897
return nil, fmt.Errorf("failed to unmarshal request: %w", err)
19021898
}
19031899
coverage := req.Coverage

‎dashboard/app/reporting_external.go‎

Lines changed: 11 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import (
88
"encoding/json"
99
"errors"
1010
"fmt"
11+
"io"
1112
"net/http"
1213

1314
"github.com/google/syzkaller/dashboard/dashapi"
@@ -18,12 +19,12 @@ import (
1819
// The external system is meant to poll for new bugs with apiReportingPoll,
1920
// and report back bug status updates with apiReportingUpdate.
2021

21-
func apiReportingPollBugs(c context.Context, r *http.Request, payload []byte) (interface{}, error) {
22+
func apiReportingPollBugs(c context.Context, r *http.Request, payload io.Reader) (interface{}, error) {
2223
if stop, err := emergentlyStopped(c); err != nil || stop {
2324
return &dashapi.PollBugsResponse{}, err
2425
}
2526
req := new(dashapi.PollBugsRequest)
26-
if err := json.Unmarshal(payload, req); err != nil {
27+
if err := json.NewDecoder(payload).Decode(req); err != nil {
2728
return nil, fmt.Errorf("failed to unmarshal request: %w", err)
2829
}
2930
reports := reportingPollBugs(c, req.Type)
@@ -38,12 +39,12 @@ func apiReportingPollBugs(c context.Context, r *http.Request, payload []byte) (i
3839
return resp, nil
3940
}
4041

41-
func apiReportingPollNotifications(c context.Context, r *http.Request, payload []byte) (interface{}, error) {
42+
func apiReportingPollNotifications(c context.Context, r *http.Request, payload io.Reader) (interface{}, error) {
4243
if stop, err := emergentlyStopped(c); err != nil || stop {
4344
return &dashapi.PollNotificationsResponse{}, err
4445
}
4546
req := new(dashapi.PollNotificationsRequest)
46-
if err := json.Unmarshal(payload, req); err != nil {
47+
if err := json.NewDecoder(payload).Decode(req); err != nil {
4748
return nil, fmt.Errorf("failed to unmarshal request: %w", err)
4849
}
4950
notifs := reportingPollNotifications(c, req.Type)
@@ -53,12 +54,12 @@ func apiReportingPollNotifications(c context.Context, r *http.Request, payload [
5354
return resp, nil
5455
}
5556

56-
func apiReportingPollClosed(c context.Context, r *http.Request, payload []byte) (interface{}, error) {
57+
func apiReportingPollClosed(c context.Context, r *http.Request, payload io.Reader) (interface{}, error) {
5758
if stop, err := emergentlyStopped(c); err != nil || stop {
5859
return &dashapi.PollClosedResponse{}, err
5960
}
6061
req := new(dashapi.PollClosedRequest)
61-
if err := json.Unmarshal(payload, req); err != nil {
62+
if err := json.NewDecoder(payload).Decode(req); err != nil {
6263
return nil, fmt.Errorf("failed to unmarshal request: %w", err)
6364
}
6465
ids, err := reportingPollClosed(c, req.IDs)
@@ -71,9 +72,9 @@ func apiReportingPollClosed(c context.Context, r *http.Request, payload []byte)
7172
return resp, nil
7273
}
7374

74-
func apiReportingUpdate(c context.Context, r *http.Request, payload []byte) (interface{}, error) {
75+
func apiReportingUpdate(c context.Context, r *http.Request, payload io.Reader) (interface{}, error) {
7576
req := new(dashapi.BugUpdate)
76-
if err := json.Unmarshal(payload, req); err != nil {
77+
if err := json.NewDecoder(payload).Decode(req); err != nil {
7778
return nil, fmt.Errorf("failed to unmarshal request: %w", err)
7879
}
7980
if req.JobID != "" {
@@ -96,9 +97,9 @@ func apiReportingUpdate(c context.Context, r *http.Request, payload []byte) (int
9697
}, nil
9798
}
9899

99-
func apiNewTestJob(c context.Context, r *http.Request, payload []byte) (interface{}, error) {
100+
func apiNewTestJob(c context.Context, r *http.Request, payload io.Reader) (interface{}, error) {
100101
req := new(dashapi.TestPatchRequest)
101-
if err := json.Unmarshal(payload, req); err != nil {
102+
if err := json.NewDecoder(payload).Decode(req); err != nil {
102103
return nil, fmt.Errorf("failed to unmarshal request: %w", err)
103104
}
104105
resp := &dashapi.TestPatchReply{}

0 commit comments

Comments
 (0)