Skip to content

Commit 580d767

Browse files
weichou1229weichou1229
authored
feat: Query events without total count (#5135)
Query events without total count to prevent large dataset query and count. Signed-off-by: bruce <[email protected]>
1 parent 3597691 commit 580d767

File tree

6 files changed

+60
-46
lines changed

6 files changed

+60
-46
lines changed

‎internal/core/data/application/event.go‎

Lines changed: 34 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -190,14 +190,6 @@ func (a *CoreDataApp) DeleteEventsByDeviceName(deviceName string, dic *di.Contai
190190
// AllEvents query events by offset and limit
191191
func (a *CoreDataApp) AllEvents(offset int, limit int, dic *di.Container) (events []dtos.Event, totalCount uint32, err errors.EdgeX) {
192192
dbClient := container.DBClientFrom(dic.Get)
193-
totalCount, err = dbClient.EventTotalCount()
194-
if err != nil {
195-
return events, totalCount, errors.NewCommonEdgeXWrapper(err)
196-
}
197-
cont, err := utils.CheckCountRange(totalCount, offset, limit)
198-
if !cont {
199-
return []dtos.Event{}, totalCount, err
200-
}
201193

202194
eventModels, err := dbClient.AllEvents(offset, limit)
203195
if err != nil {
@@ -207,6 +199,18 @@ func (a *CoreDataApp) AllEvents(offset int, limit int, dic *di.Container) (event
207199
for i, e := range eventModels {
208200
events[i] = dtos.FromEventModelToDTO(e)
209201
}
202+
if offset < 0 {
203+
return events, 0, err // skip total count
204+
}
205+
206+
totalCount, err = dbClient.EventTotalCount()
207+
if err != nil {
208+
return events, totalCount, errors.NewCommonEdgeXWrapper(err)
209+
}
210+
cont, err := utils.CheckCountRange(totalCount, offset, limit)
211+
if !cont {
212+
return []dtos.Event{}, totalCount, err
213+
}
210214
return events, totalCount, nil
211215
}
212216

@@ -216,14 +220,6 @@ func (a *CoreDataApp) EventsByDeviceName(offset int, limit int, name string, dic
216220
return events, totalCount, errors.NewCommonEdgeX(errors.KindContractInvalid, "name is empty", nil)
217221
}
218222
dbClient := container.DBClientFrom(dic.Get)
219-
totalCount, err = dbClient.EventCountByDeviceName(name)
220-
if err != nil {
221-
return events, totalCount, errors.NewCommonEdgeXWrapper(err)
222-
}
223-
cont, err := utils.CheckCountRange(totalCount, offset, limit)
224-
if !cont {
225-
return []dtos.Event{}, totalCount, err
226-
}
227223

228224
eventModels, err := dbClient.EventsByDeviceName(offset, limit, name)
229225
if err != nil {
@@ -233,20 +229,24 @@ func (a *CoreDataApp) EventsByDeviceName(offset int, limit int, name string, dic
233229
for i, e := range eventModels {
234230
events[i] = dtos.FromEventModelToDTO(e)
235231
}
236-
return events, totalCount, nil
237-
}
232+
if offset < 0 {
233+
return events, 0, err // skip total count
234+
}
238235

239-
// EventsByTimeRange query events with offset, limit and time range
240-
func (a *CoreDataApp) EventsByTimeRange(startTime int64, endTime int64, offset int, limit int, dic *di.Container) (events []dtos.Event, totalCount uint32, err errors.EdgeX) {
241-
dbClient := container.DBClientFrom(dic.Get)
242-
totalCount, err = dbClient.EventCountByTimeRange(startTime, endTime)
236+
totalCount, err = dbClient.EventCountByDeviceName(name)
243237
if err != nil {
244238
return events, totalCount, errors.NewCommonEdgeXWrapper(err)
245239
}
246240
cont, err := utils.CheckCountRange(totalCount, offset, limit)
247241
if !cont {
248242
return []dtos.Event{}, totalCount, err
249243
}
244+
return events, totalCount, nil
245+
}
246+
247+
// EventsByTimeRange query events with offset, limit and time range
248+
func (a *CoreDataApp) EventsByTimeRange(startTime int64, endTime int64, offset int, limit int, dic *di.Container) (events []dtos.Event, totalCount uint32, err errors.EdgeX) {
249+
dbClient := container.DBClientFrom(dic.Get)
250250

251251
eventModels, err := dbClient.EventsByTimeRange(startTime, endTime, offset, limit)
252252
if err != nil {
@@ -256,6 +256,18 @@ func (a *CoreDataApp) EventsByTimeRange(startTime int64, endTime int64, offset i
256256
for i, e := range eventModels {
257257
events[i] = dtos.FromEventModelToDTO(e)
258258
}
259+
if offset < 0 {
260+
return events, 0, err // skip total count
261+
}
262+
263+
totalCount, err = dbClient.EventCountByTimeRange(startTime, endTime)
264+
if err != nil {
265+
return events, totalCount, errors.NewCommonEdgeXWrapper(err)
266+
}
267+
cont, err := utils.CheckCountRange(totalCount, offset, limit)
268+
if !cont {
269+
return []dtos.Event{}, totalCount, err
270+
}
259271
return events, totalCount, nil
260272
}
261273

‎internal/core/data/controller/http/const.go‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
// Copyright (C) 2025 IOTech Ltd
2+
3+
package http
4+
5+
const (
6+
minOffset = -1 // allow using -1 to query reading data and skip the total count for pagination
7+
)

‎internal/core/data/controller/http/event.go‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -218,7 +218,7 @@ func (ec *EventController) AllEvents(c echo.Context) error {
218218
config := dataContainer.ConfigurationFrom(ec.dic.Get)
219219

220220
// parse URL query string for offset, limit
221-
offset, limit, _, err := utils.ParseGetAllObjectsRequestQueryString(c, 0, math.MaxInt32, -1, config.Service.MaxResultCount)
221+
offset, limit, _, err := utils.ParseGetAllObjectsRequestQueryString(c, minOffset, math.MaxInt32, -1, config.Service.MaxResultCount)
222222
if err != nil {
223223
return utils.WriteErrorResponse(w, ctx, lc, err, "")
224224
}
@@ -241,7 +241,7 @@ func (ec *EventController) EventsByDeviceName(c echo.Context) error {
241241
name := c.Param(common.Name)
242242

243243
// parse URL query string for offset, limit
244-
offset, limit, _, err := utils.ParseGetAllObjectsRequestQueryString(c, 0, math.MaxInt32, -1, config.Service.MaxResultCount)
244+
offset, limit, _, err := utils.ParseGetAllObjectsRequestQueryString(c, minOffset, math.MaxInt32, -1, config.Service.MaxResultCount)
245245
if err != nil {
246246
return utils.WriteErrorResponse(w, ctx, lc, err, "")
247247
}
@@ -283,7 +283,7 @@ func (ec *EventController) EventsByTimeRange(c echo.Context) error {
283283
config := dataContainer.ConfigurationFrom(ec.dic.Get)
284284

285285
// parse time range (start, end), offset, and limit from incoming request
286-
start, end, offset, limit, err := utils.ParseTimeRangeOffsetLimit(c, 0, math.MaxInt32, -1, config.Service.MaxResultCount)
286+
start, end, offset, limit, err := utils.ParseTimeRangeOffsetLimit(c, minOffset, math.MaxInt32, -1, config.Service.MaxResultCount)
287287
if err != nil {
288288
return utils.WriteErrorResponse(w, ctx, lc, err, "")
289289
}

‎internal/core/data/controller/http/event_test.go‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -619,6 +619,7 @@ func TestAllEvents(t *testing.T) {
619619
dbClientMock.On("EventTotalCount").Return(totalCount, nil)
620620
dbClientMock.On("AllEvents", 0, 20).Return(events, nil)
621621
dbClientMock.On("AllEvents", 1, 1).Return([]models.Event{events[1]}, nil)
622+
dbClientMock.On("AllEvents", -1, 1).Return([]models.Event{events[1]}, nil)
622623
dbClientMock.On("AllEvents", 4, 1).Return([]models.Event{}, errors.NewCommonEdgeX(errors.KindRangeNotSatisfiable, "query objects bounds out of range.", nil))
623624
app := application.NewCoreDataApp(dic)
624625
dic.Update(di.ServiceConstructorMap{
@@ -643,6 +644,7 @@ func TestAllEvents(t *testing.T) {
643644
}{
644645
{"Valid - get events without offset and limit", "", "", false, 3, totalCount, http.StatusOK},
645646
{"Valid - get events with offset and limit", "1", "1", false, 1, totalCount, http.StatusOK},
647+
{"Valid - set offset -1 to skip total count", "-1", "1", false, 1, 0, http.StatusOK},
646648
{"Invalid - offset out of range", "4", "1", true, 0, 0, http.StatusRequestedRangeNotSatisfiable},
647649
}
648650
for _, testCase := range tests {
@@ -710,6 +712,7 @@ func TestAllEventsByDeviceName(t *testing.T) {
710712
dbClientMock.On("EventsByDeviceName", 0, 5, testDeviceA).Return([]models.Event{events[0], events[1]}, nil)
711713
dbClientMock.On("EventsByDeviceName", 0, 5, testDeviceB).Return([]models.Event{events[2]}, nil)
712714
dbClientMock.On("EventsByDeviceName", 1, 1, testDeviceA).Return([]models.Event{events[1]}, nil)
715+
dbClientMock.On("EventsByDeviceName", -1, 1, testDeviceA).Return([]models.Event{events[1]}, nil)
713716
dbClientMock.On("EventsByDeviceName", 4, 1, testDeviceB).Return([]models.Event{}, errors.NewCommonEdgeX(errors.KindRangeNotSatisfiable, "query objects bounds out of range.", nil))
714717
app := application.NewCoreDataApp(dic)
715718
dic.Update(di.ServiceConstructorMap{
@@ -736,6 +739,7 @@ func TestAllEventsByDeviceName(t *testing.T) {
736739
{"Valid - get events with deviceName - deviceA", "0", "5", testDeviceA, false, 2, totalCountDeviceA, http.StatusOK},
737740
{"Valid - get events with deviceName - deviceB", "0", "5", testDeviceB, false, 1, totalCountDeviceB, http.StatusOK},
738741
{"Valid - get events with offset and no labels", "1", "1", testDeviceA, false, 1, totalCountDeviceA, http.StatusOK},
742+
{"Valid - set offset -1 to skip total count", "-1", "1", testDeviceA, false, 1, 0, http.StatusOK},
739743
{"Invalid - offset out of range", "4", "1", testDeviceB, true, 0, 0, http.StatusRequestedRangeNotSatisfiable},
740744
{"Invalid - get events without deviceName", "0", "10", "", true, 0, 0, http.StatusBadRequest},
741745
}
@@ -787,6 +791,7 @@ func TestAllEventsByTimeRange(t *testing.T) {
787791
dbClientMock := &dbMock.DBClient{}
788792
dbClientMock.On("EventCountByTimeRange", int64(0), int64(100)).Return(totalCount, nil)
789793
dbClientMock.On("EventsByTimeRange", int64(0), int64(100), 0, 10).Return([]models.Event{}, nil)
794+
dbClientMock.On("EventsByTimeRange", int64(0), int64(100), -1, 10).Return([]models.Event{}, nil)
790795
app := application.NewCoreDataApp(dic)
791796
dic.Update(di.ServiceConstructorMap{
792797
container.DBClientInterfaceName: func(get di.Get) interface{} {
@@ -811,6 +816,7 @@ func TestAllEventsByTimeRange(t *testing.T) {
811816
expectedStatusCode int
812817
}{
813818
{"Valid - with proper start/end/offset/limit", "0", "100", "0", "10", false, 0, totalCount, http.StatusOK},
819+
{"Valid - set offset -1 to skip total count", "0", "100", "-1", "10", false, 0, 0, http.StatusOK},
814820
{"Invalid - invalid start format", "aaa", "100", "0", "10", true, 0, totalCount, http.StatusBadRequest},
815821
{"Invalid - invalid end format", "0", "bbb", "0", "10", true, 0, totalCount, http.StatusBadRequest},
816822
{"Invalid - empty start", "", "100", "0", "10", true, 0, totalCount, http.StatusBadRequest},

‎internal/core/data/controller/http/reading.go‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
//
2-
// Copyright (C) 2021-2023 IOTech Ltd
2+
// Copyright (C) 2021-2025 IOTech Ltd
33
//
44
// SPDX-License-Identifier: Apache-2.0
55

@@ -25,8 +25,6 @@ import (
2525
"github.com/labstack/echo/v4"
2626
)
2727

28-
const minOffset = -1 // allow using -1 to query reading data and skip the total count for pagination
29-
3028
type ReadingController struct {
3129
reader io.DtoReader
3230
dic *di.Container

‎openapi/core-data.yaml‎

Lines changed: 9 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -343,23 +343,14 @@ components:
343343
- value
344344
parameters:
345345
offsetParam:
346-
in: query
347-
name: offset
348-
required: false
349-
schema:
350-
type: integer
351-
minimum: 0
352-
default: 0
353-
description: "The number of items to skip before starting to collect the result set."
354-
readingOffsetParam:
355346
in: query
356347
name: offset
357348
required: false
358349
schema:
359350
type: integer
360351
minimum: -1
361352
default: 0
362-
description: "The number of items to skip before starting to collect the result set. Specify -1 will not count the total reading to improve the performance for the large dataset."
353+
description: "The number of items to skip before starting to collect the result set. Setting this value to -1 will skip counting the total number of events or readings, which can improve performance when working with large datasets."
363354
limitParam:
364355
in: query
365356
name: limit
@@ -1147,7 +1138,7 @@ paths:
11471138
/reading/all:
11481139
parameters:
11491140
- $ref: '#/components/parameters/correlatedRequestHeader'
1150-
- $ref: '#/components/parameters/readingOffsetParam'
1141+
- $ref: '#/components/parameters/offsetParam'
11511142
- $ref: '#/components/parameters/limitParam'
11521143
get:
11531144
summary: "Given the entire range of readings sorted by origin descending, returns a portion of that range according to the offset and limit parameters. Readings returned will all inherit from BaseReading but their concrete types will be either SimpleReading or BinaryReading, potentially interleaved."
@@ -1289,7 +1280,7 @@ paths:
12891280
schema:
12901281
type: string
12911282
description: "Uniquely identifies a given device"
1292-
- $ref: '#/components/parameters/readingOffsetParam'
1283+
- $ref: '#/components/parameters/offsetParam'
12931284
- $ref: '#/components/parameters/limitParam'
12941285
get:
12951286
summary: "Given a range of readings from the specified device sorted by origin descending, returns a portion of that range according to the device name, offset and limit parameters."
@@ -1353,7 +1344,7 @@ paths:
13531344
schema:
13541345
type: string
13551346
description: The device resource name of readings.
1356-
- $ref: '#/components/parameters/readingOffsetParam'
1347+
- $ref: '#/components/parameters/offsetParam'
13571348
- $ref: '#/components/parameters/limitParam'
13581349
get:
13591350
summary: Returns a paginated list of readings whose resource name is of the specified one.
@@ -1425,7 +1416,7 @@ paths:
14251416
schema:
14261417
type: string
14271418
description: The device resource name of readings.
1428-
- $ref: '#/components/parameters/readingOffsetParam'
1419+
- $ref: '#/components/parameters/offsetParam'
14291420
- $ref: '#/components/parameters/limitParam'
14301421
get:
14311422
summary: "Returns a paginated range of readings by deviceName and resourceName"
@@ -1497,7 +1488,7 @@ paths:
14971488
schema:
14981489
type: integer
14991490
description: "Unix timestamp (nanoseconds) indicating the end of a date/time range"
1500-
- $ref: '#/components/parameters/readingOffsetParam'
1491+
- $ref: '#/components/parameters/offsetParam'
15011492
- $ref: '#/components/parameters/limitParam'
15021493
get:
15031494
summary: "Return a paginated range of readings with a create date inside the specified start/end values."
@@ -1575,7 +1566,7 @@ paths:
15751566
schema:
15761567
type: integer
15771568
description: "Unix timestamp (nanoseconds) indicating the end of a date/time range"
1578-
- $ref: '#/components/parameters/readingOffsetParam'
1569+
- $ref: '#/components/parameters/offsetParam'
15791570
- $ref: '#/components/parameters/limitParam'
15801571
get:
15811572
summary: "Return a paginated range of readings by resourceName and specified time range."
@@ -1659,7 +1650,7 @@ paths:
16591650
schema:
16601651
type: integer
16611652
description: "Unix timestamp (nanoseconds) indicating the end of a date/time range"
1662-
- $ref: '#/components/parameters/readingOffsetParam'
1653+
- $ref: '#/components/parameters/offsetParam'
16631654
- $ref: '#/components/parameters/limitParam'
16641655
get:
16651656
summary: "Return a paginated range of readings by deviceName, resourceName and specified time range."
@@ -1737,7 +1728,7 @@ paths:
17371728
schema:
17381729
type: integer
17391730
description: "Unix timestamp (nanoseconds) indicating the end of a date/time range"
1740-
- $ref: '#/components/parameters/readingOffsetParam'
1731+
- $ref: '#/components/parameters/offsetParam'
17411732
- $ref: '#/components/parameters/limitParam'
17421733
get:
17431734
summary: "Return a paginated range of readings by deviceName and specified time range while also allowing multiple resource names specified in the request body as query criteria. If resource names or request body is empty, return all the readings that meet deviceName and specified time range."

0 commit comments

Comments
 (0)