Skip to content

Commit cbde8d7

Browse files
darinkelkhoffdarinkelkhoff
committed
Merged feature/pagination-in-unique-key-helper into dev
2 parents 3e69003 + 11ff517 commit cbde8d7

File tree

2 files changed

+195
-40
lines changed

2 files changed

+195
-40
lines changed

‎qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/helpers/UniqueKeyHelper.java‎

Lines changed: 72 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,7 @@
5050
*******************************************************************************/
5151
public class UniqueKeyHelper
5252
{
53+
private static Integer pageSize = 1000;
5354

5455
/*******************************************************************************
5556
**
@@ -60,62 +61,71 @@ public static Map<List<Serializable>, Serializable> getExistingKeys(QBackendTran
6061
Map<List<Serializable>, Serializable> existingRecords = new HashMap<>();
6162
if(ukFieldNames != null)
6263
{
63-
QueryInput queryInput = new QueryInput();
64-
queryInput.setTableName(table.getName());
65-
queryInput.setTransaction(transaction);
66-
67-
QQueryFilter filter = new QQueryFilter();
68-
if(ukFieldNames.size() == 1)
69-
{
70-
List<Serializable> values = recordList.stream()
71-
.filter(r -> CollectionUtils.nullSafeIsEmpty(r.getErrors()))
72-
.map(r -> r.getValue(ukFieldNames.get(0)))
73-
.collect(Collectors.toList());
74-
filter.addCriteria(new QFilterCriteria(ukFieldNames.get(0), QCriteriaOperator.IN, values));
75-
}
76-
else
64+
for(List<QRecord> page : CollectionUtils.getPages(recordList, pageSize))
7765
{
78-
filter.setBooleanOperator(QQueryFilter.BooleanOperator.OR);
79-
for(QRecord record : recordList)
66+
QueryInput queryInput = new QueryInput();
67+
queryInput.setTableName(table.getName());
68+
queryInput.setTransaction(transaction);
69+
70+
QQueryFilter filter = new QQueryFilter();
71+
if(ukFieldNames.size() == 1)
8072
{
81-
if(CollectionUtils.nullSafeHasContents(record.getErrors()))
73+
List<Serializable> values = page.stream()
74+
.filter(r -> CollectionUtils.nullSafeIsEmpty(r.getErrors()))
75+
.map(r -> r.getValue(ukFieldNames.get(0)))
76+
.collect(Collectors.toList());
77+
78+
if(values.isEmpty())
8279
{
8380
continue;
8481
}
8582

86-
QQueryFilter subFilter = new QQueryFilter();
87-
filter.addSubFilter(subFilter);
88-
for(String fieldName : ukFieldNames)
83+
filter.addCriteria(new QFilterCriteria(ukFieldNames.get(0), QCriteriaOperator.IN, values));
84+
}
85+
else
86+
{
87+
filter.setBooleanOperator(QQueryFilter.BooleanOperator.OR);
88+
for(QRecord record : page)
8989
{
90-
Serializable value = record.getValue(fieldName);
91-
if(value == null)
90+
if(CollectionUtils.nullSafeHasContents(record.getErrors()))
9291
{
93-
subFilter.addCriteria(new QFilterCriteria(fieldName, QCriteriaOperator.IS_BLANK));
92+
continue;
9493
}
95-
else
94+
95+
QQueryFilter subFilter = new QQueryFilter();
96+
filter.addSubFilter(subFilter);
97+
for(String fieldName : ukFieldNames)
9698
{
97-
subFilter.addCriteria(new QFilterCriteria(fieldName, QCriteriaOperator.EQUALS, value));
99+
Serializable value = record.getValue(fieldName);
100+
if(value == null)
101+
{
102+
subFilter.addCriteria(new QFilterCriteria(fieldName, QCriteriaOperator.IS_BLANK));
103+
}
104+
else
105+
{
106+
subFilter.addCriteria(new QFilterCriteria(fieldName, QCriteriaOperator.EQUALS, value));
107+
}
98108
}
99109
}
100-
}
101110

102-
if(CollectionUtils.nullSafeIsEmpty(filter.getSubFilters()))
103-
{
104-
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
105-
// if we didn't build any sub-filters (because all records have errors in them), don't run a query w/ no clauses - rather - return early. //
106-
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
107-
return (existingRecords);
111+
if(CollectionUtils.nullSafeIsEmpty(filter.getSubFilters()))
112+
{
113+
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
114+
// if we didn't build any sub-filters (because all records have errors in them), don't run a query w/ no clauses - continue to next page //
115+
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
116+
continue;
117+
}
108118
}
109-
}
110119

111-
queryInput.setFilter(filter);
112-
QueryOutput queryOutput = new QueryAction().execute(queryInput);
113-
for(QRecord record : queryOutput.getRecords())
114-
{
115-
Optional<List<Serializable>> keyValues = getKeyValues(table, uniqueKey, record, allowNullKeyValuesToEqual);
116-
if(keyValues.isPresent())
120+
queryInput.setFilter(filter);
121+
QueryOutput queryOutput = new QueryAction().execute(queryInput);
122+
for(QRecord record : queryOutput.getRecords())
117123
{
118-
existingRecords.put(keyValues.get(), record.getValue(table.getPrimaryKeyField()));
124+
Optional<List<Serializable>> keyValues = getKeyValues(table, uniqueKey, record, allowNullKeyValuesToEqual);
125+
if(keyValues.isPresent())
126+
{
127+
existingRecords.put(keyValues.get(), record.getValue(table.getPrimaryKeyField()));
128+
}
119129
}
120130
}
121131
}
@@ -200,4 +210,26 @@ public boolean equals(Object obj)
200210
}
201211
}
202212

213+
214+
215+
/*******************************************************************************
216+
** Getter for pageSize
217+
**
218+
*******************************************************************************/
219+
public static Integer getPageSize()
220+
{
221+
return pageSize;
222+
}
223+
224+
225+
226+
/*******************************************************************************
227+
** Setter for pageSize
228+
**
229+
*******************************************************************************/
230+
public static void setPageSize(Integer pageSize)
231+
{
232+
UniqueKeyHelper.pageSize = pageSize;
233+
}
234+
203235
}

‎qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/tables/helpers/UniqueKeyHelperTest.java‎

Lines changed: 123 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,123 @@
1+
/*
2+
* QQQ - Low-code Application Framework for Engineers.
3+
* Copyright (C) 2021-2024. Kingsrook, LLC
4+
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
5+
* contact@kingsrook.com
6+
* https://github.com/Kingsrook/
7+
*
8+
* This program is free software: you can redistribute it and/or modify
9+
* it under the terms of the GNU Affero General Public License as
10+
* published by the Free Software Foundation, either version 3 of the
11+
* License, or (at your option) any later version.
12+
*
13+
* This program is distributed in the hope that it will be useful,
14+
* but WITHOUT ANY WARRANTY; without even the implied warranty of
15+
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16+
* GNU Affero General Public License for more details.
17+
*
18+
* You should have received a copy of the GNU Affero General Public License
19+
* along with this program. If not, see <https://www.gnu.org/licenses/>.
20+
*/
21+
22+
package com.kingsrook.qqq.backend.core.actions.tables.helpers;
23+
24+
25+
import java.io.Serializable;
26+
import java.util.List;
27+
import java.util.Map;
28+
import com.kingsrook.qqq.backend.core.BaseTest;
29+
import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
30+
import com.kingsrook.qqq.backend.core.context.QContext;
31+
import com.kingsrook.qqq.backend.core.exceptions.QException;
32+
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
33+
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput;
34+
import com.kingsrook.qqq.backend.core.model.data.QRecord;
35+
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
36+
import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore;
37+
import com.kingsrook.qqq.backend.core.utils.TestUtils;
38+
import org.junit.jupiter.api.AfterAll;
39+
import org.junit.jupiter.api.AfterEach;
40+
import org.junit.jupiter.api.BeforeAll;
41+
import org.junit.jupiter.api.BeforeEach;
42+
import org.junit.jupiter.api.Test;
43+
import static org.junit.jupiter.api.Assertions.assertEquals;
44+
45+
46+
/*******************************************************************************
47+
** Unit test for UniqueKeyHelper
48+
*******************************************************************************/
49+
class UniqueKeyHelperTest extends BaseTest
50+
{
51+
private static Integer originalPageSize;
52+
53+
/*******************************************************************************
54+
**
55+
*******************************************************************************/
56+
@BeforeAll
57+
static void beforeAll()
58+
{
59+
originalPageSize = UniqueKeyHelper.getPageSize();
60+
UniqueKeyHelper.setPageSize(5);
61+
}
62+
63+
64+
65+
/*******************************************************************************
66+
**
67+
*******************************************************************************/
68+
@AfterAll
69+
static void afterAll()
70+
{
71+
UniqueKeyHelper.setPageSize(originalPageSize);
72+
}
73+
74+
75+
76+
/*******************************************************************************
77+
**
78+
*******************************************************************************/
79+
@BeforeEach
80+
@AfterEach
81+
void beforeAndAfterEach()
82+
{
83+
MemoryRecordStore.fullReset();
84+
}
85+
86+
87+
88+
/*******************************************************************************
89+
**
90+
*******************************************************************************/
91+
@Test
92+
void testUniqueKey() throws QException
93+
{
94+
List<QRecord> recordsWithKey1Equals1AndKey2In1Through10 = List.of(
95+
new QRecord().withValue("key1", 1).withValue("key2", 1),
96+
new QRecord().withValue("key1", 1).withValue("key2", 2),
97+
new QRecord().withValue("key1", 1).withValue("key2", 3),
98+
new QRecord().withValue("key1", 1).withValue("key2", 4),
99+
new QRecord().withValue("key1", 1).withValue("key2", 5),
100+
new QRecord().withValue("key1", 1).withValue("key2", 6),
101+
new QRecord().withValue("key1", 1).withValue("key2", 7),
102+
new QRecord().withValue("key1", 1).withValue("key2", 8),
103+
new QRecord().withValue("key1", 1).withValue("key2", 9),
104+
new QRecord().withValue("key1", 1).withValue("key2", 10)
105+
);
106+
107+
InsertInput insertInput = new InsertInput();
108+
insertInput.setTableName(TestUtils.TABLE_NAME_TWO_KEYS);
109+
insertInput.setRecords(recordsWithKey1Equals1AndKey2In1Through10);
110+
InsertOutput insertOutput = new InsertAction().execute(insertInput);
111+
112+
MemoryRecordStore.resetStatistics();
113+
MemoryRecordStore.setCollectStatistics(true);
114+
115+
QTableMetaData table = QContext.getQInstance().getTable(TestUtils.TABLE_NAME_TWO_KEYS);
116+
Map<List<Serializable>, Serializable> existingKeys = UniqueKeyHelper.getExistingKeys(null, table, recordsWithKey1Equals1AndKey2In1Through10, table.getUniqueKeys().get(0), false);
117+
assertEquals(recordsWithKey1Equals1AndKey2In1Through10.size(), existingKeys.size());
118+
119+
assertEquals(2, MemoryRecordStore.getStatistics().get(MemoryRecordStore.STAT_QUERIES_RAN));
120+
}
121+
122+
123+
}

0 commit comments

Comments
 (0)