Skip to content

Commit 21aae49

Browse files
jarad0628jarad0628
authored
Merge pull request #724 from tapdata/feature/add-offset-fetch-method-v2.0.5
add OFFSET-FETCH sql format to support DB2i advance query
2 parents 17fce56 + 7205027 commit 21aae49

File tree

3 files changed

+165
-0
lines changed

3 files changed

+165
-0
lines changed

‎connectors-common/sql-core/src/main/java/io/tapdata/common/CommonDbConnector.java‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -920,6 +920,34 @@ protected void queryByAdvanceFilterWithOffsetV2(TapConnectorContext connectorCon
920920
});
921921
}
922922

923+
//for SQL Server type (with OFFSET-FETCH)
924+
protected void queryByAdvanceFilterWithOffsetFetch(TapConnectorContext connectorContext, TapAdvanceFilter filter, TapTable table, Consumer<FilterResults> consumer) throws Throwable {
925+
String sql = commonSqlMaker.buildSelectClause(table, filter, false) + getSchemaAndTable(table.getId()) + commonSqlMaker.buildSqlByAdvanceFilterWithOffsetFetch(filter);
926+
jdbcContext.query(sql, resultSet -> {
927+
FilterResults filterResults = new FilterResults();
928+
try {
929+
while (resultSet.next()) {
930+
List<String> allColumn = DbKit.getColumnsFromResultSet(resultSet);
931+
DataMap dataMap = DbKit.getRowFromResultSet(resultSet, allColumn);
932+
processDataMap(dataMap, table);
933+
filterResults.add(dataMap);
934+
if (filterResults.getResults().size() == BATCH_ADVANCE_READ_LIMIT) {
935+
consumer.accept(filterResults);
936+
filterResults = new FilterResults();
937+
}
938+
}
939+
} catch (SQLException e) {
940+
exceptionCollector.collectTerminateByServer(e);
941+
exceptionCollector.collectReadPrivileges("batchReadWithoutOffset", Collections.emptyList(), e);
942+
exceptionCollector.revealException(e);
943+
throw e;
944+
}
945+
if (EmptyKit.isNotEmpty(filterResults.getResults())) {
946+
consumer.accept(filterResults);
947+
}
948+
});
949+
}
950+
923951
protected void beginTransaction(TapConnectorContext connectorContext) throws Throwable {
924952
isTransaction = true;
925953
}
@@ -1010,6 +1038,17 @@ protected long countByAdvanceFilterV2(TapConnectorContext connectorContext, TapT
10101038
return count.get();
10111039
}
10121040

1041+
protected long countByAdvanceFilterWithOffsetFetch(TapConnectorContext connectorContext, TapTable tapTable, TapAdvanceFilter tapAdvanceFilter) throws SQLException {
1042+
AtomicLong count = new AtomicLong(0);
1043+
String sql = "SELECT COUNT(1) FROM " + getSchemaAndTable(tapTable.getId()) + commonSqlMaker.buildSqlByAdvanceFilterWithOffsetFetch(tapAdvanceFilter);
1044+
jdbcContext.query(sql, resultSet -> {
1045+
if (resultSet.next()) {
1046+
count.set(resultSet.getLong(1));
1047+
}
1048+
});
1049+
return count.get();
1050+
}
1051+
10131052
protected List<String> getAfterUniqueAutoIncrementFields(TapTable tapTable, List<TapIndex> indexList) {
10141053
return new ArrayList<>();
10151054
}

‎connectors-common/sql-core/src/main/java/io/tapdata/common/CommonSqlMaker.java‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -317,6 +317,33 @@ public void buildRowNumberClause(StringBuilder builder, TapAdvanceFilter filter)
317317
}
318318
}
319319

320+
/**
321+
* build subSql after where for advance query using OFFSET-FETCH syntax
322+
*
323+
* @param filter condition of advance query
324+
* @return where substring
325+
*/
326+
public String buildSqlByAdvanceFilterWithOffsetFetch(TapAdvanceFilter filter) {
327+
StringBuilder builder = new StringBuilder();
328+
buildWhereClause(builder, filter);
329+
buildOrderClause(builder, filter);
330+
buildOffsetFetchClause(builder, filter);
331+
return builder.toString();
332+
}
333+
334+
public void buildOffsetFetchClause(StringBuilder builder, TapAdvanceFilter filter) {
335+
if (EmptyKit.isNotNull(filter.getSkip())) {
336+
builder.append(" OFFSET ").append(filter.getSkip()).append(" ROWS ");
337+
} else if (EmptyKit.isNotNull(filter.getLimit())) {
338+
// FETCH requires OFFSET, so add OFFSET 0 if only LIMIT is specified
339+
builder.append(" OFFSET 0 ROWS ");
340+
}
341+
342+
if (EmptyKit.isNotNull(filter.getLimit())) {
343+
builder.append(" FETCH FIRST ").append(filter.getLimit()).append(" ROWS ONLY ");
344+
}
345+
}
346+
320347
/**
321348
* set value for each column in sql
322349
* e.g.

‎connectors-common/sql-core/src/test/java/io/tapdata/common/CommonSqlMakerTest.java‎

Lines changed: 99 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,12 @@
11
package io.tapdata.common;
22

33
import io.tapdata.entity.utils.DataMap;
4+
import io.tapdata.pdk.apis.entity.SortOn;
45
import io.tapdata.pdk.apis.entity.TapAdvanceFilter;
56
import org.junit.jupiter.api.Assertions;
67
import org.junit.jupiter.api.Test;
78

9+
import java.util.Arrays;
810
import java.util.HashMap;
911
import java.util.Map;
1012

@@ -76,5 +78,102 @@ void buildCommandWhereSqlParamsWithWhereTest(){
7678
Assertions.assertEquals(whereSql,actualData);
7779
}
7880

81+
@Test
82+
void buildSqlByAdvanceFilterWithOffsetFetchTest() {
83+
CommonSqlMaker commonSqlMaker = new CommonSqlMaker();
84+
TapAdvanceFilter filter = new TapAdvanceFilter();
85+
86+
// Test with both skip and limit
87+
filter.setSkip(10);
88+
filter.setLimit(20);
89+
filter.setSortOnList(Arrays.asList(new SortOn("id", SortOn.ASCENDING)));
90+
91+
String result = commonSqlMaker.buildSqlByAdvanceFilterWithOffsetFetch(filter);
92+
String expected = " ORDER BY \"id\" ASC OFFSET 10 ROWS FETCH FIRST 20 ROWS ONLY ";
93+
Assertions.assertEquals(expected, result);
94+
}
95+
96+
@Test
97+
void buildSqlByAdvanceFilterWithOffsetFetchOnlyLimitTest() {
98+
CommonSqlMaker commonSqlMaker = new CommonSqlMaker();
99+
TapAdvanceFilter filter = new TapAdvanceFilter();
100+
101+
// Test with only limit (should add OFFSET 0)
102+
filter.setLimit(15);
103+
filter.setSortOnList(Arrays.asList(new SortOn("name", SortOn.DESCENDING)));
104+
105+
String result = commonSqlMaker.buildSqlByAdvanceFilterWithOffsetFetch(filter);
106+
String expected = " ORDER BY \"name\" DESC OFFSET 0 ROWS FETCH FIRST 15 ROWS ONLY ";
107+
Assertions.assertEquals(expected, result);
108+
}
109+
110+
@Test
111+
void buildSqlByAdvanceFilterWithOffsetFetchOnlySkipTest() {
112+
CommonSqlMaker commonSqlMaker = new CommonSqlMaker();
113+
TapAdvanceFilter filter = new TapAdvanceFilter();
114+
115+
// Test with only skip
116+
filter.setSkip(5);
117+
filter.setSortOnList(Arrays.asList(new SortOn("created_at", SortOn.ASCENDING)));
118+
119+
String result = commonSqlMaker.buildSqlByAdvanceFilterWithOffsetFetch(filter);
120+
String expected = " ORDER BY \"created_at\" ASC OFFSET 5 ROWS ";
121+
Assertions.assertEquals(expected, result);
122+
}
123+
124+
@Test
125+
void buildSqlByAdvanceFilterWithOffsetFetchNoOrderByTest() {
126+
CommonSqlMaker commonSqlMaker = new CommonSqlMaker();
127+
TapAdvanceFilter filter = new TapAdvanceFilter();
128+
129+
// Test without ORDER BY but with pagination (should not add ORDER BY)
130+
filter.setSkip(10);
131+
filter.setLimit(20);
132+
133+
String result = commonSqlMaker.buildSqlByAdvanceFilterWithOffsetFetch(filter);
134+
String expected = " OFFSET 10 ROWS FETCH FIRST 20 ROWS ONLY ";
135+
Assertions.assertEquals(expected, result);
136+
}
137+
138+
@Test
139+
void buildOffsetFetchClauseTest() {
140+
CommonSqlMaker commonSqlMaker = new CommonSqlMaker();
141+
StringBuilder builder = new StringBuilder();
142+
TapAdvanceFilter filter = new TapAdvanceFilter();
143+
144+
filter.setSkip(100);
145+
filter.setLimit(50);
146+
filter.setSortOnList(Arrays.asList(new SortOn("id", SortOn.ASCENDING)));
147+
148+
commonSqlMaker.buildOffsetFetchClause(builder, filter);
149+
String result = builder.toString();
150+
String expected = " OFFSET 100 ROWS FETCH FIRST 50 ROWS ONLY ";
151+
Assertions.assertEquals(expected, result);
152+
}
153+
154+
@Test
155+
void buildSqlByAdvanceFilterWithOffsetFetchNoPaginationTest() {
156+
CommonSqlMaker commonSqlMaker = new CommonSqlMaker();
157+
TapAdvanceFilter filter = new TapAdvanceFilter();
158+
159+
// Test without any pagination parameters
160+
filter.setSortOnList(Arrays.asList(new SortOn("id", SortOn.ASCENDING)));
161+
162+
String result = commonSqlMaker.buildSqlByAdvanceFilterWithOffsetFetch(filter);
163+
String expected = " ORDER BY \"id\" ASC ";
164+
Assertions.assertEquals(expected, result);
165+
}
166+
167+
@Test
168+
void buildSqlByAdvanceFilterWithOffsetFetchEmptyTest() {
169+
CommonSqlMaker commonSqlMaker = new CommonSqlMaker();
170+
TapAdvanceFilter filter = new TapAdvanceFilter();
171+
172+
// Test with completely empty filter
173+
String result = commonSqlMaker.buildSqlByAdvanceFilterWithOffsetFetch(filter);
174+
String expected = "";
175+
Assertions.assertEquals(expected, result);
176+
}
177+
79178

80179
}

0 commit comments

Comments
 (0)