Skip to content

Commit 8d5e783

Browse files
eliranmoyaleliranmoyal
committed
scroll support with USE_SCROLL hint, also add site support
1 parent 89ca672 commit 8d5e783

File tree

10 files changed

+153
-5
lines changed

10 files changed

+153
-5
lines changed

‎src/_site/controllers.js‎

Lines changed: 57 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,9 +8,61 @@ elasticsearchSqlApp.controller('MainController', function ($scope, $http, $sce)
88
$scope.resultsRows = [];
99
$scope.searchLoading = false;
1010
$scope.explainLoading = false;
11+
$scope.nextLoading = false;
1112
$scope.resultExplan = false;
12-
13+
$scope.scrollId = null;
14+
$scope.gotNext = false;
15+
16+
$scope.nextSearch = function(){
17+
$scope.error = "";
18+
$scope.nextLoading = true;
19+
$scope.$apply();
20+
21+
22+
if($scope.scrollId == null || $scope.scrollId == "" ){
23+
$scope.error = "tryed scrolling with empty scrollId";
24+
return;
25+
}
1326

27+
$http.get($scope.url + "_search/scroll?scroll=1m&scroll_id=" + $scope.scrollId)
28+
.success(function(data, status, headers, config) {
29+
var handler = ResultHandlerFactory.create(data);
30+
var body = handler.getBody()
31+
32+
if(body.length ==null || body.length == 0){
33+
$scope.gotNext=false;
34+
}
35+
else
36+
{
37+
$scope.scrollId = handler.getScrollId();
38+
}
39+
40+
if($scope.resultsRows.length > 0){
41+
$scope.resultsRows = $scope.resultsRows.concat(handler.getBody());
42+
}
43+
else {
44+
$scope.resultsColumns = handler.getHead();
45+
$scope.resultsRows = handler.getBody();
46+
47+
}
48+
49+
50+
})
51+
.error(function(data, status, headers, config) {
52+
if(data == "") {
53+
$scope.error = "Error occured! response is not avalible.";
54+
}
55+
else {
56+
$scope.error = JSON.stringify(data);
57+
$scope.scrollId = null;
58+
}
59+
})
60+
.finally(function() {
61+
$scope.nextLoading = false;
62+
$scope.$apply()
63+
});
64+
65+
}
1466

1567
$scope.search = function() {
1668
// Reset results and error box
@@ -28,6 +80,10 @@ elasticsearchSqlApp.controller('MainController', function ($scope, $http, $sce)
2880
$http.post($scope.url + "_sql", query)
2981
.success(function(data, status, headers, config) {
3082
var handler = ResultHandlerFactory.create(data);
83+
if(handler.isScroll){
84+
$scope.gotNext=true;
85+
$scope.scrollId = handler.getScrollId();
86+
}
3187
$scope.resultsColumns = handler.getHead();
3288
$scope.resultsRows = handler.getBody();
3389

‎src/_site/index.html‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -105,6 +105,9 @@ <h1 class="page-header">SQL Query</h1>
105105
</button>
106106

107107
<button type="button" ng-click="explain()" id="explainButton" class="btn btn-info explain-button" ng-bind-html="getButtonContent(explainLoading,'Explain')" ng-cloak>
108+
</button>
109+
110+
<button type="next" ng-click="nextSearch()" id="nextButton" class="btn btn-warning next-button" ng-show="gotNext" ng-bind-html="getButtonContent(nextLoading,'Next')" ng-cloak>
108111
</button>
109112
</div>
110113

‎src/_site/query.js‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,8 +42,19 @@ var DefaultQueryResultHandler = function(data) {
4242

4343
this.data = data
4444
this.head = createScheme()
45+
this.scrollId = data["_scroll_id"]
46+
this.isScroll = this.scrollId!=null && this.scrollId!="";
4547
};
4648

49+
DefaultQueryResultHandler.prototype.isScroll = function() {
50+
return this.isScroll;
51+
};
52+
53+
DefaultQueryResultHandler.prototype.getScrollId = function() {
54+
return this.scrollId;
55+
};
56+
57+
4758
DefaultQueryResultHandler.prototype.getHead = function() {
4859
return this.head
4960
};

‎src/_site/style.css‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -123,6 +123,12 @@ body {
123123
}
124124

125125

126+
.next-button {
127+
float:left;
128+
margin-top: 10px;
129+
}
130+
131+
126132
.keyboardSection {
127133
margin-top:35px;
128134
}

‎src/main/java/org/nlpcn/es4sql/domain/Select.java‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
package org.nlpcn.es4sql.domain;
22

3+
import org.nlpcn.es4sql.domain.hints.Hint;
4+
35
import java.util.ArrayList;
46
import java.util.Arrays;
57
import java.util.List;
@@ -13,7 +15,7 @@ public class Select extends Query {
1315

1416
// Using this functions, will cause query to execute as aggregation.
1517
private final List<String> aggsFunctions = Arrays.asList("SUM", "MAX", "MIN", "AVG", "TOPHITS", "COUNT", "STATS");
16-
18+
private List<Hint> hints = new ArrayList<>();
1719
private List<Field> fields = new ArrayList<>();
1820
private List<List<Field>> groupBys = new ArrayList<>();
1921
private List<Order> orderBys = new ArrayList<>();
@@ -86,5 +88,10 @@ public void addField(Field field) {
8688
fields.add(field);
8789
}
8890

91+
public List<Hint> getHints() {
92+
return hints;
93+
}
94+
95+
8996
}
9097

‎src/main/java/org/nlpcn/es4sql/domain/hints/HintFactory.java‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,11 +35,22 @@ public static Hint getHintFromString(String hintAsString){
3535
int multiSearchSize = Integer.parseInt(number[0]);
3636
return new Hint(HintType.NL_MULTISEARCH_SIZE,new Object[]{multiSearchSize});
3737
}
38+
if(hintAsString.startsWith("! USE_SCROLL")){
39+
String[] scrollParams = getParamsFromHint(hintAsString,"! USE_SCROLL");
40+
int docsPerFetch = 10000;
41+
int timeout = 60000;
42+
if(scrollParams != null && scrollParams.length ==2) {
43+
docsPerFetch = Integer.parseInt(scrollParams[0]);
44+
timeout = Integer.parseInt(scrollParams[1]);
45+
}
46+
return new Hint(HintType.USE_SCROLL, new Object[]{docsPerFetch,timeout});
47+
}
3848
return null;
3949
}
4050

4151

4252
private static String[] getParamsFromHint(String hint, String prefix) {
53+
if(!hint.contains("(")) return null;
4354
String onlyParams = hint.replace(prefix, "").replaceAll("\\s*\\(\\s*","").replaceAll("\\s*\\,\\s*", ",").replaceAll("\\s*\\)\\s*", "");
4455
return onlyParams.split(",");
4556
}

‎src/main/java/org/nlpcn/es4sql/domain/hints/HintType.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ public enum HintType
1111
HASH_WITH_TERMS_FILTER,
1212
JOIN_LIMIT,
1313
USE_NESTED_LOOPS,
14-
NL_MULTISEARCH_SIZE;
14+
NL_MULTISEARCH_SIZE,
15+
USE_SCROLL;
1516

1617
}

‎src/main/java/org/nlpcn/es4sql/parse/SqlParser.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,8 @@ public Select parseSelect(SQLQueryExpr mySqlExpr) throws SqlParseException {
3939

4040
select.setWhere(findWhere(query.getWhere()));
4141

42+
select.getHints().addAll(parseHints(query.getHints()));
43+
4244
findLimit(query.getLimit(), select);
4345

4446
findOrderBy(query, select);

‎src/main/java/org/nlpcn/es4sql/query/DefaultQueryAction.java‎

Lines changed: 27 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
import org.elasticsearch.action.search.SearchRequestBuilder;
66
import org.elasticsearch.action.search.SearchType;
77
import org.elasticsearch.client.Client;
8+
import org.elasticsearch.common.unit.TimeValue;
89
import org.elasticsearch.index.query.BoolFilterBuilder;
910
import org.elasticsearch.index.query.BoolQueryBuilder;
1011
import org.elasticsearch.index.query.QueryBuilders;
@@ -13,6 +14,8 @@
1314
import org.nlpcn.es4sql.domain.Order;
1415
import org.nlpcn.es4sql.domain.Select;
1516
import org.nlpcn.es4sql.domain.Where;
17+
import org.nlpcn.es4sql.domain.hints.Hint;
18+
import org.nlpcn.es4sql.domain.hints.HintType;
1619
import org.nlpcn.es4sql.exception.SqlParseException;
1720
import org.nlpcn.es4sql.query.maker.FilterMaker;
1821
import org.nlpcn.es4sql.query.maker.QueryMaker;
@@ -42,12 +45,34 @@ public SqlElasticSearchRequestBuilder explain() throws SqlParseException {
4245
setLimit(select.getOffset(), select.getRowCount());
4346

4447
// set SearchType.
45-
request.setSearchType(SearchType.DFS_QUERY_THEN_FETCH);
48+
boolean usedScroll = useScrollIfNeeded();
49+
if(!usedScroll){
50+
request.setSearchType(SearchType.DFS_QUERY_THEN_FETCH);
51+
}
52+
4653
SqlElasticSearchRequestBuilder sqlElasticRequestBuilder = new SqlElasticSearchRequestBuilder(request);
4754
return sqlElasticRequestBuilder;
4855
}
4956

50-
/**
57+
private boolean useScrollIfNeeded() {
58+
Hint scrollHint = null;
59+
for(Hint hint: select.getHints()){
60+
if(hint.getType() == HintType.USE_SCROLL){
61+
scrollHint = hint;
62+
break;
63+
}
64+
}
65+
if(scrollHint!=null) {
66+
int scrollSize = (Integer) scrollHint.getParams()[0];
67+
int timeoutInMilli = (Integer) scrollHint.getParams()[1];
68+
request.setSearchType(SearchType.SCAN)
69+
.setScroll(new TimeValue(timeoutInMilli))
70+
.setSize(scrollSize);
71+
}
72+
return scrollHint !=null ;
73+
}
74+
75+
/**
5176
* Set indices and types to the search request.
5277
*/
5378
private void setIndicesAndTypes() {

‎src/test/java/org/nlpcn/es4sql/QueryTest.java‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
package org.nlpcn.es4sql;
22

3+
import org.elasticsearch.action.search.SearchRequest;
34
import org.elasticsearch.action.search.SearchRequestBuilder;
45
import org.elasticsearch.action.search.SearchResponse;
56
import org.elasticsearch.common.joda.time.DateTime;
@@ -558,9 +559,34 @@ public void isNotNullTest() throws IOException, SqlParseException, SQLFeatureNot
558559
Assert.assertEquals(1, response.getTotalHits());
559560
}
560561

562+
563+
@Test
564+
public void useScrollNoParams() throws IOException, SqlParseException, SQLFeatureNotSupportedException{
565+
SearchResponse response = getSearchResponse(String.format("SELECT /*! USE_SCROLL*/ age,gender,firstname,balance FROM %s/account LIMIT 2000", TEST_INDEX, TEST_INDEX));
566+
Assert.assertNotNull(response.getScrollId());
567+
SearchHits hits = response.getHits();
568+
Assert.assertEquals(0,hits.getHits().length);
569+
Assert.assertEquals(1000,hits.getTotalHits());
570+
}
571+
572+
@Test
573+
public void useScrollWithParams() throws IOException, SqlParseException, SQLFeatureNotSupportedException{
574+
SearchResponse response = getSearchResponse(String.format("SELECT /*! USE_SCROLL(10,5000)*/ age,gender,firstname,balance FROM %s/account ", TEST_INDEX, TEST_INDEX));
575+
Assert.assertNotNull(response.getScrollId());
576+
SearchHits hits = response.getHits();
577+
Assert.assertEquals(0,hits.getHits().length);
578+
Assert.assertEquals(1000,hits.getTotalHits());
579+
}
580+
561581
private SearchHits query(String query) throws SqlParseException, SQLFeatureNotSupportedException, SQLFeatureNotSupportedException {
562582
SearchDao searchDao = MainTestSuite.getSearchDao();
563583
SqlElasticSearchRequestBuilder select = (SqlElasticSearchRequestBuilder) searchDao.explain(query);
564584
return ((SearchResponse)select.get()).getHits();
565585
}
586+
587+
private SearchResponse getSearchResponse(String query) throws SqlParseException, SQLFeatureNotSupportedException, SQLFeatureNotSupportedException {
588+
SearchDao searchDao = MainTestSuite.getSearchDao();
589+
SqlElasticSearchRequestBuilder select = (SqlElasticSearchRequestBuilder) searchDao.explain(query);
590+
return ((SearchResponse)select.get());
591+
}
566592
}

0 commit comments

Comments
 (0)