diff --git a/pom.xml b/pom.xml
index d49ef2e5..5f2677af 100644
--- a/pom.xml
+++ b/pom.xml
@@ -82,7 +82,7 @@
2.8.5
2.3.0
4.5.9
- 2.4.0-SNAPSHOT
+ 2.10.0-SNAPSHOT
2.9.9
4.11
2.7.1
diff --git a/src/main/java/io/cdap/plugin/http/source/common/pagination/page/JsonPage.java b/src/main/java/io/cdap/plugin/http/source/common/pagination/page/JsonPage.java
index b89d5afe..3c2b4ab0 100644
--- a/src/main/java/io/cdap/plugin/http/source/common/pagination/page/JsonPage.java
+++ b/src/main/java/io/cdap/plugin/http/source/common/pagination/page/JsonPage.java
@@ -154,7 +154,9 @@ public PageEntry next() {
numPartiallyRetrieved++;
}
- resultJson.add(schemaFieldName, queryResponse.get());
+ if (!queryResponse.getRetrievedPath().equals("/")) {
+ resultJson.add(schemaFieldName, queryResponse.get());
+ }
}
String jsonString = resultJson.toString();
diff --git a/src/test/java/io/cdap/plugin/http/source/common/pagination/page/JSONPageTest.java b/src/test/java/io/cdap/plugin/http/source/common/pagination/page/JSONPageTest.java
new file mode 100644
index 00000000..ae4da90f
--- /dev/null
+++ b/src/test/java/io/cdap/plugin/http/source/common/pagination/page/JSONPageTest.java
@@ -0,0 +1,109 @@
+/*
+ * Copyright © 2022 Cask Data, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package io.cdap.plugin.http.source.common.pagination.page;
+
+import com.google.gson.JsonObject;
+import io.cdap.cdap.api.data.format.StructuredRecord;
+import io.cdap.cdap.api.data.schema.Schema;
+import io.cdap.plugin.http.source.batch.HttpBatchSourceConfig;
+import io.cdap.plugin.http.source.common.BaseHttpSourceConfig;
+import io.cdap.plugin.http.source.common.http.HttpResponse;
+import org.junit.Assert;
+import org.junit.Test;
+import org.mockito.Mockito;
+
+import java.util.ArrayList;
+
+public class JSONPageTest {
+
+ // The input schema
+ private static final Schema INPUT_SCHEMA = Schema.recordOf("input",
+ Schema.Field.of("firstName", Schema.nullableOf(Schema.of(Schema.Type.STRING))),
+ Schema.Field.of("lastName", Schema.nullableOf(Schema.of(Schema.Type.STRING))),
+ Schema.Field.of("mail", Schema.of(Schema.Type.STRING)),
+ Schema.Field.of("_id", Schema.of(Schema.Type.STRING))
+ );
+
+ private static final String JSON = "\n" +
+ "{\n" +
+ " \"_id\": \"the_id_value\",\n" +
+ " \"mail\": \"toto.tata@tutu.com\",\n" +
+ " \"firstName\": \"toto\",\n" +
+ " \"lastName\": \"tata\"\n" +
+ "}";
+
+ private static final String JSON_WITH_EMPTY = "\n" +
+ "{\n" +
+ " \"_id\": \"the_id_value\",\n" +
+ " \"mail\": \"toto.tata@tutu.com\"\n" +
+ "}";
+
+ static class BaseTestConfig extends HttpBatchSourceConfig {
+ BaseTestConfig(String referenceName) {
+ super(referenceName);
+ this.schema = INPUT_SCHEMA.toString();
+ this.url = "";
+ this.httpMethod = "GET";
+ this.oauth2Enabled = "false";
+ this.httpErrorsHandling = "2..:Success,.*:Fail";
+ this.retryPolicy = "linear";
+ this.maxRetryDuration = 10L;
+ this.linearRetryInterval = 1L;
+ this.waitTimeBetweenPages = 0L;
+ this.connectTimeout = 60;
+ this.readTimeout = 120;
+ this.format = "json";
+ this.keystoreType = "Java KeyStore (JKS)";
+ this.trustStoreType = "Java KeyStore (JKS)";
+ this.transportProtocols = "TLSv1.2";
+ }
+ }
+
+ @Test
+ public void testJSONPageNominal() {
+ HttpResponse httpResponse = Mockito.mock(HttpResponse.class);
+ Mockito.when(httpResponse.getBody()).thenReturn(JSON);
+ BaseTestConfig config = new BaseTestConfig("testJsonPageNominal");
+ JsonPage jsonPage = new JsonPage(config, httpResponse);
+ PageEntry entry = jsonPage.next();
+ StructuredRecord outputRecord = entry.getRecord();
+ StructuredRecord expectedRecord = StructuredRecord.builder(INPUT_SCHEMA)
+ .set("_id", "the_id_value")
+ .set("mail", "toto.tata@tutu.com")
+ .set("firstName", "toto")
+ .set("lastName", "tata")
+ .build();
+
+ Assert.assertEquals(expectedRecord, outputRecord);
+ }
+
+ @Test
+ public void testJSONPageWithEmpty() {
+ HttpResponse httpResponse = Mockito.mock(HttpResponse.class);
+ Mockito.when(httpResponse.getBody()).thenReturn(JSON_WITH_EMPTY);
+ BaseTestConfig config = new BaseTestConfig("testJsonPageWithEmpty");
+ JsonPage jsonPage = new JsonPage(config, httpResponse);
+ PageEntry entry = jsonPage.next();
+ StructuredRecord outputRecord = entry.getRecord();
+ StructuredRecord expectedRecord = StructuredRecord.builder(INPUT_SCHEMA)
+ .set("_id", "the_id_value")
+ .set("mail", "toto.tata@tutu.com")
+ .build();
+
+ Assert.assertEquals(expectedRecord, outputRecord);
+ }
+
+}