Skip to content

Change to use rest/api/latest/search/jql #88

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
minidragon88 wants to merge 3 commits into
base: master
Choose a base branch
from
Open

Change to use rest/api/latest/search/jql #88

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
5fb3d27
Change to use rest/api/latest/search/jql
minidragon88 Apr 28, 2025
e6ac9b9
Update version and changelog
minidragon88 Apr 28, 2025
234e132
Minor refactor
minidragon88 Apr 28, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions CHANGELOG.md
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,7 @@
## 0.3.2 - 2025-04-29
* [enhancement] Use new endpoint rest/api/latest/search/jql
* PR [#88](https://github.com/treasure-data/embulk-input-jira/pull/88)

## 0.3.1 - 2023-05-19
* [enhancement] Update library, minor code refactoring
* PR [#78](https://github.com/treasure-data/embulk-input-jira/pull/78)
Expand Down
2 changes: 1 addition & 1 deletion build.gradle
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ repositories {
mavenCentral()
}

version = "0.3.1"
version = "0.3.2"
group = "com.treasuredata.embulk.plugins"
description = "JIRA Embulk input plugin."

Expand Down
2 changes: 1 addition & 1 deletion src/main/java/org/embulk/input/jira/Constant.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ public final class Constant
public static final String DEFAULT_TIMESTAMP_PATTERN = "%Y-%m-%dT%H:%M:%S.%L%z";

public static final String CREDENTIAL_URI_PATH = "rest/api/latest/myself";
public static final String SEARCH_URI_PATH = "rest/api/latest/search";
public static final String SEARCH_URI_PATH = "rest/api/latest/search/jql";

private Constant(){}
}
21 changes: 10 additions & 11 deletions src/main/java/org/embulk/input/jira/JiraInputPlugin.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,6 @@
import java.util.SortedSet;
import java.util.TreeSet;

import static org.embulk.input.jira.Constant.GUESS_RECORDS_COUNT;
import static org.embulk.input.jira.Constant.MAX_RESULTS;
import static org.embulk.input.jira.Constant.PREVIEW_RECORDS_COUNT;

Expand Down Expand Up @@ -147,20 +146,19 @@ public TaskReport run(final TaskSource taskSource,
jiraClient.checkUserCredentials(task);
try (final PageBuilder pageBuilder = getPageBuilder(schema, output)) {
if (isPreview()) {
final List<Issue> issues = jiraClient.searchIssues(task, 0, PREVIEW_RECORDS_COUNT);
final SearchResult result = jiraClient.searchIssues(task, null, PREVIEW_RECORDS_COUNT);
final List<Issue> issues = JiraUtil.fromSearchResult(result);
issues.forEach(issue -> JiraUtil.addRecord(issue, schema, task, pageBuilder));
}
else {
int currentPage = 0;
final int totalCount = jiraClient.getTotalCount(task);
final int totalPage = JiraUtil.calculateTotalPage(totalCount, MAX_RESULTS);
LOGGER.info(String.format("Total pages (%d)", totalPage));
while (currentPage < totalPage) {
LOGGER.info(String.format("Fetching page %d/%d", (currentPage + 1), totalPage));
final List<Issue> issues = jiraClient.searchIssues(task, (currentPage * MAX_RESULTS), MAX_RESULTS);
String nextPageToken = null;
do {
final SearchResult result = jiraClient.searchIssues(task, nextPageToken, MAX_RESULTS);
nextPageToken = result.getNextPageToken();
final List<Issue> issues = JiraUtil.fromSearchResult(result);
issues.forEach(issue -> JiraUtil.addRecord(issue, schema, task, pageBuilder));
currentPage++;
}
while (nextPageToken != null);
}
pageBuilder.finish();
}
Expand All @@ -181,7 +179,8 @@ public ConfigDiff guess(final ConfigSource config)

private List<ConfigDiff> getGuessedColumns(final JiraClient jiraClient, final PluginTask task)
{
final List<Issue> issues = jiraClient.searchIssues(task, 0, GUESS_RECORDS_COUNT);
final SearchResult result = jiraClient.searchIssues(task, null, PREVIEW_RECORDS_COUNT);
final List<Issue> issues = JiraUtil.fromSearchResult(result);
if (issues.isEmpty()) {
throw new ConfigException("Could not guess schema due to empty data set");
}
Expand Down
23 changes: 23 additions & 0 deletions src/main/java/org/embulk/input/jira/SearchResult.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
package org.embulk.input.jira;

import com.google.gson.JsonArray;
import com.google.gson.JsonElement;

public class SearchResult
{
private JsonElement issues;
private String nextPageToken;

public JsonElement getIssues()
{
if (issues == null || issues.isJsonNull()) {
issues = new JsonArray();
}
return issues;
}

public String getNextPageToken()
{
return nextPageToken;
}
}
57 changes: 25 additions & 32 deletions src/main/java/org/embulk/input/jira/client/JiraClient.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,11 +1,14 @@
package org.embulk.input.jira.client;

import com.google.common.annotations.VisibleForTesting;
import com.google.gson.Gson;
import com.google.gson.JsonArray;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
import com.google.gson.JsonParser;
import com.google.gson.JsonPrimitive;

import org.apache.commons.lang3.StringUtils;
import org.apache.http.HttpStatus;
import org.apache.http.client.config.CookieSpecs;
import org.apache.http.client.config.RequestConfig;
Expand All @@ -18,8 +21,8 @@
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.http.util.EntityUtils;
import org.embulk.config.ConfigException;
import org.embulk.input.jira.Issue;
import org.embulk.input.jira.JiraInputPlugin.PluginTask;
import org.embulk.input.jira.SearchResult;
import org.embulk.input.jira.util.JiraException;
import org.embulk.input.jira.util.JiraUtil;
import org.embulk.util.retryhelper.RetryExecutor;
Expand All @@ -31,18 +34,12 @@
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map.Entry;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.StreamSupport;

import static java.util.Base64.getEncoder;
import static org.apache.http.HttpHeaders.ACCEPT;
import static org.apache.http.HttpHeaders.AUTHORIZATION;
import static org.apache.http.HttpHeaders.CONTENT_TYPE;
import static org.embulk.input.jira.Constant.HTTP_TIMEOUT;
import static org.embulk.input.jira.Constant.MIN_RESULTS;

public class JiraClient
{
Expand All @@ -64,31 +61,25 @@ public void checkUserCredentials(final PluginTask task)
}
}

public List<Issue> searchIssues(final PluginTask task, final int startAt, final int maxResults)
{
final String response = searchJiraAPI(task, startAt, maxResults);
final JsonObject result = new JsonParser().parse(response).getAsJsonObject();
return StreamSupport.stream(result.get("issues").getAsJsonArray().spliterator(), false)
.map(jsonElement -> {
final JsonObject json = jsonElement.getAsJsonObject();
final JsonObject fields = json.get("fields").getAsJsonObject();
final Set<Entry<String, JsonElement>> entries = fields.entrySet();
json.remove("fields");
// Merged all properties in fields to the object
for (final Entry<String, JsonElement> entry : entries) {
json.add(entry.getKey(), entry.getValue());
}
return new Issue(json);
})
.collect(Collectors.toList());
}

public int getTotalCount(final PluginTask task)
public SearchResult searchIssues(final PluginTask task, final String nextPageToken, final int maxResults)
{
return new JsonParser().parse(searchJiraAPI(task, 0, MIN_RESULTS)).getAsJsonObject().get("total").getAsInt();
return new Gson().fromJson(searchJiraAPI(task, nextPageToken, maxResults), SearchResult.class);
// return StreamSupport.stream(result.get("issues").getAsJsonArray().spliterator(), false)
// .map(jsonElement -> {
// final JsonObject json = jsonElement.getAsJsonObject();
// final JsonObject fields = json.get("fields").getAsJsonObject();
// final Set<Entry<String, JsonElement>> entries = fields.entrySet();
// json.remove("fields");
// // Merged all properties in fields to the object
// for (final Entry<String, JsonElement> entry : entries) {
// json.add(entry.getKey(), entry.getValue());
// }
// return new Issue(json);
// })
// .collect(Collectors.toList());
}

private String searchJiraAPI(final PluginTask task, final int startAt, final int maxResults)
private String searchJiraAPI(final PluginTask task, final String nextPageToken, final int maxResults)
{
try {
return RetryExecutor.builder()
Expand All @@ -101,7 +92,7 @@ private String searchJiraAPI(final PluginTask task, final int startAt, final int
@Override
public String call() throws Exception
{
return authorizeAndRequest(task, JiraUtil.buildSearchUrl(task.getUri()), createSearchIssuesBody(task, startAt, maxResults));
return authorizeAndRequest(task, JiraUtil.buildSearchUrl(task.getUri()), createSearchIssuesBody(task, nextPageToken, maxResults));
}

@Override
Expand Down Expand Up @@ -243,12 +234,14 @@ private HttpRequestBase createGetRequest(final PluginTask task, final String url
return request;
}

private String createSearchIssuesBody(final PluginTask task, final int startAt, final int maxResults)
private String createSearchIssuesBody(final PluginTask task, final String nextPageToken, final int maxResults)
{
final JsonObject body = new JsonObject();
final Optional<String> jql = task.getJQL();
body.add("jql", new JsonPrimitive(jql.orElse("")));
body.add("startAt", new JsonPrimitive(startAt));
if (StringUtils.isNotEmpty(nextPageToken)) {
body.add("nextPageToken", new JsonPrimitive(nextPageToken));
}
body.add("maxResults", new JsonPrimitive(maxResults));
final JsonArray fields = new JsonArray();
fields.add("*all");
Expand Down
20 changes: 20 additions & 0 deletions src/main/java/org/embulk/input/jira/util/JiraUtil.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
import org.embulk.config.ConfigSource;
import org.embulk.input.jira.Issue;
import org.embulk.input.jira.JiraInputPlugin.PluginTask;
import org.embulk.input.jira.SearchResult;
import org.embulk.spi.Column;
import org.embulk.spi.ColumnVisitor;
import org.embulk.spi.PageBuilder;
Expand All @@ -27,6 +28,8 @@
import java.time.Instant;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map.Entry;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.StreamSupport;

Expand Down Expand Up @@ -277,4 +280,21 @@ public static LinkedHashMap<String, Object> toLinkedHashMap(final JsonObject flt
}
return result;
}

public static List<Issue> fromSearchResult(final SearchResult result)
{
return StreamSupport.stream(result.getIssues().getAsJsonArray().spliterator(), false)
.map(jsonElement -> {
final JsonObject json = jsonElement.getAsJsonObject();
final JsonObject fields = json.get("fields").getAsJsonObject();
final Set<Entry<String, JsonElement>> entries = fields.entrySet();
json.remove("fields");
// Merged all properties in fields to the object
for (final Entry<String, JsonElement> entry : entries) {
json.add(entry.getKey(), entry.getValue());
}
return new Issue(json);
})
.collect(Collectors.toList());
}
}
21 changes: 12 additions & 9 deletions src/test/java/org/embulk/input/jira/JiraInputPluginTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -117,8 +117,8 @@ public void test_runDynamicSchema_withResult() throws IOException
.thenReturn(new StringEntity(searchResponse.get("body").toString()));

plugin.transaction(TestHelpers.dynamicSchemaConfig(), new Control());
// Check credential 1 + getTotal 1 + loadData 2
verify(jiraClient, times(4)).createHttpClient();
// Check credential 1 + loadData 2
verify(jiraClient, times(3)).createHttpClient();
verify(pageBuilder, times(1)).addRecord();
verify(pageBuilder, times(1)).finish();
}
Expand All @@ -137,8 +137,8 @@ public void test_run_with1RecordsResult() throws IOException
.thenReturn(new StringEntity(searchResponse.get("body").toString()));

plugin.transaction(config, new Control());
// Check credential 1 + getTotal 1 + loadData 1
verify(jiraClient, times(3)).createHttpClient();
// Check credential 1 + loadData 1
verify(jiraClient, times(2)).createHttpClient();
verify(pageBuilder, times(1)).addRecord();
verify(pageBuilder, times(1)).finish();
}
Expand All @@ -148,18 +148,21 @@ public void test_run_with2PagesResult() throws IOException
{
final JsonObject authorizeResponse = data.get("authenticateSuccess").getAsJsonObject();
final JsonObject searchResponse = data.get("2PagesResult").getAsJsonObject();
final JsonObject emptyResponse = data.get("emptyResult").getAsJsonObject();

when(statusLine.getStatusCode())
.thenReturn(authorizeResponse.get("statusCode").getAsInt())
.thenReturn(searchResponse.get("statusCode").getAsInt());
.thenReturn(searchResponse.get("statusCode").getAsInt())
.thenReturn(emptyResponse.get("statusCode").getAsInt());
when(response.getEntity())
.thenReturn(new StringEntity(authorizeResponse.get("body").toString()))
.thenReturn(new StringEntity(searchResponse.get("body").toString()));
.thenReturn(new StringEntity(searchResponse.get("body").toString()))
.thenReturn(new StringEntity(emptyResponse.get("body").toString()));

plugin.transaction(config, new Control());
// Check credential 1 + getTotal 1 + loadData 2
verify(jiraClient, times(4)).createHttpClient();
verify(pageBuilder, times(2)).addRecord();
// Check credential 1 + loadData 2
verify(jiraClient, times(3)).createHttpClient();
verify(pageBuilder, times(1)).addRecord();
verify(pageBuilder, times(1)).finish();
}

Expand Down
Loading