Skip to content

Commit 72390dc

Browse files
axlbonnetaxlbonnet
authored
Merge pull request #593 from gwen-creatis/rest_error
Requetes logs de jobs multiples et liste des exécutions pour un job
2 parents a05cec8 + aafc6c7 commit 72390dc

File tree

7 files changed

+193
-30
lines changed

7 files changed

+193
-30
lines changed

‎vip-api/src/main/java/fr/insalyon/creatis/vip/api/business/ExecutionBusiness.java‎

Lines changed: 61 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -51,10 +51,7 @@
5151
import org.springframework.beans.factory.annotation.Autowired;
5252
import org.springframework.stereotype.Service;
5353

54-
import java.util.ArrayList;
55-
import java.util.HashMap;
56-
import java.util.List;
57-
import java.util.Map;
54+
import java.util.*;
5855
import java.util.Map.Entry;
5956
import java.util.function.Supplier;
6057

@@ -96,34 +93,58 @@ public ExecutionBusiness(Supplier<User> currentUserProvider,
9693
}
9794

9895
public String getLog(String executionId, String type) throws ApiException {
96+
return getLog(executionId, null, type);
97+
}
98+
99+
public String getLog(String executionId, Integer invocationId, String type) throws ApiException {
99100
try {
100101
Simulation s = workflowBusiness.getSimulation(executionId);
101-
102102
List<Task> tasks = simulationBusiness.getJobsList(s.getID());
103-
if (tasks.size() > 2) {
104-
logger.debug("Warning: more than two tasks found for execution ID = {} ", executionId);
105-
return "too many logs found";
106-
}
103+
107104
if (tasks.isEmpty()) {
108105
logger.debug("Warning: no .sh.out log file found for execution ID = {} ", executionId);
109106
return "no log found";
110107
}
111108

112109
String extension = ".sh.app." + type;
113110

114-
String fileName = tasks.getFirst().getFileName();
111+
Task targetTask = null;
112+
113+
if (invocationId == null) {
114+
if (tasks.size() == 1) {
115+
targetTask = tasks.get(0);
116+
logger.debug("jobId is null, using the only available task with ID = {}", targetTask.getId());
117+
} else {
118+
logger.debug("jobId is null but multiple tasks found for execution ID = {}", executionId);
119+
return "jobId is required when multiple tasks exist";
120+
}
121+
} else {
122+
targetTask = tasks.stream()
123+
.filter(t -> invocationId.equals(t.getInvocationID()))
124+
.max(Comparator.comparing(Task::getCreationDate))
125+
.orElse(null);
126+
}
127+
128+
129+
130+
if (targetTask == null) {
131+
logger.debug("No job {} found for execution ID = {}", invocationId, executionId);
132+
return "no log found for job " + invocationId;
133+
}
134+
135+
String fileName = targetTask.getFileName();
115136
if (fileName != null) {
116137
return simulationBusiness.readFile(executionId, type, fileName, extension);
138+
} else {
139+
throw new ApiException("no file name for job " + invocationId + " in execution " + executionId);
117140
}
118-
else {
119-
logger.error("no file name for task of {} ", executionId);
120-
throw new ApiException("no file name for task of " + executionId);
121-
}
141+
122142
} catch (BusinessException e) {
123143
throw new ApiException(e);
124144
}
125145
}
126146

147+
127148
public Execution getExample(String executionId) throws ApiException {
128149
return getExecution(executionId, false, true);
129150
}
@@ -192,6 +213,32 @@ private Execution getExecutionFromSimulation(Simulation s, boolean summarize) th
192213
e.getReturnedFiles().get(iod.getProcessor()).add(iod.getPath());
193214
}
194215

216+
// Jobs
217+
List<Task> tasks = simulationBusiness.getJobsList(s.getID());
218+
Map<Integer, Task> latestTaskPerInvocation = new HashMap<>();
219+
220+
for (Task t : tasks) {
221+
int invId = t.getInvocationID();
222+
Task current = latestTaskPerInvocation.get(invId);
223+
224+
if (current == null || t.getCreationDate().after(current.getCreationDate())) {
225+
latestTaskPerInvocation.put(invId, t);
226+
}
227+
}
228+
229+
Map<Integer, Map<String, Object>> jobsMap = new HashMap<>();
230+
for (Map.Entry<Integer, Task> entry : latestTaskPerInvocation.entrySet()) {
231+
Task t = entry.getValue();
232+
Map<String, Object> jobInfo = new HashMap<>();
233+
jobInfo.put("status", t.getStatus().toString());
234+
jobInfo.put("exitCode", t.getExitCode());
235+
jobInfo.put("exitMessage", t.getExitMessage());
236+
jobsMap.put(entry.getKey(), jobInfo);
237+
}
238+
239+
e.setJobs(jobsMap);
240+
241+
195242
if (!(e.getStatus() == ExecutionStatus.FINISHED) && !(e.getStatus() == ExecutionStatus.KILLED) && e.getReturnedFiles().isEmpty()) {
196243
e.clearReturnedFiles();
197244
}

‎vip-api/src/main/java/fr/insalyon/creatis/vip/api/controller/processing/ExecutionController.java‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -162,13 +162,27 @@ public String getStdout(@PathVariable String executionId) throws ApiException {
162162
return executionBusiness.getLog(executionId, "out");
163163
}
164164

165+
@RequestMapping(value = "/{executionId}/jobs/{invocationId}/stdout", produces = "text/plain;charset=UTF-8")
166+
public String getJobStdout(@PathVariable String executionId, @PathVariable Integer invocationId) throws ApiException {
167+
logMethodInvocation(logger, "getJobStdout", executionId, invocationId);
168+
executionBusiness.checkIfUserCanAccessExecution(executionId);
169+
return executionBusiness.getLog(executionId, invocationId, "out");
170+
}
171+
165172
@RequestMapping(value= "/{executionId}/stderr", produces = "text/plain;charset=UTF-8")
166173
public String getStderr(@PathVariable String executionId) throws ApiException {
167174
logMethodInvocation(logger, "getStderr", executionId);
168175
executionBusiness.checkIfUserCanAccessExecution(executionId);
169176
return executionBusiness.getLog(executionId, "err");
170177
}
171178

179+
@RequestMapping(value = "/{executionId}/jobs/{invocationId}/stderr", produces = "text/plain;charset=UTF-8")
180+
public String getJobStderr(@PathVariable String executionId, @PathVariable Integer invocationId) throws ApiException {
181+
logMethodInvocation(logger, "getJobStderr", executionId, invocationId);
182+
executionBusiness.checkIfUserCanAccessExecution(executionId);
183+
return executionBusiness.getLog(executionId, invocationId, "err");
184+
}
185+
172186
@RequestMapping(value = "/{executionId}/play", method = RequestMethod.PUT)
173187
@ResponseStatus(HttpStatus.NO_CONTENT)
174188
public void playExecution(@PathVariable String executionId) throws ApiException {

‎vip-api/src/main/java/fr/insalyon/creatis/vip/api/model/Execution.java‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -57,11 +57,12 @@ public class Execution {
5757
private Long startDate;
5858
private Long endDate;
5959
private String resultsLocation;
60-
60+
private Map<Integer, Map<String, Object>> jobs; // jobId -> status
6161

6262
public Execution() {
6363
inputValues = new HashMap<>();
6464
returnedFiles = new HashMap<>();
65+
jobs = new HashMap<>();
6566
}
6667

6768
public Execution(String identifier,
@@ -85,6 +86,7 @@ public Execution(String identifier,
8586
this.startDate = startDate;
8687
this.endDate = endDate;
8788
this.resultsLocation = resultsLocation;
89+
8890
}
8991

9092
public String getIdentifier() {
@@ -186,4 +188,8 @@ public String getResultsLocation() {
186188
public void setResultsLocation(String resultsLocation) {
187189
this.resultsLocation = resultsLocation;
188190
}
191+
192+
public Map<Integer, Map<String, Object>> getJobs() { return jobs; }
193+
194+
public void setJobs(Map<Integer, Map<String, Object>> jobs) { this.jobs = jobs; }
189195
}

‎vip-api/src/test/java/fr/insalyon/creatis/vip/api/data/ExecutionTestUtils.java‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -164,7 +164,7 @@ public static Map<String,Function> getExecutionSuppliers() {
164164
Arrays.asList(
165165
"identifier", "name", "pipelineIdentifier", "timeout",
166166
"status", "inputValues", "returnedFiles", "studyIdentifier",
167-
"errorCode", "startDate", "endDate"),
167+
"errorCode", "startDate", "endDate", "jobs"),
168168
Execution::getIdentifier,
169169
Execution::getName,
170170
Execution::getPipelineIdentifier,
@@ -175,7 +175,8 @@ public static Map<String,Function> getExecutionSuppliers() {
175175
Execution::getStudyIdentifier,
176176
Execution::getErrorCode,
177177
Execution::getStartDate,
178-
Execution::getEndDate
178+
Execution::getEndDate,
179+
Execution::getJobs
179180
);
180181
}
181182

‎vip-api/src/test/java/fr/insalyon/creatis/vip/api/rest/itest/processing/ExecutionControllerIT.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -401,6 +401,11 @@ public void testInitBoutiquesExecution() throws Exception
401401

402402
Execution expectedExecution = new Execution(workflowId, "Exec test 1", appName + "/" + versionName, 0, ExecutionStatus.RUNNING, null, null, startDate.getTime(), null, null);
403403
expectedExecution.clearReturnedFiles();
404+
expectedExecution.getJobs().put(0, new HashMap<>() {{
405+
put("exitCode", 0);
406+
put("exitMessage", "Successfully executed");
407+
put("status", "COMPLETED");
408+
}});
404409

405410
setUpResourceAndEngine(appName, versionName, engineEndpoint);
406411

‎vip-application/src/main/java/fr/insalyon/creatis/vip/application/client/bean/Task.java‎

Lines changed: 88 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,13 +3,17 @@
33
import com.google.gwt.user.client.rpc.IsSerializable;
44
import fr.insalyon.creatis.vip.application.client.view.monitor.job.TaskStatus;
55

6+
import java.util.Date;
7+
68
/**
79
*
810
* @author Rafael Ferreira da Silva
911
*/
1012
public class Task implements IsSerializable {
1113

1214
private String id;
15+
private int invocationID;
16+
private Date creationDate;
1317
private TaskStatus status;
1418
private int exitCode;
1519
private String siteName;
@@ -30,11 +34,13 @@ public Task(int jobID, TaskStatus status, String command) {
3034
this.command = command;
3135
}
3236

33-
public Task(String id, TaskStatus status, String command, String fileName,
34-
int exitCode, String siteName, String nodeName, int minorStatus,
35-
String... parameters) {
37+
public Task(String id, int invocationID, Date creationDate, TaskStatus status, String command, String fileName,
38+
int exitCode, String siteName, String nodeName, int minorStatus,
39+
String... parameters) {
3640

3741
this.id = id;
42+
this.invocationID= invocationID;
43+
this.creationDate = creationDate;
3844
this.status = status;
3945
this.command = command;
4046
this.fileName = fileName;
@@ -45,6 +51,67 @@ public Task(String id, TaskStatus status, String command, String fileName,
4551
this.minorStatus = minorStatus;
4652
}
4753

54+
public enum GaswExitCode {
55+
56+
SUCCESS(0, "Successfully executed"),
57+
ERROR_READ_GRID(1, "Error downloading a file"),
58+
ERROR_WRITE_GRID(2, "Error uploading a result file"),
59+
ERROR_WRITE_LOCAL(7, "Error writing on local disk"),
60+
ERROR_GET_STD(8, "Error downloading the execution logs"),
61+
ERROR_FILE_NOT_FOUND(3, "Error: File not found"),
62+
EXECUTION_FAILED_LEGACY(6, "Execution failed"),
63+
EXECUTION_CANCELED(9, "Execution canceled"),
64+
EXECUTION_STALLED(10, "Execution stalled"),
65+
UNDEFINED(-1, "Exit code not known"),
66+
GIRDER_CLIENT_INSTALL_FAILED(20, "Error installing girder client"),
67+
BOUTIQUE_INSTALL_FAILED(21, "Error installing boutiques"),
68+
ERROR_CREATE_EXEC_DIR(22, "Error: Unable to create the execution directory"),
69+
ERROR_CD_EXEC_DIR(23, "Error: Unable to use the execution directory"),
70+
CONFIG_NOT_FOUND(24, "Error: Execution configuration file not found"),
71+
BOUTIQUE_IMPORT_FAILED(25, "Error: Import boutiques fails"),
72+
ERROR_WRITE_LFN(30, "Error uploading a result file to lfn"),
73+
ERROR_MV_FILE(31, "Error while moving a local result file"),
74+
ERROR_RESULT_FILE_EXIST(32, "Error : A local result file already exists"),
75+
ERROR_UPLOAD_GIRDER(33, "Error uploading a result file to girder"),
76+
ERROR_UPLOAD_SHANOIR(34, "Error uploading a result file to shanoir"),
77+
FAILED_CREATE_LOCAL_UPLOAD_DIR(35, "Error creating a local result directory"),
78+
ERROR_GIRDER_MKDIR(36, "Error while checking or creating a girder directory for results"),
79+
ERROR_SHANOIR_UPLOAD_SUBDIR(37, "Error: Unsupported subdirectory for shanoir results"),
80+
ERROR_DL(40, "Download error"),
81+
INVALID_SHANOIR_NIFTI(41, "Too many or none nifti file (.nii or .nii.gz) in shanoir zip, supporting only 1"),
82+
SHANOIR_DL_FAILED(42, "Error downloading a shanoir file"),
83+
EXECUTION_FAILED(50, "Execution was not successful"),
84+
SING_INVALID_IMAGE_NAME(51, "Invalid singularity/apptainer image name"),
85+
SING_IMAGE_NOT_FOUND(52, "Error: Singularity/apptainer image not found"),
86+
INVALID_CONTAINER_RUNTIME(53, "Invalid container runtime (only docker/singularity)"),
87+
TOKEN_REFRESH_TOO_LONG(60, "Error: Timeout while refreshing an OIDC token."),
88+
TOKEN_REFRESH_ERROR(61, "Error while refreshing an OIDC token");
89+
90+
private int exitCode;
91+
private String message;
92+
93+
private GaswExitCode(int exitCode, String message) {
94+
this.exitCode = exitCode;
95+
this.message = message;
96+
}
97+
98+
public String getMessage() { return message; }
99+
100+
public int getExitCode() {
101+
return this.exitCode;
102+
}
103+
104+
public static GaswExitCode fromCode(int code) {
105+
for (GaswExitCode value : values()) {
106+
if (value.exitCode == code) {
107+
return value;
108+
}
109+
}
110+
return UNDEFINED; // fallback if unknown code
111+
}
112+
}
113+
114+
48115
public String getCommand() {
49116
return command;
50117
}
@@ -120,4 +187,22 @@ public int getJobID() {
120187
public void setJobID(int jobID) {
121188
this.jobID = jobID;
122189
}
190+
191+
public int getInvocationID() {
192+
return invocationID;
193+
}
194+
195+
public void setInvocationID(int jobID) {
196+
this.invocationID = invocationID;
197+
}
198+
199+
public Date getCreationDate() {
200+
return creationDate;
201+
}
202+
203+
public void setCreationDate(int jobID) {
204+
this.creationDate = creationDate;
205+
}
206+
207+
public String getExitMessage() { return GaswExitCode.fromCode(this.exitCode).getMessage(); }
123208
}

‎vip-application/src/main/java/fr/insalyon/creatis/vip/application/server/dao/h2/SimulationData.java‎

Lines changed: 15 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -72,7 +72,7 @@ public List<Task> getTasks(int jobID) throws DAOException {
7272
List<Task> list = new ArrayList<Task>();
7373
try {
7474
PreparedStatement ps = connection.prepareStatement(
75-
"SELECT j.id, status, command, file_name, exit_code, node_site, node_name, parameters, "
75+
"SELECT j.id, j.invocation_id, creation, status, command, file_name, exit_code, node_site, node_name, parameters, "
7676
+ "ms FROM Jobs AS j LEFT JOIN ("
7777
+ " SELECT jm.id, minor_status AS ms FROM JobsMinorStatus AS jm RIGHT JOIN ( "
7878
+ " SELECT id, MAX(event_date) AS ed FROM JobsMinorStatus GROUP BY id "
@@ -102,7 +102,7 @@ public Task getTask(String taskID) throws DAOException {
102102

103103
try {
104104
PreparedStatement ps = connection.prepareStatement(
105-
"SELECT j.id, status, command, file_name, exit_code, node_site, node_name, parameters, "
105+
"SELECT j.id, j.invocation_id, creation, status, command, file_name, exit_code, node_site, node_name, parameters, "
106106
+ "ms FROM Jobs AS j LEFT JOIN ("
107107
+ " SELECT jm.id, minor_status AS ms FROM JobsMinorStatus AS jm RIGHT JOIN ( "
108108
+ " SELECT id, MAX(event_date) AS ed FROM JobsMinorStatus GROUP BY id "
@@ -172,7 +172,7 @@ private Task parseTask(ResultSet rs) throws SQLException {
172172
minorStatus = parseMinorStatus(rs.getString("ms"));
173173
}
174174

175-
return new Task(rs.getString("id"), status,
175+
return new Task(rs.getString("id"), rs.getInt("invocation_id"), rs.getTimestamp("creation"), status,
176176
rs.getString("command"), rs.getString("file_name"),
177177
rs.getInt("exit_code"), rs.getString("node_site"),
178178
rs.getString("node_name"), minorStatus,
@@ -207,12 +207,16 @@ public List<Task> getJobs() throws DAOException {
207207
try {
208208
Statement stat = connection.createStatement();
209209
ResultSet rs = stat.executeQuery(
210-
"SELECT j.id, status, command, file_name, exit_code, node_site, node_name, parameters, "
211-
+ "ms FROM Jobs AS j LEFT JOIN ("
212-
+ " SELECT jm.id, minor_status AS ms FROM JobsMinorStatus AS jm RIGHT JOIN ( "
213-
+ " SELECT id, MAX(event_date) AS ed FROM JobsMinorStatus GROUP BY id "
214-
+ " ) AS jm1 ON jm1.id = jm.id AND jm1.ed = jm.event_date "
215-
+ ") AS jm2 ON j.id = jm2.id ORDER BY j.id");
210+
"SELECT j.id, j.invocation_id, creation, status, command, file_name, exit_code, " +
211+
"node_site, node_name, parameters, ms " +
212+
"FROM Jobs AS j " +
213+
"LEFT JOIN ( " +
214+
" SELECT jm.id, minor_status AS ms FROM JobsMinorStatus AS jm " +
215+
" RIGHT JOIN ( " +
216+
" SELECT id, MAX(event_date) AS ed FROM JobsMinorStatus GROUP BY id " +
217+
" ) AS jm1 ON jm1.id = jm.id AND jm1.ed = jm.event_date " +
218+
") AS jm2 ON j.id = jm2.id " +
219+
"ORDER BY j.id");
216220

217221
while (rs.next()) {
218222
TaskStatus status = TaskStatus.valueOf(rs.getString("status"));
@@ -222,7 +226,8 @@ public List<Task> getJobs() throws DAOException {
222226
minorStatus = parseMinorStatus(rs.getString("ms"));
223227
}
224228

225-
list.add(new Task(rs.getString("id"), status,
229+
list.add(new Task( rs.getString("id"), rs.getInt("invocation_id"),
230+
rs.getTimestamp("creation"), status,
226231
rs.getString("command"), rs.getString("file_name"),
227232
rs.getInt("exit_code"), rs.getString("node_site"),
228233
rs.getString("node_name"), minorStatus,

0 commit comments

Comments
 (0)