Skip to content

Commit 9dbefc3

Browse files
spidaKevinGentilespidaKevinGentile
authored
Merge pull request #9 from spidasoftware/retry-timedout-requests-166214942
[Delivers #166214942] Retry failed requests
2 parents a4b6ed2 + 420c31a commit 9dbefc3

File tree

4 files changed

+84
-32
lines changed

4 files changed

+84
-32
lines changed

‎cmds/analyze.js‎

Lines changed: 21 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -3,18 +3,17 @@
33
const BATCH_SIZE = 100;
44

55
const Promise = require('bluebird');
6-
const request = Promise.promisify(require('request'));
76
const fs = Promise.promisifyAll(require('fs'));
87
const zlib = Promise.promisifyAll(require('zlib'));
98
const mkdirp = Promise.promisify(require('mkdirp'));
10-
const querystring = require('querystring');
119
const _ = require('lodash');
1210
const path = require('path');
1311
const co = require('co');
1412
const util = require('util');
1513

1614
const api = require('../lib/api');
1715
const { loadClientData } = require('../lib/clientData');
16+
const { requestAndRetry } = require('../lib/retrier');
1817

1918
const command = 'analyze [json..]';
2019
const desc = 'Send analysis or job in each file to CEE, then optionally poll CEE until analysis is complete and retrieve results. If polling is enabled analysis results will be written to a directory specified by the output parameter.';
@@ -59,7 +58,7 @@ module.exports = {
5958
batchJobs(argv).mapSeries(batch =>
6059
zlib.gzipAsync(JSON.stringify(batch))
6160
.then(jobData =>
62-
request({
61+
requestAndRetry({
6362
url: config.server + '/job?apiToken=' + config.apiToken,
6463
proxy: config.proxy,
6564
gzip: true,
@@ -71,15 +70,15 @@ module.exports = {
7170
'Content-Encoding': 'gzip'
7271
},
7372
body: jobData
73+
}, (response) => {
74+
try {
75+
return JSON.parse(response.body);
76+
} catch (e) {
77+
console.log(`Unable to parse response JSON: ${response.body}`);
78+
throw e;
79+
}
7480
})
75-
).then(response => {
76-
try {
77-
return JSON.parse(response.body);
78-
} catch (e) {
79-
console.log(`Unable to parse response JSON: ${response.body}`);
80-
return [];
81-
}
82-
})
81+
)
8382
).then(responses => {
8483
responses = _.flatten(responses);
8584

@@ -103,7 +102,6 @@ module.exports = {
103102
console.log('The following errors occured:');
104103
console.log(util.inspect(errors, { depth: null}));
105104
}
106-
107105
return showProgress(jobIds,argv,config).then(() => console.log('Done.'));
108106
}
109107
})
@@ -113,7 +111,7 @@ module.exports = {
113111
function pollFor(jobIds, status, config, onUpdate) {
114112
return co(function*() {
115113
while(jobIds.length > 0) {
116-
const response = yield request({
114+
const updatedJobIds = yield requestAndRetry({
117115
url: config.server + '/job/poll',
118116
proxy: config.proxy,
119117
qs: { apiToken: config.apiToken, status, ids: JSON.stringify(jobIds.slice(0,BATCH_SIZE)) },
@@ -122,10 +120,15 @@ function pollFor(jobIds, status, config, onUpdate) {
122120
'User-Agent': 'cee-cli',
123121
'Accept': 'application/json',
124122
}
123+
}, (response) => {
124+
try {
125+
return JSON.parse(response.body).map(j => j.id)
126+
} catch(e) {
127+
console.log(`Failed to poll for jobs ${response.body}`)
128+
throw e
129+
}
125130
});
126131

127-
const updatedJobIds = JSON.parse(response.body).map(j => j.id);
128-
129132
_.pullAll(jobIds,updatedJobIds);
130133
const updateP = onUpdate(updatedJobIds);
131134
if (updateP) {
@@ -158,7 +161,7 @@ function showProgress(jobIds,argv,config) {
158161
displayProgress(validCount, startedCount, finishedCount);
159162
}),
160163
pollFor(unfinishedJobs, 'FINISHED', config, jobIds =>
161-
request({
164+
requestAndRetry({
162165
url: config.server + '/job',
163166
qs: {
164167
apiToken: config.apiToken,
@@ -170,7 +173,7 @@ function showProgress(jobIds,argv,config) {
170173
'User-Agent': 'cee-cli',
171174
'Accept': 'application/json',
172175
}
173-
}).then(response =>
176+
}, (response) =>
174177
Promise.all(
175178
JSON.parse(response.body).map(result =>
176179
fs.writeFileAsync(path.join(output,result.externalId),JSON.stringify(result))
@@ -358,3 +361,4 @@ function batchJobs(argv) {
358361

359362

360363

364+

‎cmds/get.js‎

Lines changed: 20 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
'use strict';
22

33
const Promise = require('bluebird');
4-
const request = Promise.promisify(require('request'));
4+
const { requestAndRetry } = require('../lib/retrier');
55
const fs = Promise.promisifyAll(require('fs'));
66
const mkdirp = Promise.promisify(require('mkdirp'));
77
const path = require('path');
@@ -39,7 +39,7 @@ module.exports = {
3939
let jobIdsP;
4040

4141
if (typeof argv.offset !== 'undefined') {
42-
jobIdsP = request({
42+
jobIdsP = requestAndRetry({
4343
url: config.server + '/api/jobs',
4444
proxy: config.proxy,
4545
headers: {
@@ -52,14 +52,21 @@ module.exports = {
5252
offset: argv.offset
5353
},
5454
gzip: true
55-
}).then(resp => JSON.parse(resp.body).data.map(j => j.id));
55+
}, (resp) => {
56+
try {
57+
return JSON.parse(resp.body).data.map(j => j.id);
58+
} catch(e) {
59+
console.log(`Failed to get jobs at offset: ${resp.body}`)
60+
throw e;
61+
}
62+
});
5663
} else {
5764
jobIdsP = Promise.resolve(argv.jobIds);
5865
}
5966

6067
return jobIdsP.then(allJobIds =>
6168
Promise.all(_.chunk(allJobIds, 100).map(jobIds =>
62-
request({
69+
requestAndRetry({
6370
url: config.server + '/job',
6471
proxy: config.proxy,
6572
headers: {
@@ -71,10 +78,17 @@ module.exports = {
7178
ids: JSON.stringify(jobIds)
7279
},
7380
gzip: true
81+
}, (response) => {
82+
try {
83+
return JSON.parse(response.body);
84+
} catch(e) {
85+
console.log(`Failed to get jobs: ${response.body}`)
86+
throw e;
87+
}
7488
})
7589
))
76-
).then(responses => {
77-
const jobs = _.flatten(responses.map(r => JSON.parse(r.body)));
90+
).then(bodies => {
91+
const jobs = _.flatten(bodies);
7892
if (argv.stdout) {
7993
console.log(JSON.stringify(jobs));
8094
} else {

‎cmds/status.js‎

Lines changed: 12 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,6 @@
11
'use strict';
22

3-
const Promise = require('bluebird');
4-
const request = Promise.promisify(require('request'));
5-
3+
const { requestAndRetry } = require('../lib/retrier');
64
const api = require('../lib/api');
75

86
const command = 'status [jobId..]';
@@ -18,7 +16,7 @@ module.exports = {
1816
.default('config',api.defaultConfigPath),
1917
handler: argv =>
2018
api.loadConfig(argv.config).then(config =>
21-
request({
19+
requestAndRetry({
2220
url: config.server + '/job/status',
2321
proxy: config.proxy,
2422
headers: {
@@ -30,10 +28,15 @@ module.exports = {
3028
ids: JSON.stringify(argv.jobId)
3129
},
3230
gzip: true
33-
}).then(response =>
34-
JSON.parse(response.body).forEach(job =>
35-
console.log(`${job.id} -- ${job.status}`)
36-
)
37-
)
31+
}, (response) => {
32+
try {
33+
JSON.parse(response.body).forEach(job =>
34+
console.log(`${job.id} -- ${job.status}`)
35+
)
36+
} catch(e) {
37+
console.log(`Failed to get job status: ${response.body}`)
38+
throw e;
39+
}
40+
})
3841
)
3942
};

‎lib/retrier.js‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
'use strict';
2+
3+
const Promise = require('bluebird');
4+
const request = Promise.promisify(require('request'));
5+
6+
const REQUEST_RETRY_COUNT = 5;
7+
const RETRY_TIMEOUT_MS = 5000;
8+
const REQUEST_TIMEOUT_MS = 60000;
9+
10+
function rejectDelay() {
11+
return new Promise(function(resolve, reject) {
12+
setTimeout(resolve, RETRY_TIMEOUT_MS);
13+
})
14+
}
15+
16+
function requestAndRetry(requestParams, responseHandler = (response) => response, retryCount = 0){
17+
requestParams.timeout = REQUEST_TIMEOUT_MS;
18+
return request(requestParams).then(responseHandler).catch((err) => {
19+
if(retryCount < REQUEST_RETRY_COUNT){
20+
console.log(`Could not complete last request. Retrying: ${retryCount+1}/${REQUEST_RETRY_COUNT}`)
21+
return rejectDelay().then(() => requestAndRetry(requestParams, responseHandler, ++retryCount));
22+
} else {
23+
console.log("Could not complete action.");
24+
throw err
25+
}
26+
});
27+
}
28+
29+
module.exports = {
30+
requestAndRetry
31+
};

0 commit comments

Comments
 (0)