Skip to content

Commit db721dc

Browse files
Benedikt Rötschaxe312ger
Benedikt Rötsch
authored and
axe312ger
committed
feat(task-listing): implement listr for a proper overview of all export tasks
1 parent baf570c commit db721dc

File tree

5 files changed

+693
-566
lines changed

5 files changed

+693
-566
lines changed

‎bin/contentful-export‎

Lines changed: 1 addition & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,12 @@
11
#!/usr/bin/env node
2-
var log = require('npmlog')
32
var runContentfulExport = require('../dist/index')
43
var usageParams = require('../dist/usageParams')
54

6-
// welcome the user and let them know what's gonna happen
7-
log.info('Contentful Export Tool')
8-
log.info('Fetching Space data ...')
9-
105
runContentfulExport(usageParams)
116
.then((result) => {
127
process.exit(0)
138
})
149
.catch((err) => {
15-
log.error('export', 'Failed with\n', err)
10+
console.error(err)
1611
process.exit(1)
1712
})

‎lib/index.js‎

Lines changed: 115 additions & 87 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,9 @@ import getFullSourceSpace from 'contentful-batch-libs/dist/get/get-full-source-s
66
import Promise from 'bluebird'
77
import jsonStringifySafe from 'json-stringify-safe'
88
import log from 'npmlog'
9+
import Listr from 'listr'
10+
import UpdateRenderer from 'listr-update-renderer'
11+
import VerboseRenderer from 'listr-verbose-renderer'
912
import { resolve } from 'path'
1013
import { startCase } from 'lodash'
1114
import Table from 'cli-table2'
@@ -22,7 +25,8 @@ export default function runContentfulExport (usageParams) {
2225
skipContent: false,
2326
skipWebhooks: false,
2427
maxAllowedLimit: 1000,
25-
saveFile: true
28+
saveFile: true,
29+
useVerboseRenderer: false
2630
}
2731

2832
summary.startTime = moment()
@@ -52,104 +56,128 @@ export default function runContentfulExport (usageParams) {
5256
opts.errorLogFile = opts.exportDir + '/contentful-export-' + Date.now() + '.log'
5357
}
5458

55-
const clients = createClients(opts)
56-
return getFullSourceSpace({
57-
managementClient: clients.source.management,
58-
spaceId: clients.source.spaceId,
59-
maxAllowedLimit: opts.maxAllowedLimit,
60-
includeDrafts: opts.includeDrafts,
61-
skipContentModel: opts.skipContentModel,
62-
skipContent: opts.skipContent,
63-
skipWebhooks: opts.skipWebhooks,
64-
skipRoles: opts.skipRoles
65-
})
66-
.then((response) => {
67-
if (!opts.downloadAssets) {
68-
return response
59+
const listrOptions = opts.useVerboseRenderer
60+
? {
61+
renderer: VerboseRenderer
62+
}
63+
: {
64+
renderer: UpdateRenderer,
65+
collapse: false
66+
}
67+
68+
const tasks = new Listr([
69+
{
70+
title: 'Initialize clients',
71+
task: (ctx) => {
72+
ctx.clients = createClients(opts)
6973
}
70-
summary.assetDownloads = {
71-
successCount: 0,
72-
warningCount: 0,
73-
errorCount: 0
74+
},
75+
{
76+
title: 'Fetching data from space',
77+
task: (ctx) => {
78+
return getFullSourceSpace({
79+
managementClient: ctx.clients.source.management,
80+
spaceId: ctx.clients.source.spaceId,
81+
maxAllowedLimit: opts.maxAllowedLimit,
82+
includeDrafts: opts.includeDrafts,
83+
skipContentModel: opts.skipContentModel,
84+
skipContent: opts.skipContent,
85+
skipWebhooks: opts.skipWebhooks,
86+
skipRoles: opts.skipRoles,
87+
listrOptions
88+
})
7489
}
75-
76-
log.info('export', 'Downloading ' + response.assets.length + ' assets')
77-
78-
return Promise.map(response.assets, (asset) => {
79-
if (!asset.fields.file) {
80-
log.warn('export', '-> asset has no file(s)', jsonStringifySafe(asset))
81-
summary.assetDownloads.warningCount++
82-
return
83-
}
84-
const locales = Object.keys(asset.fields.file)
85-
return Promise.mapSeries(locales, (locale) => {
86-
const url = asset.fields.file[locale].url || asset.fields.file[locale].upload
87-
if (!url) {
88-
log.warn('export', '-> asset no file(s) for locale', locale, jsonStringifySafe(asset))
89-
summary.assetDownloads.warningCount++
90+
},
91+
{
92+
title: 'Download assets',
93+
task: (ctx) => {
94+
let successCount = 0
95+
let warningCount = 0
96+
let errorCount = 0
97+
98+
log.info('export', `Downloading ${ctx.data.assets.length} assets`)
99+
100+
return Promise.map(ctx.data.assets, (asset) => {
101+
if (!asset.fields.file) {
102+
log.warn('-> asset has no file(s)', jsonStringifySafe(asset))
103+
warningCount++
90104
return
91105
}
92-
return downloadAsset(url, opts.exportDir)
93-
.then((downLoadedFile) => {
94-
log.info('export', '-> ' + downLoadedFile)
95-
summary.assetDownloads.successCount++
96-
})
97-
.catch((error) => {
98-
log.error('export', '-> error downloading ' + url + ' => ' + error.message)
99-
log.error('export', JSON.stringify(error, null, 2))
100-
summary.assetDownloads.errorCount++
106+
const locales = Object.keys(asset.fields.file)
107+
return Promise.mapSeries(locales, (locale) => {
108+
const url = asset.fields.file[locale].url || asset.fields.file[locale].upload
109+
return downloadAsset(url, opts.exportDir)
110+
.then((downLoadedFile) => {
111+
log.info('export', '-> ' + downLoadedFile)
112+
successCount++
113+
})
114+
.catch((error) => {
115+
log.error('export', '-> error downloading ' + url + ' => ' + error.message)
116+
log.error('export', JSON.stringify(error, null, 2))
117+
errorCount++
118+
})
101119
})
120+
}, {
121+
concurrency: 6
102122
})
103-
}, {
104-
concurrency: 6
105-
})
106-
.then(() => {
107-
log.info('export', 'Finished loading files for ' + response.assets.length + ' assets.')
108-
return response
109-
})
110-
})
111-
.then((response) => {
112-
if (opts.saveFile) {
123+
.then(() => {
124+
ctx.assetDownloads = {
125+
successCount,
126+
warningCount,
127+
errorCount
128+
}
129+
})
130+
},
131+
skip: () => !opts.downloadAssets
132+
},
133+
{
134+
title: 'Writing data to file',
135+
task: (ctx) => {
113136
fs.existsSync(opts.exportDir) || fs.mkdirSync(opts.exportDir)
114-
const responseFile = `${opts.exportDir}/contentful-export-${clients.source.spaceId}-${Date.now()}.json`
115-
log.info('export', 'Writing space data to json file at : ' + responseFile)
116-
fs.writeFileSync(responseFile, jsonStringifySafe(response, null, 4))
117-
}
118-
return response
119-
})
120-
.then((response) => {
121-
const responseTable = new Table()
137+
ctx.responseFile = `${opts.exportDir}/contentful-export-${ctx.clients.source.spaceId}-${Date.now()}.json`
138+
return fs.writeFileSync(ctx.responseFile, jsonStringifySafe(ctx.data, null, 4))
139+
},
140+
skip: () => !opts.saveFile
141+
}
142+
], listrOptions)
143+
144+
return tasks.run({
145+
data: {}
146+
})
147+
.then((ctx) => {
148+
const responseTable = new Table()
122149

123-
responseTable.push([{colSpan: 2, content: 'Exported entities'}])
150+
responseTable.push([{colSpan: 2, content: 'Exported entities'}])
124151

125-
Object.keys(response).forEach((type) => {
126-
responseTable.push([startCase(type), response[type].length])
127-
})
152+
Object.keys(ctx.data).forEach((type) => {
153+
responseTable.push([startCase(type), ctx.data[type].length])
154+
})
128155

129-
console.log(responseTable.toString())
156+
console.log(responseTable.toString())
130157

131-
if ('assetDownloads' in summary) {
132-
const downloadsTable = new Table()
133-
downloadsTable.push([{colSpan: 2, content: 'Asset file download results'}])
134-
downloadsTable.push(['Successful', summary.assetDownloads.successCount])
135-
downloadsTable.push(['Warnings ', summary.assetDownloads.warningCount])
136-
downloadsTable.push(['Errors ', summary.assetDownloads.errorCount])
137-
console.log(downloadsTable.toString())
138-
}
158+
if ('assetDownloads' in summary) {
159+
const downloadsTable = new Table()
160+
downloadsTable.push([{colSpan: 2, content: 'Asset file download results'}])
161+
downloadsTable.push(['Successful', ctx.assetDownloads.successCount])
162+
downloadsTable.push(['Warnings ', ctx.assetDownloads.warningCount])
163+
downloadsTable.push(['Errors ', ctx.assetDownloads.errorCount])
164+
console.log(downloadsTable.toString())
165+
}
139166

140-
const durationHuman = summary.startTime.fromNow(true)
141-
const durationSeconds = moment().diff(summary.startTime, 'seconds')
167+
const durationHuman = summary.startTime.fromNow(true)
168+
const durationSeconds = moment().diff(summary.startTime, 'seconds')
142169

143-
log.info('export', `The export took ${durationHuman} (${durationSeconds}s)`)
144-
145-
return response
146-
})
147-
.catch((err) => {
148-
const { sourceSpace, errorLogFile } = opts
149-
dumpErrorBuffer({
150-
sourceSpace,
151-
errorLogFile
152-
})
153-
throw err
170+
log.info('export', `The export took ${durationHuman} (${durationSeconds}s)`)
171+
if (opts.saveFile) {
172+
log.info('export', `Stored space data to json file at: ${ctx.responseFile}`)
173+
}
174+
})
175+
.catch((err) => {
176+
const { sourceSpace, errorLogFile } = opts
177+
dumpErrorBuffer({
178+
sourceSpace,
179+
errorLogFile
154180
})
181+
throw err
182+
})
155183
}

‎lib/usageParams.js‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,11 @@ export default yargs
6767
type: 'boolean',
6868
default: true
6969
})
70+
.option('use-verbose-renderer', {
71+
describe: 'Full path to the error log file',
72+
type: 'boolean',
73+
default: false
74+
})
7075
.config('config', 'An optional configuration JSON file containing all the options for a single run')
7176
.check(function (argv) {
7277
if (!argv.spaceId) {

‎package.json‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,9 @@
5353
"contentful-batch-libs": "^5.3.0",
5454
"fs-extra": "^2.1.2",
5555
"json-stringify-safe": "^5.0.1",
56+
"listr": "^0.11.0",
57+
"listr-update-renderer": "^0.2.0",
58+
"listr-verbose-renderer": "^0.4.0",
5659
"lodash": "^4.0.0",
5760
"moment": "^2.18.1",
5861
"npmlog": "^4.0.0",

0 commit comments

Comments
 (0)