Add concurrent files downloading using Listr

Author: thedoorbell

bin/page-loader.js

@@ -12,7 +12,7 @@ program
   .action((url, options) => {
     downloadPage(url, options.output)
       .then((filePath) => {
-        console.log(filePath)
+        console.log(`Page was successfully downloaded into ${filePath}`)
         process.exit(0)
       })
       .catch((err) => {

src/downloadFiles.js

@@ -3,9 +3,55 @@ import fsp from 'fs/promises'
 import debug from 'debug'
 import getData from './getData.js'
 import generateName from './generateName.js'
+import Listr from 'listr'
 
 const log = debug('page-loader')
 
+const tasks = ($, links, url, filesDirPath, selector, attr) => new Listr(links
+  .filter((link) => {
+    const absLink = new URL(link, url)
+    if (absLink.hostname === new URL(url).hostname) {
+      return link
+    }
+  })
+  .map((link) => {
+    const absLink = new URL(link, url)
+    return {
+      title: `${absLink.href}`,
+      task: () => downloadFile($, link, absLink, filesDirPath, selector, attr),
+    }
+  }), {
+  concurrent: true,
+  exitOnError: false,
+})
+
+const downloadFile = ($, link, absLink, filesDirPath, selector, attr) => {
+  const fileName = path.extname(absLink.pathname)
+    ? generateName(absLink.href).replace(/-(?=[a-zA-Z0-9]+$)/, '.')
+    : generateName(absLink.href) + '.html'
+  const filePath = path.join(filesDirPath, fileName)
+
+  log('download file %s', absLink.href)
+  return getData(absLink.href)
+    .then((data) => {
+      log('write to file %s', filePath)
+      return fsp.writeFile(filePath, data)
+    })
+    .then(() => {
+      const newLink = path.join(path.basename(filesDirPath), fileName)
+      log('change link from %s to %s in HTML', link, newLink)
+      $(selector).each((i, tag) => {
+        if ($(tag).attr(attr) === link) {
+          $(tag).attr(attr, newLink)
+        }
+      })
+    })
+    .catch((err) => {
+      log('failed to download %s: %s', absLink.href, err.message)
+      throw new Error(`Cant load file ${absLink.href}`)
+    })
+}
+
 const downloadFiles = ($, url, filesDirPath, selector, attr) => {
   const links = $(selector).map((i, tag) => $(tag).attr(attr)).get()
 
@@ -16,38 +62,11 @@ const downloadFiles = ($, url, filesDirPath, selector, attr) => {
 
   log(`download %ss`, selector)
   return fsp.mkdir(filesDirPath, { recursive: true })
-    .then(() => Promise.all(links.map((link) => {
-      const absLink = new URL(link, url)
-      if (absLink.hostname !== new URL(url).hostname) {
-        log('skip external link %s', absLink.href)
-        return Promise.resolve()
-      }
-      const fileName = path.extname(absLink.pathname)
-        ? generateName(absLink.href).replace(/-(?=[a-zA-Z0-9]+$)/, '.')
-        : generateName(absLink.href) + '.html'
-      const filePath = path.join(filesDirPath, fileName)
-
-      log('download file %s', absLink.href)
-      return getData(absLink.href)
-        .then((data) => {
-          log('write to file %s', filePath)
-          return fsp.writeFile(filePath, data)
-        })
-        .then(() => {
-          const newLink = path.join(path.basename(filesDirPath), fileName)
-          log('change link from %s to %s in HTML', link, newLink)
-          $(selector).each((i, tag) => {
-            if ($(tag).attr(attr) === link) {
-              $(tag).attr(attr, newLink)
-            }
-          })
-        })
-        .catch((err) => {
-          log('failed to download %s: %s', absLink.href, err.message)
-          throw new Error(`Cant load file ${absLink.href}`)
-        })
-    }),
-    ))
+    .then(() => tasks($, links, url, filesDirPath, selector, attr)
+      .run().catch((err) => {
+        log('failed to download files: %s', err.message)
+        throw err
+      }))
 }
 
 export default downloadFiles

src/index.js

@@ -6,6 +6,7 @@ import debug from 'debug'
 import getData from './getData.js'
 import downloadFiles from './downloadFiles.js'
 import generateName from './generateName.js'
+// import Listr from 'listr'
 
 const log = debug('page-loader')