Add download files logic & refactor

Author: thedoorbell

src/downloadFiles.js

@@ -0,0 +1,37 @@
+import path from 'path'
+import fsp from 'fs/promises'
+import getData from './getData.js'
+import generateName from './generateName.js'
+
+const downloadFiles = ($, url, filesDirPath, selector, attr) => {
+  const links = $(selector).map((i, tag) => $(tag).attr(attr)).get()
+
+  if (links.length === 0) {
+    return Promise.resolve()
+  }
+
+  return fsp.mkdir(filesDirPath, { recursive: true })
+    .then(() => Promise.all(links.map((link) => {
+      const absLink = new URL(link, url)
+      if (absLink.hostname !== new URL(url).hostname) {
+        return Promise.resolve()
+      }
+      const fileName = path.extname(absLink.pathname)
+        ? generateName(absLink.href).replace(/-(?=[a-zA-Z0-9]+$)/, '.')
+        : generateName(absLink.href) + '.html'
+      const filePath = path.join(filesDirPath, fileName)
+
+      return getData(absLink.href)
+        .then(data => fsp.writeFile(filePath, data))
+        .then(() => {
+          $(selector).each((i, tag) => {
+            if ($(tag).attr(attr) === link) {
+              $(tag).attr(attr, path.join(path.basename(filesDirPath), fileName))
+            }
+          })
+        })
+    }),
+    ))
+}
+
+export default downloadFiles

src/generateName.js

@@ -0,0 +1,6 @@
+const generateName = (url) => {
+  const href = new URL(url)
+  return (href.hostname + href.pathname).replace(/[^a-zA-Z0-9]/g, '-')
+}
+
+export default generateName

src/getData.js

@@ -1,12 +1,7 @@
 import axios from 'axios'
-import path from 'path'
 
-const getData = (url) => {
-  const imgExtenstions = ['.png', '.jpg', '.jpeg', '.svg']
-  if (imgExtenstions.includes(path.extname(url).toLowerCase())) {
-    return axios.get(url, { responseType: 'arraybuffer' }).then(response => response.data)
-  }
-  return axios.get(url).then(response => response.data)
-}
+const getData = url => axios
+  .get(url, { responseType: 'arraybuffer' })
+  .then(response => response.data)
 
 export default getData

src/getLinks.js

@@ -1,55 +1,29 @@
 import path from 'path'
 import fsp from 'fs/promises'
 import * as cheerio from 'cheerio'
-import getData from './getData.js'
-import getLinks from './getLinks.js'
 import * as prettier from 'prettier'
+import getData from './getData.js'
+import downloadFiles from './downloadFiles.js'
+import generateName from './generateName.js'
 
 const getAbsolutePath = dirpath => path.resolve(process.cwd(), dirpath)
 
-const generateName = (url) => {
-  const href = new URL(url)
-  return (href.hostname + href.pathname).replace(/[^a-zA-Z0-9]/g, '-')
-}
-
 const downloadPage = (url, outputDir) => {
   const fileName = generateName(url) + '.html'
   const filePath = path.join(getAbsolutePath(outputDir), fileName)
   const filesDirName = generateName(url) + '_files'
   const filesDirPath = path.join(getAbsolutePath(outputDir), filesDirName)
-  let links
   let $
 
   return getData(url)
     .then((html) => {
       $ = cheerio.load(html)
-      links = getLinks(html)
-
-      if (links.length === 0) {
-        return null
-      }
-      return fsp.mkdir(filesDirPath)
-    })
-    .then(() => {
-      if (links.length === 0) {
-        return null
-      }
-      return Promise.all(links.map((link) => {
-        const absoluteLink = new URL(link, url).href
-        const imageName = generateName(absoluteLink).replace(/-(?=[a-zA-Z0-9]+$)/, '.')
-        const imagePath = path.join(filesDirPath, imageName)
 
-        return getData(absoluteLink)
-          .then(imgData => fsp.writeFile(imagePath, imgData))
-          .then(() => {
-            $('img').each((i, tag) => {
-              if ($(tag).attr('src') === link) {
-                $(tag).attr('src', path.join(filesDirName, imageName))
-              }
-            })
-          })
-      }),
-      )
+      return Promise.all([
+        downloadFiles($, url, filesDirPath, 'img', 'src'),
+        downloadFiles($, url, filesDirPath, 'link', 'href'),
+        downloadFiles($, url, filesDirPath, 'script', 'src'),
+      ])
     })
     .then(() => prettier.format($.html(), { parser: 'html' }))
     .then(formattedHtml => fsp.writeFile(filePath, formattedHtml))