Added the method of getting the request result through the callback function

Author: coder-hxl

src/index.ts

@@ -21,12 +21,13 @@ import {
   IFetchFileConfig,
   IFetchPollingConfig,
   IFetchBaseConifg,
-  IFetchCommon,
   IFileInfo,
   IFetchHTML,
   IRequestResItem,
   IRequestConfig,
-  IIntervalTime
+  IIntervalTime,
+  IFetchCommon,
+  IFetchCommonArr
 } from './types'
 
 export default class XCrawl {
@@ -73,23 +74,24 @@ export default class XCrawl {
 
   private async useBatchRequestByMode(
     requestConifg: IRequestConfig | IRequestConfig[],
-    intervalTime: IIntervalTime | undefined
+    intervalTime: IIntervalTime | undefined,
+    callback: (requestResItem: IRequestResItem) => void
   ) {
     const requestConfigQueue = isArray(requestConifg)
       ? requestConifg
       : [requestConifg]
 
-    let requestRes: IRequestResItem[] = []
     if (this.baseConfig.mode !== 'sync') {
-      requestRes = await batchRequest(requestConfigQueue, intervalTime)
+      await batchRequest(requestConfigQueue, intervalTime, callback)
     } else {
-      requestRes = await syncBatchRequest(requestConfigQueue, intervalTime)
+      await syncBatchRequest(requestConfigQueue, intervalTime, callback)
     }
-
-    return requestRes
   }
 
-  async fetchHTML(config: IFetchHTMLConfig): Promise<IFetchHTML> {
+  async fetchHTML(
+    config: IFetchHTMLConfig,
+    callback?: (res: IFetchHTML) => void
+  ): Promise<IFetchHTML> {
     const { requestConifg } = this.mergeConfig({
       requestConifg: isString(config) ? { url: config } : config
     })
@@ -105,44 +107,50 @@ export default class XCrawl {
       }
     }
 
+    if (callback) {
+      callback(res)
+    }
+
     return res
   }
 
-  async fetchData<T = any>(config: IFetchDataConfig): Promise<IFetchCommon<T>> {
+  async fetchData<T = any>(
+    config: IFetchDataConfig,
+    callback?: (res: IFetchCommon<T>) => void
+  ): Promise<IFetchCommonArr<T>> {
     const { requestConifg, intervalTime } = this.mergeConfig(config)
 
-    const requestRes = await this.useBatchRequestByMode(
-      requestConifg,
-      intervalTime
-    )
-
-    const container: IFetchCommon<T> = []
-
-    requestRes.forEach((item) => {
-      const contentType = item.headers['content-type'] ?? ''
-      const rawData = item.data
+    const container: IFetchCommonArr<T> = []
+    function handleResItem(requestResItem: IRequestResItem) {
+      const contentType = requestResItem.headers['content-type'] ?? ''
+      const rawData = requestResItem.data
 
       const data = contentType.includes('text')
         ? rawData.toString()
         : JSON.parse(rawData.toString())
 
-      container.push({ ...item, data })
-    })
+      const itemRes = { ...requestResItem, data }
+
+      if (callback) {
+        callback(itemRes)
+      }
+
+      container.push(itemRes)
+    }
+
+    await this.useBatchRequestByMode(requestConifg, intervalTime, handleResItem)
 
     return container
   }
 
-  async fetchFile(config: IFetchFileConfig): Promise<IFetchCommon<IFileInfo>> {
+  async fetchFile(
+    config: IFetchFileConfig,
+    callback?: (res: IFetchCommon<IFileInfo>) => void
+  ): Promise<IFetchCommonArr<IFileInfo>> {
     const { requestConifg, intervalTime, fileConfig } = this.mergeConfig(config)
 
-    const requestRes = await this.useBatchRequestByMode(
-      requestConifg,
-      intervalTime
-    )
-
-    const container: IFetchCommon<IFileInfo> = []
-
-    requestRes.forEach((requestResItem) => {
+    const container: IFetchCommonArr<IFileInfo> = []
+    function handleResItem(requestResItem: IRequestResItem) {
       const { id, headers, data } = requestResItem
 
       const mimeType = headers['content-type'] ?? ''
@@ -156,16 +164,24 @@ export default class XCrawl {
       try {
         fs.writeFileSync(filePath, data)
 
-        container.push({
+        const res = {
           ...requestResItem,
           data: { fileName, mimeType, size: data.length, filePath }
-        })
+        }
+
+        if (callback) {
+          callback(res)
+        }
+
+        container.push(res)
       } catch (error: any) {
         log(logError(`File save error at id ${id}: ${error.message}`))
       }
-    })
+    }
+
+    await this.useBatchRequestByMode(requestConifg, intervalTime, handleResItem)
 
-    const saveTotal = requestRes.length
+    const saveTotal = isArray(requestConifg) ? requestConifg.length : 1
     const success = container.length
     const error = saveTotal - success
     log(
@@ -188,12 +204,12 @@ export default class XCrawl {
     const total = year + month + day + hour + minute
 
     let count = 0
-    function cb() {
+    function startCallback() {
       console.log(logWarn(`Start the ${logWarn.bold(++count)} polling`))
       callback(count)
     }
 
-    cb()
-    setInterval(cb, total)
+    startCallback()
+    setInterval(startCallback, total)
   }
 }

src/request.ts

@@ -163,7 +163,8 @@ async function useSleepByBatch(
 
 export async function batchRequest(
   requestConifgs: IRequestConfig[],
-  intervalTime: IIntervalTime | undefined
+  intervalTime: IIntervalTime | undefined,
+  callback: (requestResItem: IRequestResItem) => void
 ) {
   const isHaveIntervalTime = !isUndefined(intervalTime)
   const isNumberIntervalTime = isNumber(intervalTime)
@@ -172,9 +173,10 @@ export async function batchRequest(
     `Begin execution, mode: async, total: ${logNumber(requestConifgs.length)} `
   )
 
-  const requestQueue: Promise<IRequestResItem | string>[] = []
-
   let index = 0
+  let successTotal = 0
+  let errorTotal = 0
+  const requestQueue: Promise<undefined | string>[] = []
   for (const requestConifg of requestConifgs) {
     const id = ++index
 
@@ -187,12 +189,14 @@ export async function batchRequest(
 
     const requestItem = request(requestConifg)
       .catch((error: any) => {
+        errorTotal++
         return `Request ${id} is an error: ${error.message}`
       })
       .then((requestRes) => {
         if (typeof requestRes === 'string') return requestRes
 
-        return { id, ...requestRes }
+        successTotal++
+        callback({ id, ...requestRes })
       })
 
     requestQueue.push(requestItem)
@@ -202,32 +206,20 @@ export async function batchRequest(
 
   const res = await Promise.all(requestQueue)
 
-  const success: IRequestResItem[] = []
-  const error: string[] = []
-
-  // 通过类型分类
-  res.forEach((item) => {
-    if (typeof item === 'string') {
-      return error.push(item)
-    }
-
-    success.push(item)
-  })
-
-  error.forEach((message) => log(logError(message)))
+  // 打印错误消息
+  res.forEach((item) => (item ? log(logError(item)) : ''))
 
   log(
     `requestsTotal: ${logNumber(requestConifgs.length)}, success: ${logSuccess(
-      success.length
-    )}, error: ${logError(error.length)}`
+      successTotal
+    )}, error: ${logError(errorTotal)}`
   )
-
-  return success
 }
 
 export async function syncBatchRequest(
   requestConifgs: IRequestConfig[],
-  intervalTime: IIntervalTime | undefined
+  intervalTime: IIntervalTime | undefined,
+  callback: (requestResItem: IRequestResItem) => void
 ) {
   const isHaveIntervalTime = !isUndefined(intervalTime)
   const isNumberIntervalTime = isNumber(intervalTime)
@@ -239,7 +231,6 @@ export async function syncBatchRequest(
   let id = 0
   let successTotal = 0
   let errorTotal = 0
-  const requestRes: IRequestResItem[] = []
   for (const requestConifg of requestConifgs) {
     id++
 
@@ -250,15 +241,22 @@ export async function syncBatchRequest(
       id
     )
 
+    let isRequestSuccess = true
+    let requestResItem: IRequestResItem | null = null
     try {
-      const requestResItem = await request(requestConifg)
-      requestRes.push({ id, ...requestResItem })
+      const requestRes = await request(requestConifg)
+      requestResItem = { id, ...requestRes }
       log(logSuccess(`Request ${logNumber(id)} is an success`))
       successTotal++
     } catch (error: any) {
+      isRequestSuccess = false
       log(logError(`Request ${id} is an error: ${error.message}`))
       errorTotal++
     }
+
+    if (isRequestSuccess && callback) {
+      callback(requestResItem as IRequestResItem)
+    }
   }
 
   log(logSuccess('All requests are over!'))
@@ -268,6 +266,4 @@ export async function syncBatchRequest(
       successTotal
     )}, error: ${logError(errorTotal)}`
   )
-
-  return requestRes
 }

src/types.ts

@@ -89,12 +89,14 @@ export interface IFetchPollingConfig {
   m?: number
 }
 
-export type IFetchCommon<T> = {
+export interface IFetchCommon<T> {
   id: number
   statusCode: number | undefined
   headers: IncomingHttpHeaders
   data: T
-}[]
+}
+
+export type IFetchCommonArr<T> = IFetchCommon<T>[]
 
 export interface IFileInfo {
   fileName: string