@@ -10,7 +10,7 @@ import querystring from 'node:querystring'
10
10
11
11
import HttpsProxyAgent from 'https-proxy-agent'
12
12
13
- import { isUndefined } from './utils'
13
+ import { isObject , isUndefined } from './utils'
14
14
15
15
import { AnyObject } from './types/common'
16
16
import { LoaderCrawlDataDetail , LoaderCrawlFileDetail } from './api'
@@ -22,21 +22,30 @@ export interface Request {
22
22
data : Buffer
23
23
}
24
24
25
+ interface ContentConfig {
26
+ protocol : 'http:' | 'https:'
27
+ data : string | undefined
28
+
29
+ requestConfig : RequestOptions
30
+ }
31
+
25
32
function parseHeaders (
26
- rawConfig : LoaderCrawlDataDetail & LoaderCrawlFileDetail ,
27
- config : RequestOptions
33
+ rawRequestConfig : LoaderCrawlDataDetail & LoaderCrawlFileDetail ,
34
+ contentConfig : ContentConfig
28
35
) {
29
- const rawHeaders = rawConfig . headers ?? { }
36
+ const rawHeaders = rawRequestConfig . headers ?? { }
37
+ const { requestConfig, data } = contentConfig
38
+
30
39
const headers : AnyObject = {
31
40
'User-Agent' :
32
41
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36' ,
33
42
...rawHeaders
34
43
}
35
44
36
- if ( config . method === 'POST' && rawConfig . data ) {
45
+ if ( ! isUndefined ( data ) ) {
37
46
const defaultHeaderConfig = [
38
47
{ key : 'Content-Type' , value : 'application/json' } ,
39
- { key : 'Content-Length' , value : Buffer . byteLength ( rawConfig . data ) }
48
+ { key : 'Content-Length' , value : Buffer . byteLength ( data ) }
40
49
]
41
50
42
51
defaultHeaderConfig . forEach ( ( item ) => {
@@ -48,56 +57,54 @@ function parseHeaders(
48
57
} )
49
58
}
50
59
51
- return headers
60
+ requestConfig . headers = headers
52
61
}
53
62
54
- function handleRequestConfig (
55
- rawConfig : LoaderCrawlDataDetail & LoaderCrawlFileDetail
56
- ) : RequestOptions {
57
- const { protocol, hostname, port, pathname, search } = new Url . URL (
58
- rawConfig . url
59
- )
60
- const isHttp = protocol === 'http:'
63
+ function createContentConfig (
64
+ rawRequestConfig : LoaderCrawlDataDetail & LoaderCrawlFileDetail
65
+ ) : ContentConfig {
66
+ const { data : rawData , url, params, proxyUrl } = rawRequestConfig
67
+ const { protocol, hostname, port, pathname, search } = new Url . URL ( url )
61
68
62
69
let path = pathname
63
- if ( search || rawConfig . params ) {
70
+ if ( search || params ) {
64
71
if ( search ) {
65
- path += `${ search } ${
66
- rawConfig . params ? '&' + querystring . stringify ( rawConfig . params ) : ''
67
- } `
72
+ path += `${ search } ${ params ? '&' + querystring . stringify ( params ) : '' } `
68
73
} else {
69
- path += `?${ querystring . stringify ( rawConfig . params ) } `
74
+ path += `?${ querystring . stringify ( params ) } `
70
75
}
71
76
}
72
77
73
- const config : RequestOptions = {
74
- agent : rawConfig . proxyUrl
75
- ? HttpsProxyAgent ( rawConfig . proxyUrl )
76
- : isHttp
77
- ? new http . Agent ( )
78
- : new https . Agent ( ) ,
79
-
80
- protocol,
81
- hostname,
82
- port,
83
- path,
84
-
85
- method : rawConfig . method ?. toLocaleUpperCase ( ) ?? 'GET' ,
86
- headers : { } ,
87
- timeout : rawConfig . timeout
78
+ const contentConfig : ContentConfig = {
79
+ requestConfig : {
80
+ agent : proxyUrl
81
+ ? HttpsProxyAgent ( proxyUrl )
82
+ : protocol === 'http:'
83
+ ? new http . Agent ( )
84
+ : new https . Agent ( ) ,
85
+
86
+ protocol,
87
+ hostname,
88
+ port,
89
+ path,
90
+
91
+ method : rawRequestConfig . method ?. toLocaleUpperCase ( ) ?? 'GET' ,
92
+ headers : { } ,
93
+ timeout : rawRequestConfig . timeout
94
+ } ,
95
+
96
+ protocol : protocol as 'http:' | 'https:' ,
97
+ data : isObject ( rawData ) ? JSON . stringify ( rawData ) : rawData
88
98
}
89
99
90
- config . headers = parseHeaders ( rawConfig , config )
100
+ parseHeaders ( rawRequestConfig , contentConfig )
91
101
92
- return config
102
+ return contentConfig
93
103
}
94
104
95
105
export function request ( config : LoaderCrawlDataDetail & LoaderCrawlFileDetail ) {
96
106
return new Promise < Request > ( ( resolve , reject ) => {
97
- const isDataUndefine = isUndefined ( config . data )
98
- config . data = ! isDataUndefine ? JSON . stringify ( config . data ) : config . data
99
-
100
- const requestConfig = handleRequestConfig ( config )
107
+ const { requestConfig, protocol, data } = createContentConfig ( config )
101
108
102
109
function handleRes ( res : IncomingMessage ) {
103
110
const { statusCode, headers } = res
@@ -118,12 +125,10 @@ export function request(config: LoaderCrawlDataDetail & LoaderCrawlFileDetail) {
118
125
} )
119
126
}
120
127
121
- let req : ClientRequest
122
- if ( requestConfig . protocol === 'http:' ) {
123
- req = http . request ( requestConfig , handleRes )
124
- } else {
125
- req = https . request ( requestConfig , handleRes )
126
- }
128
+ const req : ClientRequest =
129
+ protocol === 'http:'
130
+ ? http . request ( requestConfig , handleRes )
131
+ : https . request ( requestConfig , handleRes )
127
132
128
133
req . on ( 'timeout' , ( ) => {
129
134
reject ( new Error ( `Timeout ${ config . timeout } ms` ) )
@@ -134,7 +139,7 @@ export function request(config: LoaderCrawlDataDetail & LoaderCrawlFileDetail) {
134
139
} )
135
140
136
141
// 其他处理
137
- if ( requestConfig . method === 'POST' && ! isDataUndefine ) {
142
+ if ( ! isUndefined ( data ) ) {
138
143
req . write ( config . data )
139
144
}
140
145
0 commit comments