Skip to content

Commit 29bd7c2

Browse files
authored
Merge pull request #47 from coder-hxl/fix/46
fix(crawlData): data conversion process
2 parents 8fccbc7 + d513a16 commit 29bd7c2

File tree

1 file changed

+52
-47
lines changed

1 file changed

+52
-47
lines changed

src/request.ts

+52-47
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ import querystring from 'node:querystring'
1010

1111
import HttpsProxyAgent from 'https-proxy-agent'
1212

13-
import { isUndefined } from './utils'
13+
import { isObject, isUndefined } from './utils'
1414

1515
import { AnyObject } from './types/common'
1616
import { LoaderCrawlDataDetail, LoaderCrawlFileDetail } from './api'
@@ -22,21 +22,30 @@ export interface Request {
2222
data: Buffer
2323
}
2424

25+
interface ContentConfig {
26+
protocol: 'http:' | 'https:'
27+
data: string | undefined
28+
29+
requestConfig: RequestOptions
30+
}
31+
2532
function parseHeaders(
26-
rawConfig: LoaderCrawlDataDetail & LoaderCrawlFileDetail,
27-
config: RequestOptions
33+
rawRequestConfig: LoaderCrawlDataDetail & LoaderCrawlFileDetail,
34+
contentConfig: ContentConfig
2835
) {
29-
const rawHeaders = rawConfig.headers ?? {}
36+
const rawHeaders = rawRequestConfig.headers ?? {}
37+
const { requestConfig, data } = contentConfig
38+
3039
const headers: AnyObject = {
3140
'User-Agent':
3241
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36',
3342
...rawHeaders
3443
}
3544

36-
if (config.method === 'POST' && rawConfig.data) {
45+
if (!isUndefined(data)) {
3746
const defaultHeaderConfig = [
3847
{ key: 'Content-Type', value: 'application/json' },
39-
{ key: 'Content-Length', value: Buffer.byteLength(rawConfig.data) }
48+
{ key: 'Content-Length', value: Buffer.byteLength(data) }
4049
]
4150

4251
defaultHeaderConfig.forEach((item) => {
@@ -48,56 +57,54 @@ function parseHeaders(
4857
})
4958
}
5059

51-
return headers
60+
requestConfig.headers = headers
5261
}
5362

54-
function handleRequestConfig(
55-
rawConfig: LoaderCrawlDataDetail & LoaderCrawlFileDetail
56-
): RequestOptions {
57-
const { protocol, hostname, port, pathname, search } = new Url.URL(
58-
rawConfig.url
59-
)
60-
const isHttp = protocol === 'http:'
63+
function createContentConfig(
64+
rawRequestConfig: LoaderCrawlDataDetail & LoaderCrawlFileDetail
65+
): ContentConfig {
66+
const { data: rawData, url, params, proxyUrl } = rawRequestConfig
67+
const { protocol, hostname, port, pathname, search } = new Url.URL(url)
6168

6269
let path = pathname
63-
if (search || rawConfig.params) {
70+
if (search || params) {
6471
if (search) {
65-
path += `${search}${
66-
rawConfig.params ? '&' + querystring.stringify(rawConfig.params) : ''
67-
}`
72+
path += `${search}${params ? '&' + querystring.stringify(params) : ''}`
6873
} else {
69-
path += `?${querystring.stringify(rawConfig.params)}`
74+
path += `?${querystring.stringify(params)}`
7075
}
7176
}
7277

73-
const config: RequestOptions = {
74-
agent: rawConfig.proxyUrl
75-
? HttpsProxyAgent(rawConfig.proxyUrl)
76-
: isHttp
77-
? new http.Agent()
78-
: new https.Agent(),
79-
80-
protocol,
81-
hostname,
82-
port,
83-
path,
84-
85-
method: rawConfig.method?.toLocaleUpperCase() ?? 'GET',
86-
headers: {},
87-
timeout: rawConfig.timeout
78+
const contentConfig: ContentConfig = {
79+
requestConfig: {
80+
agent: proxyUrl
81+
? HttpsProxyAgent(proxyUrl)
82+
: protocol === 'http:'
83+
? new http.Agent()
84+
: new https.Agent(),
85+
86+
protocol,
87+
hostname,
88+
port,
89+
path,
90+
91+
method: rawRequestConfig.method?.toLocaleUpperCase() ?? 'GET',
92+
headers: {},
93+
timeout: rawRequestConfig.timeout
94+
},
95+
96+
protocol: protocol as 'http:' | 'https:',
97+
data: isObject(rawData) ? JSON.stringify(rawData) : rawData
8898
}
8999

90-
config.headers = parseHeaders(rawConfig, config)
100+
parseHeaders(rawRequestConfig, contentConfig)
91101

92-
return config
102+
return contentConfig
93103
}
94104

95105
export function request(config: LoaderCrawlDataDetail & LoaderCrawlFileDetail) {
96106
return new Promise<Request>((resolve, reject) => {
97-
const isDataUndefine = isUndefined(config.data)
98-
config.data = !isDataUndefine ? JSON.stringify(config.data) : config.data
99-
100-
const requestConfig = handleRequestConfig(config)
107+
const { requestConfig, protocol, data } = createContentConfig(config)
101108

102109
function handleRes(res: IncomingMessage) {
103110
const { statusCode, headers } = res
@@ -118,12 +125,10 @@ export function request(config: LoaderCrawlDataDetail & LoaderCrawlFileDetail) {
118125
})
119126
}
120127

121-
let req: ClientRequest
122-
if (requestConfig.protocol === 'http:') {
123-
req = http.request(requestConfig, handleRes)
124-
} else {
125-
req = https.request(requestConfig, handleRes)
126-
}
128+
const req: ClientRequest =
129+
protocol === 'http:'
130+
? http.request(requestConfig, handleRes)
131+
: https.request(requestConfig, handleRes)
127132

128133
req.on('timeout', () => {
129134
reject(new Error(`Timeout ${config.timeout}ms`))
@@ -134,7 +139,7 @@ export function request(config: LoaderCrawlDataDetail & LoaderCrawlFileDetail) {
134139
})
135140

136141
// 其他处理
137-
if (requestConfig.method === 'POST' && !isDataUndefine) {
142+
if (!isUndefined(data)) {
138143
req.write(config.data)
139144
}
140145

0 commit comments

Comments
 (0)