Skip to content

Commit f843c9f

Browse files
coder-hxlcoder-hxl
committed
other
1 parent d2409e6 commit f843c9f

File tree

7 files changed

+160
-77
lines changed

7 files changed

+160
-77
lines changed

‎README.md

+44-11
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,8 @@ XCrawl is a Nodejs multifunctional crawler library.
88

99
- Crawl HTML, JSON, file resources, etc. with simple configuration
1010
- Use the JSDOM library to parse HTML, or parse HTML by yourself
11-
- Optional mode asynchronous/synchronous for batch requests
11+
- The request method supports asynchronous/synchronous
12+
- Support Promise/Callback
1213
- Polling function
1314
- Anthropomorphic request interval
1415
- Written in TypeScript
@@ -47,6 +48,7 @@ XCrawl is a Nodejs multifunctional crawler library.
4748
* [IFetchFileConfig](#IFetchFileConfig)
4849
* [IFetchPollingConfig](#IFetchPollingConfig)
4950
* [IFetchCommon](#IFetchCommon)
51+
* [IFetchCommonArr](#IFetchCommonArr)
5052
* [IFileInfo](#IFileInfo)
5153
* [IFetchHTML](#IFetchHTML)
5254
- [More](#More)
@@ -92,10 +94,26 @@ Create a crawler instance via new XCrawl. The request queue is maintained by the
9294
```ts
9395
class XCrawl {
9496
constructor(baseConfig?: IXCrawlBaseConifg)
95-
fetchHTML(config: IFetchHTMLConfig): Promise<IFetchHTML>
96-
fetchData<T = any>(config: IFetchDataConfig): Promise<IFetchCommon<T>>
97-
fetchFile(config: IFetchFileConfig): Promise<IFetchCommon<IFileInfo>>
98-
fetchPolling(config: IFetchPollingConfig, callback: (count: number) => void): void
97+
98+
fetchHTML(
99+
config: IFetchHTMLConfig,
100+
callback?: (res: IFetchHTML) => void
101+
): Promise<IFetchHTML>
102+
103+
fetchData<T = any>(
104+
config: IFetchDataConfig,
105+
callback?: (res: IFetchCommon<T>) => void
106+
): Promise<IFetchCommonArr<T>>
107+
108+
fetchFile(
109+
config: IFetchFileConfig,
110+
callback?: (res: IFetchCommon<IFileInfo>) => void
111+
): Promise<IFetchCommonArr<IFileInfo>>
112+
113+
fetchPolling(
114+
config: IFetchPollingConfig,
115+
callback: (count: number) => void
116+
): void
99117
}
100118
```
101119
@@ -142,7 +160,10 @@ fetchHTML is the method of the above [myXCrawl](https://github.com/coder-hxl/x-c
142160
#### Type
143161
144162
```ts
145-
function fetchHTML(config: IFetchHTMLConfig): Promise<IFetchHTML>
163+
fetchHTML(
164+
config: IFetchHTMLConfig,
165+
callback?: (res: IFetchHTML) => void
166+
): Promise<IFetchHTML>
146167
```
147168
148169
#### Example
@@ -161,7 +182,10 @@ fetchData is the method of the above [myXCrawl](#Example-1) instance, which is u
161182
#### Type
162183
163184
```ts
164-
function fetchData<T = any>(config: IFetchDataConfig): Promise<IFetchCommon<T>>
185+
fetchData<T = any>(
186+
config: IFetchDataConfig,
187+
callback?: (res: IFetchCommon<T>) => void
188+
): Promise<IFetchCommonArr<T>>
165189
```
166190
167191
#### Example
@@ -188,7 +212,10 @@ fetchFile is the method of the above [myXCrawl](#Example-1) instance, which is u
188212
#### Type
189213
190214
```ts
191-
function fetchFile(config: IFetchFileConfig): Promise<IFetchCommon<IFileInfo>>
215+
fetchFile(
216+
config: IFetchFileConfig,
217+
callback?: (res: IFetchCommon<IFileInfo>) => void
218+
): Promise<IFetchCommonArr<IFileInfo>>
192219
```
193220
194221
#### Example
@@ -331,12 +358,18 @@ interface IFetchPollingConfig {
331358
### IFetchCommon
332359
333360
```ts
334-
type IFetchCommon<T> = {
361+
interface IFetchCommon<T> {
335362
id: number
336363
statusCode: number | undefined
337-
headers: IncomingHttpHeaders // node:http type
364+
headers: IncomingHttpHeaders // node:http 类型
338365
data: T
339-
}[]
366+
}
367+
```
368+
369+
### IFetchCommonArr
370+
371+
```ts
372+
type IFetchCommonArr<T> = IFetchCommon<T>[]
340373
```
341374
342375
### IFileInfo

‎document/cn.md

+43-10
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,8 @@ XCrawl 是 Nodejs 多功能爬虫库。
88

99
- 只需简单的配置即可抓取 HTML 、JSON、文件资源等等
1010
- 使用 JSDOM 库对 HTML 解析，也可自行解析 HTML
11-
- 批量请求时可选择模式 异步/同步
11+
- 请求方式支持 异步/同步
12+
- 支持 Promise/Callback
1213
- 轮询功能
1314
- 拟人化的请求间隔时间
1415
- 使用 TypeScript 编写
@@ -47,6 +48,7 @@ XCrawl 是 Nodejs 多功能爬虫库。
4748
* [IFetchFileConfig](#IFetchFileConfig)
4849
* [IFetchPollingConfig](#IFetchPollingConfig)
4950
* [IFetchCommon](#IFetchCommon)
51+
* [IFetchCommonArr](#IFetchCommonArr)
5052
* [IFileInfo](#IFileInfo)
5153
* [IFetchHTML](#IFetchHTML)
5254
- [更多](#更多)
@@ -104,10 +106,26 @@ myXCrawl.fetchPolling({ d: 1 }, () => {
104106
```ts
105107
class XCrawl {
106108
constructor(baseConfig?: IXCrawlBaseConifg)
107-
fetchHTML(config: IFetchHTMLConfig): Promise<IFetchHTML>
108-
fetchData<T = any>(config: IFetchDataConfig): Promise<IFetchCommon<T>>
109-
fetchFile(config: IFetchFileConfig): Promise<IFetchCommon<IFileInfo>>
110-
fetchPolling(config: IFetchPollingConfig, callback: (count: number) => void): void
109+
110+
fetchHTML(
111+
config: IFetchHTMLConfig,
112+
callback?: (res: IFetchHTML) => void
113+
): Promise<IFetchHTML>
114+
115+
fetchData<T = any>(
116+
config: IFetchDataConfig,
117+
callback?: (res: IFetchCommon<T>) => void
118+
): Promise<IFetchCommonArr<T>>
119+
120+
fetchFile(
121+
config: IFetchFileConfig,
122+
callback?: (res: IFetchCommon<IFileInfo>) => void
123+
): Promise<IFetchCommonArr<IFileInfo>>
124+
125+
fetchPolling(
126+
config: IFetchPollingConfig,
127+
callback: (count: number) => void
128+
): void
111129
}
112130
```
113131
@@ -154,7 +172,10 @@ fetchHTML 是 [myXCrawl](https://github.com/coder-hxl/x-crawl/blob/main/document
154172
#### 类型
155173
156174
```ts
157-
function fetchHTML(config: IFetchHTMLConfig): Promise<IFetchHTML>
175+
fetchHTML(
176+
config: IFetchHTMLConfig,
177+
callback?: (res: IFetchHTML) => void
178+
): Promise<IFetchHTML>
158179
```
159180
160181
#### 示例
@@ -173,7 +194,10 @@ fetch 是 [myXCrawl](#示例-1) 实例的方法，通常用于爬取 API ，可
173194
#### 类型
174195
175196
```ts
176-
function fetchData<T = any>(config: IFetchDataConfig): Promise<IFetchCommon<T>>
197+
fetchData<T = any>(
198+
config: IFetchDataConfig,
199+
callback?: (res: IFetchCommon<T>) => void
200+
): Promise<IFetchCommonArr<T>>
177201
```
178202
179203
#### 示例
@@ -200,7 +224,10 @@ fetchFile 是 [myXCrawl](#示例-1) 实例的方法，通常用于爬取文件
200224
#### 类型
201225
202226
```ts
203-
function fetchFile(config: IFetchFileConfig): Promise<IFetchCommon<IFileInfo>>
227+
fetchFile(
228+
config: IFetchFileConfig,
229+
callback?: (res: IFetchCommon<IFileInfo>) => void
230+
): Promise<IFetchCommonArr<IFileInfo>>
204231
```
205232
206233
#### 示例
@@ -343,12 +370,18 @@ interface IFetchPollingConfig {
343370
### IFetchCommon
344371
345372
```ts
346-
type IFetchCommon<T> = {
373+
interface IFetchCommon<T> {
347374
id: number
348375
statusCode: number | undefined
349376
headers: IncomingHttpHeaders // node:http 类型
350377
data: T
351-
}[]
378+
}
379+
```
380+
381+
### IFetchCommonArr
382+
383+
```ts
384+
type IFetchCommonArr<T> = IFetchCommon<T>[]
352385
```
353386
354387
### IFileInfo

‎package.json

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
{
22
"private": true,
33
"name": "x-crawl",
4-
"version": "0.4.0",
4+
"version": "1.0.0",
55
"author": "CoderHxl",
66
"description": "XCrawl is a Nodejs multifunctional crawler library.",
77
"license": "MIT",

‎publish/README.md

+44-11
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,8 @@ XCrawl is a Nodejs multifunctional crawler library.
88

99
- Crawl HTML, JSON, file resources, etc. with simple configuration
1010
- Use the JSDOM library to parse HTML, or parse HTML by yourself
11-
- Optional mode asynchronous/synchronous for batch requests
11+
- The request method supports asynchronous/synchronous
12+
- Support Promise/Callback
1213
- Polling function
1314
- Anthropomorphic request interval
1415
- Written in TypeScript
@@ -47,6 +48,7 @@ XCrawl is a Nodejs multifunctional crawler library.
4748
* [IFetchFileConfig](#IFetchFileConfig)
4849
* [IFetchPollingConfig](#IFetchPollingConfig)
4950
* [IFetchCommon](#IFetchCommon)
51+
* [IFetchCommonArr](#IFetchCommonArr)
5052
* [IFileInfo](#IFileInfo)
5153
* [IFetchHTML](#IFetchHTML)
5254
- [More](#More)
@@ -92,10 +94,26 @@ Create a crawler instance via new XCrawl. The request queue is maintained by the
9294
```ts
9395
class XCrawl {
9496
constructor(baseConfig?: IXCrawlBaseConifg)
95-
fetchHTML(config: IFetchHTMLConfig): Promise<IFetchHTML>
96-
fetchData<T = any>(config: IFetchDataConfig): Promise<IFetchCommon<T>>
97-
fetchFile(config: IFetchFileConfig): Promise<IFetchCommon<IFileInfo>>
98-
fetchPolling(config: IFetchPollingConfig, callback: (count: number) => void): void
97+
98+
fetchHTML(
99+
config: IFetchHTMLConfig,
100+
callback?: (res: IFetchHTML) => void
101+
): Promise<IFetchHTML>
102+
103+
fetchData<T = any>(
104+
config: IFetchDataConfig,
105+
callback?: (res: IFetchCommon<T>) => void
106+
): Promise<IFetchCommonArr<T>>
107+
108+
fetchFile(
109+
config: IFetchFileConfig,
110+
callback?: (res: IFetchCommon<IFileInfo>) => void
111+
): Promise<IFetchCommonArr<IFileInfo>>
112+
113+
fetchPolling(
114+
config: IFetchPollingConfig,
115+
callback: (count: number) => void
116+
): void
99117
}
100118
```
101119
@@ -142,7 +160,10 @@ fetchHTML is the method of the above [myXCrawl](https://github.com/coder-hxl/x-c
142160
#### Type
143161
144162
```ts
145-
function fetchHTML(config: IFetchHTMLConfig): Promise<IFetchHTML>
163+
fetchHTML(
164+
config: IFetchHTMLConfig,
165+
callback?: (res: IFetchHTML) => void
166+
): Promise<IFetchHTML>
146167
```
147168
148169
#### Example
@@ -161,7 +182,10 @@ fetchData is the method of the above [myXCrawl](#Example-1) instance, which is u
161182
#### Type
162183
163184
```ts
164-
function fetchData<T = any>(config: IFetchDataConfig): Promise<IFetchCommon<T>>
185+
fetchData<T = any>(
186+
config: IFetchDataConfig,
187+
callback?: (res: IFetchCommon<T>) => void
188+
): Promise<IFetchCommonArr<T>>
165189
```
166190
167191
#### Example
@@ -188,7 +212,10 @@ fetchFile is the method of the above [myXCrawl](#Example-1) instance, which is u
188212
#### Type
189213
190214
```ts
191-
function fetchFile(config: IFetchFileConfig): Promise<IFetchCommon<IFileInfo>>
215+
fetchFile(
216+
config: IFetchFileConfig,
217+
callback?: (res: IFetchCommon<IFileInfo>) => void
218+
): Promise<IFetchCommonArr<IFileInfo>>
192219
```
193220
194221
#### Example
@@ -331,12 +358,18 @@ interface IFetchPollingConfig {
331358
### IFetchCommon
332359
333360
```ts
334-
type IFetchCommon<T> = {
361+
interface IFetchCommon<T> {
335362
id: number
336363
statusCode: number | undefined
337-
headers: IncomingHttpHeaders // node:http type
364+
headers: IncomingHttpHeaders // node:http 类型
338365
data: T
339-
}[]
366+
}
367+
```
368+
369+
### IFetchCommonArr
370+
371+
```ts
372+
type IFetchCommonArr<T> = IFetchCommon<T>[]
340373
```
341374
342375
### IFileInfo

‎publish/package.json

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
22
"name": "x-crawl",
3-
"version": "0.4.0",
3+
"version": "1.0.0",
44
"author": "CoderHxl",
55
"description": "XCrawl is a Nodejs multifunctional crawler library.",
66
"license": "MIT",

‎test/start/index.js

+1-1
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)