spider_hs_code/my_book_spider.py at master · xiphodon/spider_hs_code · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Time    : 2018/6/11 15:23
# @Author  : GuoChang
# @Site    : https://github.com/xiphodon
# @File    : my_book_spider.py
# @Software: PyCharm

import requests
from lxml import etree
import os
import json
import time

home_url = r'https://www.wuxiaworld.com'

home_path = r'E:\work_all\topease\my_book_spider'
chapter_json_path = os.path.join(home_path, 'chapter_list.json')

book_text_path = os.path.join(home_path, 'coiling-dragon.txt')

headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64; rv:49.0) Gecko/20100101 Firefox/49.0',
    'Connection': 'keep-alive'
}


def while_request_get(page_url, times=1000):
    """
    循环请求
    :return:
    """
    while_times = 0
    while True:
        try:
            result = requests.get(page_url, headers=headers, timeout=20)
            # result = requests.get(page_url, headers=headers, proxies=proxies, timeout=5)
        except Exception as e:
            if while_times < times:
                while_times += 1
                print('**********', '尝试重新链接', while_times, '次:', page_url.replace(home_url, ''))
                continue
            else:
                raise e
        else:
            return result


def get_book_chapter_json():
    """
    存取该书籍章节为json
    :return:
    """

    if os.path.exists(chapter_json_path):
        with open(chapter_json_path, 'r', encoding='utf8') as fp:
            data_json = json.loads(fp.read())
        print('get chapter json from file')
        return data_json

    result = while_request_get(home_url + '/novel/coiling-dragon')

    seletor = etree.HTML(result.text)

    chapter_a_list = seletor.xpath('.//div[@id="collapse-0"]//ul/li[@class="chapter-item"]/a')

    chapter_list_json = list()
    for a_node in chapter_a_list:
        chapter_href = ''.join(a_node.xpath('./@href'))
        chapter_name = ''.join(a_node.xpath('./span/text()'))
        chapter_list_json.append({
            'chapter_name': chapter_name,
            'chapter_href': home_url + chapter_href
        })

    with open(chapter_json_path, 'w', encoding='utf8') as fp:
        fp.write(json.dumps(chapter_list_json))

    print('get chapter json from web')
    return chapter_list_json


def download_chapter_content_to_file(data_json):
    """
    下载章节内容至文件
    :param data_json:
    :return:
    """

    book_content_text = ''

    for item_chapter in data_json:
        chapter_name = item_chapter['chapter_name']
        chapter_href = item_chapter['chapter_href']

        print(chapter_name)

        result = while_request_get(chapter_href)

        seletor = etree.HTML(result.text)

        # chapter_title = ''.join(seletor.xpath('.//div[@class="p-15"]//div/h4/text()'))

        chapter_content_p_list = seletor.xpath('.//div[@class="p-15"]/div[@class="fr-view"]/p')

        chapter_content_text = ''
        for p_node in chapter_content_p_list:
            p_node_text = p_node.xpath('string()')

            if p_node_text.strip() == 'Previous Chapter':
                continue

            chapter_content_text += p_node_text + '\n'

        book_content_text += chapter_content_text

        print(len(chapter_content_text))
        time.sleep(0.5)

    with open(book_text_path, 'w', encoding='utf8') as fp:
        fp.write(book_content_text)


def start():
    """
    启动入口
    :return:
    """
    data_json = get_book_chapter_json()
    download_chapter_content_to_file(data_json)


if __name__ == '__main__':
    start()