zhw_crawler/readjson.py at master · askender/zhw_crawler · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import json
import sys
reload(sys)
sys.setdefaultencoding('utf-8')
import re
import os


def get_list():
    ids = []
    with open('xx_bad') as f:
        for i in f:
            i = i.replace('\n', '')
            ids.append(i)
    return ids


def get_doc(i):
    path = 'html/airjson/%s.json' % i
    if not os.path.isfile(path):
        return []
    with open(path) as f:
        doc = f.read()
        doc = doc.replace('\n', '')
        comma_re = re.compile(',+')
        doc = comma_re.sub(',', doc)
        doc = doc.replace('[,', '[')
        doc = doc[4:]
        if doc.endswith('</html>'):
            print i
            os.remove(path)
            return []
        try:
            doc = json.loads(doc)
        except:
            return []
    return doc


def get_idx(idx_old, doc):
    for idx, i in enumerate(doc):
        idx_new = "%s[%s]" % (idx_old, idx)
        if isinstance(i, (list)):
            get_idx(idx_new, i)
        else:
            pass
            # print idx_new, i

for i in get_list():
    doc = get_doc(i)
    get_idx('', doc)


# https://www.google.com/search?tbm=map&hl=zh-CN&q=ABG%20airport