SunbizVerify/caliScrape.py at master · lopezdp/SunbizVerify · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
#!/usr/bin/env python3

import mechanize
import argparse
import re
import json
from urllib.parse import quote
from bs4 import BeautifulSoup
import ssl

MAX_SIZE = 5
#TODO MUST BE REFACTORED!
#TODO DON'T turn this in without refactoring!
#TODO Merge with Florida search
def searchState(searchQuery):

    browser = mechanize.Browser()
    browser.set_handle_robots(False)
    browser.addheaders = [('User-agent', 'Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.0.1) Gecko/2008071615 Fedora/3.0.1-1.fc9 Firefox/3.0.1')]

    context = ssl._create_unverified_context()
    browser.set_ca_data(context=context)

    formattedInput = quote(searchQuery)
    #print('url: ', f'https://businesssearch.sos.ca.gov/CBS/SearchResults?filing=False&SearchType=CORP&SearchCriteria={formattedInput}&SearchSubType=Keyword')
    #TODO: Remove hard coded url
    response = browser.open(f'https://businesssearch.sos.ca.gov/CBS/SearchResults?filing=False&SearchType=CORP&SearchCriteria={formattedInput}&SearchSubType=Keyword')
    html = response.read() # TODO: FIND OUT WHY THIS WORKS!!!!
    #print(html)
    soup = BeautifulSoup(html, 'html.parser')
    tags = soup.find_all('td')
    activeBusinesses = []
    #print(tags)

    for index, tag in enumerate(tags):
        # Get the string content from the html tag...
        tagString = str(tag.string).lstrip() if tag.string != None else ""

        #NOTE: len() on a list is an O(1) operation...
        if(len(activeBusinesses) < MAX_SIZE and re.search('Active', tagString, re.I)):
            activeBusinesses.append(tags[index + 1]['data-order'])
            #print(tags[index + 1]['data-order'])

    return activeBusinesses

def main(args):
    results = searchState(args.searchQuery)

    # Output results
    if(args.json):
        print(json.dumps({'data' : results, 'state' : 'California'}, sort_keys=True,
                         indent=4, separators=(',', ': ')))
    else:
        print(f'Found {len(results)} active businesses while searching for {args.searchQuery}')
        print("--------- Results ---------")
        for result in results:
            print(result)

    # TODO: Return results in JSON or as a python list
    return json.dumps({'data' : results, 'state' : 'California'}, sort_keys=True,
                        indent=None, separators=(',', ': '))


if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='Searches for an active business entity on https://businesssearch.sos.ca.gov/ by \'scraping\' the website.')
    parser.add_argument("searchQuery", help='The name of the business to search for.')
    parser.add_argument("-j", "--json",  help='Print results in JSON', action='store_true')
    args = parser.parse_args()
    main(args)