Improved readability and removed unnecessary functionality

Author: mariostoev

.idea/inspectionProfiles/Project_Default.xml

@@ -8,13 +8,15 @@
 
 def http_request(url, payload=None):
 
+    """ Makes http request to a URL address """
+
     if payload is None:
         payload = {}
 
     content = requests.get(url, params=payload, verify=False)
     content.raise_for_status()  # Raise HTTPError for bad requests (4xx or 5xx)
 
-    return content, content.url
+    return content.text, content.url
 
 
 class Connector(object):
@@ -27,13 +29,17 @@ def __init__(self, scrape_function, tasks):
 
     async def __http_request__async(self, url, session):
 
+        """ Sends asynchronous http request to URL address and scrapes the webpage. """
+
         async with session.get(url) as response:
             page_html = await response.read()
 
             return self.scrape_function(page_html, url)
 
     async def __async_scraper(self):
 
+        """ Appends URL's into tasks and gathers their output. """
+
         async_tasks = []
         async with aiohttp.ClientSession() as session:
             for n in self.tasks:

finviz/request_functions.py

@@ -16,7 +16,7 @@ def create_connection():
 
 def export_to_csv(headers, data):
 
-    with open('/screener_results.csv', 'w', newline='') as output_file:
+    with open('screener_results.csv', 'w', newline='') as output_file:
         dict_writer = csv.DictWriter(output_file, headers)
         dict_writer.writeheader()
 
@@ -61,15 +61,3 @@ def export_to_db(headers, data):
 
     conn.commit()
     conn.close()
-
-
-def select_from_db():
-
-    conn = create_connection()
-    c = conn.cursor()
-    c.execute("SELECT * FROM screener_results")
-
-    rows = c.fetchall()
-
-    for row in rows:
-        print(row)

finviz/save_data.py

@@ -3,13 +3,20 @@
 
 
 def get_total_rows(page_content):
+
+    """
+    Gets the total rows of the table. This function is called when the user does not provide a number of rows that have to be scraped.
+    """
+
     total_element = page_content.cssselect('td[width="140"]')
 
     return int(etree.tostring(total_element[0]).decode("utf-8").split('</b>')[1].split(' ')[0])
 
 
 def get_page_urls(page_content, rows, url):
 
+    """ Gets the page URL addresses """
+
     try:
         total_pages = int([i.text.split('/')[1] for i in page_content.cssselect('option[value="1"]')][0])
     except IndexError:

finviz/scraper_functions.py

@@ -1,8 +1,8 @@
-from .save_data import export_to_db, select_from_db, export_to_csv
+from finviz.request_functions import Connector, http_request
+from .save_data import export_to_db, export_to_csv
 from urllib.parse import urlencode
 from lxml import html
 from lxml import etree
-import finviz.request_functions as send
 import finviz.scraper_functions as scrape
 
 
@@ -11,45 +11,53 @@ class Screener(object):
     def __init__(self, tickers=None, filters=None, rows=None, order='', signal='', table='Overview'):
 
         if tickers is None:
-            self.tickers = []
+            self._tickers = []
         else:
-            self.tickers = tickers
+            self._tickers = tickers
 
         if filters is None:
-            self.filters = []
+            self._filters = []
         else:
-            self.filters = filters
-
-        self.rows = rows
-        self.order = order
-        self.signal = signal
-        self.table = table
-        self.page_content = None
-        self.url = None
-        self.headers = None
-        self.page_urls = None
-        self.data = None
+            self._filters = filters
 
-        self.__search_screener()
+        self._table_types = {
+            'Overview': '110',
+            'Valuation': '120',
+            'Ownership': '130',
+            'Performance': '140',
+            'Custom': '150',
+            'Financial': '160',
+            'Technical': '170'
+        }
 
-    def to_sqlite(self):
+        self._page_unparsed, self._url = http_request('https://finviz.com/screener.ashx', payload={
+                                                   'v': self._table_types[table],
+                                                   't': ','.join(self._tickers),
+                                                   'f': ','.join(self._filters),
+                                                   'o': order,
+                                                   's': signal
+                                                   })
 
-        export_to_db(self.headers, self.data)
+        self._page_content = html.fromstring(self._page_unparsed)
+        self._headers = self.__get_table_headers()
 
-    def display_db(self):
+        if rows is None:
+            self._rows = scrape.get_total_rows(self._page_content)
+        else:
+            self._rows = rows
 
-        select_from_db()
+        self.data = None
+        self.__search_screener()
 
-    def to_csv(self):
+    def to_sqlite(self):
+        export_to_db(self._headers, self.data)
 
-        export_to_csv(self.headers, self.data)
+    def to_csv(self):
+        export_to_csv(self._headers, self.data)
 
-    def get_charts(self, period='d', size='l', chart_type='c', ta=None, save_to=None):
+    def get_charts(self, period='d', size='l', chart_type='c', ta='1'):
 
-        if ta is True or None:  # Charts include TA by default
-            ta = '1'
-        else:
-            ta = '0'
+        """ Asynchronously downloads charts of tickers displayed by the screener. """
 
         payload = {
             'ty': chart_type,
@@ -65,13 +73,14 @@ def get_charts(self, period='d', size='l', chart_type='c', ta=None, save_to=None
             for row in page:
                 chart_urls.append(base_url + '&t={}'.format(row.get('Ticker')))
 
-        async_connector = send.Connector(scrape.download_image, chart_urls)
-        async_connector.directory = save_to
+        async_connector = Connector(scrape.download_image, chart_urls)
         async_connector.run_connector()
 
     def __get_table_headers(self):
 
-        first_row = self.page_content.cssselect('tr[valign="middle"]')
+        """ Scrapes the table headers from the initial page. """
+
+        first_row = self._page_content.cssselect('tr[valign="middle"]')
 
         headers = []
         for table_content in first_row[0]:
@@ -82,10 +91,12 @@ def __get_table_headers(self):
             else:
                 headers.append(table_content.text)
 
-        self.headers = headers
+        return headers
 
     def __get_table_data(self, page=None, url=None):
 
+        """ Returns the data, from each row of the table, inside a dictionary ."""
+
         def parse_row(line):
 
             row_data = []
@@ -104,46 +115,21 @@ def parse_row(line):
 
         for row in all_rows:
 
-            if int(row[0].text) is self.rows:
-                values = dict(zip(self.headers, parse_row(row)))
+            if int(row[0].text) is self._rows:
+                values = dict(zip(self._headers, parse_row(row)))
                 data_sets.append(values)
                 break
 
             else:
-                values = dict(zip(self.headers, parse_row(row)))
+                values = dict(zip(self._headers, parse_row(row)))
                 data_sets.append(values)
 
         return data_sets
 
     def __search_screener(self):
 
-        table = {
-            'Overview': '110',
-            'Valuation': '120',
-            'Ownership': '130',
-            'Performance': '140',
-            'Custom': '150',
-            'Financial': '160',
-            'Technical': '170'
-        }
-
-        payload = {
-            'v': table[self.table],
-            't': ','.join(self.tickers),
-            'f': ','.join(self.filters),
-            'o': self.order,
-            's': self.signal
-        }
-
-        self.page_content, self.url = send.http_request('https://finviz.com/screener.ashx', payload)
-        self.page_content = html.fromstring(self.page_content.text)  # Parses the page with the default lxml parser
-
-        self.__get_table_headers()
-
-        if self.rows is None:
-            self.rows = scrape.get_total_rows(self.page_content)
-
-        self.page_urls = scrape.get_page_urls(self.page_content, self.rows, self.url)
+        """ Saves data from the FinViz screener. """
 
-        async_connector = send.Connector(self.__get_table_data, self.page_urls)
+        page_urls = scrape.get_page_urls(self._page_content, self._rows, self._url)
+        async_connector = Connector(self.__get_table_data, page_urls)
         self.data = async_connector.run_connector()

finviz/screener.py

@@ -2,15 +2,15 @@
 setup(
   name = 'finviz',
   packages = ['finviz'],
-  version = '1.1',
+  version = '1.1.1',
   license='MIT',
   description = 'Unofficial API for finviz.com',
   author = 'Mario Stoev',
-  author_email = '[email protected]',      # Type in your E-Mail
-  url = 'https://github.com/mariostoev/finviz',   # Provide either the link to your github or to your website
-  download_url = 'https://github.com/user/reponame/archive/v_01.tar.gz',    # I explain this later on
-  keywords = ['finviz', 'api', 'screener', 'finviz api', 'charts', 'scraper'],   # Keywords that define your package best
-  install_requires=[            # I get to this in a second
+  author_email = '[email protected]',
+  url = 'https://github.com/mariostoev/finviz',
+  download_url = 'https://github.com/user/reponame/archive/v_01.tar.gz',
+  keywords = ['finviz', 'api', 'screener', 'finviz api', 'charts', 'scraper'],
+  install_requires=[
           'lxml',
           'requests',
           'aiohttp',