PyPi release and support for downloading charts

Author: mariostoev

.idea/sqldialects.xml

@@ -1 +1 @@
-from finviz.screener import Screener
+from finviz.screener import Screener

finviz/__init__.py

@@ -30,16 +30,16 @@ async def __http_request__async(self, url, session):
         async with session.get(url) as response:
             page_html = await response.read()
 
-            return self.scrape_function(page_html)
+            return self.scrape_function(page_html, url)
 
     async def __async_scraper(self):
 
-        tasks = []
+        async_tasks = []
         async with aiohttp.ClientSession() as session:
             for n in self.tasks:
-                tasks.append(self.__http_request__async(n, session))
+                async_tasks.append(self.__http_request__async(n, session))
 
-            self.data = await asyncio.gather(*tasks)
+            self.data = await asyncio.gather(*async_tasks)
 
     def run_connector(self):
 

finviz/request_functions.py

@@ -1,27 +1,29 @@
 import csv
-import os
 import sqlite3
 import re
 
+
 def create_connection():
-    
+
     sqlite_file = "../screener.sqlite"
-    
+
     try:
         conn = sqlite3.connect(sqlite_file)
         return conn
     except sqlite3.Error as error:
         raise ("An error has occurred while connecting to the database: ", error.args[0])
 
-def export_to_csv(headers, data, directory):
 
-    with open(directory + '/screener_results.csv', 'w', newline='') as output_file:
+def export_to_csv(headers, data):
+
+    with open('/screener_results.csv', 'w', newline='') as output_file:
         dict_writer = csv.DictWriter(output_file, headers)
         dict_writer.writeheader()
 
         for n in data:
             dict_writer.writerows(n)
 
+
 def export_to_db(headers, data):
 
     field_list = ""
@@ -71,8 +73,3 @@ def select_from_db():
 
     for row in rows:
         print(row)
-
-    rows = c.fetchall()
-
-    for row in rows:
-        print(row)

finviz/save_data.py

@@ -1,4 +1,5 @@
 from lxml import etree
+import os
 
 
 def get_total_rows(page_content):
@@ -27,3 +28,14 @@ def get_page_urls(page_content, rows, url):
             urls.append(url + '&r={}'.format(str(sequence)))
 
     return urls
+
+
+def download_image(page_content, url):
+
+    file_name = url.split('t=')[1] + '.jpg'
+
+    if not os.path.exists('charts'):
+        os.mkdir('charts')
+
+    with open('charts/' + file_name, 'wb') as handle:
+        handle.write(page_content)

finviz/scraper_functions.py

@@ -1,13 +1,14 @@
 from .save_data import export_to_db, select_from_db, export_to_csv
+from urllib.parse import urlencode
 from lxml import html
 from lxml import etree
 import finviz.request_functions as send
 import finviz.scraper_functions as scrape
-from os import getcwd
+
 
 class Screener(object):
 
-    def __init__(self, tickers=None, filters=None, order='', rows=None, signal='', table='Overview'):
+    def __init__(self, tickers=None, filters=None, rows=None, order='', signal='', table='Overview'):
 
         if tickers is None:
             self.tickers = []
@@ -31,43 +32,42 @@ def __init__(self, tickers=None, filters=None, order='', rows=None, signal='', t
 
         self.__search_screener()
 
-    def to_csv(self, directory=None):
-
-        if directory is None:
-            directory = getcwd()
-
-        export_to_csv(self.headers, self.data, directory)
-
     def to_sqlite(self):
+
         export_to_db(self.headers, self.data)
 
     def display_db(self):
-        select_from_db()
 
-    def __get_total_rows(self):
+        select_from_db()
 
-        total_element = self.page_content.cssselect('td[width="140"]')
-        self.rows = int(etree.tostring(total_element[0]).decode("utf-8").split('</b>')[1].split(' ')[0])
+    def to_csv(self):
 
-    def __get_page_urls(self):
+        export_to_csv(self.headers, self.data)
 
-        try:
-            total_pages = int([i.text.split('/')[1] for i in self.page_content.cssselect('option[value="1"]')][0])
-        except IndexError:  # No results found
-            return None
+    def get_charts(self, period='d', size='l', chart_type='c', ta=None, save_to=None):
 
-        urls = []
+        if ta is True or None:  # Charts include TA by default
+            ta = '1'
+        else:
+            ta = '0'
 
-        for page_number in range(1, total_pages + 1):
+        payload = {
+            'ty': chart_type,
+            'ta': ta,
+            'p': period,
+            's': size
+        }
 
-            sequence = 1 + (page_number - 1) * 20
+        base_url = 'https://finviz.com/chart.ashx?' + urlencode(payload)
+        chart_urls = []
 
-            if sequence - 20 <= self.rows < sequence:
-                break
-            else:
-                urls.append(self.url + '&r={}'.format(str(sequence)))
+        for page in self.data:
+            for row in page:
+                chart_urls.append(base_url + '&t={}'.format(row.get('Ticker')))
 
-        self.page_urls = urls
+        async_connector = send.Connector(scrape.download_image, chart_urls)
+        async_connector.directory = save_to
+        async_connector.run_connector()
 
     def __get_table_headers(self):
 
@@ -84,7 +84,7 @@ def __get_table_headers(self):
 
         self.headers = headers
 
-    def __get_table_data(self, page=None):
+    def __get_table_data(self, page=None, url=None):
 
         def parse_row(line):
 

finviz/screener.py

@@ -0,0 +1,3 @@
+# Inside of setup.cfg
+[metadata]
+description-file = README.md

setup.cfg

@@ -0,0 +1,26 @@
+from distutils.core import setup
+setup(
+  name = 'finviz',
+  packages = ['finviz'],
+  version = '1.1',
+  license='MIT',
+  description = 'Unofficial API for finviz.com',
+  author = 'Mario Stoev',
+  author_email = '[email protected]',      # Type in your E-Mail
+  url = 'https://github.com/mariostoev/finviz',   # Provide either the link to your github or to your website
+  download_url = 'https://github.com/user/reponame/archive/v_01.tar.gz',    # I explain this later on
+  keywords = ['finviz', 'api', 'screener', 'finviz api', 'charts', 'scraper'],   # Keywords that define your package best
+  install_requires=[            # I get to this in a second
+          'lxml',
+          'requests',
+          'aiohttp',
+          'urllib3'
+      ],
+  classifiers=[
+    'Development Status :: 5 - Production/Stable',
+    'Intended Audience :: Developers',
+    'Topic :: Software Development :: Build Tools',
+    'License :: OSI Approved :: MIT License',
+    'Programming Language :: Python :: 3.6',
+  ],
+)