Update

.gitignore

@@ -1,2 +1,6 @@
-db.db
-*.pyc
+/db.*
+/*.db
+/*.pyc
+/*.log
+/*.bak
+/*.csv

README.md

@@ -1,15 +1,35 @@
-# GFYP - Go "Find Your" Phishers
+# GFYP - Go Find Your Phishers
 
-Having experimented with dnstwist for some time now, it dawned on me that it could be useful for alerting of new domains as they pop up related to your domains of interest. 
-
-For more information on dnstwist, see https://github.com/elceef/dnstwist
+This tool augments [dnstwist](https://github.com/elceef/dnstwist) with a database that tracks identified phishing sites over times, and provides email alerts when new ones are discovered.
 
 ## Installation
-* pip install -r requirements.txt
-* Add notification email information (username/password) to the top of core.py
-* python util.py build
-* python util.py add (domain name) (email address)
-* python core.py (or set it as a cron job)  - Start getting information reports hourly.
-
-## Notes
-Yes I know some of the coding is horrible. I'm going to be cleaning it up sometime in the next few weeks here when I get some time, just wanting to share it for now.
+
+    $ pip install -r requirements.txt
+
+## Configuration
+
+1. Initialize database with `python util.py build`
+2. Either add your SMTP credentials by hard-coding them in core.py, or set the
+following environment variables:
+    * `GFYP_EMAIL_USERNAME`
+    * `GFYP_EMAIL_PASSWORD`
+    * `GFYP_EMAIL_SMTPSERVER`
+
+Ex.
+
+    $ export [email protected]
+    $ export GFYP_EMAIL_PASSWORD=ilovemallory
+    $ export GFYP_EMAIL_SMTPSERVER=smtp.example.com
+
+## Usage
+
+    # add domain to list for which to hunt phishing domains
+    python util.py add (domain name) (email address) [optional: path to csv containing additional TLDs to check]
+    # start searching process
+    python core.py # or set it as a cron job to regular reports
+
+## Troubleshooting
+
+### GMail
+
+If using GMail as an SMTP provider, you may first need to log into GMail in the web interface and enable the "Allow less secure apps" option in the "Sign-in & security" section.

common.py

@@ -0,0 +1,35 @@
+"""Shared functions and constants"""
+import sys
+import logging
+
+LOG_FILENAME = 'gfyp.log'
+
+BOLD = ""
+END = ""
+if sys.platform != 'win32' and sys.stdout.isatty():
+    BOLD = "\033[1m"
+    END = "\033[0m"
+
+def pretty_print(string):
+    """For *nix systems, augment TTY output. For others, strip such syntax."""
+    string = string.replace('$BOLD$', BOLD)
+    string = string.replace('$END$', END)
+    print(string)
+
+def log(msg, level=logging.INFO):
+    """Add a string to the log file."""
+    logging.basicConfig(filename=LOG_FILENAME,
+                        format='%(asctime)s:%(levelname)s:%(message)s',
+                        level=logging.INFO)
+    if level == logging.DEBUG:
+        logging.debug(msg)
+    elif level == logging.INFO:
+        logging.info(msg)
+    elif level == logging.WARNING:
+        logging.warning(msg)
+    elif level == logging.ERROR:
+        logging.error(msg)
+    elif level == logging.CRITICAL:
+        logging.critical(msg)
+    else:
+        raise ValueError(str(level))

core.py

@@ -12,54 +12,177 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import sqlite3
+"""Checks database for new phishing entries and executes alerts."""
+
+import os
 import sys
-from dnslib import dnslib
 import smtplib
+import logging
+
+from dnslib import dnslib #dnslib.py
+import gfyp_db #gfyp_db.py
+from common import pretty_print, log #common.py
+
+#SET EMAIL SETTINGS HERE IF NOT USING ENVIRONMENT VARIABLES
+EMAIL_USERNAME = None
+EMAIL_PASSWORD = None
+EMAIL_SMTPSERVER = None
+
+def send_email(smtp_auth, recipient, subject, body):
+    """Send email via SMTP.
+    Args:
+        smtp_auth (dict): Contains 'username' (str), 'password' (str), and
+            'server' (str).
+        recipient (str): The email address to send to
+        subject (str)
+        body (str)
+
+    http://stackoverflow.com/questions/10147455/trying-to-send-email-gmail-as-mail-provider-using-python
+    """
+    email_to = [recipient]
+
+    #Sending message, first construct actual message
+    message = ("From: %s\nTo: %s\nSubject: %s\n\n%s" %
+               (smtp_auth['username'], ", ".join(email_to), subject, body))
+    try:
+        server_ssl = smtplib.SMTP_SSL(smtp_auth['server'], 465)
+        server_ssl.ehlo()
+        server_ssl.login(smtp_auth['username'], smtp_auth['password'])
+        server_ssl.sendmail(smtp_auth['username'], email_to, message)
+        server_ssl.close()
+    except Exception as err:
+        msg = "Failed to send mail: %s" % str(err)
+        log(msg, logging.ERROR)
+        sys.exit(msg)
+
+    msg = "Email sent to %s." % recipient
+    print(msg)
+    log(msg)
+
+def check_and_send_alert(smtp_auth, alert_email, domain, escape_alert=False,
+                         db_con=None):
+    """Consult DB whether an alert needs to be sent for domain, and send one.
+    Args:
+        smtp_auth (dict): Credentials for SMTP server, including 'username',
+            'password', and 'server'.
+        alert_email (str)
+        domain (str)
+        escape_alert (bool): Whether or not to escape periods in the email body
+            in order to avoid spam filtering. (Default: False)
+        db_con (None or `gfyp_db.DatabaseConnection`): This can optionally
+            provide a database connection to reuse. Otherwise, a new one will
+            be created.
+    """
+    msg = "Now checking %s - %s" % (alert_email, domain)
+    print(msg)
+    log(msg)
+    close_db = False
+    if db_con is None:
+        db_con = gfyp_db.DatabaseConnection()
+        close_db = True
+    body = ""
+    dns_check = dnslib()
+    entries = dns_check.checkDomain(domain)
+    msg = "DNSTwist found %d variant domains from %s." % (len(entries), domain)
+    print(msg)
+    log(msg)
+    num_new_entries = 0
+    for domain_found, domain_info in entries:
+        found_entries = db_con.get_matching_found_domains(domain_found)
+        entries_iter = found_entries.fetchall()
+
+        if len(entries_iter) == 0:
+            db_con.add_discovered_domain(domain_found, domain_info)
+            body += "\r\n\r\n%s - %s" % (domain_found, domain_info)
+            num_new_entries += 1
 
-EMAIL_USERNAME = "EMAIL_ADDRESS_GOES_HERE"
-EMAIL_PASSWORD = "EMAIL_PASSWORD_GOES_HERE"
-EMAIL_STMPSERVER = "SMTP_SERVER_GOES_HERE"
-
-#Code below taken from: http://stackoverflow.com/questions/10147455/trying-to-send-email-gmail-as-mail-provider-using-python
-def send_email(recipient, subject, body):
-
-	FROM = EMAIL_USERNAME
-	TO = [recipient]
-	SUBJECT = subject
-	TEXT = body
-
-	#Sending message, first construct actual message
-	message = """\From: %s\nTo: %s\nSubject: %s\n\n%s
-	""" % (FROM, ", ".join(TO), SUBJECT, TEXT)
-	try:
-		server_ssl = smtplib.SMTP_SSL(EMAIL_STMPSERVER, 465)
-		server_ssl.ehlo()
-		server_ssl.login(EMAIL_USERNAME, EMAIL_PASSWORD)  
-		server_ssl.sendmail(FROM, TO, message)
-		server_ssl.close()
-	except:
-        	print "failed to send mail"
+    if body != "":
+        recipient = alert_email
+        subject = 'GFYP - New Entries for %s' % domain
+        if escape_alert:
+            body = body.replace('.', '[.]')
+        send_email(smtp_auth, recipient, subject, body)
+
+    msg = "Found %d new domain variants from %s" % (num_new_entries, domain)
+    print(msg)
+    log(msg)
+
+    if close_db:
+        db_con.conn.close()
 
 def main():
-	dnsCheck = dnslib()
-	conn = sqlite3.connect('db.db')
-	c = conn.cursor()
-	domainentries =  c.execute('SELECT * FROM lookupTable').fetchall()
-	for row in domainentries:
-		print "Now checking %s - %s" % (row[0], row[1])
-		body = ""
-		entries = dnsCheck.checkDomain(row[1])
-		for entry in entries:
-			foundEntries = c.execute("SELECT * FROM foundDomains WHERE domainName = '%s'" % entry[0])
-			entriesIter = foundEntries.fetchall()
-			if len(entriesIter) == 0:
-				c.execute("INSERT INTO foundDomains VALUES ('%s','%s')" % (entry[0],entry[1]))
-				body = body+"\r\n\r\n%s - %s" % (entry[0],entry[1])
-		if body != "":
-			send_email(row[0],'GFYP - New Entries for %s' % row[1],body) 
-	conn.commit()
-	conn.close()
+    """Description: Search for new domain variants and email alerts for new ones.
+    """
+    args = get_args()
+    #Get configuration from env variables or fallback to hard-coded values
+    smtp_auth = dict()
+    smtp_auth['username'] = os.getenv('GFYP_EMAIL_USERNAME', EMAIL_USERNAME)
+    smtp_auth['password'] = os.getenv('GFYP_EMAIL_PASSWORD', EMAIL_PASSWORD)
+    smtp_auth['server'] = os.getenv('GFYP_EMAIL_SMTPSERVER', EMAIL_SMTPSERVER)
+    for key, value in list(smtp_auth.items()):
+        if value is None:
+            msg = "Fatal error: Email setting '%s' has not been set." % key
+            log(msg, logging.ERROR)
+            sys.exit(msg)
+
+    if any([EMAIL_USERNAME, EMAIL_PASSWORD, EMAIL_SMTPSERVER]):
+        msg = ("WARNING: You have hard-coded credentials into a code file. Do "
+               "not commit it to a public Git repo!")
+        print(msg)
+        log(msg, logging.WARNING)
+
+    with gfyp_db.DatabaseConnection() as db_con:
+        if db_con.is_db_current():
+            domain_entries = db_con.get_watch_entries()
+
+            if len(domain_entries) == 0:
+                msg = ("No domains have been added for watching/alerts. Use "
+                       "util.py to add domains.")
+                print(msg)
+                log(msg)
+
+            for row in domain_entries:
+                alert_email = row[0]
+                domain = row[1]
+                check_and_send_alert(
+                    smtp_auth, alert_email, domain,
+                    escape_alert=args['escape_alert'], db_con=db_con)
+        else:
+            msg = "GFYP database is not current. Please run 'python util.py migrate' to update to the current schema"
+            print(msg)
+            log(msg,logging.ERROR)
+
+def usage():
+    """Print usage info."""
+    usage_str = (
+        "GFYP Core - Find domain variants and send alerts\n"
+        "usage: python core.py [$BOLD$-escapealert$END$]\n"
+        "Options:\n"
+        "    $BOLD$-escapealert$END$ - Escape periods in email alert to avoid "
+        "spam filter")
+    pretty_print(usage_str)
+    sys.exit()
+
+def get_args():
+    """Get command line arguments.
+
+    Current arguments:
+        * escape_alert (bool): Whether to escape periods in alert email.
+    """
+    args = dict()
+    args['escape_alert'] = False
+    if len(sys.argv) == 1:
+        return args
+    elif len(sys.argv) == 2:
+        if sys.argv[1] == '-escapealert':
+            args['escape_alert'] = True
+        else:
+            log("Invalid arguments: %s" % sys.argv, logging.ERROR)
+            usage()
+    else:
+        log("Invalid arguments: %s" % sys.argv, logging.ERROR)
+        usage()
+    return args
 
 if __name__ == "__main__":
     main()