Merge branch 'release-0.2'

Author: bbengfort

.gitignore

@@ -57,3 +57,4 @@ target/
 # Local configurations
 conf/baleen.yaml
 fixtures/corpus
+fixtures/ddl-feedly.opml

baleen/config.py

@@ -38,7 +38,6 @@ class BaleenConfiguration(confire.Configuration):
     Meaningful defaults and required configurations.
 
     debug:    the app will print or log debug statements
-    testing:  the app will not overwrite important resources
     database: connection information for mongo
     """
 
@@ -48,10 +47,11 @@ class BaleenConfiguration(confire.Configuration):
         os.path.abspath("conf/baleen.yaml"),     # Local configuration
     ]
 
-    debug    = True
-    testing  = True
-    database = MongoConfiguration()
-
+    debug      = True
+    database   = MongoConfiguration()
+    logfile    = 'baleen.log'                    # Location to write log
+    loglevel   = 'DEBUG'                         # Log messages to record
+    fetch_html = True                            # Actually fetch HTML link
 
 ## Load settings immediately for import
 settings = BaleenConfiguration.load()

baleen/feed.py

@@ -22,14 +22,16 @@
 ## Imports
 ##########################################################################
 
+import requests
 import feedparser
 import baleen.models as db
 
 from copy import copy
 from baleen.opml import OPML
 from collections import Counter
-from baleen.utils import localnow
-from baleen.logger import IngestLogger
+from baleen.config import settings
+from baleen.utils.timez import localnow
+from baleen.utils.logger import IngestLogger
 from dateutil import parser as dtparser
 
 ##########################################################################
@@ -51,8 +53,10 @@ def get_feed_urls(self):
             for url in self.feed_urls:
                 yield url
         else:
-            raise NotImplementedError(("Subclasses must either provide a list of",
-                                       " feed_urls or override get_feed_urls."))
+            raise NotImplementedError((
+                "Subclasses must either provide a list of "
+                "feed_urls or override get_feed_urls."
+            ))
 
     def feeds(self):
         """
@@ -131,8 +135,35 @@ def wrangle(self, entry):
             post['mimetype'] = selected.get('type')
             post['content']  = selected.get('value')
 
+        ## Fetch the content if requested.
+        if settings.fetch_html:
+            page = self.fetch(post.get('url'))
+            if page:
+                post['content'] = page
+
         return post
 
+    def fetch(self, url):
+        """
+        Fetches the given url and returns the content, capturing errors.
+        """
+        try:
+            response = requests.get(url)
+            if response.status_code == 200:
+                return response.text
+            else:
+                self.logger.error(
+                    "Could not fetch '{}': {} {}".format(
+                        url, response.status_code, response.reason
+                    )
+                )
+        except Exception as e:
+            self.logger.error(
+                "Could not fetch '{}': {}".format(
+                    url, str(e)
+                )
+            )
+
     def fields(self):
         """
         Returns a count of all the available fields in every entry.
@@ -185,7 +216,7 @@ def get_feed_urls(self):
 
         NOTE: You must connect to the MongoDB before calling this method!
         """
-        for feed in db.Feed.objects.only('link'):
+        for feed in db.Feed.objects(active=True).only('link'):
             yield feed.link
 
     def update_feed(self, feed, result):
@@ -308,7 +339,7 @@ def __init__(self, path):
 
     def get_feed_urls(self):
         for feed in self.opml:
-            yield feed['link']
+            yield feed['xmlUrl']
 
     def __len__(self):
         return len(self.opml)

baleen/models.py

@@ -71,6 +71,7 @@ class Feed(me.DynamicDocument):
     link      = me.URLField(required=True, unique=True)
     urls      = me.DictField()
     category  = me.StringField(required=True)
+    active    = me.BooleanField(default=True)
     fetched   = me.DateTimeField(default=None)
     created   = me.DateTimeField(default=datetime.now, required=True)
     updated   = me.DateTimeField(default=datetime.now, required=True)

baleen/opml.py

@@ -38,9 +38,9 @@ def ingest(path, **kwargs):
     for feed in opml:
         feed.pop('type')                    # Unneeded for database
         feed.pop('text')                    # Unneeded for database
-        feed['link'] = feed.pop('xmlurl')   # Rename the XML URL
+        feed['link'] = feed.pop('xmlUrl')   # Rename the XML URL
         feed['urls'] = {
-            'htmlurl': feed.pop('htmlurl')  # Add htmlurl to urls
+            'htmlurl': feed.pop('htmlUrl')  # Add htmlurl to urls
         }
         feed = db.Feed(**feed)              # Construct without an ObjectId
 
@@ -68,7 +68,7 @@ def categories(self):
         Reads the file to capture all the categories
         """
         with open(self.path, 'r') as data:
-            soup = BeautifulSoup(data)
+            soup = BeautifulSoup(data, 'xml')
             for topic in soup.select('body > outline'):
                 yield topic['title']
 
@@ -87,7 +87,7 @@ def __iter__(self):
         from the OPML file; also captures category data.
         """
         with open(self.path, 'r') as data:
-            soup = BeautifulSoup(data)
+            soup = BeautifulSoup(data, 'xml')
             for topic in soup.select('body > outline'):
                 for feed in topic.find_all('outline'):
                     data = feed.attrs.copy()

baleen/utils/__init__.py

@@ -0,0 +1,21 @@
+# baleen.utils
+# Utilities and helpers functions for the Baleen project.
+#
+# Author:   Benjamin Bengfort <bengfort@cs.umd.edu>
+# Created:  Sun Feb 21 15:00:06 2016 -0500
+#
+# Copyright (C) 2016 University of Maryland
+# For license information, see LICENSE.txt
+#
+# ID: __init__.py [] benjamin@bengfort.com $
+
+"""
+Utilities and helpers functions for the Baleen project.
+"""
+
+##########################################################################
+## Imports
+##########################################################################
+
+# For the log configuration to work
+from . import mongolog

baleen/logger.py baleen/utils/logger.pybaleen/logger.py renamed to baleen/utils/logger.py

@@ -1,4 +1,4 @@
-# baleen.logger
+# baleen.utils.logger
 # Logging utility for Baleen
 #
 # Author:   Benjamin Bengfort <benjamin@bengfort.com>
@@ -19,7 +19,7 @@
 
 import logging
 import logging.config
-from baleen.utils import *
+from baleen.utils.timez import *
 from baleen.config import settings
 
 ##########################################################################
@@ -41,32 +41,32 @@
             'class': 'logging.NullHandler',
         },
         'console': {
-            'level': 'DEBUG',
+            'level': 'WARNING',
             'class': 'logging.StreamHandler',
             'formatter': 'simple',
         },
         'logfile': {
             'level': 'INFO',
             'class': 'logging.handlers.RotatingFileHandler',
-            'filename': 'baleen.log',
-            'maxBytes': '16777216', # 16 MB
+            'filename': settings.logfile,
+            'maxBytes': '536870912', # 512 MB
             'formatter': 'simple',
         },
         'mongolog': {
             'level': 'INFO',
-            'class': 'baleen.mongolog.MongoHandler',
+            'class': 'baleen.utils.mongolog.MongoHandler',
         }
     },
     'loggers': {
         'baleen': {
-            'level': 'DEBUG',
-            'handlers': ['console'],
+            'level': settings.loglevel,
+            'handlers': ['logfile'],
             'propagagte': True,
         },
         'baleen.ingest': {
             'level': 'INFO',
             'handlers': ['logfile', 'mongolog'],
-            'propagate': True,
+            'propagate': False,
         }
     },
 }
@@ -121,10 +121,10 @@ def error(self, message, *args, **kwargs):
     def critical(self, message, *args, **kwargs):
         return self.log(logging.CRITICAL, message, *args, **kwargs)
 
+
 class IngestLogger(Logger):
     """
-    Performs logging for the coruscate process with the log options above.
+    Performs logging for the baleen process with the log options above.
     """
 
     logger = logging.getLogger('baleen.ingest')
-

baleen/mongolog.py baleen/utils/mongolog.pybaleen/mongolog.py renamed to baleen/utils/mongolog.py

@@ -1,4 +1,4 @@
-# baleen.mongolog
+# baleen.utils.mongolog
 # Handlers and formatters for logging to Mongo
 #
 # Author:   Benjamin Bengfort <benjamin@bengfort.com>
@@ -20,12 +20,12 @@
 import getpass
 import logging
 import logging.config
-from baleen.utils import *
+from baleen.utils.timez import *
 from baleen.config import settings
 
 from datetime import datetime
 from socket import gethostname
-from pymongo import Connection
+from pymongo import MongoClient
 from pymongo.errors import OperationFailure, PyMongoError
 
 ##########################################################################
@@ -44,7 +44,6 @@ def format(self, record):
 
         ## Get the dictionary ready for Mongo
         data    = record.__dict__.copy()
-        print data
 
         ## Get the log message as intended via super
         message   = super(MongoFormatter, self).format(record)
@@ -112,7 +111,7 @@ def connect(self):
         Connect to the Mongo database.
         """
         try:
-            self.connection = Connection(host=self.host, port=self.port)
+            self.connection = MongoClient(host=self.host, port=self.port)
         except PyMongoError:
             if self.fail_silently:
                 return
@@ -127,7 +126,6 @@ def close(self):
         Close the connection to the Mongo database.
         """
         if self.connection is not None:
-            self.connection.disconnect()
             self.connection = None
 
     def emit(self, record):

baleen/utils.py baleen/utils/timez.pybaleen/utils.py renamed to baleen/utils/timez.py

@@ -1,4 +1,4 @@
-# baleen.utils
+# baleen.utils.timez
 # Utility functions for Baleen
 #
 # Author:   Benjamin Bengfort <benjamin@bengfort.com>
@@ -62,13 +62,3 @@ def strptimez(dtstr, dtfmt):
     delta  = timedelta(hours = offset/100)
     utctsp = datetime.strptime(dtstr, dtfmt) - delta
     return utctsp.replace(tzinfo=tzutc())
-
-def dthandler(obj, dtftmt="%Y-%m-%dT%H:%M:%S"):
-    """
-    JSON helper function that provides a handler for Python datetime objects,
-    returning the ISO 8601 format.
-    """
-    dthandler = None
-    if isinstance(obj, datetime) or isinstance(obj, date):
-       dthandler = obj.strftime(dtftmt)
-    return dthandler

baleen/version.py

@@ -19,7 +19,7 @@
 
 __version_info__ = {
     'major': 0,
-    'minor': 1,
+    'minor': 2,
     'micro': 0,
     'releaselevel': 'final',
     'serial': 0,