Merge pull request #18 from al4/master

Flesh out /stats API

Author: al4

Vagrantfile

@@ -43,13 +43,14 @@ Vagrant.configure(2) do |config|
   # backing providers for Vagrant. These expose provider-specific options.
   # Example for VirtualBox:
   #
-  # config.vm.provider "virtualbox" do |vb|
+  config.vm.provider "virtualbox" do |vb|
   #   # Display the VirtualBox GUI when booting the machine
   #   vb.gui = true
   #
   #   # Customize the amount of memory on the VM:
   #   vb.memory = "1024"
-  # end
+    vb.cpus = "2"
+  end
   #
   # View the documentation for the provider you are using for more
   # information on available options.
@@ -77,6 +78,9 @@ Vagrant.configure(2) do |config|
 
     # Build tools
     sudo apt-get -y install build-essential git-buildpackage debhelper python-dev dh-systemd
+    wget -P /tmp/ \
+        'https://launchpad.net/ubuntu/+archive/primary/+files/dh-virtualenv_0.11-1_all.deb'
+    dpkg -i /tmp/dh-virtualenv_0.11-1_all.deb
 
     sudo pip install --upgrade pip
     sudo pip install sphinx sphinxcontrib-httpdomain

debian/changelog

@@ -1,3 +1,94 @@
+orlo (0.1.1) stable; urgency=medium
+
+  * Cast package_rollback as a boolean
+  * Add log files for gunicorn
+  * Separate debug mode and debug logging
+  * Return 400 when getting /releases with no filter
+  * Remove skip from documentation, it was renamed to offset
+  * Fix tests, require filter on GET /releases
+  * Bump vagrant box cpus to 2
+  * Implement stats by date/time
+  * Fix query in release stats to group by release
+  * Add boolean True to acceptable success values
+  * Bump version
+  * Cast package_rollback as a boolean
+  * Add log files for gunicorn
+  * Separate debug mode and debug logging
+  * Return 400 when getting /releases with no filter
+  * Remove skip from documentation, it was renamed to offset
+  * Fix tests, require filter on GET /releases
+  * Bump vagrant box cpus to 2
+  * Implement stats by date/time
+  * Fix query in release stats to group by release
+  * Add boolean True to acceptable success values
+  * Bump version
+
+ -- Alex Forbes <[email protected]>  Mon, 01 Feb 2016 18:19:03 +0000
+
+orlo (0.1.0) stable; urgency=medium
+
+  * Move orlo to /api/ in nginx
+  * Add ability to filter on rollback
+  * Add DB created to vagrant file
+  * Add more get_releases package filters and "latest" option
+  * Abstract filtering logic +
+  * Show easy to understand json message when filtering on invalid field
+  * Fix last/latest parameter name
+  * Use limit(1) instead of first for latest query
+  * Move helper functions from views.py to util.py
+  * Rename views.py to route_api.py
+  * Move filter function to util
+  * Start adding stats endpoints and functions
+  * Add index to stime field on Release
+  * Add __version__ attribute to package
+  * Add _version.py to gitignore
+  * Move cli interface into python page
+  * Add queries, for use by the stats and info routes
+  * Implement more /info endpoints, separate tests from stats
+  * Refactor non-endpoint tests to use internal methods rather than http
+  * Remove data.py
+  * Make the release counting function (now count_releases) more generic
+  * Add rollback filter to count_packages
+  * Comments, plus bump version
+  * Install requirements in vagrant file
+  * Fix platform name in log message
+  * Implement /stats endpoints
+  * Add /stats/package and consolidate the dictionary creation
+  * Handle poorly formatted time gracefully in stats endpoints
+  * Bump version
+  * Reverse Release-Package relationship
+  * Fix missing arguments passed to count_releases and implement /stats
+  * Fix to package_versions
+  * Rename /info urls
+  * Documentation updates
+  * Update Travis config to use Postgres
+  * Remove password from test postgres DB
+  * Fix postgres command
+  * Fix database setup
+  * Fix quotes in travis DB string
+  * Fix package_versions under postgres
+  * Set Flask packages to >= $version in requirements.txt
+  * Change /info/user endpoint
+  * Remove print statement, fix minor documentation bugs in /info
+  * Move platform in /info routes to query parameter
+  * Stream output of GET /releases to reduce memory usage
+  * Move /releases streaming json generator to util.py
+  * Add example curl to documentation for /import
+  * Abstract release logic away from the get_releases route
+  * Add status to get_releases parameters
+  * Remove "latest" filter option in favour of "desc" and "limit"
+  * Rename test_import
+  * Add offset to get_releases
+  * Ensure limit and offset are ints
+  * Implement time-based stats for charts
+  * Move orlo.conf to orlo/orlo.ini
+  * Rename package from python-orlo to orlo
+  * Deb packaging fixes
+  * Add tests for stop package
+  * Bump version to 0.1.0
+
+ -- Alex Forbes <[email protected]>  Tue, 19 Jan 2016 16:07:52 +0000
+
 python-orlo (0.0.4) stable; urgency=medium
 
   * Update debian description

orlo/__init__.py

@@ -1,4 +1,5 @@
 from flask import Flask
+import logging
 from logging.handlers import RotatingFileHandler
 
 from orlo.config import config
@@ -15,8 +16,15 @@
 if config.getboolean('db', 'echo_queries'):
     app.config['SQLALCHEMY_ECHO'] = True
 
-if config.getboolean('logging', 'debug'):
+# Debug mode ignores all custom logging and should only be used in
+# local testing...
+if config.getboolean('main', 'debug_mode'):
     app.debug = True
+
+# ...as opposed to loglevel debug, which can be used anywhere
+if config.getboolean('logging', 'debug'):
+    app.logger.setLevel(logging.DEBUG)
+
 app.logger.debug('Debug enabled')
 
 if not config.getboolean('main', 'strict_slashes'):

orlo/cli.py

@@ -19,8 +19,10 @@ def parse_args():
 
     p_database = argparse.ArgumentParser(add_help=False)
     p_server = argparse.ArgumentParser(add_help=False)
-    p_server.add_argument('--host', '-H', dest='host', default='127.0.0.1', help="Address to listen on")
-    p_server.add_argument('--port', '-P', dest='port', type=int, default=5000, help="Port to listen on")
+    p_server.add_argument('--host', '-H', dest='host', default='127.0.0.1',
+                          help="Address to listen on")
+    p_server.add_argument('--port', '-P', dest='port', type=int, default=5000,
+                          help="Port to listen on")
 
     subparsers = parser.add_subparsers(dest='action')
     sp_config = subparsers.add_parser(

orlo/config.py

@@ -5,6 +5,7 @@
 config = ConfigParser.ConfigParser()
 
 config.add_section('main')
+config.set('main', 'debug_mode', 'false')
 config.set('main', 'propagate_exceptions', 'true')
 config.set('main', 'time_format', '%Y-%m-%dT%H:%M:%SZ')
 config.set('main', 'time_zone', 'UTC')

orlo/exceptions.py

@@ -5,6 +5,7 @@
 
 class OrloError(Exception):
     status_code = 500
+
     def __init__(self, message, status_code=None, payload=None):
         Exception.__init__(self)
         self.message = message
@@ -28,4 +29,3 @@ class DatabaseError(OrloError):
 
 class OrloWorkflowError(OrloError):
     status_code = 400
-

orlo/queries.py

@@ -5,7 +5,6 @@
 from orlo import app
 from orlo.orm import db, Release, Platform, Package, release_platform
 from orlo.exceptions import OrloError, InvalidUsage
-from orlo.util import is_int
 from collections import OrderedDict
 
 __author__ = 'alforbes'
@@ -15,7 +14,7 @@
 """
 
 
-def _filter_release_status(query, status):
+def filter_release_status(query, status):
     """
     Filter the given query by the given release status
 
@@ -43,7 +42,7 @@ def _filter_release_status(query, status):
     return query
 
 
-def _filter_release_rollback(query, rollback):
+def filter_release_rollback(query, rollback):
     """
     Filter the given query by whether the releases are rollbacks or not
 
@@ -81,12 +80,13 @@ def apply_filters(query, args):
         if field == 'latest':  # this is not a comparison
             continue
 
-        # special logic for these ones, as they are package attributes
+        # special logic for these ones, as they are release attributes that
+        # are JIT calculated from package attributes
         if field == 'status':
-            query = _filter_release_status(query, value)
+            query = filter_release_status(query, value)
             continue
         if field == 'rollback':
-            query = _filter_release_rollback(query, value)
+            query = filter_release_rollback(query, value)
             continue
 
         if field.startswith('package_'):
@@ -189,11 +189,15 @@ def releases(**kwargs):
     query = query.order_by(stime_field())
 
     if limit:
-        if not is_int(limit):
+        try:
+            limit = int(limit)
+        except ValueError:
             raise InvalidUsage("limit must be a valid integer value")
         query = query.limit(limit)
     if offset:
-        if not is_int(offset):
+        try:
+            offset = int(offset)
+        except ValueError:
             raise InvalidUsage("offset must be a valid integer value")
         query = query.offset(offset)
 
@@ -420,10 +424,10 @@ def count_releases(user=None, package=None, team=None, platform=None, status=Non
         query = query.filter(Release.stime <= ftime)
 
     if rollback is not None:
-        query = _filter_release_rollback(query, rollback)
+        query = filter_release_rollback(query, rollback)
 
     if status:
-        query = _filter_release_status(query, status)
+        query = filter_release_status(query, status)
 
     return query
 
@@ -500,116 +504,3 @@ def platform_list():
     return query
 
 
-def stats_release_time(unit, summarize_by_unit=False, **kwargs):
-    """
-    Return stats by time from the given arguments
-
-    Functions in this file usually return a query object, but here we are
-    returning the result, as there are several queries in play.
-
-    :param summarize_by_unit: Passed to add_release_by_time_to_dict()
-    :param unit: Passed to add_release_by_time_to_dict()
-    """
-
-    root_query = db.session.query(Release.id, Release.stime).join(Package)
-    root_query = apply_filters(root_query, kwargs)
-
-    # Build queries for the individual stats
-    q_normal_successful = _filter_release_status(
-            _filter_release_rollback(root_query, rollback=False), 'SUCCESSFUL'
-    )
-    q_normal_failed = _filter_release_status(
-            _filter_release_rollback(root_query, rollback=False), 'FAILED'
-    )
-    q_rollback_successful = _filter_release_status(
-            _filter_release_rollback(root_query, rollback=True), 'SUCCESSFUL'
-    )
-    q_rollback_failed = _filter_release_status(
-            _filter_release_rollback(root_query, rollback=True), 'FAILED'
-    )
-
-    output_dict = OrderedDict()
-
-    add_releases_by_time_to_dict(
-            q_normal_successful, output_dict, ('normal', 'successful'), unit, summarize_by_unit)
-    add_releases_by_time_to_dict(
-            q_normal_failed, output_dict, ('normal', 'failed'), unit, summarize_by_unit)
-    add_releases_by_time_to_dict(
-            q_rollback_successful, output_dict, ('rollback', 'successful'), unit,
-            summarize_by_unit)
-    add_releases_by_time_to_dict(
-            q_rollback_failed, output_dict, ('rollback', 'failed'), unit, summarize_by_unit)
-
-    return output_dict
-
-
-def add_releases_by_time_to_dict(query, releases_dict, t_category, unit='month',
-                                 summarize_by_unit=False):
-    """
-    Take a query and add each of its releases to a dictionary, broken down by time
-
-    :param dict releases_dict: Dict to add to
-    :param tuple t_category: tuple of headings, i.e. (<normal|rollback>, <successful|failed>)
-    :param query query: Query object to retrieve releases from
-    :param string unit: Can be 'iso', 'hour', 'day', 'week', 'month', 'year',
-    :param boolean summarize_by_unit: Only break down releases by the given unit, i.e. only one
-        layer deep
-    :return:
-    """
-
-    for release in query:
-        if summarize_by_unit:
-            tree_args = [str(getattr(release.stime, unit))]
-        else:
-            if unit == 'year':
-                tree_args = [str(release.stime.year)]
-            elif unit == 'month':
-                tree_args = [str(release.stime.year), str(release.stime.month)]
-            elif unit == 'week':
-                # First two args of isocalendar(), year and week
-                tree_args = [str(i) for i in release.stime.isocalendar()][0:2]
-            elif unit == 'iso':
-                tree_args = [str(i) for i in release.stime.isocalendar()]
-            elif unit == 'day':
-                tree_args = [str(release.stime.year), str(release.stime.month),
-                             str(release.stime.day)]
-            elif unit == 'hour':
-                tree_args = [str(release.stime.year), str(release.stime.month),
-                             str(release.stime.day), str(release.stime.hour)]
-            else:
-                raise InvalidUsage(
-                    'Invalid unit "{}" specified for release breakdown'.format(
-                        unit))
-        # Append categories
-        print(tree_args)
-        tree_args += t_category
-        append_tree_recursive(releases_dict, tree_args[0], tree_args)
-
-
-def append_tree_recursive(tree, parent, nodes):
-    """
-    Recursively place the nodes under each other
-
-    :param dict tree: The dictionary we are operating on
-    :param parent: The parent for this node
-    :param nodes: The list of nodes
-    :return:
-    """
-    print('Called recursive function with args:\n{}, {}, {}'.format(
-            str(tree), str(parent), str(nodes)))
-    try:
-        # Get the child, one after the parent
-        child = nodes[nodes.index(parent) + 1]
-    except IndexError:
-        # Must be at end
-        if parent in tree:
-            tree[parent] += 1
-        else:
-            tree[parent] = 1
-        return tree
-
-    # Otherwise recurse again
-    if parent not in tree:
-        tree[parent] = {}
-    # Child becomes the parent
-    append_tree_recursive(tree[parent], child, nodes)