Configured unit tests for projects

Miha Pleško · matejart · commit 58b88e9e7288 · 2016-03-21T14:40:19.000+01:00
Resolved bug that unit tests are
adding messages to the main queue.
Solution: tests use separated Celery
queue (dice_deploy_tests) with own
dummy worker (worker-tests).
Also added Celery dashboard to
control messages with ease. Actually
just installed Celery Flower module
and it works.
Implemented upstart scripts for both
Celery workers (main and test) and also
for celery dashboard.
diff --git a/.gitignore b/.gitignore
@@ -7,3 +7,5 @@
 .idea/*
 dice_deploy_django/uploads/*
 db.sqlite3
+dice_deploy_django/celery.err
+dice_deploy_django/celery.out
diff --git a/Vagrantfile b/Vagrantfile
@@ -4,11 +4,21 @@ Vagrant.configure(2) do |config|
   config.vm.box = "ubuntu/trusty64"
 
   config.vm.network "private_network", type: "dhcp"
+  # django server
   config.vm.network "forwarded_port", guest: 8080, host: 7080
+  # celery flower dashboard (main)
+  config.vm.network "forwarded_port", guest: 5556, host: 8056
+  # celery flower dashboard (tests)
+  config.vm.network "forwarded_port", guest: 5555, host: 8055
+
 
   config.vm.provision "shell", path: "provision-root.sh"
   config.vm.provision "shell", path: "provision-user.sh", privileged: false
 
+  # setup and run celery upstart service
+  config.vm.provision "shell", inline: "cp /vagrant/install/upstart-services/* /etc/init/"
+  config.vm.provision "shell", inline: "service celery-service start"
+
   config.vm.synced_folder "dice_deploy_django", "/home/vagrant/dice_deploy_django"
 
   config.vm.provider "virtualbox" do |v|
diff --git a/dice_deploy_django/cfy_wrapper/management/__init__.py b/dice_deploy_django/cfy_wrapper/management/__init__.py
diff --git a/dice_deploy_django/cfy_wrapper/management/commands/__init__.py b/dice_deploy_django/cfy_wrapper/management/commands/__init__.py
diff --git a/dice_deploy_django/cfy_wrapper/management/commands/celery-service.py b/dice_deploy_django/cfy_wrapper/management/commands/celery-service.py
@@ -0,0 +1,52 @@
+from django.core.management.base import BaseCommand, CommandError
+from django.conf import settings
+import subprocess
+
+"""
+Example calls:
+python manage.py celery-service start
+python manage.py celery-service stop
+"""
+
+
+class Command(BaseCommand):
+    help = 'Starts or stops celery service.'
+
+    def add_arguments(self, parser):
+        parser.add_argument('command', type=str, help='start|stop')
+        parser.add_argument('--unit_tests', action='store_true',
+                            help='Add this flag to run celery-test-service instead of celery-service')
+        parser.set_defaults(unit_tests=False)
+
+    def handle(self, *args, **options):
+        print('unit_tests: %s' % options['unit_tests'])
+
+        if options['command'].lower() == 'start':
+            print('Starting celery for app %s' % settings.CELERY_APP_NAME)
+            p = subprocess.Popen([
+                'sudo',
+                'service',
+                'celery-service' if not options['unit_tests'] else 'celery-test-service',
+                'start'
+            ], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+            out, err = p.communicate()
+            if 'start/running' in out or 'already running' in err:
+                print('Success')
+            else:
+                print('ERROR: %s\n%s' % (out, err))
+        elif options['command'].lower() == 'stop':
+            print('Stopping celery for app %s' % settings.CELERY_APP_NAME)
+            p = subprocess.Popen([
+                'sudo',
+                'service',
+                'celery-service' if not options['unit_tests'] else 'celery-test-service',
+                'stop'
+            ], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+            out, err = p.communicate()
+            if 'stop/waiting' in out:
+                print('Success')
+            else:
+                print('ERROR: %s\n%s' % (out, err))
+        else:
+            raise CommandError('Invalid command argument, got: %s' % options['command'])
+
diff --git a/dice_deploy_django/dice_deploy/celery.py b/dice_deploy_django/dice_deploy/celery.py
@@ -8,6 +8,7 @@
 from celery import Celery
 from django.conf import settings
 
-app = Celery("dice_deploy")
+# app = Celery(settings.CELERY_APP_NAME)
+app = Celery('BUREK')
 app.config_from_object("django.conf:settings")
 app.autodiscover_tasks(lambda: settings.INSTALLED_APPS)
diff --git a/dice_deploy_django/dice_deploy/settings.py b/dice_deploy_django/dice_deploy/settings.py
@@ -24,6 +24,7 @@
     "django.contrib.contenttypes",
     "rest_framework",
     "cfy_wrapper",
+    "unit_tests",  # must be app in order for celery-test-service to work
 )
 
 MIDDLEWARE_CLASSES = (
@@ -60,6 +61,13 @@
 CELERY_ACCEPT_CONTENT = ['json']
 CELERY_ENABLE_UTC = True
 
+BROKER_CONNECTION_MAX_RETRIES = 0
+BROKER_FAILOVER_STRATEGY = "round-robin"
+BROKER_HEARTBEAT = 10
+CELERY_SEND_EVENTS = True
+
+CELERY_APP_NAME = 'dice_deploy'
+
 # Cloudify settings
 CFY_MANAGER_URL = "172.16.95.77"
 POOL_SLEEP_INTERVAL = 3  # In seconds
diff --git a/dice_deploy_django/dice_deploy/settings_tests.py b/dice_deploy_django/dice_deploy/settings_tests.py
@@ -1,4 +1,14 @@
 from dice_deploy.settings import *
 import os
 
+# tests will check this flag pripr running
+IS_TEST_SETTINGS = True
+
 TEST_FILES_DIR = os.path.join(BASE_DIR, 'unit_tests', 'files')
+TEST_FILE_BLUEPRINT_EXAMPLE = os.path.join(TEST_FILES_DIR, 'blueprints', 'example_blueprint.tar.gz')
+
+MEDIA_ROOT = os.path.join(BASE_DIR, "uploads_tests")
+
+# make celery synchronous
+# CELERY_ALWAYS_EAGER = True
+CELERY_APP_NAME = 'dice_deploy_tests'
diff --git a/dice_deploy_django/requirements.in b/dice_deploy_django/requirements.in
@@ -4,3 +4,4 @@ markdown
 cloudify_rest_client
 celery
 enum34
+flower
diff --git a/dice_deploy_django/requirements.txt b/dice_deploy_django/requirements.txt
@@ -16,3 +16,4 @@ kombu==3.0.28             # via celery
 markdown==2.6.2
 pytz==2015.6              # via celery
 requests==2.7.0           # via cloudify-rest-client
+flower==0.8.4
diff --git a/dice_deploy_django/unit_tests/celery.py b/dice_deploy_django/unit_tests/celery.py
@@ -0,0 +1,13 @@
+from __future__ import absolute_import
+
+import os
+
+# This is set here since django.conf import needs to have this set
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "dice_deploy.settings")
+
+from celery import Celery
+from django.conf import settings
+
+app = Celery(settings.CELERY_APP_NAME)
+app.config_from_object("django.conf:settings")
+app.autodiscover_tasks(lambda: settings.INSTALLED_APPS)
diff --git a/dice_deploy_django/unit_tests/test_cfy_wrapper_api.py b/dice_deploy_django/unit_tests/test_cfy_wrapper_api.py
@@ -1,22 +1,48 @@
 from django.core.urlresolvers import reverse
 from rest_framework import status
 from rest_framework.test import APITestCase
-from dice_deploy import settings_tests
+from django.conf import settings
+from rest_framework.exceptions import NotFound
 import os
+from cfy_wrapper.models import Blueprint
+import shutil
+from django.core import management
 
 
 class AccountTests(APITestCase):
+    def setUp(self):
+        # prevent data loss due to not setting settings_tests.py as settings module
+        try:
+            is_test_settings = settings.IS_TEST_SETTINGS
+        except Exception, e:
+            raise Exception(
+                'Must run with dice_deploy/settings_tests.py. Aborting to prevent data loss.'
+            )
+
+        # clean uploads_tests folder prior starting
+        if os.path.exists(settings.MEDIA_ROOT):
+            shutil.rmtree(settings.MEDIA_ROOT)
+        os.mkdir(settings.MEDIA_ROOT)
+
+        # start celery
+        management.call_command('celery-service', 'start', '--unit_tests')
+
+    def tearDown(self):
+        shutil.rmtree(settings.MEDIA_ROOT)
+
+        # stop celery
+        # management.call_command('celery-service', 'stop', '--unit_tests')
+
     def test_upload_blueprint(self):
         """ Test uploading a new blueprint """
         url = reverse('blueprints')
 
-        example = None
-        with open(os.path.join(settings_tests.TEST_FILES_DIR, 'example.tar.gz'), 'rb') as f:
-            example = f.read()
-
-        data = {'file': example}
-        response = self.client.put(url, data)
+        with open(os.path.join(settings.TEST_FILE_BLUEPRINT_EXAMPLE), 'rb') as f:
+            # example = f.read()
+            data = {'file': f}
+            response = self.client.put(url, data)
 
+        # check HTTP response
         self.assertEqual(
             response.status_code, status.HTTP_201_CREATED,
             msg='Recieved bad status: %d. Data was: %s' % (response.status_code, response.data)
@@ -25,3 +51,17 @@ def test_upload_blueprint(self):
 
         if 'cfy_id' not in response.data:
             raise AssertionError('Missing "cfy" key in response')
+
+        # check DB state
+        try:
+            blueprint = Blueprint.get(response.data['cfy_id'])
+        except NotFound:
+            raise AssertionError('Blueprint was not found in DB, but it should be there.')
+
+        # check filesystem
+        with open(blueprint.archive.path) as f:
+            data = f.read()
+        if not data:
+            raise AssertionError('Blueprint .tar.gz file could not be found/opened on filesystem')
+
+
diff --git a/install/upstart-services/celery-service.conf b/install/upstart-services/celery-service.conf
@@ -0,0 +1,39 @@
+description "Start and stop celery daemon"
+
+#
+# put this file into /etc/init to enable commands:
+# - sudo service celery-service start|stop|restart
+#
+
+# variables - NOTE: they must all be direct. Cross dependencies will only evaluate in "script", "pre-start", ... (known upstart bug)
+env APP_NAME=dice_deploy
+env DJANGO_HOME_DIR=/home/vagrant/dice_deploy_django
+
+kill signal SIGINT
+
+# Make sure network and fs is up, and start in runlevels 2-5
+start on (net-device-up
+          and local-filesystems
+          and runlevel [2345])
+# Stop in runlevels 0,1 and 6
+stop on runlevel [016]
+
+# automatically respawn if program stopped unexpectedly
+respawn
+
+# wait a few seconds before respawning
+post-stop exec sleep 5
+
+# cd to code path and run node, with the right switches
+setuid vagrant
+script
+    # activate virtualenv
+    . /home/vagrant/venv/bin/activate
+
+    cd $DJANGO_HOME_DIR
+
+    # run commend
+    # exec celery worker -A $APP_NAME
+    exec celery flower -A $APP_NAME --port=5556 --broker_api=http://guest:guest@localhost:15672/api/
+end script
+
diff --git a/install/upstart-services/celery-test-service.conf b/install/upstart-services/celery-test-service.conf
@@ -0,0 +1,38 @@
+description "Start and stop celery daemon"
+
+#
+# put this file into /etc/init to enable commands:
+# - sudo service celery-service start|stop|restart
+#
+
+# variables - NOTE: they must all be direct. Cross dependencies will only evaluate in "script", "pre-start", ... (known upstart bug)
+env APP_NAME=unit_tests
+env DJANGO_HOME_DIR=/home/vagrant/dice_deploy_django
+
+kill signal SIGINT
+
+# Make sure network and fs is up, and start in runlevels 2-5
+start on (net-device-up
+          and local-filesystems
+          and runlevel [2345])
+# Stop in runlevels 0,1 and 6
+stop on runlevel [016]
+
+# automatically respawn if program stopped unexpectedly
+respawn
+
+# wait a few seconds before respawning
+post-stop exec sleep 5
+
+# cd to code path and run node, with the right switches
+setuid vagrant
+script
+    # activate virtualenv
+    . /home/vagrant/venv/bin/activate
+
+    cd $DJANGO_HOME_DIR
+
+    # run commend
+    exec celery flower -A $APP_NAME --port=5555 --broker_api=http://guest:guest@localhost:15672/api/
+end script
+
diff --git a/provision-root.sh b/provision-root.sh
@@ -1,4 +1,4 @@
 #!/bin/sh
 
 apt-get update
-apt-get install -y rabbitmq-server python-virtualenv python-dev
+apt-get install -y rabbitmq-server python-virtualenv python-dev dos2unix

-Original file line number
+Diff line change
 cloudify_rest_client
 celery
 enum34
 +flower