use uwsgi instead of paste

.gitignore

@@ -24,6 +24,7 @@ parts
 bin
 local
 /src/*_theme
+/lib
 lib64
 eggs
 develop-eggs

CHANGES.txt

@@ -4,6 +4,19 @@ adhocracy Changelog
 1.2beta2dev
 -----------
 
+- link python lib and inclue directories, uwsgi needs this.
+
+  Upgrade manual installation:
+  ____________________________
+
+  * cd python
+  * bin/buildout 
+  * cd ..
+  * bin/buildout
+
+- run adhocarcy with uwsgi instead of paster to improve performance, 
+  for development keep with paster [joka]
+
 - added varnish reverse proxy [joka]
 
 - Readd bin/test to run tests (wrapper for py.test) [joka]

INSTALLATION.rst

@@ -70,13 +70,13 @@ Check out Adhocracy:
 
     $ git clone https://github.com/liqd/adhocracy
     $ cd adhocracy
-    
+
     $ git submodule init
     $ git submodule update
 
 ## Setup an isolated python environment to run Adhocracy
 
-To install Adhocracy you need python (2.6|2.7) with PIL (python imaging) but 
+To install Adhocracy you need python (2.6|2.7) with PIL (python imaging) but
 no other system-packages.
 
 Compile python and PIL with the included python buildout::
@@ -92,7 +92,7 @@ Run buildout:
 
 ::
 
-    $ bin/python bootstrap.py 
+    $ bin/python bootstrap.py
     $ bin/buildout
 
 Start Adhocracy and dependent servers:
@@ -109,6 +109,7 @@ have to setup the Adhocracy database manually:
 
     $ bin/paster setup-app etc/adhocracy.ini --name=content
 
+
 Run Adhocracy
 -------------
 
@@ -143,6 +144,14 @@ Start the Adhocracy server in foreground mode:
     $ bin/supervisorctl stop adhocracy
     $ bin/paster serve etc/adhocracy.ini
 
+To run adhocarcy with uwsgi instead of paster:
+
+::
+    $ bin/uwsgi --ini-paste etc/adhocracy.ini
+
+To run the ipdb python debuger with uwsgi use "IPDB()" instead of "ipdb.set_trace()".
+
+
 Buildout configuration
 ----------------------
 

build.sh

@@ -274,7 +274,7 @@ if [ '!' -e python/buildout.python/src ]; then
 fi
 
 # Install local python if necessary
-if [ '!' -x bin/python ]; then
+if [ '!' -x bin/python ] || [ '!' -e lib ]; then
     if [ '!' -f python/bin/buildout ]; then
         (cd python && python bootstrap.py)
     fi

buildout.cfg

@@ -11,6 +11,7 @@ extends =
     buildouts/base.cfg
     buildouts/lxml.cfg
     buildouts/adhocracy.cfg
+    buildouts/uwsgi.cfg
 #    buildouts/adhocracy_theming.cfg
     buildouts/stylesheets.cfg
     buildouts/memcache.cfg

buildouts/adhocracy.cfg

@@ -152,18 +152,19 @@ command =
 
 # python with all required eggs
 [adhocpy]
-recipe = zc.recipe.egg
+recipe = minitage.recipe.scripts 
 eggs =  ${buildout:eggs}
 extra-paths = ./
 interpreter = adhocpy
 scripts = adhocpy
+env-file = ${buildout:bin-directory}/adhocpy-env
 
 ##############################################################################
 # Set up supervisor to run it all
 ##############################################################################
 
 [supervisor]
-environment = LD_LIBRARY_PATH="${buildout:directory}/python/python-2.7/lib/"
+environment = LD_LIBRARY_PATH="${buildout:directory}/lib/"
 programs +=
     ${buildout:adhocracy_worker-supervisor}
     ${buildout:adhocracy-supervisor}

buildouts/uwsgi.cfg

@@ -0,0 +1,115 @@
+##############################################################################
+# Buildout to install uwsgi https://github.com/unbit/uwsgi
+#
+# requires:
+#
+#   - bin/adhocracpy-env to set the PYTHONPATH variable
+#
+##############################################################################
+
+[buildout]
+
+parts +=
+    uwsgi
+    uwsgi_bin
+
+adhocracy-supervisor =
+    45 adhocracy (environment=${supervisor:environment} stopsignal=INT redirect_stderr=true stdout_logfile=var/log/adhocracy.log stderr_logfile=NONE) ${buildout:bin-directory}/uwsgi [--ini-paste ${buildout:directory}/etc/adhocracy.ini]
+
+##############################################################################
+# System settings
+##############################################################################
+
+[domains]
+uwsgi = 127.0.0.1
+
+[ports]
+uwsgi = 5008
+
+[urls]
+uwsgi = http://projects.unbit.it/downloads/uwsgi-1.9.12.tar.gz
+
+###############################
+# Install and configure uwsgi #
+###############################
+
+# compile options
+[uwsgi_build_conf]
+recipe = collective.recipe.template
+output = ${buildout:etc-directory}/uwsgi_buildconf.ini
+input = ${buildout:etc-directory}/uwsgi_buildconf.ini.in 
+bin-name = ${buildout:bin-directory}/uwsgi
+plugin-dir = ${buildout:var-directory}/uwsgi_plugins
+main-plugin = python, gevent
+
+[uwsgi_build_env]
+UWSGI_PROFILE = ${uwsgi_build_profile:output}
+PYTHONHOME = ${buildout:directory}
+
+# compile egg
+[uwsgi]
+recipe = minitage.recipe.du 
+url = ${urls:uwsgi}
+eggs = 
+    ${buildout:eggs} 
+    uwsgi 
+environment = uwsgi_build_env
+
+# uwsgi bin
+[uwsgi_bin]
+recipe = collective.recipe.template
+output = ${buildout:bin-directory}/uwsgi
+input = inline:
+    #!/bin/bash
+    source ${adhocpy:env-file}
+    ${buildout:parts-directory}/uwsgi/bin/uwsgi "$@"
+mode = 755 
+
+# configure
+[uwsgi_conf]
+conf =
+    #basics
+
+    http = ${adhocracy:host}:${ports:main} 
+    pidfile = ${buildout:directory}/var/uwsgi.pid 
+    master = true
+    chdir =  ${buildout:directory}/src
+    home = ${buildout:directory}
+
+    # extended
+
+    # Reload if ini changes
+    touch-reload = true  
+    # stop uwsgi if the wsgi app is not available
+    need-app = true
+    # allow threads
+    enable-threads = true
+    #Set close-on-exec on sockets (could be required for spawning processes in requests).
+    close-on-exec = true
+    #Automatically kill workers if master dies (can be dangerous for availability).
+    no-orphans = true
+    # Try to remove all of the generated files/sockets (UNIX sockets and pidfiles) upon exit.
+    vacuum = true
+
+    #performance tuning
+
+    processes = 1
+    worker = 20
+    post-buffering = 4096
+    max-requests = 1000
+    # Reload a worker if its address space usage is higher than the specified value (in megabytes).
+    reload-on-as = 128 
+    #gracefull dead
+    harakiri = 60
+    harakiri-verbose = true
+
+    #logging
+
+    log-date = true
+    log-slow = true
+
+    # debug
+
+    # IPDB(); opens python shell
+    shared-import = ${buildout:directory}/scripts/interact.py
+    # memory-report = true

etc/adhocracy.ini.in

@@ -7,8 +7,6 @@
 ############################################################## 
 
 [DEFAULT]
-{% if parts.adhocracy.debug == 'True' %}debug = True  {% end %}
-{% if parts.adhocracy.debug != 'True' %}debug = False {% end %}
 
 # Uncomment and replace with the address which should receive any error reports
 email_to = ${parts.adhocracy['email.to']}
@@ -24,7 +22,10 @@ port = ${parts.ports.main}
 {% if parts.adhocracy.debug != 'True' %}use_threadpool = True {% end %}
 threadpool_workers = 20
 
-
+{% if "uwsgi_conf" in parts %}
+[uwsgi]
+${parts.uwsgi_conf.conf}
+{% end %}
 
 {% if parts.adhocracy.themed == 'False' %}
 ###################not themed adhocracy ###################

etc/uwsgi_buildconf.ini.in

@@ -0,0 +1,50 @@
+; this is mainly copied from the uwsgi buildconf/default.ini
+; trying to surface some of the build options to buildout
+; properties.
+[uwsgi]
+bin_name = ${:bin-name}
+plugin_dir = ${:plugin-dir}
+main_plugin = ${:main-plugin}
+xml = true 
+ini = true
+yaml = true
+json = auto
+sqlite3 = auto
+zeromq = auto
+snmp = true
+spooler = true
+embedded = true
+ssl = auto
+udp = true
+multicast = true
+threading = true
+minterpreters = true
+async = true
+ldap = auto
+pcre = auto
+routing = auto
+alarm = auto
+ipv6 = false
+debug = false
+unbit = false
+xml_implementation = libxml2
+yaml_implementation = auto
+malloc_implementation = libc
+extras =
+plugins =
+append_version =
+embedded_plugins = %(main_plugin)s, ping, cache, nagios, rrdtool, carbon, rpc, corerouter, fastrouter, http, ugreen, signal, syslog, rsyslog, logsocket, router_uwsgi, router_redirect, router_basicauth, zergpool, redislog, mongodblog, router_rewrite, router_http, logfile, router_cache, rawrouter
+as_shared_library = false
+
+locking = auto
+event = auto
+timer = auto
+filemonitor = auto
+
+blacklist =
+whitelist =
+
+embed_files =
+
+embed_config =
+

python/buildout.cfg

@@ -13,19 +13,34 @@ parts =
     ${buildout:links-parts}
 
 python-buildout-root = ${buildout:directory}/buildout.python/src
-pil-install-args = Pillow==2.0.0
+pil-install-args = Pillow==2.0
+
 
 [install-links]
 recipe = plone.recipe.command
 prefix = ${buildout:directory}/..
-command =
+command = 
     #!/bin/sh
     mkdir -p ${install-links:prefix}/bin/
+
+    # remove exiting links 
+    rm ${:prefix}/bin/python 
+    for i in $(ls ${:prefix}/bin/*[23].*); do
+        rm $i;
+    done
+    rm ${:prefix}/include;
+    rm ${:prefix}/lib;
 
-    #Symlink default python (2.7)
-    DEST=${install-links:prefix}/bin/python;
-    ln -vfs ${buildout:directory}/python-2.7/bin/python2.7 $DEST;     
-update-command = ${:command}
+    # Symlink include, lib directory, uwsgi needs this there
+    ln -s ${buildout:directory}/python-2.7/include ${:prefix}/include
+    ln -s ${buildout:directory}/python-2.7/lib ${:prefix}/lib
 
+    # Symlink python, easy_install
+    for i in $(ls ${buildout:directory}/python-[23].*/bin/*[23].*); do
+        DEST=${:prefix}/bin/`basename $i`;
+        ln -s $i $DEST;
+    done 
+    ln -s ${:prefix}/bin/python2.7 ${:prefix}/bin/python
 
 
+update-command = ${:command}

scripts/interact.py

@@ -0,0 +1,31 @@
+"""uwsgi development helpers: open interactive python shells"""
+import os
+import sys
+import inspect
+import code
+
+
+class RestoredStandardInputContext(object):
+
+    def __enter__(self):
+        self.backup_stdin = os.dup(sys.stdin.fileno())
+        os.dup2(sys.stdout.fileno(), sys.stdin.fileno())
+
+    def __exit__(self, error_type, error, traceback):
+        os.dup2(self.backup_stdin, sys.stdin.fileno())
+
+
+def interact(locals=None, plain=False):
+
+    with RestoredStandardInputContext():
+        code.interact(local=locals or inspect.currentframe().f_back.f_locals)
+
+
+def interact_ipdb():
+
+    with RestoredStandardInputContext():
+        import ipdb
+        ipdb.set_trace()
+
+__builtins__['PDB'] = interact
+__builtins__['IPDB'] = interact_ipdb